864 lines
30 KiB
C
864 lines
30 KiB
C
// Physical frame database.
|
|
// This file consumes the boot memory map and builds one PFN descriptor for each
|
|
// managed physical frame. The first implementation still allocates by scanning,
|
|
// but callers now get typed, reference-counted frames instead of anonymous
|
|
// bitmap bits.
|
|
|
|
#include <kernel/mm_pfn.h>
|
|
#include <kernel/ke_spinlock.h>
|
|
#include <kernel/ke_irql.h>
|
|
#include <string.h>
|
|
|
|
extern KSTATUS MmVadTrimOnePageForPressure(void);
|
|
|
|
static uint64_t gPhysBase = 0;
|
|
static MM_PFN* gPfnDatabase = (MM_PFN*)0;
|
|
static size_t gMaxPfn = 0;
|
|
static size_t gTotalFrames = 0;
|
|
static size_t gFreeFrames = 0;
|
|
static SPINLOCK gPfnLock = SPINLOCK_INIT;
|
|
static uint64_t gPfnAllocations = 0;
|
|
static uint64_t gPfnFrees = 0;
|
|
static uint64_t gPfnRetains = 0;
|
|
static uint64_t gPfnReleases = 0;
|
|
static uint64_t gPfnPins = 0;
|
|
static uint64_t gPfnUnpins = 0;
|
|
static uint64_t gPfnFailedAllocations = 0;
|
|
static uint64_t gPfnFailedContiguousAllocations = 0;
|
|
static uint64_t gPfnFailedInvalidRange = 0;
|
|
static uint64_t gPfnFailedEmptyRange = 0;
|
|
static uint64_t gPfnFailedReserveDenied = 0;
|
|
static uint64_t gPfnFailedNoFrame = 0;
|
|
static uint64_t gPfnFailedContiguousInvalidRequest = 0;
|
|
static uint64_t gPfnFailedContiguousReserveDenied = 0;
|
|
static uint64_t gPfnFailedContiguousNoRun = 0;
|
|
static uint64_t gPfnReclaimAttempts = 0;
|
|
static uint64_t gPfnReclaimPagesRequested = 0;
|
|
static uint64_t gPfnReclaimPagesSucceeded = 0;
|
|
static uint64_t gPfnReclaimPagesFailed = 0;
|
|
static uint64_t gPfnReclaimReentrantSkips = 0;
|
|
static volatile int gPfnReclaiming = 0;
|
|
|
|
static PHYSICAL_ADDRESS KiClaimPfnLocked(size_t pfn, MM_PFN_USAGE usage);
|
|
|
|
#define MM_PFN_KERNEL_RESERVE_FRAMES 128u
|
|
#define MM_PFN_RECLAIM_LOW_WATER 512u
|
|
#define MM_PFN_RECLAIM_HIGH_WATER 768u
|
|
|
|
// Converts a physical address into the direct-map alias used during early memory setup.
|
|
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS phys) {
|
|
return (void*)(phys + gPhysBase);
|
|
}
|
|
|
|
// Rounds a physical address up to the next whole frame boundary.
|
|
static inline PHYSICAL_ADDRESS KiAlignUp(PHYSICAL_ADDRESS value) {
|
|
return (value + PAGE_SIZE - 1) & PAGE_MASK;
|
|
}
|
|
|
|
// Rounds a physical address down to the previous whole frame boundary.
|
|
static inline PHYSICAL_ADDRESS KiAlignDown(PHYSICAL_ADDRESS value) {
|
|
return value & PAGE_MASK;
|
|
}
|
|
|
|
// Returns whether a boot memory type describes RAM that the PFN database should track.
|
|
static int KiMemoryTypeManaged(MEMORY_TYPE type) {
|
|
return type == MemoryUsable || type == MemoryBootloader || type == MemoryKernel;
|
|
}
|
|
|
|
// Returns whether an allocation may consume the emergency kernel reserve.
|
|
static int KiPfnUsageMayUseReserve(MM_PFN_USAGE usage) {
|
|
return usage == MmPfnUseKernel ||
|
|
usage == MmPfnUsePageTable ||
|
|
usage == MmPfnUseDma ||
|
|
usage == MmPfnUseBootloader ||
|
|
usage == MmPfnUsePfnDatabase ||
|
|
usage == MmPfnUseMmio;
|
|
}
|
|
|
|
// Converts allocation purpose into the initial PFN state.
|
|
static MM_PFN_STATE KiAllocatedStateForUsage(MM_PFN_USAGE usage) {
|
|
(void)usage;
|
|
return MmPfnStateActive;
|
|
}
|
|
|
|
// Marks a PFN as used by a permanent structure that must never be returned to the free list.
|
|
static void KiReservePfnLocked(size_t pfn, MM_PFN_USAGE usage) {
|
|
if (pfn >= gMaxPfn)
|
|
return;
|
|
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) {
|
|
if (gFreeFrames)
|
|
gFreeFrames--;
|
|
}
|
|
|
|
if (entry->State == MmPfnStateInvalid)
|
|
gTotalFrames++;
|
|
|
|
entry->State = MmPfnStateActive;
|
|
entry->Usage = usage;
|
|
entry->RefCount = 1;
|
|
entry->PinCount = 1;
|
|
entry->Flags = 0;
|
|
entry->MappedCount = 0;
|
|
entry->Owner = (void*)0;
|
|
entry->BackingPage = 0;
|
|
}
|
|
|
|
// Marks a page-aligned RAM range as managed by the PFN database.
|
|
static void KiRegisterManagedRange(PHYSICAL_ADDRESS base,
|
|
PHYSICAL_ADDRESS end,
|
|
MM_PFN_STATE state,
|
|
MM_PFN_USAGE usage) {
|
|
size_t firstPfn = (size_t)(base / PAGE_SIZE);
|
|
size_t count = (size_t)((end - base) / PAGE_SIZE);
|
|
|
|
for (size_t i = 0; i < count; i++) {
|
|
size_t pfn = firstPfn + i;
|
|
if (pfn >= gMaxPfn)
|
|
break;
|
|
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateInvalid)
|
|
gTotalFrames++;
|
|
|
|
if (entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) {
|
|
if (gFreeFrames)
|
|
gFreeFrames--;
|
|
}
|
|
|
|
entry->State = state;
|
|
entry->Usage = usage;
|
|
entry->RefCount = (state == MmPfnStateFree || state == MmPfnStateZeroed) ? 0 : 1;
|
|
entry->PinCount = 0;
|
|
entry->Flags = 0;
|
|
entry->MappedCount = 0;
|
|
entry->Owner = (void*)0;
|
|
entry->BackingPage = 0;
|
|
|
|
if (state == MmPfnStateFree || state == MmPfnStateZeroed)
|
|
gFreeFrames++;
|
|
}
|
|
}
|
|
|
|
// Returns whether the future page-replacement code may consider this frame a candidate.
|
|
static int KiPfnEvictableLocked(const MM_PFN* entry) {
|
|
if (!entry || entry->PinCount)
|
|
return 0;
|
|
if (entry->State != MmPfnStateActive && entry->State != MmPfnStateStandby && entry->State != MmPfnStateModified)
|
|
return 0;
|
|
return entry->Usage == MmPfnUseUser ||
|
|
entry->Usage == MmPfnUseCleanFileCache ||
|
|
entry->Usage == MmPfnUseImage ||
|
|
entry->Usage == MmPfnUseMappedFile ||
|
|
entry->Usage == MmPfnUseAnonymous ||
|
|
entry->Usage == MmPfnUseCow;
|
|
}
|
|
|
|
// Returns whether a standby frame can be counted as immediately reclaimable.
|
|
static int KiPfnReclaimableLocked(const MM_PFN* entry) {
|
|
if (!entry || entry->PinCount || entry->MappedCount)
|
|
return 0;
|
|
if (entry->State != MmPfnStateStandby || entry->RefCount != 1)
|
|
return 0;
|
|
return entry->Usage == MmPfnUseCleanFileCache || entry->Usage == MmPfnUseImage;
|
|
}
|
|
|
|
// Returns true when a PFN can satisfy a new allocation.
|
|
static int KiPfnAvailableLocked(size_t pfn) {
|
|
if (pfn >= gMaxPfn)
|
|
return 0;
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
return entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed;
|
|
}
|
|
|
|
// Tries to create breathing room by swapping private user pages before an allocation fails.
|
|
static void KiPfnReclaimForPressure(void) {
|
|
if (!KeCanWait())
|
|
return;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
if (gPfnReclaiming) {
|
|
gPfnReclaimReentrantSkips++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return;
|
|
}
|
|
if (gFreeFrames >= MM_PFN_RECLAIM_LOW_WATER) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return;
|
|
}
|
|
|
|
gPfnReclaiming = 1;
|
|
gPfnReclaimAttempts++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
|
|
for (uint32_t i = 0; i < 64; ++i) {
|
|
flags = KeAcquireSpinlock(&gPfnLock);
|
|
if (gFreeFrames >= MM_PFN_RECLAIM_HIGH_WATER) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
break;
|
|
}
|
|
gPfnReclaimPagesRequested++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
|
|
KSTATUS s = MmVadTrimOnePageForPressure();
|
|
|
|
flags = KeAcquireSpinlock(&gPfnLock);
|
|
if (KSUCCESS(s))
|
|
gPfnReclaimPagesSucceeded++;
|
|
else
|
|
gPfnReclaimPagesFailed++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
|
|
if (!KSUCCESS(s))
|
|
break;
|
|
}
|
|
|
|
flags = KeAcquireSpinlock(&gPfnLock);
|
|
gPfnReclaiming = 0;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
}
|
|
|
|
// Claims a free PFN in the requested range while the PFN lock is held.
|
|
static PHYSICAL_ADDRESS KiClaimAvailableInRangeLocked(size_t firstPfn, size_t limitPfn, MM_PFN_USAGE usage) {
|
|
for (size_t pfn = limitPfn; pfn > firstPfn; pfn--) {
|
|
size_t candidate = pfn - 1;
|
|
if (KiPfnAvailableLocked(candidate))
|
|
return KiClaimPfnLocked(candidate, usage);
|
|
}
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
// Claims one free PFN for the requested usage.
|
|
static PHYSICAL_ADDRESS KiClaimPfnLocked(size_t pfn, MM_PFN_USAGE usage) {
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
entry->State = KiAllocatedStateForUsage(usage);
|
|
entry->Usage = usage;
|
|
entry->RefCount = 1;
|
|
entry->PinCount = (usage == MmPfnUsePageTable) ? 1 : 0;
|
|
entry->Flags = 0;
|
|
entry->MappedCount = 0;
|
|
entry->Owner = (void*)0;
|
|
entry->BackingPage = 0;
|
|
if (gFreeFrames)
|
|
gFreeFrames--;
|
|
gPfnAllocations++;
|
|
return MmPfnAddress(pfn);
|
|
}
|
|
|
|
// Initializes the PFN database from the bootloader memory map.
|
|
bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
|
|
if (!map)
|
|
return false;
|
|
|
|
gPhysBase = physicalWindowBase;
|
|
|
|
PHYSICAL_ADDRESS maxEnd = 0;
|
|
for (uint32_t i = 0; i < map->RegionCount; i++) {
|
|
const MEMORY_REGION* r = &map->Regions[i];
|
|
if (!KiMemoryTypeManaged(r->Type))
|
|
continue;
|
|
|
|
PHYSICAL_ADDRESS base = KiAlignUp(r->Base);
|
|
PHYSICAL_ADDRESS end = r->Base + r->Length;
|
|
if (end < r->Base)
|
|
end = (PHYSICAL_ADDRESS)-1;
|
|
end = KiAlignDown(end);
|
|
if (end <= base)
|
|
continue;
|
|
if (end > maxEnd)
|
|
maxEnd = end;
|
|
}
|
|
|
|
gMaxPfn = (size_t)(maxEnd / PAGE_SIZE);
|
|
if (!gMaxPfn)
|
|
return false;
|
|
|
|
size_t pfnBytes = gMaxPfn * sizeof(MM_PFN);
|
|
size_t pfnPages = (pfnBytes + PAGE_SIZE - 1) / PAGE_SIZE;
|
|
PHYSICAL_ADDRESS metadataSize = (PHYSICAL_ADDRESS)pfnPages * PAGE_SIZE;
|
|
|
|
const PHYSICAL_ADDRESS metadataMin = 0x100000ULL;
|
|
PHYSICAL_ADDRESS metadataPhys = 0;
|
|
int metadataFound = 0;
|
|
|
|
for (uint32_t i = 0; i < map->RegionCount; i++) {
|
|
const MEMORY_REGION* r = &map->Regions[i];
|
|
if (r->Type != MemoryUsable)
|
|
continue;
|
|
|
|
PHYSICAL_ADDRESS base = KiAlignUp(r->Base);
|
|
if (base < metadataMin)
|
|
base = metadataMin;
|
|
PHYSICAL_ADDRESS end = r->Base + r->Length;
|
|
if (end < r->Base)
|
|
end = (PHYSICAL_ADDRESS)-1;
|
|
end = KiAlignDown(end);
|
|
if (end <= base)
|
|
continue;
|
|
|
|
if (end - base >= metadataSize) {
|
|
metadataPhys = base;
|
|
metadataFound = 1;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!metadataFound)
|
|
return false;
|
|
|
|
gPfnDatabase = (MM_PFN*)KiPhysToVirt(metadataPhys);
|
|
memset(gPfnDatabase, 0, metadataSize);
|
|
gTotalFrames = 0;
|
|
gFreeFrames = 0;
|
|
gPfnLock = (SPINLOCK)SPINLOCK_INIT;
|
|
gPfnAllocations = 0;
|
|
gPfnFrees = 0;
|
|
gPfnRetains = 0;
|
|
gPfnReleases = 0;
|
|
gPfnPins = 0;
|
|
gPfnUnpins = 0;
|
|
gPfnFailedAllocations = 0;
|
|
gPfnFailedContiguousAllocations = 0;
|
|
gPfnFailedInvalidRange = 0;
|
|
gPfnFailedEmptyRange = 0;
|
|
gPfnFailedReserveDenied = 0;
|
|
gPfnFailedNoFrame = 0;
|
|
gPfnFailedContiguousInvalidRequest = 0;
|
|
gPfnFailedContiguousReserveDenied = 0;
|
|
gPfnFailedContiguousNoRun = 0;
|
|
gPfnReclaimAttempts = 0;
|
|
gPfnReclaimPagesRequested = 0;
|
|
gPfnReclaimPagesSucceeded = 0;
|
|
gPfnReclaimPagesFailed = 0;
|
|
gPfnReclaimReentrantSkips = 0;
|
|
|
|
for (uint32_t i = 0; i < map->RegionCount; i++) {
|
|
const MEMORY_REGION* r = &map->Regions[i];
|
|
if (!KiMemoryTypeManaged(r->Type))
|
|
continue;
|
|
|
|
PHYSICAL_ADDRESS base = KiAlignUp(r->Base);
|
|
PHYSICAL_ADDRESS end = r->Base + r->Length;
|
|
if (end < r->Base)
|
|
end = (PHYSICAL_ADDRESS)-1;
|
|
end = KiAlignDown(end);
|
|
if (end <= base)
|
|
continue;
|
|
|
|
if (r->Type == MemoryUsable) {
|
|
KiRegisterManagedRange(base, end, MmPfnStateFree, MmPfnUseNone);
|
|
} else if (r->Type == MemoryBootloader) {
|
|
KiRegisterManagedRange(base, end, MmPfnStateActive, MmPfnUseBootloader);
|
|
} else {
|
|
KiRegisterManagedRange(base, end, MmPfnStateActive, MmPfnUseKernel);
|
|
for (size_t pfn = (size_t)(base / PAGE_SIZE); pfn < (size_t)(end / PAGE_SIZE) && pfn < gMaxPfn; pfn++)
|
|
gPfnDatabase[pfn].PinCount = 1;
|
|
}
|
|
}
|
|
|
|
// PFN 0 remains reserved because a zero physical address is the allocation failure value.
|
|
KiReservePfnLocked(0, MmPfnUseKernel);
|
|
|
|
// The PFN database owns its storage permanently. It is structural metadata,
|
|
// not an allocation that can later be freed through a stale physical address.
|
|
size_t metadataFirstPfn = (size_t)(metadataPhys / PAGE_SIZE);
|
|
for (size_t i = 0; i < pfnPages; i++)
|
|
KiReservePfnLocked(metadataFirstPfn + i, MmPfnUsePfnDatabase);
|
|
|
|
return true;
|
|
}
|
|
|
|
// Converts a physical frame address to a PFN index.
|
|
size_t MmPfnIndex(PHYSICAL_ADDRESS frame) {
|
|
if (frame & (PAGE_SIZE - 1))
|
|
return MM_PFN_INVALID_INDEX;
|
|
size_t pfn = (size_t)(frame / PAGE_SIZE);
|
|
if (pfn >= gMaxPfn)
|
|
return MM_PFN_INVALID_INDEX;
|
|
return pfn;
|
|
}
|
|
|
|
// Converts a PFN index to a physical frame address.
|
|
PHYSICAL_ADDRESS MmPfnAddress(size_t pfn) {
|
|
return (PHYSICAL_ADDRESS)pfn * PAGE_SIZE;
|
|
}
|
|
|
|
// Returns the PFN descriptor for a page-aligned physical frame.
|
|
__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return (MM_PFN*)0;
|
|
return &gPfnDatabase[pfn];
|
|
}
|
|
|
|
// Returns the direct-map virtual address for a managed physical frame.
|
|
__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame) {
|
|
if (MmPfnIndex(frame) == MM_PFN_INVALID_INDEX)
|
|
return (void*)0;
|
|
return KiPhysToVirt(frame);
|
|
}
|
|
|
|
// Allocates one physical frame for the requested kernel subsystem.
|
|
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage) {
|
|
return MmPfnAllocateInRange(PAGE_SIZE, (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE, usage);
|
|
}
|
|
|
|
// Allocates one physical frame below a physical address limit.
|
|
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage) {
|
|
return MmPfnAllocateInRange(PAGE_SIZE, maxExclusive, usage);
|
|
}
|
|
|
|
// Allocates one physical frame inside a physical address range.
|
|
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive,
|
|
PHYSICAL_ADDRESS maxExclusive,
|
|
MM_PFN_USAGE usage) {
|
|
minInclusive = KiAlignUp(minInclusive);
|
|
maxExclusive = KiAlignDown(maxExclusive);
|
|
if (maxExclusive <= minInclusive) {
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
gPfnFailedAllocations++;
|
|
gPfnFailedInvalidRange++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
size_t firstPfn = (size_t)(minInclusive / PAGE_SIZE);
|
|
size_t limitPfn = (size_t)(maxExclusive / PAGE_SIZE);
|
|
if (limitPfn > gMaxPfn)
|
|
limitPfn = gMaxPfn;
|
|
if (firstPfn >= limitPfn) {
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
gPfnFailedAllocations++;
|
|
gPfnFailedEmptyRange++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
KiPfnReclaimForPressure();
|
|
|
|
int mayUseReserve = KiPfnUsageMayUseReserve(usage);
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
if (!mayUseReserve && gFreeFrames <= MM_PFN_KERNEL_RESERVE_FRAMES) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
} else {
|
|
PHYSICAL_ADDRESS phys = KiClaimAvailableInRangeLocked(firstPfn, limitPfn, usage);
|
|
if (phys) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return phys;
|
|
}
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
}
|
|
|
|
KiPfnReclaimForPressure();
|
|
|
|
flags = KeAcquireSpinlock(&gPfnLock);
|
|
if (!mayUseReserve && gFreeFrames <= MM_PFN_KERNEL_RESERVE_FRAMES) {
|
|
gPfnFailedAllocations++;
|
|
gPfnFailedReserveDenied++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
PHYSICAL_ADDRESS phys = KiClaimAvailableInRangeLocked(firstPfn, limitPfn, usage);
|
|
if (phys) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return phys;
|
|
}
|
|
gPfnFailedAllocations++;
|
|
gPfnFailedNoFrame++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
// Allocates a physically contiguous run for DMA or hardware structures.
|
|
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) {
|
|
if (!count || count > gMaxPfn) {
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
gPfnFailedContiguousAllocations++;
|
|
gPfnFailedContiguousInvalidRequest++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
if (!KiPfnUsageMayUseReserve(usage))
|
|
KiPfnReclaimForPressure();
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
if (!KiPfnUsageMayUseReserve(usage) && gFreeFrames < MM_PFN_KERNEL_RESERVE_FRAMES + count) {
|
|
gPfnFailedContiguousAllocations++;
|
|
gPfnFailedContiguousReserveDenied++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
size_t runStart = 0;
|
|
size_t runLength = 0;
|
|
|
|
for (size_t pfn = 1; pfn < gMaxPfn; pfn++) {
|
|
if (KiPfnAvailableLocked(pfn)) {
|
|
if (!runLength)
|
|
runStart = pfn;
|
|
runLength++;
|
|
if (runLength == count) {
|
|
for (size_t i = 0; i < count; i++)
|
|
KiClaimPfnLocked(runStart + i, usage);
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return MmPfnAddress(runStart);
|
|
}
|
|
} else {
|
|
runLength = 0;
|
|
}
|
|
}
|
|
|
|
gPfnFailedContiguousAllocations++;
|
|
gPfnFailedContiguousNoRun++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return (PHYSICAL_ADDRESS)0;
|
|
}
|
|
|
|
// Increments the reference count on an allocated frame.
|
|
__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
if (entry->RefCount == UINT16_MAX) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_FAILURE;
|
|
}
|
|
entry->RefCount++;
|
|
gPfnRetains++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Releases a reference and returns the frame to the free list when the last owner leaves.
|
|
__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
if (entry->PinCount) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_ACCESS_DENIED;
|
|
}
|
|
gPfnReleases++;
|
|
if (entry->RefCount > 1) {
|
|
entry->RefCount--;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
if (entry->MappedCount) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_ACCESS_DENIED;
|
|
}
|
|
|
|
entry->State = MmPfnStateFree;
|
|
entry->Usage = MmPfnUseNone;
|
|
entry->RefCount = 0;
|
|
entry->PinCount = 0;
|
|
entry->Flags = 0;
|
|
entry->MappedCount = 0;
|
|
entry->Owner = (void*)0;
|
|
entry->BackingPage = 0;
|
|
gFreeFrames++;
|
|
gPfnFrees++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Frees a frame that is expected to have a single owner.
|
|
__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame) {
|
|
return MmPfnRelease(frame);
|
|
}
|
|
|
|
// Pins a frame so later page replacement cannot evict it.
|
|
__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
if (entry->PinCount == UINT16_MAX) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_FAILURE;
|
|
}
|
|
entry->PinCount++;
|
|
gPfnPins++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Removes one pin from a frame.
|
|
__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (!entry->PinCount) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
entry->PinCount--;
|
|
gPfnUnpins++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Records that a VAD now maps this frame into a user address space.
|
|
__declspec(dllexport) KSTATUS MmPfnAddMapping(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
if (entry->MappedCount == UINT32_MAX) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_FAILURE;
|
|
}
|
|
entry->MappedCount++;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Removes one user VAD mapping reference from a frame.
|
|
__declspec(dllexport) KSTATUS MmPfnRemoveMapping(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed ||
|
|
entry->MappedCount == 0) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
entry->MappedCount--;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Moves a clean, unshared frame to standby so pressure can reclaim it later.
|
|
__declspec(dllexport) KSTATUS MmPfnMoveToStandby(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State != MmPfnStateActive || entry->PinCount || entry->MappedCount || entry->RefCount != 1) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_ACCESS_DENIED;
|
|
}
|
|
if (entry->Usage != MmPfnUseCleanFileCache && entry->Usage != MmPfnUseImage) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
|
|
entry->State = MmPfnStateStandby;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Reactivates a standby frame after a cache or image lookup found it still useful.
|
|
__declspec(dllexport) KSTATUS MmPfnActivate(PHYSICAL_ADDRESS frame) {
|
|
size_t pfn = MmPfnIndex(frame);
|
|
if (pfn == MM_PFN_INVALID_INDEX)
|
|
return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State != MmPfnStateStandby || !entry->RefCount) {
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_INVALID_PARAMETER;
|
|
}
|
|
|
|
entry->State = MmPfnStateActive;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// Returns the number of frames that are immediately available for allocation.
|
|
size_t MmFreeFrameCount(void) {
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
size_t count = gFreeFrames;
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
return count;
|
|
}
|
|
|
|
// Returns the number of RAM frames managed by the PFN database.
|
|
size_t MmTotalFrameCount(void) {
|
|
return gTotalFrames;
|
|
}
|
|
|
|
// Returns detailed PFN state counters for VMM self-tests and diagnostics.
|
|
void MmPfnQueryStats(MM_PFN_STATS* out) {
|
|
if (!out)
|
|
return;
|
|
|
|
MM_PFN_STATS stats;
|
|
memset(&stats, 0, sizeof(stats));
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
stats.TotalFrames = (uint64_t)gTotalFrames;
|
|
stats.FreeFrames = (uint64_t)gFreeFrames;
|
|
stats.UsedFrames = stats.TotalFrames - stats.FreeFrames;
|
|
|
|
for (size_t pfn = 0; pfn < gMaxPfn; pfn++) {
|
|
const MM_PFN* entry = &gPfnDatabase[pfn];
|
|
if (entry->State == MmPfnStateInvalid)
|
|
continue;
|
|
switch ((MM_PFN_STATE)entry->State) {
|
|
case MmPfnStateReserved: stats.ReservedFrames++; break;
|
|
case MmPfnStateZeroed: stats.ZeroedFrames++; break;
|
|
case MmPfnStateActive: stats.ActiveFrames++; break;
|
|
case MmPfnStateStandby: stats.StandbyFrames++; break;
|
|
case MmPfnStateModified: stats.ModifiedFrames++; break;
|
|
default: break;
|
|
}
|
|
if (entry->PinCount)
|
|
stats.PinnedFrames++;
|
|
if (entry->MappedCount)
|
|
stats.MappedFrames++;
|
|
if (KiPfnEvictableLocked(entry))
|
|
stats.EvictableFrames++;
|
|
else if (entry->State != MmPfnStateFree && entry->State != MmPfnStateZeroed)
|
|
stats.NonEvictableFrames++;
|
|
if (KiPfnReclaimableLocked(entry))
|
|
stats.ReclaimableFrames++;
|
|
|
|
switch ((MM_PFN_USAGE)entry->Usage) {
|
|
case MmPfnUseKernel: stats.KernelFrames++; break;
|
|
case MmPfnUsePageTable: stats.PageTableFrames++; break;
|
|
case MmPfnUseUser: stats.UserFrames++; break;
|
|
case MmPfnUseCleanFileCache:
|
|
stats.CleanFileCacheFrames++;
|
|
if (entry->State == MmPfnStateStandby) stats.StandbyCleanFileCacheFrames++;
|
|
else if (entry->State == MmPfnStateActive) stats.ActiveCleanFileCacheFrames++;
|
|
break;
|
|
case MmPfnUseImage:
|
|
stats.ImageFrames++;
|
|
if (entry->State == MmPfnStateStandby) stats.StandbyImageFrames++;
|
|
else if (entry->State == MmPfnStateActive) stats.ActiveImageFrames++;
|
|
break;
|
|
case MmPfnUseMappedFile: stats.MappedFileFrames++; break;
|
|
case MmPfnUseAnonymous: stats.AnonymousFrames++; break;
|
|
case MmPfnUseCow: stats.CowFrames++; break;
|
|
case MmPfnUseSurface: stats.SurfaceFrames++; break;
|
|
case MmPfnUseDriver: stats.DriverFrames++; break;
|
|
case MmPfnUseDma: stats.DmaFrames++; break;
|
|
case MmPfnUseBootloader: stats.BootloaderFrames++; break;
|
|
case MmPfnUsePfnDatabase: stats.PfnDatabaseFrames++; break;
|
|
case MmPfnUseMmio: stats.MmioFrames++; break;
|
|
default: break;
|
|
}
|
|
}
|
|
|
|
stats.AvailableFrames = stats.FreeFrames + stats.ReclaimableFrames;
|
|
|
|
stats.AllocationCount = gPfnAllocations;
|
|
stats.FreeCount = gPfnFrees;
|
|
stats.RetainCount = gPfnRetains;
|
|
stats.ReleaseCount = gPfnReleases;
|
|
stats.PinCount = gPfnPins;
|
|
stats.UnpinCount = gPfnUnpins;
|
|
stats.FailedAllocations = gPfnFailedAllocations;
|
|
stats.FailedContiguousAllocations = gPfnFailedContiguousAllocations;
|
|
stats.FailedInvalidRange = gPfnFailedInvalidRange;
|
|
stats.FailedEmptyRange = gPfnFailedEmptyRange;
|
|
stats.FailedReserveDenied = gPfnFailedReserveDenied;
|
|
stats.FailedNoFrame = gPfnFailedNoFrame;
|
|
stats.FailedContiguousInvalidRequest = gPfnFailedContiguousInvalidRequest;
|
|
stats.FailedContiguousReserveDenied = gPfnFailedContiguousReserveDenied;
|
|
stats.FailedContiguousNoRun = gPfnFailedContiguousNoRun;
|
|
stats.ReclaimAttempts = gPfnReclaimAttempts;
|
|
stats.ReclaimPagesRequested = gPfnReclaimPagesRequested;
|
|
stats.ReclaimPagesSucceeded = gPfnReclaimPagesSucceeded;
|
|
stats.ReclaimPagesFailed = gPfnReclaimPagesFailed;
|
|
stats.ReclaimReentrantSkips = gPfnReclaimReentrantSkips;
|
|
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
*out = stats;
|
|
}
|
|
|
|
// Returns the PMM counters expected by the existing stats syscall.
|
|
void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out) {
|
|
if (!out)
|
|
return;
|
|
|
|
MM_PHYSICAL_STATS stats;
|
|
memset(&stats, 0, sizeof(stats));
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
|
stats.TotalFrames = (uint64_t)gTotalFrames;
|
|
stats.FreeFrames = (uint64_t)gFreeFrames;
|
|
stats.UsedFrames = stats.TotalFrames - stats.FreeFrames;
|
|
|
|
uint64_t run = 0;
|
|
for (size_t pfn = 1; pfn < gMaxPfn; pfn++) {
|
|
if (KiPfnAvailableLocked(pfn)) {
|
|
run++;
|
|
continue;
|
|
}
|
|
|
|
if (run) {
|
|
stats.FreeRunCount++;
|
|
if (run > stats.LargestFreeRun)
|
|
stats.LargestFreeRun = run;
|
|
if (run == 1) stats.FreeRunHistogram[0]++;
|
|
else if (run <= 3) stats.FreeRunHistogram[1]++;
|
|
else if (run <= 7) stats.FreeRunHistogram[2]++;
|
|
else if (run <= 15) stats.FreeRunHistogram[3]++;
|
|
else if (run <= 63) stats.FreeRunHistogram[4]++;
|
|
else if (run <= 255) stats.FreeRunHistogram[5]++;
|
|
else if (run <= 1023) stats.FreeRunHistogram[6]++;
|
|
else stats.FreeRunHistogram[7]++;
|
|
run = 0;
|
|
}
|
|
}
|
|
|
|
if (run) {
|
|
stats.FreeRunCount++;
|
|
if (run > stats.LargestFreeRun)
|
|
stats.LargestFreeRun = run;
|
|
if (run == 1) stats.FreeRunHistogram[0]++;
|
|
else if (run <= 3) stats.FreeRunHistogram[1]++;
|
|
else if (run <= 7) stats.FreeRunHistogram[2]++;
|
|
else if (run <= 15) stats.FreeRunHistogram[3]++;
|
|
else if (run <= 63) stats.FreeRunHistogram[4]++;
|
|
else if (run <= 255) stats.FreeRunHistogram[5]++;
|
|
else if (run <= 1023) stats.FreeRunHistogram[6]++;
|
|
else stats.FreeRunHistogram[7]++;
|
|
}
|
|
|
|
KeReleaseSpinlock(&gPfnLock, flags);
|
|
*out = stats;
|
|
}
|