1451 lines
51 KiB
C
1451 lines
51 KiB
C
// mm_virt.c - virtual memory manager
|
|
//
|
|
// Page tables:
|
|
// One PML4 shared by all processes.
|
|
// Upper half (bit 47 set) has U/S = 0 - ring-3 access always faults.
|
|
// Lower half pages are PAGE_USER.
|
|
//
|
|
// Kernel pool:
|
|
// A boundary-tag (header + footer) free-list allocator starting at POOL_BASE.
|
|
// Pages are committed on demand from the PMM.
|
|
//
|
|
// User heap:
|
|
// A single best-fit free list in the lower half.
|
|
// Page-granular lower-half regions owned by PROCESS metadata. Sub-page
|
|
// allocation is a user-mode runtime concern.
|
|
#include <stddef.h>
|
|
#include <string.h>
|
|
#include <intrin.h>
|
|
#include <kernel/mm_virt.h>
|
|
#include <kernel/ke_process.h>
|
|
#include <kernel/ke_cpu.h>
|
|
#include <kernel/ke_debug.h>
|
|
#include <kernel/ke_spinlock.h>
|
|
#include <kernel/ke_irql.h>
|
|
#include <kernel/hal/hal_apic.h>
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers - physical window access
|
|
// ---------------------------------------------------------------------------
|
|
static uint64_t gPhysBase = PHYS_MAP_BASE;
|
|
|
|
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
|
|
return (void *)(p + gPhysBase);
|
|
}
|
|
static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
|
|
return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// PML4 / page-table helpers
|
|
// ---------------------------------------------------------------------------
|
|
static PHYSICAL_ADDRESS gPml4Phys = 0;
|
|
static SPINLOCK gPageTableLock = SPINLOCK_INIT;
|
|
|
|
static volatile long gTlbShootdownLock = 0;
|
|
static volatile long gTlbShootdownActive = 0;
|
|
static volatile long gTlbShootdownAcks = 0;
|
|
static uintptr_t gTlbShootdownVirt = 0;
|
|
|
|
#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u
|
|
|
|
// Extract the page-table index for each level from a virtual address.
|
|
#define PML4_IDX(va) (((va) >> 39) & 0x1FF)
|
|
#define PDPT_IDX(va) (((va) >> 30) & 0x1FF)
|
|
#define PDT_IDX(va) (((va) >> 21) & 0x1FF)
|
|
#define PT_IDX(va) (((va) >> 12) & 0x1FF)
|
|
|
|
#define PAGE_PS (1ULL << 7)
|
|
#define PAGE_FLAGS_MASK (0xFFFULL | PAGE_NX)
|
|
#define PAGE_FRAME_4K_MASK 0x000FFFFFFFFFF000ULL
|
|
#define PAGE_FRAME_2M_MASK 0x000FFFFFFFE00000ULL
|
|
#define PAGE_FRAME_1G_MASK 0x000FFFFFC0000000ULL
|
|
|
|
// Ensure a page-table entry at 'parentPhys[index]' points to a child table.
|
|
// Allocates and zeros a new frame if the entry is not present.
|
|
// Returns the physical address of the child table, or 0 on OOM.
|
|
static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
|
|
unsigned index, uint64_t extraFlags) {
|
|
uint64_t* entry = KiPtePtr(parentPhys, index);
|
|
if (*entry & PAGE_PRESENT) {
|
|
*entry |= extraFlags;
|
|
return *entry & PAGE_FRAME_4K_MASK;
|
|
}
|
|
PHYSICAL_ADDRESS childPhys = MmAllocateFrame();
|
|
if (!childPhys) return 0;
|
|
memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE);
|
|
*entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags;
|
|
return childPhys;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// TLB/cache coherency hooks
|
|
// ---------------------------------------------------------------------------
|
|
void MmFlushTlbLocal(uintptr_t virt) {
|
|
__invlpg((void*)virt);
|
|
}
|
|
|
|
static void KiAcquireTlbShootdownLock(void) {
|
|
// Do not use the normal spinlock here: it masks interrupts while spinning.
|
|
// A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
|
|
// concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is
|
|
// meant to avoid.
|
|
while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0)
|
|
_mm_pause();
|
|
_ReadWriteBarrier();
|
|
}
|
|
|
|
static void KiReleaseTlbShootdownLock(void) {
|
|
_ReadWriteBarrier();
|
|
_InterlockedExchange(&gTlbShootdownLock, 0);
|
|
}
|
|
|
|
void MmFlushTlbForKernelMapping(uintptr_t virt) {
|
|
MmFlushTlbLocal(virt);
|
|
|
|
if (gCpuCount <= 1)
|
|
return;
|
|
|
|
KCPU* self = KeGetCurrentCpu();
|
|
uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU;
|
|
|
|
KiAcquireTlbShootdownLock();
|
|
|
|
long targets = 0;
|
|
gTlbShootdownVirt = virt;
|
|
_InterlockedExchange(&gTlbShootdownAcks, 0);
|
|
_ReadWriteBarrier();
|
|
_InterlockedExchange(&gTlbShootdownActive, 1);
|
|
_ReadWriteBarrier();
|
|
|
|
for (uint32_t i = 0; i < gCpuCount; i++) {
|
|
KCPU* cpu = &gCpus[i];
|
|
if (i == selfIndex)
|
|
continue;
|
|
if (!cpu->Present || !cpu->Started || !cpu->Online)
|
|
continue;
|
|
targets++;
|
|
HalApicSendTlbShootdownIpi(cpu->LapicId);
|
|
}
|
|
|
|
for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) {
|
|
if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets)
|
|
break;
|
|
_mm_pause();
|
|
}
|
|
|
|
long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0);
|
|
_InterlockedExchange(&gTlbShootdownActive, 0);
|
|
_ReadWriteBarrier();
|
|
KiReleaseTlbShootdownLock();
|
|
|
|
if (acks < targets) {
|
|
KePanic(KE_PANIC_CODE_GENERIC,
|
|
"TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n",
|
|
(uint64_t)virt, targets, acks, selfIndex);
|
|
}
|
|
}
|
|
|
|
static int KiHandleTlbShootdown(void) {
|
|
if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
|
|
return 0;
|
|
|
|
uintptr_t va = gTlbShootdownVirt;
|
|
_ReadWriteBarrier();
|
|
MmFlushTlbLocal(va);
|
|
_ReadWriteBarrier();
|
|
_InterlockedIncrement(&gTlbShootdownAcks);
|
|
return 1;
|
|
}
|
|
|
|
int MmHandleTlbShootdownNmi(void) {
|
|
return KiHandleTlbShootdown();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Public mapping
|
|
// ---------------------------------------------------------------------------
|
|
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode) {
|
|
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
|
|
|
|
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
|
|
|
|
PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
|
|
if (!pdpt) {
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return STATUS_NO_MEMORY;
|
|
}
|
|
PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
|
|
if (!pdt) {
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return STATUS_NO_MEMORY;
|
|
}
|
|
PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag);
|
|
if (!pt) {
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return STATUS_NO_MEMORY;
|
|
}
|
|
|
|
uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
|
|
if ((*pte & PAGE_PRESENT) && mode == MmMapFailIfPresent) {
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return STATUS_ALREADY_EXISTS;
|
|
}
|
|
*pte = phys | flags;
|
|
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
MmFlushTlbForKernelMapping(virt);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
|
|
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) {
|
|
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
|
|
|
|
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
|
|
if (!(*e4 & PAGE_PRESENT)) {
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return STATUS_NOT_FOUND;
|
|
}
|
|
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt));
|
|
if (!(*e3 & PAGE_PRESENT)) {
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return STATUS_NOT_FOUND;
|
|
}
|
|
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt));
|
|
if (!(*e2 & PAGE_PRESENT)) {
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return STATUS_NOT_FOUND;
|
|
}
|
|
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt));
|
|
*pte = 0;
|
|
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
MmFlushTlbForKernelMapping(virt);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
|
|
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags) {
|
|
if (!size) return STATUS_SUCCESS;
|
|
|
|
// Round down to page boundary, round size up.
|
|
uintptr_t base = virt & PAGE_MASK;
|
|
uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
|
|
|
|
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
|
|
KSTATUS status = STATUS_SUCCESS;
|
|
uintptr_t flushEnd = base;
|
|
|
|
for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
|
|
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
|
|
if (!(*e4 & PAGE_PRESENT)) {
|
|
status = STATUS_NOT_FOUND;
|
|
break;
|
|
}
|
|
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
|
|
if (!(*e3 & PAGE_PRESENT)) {
|
|
status = STATUS_NOT_FOUND;
|
|
break;
|
|
}
|
|
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
|
|
if (!(*e2 & PAGE_PRESENT)) {
|
|
status = STATUS_NOT_FOUND;
|
|
break;
|
|
}
|
|
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
|
|
if (!(*pte & PAGE_PRESENT)) {
|
|
status = STATUS_NOT_FOUND;
|
|
break;
|
|
}
|
|
|
|
// Preserve the physical address, replace everything else.
|
|
PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK;
|
|
*pte = phys | flags;
|
|
flushEnd = va + PAGE_SIZE;
|
|
}
|
|
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
|
|
for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE)
|
|
MmFlushTlbForKernelMapping(va);
|
|
|
|
return status;
|
|
}
|
|
|
|
|
|
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
|
|
if (!gPml4Phys) return 0; // Not initialized
|
|
|
|
PHYSICAL_ADDRESS result = 0;
|
|
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
|
|
|
|
uint64_t pml4_idx = (virt >> 39) & 0x1FF;
|
|
uint64_t pdpt_idx = (virt >> 30) & 0x1FF;
|
|
uint64_t pd_idx = (virt >> 21) & 0x1FF;
|
|
uint64_t pt_idx = (virt >> 12) & 0x1FF;
|
|
|
|
uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
|
|
if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out;
|
|
|
|
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK);
|
|
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out;
|
|
|
|
// Check for 1GB huge pages (bit 7 - PS)
|
|
if (pdpt[pdpt_idx] & PAGE_PS) {
|
|
result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF);
|
|
goto out;
|
|
}
|
|
|
|
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK);
|
|
if (!(pd[pd_idx] & PAGE_PRESENT)) goto out;
|
|
|
|
// Check for 2MB huge pages (bit 7 - PS)
|
|
if (pd[pd_idx] & PAGE_PS) {
|
|
result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF);
|
|
goto out;
|
|
}
|
|
|
|
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK);
|
|
if (!(pt[pt_idx] & PAGE_PRESENT)) goto out;
|
|
|
|
// Extract the physical frame and add the offset within the page
|
|
result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
|
|
|
|
out:
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return result;
|
|
}
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Page query and user/kernel copy boundary helpers
|
|
// ---------------------------------------------------------------------------
|
|
__declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
|
|
PHYSICAL_ADDRESS* physOut,
|
|
uint64_t* flagsOut) {
|
|
if (!gPml4Phys) return STATUS_NOT_FOUND;
|
|
|
|
KSTATUS status = STATUS_NOT_FOUND;
|
|
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
|
|
|
|
uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys);
|
|
uint64_t e4 = pml4[PML4_IDX(virt)];
|
|
if (!(e4 & PAGE_PRESENT)) goto out;
|
|
|
|
uint64_t effectiveWrite = e4 & PAGE_WRITE;
|
|
uint64_t effectiveUser = e4 & PAGE_USER;
|
|
uint64_t effectiveNx = e4 & PAGE_NX;
|
|
|
|
uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK);
|
|
uint64_t e3 = pdpt[PDPT_IDX(virt)];
|
|
if (!(e3 & PAGE_PRESENT)) goto out;
|
|
effectiveWrite &= e3 & PAGE_WRITE;
|
|
effectiveUser &= e3 & PAGE_USER;
|
|
effectiveNx |= e3 & PAGE_NX;
|
|
|
|
if (e3 & PAGE_PS) {
|
|
if (physOut)
|
|
*physOut = (e3 & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFFULL);
|
|
if (flagsOut) {
|
|
uint64_t flags = (e3 & PAGE_FLAGS_MASK) | effectiveNx;
|
|
if (!effectiveWrite) flags &= ~PAGE_WRITE;
|
|
if (!effectiveUser) flags &= ~PAGE_USER;
|
|
*flagsOut = flags;
|
|
}
|
|
status = STATUS_SUCCESS;
|
|
goto out;
|
|
}
|
|
|
|
uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK);
|
|
uint64_t e2 = pd[PDT_IDX(virt)];
|
|
if (!(e2 & PAGE_PRESENT)) goto out;
|
|
effectiveWrite &= e2 & PAGE_WRITE;
|
|
effectiveUser &= e2 & PAGE_USER;
|
|
effectiveNx |= e2 & PAGE_NX;
|
|
|
|
if (e2 & PAGE_PS) {
|
|
if (physOut)
|
|
*physOut = (e2 & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFFULL);
|
|
if (flagsOut) {
|
|
uint64_t flags = (e2 & PAGE_FLAGS_MASK) | effectiveNx;
|
|
if (!effectiveWrite) flags &= ~PAGE_WRITE;
|
|
if (!effectiveUser) flags &= ~PAGE_USER;
|
|
*flagsOut = flags;
|
|
}
|
|
status = STATUS_SUCCESS;
|
|
goto out;
|
|
}
|
|
|
|
uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK);
|
|
uint64_t e1 = pt[PT_IDX(virt)];
|
|
if (!(e1 & PAGE_PRESENT)) goto out;
|
|
effectiveWrite &= e1 & PAGE_WRITE;
|
|
effectiveUser &= e1 & PAGE_USER;
|
|
effectiveNx |= e1 & PAGE_NX;
|
|
|
|
if (physOut)
|
|
*physOut = (e1 & PAGE_FRAME_4K_MASK) + (virt & 0xFFFULL);
|
|
if (flagsOut) {
|
|
uint64_t flags = (e1 & PAGE_FLAGS_MASK) | effectiveNx;
|
|
if (!effectiveWrite) flags &= ~PAGE_WRITE;
|
|
if (!effectiveUser) flags &= ~PAGE_USER;
|
|
*flagsOut = flags;
|
|
}
|
|
status = STATUS_SUCCESS;
|
|
|
|
out:
|
|
KeReleaseSpinlock(&gPageTableLock, lockFlags);
|
|
return status;
|
|
}
|
|
|
|
|
|
static int KiUserRangeValid(uintptr_t addr, size_t size) {
|
|
if (size == 0) return 1;
|
|
if (addr < USER_BASE) return 0;
|
|
if (addr >= USER_LIMIT) return 0;
|
|
if (size > (size_t)(USER_LIMIT - addr)) return 0;
|
|
return 1;
|
|
}
|
|
|
|
static KSTATUS KiProbeUserRange(uintptr_t addr, size_t size, uint64_t requiredFlags) {
|
|
if (!KiUserRangeValid(addr, size))
|
|
return STATUS_ACCESS_DENIED;
|
|
|
|
if (size == 0)
|
|
return STATUS_SUCCESS;
|
|
|
|
uintptr_t va = addr & PAGE_MASK;
|
|
uintptr_t last = (addr + size - 1) & PAGE_MASK;
|
|
|
|
for (;;) {
|
|
uint64_t flags = 0;
|
|
KSTATUS s = MmQueryPage(va, (PHYSICAL_ADDRESS*)0, &flags);
|
|
if (!KSUCCESS(s))
|
|
return STATUS_ACCESS_DENIED;
|
|
if ((flags & requiredFlags) != requiredFlags)
|
|
return STATUS_ACCESS_DENIED;
|
|
if (va == last)
|
|
break;
|
|
va += PAGE_SIZE;
|
|
}
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS MmProbeUserRead(const void* ptr, size_t size) {
|
|
return KiProbeUserRange((uintptr_t)ptr, size, PAGE_PRESENT | PAGE_USER);
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size) {
|
|
return KiProbeUserRange((uintptr_t)ptr, size, PAGE_PRESENT | PAGE_USER | PAGE_WRITE);
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size) {
|
|
if (!kernelDst && size) return STATUS_INVALID_PARAMETER;
|
|
|
|
KSTATUS s = MmProbeUserRead(userSrc, size);
|
|
if (!KSUCCESS(s)) return s;
|
|
|
|
if (size) memcpy(kernelDst, userSrc, size);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size) {
|
|
if (!kernelSrc && size) return STATUS_INVALID_PARAMETER;
|
|
|
|
KSTATUS s = MmProbeUserWrite(userDst, size);
|
|
if (!KSUCCESS(s)) return s;
|
|
|
|
if (size) memcpy(userDst, kernelSrc, size);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Boundary-tag allocator (used for both pool and user heap)
|
|
// ---------------------------------------------------------------------------
|
|
// Each allocation has a BLOCK_HDR immediately before the user data, and a
|
|
// BLOCK_FTR immediately after. Both carry the block size and a 'free' flag.
|
|
// This allows O(1) coalescing of adjacent free blocks.
|
|
|
|
#define BLOCK_MAGIC 0xB10CB10CUL
|
|
|
|
typedef struct BLOCK_HDR {
|
|
uint32_t Magic;
|
|
uint32_t Free; // 1 = free, 0 = used
|
|
size_t Size; // usable bytes (excludes header + footer)
|
|
struct BLOCK_HDR* PrevFree;
|
|
struct BLOCK_HDR* NextFree;
|
|
} BLOCK_HDR;
|
|
|
|
typedef struct BLOCK_FTR {
|
|
size_t Size;
|
|
uint32_t Free;
|
|
uint32_t Magic;
|
|
} BLOCK_FTR;
|
|
|
|
#define HDR_SIZE (sizeof(BLOCK_HDR))
|
|
#define FTR_SIZE (sizeof(BLOCK_FTR))
|
|
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
|
|
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
|
|
|
|
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
|
|
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
|
|
}
|
|
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
|
|
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
|
|
}
|
|
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
|
|
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
|
|
}
|
|
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
|
|
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
|
|
}
|
|
|
|
typedef struct HEAP {
|
|
BLOCK_HDR* FreeList; // doubly-linked list of free blocks
|
|
uintptr_t Start; // virtual base of the arena
|
|
uintptr_t Brk; // current committed end (page-aligned)
|
|
uintptr_t Limit; // hard upper bound
|
|
uint64_t MapFlags; // PAGE_* flags for new committed pages
|
|
SPINLOCK Lock;
|
|
} HEAP;
|
|
|
|
static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) {
|
|
if (!h || !size) return 1;
|
|
if (start < h->Start) return 0;
|
|
if (start > (uintptr_t)-1 - (size - 1)) return 0;
|
|
|
|
uintptr_t begin = start & PAGE_MASK;
|
|
uintptr_t end = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
|
|
if (end < begin || end > h->Limit) return 0;
|
|
|
|
for (uintptr_t va = begin; va < end; va += PAGE_SIZE) {
|
|
PHYSICAL_ADDRESS phys = 0;
|
|
uint64_t flags = 0;
|
|
KSTATUS qs = MmQueryPage(va, &phys, &flags);
|
|
if (KSUCCESS(qs) && (flags & PAGE_PRESENT))
|
|
continue;
|
|
|
|
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
|
if (!frame) return 0;
|
|
KSTATUS ms = MmMapPage(va, frame, h->MapFlags, MmMapFailIfPresent);
|
|
if (!KSUCCESS(ms)) {
|
|
MmFreeFrame(frame);
|
|
return 0;
|
|
}
|
|
memset((void*)va, 0, PAGE_SIZE);
|
|
DbgPrint("MM: repaired missing heap page va=%016llx\n", (unsigned long long)va);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
// Extend heap by at least 'need' bytes (rounded up to page boundary).
|
|
static int KiHeapGrow(HEAP* h, size_t need) {
|
|
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
|
if (h->Brk + grow > h->Limit) return 0;
|
|
|
|
for (uintptr_t va = h->Brk; va < h->Brk + grow; va += PAGE_SIZE) {
|
|
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
|
if (!frame) return 0;
|
|
if (!KSUCCESS(MmMapPage(va, frame, h->MapFlags, MmMapFailIfPresent))) {
|
|
MmFreeFrame(frame);
|
|
return 0;
|
|
}
|
|
memset((void *)va, 0, PAGE_SIZE);
|
|
}
|
|
|
|
if (!KiHeapEnsureMapped(h, h->Brk, grow))
|
|
return 0;
|
|
|
|
// Make the new pages a single free block.
|
|
BLOCK_HDR* blk = (BLOCK_HDR *)h->Brk;
|
|
blk->Magic = BLOCK_MAGIC;
|
|
blk->Free = 1;
|
|
blk->Size = grow - OVERHEAD;
|
|
blk->PrevFree = (BLOCK_HDR *)0;
|
|
blk->NextFree = h->FreeList;
|
|
if (h->FreeList) h->FreeList->PrevFree = blk;
|
|
h->FreeList = blk;
|
|
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
|
KiHdrToFtr(blk)->Free = 1;
|
|
KiHdrToFtr(blk)->Size = blk->Size;
|
|
|
|
h->Brk += grow;
|
|
return 1;
|
|
}
|
|
|
|
// Unlink a free block from the free list.
|
|
static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
|
|
if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
|
|
else h->FreeList = blk->NextFree;
|
|
if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
|
|
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
|
|
}
|
|
|
|
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
|
|
(void)h;
|
|
if (!blk || blk->Free == 0) return;
|
|
|
|
// Do not decommit kernel-pool tail pages for now. The pool allocator keeps
|
|
// free-list nodes and boundary tags inside the managed arena; shrinking the
|
|
// mapped tail while large launch-failure cleanup is walking/freeing many
|
|
// objects can leave stale metadata pointers into unmapped pages. Physical
|
|
// pages are still reclaimed for user regions by MmDestroyUserRegion(); pool
|
|
// pages remain committed until we have a dedicated, audited heap shrinker.
|
|
KiHdrToFtr(blk)->Size = blk->Size;
|
|
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
|
KiHdrToFtr(blk)->Free = 1;
|
|
}
|
|
|
|
static void* KiHeapAlloc(HEAP* h, size_t size) {
|
|
if (!size) return (void *)0;
|
|
size = (size + 7) & ~7ULL; // 8-byte align
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
|
|
|
|
if (h->Brk > h->Start && !KiHeapEnsureMapped(h, h->Start, h->Brk - h->Start)) {
|
|
KeReleaseSpinlock(&h->Lock, flags);
|
|
return (void*)0;
|
|
}
|
|
|
|
// Best-fit search.
|
|
BLOCK_HDR* best = (BLOCK_HDR *)0;
|
|
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
|
|
if (cur->Size >= size) {
|
|
if (!best || cur->Size < best->Size) best = cur;
|
|
if (best->Size == size) break;
|
|
}
|
|
}
|
|
|
|
if (!best) {
|
|
if (!KiHeapGrow(h, size + OVERHEAD)) {
|
|
KeReleaseSpinlock(&h->Lock, flags);
|
|
return (void *)0;
|
|
}
|
|
// Re-search; KiHeapGrow inserted at the front.
|
|
best = h->FreeList;
|
|
while (best && best->Size < size) best = best->NextFree;
|
|
if (!best) {
|
|
KeReleaseSpinlock(&h->Lock, flags);
|
|
return (void *)0;
|
|
}
|
|
}
|
|
|
|
KiFreeListRemove(h, best);
|
|
|
|
// Split the block if the remainder is big enough.
|
|
if (best->Size >= size + OVERHEAD + sizeof(void*)) {
|
|
BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE);
|
|
rem->Magic = BLOCK_MAGIC;
|
|
rem->Free = 1;
|
|
rem->Size = best->Size - size - OVERHEAD;
|
|
rem->PrevFree = (BLOCK_HDR *)0;
|
|
rem->NextFree = h->FreeList;
|
|
if (h->FreeList) h->FreeList->PrevFree = rem;
|
|
h->FreeList = rem;
|
|
KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
|
|
KiHdrToFtr(rem)->Free = 1;
|
|
KiHdrToFtr(rem)->Size = rem->Size;
|
|
|
|
best->Size = size;
|
|
}
|
|
|
|
best->Free = 0;
|
|
KiHdrToFtr(best)->Free = 0;
|
|
KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
|
|
KiHdrToFtr(best)->Size = best->Size;
|
|
|
|
memset((char *)best + HDR_SIZE, 0, size);
|
|
KeReleaseSpinlock(&h->Lock, flags);
|
|
return (char *)best + HDR_SIZE;
|
|
}
|
|
|
|
static void KiHeapFree(HEAP* h, void* ptr) {
|
|
if (!ptr) return;
|
|
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
|
|
if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
|
|
|
|
blk->Free = 1;
|
|
|
|
// Coalesce with next block if it is free.
|
|
BLOCK_HDR* next = KiNextBlock(blk);
|
|
if ((uintptr_t)next < h->Brk && next->Magic == BLOCK_MAGIC && next->Free) {
|
|
KiFreeListRemove(h, next);
|
|
blk->Size += OVERHEAD + next->Size;
|
|
KiHdrToFtr(blk)->Size = blk->Size;
|
|
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
|
KiHdrToFtr(blk)->Free = 1;
|
|
}
|
|
|
|
// Coalesce with previous block if it is free.
|
|
if ((uintptr_t)blk > h->Start) {
|
|
BLOCK_FTR* prevFtr = KiPrevFtr(blk);
|
|
if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free) {
|
|
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
|
|
KiFreeListRemove(h, prev);
|
|
prev->Size += OVERHEAD + blk->Size;
|
|
KiHdrToFtr(prev)->Size = prev->Size;
|
|
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
|
|
KiHdrToFtr(prev)->Free = 1;
|
|
blk = prev;
|
|
}
|
|
}
|
|
|
|
// Insert at head of free list.
|
|
blk->PrevFree = (BLOCK_HDR *)0;
|
|
blk->NextFree = h->FreeList;
|
|
if (h->FreeList) h->FreeList->PrevFree = blk;
|
|
h->FreeList = blk;
|
|
|
|
KiHdrToFtr(blk)->Free = 1;
|
|
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
|
|
|
KiHeapTryDecommitTail(h, blk);
|
|
|
|
KeReleaseSpinlock(&h->Lock, flags);
|
|
}
|
|
|
|
static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
|
|
// If ptr is NULL, realloc acts like malloc
|
|
if (!ptr) {
|
|
return KiHeapAlloc(h, new_size);
|
|
}
|
|
|
|
// If size is 0, realloc acts like free
|
|
if (new_size == 0) {
|
|
KiHeapFree(h, ptr);
|
|
return (void*)0;
|
|
}
|
|
|
|
// Retrieve the header of the existing block
|
|
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
|
|
|
|
// Sanity checks to prevent operating on corrupted or already-freed pointers
|
|
if (blk->Magic != BLOCK_MAGIC || blk->Free) {
|
|
return (void*)0;
|
|
}
|
|
|
|
// Align the requested size (matches KiHeapAlloc 8-byte alignment)
|
|
size_t aligned_size = (new_size + 7) & ~7ULL;
|
|
|
|
// If the current block is already big enough, we can reuse it in-place.
|
|
// (Note: To save space, you could optionally split the block here if
|
|
// the new size is significantly smaller, similar to KiHeapAlloc's logic).
|
|
if (blk->Size >= aligned_size) {
|
|
return ptr;
|
|
}
|
|
|
|
// The block is too small. Allocate a new, larger block.
|
|
void* new_ptr = KiHeapAlloc(h, aligned_size);
|
|
if (!new_ptr) {
|
|
return (void*)0; // Out of memory, original block is left untouched
|
|
}
|
|
memset(new_ptr, 0, aligned_size);
|
|
|
|
// Copy the old data to the new block.
|
|
// We only copy up to the old block's usable size.
|
|
memcpy(new_ptr, ptr, blk->Size);
|
|
|
|
// Free the old block
|
|
KiHeapFree(h, ptr);
|
|
|
|
return new_ptr;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Kernel pool
|
|
// ---------------------------------------------------------------------------
|
|
static HEAP gPoolHeap;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// User virtual region arena
|
|
// ---------------------------------------------------------------------------
|
|
typedef struct USER_VA_REGION {
|
|
LIST_ENTRY ProcessLink;
|
|
LIST_ENTRY GlobalLink;
|
|
PROCESS* Owner;
|
|
|
|
uintptr_t ReservationBase;
|
|
size_t ReservationSize;
|
|
uintptr_t Base;
|
|
size_t Size;
|
|
size_t CommittedSize;
|
|
|
|
uint64_t PageFlags;
|
|
USER_REGION_TYPE Type;
|
|
uint32_t Flags;
|
|
} USER_VA_REGION;
|
|
|
|
static LIST_ENTRY gUserRegionList;
|
|
static SPINLOCK gUserRegionLock = SPINLOCK_INIT;
|
|
static int gUserRegionListReady = 0;
|
|
|
|
#define USER_REGION_ARENA_BASE 0x0000100000000000ULL
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Module init
|
|
// ---------------------------------------------------------------------------
|
|
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
|
|
gPml4Phys = bootPml4Phys;
|
|
gPageTableLock = (SPINLOCK)SPINLOCK_INIT;
|
|
|
|
// Kernel pool heap.
|
|
gPoolHeap.FreeList = (BLOCK_HDR *)0;
|
|
gPoolHeap.Start = POOL_BASE;
|
|
gPoolHeap.Brk = POOL_BASE;
|
|
gPoolHeap.Limit = POOL_LIMIT;
|
|
gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
|
|
gPoolHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
|
|
|
|
// Pre-commit the requested pool size.
|
|
if (poolSize) {
|
|
poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
|
KiHeapGrow(&gPoolHeap, poolSize);
|
|
}
|
|
|
|
// User VA arena. One global lower-half virtual-address bucket, with
|
|
// per-process ownership tracked by USER_VA_REGION descriptors.
|
|
LIST_INIT(&gUserRegionList);
|
|
gUserRegionLock = (SPINLOCK)SPINLOCK_INIT;
|
|
gUserRegionListReady = 1;
|
|
}
|
|
|
|
void MmActivate(void) {
|
|
__writecr3(gPml4Phys);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Public pool API
|
|
// ---------------------------------------------------------------------------
|
|
__declspec(dllexport) void* MmAllocatePool(size_t size) { return KiHeapAlloc(&gPoolHeap, size); }
|
|
__declspec(dllexport) void MmFreePool(void* ptr) { KiHeapFree(&gPoolHeap, ptr); }
|
|
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); }
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// User virtual region API
|
|
// ---------------------------------------------------------------------------
|
|
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
|
|
if (align < PAGE_SIZE) align = PAGE_SIZE;
|
|
if (align & (align - 1)) return 0;
|
|
return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1);
|
|
}
|
|
|
|
static inline size_t KiAlignUpSize(size_t v, size_t align) {
|
|
if (align < PAGE_SIZE) align = PAGE_SIZE;
|
|
if (align & (align - 1)) return 0;
|
|
return (v + align - 1) & ~(align - 1);
|
|
}
|
|
|
|
static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) {
|
|
uintptr_t aEnd = aBase + aSize;
|
|
uintptr_t bEnd = bBase + bSize;
|
|
return aBase < bEnd && bBase < aEnd;
|
|
}
|
|
|
|
static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) {
|
|
if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER;
|
|
if (!gUserRegionListReady) return STATUS_FAILURE;
|
|
|
|
uintptr_t candidate = KiAlignUpPtr(USER_REGION_ARENA_BASE, alignment);
|
|
if (!candidate) return STATUS_INVALID_PARAMETER;
|
|
|
|
for (;;) {
|
|
if (candidate < USER_REGION_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate))
|
|
return STATUS_NO_MEMORY;
|
|
|
|
uintptr_t next = candidate;
|
|
int collided = 0;
|
|
|
|
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
|
if (KiRangesOverlap(candidate, reservationSize, r->ReservationBase, r->ReservationSize)) {
|
|
uintptr_t end = r->ReservationBase + r->ReservationSize;
|
|
next = KiAlignUpPtr(end, alignment);
|
|
if (!next || next <= candidate) return STATUS_NO_MEMORY;
|
|
collided = 1;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!collided) {
|
|
*baseOut = candidate;
|
|
return STATUS_SUCCESS;
|
|
}
|
|
candidate = next;
|
|
}
|
|
}
|
|
|
|
static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) {
|
|
if (!process || !base) return (USER_VA_REGION*)0;
|
|
uintptr_t b = (uintptr_t)base;
|
|
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
|
if (r->Base == b) return r;
|
|
}
|
|
return (USER_VA_REGION*)0;
|
|
}
|
|
|
|
static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) {
|
|
if (!process || !base || !size) return (USER_VA_REGION*)0;
|
|
if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0;
|
|
uintptr_t end = base + size;
|
|
|
|
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
|
uintptr_t regionEnd = r->Base + r->CommittedSize;
|
|
if (base >= r->Base && end <= regionEnd) return r;
|
|
}
|
|
return (USER_VA_REGION*)0;
|
|
}
|
|
|
|
static void KiUnmapUserRegionPayload(USER_VA_REGION* r) {
|
|
if (!r) return;
|
|
uintptr_t end = r->Base + r->CommittedSize;
|
|
for (uintptr_t va = r->Base; va < end; va += PAGE_SIZE) {
|
|
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va);
|
|
if (!phys) continue;
|
|
if (KSUCCESS(MmUnmapPage(va))) {
|
|
MmFreeFrame(phys);
|
|
}
|
|
}
|
|
}
|
|
|
|
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
|
|
uint64_t pageFlags, uint32_t regionFlags,
|
|
USER_REGION_TYPE type, void** outBase) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
if (!process || !outBase || !size) return STATUS_INVALID_PARAMETER;
|
|
*outBase = (void*)0;
|
|
|
|
if (!(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER;
|
|
if (!(pageFlags & PAGE_PRESENT)) pageFlags |= PAGE_PRESENT;
|
|
|
|
size_t payloadSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
|
if (payloadSize < size) return STATUS_NO_MEMORY;
|
|
|
|
size_t guardLow = (regionFlags & USER_REGION_GUARD_LOW) ? PAGE_SIZE : 0;
|
|
size_t guardHigh = (regionFlags & USER_REGION_GUARD_HIGH) ? PAGE_SIZE : 0;
|
|
size_t reservationSize = guardLow + payloadSize + guardHigh;
|
|
if (reservationSize < payloadSize) return STATUS_NO_MEMORY;
|
|
|
|
if (alignment < PAGE_SIZE) alignment = PAGE_SIZE;
|
|
if (alignment & (alignment - 1)) return STATUS_INVALID_PARAMETER;
|
|
|
|
USER_VA_REGION* r = (USER_VA_REGION*)MmAllocatePool(sizeof(USER_VA_REGION));
|
|
if (!r) return STATUS_NO_MEMORY;
|
|
memset(r, 0, sizeof(*r));
|
|
LIST_INIT(&r->ProcessLink);
|
|
LIST_INIT(&r->GlobalLink);
|
|
|
|
uintptr_t lockFlags = KeAcquireSpinlock(&gUserRegionLock);
|
|
uintptr_t reservationBase = 0;
|
|
KSTATUS s = KiFindUserVaRange(reservationSize, alignment, &reservationBase);
|
|
if (KSUCCESS(s)) {
|
|
r->Owner = process;
|
|
r->ReservationBase = reservationBase;
|
|
r->ReservationSize = reservationSize;
|
|
r->Base = reservationBase + guardLow;
|
|
r->Size = size;
|
|
r->CommittedSize = payloadSize;
|
|
r->PageFlags = pageFlags;
|
|
r->Type = type;
|
|
r->Flags = regionFlags;
|
|
ListInsertTail(&gUserRegionList, &r->GlobalLink);
|
|
}
|
|
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
|
|
|
|
if (!KSUCCESS(s)) {
|
|
MmFreePool(r);
|
|
return s;
|
|
}
|
|
|
|
uintptr_t va = r->Base;
|
|
for (; va < r->Base + payloadSize; va += PAGE_SIZE) {
|
|
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
|
if (!frame) {
|
|
s = STATUS_NO_MEMORY;
|
|
break;
|
|
}
|
|
s = MmMapPage(va, frame, pageFlags, MmMapFailIfPresent);
|
|
if (!KSUCCESS(s)) {
|
|
MmFreeFrame(frame);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!KSUCCESS(s)) {
|
|
r->CommittedSize = (va > r->Base) ? (size_t)(va - r->Base) : 0;
|
|
KiUnmapUserRegionPayload(r);
|
|
lockFlags = KeAcquireSpinlock(&gUserRegionLock);
|
|
ListRemove(&r->GlobalLink);
|
|
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
|
|
MmFreePool(r);
|
|
return s;
|
|
}
|
|
|
|
if (regionFlags & USER_REGION_ZERO) {
|
|
memset((void*)r->Base, 0, payloadSize);
|
|
}
|
|
|
|
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
ListInsertTail(&process->UserRegions, &r->ProcessLink);
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
|
|
*outBase = (void*)r->Base;
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t start = (uintptr_t)base;
|
|
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
|
|
|
|
uintptr_t alignedStart = start & PAGE_MASK;
|
|
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
|
|
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
|
|
|
|
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
|
|
uintptr_t regionBase = r ? r->Base : 0;
|
|
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
|
|
uint64_t defaultFlags = r ? r->PageFlags : 0;
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
|
|
if (!r) {
|
|
DbgPrint("MM: ensure user mapping failed: pid=%u range=%016llx-%016llx not inside owned committed region\n",
|
|
process->ProcessId,
|
|
(unsigned long long)alignedStart,
|
|
(unsigned long long)alignedEnd);
|
|
return STATUS_NOT_FOUND;
|
|
}
|
|
|
|
uint64_t flags = pageFlags ? pageFlags : defaultFlags;
|
|
flags |= PAGE_PRESENT | PAGE_USER;
|
|
|
|
uint32_t committed = 0;
|
|
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
|
|
if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED;
|
|
|
|
PHYSICAL_ADDRESS phys = 0;
|
|
uint64_t existingFlags = 0;
|
|
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
|
|
if (KSUCCESS(qs) && (existingFlags & PAGE_PRESENT))
|
|
continue;
|
|
|
|
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
|
if (!frame) return STATUS_NO_MEMORY;
|
|
|
|
KSTATUS ms = MmMapPage(va, frame, flags, MmMapFailIfPresent);
|
|
if (!KSUCCESS(ms)) {
|
|
MmFreeFrame(frame);
|
|
return ms;
|
|
}
|
|
memset((void*)va, 0, PAGE_SIZE);
|
|
committed++;
|
|
}
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t start = (uintptr_t)base;
|
|
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
|
|
|
|
uintptr_t alignedStart = start & PAGE_MASK;
|
|
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
|
|
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
|
|
|
|
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
|
|
uintptr_t regionBase = r ? r->Base : 0;
|
|
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
|
|
if (!r) {
|
|
DbgPrint("MM: protect user region failed: pid=%u range=%016llx-%016llx not owned/committed\n",
|
|
process->ProcessId,
|
|
(unsigned long long)alignedStart,
|
|
(unsigned long long)alignedEnd);
|
|
return STATUS_NOT_FOUND;
|
|
}
|
|
|
|
if (pageFlags == 0) {
|
|
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
|
|
if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED;
|
|
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va);
|
|
if (phys && KSUCCESS(MmUnmapPage(va)))
|
|
MmFreeFrame(phys);
|
|
}
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
uint64_t finalFlags = pageFlags | PAGE_PRESENT | PAGE_USER;
|
|
|
|
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
|
|
if (va < regionBase || va >= regionEnd) {
|
|
DbgPrint("MM: protect user region hit guard/outside page pid=%u va=%016llx region=%016llx-%016llx\n",
|
|
process->ProcessId,
|
|
(unsigned long long)va,
|
|
(unsigned long long)regionBase,
|
|
(unsigned long long)regionEnd);
|
|
return STATUS_ACCESS_DENIED;
|
|
}
|
|
|
|
PHYSICAL_ADDRESS phys = 0;
|
|
uint64_t existingFlags = 0;
|
|
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
|
|
if (!KSUCCESS(qs) || !(existingFlags & PAGE_PRESENT)) {
|
|
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
|
if (!frame) return STATUS_NO_MEMORY;
|
|
KSTATUS ms = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MmMapFailIfPresent);
|
|
if (!KSUCCESS(ms)) {
|
|
MmFreeFrame(frame);
|
|
return ms;
|
|
}
|
|
memset((void*)va, 0, PAGE_SIZE);
|
|
DbgPrint("MM: committed missing user page during protect pid=%u va=%016llx\n",
|
|
process->ProcessId, (unsigned long long)va);
|
|
}
|
|
|
|
KSTATUS ps = MmSetPageProtect(va, PAGE_SIZE, finalFlags);
|
|
if (!KSUCCESS(ps)) {
|
|
PHYSICAL_ADDRESS dbgPhys = 0;
|
|
uint64_t dbgFlags = 0;
|
|
KSTATUS dbg = MmQueryPage(va, &dbgPhys, &dbgFlags);
|
|
DbgPrint("MM: protect user page failed pid=%u va=%016llx st=%d query=%d phys=%016llx flags=%016llx final=%016llx\n",
|
|
process->ProcessId,
|
|
(unsigned long long)va,
|
|
ps, dbg,
|
|
(unsigned long long)dbgPhys,
|
|
(unsigned long long)dbgFlags,
|
|
(unsigned long long)finalFlags);
|
|
return ps;
|
|
}
|
|
}
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
if (!process || !base) return STATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
|
|
if (!r) {
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
return STATUS_NOT_FOUND;
|
|
}
|
|
ListRemove(&r->ProcessLink);
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
|
|
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
|
|
ListRemove(&r->GlobalLink);
|
|
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
|
|
|
|
KiUnmapUserRegionPayload(r);
|
|
MmFreePool(r);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
void MmDestroyAllUserRegions(PROCESS* process) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
if (!process) return;
|
|
|
|
for (;;) {
|
|
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
LIST_ENTRY* node = ListPopHead(&process->UserRegions);
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
if (!node) break;
|
|
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
|
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
|
|
ListRemove(&r->GlobalLink);
|
|
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
|
|
KiUnmapUserRegionPayload(r);
|
|
MmFreePool(r);
|
|
}
|
|
}
|
|
|
|
KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut,
|
|
uint64_t* pageFlagsOut, USER_REGION_TYPE* typeOut, uint32_t* regionFlagsOut) {
|
|
if (!process || !address) return STATUS_INVALID_PARAMETER;
|
|
uintptr_t addr = (uintptr_t)address;
|
|
|
|
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
|
uintptr_t end = r->Base + r->CommittedSize;
|
|
if (addr >= r->Base && addr < end) {
|
|
if (baseOut) *baseOut = (void*)r->Base;
|
|
if (sizeOut) *sizeOut = r->CommittedSize;
|
|
if (pageFlagsOut) {
|
|
PHYSICAL_ADDRESS phys = 0;
|
|
uint64_t flags = 0;
|
|
KSTATUS qs = MmQueryPage(addr & PAGE_MASK, &phys, &flags);
|
|
*pageFlagsOut = KSUCCESS(qs) ? flags : 0;
|
|
}
|
|
if (typeOut) *typeOut = r->Type;
|
|
if (regionFlagsOut) *regionFlagsOut = r->Flags;
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
return STATUS_SUCCESS;
|
|
}
|
|
}
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
return STATUS_NOT_FOUND;
|
|
}
|
|
|
|
size_t MmGetUserRegionSize(PROCESS* process, const void* base) {
|
|
if (!process || !base) return 0;
|
|
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
|
|
size_t size = r ? r->Size : 0;
|
|
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
|
return size;
|
|
}
|
|
|
|
size_t MmGetUserRegionSizeByBase(const void* base) {
|
|
if (!base) return 0;
|
|
uintptr_t b = (uintptr_t)base;
|
|
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
|
|
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
|
if (r->Base == b) {
|
|
size_t size = r->Size;
|
|
KeReleaseSpinlock(&gUserRegionLock, flags);
|
|
return size;
|
|
}
|
|
}
|
|
KeReleaseSpinlock(&gUserRegionLock, flags);
|
|
return 0;
|
|
}
|
|
|
|
|
|
static const char* KiUserRegionTypeName(USER_REGION_TYPE type) {
|
|
switch (type) {
|
|
case UserRegionGeneric: return "generic";
|
|
case UserRegionHeap: return "heap";
|
|
case UserRegionImage: return "image";
|
|
case UserRegionStack: return "stack";
|
|
case UserRegionTeb: return "teb";
|
|
case UserRegionPeb: return "peb";
|
|
case UserRegionEnvironment: return "env";
|
|
case UserRegionMappedFile: return "mapped";
|
|
case UserRegionShared: return "shared";
|
|
default: return "?";
|
|
}
|
|
}
|
|
|
|
void MmDebugDumpUserRegions(PROCESS* process) {
|
|
if (!process) {
|
|
DbgPrint("MM: user regions: <no process>\n");
|
|
return;
|
|
}
|
|
|
|
DbgPrint("MM: user regions for pid=%u name=%s:\n", process->ProcessId, process->Name);
|
|
uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock);
|
|
uint32_t count = 0;
|
|
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
|
DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx flags=%x pte=%llx\n",
|
|
count,
|
|
KiUserRegionTypeName(r->Type),
|
|
(unsigned long long)r->ReservationBase,
|
|
(unsigned long long)(r->ReservationBase + r->ReservationSize),
|
|
(unsigned long long)r->Base,
|
|
(unsigned long long)r->Size,
|
|
(unsigned long long)r->CommittedSize,
|
|
r->Flags,
|
|
(unsigned long long)r->PageFlags);
|
|
count++;
|
|
}
|
|
KeReleaseSpinlock(&process->UserRegionLock, flags);
|
|
if (!count) DbgPrint(" <none>\n");
|
|
}
|
|
|
|
static void KiDebugDescribePoolAddress(uintptr_t addr) {
|
|
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
|
|
(unsigned long long)gPoolHeap.Start,
|
|
(unsigned long long)gPoolHeap.Brk,
|
|
(unsigned long long)gPoolHeap.Limit,
|
|
(unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
|
|
|
|
if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
|
|
if (addr >= gPoolHeap.Brk) {
|
|
DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
|
|
(unsigned long long)addr,
|
|
(unsigned long long)(addr - gPoolHeap.Brk));
|
|
return;
|
|
}
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
|
|
for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
|
|
(uintptr_t)cur < gPoolHeap.Brk;
|
|
cur = KiNextBlock(cur)) {
|
|
if (cur->Magic != BLOCK_MAGIC) {
|
|
DbgPrint("MM: pool walk corrupt at hdr=%p magic=%08x\n", cur, cur->Magic);
|
|
break;
|
|
}
|
|
uintptr_t hdr = (uintptr_t)cur;
|
|
uintptr_t data = hdr + HDR_SIZE;
|
|
uintptr_t end = data + cur->Size;
|
|
uintptr_t ftr = end;
|
|
if (addr >= hdr && addr < ftr + FTR_SIZE) {
|
|
DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx free=%u\n",
|
|
(unsigned long long)addr,
|
|
(unsigned long long)hdr,
|
|
(unsigned long long)data,
|
|
(unsigned long long)end,
|
|
(unsigned long long)cur->Size,
|
|
cur->Free);
|
|
break;
|
|
}
|
|
if (KiNextBlock(cur) <= cur) {
|
|
DbgPrint("MM: pool walk stopped: non-advancing block hdr=%p size=%llx\n", cur, (unsigned long long)cur->Size);
|
|
break;
|
|
}
|
|
}
|
|
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
|
|
}
|
|
|
|
void MmDebugDescribeAddress(uintptr_t addr) {
|
|
PHYSICAL_ADDRESS phys = 0;
|
|
uint64_t flags = 0;
|
|
KSTATUS qs = MmQueryPage(addr, &phys, &flags);
|
|
DbgPrint("MM: describe va=%016llx query=%d phys=%016llx flags=%016llx\n",
|
|
(unsigned long long)addr,
|
|
qs,
|
|
(unsigned long long)phys,
|
|
(unsigned long long)flags);
|
|
|
|
if (addr >= POOL_BASE && addr < POOL_LIMIT) {
|
|
KiDebugDescribePoolAddress(addr);
|
|
return;
|
|
}
|
|
|
|
if (addr >= USER_REGION_ARENA_BASE && addr < USER_LIMIT) {
|
|
uintptr_t gl = KeAcquireSpinlock(&gUserRegionLock);
|
|
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
|
if (addr >= r->ReservationBase && addr < r->ReservationBase + r->ReservationSize) {
|
|
DbgPrint("MM: address belongs to pid=%u name=%s region=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n",
|
|
r->Owner ? r->Owner->ProcessId : 0,
|
|
r->Owner ? r->Owner->Name : "?",
|
|
KiUserRegionTypeName(r->Type),
|
|
(unsigned long long)r->ReservationBase,
|
|
(unsigned long long)(r->ReservationBase + r->ReservationSize),
|
|
(unsigned long long)r->Base,
|
|
(unsigned long long)r->Size,
|
|
r->Flags);
|
|
KeReleaseSpinlock(&gUserRegionLock, gl);
|
|
return;
|
|
}
|
|
}
|
|
KeReleaseSpinlock(&gUserRegionLock, gl);
|
|
DbgPrint("MM: address is in user arena but not reserved by any USER_VA_REGION\n");
|
|
}
|
|
}
|
|
|
|
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) {
|
|
void* base = (void*)0;
|
|
if (!process || !size) return (void*)0;
|
|
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
|
|
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
|
USER_REGION_ZERO,
|
|
UserRegionHeap,
|
|
&base);
|
|
return KSUCCESS(s) ? base : (void*)0;
|
|
}
|
|
|
|
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) {
|
|
if (!process || !ptr) return;
|
|
(void)MmDestroyUserRegion(process, ptr);
|
|
}
|
|
|
|
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) {
|
|
if (!process) return (void*)0;
|
|
if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize);
|
|
if (!newSize) {
|
|
MmFreeUserMemoryForProcess(process, ptr);
|
|
return (void*)0;
|
|
}
|
|
|
|
size_t oldSize = MmGetUserRegionSize(process, ptr);
|
|
if (!oldSize) return (void*)0;
|
|
if (newSize <= oldSize) return ptr;
|
|
|
|
void* newPtr = MmAllocateUserMemoryForProcess(process, newSize);
|
|
if (!newPtr) return (void*)0;
|
|
memcpy(newPtr, ptr, oldSize);
|
|
MmFreeUserMemoryForProcess(process, ptr);
|
|
return newPtr;
|
|
}
|
|
|
|
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
|
|
PROCESS* process = KeGetCurrentProcess();
|
|
return MmAllocateUserMemoryForProcess(process, size);
|
|
}
|
|
|
|
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
|
|
PROCESS* process = KeGetCurrentProcess();
|
|
MmFreeUserMemoryForProcess(process, ptr);
|
|
}
|
|
|
|
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
|
|
PROCESS* process = KeGetCurrentProcess();
|
|
return MmReallocateUserMemoryForProcess(process, ptr, newSize);
|
|
}
|
|
|
|
__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) {
|
|
void* base = (void*)0;
|
|
if (!process || !size) return (void*)0;
|
|
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
|
|
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
|
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
|
|
UserRegionImage,
|
|
&base);
|
|
return KSUCCESS(s) ? base : (void*)0;
|
|
}
|
|
|
|
__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) {
|
|
(void)size;
|
|
if (!process || !ptr) return;
|
|
(void)MmDestroyUserRegion(process, ptr);
|
|
}
|
|
|
|
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
|
|
PROCESS* process = KeGetCurrentProcess();
|
|
return MmAllocateUserImageForProcess(process, size);
|
|
}
|
|
|
|
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
|
|
PROCESS* process = KeGetCurrentProcess();
|
|
MmFreeUserImageForProcess(process, ptr, size);
|
|
}
|
|
|
|
size_t MmGetKernelPoolUsage(void) {
|
|
// Returns bytes committed to the kernel pool
|
|
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
|
|
}
|
|
|
|
size_t MmGetUserHeapUsage(void) {
|
|
size_t total = 0;
|
|
if (!gUserRegionListReady) return 0;
|
|
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
|
|
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
|
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
|
total += r->CommittedSize;
|
|
}
|
|
KeReleaseSpinlock(&gUserRegionLock, flags);
|
|
return total;
|
|
}
|