// mm_virt.c - virtual memory manager // // Page tables: // One PML4 shared by all processes. // Upper half (bit 47 set) has U/S = 0 - ring-3 access always faults. // Lower half pages are PAGE_USER. // // Kernel pool: // A boundary-tag (header + footer) free-list allocator starting at POOL_BASE. // Pages are committed on demand from the PMM. // // User heap: // A single best-fit free list in the lower half. // Page-granular lower-half regions owned by PROCESS metadata. Sub-page // allocation is a user-mode runtime concern. #include #include #include #include #include #include #include #include #include #include // --------------------------------------------------------------------------- // Helpers - physical window access // --------------------------------------------------------------------------- static uint64_t gPhysBase = PHYS_MAP_BASE; static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) { return (void *)(p + gPhysBase); } static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) { return (uint64_t *)KiPhysToVirt(tablePhys) + idx; } // --------------------------------------------------------------------------- // PML4 / page-table helpers // --------------------------------------------------------------------------- static PHYSICAL_ADDRESS gPml4Phys = 0; static SPINLOCK gPageTableLock = SPINLOCK_INIT; static volatile long gTlbShootdownLock = 0; static volatile long gTlbShootdownActive = 0; static volatile long gTlbShootdownAcks = 0; static uintptr_t gTlbShootdownVirt = 0; #define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u // Extract the page-table index for each level from a virtual address. #define PML4_IDX(va) (((va) >> 39) & 0x1FF) #define PDPT_IDX(va) (((va) >> 30) & 0x1FF) #define PDT_IDX(va) (((va) >> 21) & 0x1FF) #define PT_IDX(va) (((va) >> 12) & 0x1FF) #define PAGE_PS (1ULL << 7) #define PAGE_FLAGS_MASK (0xFFFULL | PAGE_NX) #define PAGE_FRAME_4K_MASK 0x000FFFFFFFFFF000ULL #define PAGE_FRAME_2M_MASK 0x000FFFFFFFE00000ULL #define PAGE_FRAME_1G_MASK 0x000FFFFFC0000000ULL // Ensure a page-table entry at 'parentPhys[index]' points to a child table. // Allocates and zeros a new frame if the entry is not present. // Returns the physical address of the child table, or 0 on OOM. static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys, unsigned index, uint64_t extraFlags) { uint64_t* entry = KiPtePtr(parentPhys, index); if (*entry & PAGE_PRESENT) { *entry |= extraFlags; return *entry & PAGE_FRAME_4K_MASK; } PHYSICAL_ADDRESS childPhys = MmAllocateFrame(); if (!childPhys) return 0; memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE); *entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags; return childPhys; } // --------------------------------------------------------------------------- // TLB/cache coherency hooks // --------------------------------------------------------------------------- void MmFlushTlbLocal(uintptr_t virt) { __invlpg((void*)virt); } static void KiAcquireTlbShootdownLock(void) { // Do not use the normal spinlock here: it masks interrupts while spinning. // A CPU spinning with IF=0 on the shootdown lock could not acknowledge a // concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is // meant to avoid. while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0) _mm_pause(); _ReadWriteBarrier(); } static void KiReleaseTlbShootdownLock(void) { _ReadWriteBarrier(); _InterlockedExchange(&gTlbShootdownLock, 0); } void MmFlushTlbForKernelMapping(uintptr_t virt) { MmFlushTlbLocal(virt); if (gCpuCount <= 1) return; KCPU* self = KeGetCurrentCpu(); uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU; KiAcquireTlbShootdownLock(); long targets = 0; gTlbShootdownVirt = virt; _InterlockedExchange(&gTlbShootdownAcks, 0); _ReadWriteBarrier(); _InterlockedExchange(&gTlbShootdownActive, 1); _ReadWriteBarrier(); for (uint32_t i = 0; i < gCpuCount; i++) { KCPU* cpu = &gCpus[i]; if (i == selfIndex) continue; if (!cpu->Present || !cpu->Started || !cpu->Online) continue; targets++; HalApicSendTlbShootdownIpi(cpu->LapicId); } for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) { if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets) break; _mm_pause(); } long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0); _InterlockedExchange(&gTlbShootdownActive, 0); _ReadWriteBarrier(); KiReleaseTlbShootdownLock(); if (acks < targets) { KePanic(KE_PANIC_CODE_GENERIC, "TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n", (uint64_t)virt, targets, acks, selfIndex); } } static int KiHandleTlbShootdown(void) { if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0) return 0; uintptr_t va = gTlbShootdownVirt; _ReadWriteBarrier(); MmFlushTlbLocal(va); _ReadWriteBarrier(); _InterlockedIncrement(&gTlbShootdownAcks); return 1; } int MmHandleTlbShootdownNmi(void) { return KiHandleTlbShootdown(); } // --------------------------------------------------------------------------- // Public mapping // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode) { unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0; uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag); if (!pdpt) { KeReleaseSpinlock(&gPageTableLock, lockFlags); return STATUS_NO_MEMORY; } PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag); if (!pdt) { KeReleaseSpinlock(&gPageTableLock, lockFlags); return STATUS_NO_MEMORY; } PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag); if (!pt) { KeReleaseSpinlock(&gPageTableLock, lockFlags); return STATUS_NO_MEMORY; } uint64_t* pte = KiPtePtr(pt, PT_IDX(virt)); if ((*pte & PAGE_PRESENT) && mode == MmMapFailIfPresent) { KeReleaseSpinlock(&gPageTableLock, lockFlags); return STATUS_ALREADY_EXISTS; } *pte = phys | flags; KeReleaseSpinlock(&gPageTableLock, lockFlags); MmFlushTlbForKernelMapping(virt); return STATUS_SUCCESS; } __declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) { uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt)); if (!(*e4 & PAGE_PRESENT)) { KeReleaseSpinlock(&gPageTableLock, lockFlags); return STATUS_NOT_FOUND; } uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt)); if (!(*e3 & PAGE_PRESENT)) { KeReleaseSpinlock(&gPageTableLock, lockFlags); return STATUS_NOT_FOUND; } uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt)); if (!(*e2 & PAGE_PRESENT)) { KeReleaseSpinlock(&gPageTableLock, lockFlags); return STATUS_NOT_FOUND; } uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt)); *pte = 0; KeReleaseSpinlock(&gPageTableLock, lockFlags); MmFlushTlbForKernelMapping(virt); return STATUS_SUCCESS; } __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags) { if (!size) return STATUS_SUCCESS; // Round down to page boundary, round size up. uintptr_t base = virt & PAGE_MASK; uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK; uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); KSTATUS status = STATUS_SUCCESS; uintptr_t flushEnd = base; for (uintptr_t va = base; va < end; va += PAGE_SIZE) { uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va)); if (!(*e4 & PAGE_PRESENT)) { status = STATUS_NOT_FOUND; break; } uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va)); if (!(*e3 & PAGE_PRESENT)) { status = STATUS_NOT_FOUND; break; } if (*e3 & PAGE_PS) { status = STATUS_NOT_SUPPORTED; break; } uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va)); if (!(*e2 & PAGE_PRESENT)) { status = STATUS_NOT_FOUND; break; } if (*e2 & PAGE_PS) { status = STATUS_NOT_SUPPORTED; break; } uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va)); if (!(*pte & PAGE_PRESENT)) { status = STATUS_NOT_FOUND; break; } // Preserve the physical address, replace everything else. PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK; *pte = phys | flags; flushEnd = va + PAGE_SIZE; } KeReleaseSpinlock(&gPageTableLock, lockFlags); for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE) MmFlushTlbForKernelMapping(va); return status; } __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) { if (!gPml4Phys) return 0; // Not initialized PHYSICAL_ADDRESS result = 0; uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); uint64_t pml4_idx = (virt >> 39) & 0x1FF; uint64_t pdpt_idx = (virt >> 30) & 0x1FF; uint64_t pd_idx = (virt >> 21) & 0x1FF; uint64_t pt_idx = (virt >> 12) & 0x1FF; uint64_t* pml4 = KiPhysToVirt(gPml4Phys); if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out; uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK); if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out; // Check for 1GB huge pages (bit 7 - PS) if (pdpt[pdpt_idx] & PAGE_PS) { result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF); goto out; } uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK); if (!(pd[pd_idx] & PAGE_PRESENT)) goto out; // Check for 2MB huge pages (bit 7 - PS) if (pd[pd_idx] & PAGE_PS) { result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF); goto out; } uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK); if (!(pt[pt_idx] & PAGE_PRESENT)) goto out; // Extract the physical frame and add the offset within the page result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF); out: KeReleaseSpinlock(&gPageTableLock, lockFlags); return result; } // --------------------------------------------------------------------------- // Page query and user/kernel copy boundary helpers // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut) { if (!gPml4Phys) return STATUS_NOT_FOUND; KSTATUS status = STATUS_NOT_FOUND; uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys); uint64_t e4 = pml4[PML4_IDX(virt)]; if (!(e4 & PAGE_PRESENT)) goto out; uint64_t effectiveWrite = e4 & PAGE_WRITE; uint64_t effectiveUser = e4 & PAGE_USER; uint64_t effectiveNx = e4 & PAGE_NX; uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK); uint64_t e3 = pdpt[PDPT_IDX(virt)]; if (!(e3 & PAGE_PRESENT)) goto out; effectiveWrite &= e3 & PAGE_WRITE; effectiveUser &= e3 & PAGE_USER; effectiveNx |= e3 & PAGE_NX; if (e3 & PAGE_PS) { if (physOut) *physOut = (e3 & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFFULL); if (flagsOut) { uint64_t flags = (e3 & PAGE_FLAGS_MASK) | effectiveNx; if (!effectiveWrite) flags &= ~PAGE_WRITE; if (!effectiveUser) flags &= ~PAGE_USER; *flagsOut = flags; } status = STATUS_SUCCESS; goto out; } uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK); uint64_t e2 = pd[PDT_IDX(virt)]; if (!(e2 & PAGE_PRESENT)) goto out; effectiveWrite &= e2 & PAGE_WRITE; effectiveUser &= e2 & PAGE_USER; effectiveNx |= e2 & PAGE_NX; if (e2 & PAGE_PS) { if (physOut) *physOut = (e2 & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFFULL); if (flagsOut) { uint64_t flags = (e2 & PAGE_FLAGS_MASK) | effectiveNx; if (!effectiveWrite) flags &= ~PAGE_WRITE; if (!effectiveUser) flags &= ~PAGE_USER; *flagsOut = flags; } status = STATUS_SUCCESS; goto out; } uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK); uint64_t e1 = pt[PT_IDX(virt)]; if (!(e1 & PAGE_PRESENT)) goto out; effectiveWrite &= e1 & PAGE_WRITE; effectiveUser &= e1 & PAGE_USER; effectiveNx |= e1 & PAGE_NX; if (physOut) *physOut = (e1 & PAGE_FRAME_4K_MASK) + (virt & 0xFFFULL); if (flagsOut) { uint64_t flags = (e1 & PAGE_FLAGS_MASK) | effectiveNx; if (!effectiveWrite) flags &= ~PAGE_WRITE; if (!effectiveUser) flags &= ~PAGE_USER; *flagsOut = flags; } status = STATUS_SUCCESS; out: KeReleaseSpinlock(&gPageTableLock, lockFlags); return status; } static int KiUserRangeValid(uintptr_t addr, size_t size) { if (size == 0) return 1; if (addr < USER_BASE) return 0; if (addr >= USER_LIMIT) return 0; if (size > (size_t)(USER_LIMIT - addr)) return 0; return 1; } static KSTATUS KiProbeUserRange(uintptr_t addr, size_t size, uint64_t requiredFlags) { if (!KiUserRangeValid(addr, size)) return STATUS_ACCESS_DENIED; if (size == 0) return STATUS_SUCCESS; uintptr_t va = addr & PAGE_MASK; uintptr_t last = (addr + size - 1) & PAGE_MASK; for (;;) { uint64_t flags = 0; KSTATUS s = MmQueryPage(va, (PHYSICAL_ADDRESS*)0, &flags); if (!KSUCCESS(s)) return STATUS_ACCESS_DENIED; if ((flags & requiredFlags) != requiredFlags) return STATUS_ACCESS_DENIED; if (va == last) break; va += PAGE_SIZE; } return STATUS_SUCCESS; } __declspec(dllexport) KSTATUS MmProbeUserRead(const void* ptr, size_t size) { return KiProbeUserRange((uintptr_t)ptr, size, PAGE_PRESENT | PAGE_USER); } __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size) { return KiProbeUserRange((uintptr_t)ptr, size, PAGE_PRESENT | PAGE_USER | PAGE_WRITE); } __declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size) { if (!kernelDst && size) return STATUS_INVALID_PARAMETER; KSTATUS s = MmProbeUserRead(userSrc, size); if (!KSUCCESS(s)) return s; if (size) memcpy(kernelDst, userSrc, size); return STATUS_SUCCESS; } __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size) { if (!kernelSrc && size) return STATUS_INVALID_PARAMETER; KSTATUS s = MmProbeUserWrite(userDst, size); if (!KSUCCESS(s)) return s; if (size) memcpy(userDst, kernelSrc, size); return STATUS_SUCCESS; } // --------------------------------------------------------------------------- // Boundary-tag allocator (used for both pool and user heap) // --------------------------------------------------------------------------- // Each allocation has a BLOCK_HDR immediately before the user data, and a // BLOCK_FTR immediately after. Both carry the block size and a 'free' flag. // This allows O(1) coalescing of adjacent free blocks. #define BLOCK_MAGIC 0xB10CB10CUL typedef struct BLOCK_HDR { uint32_t Magic; uint32_t Free; // 1 = free, 0 = used size_t Size; // usable bytes (excludes header + footer) struct BLOCK_HDR* PrevFree; struct BLOCK_HDR* NextFree; } BLOCK_HDR; typedef struct BLOCK_FTR { size_t Size; uint32_t Free; uint32_t Magic; } BLOCK_FTR; #define HDR_SIZE (sizeof(BLOCK_HDR)) #define FTR_SIZE (sizeof(BLOCK_FTR)) #define OVERHEAD (HDR_SIZE + FTR_SIZE) #define MIN_BLOCK (OVERHEAD + sizeof(void*)) static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) { return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size); } static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) { return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size); } static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) { return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE); } static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) { return (BLOCK_FTR *)((char *)h - FTR_SIZE); } typedef struct HEAP { BLOCK_HDR* FreeList; // doubly-linked list of free blocks uintptr_t Start; // virtual base of the arena uintptr_t Brk; // current committed end (page-aligned) uintptr_t Limit; // hard upper bound uint64_t MapFlags; // PAGE_* flags for new committed pages SPINLOCK Lock; } HEAP; static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) { if (!h || !size) return 1; if (start < h->Start) return 0; if (start > (uintptr_t)-1 - (size - 1)) return 0; uintptr_t begin = start & PAGE_MASK; uintptr_t end = (start + size + PAGE_SIZE - 1) & PAGE_MASK; if (end < begin || end > h->Limit) return 0; for (uintptr_t va = begin; va < end; va += PAGE_SIZE) { PHYSICAL_ADDRESS phys = 0; uint64_t flags = 0; KSTATUS qs = MmQueryPage(va, &phys, &flags); if (KSUCCESS(qs) && (flags & PAGE_PRESENT)) continue; PHYSICAL_ADDRESS frame = MmAllocateFrame(); if (!frame) return 0; KSTATUS ms = MmMapPage(va, frame, h->MapFlags, MmMapFailIfPresent); if (!KSUCCESS(ms)) { MmFreeFrame(frame); return 0; } memset((void*)va, 0, PAGE_SIZE); DbgPrint("MM: repaired missing heap page va=%016llx\n", (unsigned long long)va); } return 1; } // Extend heap by at least 'need' bytes (rounded up to page boundary). static int KiHeapGrow(HEAP* h, size_t need) { size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); if (h->Brk + grow > h->Limit) return 0; for (uintptr_t va = h->Brk; va < h->Brk + grow; va += PAGE_SIZE) { PHYSICAL_ADDRESS frame = MmAllocateFrame(); if (!frame) return 0; if (!KSUCCESS(MmMapPage(va, frame, h->MapFlags, MmMapFailIfPresent))) { MmFreeFrame(frame); return 0; } memset((void *)va, 0, PAGE_SIZE); } if (!KiHeapEnsureMapped(h, h->Brk, grow)) return 0; // Make the new pages a single free block. BLOCK_HDR* blk = (BLOCK_HDR *)h->Brk; blk->Magic = BLOCK_MAGIC; blk->Free = 1; blk->Size = grow - OVERHEAD; blk->PrevFree = (BLOCK_HDR *)0; blk->NextFree = h->FreeList; if (h->FreeList) h->FreeList->PrevFree = blk; h->FreeList = blk; KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; KiHdrToFtr(blk)->Free = 1; KiHdrToFtr(blk)->Size = blk->Size; h->Brk += grow; return 1; } // Unlink a free block from the free list. static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) { if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree; else h->FreeList = blk->NextFree; if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree; blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0; } static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) { (void)h; if (!blk || blk->Free == 0) return; // Do not decommit kernel-pool tail pages for now. The pool allocator keeps // free-list nodes and boundary tags inside the managed arena; shrinking the // mapped tail while large launch-failure cleanup is walking/freeing many // objects can leave stale metadata pointers into unmapped pages. Physical // pages are still reclaimed for user regions by MmDestroyUserRegion(); pool // pages remain committed until we have a dedicated, audited heap shrinker. KiHdrToFtr(blk)->Size = blk->Size; KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; KiHdrToFtr(blk)->Free = 1; } static void* KiHeapAlloc(HEAP* h, size_t size) { if (!size) return (void *)0; size = (size + 7) & ~7ULL; // 8-byte align uintptr_t flags = KeAcquireSpinlock(&h->Lock); if (h->Brk > h->Start && !KiHeapEnsureMapped(h, h->Start, h->Brk - h->Start)) { KeReleaseSpinlock(&h->Lock, flags); return (void*)0; } // Best-fit search. BLOCK_HDR* best = (BLOCK_HDR *)0; for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) { if (cur->Size >= size) { if (!best || cur->Size < best->Size) best = cur; if (best->Size == size) break; } } if (!best) { if (!KiHeapGrow(h, size + OVERHEAD)) { KeReleaseSpinlock(&h->Lock, flags); return (void *)0; } // Re-search; KiHeapGrow inserted at the front. best = h->FreeList; while (best && best->Size < size) best = best->NextFree; if (!best) { KeReleaseSpinlock(&h->Lock, flags); return (void *)0; } } KiFreeListRemove(h, best); // Split the block if the remainder is big enough. if (best->Size >= size + OVERHEAD + sizeof(void*)) { BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE); rem->Magic = BLOCK_MAGIC; rem->Free = 1; rem->Size = best->Size - size - OVERHEAD; rem->PrevFree = (BLOCK_HDR *)0; rem->NextFree = h->FreeList; if (h->FreeList) h->FreeList->PrevFree = rem; h->FreeList = rem; KiHdrToFtr(rem)->Magic = BLOCK_MAGIC; KiHdrToFtr(rem)->Free = 1; KiHdrToFtr(rem)->Size = rem->Size; best->Size = size; } best->Free = 0; KiHdrToFtr(best)->Free = 0; KiHdrToFtr(best)->Magic = BLOCK_MAGIC; KiHdrToFtr(best)->Size = best->Size; memset((char *)best + HDR_SIZE, 0, size); KeReleaseSpinlock(&h->Lock, flags); return (char *)best + HDR_SIZE; } static void KiHeapFree(HEAP* h, void* ptr) { if (!ptr) return; BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE); if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt uintptr_t flags = KeAcquireSpinlock(&h->Lock); blk->Free = 1; // Coalesce with next block if it is free. BLOCK_HDR* next = KiNextBlock(blk); if ((uintptr_t)next < h->Brk && next->Magic == BLOCK_MAGIC && next->Free) { KiFreeListRemove(h, next); blk->Size += OVERHEAD + next->Size; KiHdrToFtr(blk)->Size = blk->Size; KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; KiHdrToFtr(blk)->Free = 1; } // Coalesce with previous block if it is free. if ((uintptr_t)blk > h->Start) { BLOCK_FTR* prevFtr = KiPrevFtr(blk); if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free) { BLOCK_HDR* prev = KiFtrToHdr(prevFtr); KiFreeListRemove(h, prev); prev->Size += OVERHEAD + blk->Size; KiHdrToFtr(prev)->Size = prev->Size; KiHdrToFtr(prev)->Magic = BLOCK_MAGIC; KiHdrToFtr(prev)->Free = 1; blk = prev; } } // Insert at head of free list. blk->PrevFree = (BLOCK_HDR *)0; blk->NextFree = h->FreeList; if (h->FreeList) h->FreeList->PrevFree = blk; h->FreeList = blk; KiHdrToFtr(blk)->Free = 1; KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; KiHeapTryDecommitTail(h, blk); KeReleaseSpinlock(&h->Lock, flags); } static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) { // If ptr is NULL, realloc acts like malloc if (!ptr) { return KiHeapAlloc(h, new_size); } // If size is 0, realloc acts like free if (new_size == 0) { KiHeapFree(h, ptr); return (void*)0; } // Retrieve the header of the existing block BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE); // Sanity checks to prevent operating on corrupted or already-freed pointers if (blk->Magic != BLOCK_MAGIC || blk->Free) { return (void*)0; } // Align the requested size (matches KiHeapAlloc 8-byte alignment) size_t aligned_size = (new_size + 7) & ~7ULL; // If the current block is already big enough, we can reuse it in-place. // (Note: To save space, you could optionally split the block here if // the new size is significantly smaller, similar to KiHeapAlloc's logic). if (blk->Size >= aligned_size) { return ptr; } // The block is too small. Allocate a new, larger block. void* new_ptr = KiHeapAlloc(h, aligned_size); if (!new_ptr) { return (void*)0; // Out of memory, original block is left untouched } memset(new_ptr, 0, aligned_size); // Copy the old data to the new block. // We only copy up to the old block's usable size. memcpy(new_ptr, ptr, blk->Size); // Free the old block KiHeapFree(h, ptr); return new_ptr; } // --------------------------------------------------------------------------- // Kernel pool // --------------------------------------------------------------------------- static HEAP gPoolHeap; // --------------------------------------------------------------------------- // User virtual region arena // --------------------------------------------------------------------------- typedef struct USER_VA_REGION { LIST_ENTRY ProcessLink; LIST_ENTRY GlobalLink; PROCESS* Owner; uintptr_t ReservationBase; size_t ReservationSize; uintptr_t Base; size_t Size; size_t CommittedSize; uint64_t PageFlags; USER_REGION_TYPE Type; uint32_t Flags; } USER_VA_REGION; static LIST_ENTRY gUserRegionList; static SPINLOCK gUserRegionLock = SPINLOCK_INIT; static int gUserRegionListReady = 0; #define USER_REGION_ARENA_BASE 0x0000100000000000ULL // --------------------------------------------------------------------------- // Module init // --------------------------------------------------------------------------- void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) { gPml4Phys = bootPml4Phys; gPageTableLock = (SPINLOCK)SPINLOCK_INIT; // Kernel pool heap. gPoolHeap.FreeList = (BLOCK_HDR *)0; gPoolHeap.Start = POOL_BASE; gPoolHeap.Brk = POOL_BASE; gPoolHeap.Limit = POOL_LIMIT; gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX; gPoolHeap.Lock = (SPINLOCK)SPINLOCK_INIT; // Pre-commit the requested pool size. if (poolSize) { poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); KiHeapGrow(&gPoolHeap, poolSize); } // User VA arena. One global lower-half virtual-address bucket, with // per-process ownership tracked by USER_VA_REGION descriptors. LIST_INIT(&gUserRegionList); gUserRegionLock = (SPINLOCK)SPINLOCK_INIT; gUserRegionListReady = 1; } void MmActivate(void) { __writecr3(gPml4Phys); } // --------------------------------------------------------------------------- // Public pool API // --------------------------------------------------------------------------- __declspec(dllexport) void* MmAllocatePool(size_t size) { return KiHeapAlloc(&gPoolHeap, size); } __declspec(dllexport) void MmFreePool(void* ptr) { KiHeapFree(&gPoolHeap, ptr); } __declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); } // --------------------------------------------------------------------------- // User virtual region API // --------------------------------------------------------------------------- static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) { if (align < PAGE_SIZE) align = PAGE_SIZE; if (align & (align - 1)) return 0; return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1); } static inline size_t KiAlignUpSize(size_t v, size_t align) { if (align < PAGE_SIZE) align = PAGE_SIZE; if (align & (align - 1)) return 0; return (v + align - 1) & ~(align - 1); } static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) { uintptr_t aEnd = aBase + aSize; uintptr_t bEnd = bBase + bSize; return aBase < bEnd && bBase < aEnd; } static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) { if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER; if (!gUserRegionListReady) return STATUS_FAILURE; uintptr_t candidate = KiAlignUpPtr(USER_REGION_ARENA_BASE, alignment); if (!candidate) return STATUS_INVALID_PARAMETER; for (;;) { if (candidate < USER_REGION_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate)) return STATUS_NO_MEMORY; uintptr_t next = candidate; int collided = 0; for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); if (KiRangesOverlap(candidate, reservationSize, r->ReservationBase, r->ReservationSize)) { uintptr_t end = r->ReservationBase + r->ReservationSize; next = KiAlignUpPtr(end, alignment); if (!next || next <= candidate) return STATUS_NO_MEMORY; collided = 1; break; } } if (!collided) { *baseOut = candidate; return STATUS_SUCCESS; } candidate = next; } } static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) { if (!process || !base) return (USER_VA_REGION*)0; uintptr_t b = (uintptr_t)base; for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); if (r->Base == b) return r; } return (USER_VA_REGION*)0; } static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) { if (!process || !base || !size) return (USER_VA_REGION*)0; if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0; uintptr_t end = base + size; for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); uintptr_t regionEnd = r->Base + r->CommittedSize; if (base >= r->Base && end <= regionEnd) return r; } return (USER_VA_REGION*)0; } static void KiUnmapUserRegionPayload(USER_VA_REGION* r) { if (!r) return; uintptr_t end = r->Base + r->CommittedSize; for (uintptr_t va = r->Base; va < end; va += PAGE_SIZE) { PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); if (!phys) continue; if (KSUCCESS(MmUnmapPage(va))) { MmFreeFrame(phys); } } } KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment, uint64_t pageFlags, uint32_t regionFlags, USER_REGION_TYPE type, void** outBase) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !outBase || !size) return STATUS_INVALID_PARAMETER; *outBase = (void*)0; if (!(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER; if (!(pageFlags & PAGE_PRESENT)) pageFlags |= PAGE_PRESENT; size_t payloadSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); if (payloadSize < size) return STATUS_NO_MEMORY; size_t guardLow = (regionFlags & USER_REGION_GUARD_LOW) ? PAGE_SIZE : 0; size_t guardHigh = (regionFlags & USER_REGION_GUARD_HIGH) ? PAGE_SIZE : 0; size_t reservationSize = guardLow + payloadSize + guardHigh; if (reservationSize < payloadSize) return STATUS_NO_MEMORY; if (alignment < PAGE_SIZE) alignment = PAGE_SIZE; if (alignment & (alignment - 1)) return STATUS_INVALID_PARAMETER; USER_VA_REGION* r = (USER_VA_REGION*)MmAllocatePool(sizeof(USER_VA_REGION)); if (!r) return STATUS_NO_MEMORY; memset(r, 0, sizeof(*r)); LIST_INIT(&r->ProcessLink); LIST_INIT(&r->GlobalLink); uintptr_t lockFlags = KeAcquireSpinlock(&gUserRegionLock); uintptr_t reservationBase = 0; KSTATUS s = KiFindUserVaRange(reservationSize, alignment, &reservationBase); if (KSUCCESS(s)) { r->Owner = process; r->ReservationBase = reservationBase; r->ReservationSize = reservationSize; r->Base = reservationBase + guardLow; r->Size = size; r->CommittedSize = payloadSize; r->PageFlags = pageFlags; r->Type = type; r->Flags = regionFlags; ListInsertTail(&gUserRegionList, &r->GlobalLink); } KeReleaseSpinlock(&gUserRegionLock, lockFlags); if (!KSUCCESS(s)) { MmFreePool(r); return s; } uintptr_t va = r->Base; for (; va < r->Base + payloadSize; va += PAGE_SIZE) { PHYSICAL_ADDRESS frame = MmAllocateFrame(); if (!frame) { s = STATUS_NO_MEMORY; break; } s = MmMapPage(va, frame, pageFlags, MmMapFailIfPresent); if (!KSUCCESS(s)) { MmFreeFrame(frame); break; } } if (!KSUCCESS(s)) { r->CommittedSize = (va > r->Base) ? (size_t)(va - r->Base) : 0; KiUnmapUserRegionPayload(r); lockFlags = KeAcquireSpinlock(&gUserRegionLock); ListRemove(&r->GlobalLink); KeReleaseSpinlock(&gUserRegionLock, lockFlags); MmFreePool(r); return s; } if (regionFlags & USER_REGION_ZERO) { memset((void*)r->Base, 0, payloadSize); } uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); ListInsertTail(&process->UserRegions, &r->ProcessLink); KeReleaseSpinlock(&process->UserRegionLock, procFlags); *outBase = (void*)r->Base; return STATUS_SUCCESS; } KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !base || !size) return STATUS_INVALID_PARAMETER; uintptr_t start = (uintptr_t)base; if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY; uintptr_t alignedStart = start & PAGE_MASK; uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY; uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart); uintptr_t regionBase = r ? r->Base : 0; uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0; uint64_t defaultFlags = r ? r->PageFlags : 0; KeReleaseSpinlock(&process->UserRegionLock, procFlags); if (!r) { DbgPrint("MM: ensure user mapping failed: pid=%u range=%016llx-%016llx not inside owned committed region\n", process->ProcessId, (unsigned long long)alignedStart, (unsigned long long)alignedEnd); return STATUS_NOT_FOUND; } uint64_t flags = pageFlags ? pageFlags : defaultFlags; flags |= PAGE_PRESENT | PAGE_USER; uint32_t committed = 0; for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED; PHYSICAL_ADDRESS phys = 0; uint64_t existingFlags = 0; KSTATUS qs = MmQueryPage(va, &phys, &existingFlags); if (KSUCCESS(qs) && (existingFlags & PAGE_PRESENT)) continue; PHYSICAL_ADDRESS frame = MmAllocateFrame(); if (!frame) return STATUS_NO_MEMORY; KSTATUS ms = MmMapPage(va, frame, flags, MmMapFailIfPresent); if (!KSUCCESS(ms)) { MmFreeFrame(frame); return ms; } memset((void*)va, 0, PAGE_SIZE); committed++; } return STATUS_SUCCESS; } KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !base || !size) return STATUS_INVALID_PARAMETER; uintptr_t start = (uintptr_t)base; if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY; uintptr_t alignedStart = start & PAGE_MASK; uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY; uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart); uintptr_t regionBase = r ? r->Base : 0; uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0; KeReleaseSpinlock(&process->UserRegionLock, procFlags); if (!r) { DbgPrint("MM: protect user region failed: pid=%u range=%016llx-%016llx not owned/committed\n", process->ProcessId, (unsigned long long)alignedStart, (unsigned long long)alignedEnd); return STATUS_NOT_FOUND; } if (pageFlags == 0) { for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED; PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); if (phys && KSUCCESS(MmUnmapPage(va))) MmFreeFrame(phys); } return STATUS_SUCCESS; } uint64_t finalFlags = pageFlags | PAGE_PRESENT | PAGE_USER; for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { if (va < regionBase || va >= regionEnd) { DbgPrint("MM: protect user region hit guard/outside page pid=%u va=%016llx region=%016llx-%016llx\n", process->ProcessId, (unsigned long long)va, (unsigned long long)regionBase, (unsigned long long)regionEnd); return STATUS_ACCESS_DENIED; } PHYSICAL_ADDRESS phys = 0; uint64_t existingFlags = 0; KSTATUS qs = MmQueryPage(va, &phys, &existingFlags); if (!KSUCCESS(qs) || !(existingFlags & PAGE_PRESENT)) { PHYSICAL_ADDRESS frame = MmAllocateFrame(); if (!frame) return STATUS_NO_MEMORY; KSTATUS ms = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MmMapFailIfPresent); if (!KSUCCESS(ms)) { MmFreeFrame(frame); return ms; } memset((void*)va, 0, PAGE_SIZE); DbgPrint("MM: committed missing user page during protect pid=%u va=%016llx\n", process->ProcessId, (unsigned long long)va); } KSTATUS ps = MmSetPageProtect(va, PAGE_SIZE, finalFlags); if (!KSUCCESS(ps)) { PHYSICAL_ADDRESS dbgPhys = 0; uint64_t dbgFlags = 0; KSTATUS dbg = MmQueryPage(va, &dbgPhys, &dbgFlags); DbgPrint("MM: protect user page failed pid=%u va=%016llx st=%d query=%d phys=%016llx flags=%016llx final=%016llx\n", process->ProcessId, (unsigned long long)va, ps, dbg, (unsigned long long)dbgPhys, (unsigned long long)dbgFlags, (unsigned long long)finalFlags); return ps; } } return STATUS_SUCCESS; } KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !base) return STATUS_INVALID_PARAMETER; uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); USER_VA_REGION* r = KiFindUserRegionLocked(process, base); if (!r) { KeReleaseSpinlock(&process->UserRegionLock, procFlags); return STATUS_NOT_FOUND; } ListRemove(&r->ProcessLink); KeReleaseSpinlock(&process->UserRegionLock, procFlags); uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock); ListRemove(&r->GlobalLink); KeReleaseSpinlock(&gUserRegionLock, globalFlags); KiUnmapUserRegionPayload(r); MmFreePool(r); return STATUS_SUCCESS; } void MmDestroyAllUserRegions(PROCESS* process) { KeAssertIrql(PASSIVE_LEVEL); if (!process) return; for (;;) { uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); LIST_ENTRY* node = ListPopHead(&process->UserRegions); KeReleaseSpinlock(&process->UserRegionLock, procFlags); if (!node) break; USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock); ListRemove(&r->GlobalLink); KeReleaseSpinlock(&gUserRegionLock, globalFlags); KiUnmapUserRegionPayload(r); MmFreePool(r); } } KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, USER_REGION_TYPE* typeOut, uint32_t* regionFlagsOut) { if (!process || !address) return STATUS_INVALID_PARAMETER; uintptr_t addr = (uintptr_t)address; uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); uintptr_t end = r->Base + r->CommittedSize; if (addr >= r->Base && addr < end) { if (baseOut) *baseOut = (void*)r->Base; if (sizeOut) *sizeOut = r->CommittedSize; if (pageFlagsOut) { PHYSICAL_ADDRESS phys = 0; uint64_t flags = 0; KSTATUS qs = MmQueryPage(addr & PAGE_MASK, &phys, &flags); *pageFlagsOut = KSUCCESS(qs) ? flags : 0; } if (typeOut) *typeOut = r->Type; if (regionFlagsOut) *regionFlagsOut = r->Flags; KeReleaseSpinlock(&process->UserRegionLock, procFlags); return STATUS_SUCCESS; } } KeReleaseSpinlock(&process->UserRegionLock, procFlags); return STATUS_NOT_FOUND; } void MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t* reservedBytesOut, size_t* committedBytesOut) { size_t regions = 0; size_t reserved = 0; size_t committed = 0; if (process) { uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock); for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); regions++; reserved += r->ReservationSize; committed += r->CommittedSize; } KeReleaseSpinlock(&process->UserRegionLock, flags); } if (regionCountOut) *regionCountOut = regions; if (reservedBytesOut) *reservedBytesOut = reserved; if (committedBytesOut) *committedBytesOut = committed; } size_t MmGetUserRegionSize(PROCESS* process, const void* base) { if (!process || !base) return 0; uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); USER_VA_REGION* r = KiFindUserRegionLocked(process, base); size_t size = r ? r->Size : 0; KeReleaseSpinlock(&process->UserRegionLock, procFlags); return size; } size_t MmGetUserRegionSizeByBase(const void* base) { if (!base) return 0; uintptr_t b = (uintptr_t)base; uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock); for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); if (r->Base == b) { size_t size = r->Size; KeReleaseSpinlock(&gUserRegionLock, flags); return size; } } KeReleaseSpinlock(&gUserRegionLock, flags); return 0; } static const char* KiUserRegionTypeName(USER_REGION_TYPE type) { switch (type) { case UserRegionGeneric: return "generic"; case UserRegionHeap: return "heap"; case UserRegionImage: return "image"; case UserRegionStack: return "stack"; case UserRegionTeb: return "teb"; case UserRegionPeb: return "peb"; case UserRegionEnvironment: return "env"; case UserRegionMappedFile: return "mapped"; case UserRegionShared: return "shared"; default: return "?"; } } void MmDebugDumpUserRegions(PROCESS* process) { if (!process) { DbgPrint("MM: user regions: \n"); return; } DbgPrint("MM: user regions for pid=%u name=%s:\n", process->ProcessId, process->Name); uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock); uint32_t count = 0; for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx flags=%x pte=%llx\n", count, KiUserRegionTypeName(r->Type), (unsigned long long)r->ReservationBase, (unsigned long long)(r->ReservationBase + r->ReservationSize), (unsigned long long)r->Base, (unsigned long long)r->Size, (unsigned long long)r->CommittedSize, r->Flags, (unsigned long long)r->PageFlags); count++; } KeReleaseSpinlock(&process->UserRegionLock, flags); if (!count) DbgPrint(" \n"); } static void KiDebugDescribePoolAddress(uintptr_t addr) { DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n", (unsigned long long)gPoolHeap.Start, (unsigned long long)gPoolHeap.Brk, (unsigned long long)gPoolHeap.Limit, (unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024)); if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return; if (addr >= gPoolHeap.Brk) { DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n", (unsigned long long)addr, (unsigned long long)(addr - gPoolHeap.Brk)); return; } uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock); for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start; (uintptr_t)cur < gPoolHeap.Brk; cur = KiNextBlock(cur)) { if (cur->Magic != BLOCK_MAGIC) { DbgPrint("MM: pool walk corrupt at hdr=%p magic=%08x\n", cur, cur->Magic); break; } uintptr_t hdr = (uintptr_t)cur; uintptr_t data = hdr + HDR_SIZE; uintptr_t end = data + cur->Size; uintptr_t ftr = end; if (addr >= hdr && addr < ftr + FTR_SIZE) { DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx free=%u\n", (unsigned long long)addr, (unsigned long long)hdr, (unsigned long long)data, (unsigned long long)end, (unsigned long long)cur->Size, cur->Free); break; } if (KiNextBlock(cur) <= cur) { DbgPrint("MM: pool walk stopped: non-advancing block hdr=%p size=%llx\n", cur, (unsigned long long)cur->Size); break; } } KeReleaseSpinlock(&gPoolHeap.Lock, flags); } void MmDebugDescribeAddress(uintptr_t addr) { PHYSICAL_ADDRESS phys = 0; uint64_t flags = 0; KSTATUS qs = MmQueryPage(addr, &phys, &flags); DbgPrint("MM: describe va=%016llx query=%d phys=%016llx flags=%016llx\n", (unsigned long long)addr, qs, (unsigned long long)phys, (unsigned long long)flags); if (addr >= POOL_BASE && addr < POOL_LIMIT) { KiDebugDescribePoolAddress(addr); return; } if (addr >= USER_REGION_ARENA_BASE && addr < USER_LIMIT) { uintptr_t gl = KeAcquireSpinlock(&gUserRegionLock); for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); if (addr >= r->ReservationBase && addr < r->ReservationBase + r->ReservationSize) { DbgPrint("MM: address belongs to pid=%u name=%s region=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n", r->Owner ? r->Owner->ProcessId : 0, r->Owner ? r->Owner->Name : "?", KiUserRegionTypeName(r->Type), (unsigned long long)r->ReservationBase, (unsigned long long)(r->ReservationBase + r->ReservationSize), (unsigned long long)r->Base, (unsigned long long)r->Size, r->Flags); KeReleaseSpinlock(&gUserRegionLock, gl); return; } } KeReleaseSpinlock(&gUserRegionLock, gl); DbgPrint("MM: address is in user arena but not reserved by any USER_VA_REGION\n"); } } __declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) { void* base = (void*)0; if (!process || !size) return (void*)0; KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, USER_REGION_ZERO, UserRegionHeap, &base); return KSUCCESS(s) ? base : (void*)0; } __declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) { if (!process || !ptr) return; (void)MmDestroyUserRegion(process, ptr); } __declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) { if (!process) return (void*)0; if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize); if (!newSize) { MmFreeUserMemoryForProcess(process, ptr); return (void*)0; } size_t oldSize = MmGetUserRegionSize(process, ptr); if (!oldSize) return (void*)0; if (newSize <= oldSize) return ptr; void* newPtr = MmAllocateUserMemoryForProcess(process, newSize); if (!newPtr) return (void*)0; memcpy(newPtr, ptr, oldSize); MmFreeUserMemoryForProcess(process, ptr); return newPtr; } __declspec(dllexport) void* MmAllocateUserMemory(size_t size) { PROCESS* process = KeGetCurrentProcess(); return MmAllocateUserMemoryForProcess(process, size); } __declspec(dllexport) void MmFreeUserMemory(void* ptr) { PROCESS* process = KeGetCurrentProcess(); MmFreeUserMemoryForProcess(process, ptr); } __declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { PROCESS* process = KeGetCurrentProcess(); return MmReallocateUserMemoryForProcess(process, ptr, newSize); } __declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) { void* base = (void*)0; if (!process || !size) return (void*)0; KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH, UserRegionImage, &base); return KSUCCESS(s) ? base : (void*)0; } __declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) { (void)size; if (!process || !ptr) return; (void)MmDestroyUserRegion(process, ptr); } __declspec(dllexport) void* MmAllocateUserImage(size_t size) { PROCESS* process = KeGetCurrentProcess(); return MmAllocateUserImageForProcess(process, size); } __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) { PROCESS* process = KeGetCurrentProcess(); MmFreeUserImageForProcess(process, ptr, size); } size_t MmGetKernelPoolUsage(void) { // Returns bytes committed to the kernel pool return (size_t)(gPoolHeap.Brk - gPoolHeap.Start); } size_t MmGetUserHeapUsage(void) { size_t total = 0; if (!gUserRegionListReady) return 0; uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock); for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); total += r->CommittedSize; } KeReleaseSpinlock(&gUserRegionLock, flags); return total; }