maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move keyboard and mouse out of HAL into a standalone ps2.sys driver, implement usb stack with xhci controller discovery, hid keyboard/mouse, and bulk-only mass storage, add pci msi support with capability chain walking and software irq vector allocation (slots 16-23), rewrite user virtual memory as a USER_VA_REGION descriptor arena with per-process ownership tracking and guard page support, replacing the old bump allocator and top-down image region, add MmCreateUserRegion/MmDestroyAllUserRegions so process teardown reclaims all user va in one pass, allocate a pool-backed shadow framebuffer in hal_screen to fix cursor tearing and scroll corruption, reserve ap trampoline page below 1mb in uefi bootloader before boot services exit, rewrite memset/memcpy/memmove/memcmp with __stosq/__movsq intrinsics, promote driver flag literals to macros and add multi-tu STANDALONE_DRIVERS table to build.c, split rtl8139/rtl816x into header+source, rename KbReadScancode to KbReadKey, improve page fault output with MmDebugDescribeAddress and user region dump, fix %p to %016llX throughout.

Browse Source
This commit is contained in:
maksw committed 2026-06-02 00:01:20 +02:00
1 parent b970d7f59c
commit b838f8d83d
130 files changed
+8137 -2932
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/mm_virt.c
+628 -94
View File
@@ -16,6 +16,7 @@
#include <string.h>
#include <intrin.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_process.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
@@ -139,7 +140,7 @@ void MmFlushTlbForKernelMapping(uintptr_t virt) {
if (acks < targets) {
KePanic(KE_PANIC_CODE_GENERIC,
"TLB shootdown timeout: va=0x%llX targets=%ld acks=%ld selfCpu=%u\n",
"TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n",
(uint64_t)virt, targets, acks, selfIndex);
}
}
@@ -506,6 +507,35 @@ typedef struct HEAP {
SPINLOCK Lock;
} HEAP;
static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) {
if (!h || !size) return 1;
if (start < h->Start) return 0;
if (start > (uintptr_t)-1 - (size - 1)) return 0;
uintptr_t begin = start & PAGE_MASK;
uintptr_t end = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
if (end < begin || end > h->Limit) return 0;
for (uintptr_t va = begin; va < end; va += PAGE_SIZE) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
KSTATUS qs = MmQueryPage(va, &phys, &flags);
if (KSUCCESS(qs) && (flags & PAGE_PRESENT))
continue;
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) return 0;
KSTATUS ms = MmMapPage(va, frame, h->MapFlags);
if (!KSUCCESS(ms)) {
MmFreeFrame(frame);
return 0;
}
memset((void*)va, 0, PAGE_SIZE);
DbgPrint("MM: repaired missing heap page va=%016llx\n", (unsigned long long)va);
}
return 1;
}
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int KiHeapGrow(HEAP* h, size_t need) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
@@ -521,6 +551,9 @@ static int KiHeapGrow(HEAP* h, size_t need) {
memset((void *)va, 0, PAGE_SIZE);
}
if (!KiHeapEnsureMapped(h, h->Brk, grow))
return 0;
// Make the new pages a single free block.
BLOCK_HDR* blk = (BLOCK_HDR *)h->Brk;
blk->Magic = BLOCK_MAGIC;
@@ -547,34 +580,18 @@ static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
}
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
(void)h;
if (!blk || blk->Free == 0) return;
while (h->Brk > h->Start) {
uintptr_t blockEnd = (uintptr_t)blk + HDR_SIZE + blk->Size + FTR_SIZE;
if (blockEnd != h->Brk) break;
// Keep at least one page for the block header/footer and a small payload.
if (blk->Size < PAGE_SIZE + sizeof(void*)) break;
uintptr_t pageStart = h->Brk - PAGE_SIZE;
if (pageStart <= (uintptr_t)blk) break; // would unmap the page containing the header
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(pageStart);
if (!phys) break;
if (!KSUCCESS(MmUnmapPage(pageStart))) {
break;
}
MmFreeFrame(phys);
h->Brk -= PAGE_SIZE;
blk->Size -= PAGE_SIZE;
}
if (blk->Free) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
// Do not decommit kernel-pool tail pages for now. The pool allocator keeps
// free-list nodes and boundary tags inside the managed arena; shrinking the
// mapped tail while large launch-failure cleanup is walking/freeing many
// objects can leave stale metadata pointers into unmapped pages. Physical
// pages are still reclaimed for user regions by MmDestroyUserRegion(); pool
// pages remain committed until we have a dedicated, audited heap shrinker.
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
static void* KiHeapAlloc(HEAP* h, size_t size) {
@@ -583,6 +600,11 @@ static void* KiHeapAlloc(HEAP* h, size_t size) {
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (h->Brk > h->Start && !KiHeapEnsureMapped(h, h->Start, h->Brk - h->Start)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void*)0;
}
// Best-fit search.
BLOCK_HDR* best = (BLOCK_HDR *)0;
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
@@ -735,12 +757,29 @@ static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
static HEAP gPoolHeap;
// ---------------------------------------------------------------------------
// User heap
// User virtual region arena
// ---------------------------------------------------------------------------
static HEAP gUserHeap;
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
static SPINLOCK gUserImageLock = SPINLOCK_INIT;
static uintptr_t gUserImageNext = USER_LIMIT;
typedef struct USER_VA_REGION {
LIST_ENTRY ProcessLink;
LIST_ENTRY GlobalLink;
PROCESS* Owner;
uintptr_t ReservationBase;
size_t ReservationSize;
uintptr_t Base;
size_t Size;
size_t CommittedSize;
uint64_t PageFlags;
USER_REGION_TYPE Type;
uint32_t Flags;
} USER_VA_REGION;
static LIST_ENTRY gUserRegionList;
static SPINLOCK gUserRegionLock = SPINLOCK_INIT;
static int gUserRegionListReady = 0;
#define USER_REGION_ARENA_BASE 0x0000100000000000ULL
// ---------------------------------------------------------------------------
// Module init
@@ -763,20 +802,11 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
KiHeapGrow(&gPoolHeap, poolSize);
}
// User heap.
gUserHeap.FreeList = (BLOCK_HDR *)0;
gUserHeap.Start = USER_BASE;
gUserHeap.Brk = USER_BASE;
gUserHeap.Limit = USER_LIMIT;
gUserHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX;
gUserHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
// Executable image mappings grow downward from the top of user space.
// The regular user heap grows upward from USER_BASE. Keeping them in
// separate page-aligned regions prevents W^X section protection from
// changing heap metadata or adjacent allocations.
gUserImageLock = (SPINLOCK)SPINLOCK_INIT;
gUserImageNext = USER_LIMIT;
// User VA arena. One global lower-half virtual-address bucket, with
// per-process ownership tracked by USER_VA_REGION descriptors.
LIST_INIT(&gUserRegionList);
gUserRegionLock = (SPINLOCK)SPINLOCK_INIT;
gUserRegionListReady = 1;
}
void MmActivate(void) {
@@ -791,69 +821,86 @@ __declspec(dllexport) void MmFreePool(void* ptr) { KiHeap
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); }
// ---------------------------------------------------------------------------
// Public user heap API
// User virtual region API
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapAlloc(&gUserHeap, size);
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
if (align & (align - 1)) return 0;
return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1);
}
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
KeAssertIrql(PASSIVE_LEVEL);
KiHeapFree(&gUserHeap, ptr);
static inline size_t KiAlignUpSize(size_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
if (align & (align - 1)) return 0;
return (v + align - 1) & ~(align - 1);
}
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapRealloc(&gUserHeap, ptr, newSize);
static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) {
uintptr_t aEnd = aBase + aSize;
uintptr_t bEnd = bBase + bSize;
return aBase < bEnd && bBase < aEnd;
}
static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) {
if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER;
if (!gUserRegionListReady) return STATUS_FAILURE;
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
if (!size) return (void*)0;
uintptr_t candidate = KiAlignUpPtr(USER_REGION_ARENA_BASE, alignment);
if (!candidate) return STATUS_INVALID_PARAMETER;
size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (pagesSize < size) return (void*)0;
for (;;) {
if (candidate < USER_REGION_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate))
return STATUS_NO_MEMORY;
uintptr_t flags = KeAcquireSpinlock(&gUserImageLock);
uintptr_t next = candidate;
int collided = 0;
if (pagesSize > gUserImageNext || gUserImageNext - pagesSize < gUserHeap.Brk) {
KeReleaseSpinlock(&gUserImageLock, flags);
return (void*)0;
}
uintptr_t base = gUserImageNext - pagesSize;
gUserImageNext = base;
KeReleaseSpinlock(&gUserImageLock, flags);
for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) {
MmFreeUserImage((void*)base, va - base);
return (void*)0;
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
if (KiRangesOverlap(candidate, reservationSize, r->ReservationBase, r->ReservationSize)) {
uintptr_t end = r->ReservationBase + r->ReservationSize;
next = KiAlignUpPtr(end, alignment);
if (!next || next <= candidate) return STATUS_NO_MEMORY;
collided = 1;
break;
}
}
KSTATUS s = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
if (!KSUCCESS(s)) {
MmFreeFrame(frame);
MmFreeUserImage((void*)base, va - base);
return (void*)0;
}
}
memset((void*)base, 0, pagesSize);
return (void*)base;
if (!collided) {
*baseOut = candidate;
return STATUS_SUCCESS;
}
candidate = next;
}
}
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
if (!ptr || !size) return;
static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) {
if (!process || !base) return (USER_VA_REGION*)0;
uintptr_t b = (uintptr_t)base;
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
if (r->Base == b) return r;
}
return (USER_VA_REGION*)0;
}
uintptr_t base = (uintptr_t)ptr & PAGE_MASK;
size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (pagesSize < size) return;
static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) {
if (!process || !base || !size) return (USER_VA_REGION*)0;
if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0;
uintptr_t end = base + size;
for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) {
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
uintptr_t regionEnd = r->Base + r->CommittedSize;
if (base >= r->Base && end <= regionEnd) return r;
}
return (USER_VA_REGION*)0;
}
static void KiUnmapUserRegionPayload(USER_VA_REGION* r) {
if (!r) return;
uintptr_t end = r->Base + r->CommittedSize;
for (uintptr_t va = r->Base; va < end; va += PAGE_SIZE) {
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va);
if (!phys) continue;
if (KSUCCESS(MmUnmapPage(va))) {
@@ -862,12 +909,499 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
}
}
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t regionFlags,
USER_REGION_TYPE type, void** outBase) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !outBase || !size) return STATUS_INVALID_PARAMETER;
*outBase = (void*)0;
if (!(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER;
if (!(pageFlags & PAGE_PRESENT)) pageFlags |= PAGE_PRESENT;
size_t payloadSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (payloadSize < size) return STATUS_NO_MEMORY;
size_t guardLow = (regionFlags & USER_REGION_GUARD_LOW) ? PAGE_SIZE : 0;
size_t guardHigh = (regionFlags & USER_REGION_GUARD_HIGH) ? PAGE_SIZE : 0;
size_t reservationSize = guardLow + payloadSize + guardHigh;
if (reservationSize < payloadSize) return STATUS_NO_MEMORY;
if (alignment < PAGE_SIZE) alignment = PAGE_SIZE;
if (alignment & (alignment - 1)) return STATUS_INVALID_PARAMETER;
USER_VA_REGION* r = (USER_VA_REGION*)MmAllocatePool(sizeof(USER_VA_REGION));
if (!r) return STATUS_NO_MEMORY;
memset(r, 0, sizeof(*r));
LIST_INIT(&r->ProcessLink);
LIST_INIT(&r->GlobalLink);
uintptr_t lockFlags = KeAcquireSpinlock(&gUserRegionLock);
uintptr_t reservationBase = 0;
KSTATUS s = KiFindUserVaRange(reservationSize, alignment, &reservationBase);
if (KSUCCESS(s)) {
r->Owner = process;
r->ReservationBase = reservationBase;
r->ReservationSize = reservationSize;
r->Base = reservationBase + guardLow;
r->Size = size;
r->CommittedSize = payloadSize;
r->PageFlags = pageFlags;
r->Type = type;
r->Flags = regionFlags;
ListInsertTail(&gUserRegionList, &r->GlobalLink);
}
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
if (!KSUCCESS(s)) {
MmFreePool(r);
return s;
}
uintptr_t va = r->Base;
for (; va < r->Base + payloadSize; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) {
s = STATUS_NO_MEMORY;
break;
}
s = MmMapPage(va, frame, pageFlags);
if (!KSUCCESS(s)) {
MmFreeFrame(frame);
break;
}
}
if (!KSUCCESS(s)) {
r->CommittedSize = (va > r->Base) ? (size_t)(va - r->Base) : 0;
KiUnmapUserRegionPayload(r);
lockFlags = KeAcquireSpinlock(&gUserRegionLock);
ListRemove(&r->GlobalLink);
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
MmFreePool(r);
return s;
}
if (regionFlags & USER_REGION_ZERO) {
memset((void*)r->Base, 0, payloadSize);
}
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
ListInsertTail(&process->UserRegions, &r->ProcessLink);
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
*outBase = (void*)r->Base;
return STATUS_SUCCESS;
}
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
uintptr_t start = (uintptr_t)base;
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
uintptr_t alignedStart = start & PAGE_MASK;
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
uintptr_t regionBase = r ? r->Base : 0;
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
uint64_t defaultFlags = r ? r->PageFlags : 0;
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
if (!r) {
DbgPrint("MM: ensure user mapping failed: pid=%u range=%016llx-%016llx not inside owned committed region\n",
process->ProcessId,
(unsigned long long)alignedStart,
(unsigned long long)alignedEnd);
return STATUS_NOT_FOUND;
}
uint64_t flags = pageFlags ? pageFlags : defaultFlags;
flags |= PAGE_PRESENT | PAGE_USER;
uint32_t committed = 0;
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED;
PHYSICAL_ADDRESS phys = 0;
uint64_t existingFlags = 0;
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
if (KSUCCESS(qs) && (existingFlags & PAGE_PRESENT))
continue;
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) return STATUS_NO_MEMORY;
KSTATUS ms = MmMapPage(va, frame, flags);
if (!KSUCCESS(ms)) {
MmFreeFrame(frame);
return ms;
}
memset((void*)va, 0, PAGE_SIZE);
committed++;
}
return STATUS_SUCCESS;
}
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
uintptr_t start = (uintptr_t)base;
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
uintptr_t alignedStart = start & PAGE_MASK;
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
uintptr_t regionBase = r ? r->Base : 0;
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
if (!r) {
DbgPrint("MM: protect user region failed: pid=%u range=%016llx-%016llx not owned/committed\n",
process->ProcessId,
(unsigned long long)alignedStart,
(unsigned long long)alignedEnd);
return STATUS_NOT_FOUND;
}
uint64_t finalFlags = pageFlags | PAGE_PRESENT | PAGE_USER;
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
if (va < regionBase || va >= regionEnd) {
DbgPrint("MM: protect user region hit guard/outside page pid=%u va=%016llx region=%016llx-%016llx\n",
process->ProcessId,
(unsigned long long)va,
(unsigned long long)regionBase,
(unsigned long long)regionEnd);
return STATUS_ACCESS_DENIED;
}
PHYSICAL_ADDRESS phys = 0;
uint64_t existingFlags = 0;
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
if (!KSUCCESS(qs) || !(existingFlags & PAGE_PRESENT)) {
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) return STATUS_NO_MEMORY;
KSTATUS ms = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
if (!KSUCCESS(ms)) {
MmFreeFrame(frame);
return ms;
}
memset((void*)va, 0, PAGE_SIZE);
DbgPrint("MM: committed missing user page during protect pid=%u va=%016llx\n",
process->ProcessId, (unsigned long long)va);
}
KSTATUS ps = MmSetPageProtect(va, PAGE_SIZE, finalFlags);
if (!KSUCCESS(ps)) {
PHYSICAL_ADDRESS dbgPhys = 0;
uint64_t dbgFlags = 0;
KSTATUS dbg = MmQueryPage(va, &dbgPhys, &dbgFlags);
DbgPrint("MM: protect user page failed pid=%u va=%016llx st=%d query=%d phys=%016llx flags=%016llx final=%016llx\n",
process->ProcessId,
(unsigned long long)va,
ps, dbg,
(unsigned long long)dbgPhys,
(unsigned long long)dbgFlags,
(unsigned long long)finalFlags);
return ps;
}
}
return STATUS_SUCCESS;
}
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base) return STATUS_INVALID_PARAMETER;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
if (!r) {
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
return STATUS_NOT_FOUND;
}
ListRemove(&r->ProcessLink);
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
ListRemove(&r->GlobalLink);
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
KiUnmapUserRegionPayload(r);
MmFreePool(r);
return STATUS_SUCCESS;
}
void MmDestroyAllUserRegions(PROCESS* process) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process) return;
for (;;) {
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
LIST_ENTRY* node = ListPopHead(&process->UserRegions);
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
if (!node) break;
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
ListRemove(&r->GlobalLink);
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
KiUnmapUserRegionPayload(r);
MmFreePool(r);
}
}
size_t MmGetUserRegionSize(PROCESS* process, const void* base) {
if (!process || !base) return 0;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
size_t size = r ? r->Size : 0;
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
return size;
}
size_t MmGetUserRegionSizeByBase(const void* base) {
if (!base) return 0;
uintptr_t b = (uintptr_t)base;
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
if (r->Base == b) {
size_t size = r->Size;
KeReleaseSpinlock(&gUserRegionLock, flags);
return size;
}
}
KeReleaseSpinlock(&gUserRegionLock, flags);
return 0;
}
static const char* KiUserRegionTypeName(USER_REGION_TYPE type) {
switch (type) {
case UserRegionGeneric: return "generic";
case UserRegionHeap: return "heap";
case UserRegionImage: return "image";
case UserRegionStack: return "stack";
case UserRegionTeb: return "teb";
case UserRegionPeb: return "peb";
case UserRegionEnvironment: return "env";
case UserRegionMappedFile: return "mapped";
case UserRegionShared: return "shared";
default: return "?";
}
}
void MmDebugDumpUserRegions(PROCESS* process) {
if (!process) {
DbgPrint("MM: user regions: <no process>\n");
return;
}
DbgPrint("MM: user regions for pid=%u name=%s:\n", process->ProcessId, process->Name);
uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock);
uint32_t count = 0;
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx flags=%x pte=%llx\n",
count,
KiUserRegionTypeName(r->Type),
(unsigned long long)r->ReservationBase,
(unsigned long long)(r->ReservationBase + r->ReservationSize),
(unsigned long long)r->Base,
(unsigned long long)r->Size,
(unsigned long long)r->CommittedSize,
r->Flags,
(unsigned long long)r->PageFlags);
count++;
}
KeReleaseSpinlock(&process->UserRegionLock, flags);
if (!count) DbgPrint(" <none>\n");
}
static void KiDebugDescribePoolAddress(uintptr_t addr) {
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
(unsigned long long)gPoolHeap.Start,
(unsigned long long)gPoolHeap.Brk,
(unsigned long long)gPoolHeap.Limit,
(unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
if (addr >= gPoolHeap.Brk) {
DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
(unsigned long long)addr,
(unsigned long long)(addr - gPoolHeap.Brk));
return;
}
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
(uintptr_t)cur < gPoolHeap.Brk;
cur = KiNextBlock(cur)) {
if (cur->Magic != BLOCK_MAGIC) {
DbgPrint("MM: pool walk corrupt at hdr=%p magic=%08x\n", cur, cur->Magic);
break;
}
uintptr_t hdr = (uintptr_t)cur;
uintptr_t data = hdr + HDR_SIZE;
uintptr_t end = data + cur->Size;
uintptr_t ftr = end;
if (addr >= hdr && addr < ftr + FTR_SIZE) {
DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx free=%u\n",
(unsigned long long)addr,
(unsigned long long)hdr,
(unsigned long long)data,
(unsigned long long)end,
(unsigned long long)cur->Size,
cur->Free);
break;
}
if (KiNextBlock(cur) <= cur) {
DbgPrint("MM: pool walk stopped: non-advancing block hdr=%p size=%llx\n", cur, (unsigned long long)cur->Size);
break;
}
}
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
}
void MmDebugDescribeAddress(uintptr_t addr) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
KSTATUS qs = MmQueryPage(addr, &phys, &flags);
DbgPrint("MM: describe va=%016llx query=%d phys=%016llx flags=%016llx\n",
(unsigned long long)addr,
qs,
(unsigned long long)phys,
(unsigned long long)flags);
if (addr >= POOL_BASE && addr < POOL_LIMIT) {
KiDebugDescribePoolAddress(addr);
return;
}
if (addr >= USER_REGION_ARENA_BASE && addr < USER_LIMIT) {
uintptr_t gl = KeAcquireSpinlock(&gUserRegionLock);
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
if (addr >= r->ReservationBase && addr < r->ReservationBase + r->ReservationSize) {
DbgPrint("MM: address belongs to pid=%u name=%s region=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n",
r->Owner ? r->Owner->ProcessId : 0,
r->Owner ? r->Owner->Name : "?",
KiUserRegionTypeName(r->Type),
(unsigned long long)r->ReservationBase,
(unsigned long long)(r->ReservationBase + r->ReservationSize),
(unsigned long long)r->Base,
(unsigned long long)r->Size,
r->Flags);
KeReleaseSpinlock(&gUserRegionLock, gl);
return;
}
}
KeReleaseSpinlock(&gUserRegionLock, gl);
DbgPrint("MM: address is in user arena but not reserved by any USER_VA_REGION\n");
}
}
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) {
void* base = (void*)0;
if (!process || !size) return (void*)0;
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
USER_REGION_ZERO,
UserRegionHeap,
&base);
return KSUCCESS(s) ? base : (void*)0;
}
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) {
if (!process || !ptr) return;
(void)MmDestroyUserRegion(process, ptr);
}
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) {
if (!process) return (void*)0;
if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize);
if (!newSize) {
MmFreeUserMemoryForProcess(process, ptr);
return (void*)0;
}
size_t oldSize = MmGetUserRegionSize(process, ptr);
if (!oldSize) return (void*)0;
if (newSize <= oldSize) return ptr;
void* newPtr = MmAllocateUserMemoryForProcess(process, newSize);
if (!newPtr) return (void*)0;
memcpy(newPtr, ptr, oldSize);
MmFreeUserMemoryForProcess(process, ptr);
return newPtr;
}
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
PROCESS* process = KeGetCurrentProcess();
return MmAllocateUserMemoryForProcess(process, size);
}
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
PROCESS* process = KeGetCurrentProcess();
MmFreeUserMemoryForProcess(process, ptr);
}
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
PROCESS* process = KeGetCurrentProcess();
return MmReallocateUserMemoryForProcess(process, ptr, newSize);
}
__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) {
void* base = (void*)0;
if (!process || !size) return (void*)0;
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
UserRegionImage,
&base);
return KSUCCESS(s) ? base : (void*)0;
}
__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) {
(void)size;
if (!process || !ptr) return;
(void)MmDestroyUserRegion(process, ptr);
}
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
PROCESS* process = KeGetCurrentProcess();
return MmAllocateUserImageForProcess(process, size);
}
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
PROCESS* process = KeGetCurrentProcess();
MmFreeUserImageForProcess(process, ptr, size);
}
size_t MmGetKernelPoolUsage(void) {
// Returns bytes committed to the kernel pool
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
}
size_t MmGetUserHeapUsage(void) {
// Returns bytes committed to the user heap
return (size_t)(gUserHeap.Brk - gUserHeap.Start);
size_t total = 0;
if (!gUserRegionListReady) return 0;
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
total += r->CommittedSize;
}
KeReleaseSpinlock(&gUserRegionLock, flags);
return total;
}