maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace the VHD/diskpart disk image pipeline with a directory-based vvfat root, we'll see when it breaks, fix /O2, add an all_tests, generalize the fat32 driver into a fat 16+32 driver, change policy on volume drivers, collapse KCPU's fields into a single atomic KE_CPU_STATE state machine, move scheduler ticking to per-CPU LAPIC timers, gate verbose SMP topology logging behind selftest diagnostics and add SMP pool-checkpoint validation around AP admission, add checked intrusive list helpers behind a new KE_DEBUG_LIST_CHECKS flag, add ObInitializeObjectHeader/ObTryReferenceObject, add waitable display clock objects and a SYS_OPEN_DISPLAY_CLOCK syscall for vsync-style waits, refactor kernel input into a per-queue KE_INPUT_CONTEXT with its own key/mouse ring buffers and DPC-driven key repeat, add KeInputQueuePeekEvent/ReadEvent/WaitKey and a SYS_QUERY_GLOBAL_INPUT_STATE syscall, split the kernel pool allocator out of mm_virt, new video files!

Browse Source
This commit is contained in:
maksw committed 2026-07-12 00:10:21 +02:00
1 parent 485a6bbfe7
commit 8b254f84f4
74 files changed
+4488 -2503
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/mm_virt.c
+5 -757
View File
@@ -1,5 +1,5 @@
// Kernel virtual-memory manager. This file owns kernel pool allocation, vad bookkeeping, and user-copy validation.
// Page-table construction lives in mm_pte.c so the higher VMM can grow toward VADs, faults, COW, and swap without mixing those policies with raw PTE edits.
// Kernel virtual-memory manager. This file owns VAD bookkeeping, faults, and user-copy validation.
// Pool allocation lives in mm_pool.c and page-table construction lives in mm_pte.c.
// Virtual address layout:
// 0000000000001000-00007FFFFFFFF000 is the lower-half user range.
@@ -12,9 +12,6 @@
// The current VM model uses one shared PML4, so user pointers are globally
// meaningful while protection and lifetime are still tracked per process VAD.
// Kernel pool:
// A boundary-tag allocator starts at POOL_BASE and commits pages on demand.
// User mappings:
// VADs separate reservation from commitment. Committed pages become resident
// when a user fault or kernel prefault path materializes them.
@@ -22,6 +19,7 @@
#include <string.h>
#include <intrin.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_pool.h>
#include <kernel/mm_pte.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_swap.h>
@@ -86,649 +84,6 @@ __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, s
return KiCopyToUserPaged(userDst, kernelSrc, size);
}
// Boundary-tag allocator (used for both pool and user heap)
// Each allocation has a BLOCK_HDR immediately before the user data, and a
// BLOCK_FTR immediately after. Both carry the block size and a 'free' flag.
// This allows O(1) coalescing of adjacent free blocks.
#define BLOCK_MAGIC 0xB10CB10CUL
#define BLOCK_STATE_USED 0u
#define BLOCK_STATE_FREE 1u
#define BLOCK_STATE_QUARANTINE 2u
#define BLOCK_FLAG_CANARY 0x00000001u
#define BLOCK_FLAG_POISONED 0x00000002u
#define POOL_TAIL_CANARY_BASE 0xC0FFEE1234ABCDEFULL
#define POOL_POISON_FREE 0xDD
#define POOL_QUARANTINE_SLOTS 256u
#define POOL_TAG_DEFAULT 0x6C6F6F70u /* 'pool' */
typedef struct BLOCK_HDR {
uint32_t Magic;
uint32_t Free; // BLOCK_STATE_*
size_t Size; // usable allocator bytes (excludes header + footer)
struct BLOCK_HDR* PrevFree;
struct BLOCK_HDR* NextFree;
// Debug metadata. Present in all builds so boot.ini bits can enable
// diagnostics without changing allocator layout or ABI.
size_t RequestedSize;
uint32_t Tag;
uint32_t Flags;
uint64_t Sequence;
uintptr_t AllocCaller;
uintptr_t FreeCaller;
} BLOCK_HDR;
typedef struct BLOCK_FTR {
size_t Size;
uint32_t Free;
uint32_t Magic;
} BLOCK_FTR;
#define HDR_SIZE (sizeof(BLOCK_HDR))
#define FTR_SIZE (sizeof(BLOCK_FTR))
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
// Implements the hdr To Ftr operation.
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
}
// Implements the ftr To Hdr operation.
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
}
// Implements the next Block operation.
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
}
// Implements the prev Ftr operation.
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
}
typedef struct HEAP HEAP;
struct HEAP {
BLOCK_HDR* FreeList; // doubly-linked list of free blocks
uintptr_t Start; // virtual base of the arena
uintptr_t Brk; // current committed end (page-aligned)
uintptr_t Limit; // hard upper bound
uint64_t MapFlags; // PAGE_* flags for new committed pages
SPINLOCK Lock;
int DecommitTailOnFree;
uint64_t NextSequence;
BLOCK_HDR* Quarantine[POOL_QUARANTINE_SLOTS];
size_t QuarantineCursor;
size_t QuarantineCount;
};
static inline uint32_t KiPoolDebugFlags(void) {
return KeGetDebugFlags() & (KE_DEBUG_POOL_CANARY |
KE_DEBUG_POOL_POISON |
KE_DEBUG_POOL_QUARANTINE |
KE_DEBUG_POOL_CALLERS);
}
static inline uint64_t KiPoolTailCanaryValue(const BLOCK_HDR* blk) {
return POOL_TAIL_CANARY_BASE ^
((uint64_t)(uintptr_t)blk << 13) ^
((uint64_t)blk->RequestedSize << 1) ^
blk->Sequence;
}
static inline uint8_t* KiBlockUserBase(BLOCK_HDR* blk) {
return (uint8_t*)blk + HDR_SIZE;
}
static inline uint64_t* KiBlockTailCanary(BLOCK_HDR* blk) {
return (uint64_t*)(void*)(KiBlockUserBase(blk) + blk->RequestedSize);
}
static const char* KiPoolStateName(uint32_t state) {
switch (state) {
case BLOCK_STATE_USED: return "used";
case BLOCK_STATE_FREE: return "free";
case BLOCK_STATE_QUARANTINE: return "quarantine";
default: return "bad-state";
}
}
static _Noreturn void KiPoolPanic(const char* reason, HEAP* h, BLOCK_HDR* blk, const void* ptr) {
KePanic(KE_PANIC_CODE_GENERIC,
"MM: pool corruption: %s ptr=0x%016llX hdr=0x%016llX heap=[%016llx,%016llx) magic=%08x state=%u(%s) size=%llu req=%llu tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX\n",
reason ? reason : "unknown",
ptr,
blk,
(unsigned long long)(h ? h->Start : 0),
(unsigned long long)(h ? h->Brk : 0),
blk ? blk->Magic : 0,
blk ? blk->Free : 0,
blk ? KiPoolStateName(blk->Free) : "<null>",
(unsigned long long)(blk ? blk->Size : 0),
(unsigned long long)(blk ? blk->RequestedSize : 0),
blk ? blk->Tag : 0,
blk ? blk->Flags : 0,
(unsigned long long)(blk ? blk->Sequence : 0),
blk ? (void*)blk->AllocCaller : (void*)0,
blk ? (void*)blk->FreeCaller : (void*)0);
}
static void KiPoolValidateFooterLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!h || !blk)
KiPoolPanic(where, h, blk, blk ? KiBlockUserBase(blk) : (void*)0);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
if ((uintptr_t)blk < h->Start || (uintptr_t)blk >= h->Brk)
KiPoolPanic("block outside heap", h, blk, KiBlockUserBase(blk));
if (blk->Size > (size_t)(h->Brk - (uintptr_t)blk - OVERHEAD))
KiPoolPanic("block size outside heap", h, blk, KiBlockUserBase(blk));
BLOCK_FTR* ftr = KiHdrToFtr(blk);
if ((uintptr_t)ftr + FTR_SIZE > h->Brk)
KiPoolPanic("footer outside heap", h, blk, KiBlockUserBase(blk));
if (ftr->Magic != BLOCK_MAGIC || ftr->Size != blk->Size || ftr->Free != blk->Free)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
static void KiPoolWriteTailCanary(BLOCK_HDR* blk) {
if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
return;
*KiBlockTailCanary(blk) = KiPoolTailCanaryValue(blk);
}
static void KiPoolValidateTailCanaryLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
return;
if (blk->RequestedSize + sizeof(uint64_t) > blk->Size)
KiPoolPanic("invalid canary layout", h, blk, KiBlockUserBase(blk));
uint64_t actual = *KiBlockTailCanary(blk);
uint64_t expect = KiPoolTailCanaryValue(blk);
if (actual != expect)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
static void KiPoolPoisonFreedPayload(BLOCK_HDR* blk) {
if (!blk)
return;
size_t bytes = blk->RequestedSize;
if (bytes > blk->Size)
bytes = blk->Size;
memset(KiBlockUserBase(blk), POOL_POISON_FREE, bytes);
blk->Flags |= BLOCK_FLAG_POISONED;
KiPoolWriteTailCanary(blk);
}
static void KiPoolValidateFreedPoisonLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!blk || !(blk->Flags & BLOCK_FLAG_POISONED))
return;
size_t bytes = blk->RequestedSize;
if (bytes > blk->Size)
bytes = blk->Size;
const uint8_t* p = KiBlockUserBase(blk);
for (size_t i = 0; i < bytes; i++) {
if (p[i] != POOL_POISON_FREE)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
}
// Rolls back a failed heap grow attempt. KiHeapGrow maps pages before the
// new free block is linked into the allocator; if a later frame allocation or
// page-table allocation fails, those earlier mappings must be returned or the
// pool silently leaks physical frames and leaves stale mappings below Brk.
static void KiHeapRollbackGrow(uintptr_t start, uintptr_t end) {
for (uintptr_t va = start; va < end; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPteTranslate(va);
if (!frame)
continue;
if (KSUCCESS(MmPteUnmapPage(va)))
(void)MmPfnFree(frame);
}
}
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int KiHeapGrow(HEAP* h, size_t need) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (h->Brk + grow > h->Limit) return 0;
uintptr_t start = h->Brk;
uintptr_t mappedEnd = start;
for (uintptr_t va = start; va < start + grow; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame) {
KiHeapRollbackGrow(start, mappedEnd);
return 0;
}
if (!KSUCCESS(MmPteMapPage(va, frame, h->MapFlags, MmPteMapFailIfPresent))) {
MmPfnFree(frame);
KiHeapRollbackGrow(start, mappedEnd);
return 0;
}
mappedEnd = va + PAGE_SIZE;
memset((void *)va, 0, PAGE_SIZE);
}
// The grow path owns the only transition from reserved heap VA to mapped heap
// pages, so a successful loop above is the heap's committed-frontier proof.
// Re-querying every heap page on later allocations only burns page-table locks.
// Missing pool pages are corruption now, not something the allocator repairs.
uintptr_t oldBrk = h->Brk;
uintptr_t newBrk = oldBrk + grow;
// If the current tail block is already free, extend it in-place instead of
// leaving adjacent free blocks. This matters once tail decommit is enabled:
// a later grow should rebuild the same tail block, not fragment the arena
// into many page-sized free runs.
if (oldBrk > h->Start) {
BLOCK_FTR* prevFtr = (BLOCK_FTR*)(oldBrk - FTR_SIZE);
if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free == BLOCK_STATE_FREE) {
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
if ((uintptr_t)prev >= h->Start && prev->Magic == BLOCK_MAGIC && prev->Free == BLOCK_STATE_FREE) {
prev->Size += grow;
h->Brk = newBrk;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(prev)->Size = prev->Size;
return 1;
}
}
}
// Make the new pages a single free block.
BLOCK_HDR* blk = (BLOCK_HDR *)oldBrk;
blk->Magic = BLOCK_MAGIC;
blk->Free = BLOCK_STATE_FREE;
blk->Size = grow - OVERHEAD;
blk->PrevFree = (BLOCK_HDR *)0;
blk->NextFree = h->FreeList;
blk->RequestedSize = 0;
blk->Tag = 0;
blk->Flags = 0;
blk->Sequence = 0;
blk->AllocCaller = 0;
blk->FreeCaller = 0;
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
h->Brk = newBrk;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Size = blk->Size;
return 1;
}
// Unlink a free block from the free list.
static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
else h->FreeList = blk->NextFree;
if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
}
// Implements the heap Try Decommit Tail operation.
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk || blk->Free == BLOCK_STATE_USED) return;
// Do not eagerly decommit the kernel pool on every free. Large transient
// syscall bounce buffers, especially SysBlitScreen's frame-sized buffer,
// are often allocated/freed once per frame. If the tail block is unmapped
// immediately, the next frame has to map the same pages again, producing
// massive page-table churn and global TLB shootdowns. Keep the committed
// pool high-water resident during normal operation; add an explicit idle or
// pressure pool trim later if needed.
if (!h->DecommitTailOnFree) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
uintptr_t blockBase = (uintptr_t)blk;
uintptr_t oldBrk = h->Brk;
// Only the last free block can return committed heap pages. Interior free
// blocks must remain mapped because the allocator's boundary tags and free
// list links are stored inside those blocks.
if ((uintptr_t)KiNextBlock(blk) != oldBrk) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
// Keep enough mapped space for the tail block's header, footer, and a small
// usable payload, then decommit whole pages above that. The footer moves to
// the new Brk so future coalescing can still walk the boundary tag chain.
uintptr_t minEnd = blockBase + MIN_BLOCK;
uintptr_t newBrk = (minEnd + PAGE_SIZE - 1) & ~(uintptr_t)(PAGE_SIZE - 1);
if (newBrk >= oldBrk) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
for (uintptr_t va = newBrk; va < oldBrk; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPteTranslate(va);
if (!frame)
continue;
if (KSUCCESS(MmPteUnmapPage(va)))
(void)MmPfnFree(frame);
}
h->Brk = newBrk;
blk->Size = newBrk - blockBase - OVERHEAD;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
}
// Releases a previously quarantined block into the real free list. Caller holds h->Lock.
static void KiHeapReleaseFreeBlockLocked(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk) return;
KiPoolValidateFooterLocked(h, blk, "release free footer corrupt");
KiPoolValidateTailCanaryLocked(h, blk, "release free tail canary corrupt");
KiPoolValidateFreedPoisonLocked(h, blk, "use-after-free payload write detected");
blk->Free = BLOCK_STATE_FREE;
blk->RequestedSize = 0;
blk->Tag = 0;
blk->Flags = 0;
blk->AllocCaller = 0;
// Keep FreeCaller/Sequence for diagnostics until the block is reused.
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
// Coalesce with next block if it is really free. Quarantined neighbors are
// intentionally not coalesced; their payload must remain intact for UAF checks.
BLOCK_HDR* next = KiNextBlock(blk);
if ((uintptr_t)next < h->Brk) {
KiPoolValidateFooterLocked(h, next, "next footer corrupt while freeing");
if (next->Free == BLOCK_STATE_FREE) {
KiFreeListRemove(h, next);
blk->Size += OVERHEAD + next->Size;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
}
}
// Coalesce with previous block if it is really free.
if ((uintptr_t)blk > h->Start) {
BLOCK_FTR* prevFtr = KiPrevFtr(blk);
if (prevFtr->Magic == BLOCK_MAGIC) {
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
KiPoolValidateFooterLocked(h, prev, "previous footer corrupt while freeing");
if (prev->Free == BLOCK_STATE_FREE) {
KiFreeListRemove(h, prev);
prev->Size += OVERHEAD + blk->Size;
KiHdrToFtr(prev)->Size = prev->Size;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE;
blk = prev;
}
}
}
// Insert at head of free list.
blk->PrevFree = (BLOCK_HDR *)0;
blk->NextFree = h->FreeList;
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
KiHeapTryDecommitTail(h, blk);
}
static void KiHeapReleaseQuarantineSlotLocked(HEAP* h, size_t index) {
if (!h || index >= POOL_QUARANTINE_SLOTS)
return;
BLOCK_HDR* old = h->Quarantine[index];
if (!old)
return;
h->Quarantine[index] = (BLOCK_HDR*)0;
if (h->QuarantineCount)
h->QuarantineCount--;
if (old->Free != BLOCK_STATE_QUARANTINE)
KiPoolPanic("quarantine slot state corrupt", h, old, KiBlockUserBase(old));
KiHeapReleaseFreeBlockLocked(h, old);
}
static void KiHeapDrainQuarantineLocked(HEAP* h) {
if (!h)
return;
for (size_t i = 0; i < POOL_QUARANTINE_SLOTS; i++)
KiHeapReleaseQuarantineSlotLocked(h, i);
h->QuarantineCursor = 0;
}
static void KiHeapQuarantineBlockLocked(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk)
return;
size_t idx = h->QuarantineCursor % POOL_QUARANTINE_SLOTS;
KiHeapReleaseQuarantineSlotLocked(h, idx);
blk->Free = BLOCK_STATE_QUARANTINE;
KiHdrToFtr(blk)->Free = BLOCK_STATE_QUARANTINE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
h->Quarantine[idx] = blk;
h->QuarantineCursor = (idx + 1u) % POOL_QUARANTINE_SLOTS;
h->QuarantineCount++;
}
// Implements the heap Alloc operation.
static void* KiHeapAllocEx(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t caller) {
if (!requestedSize) return (void *)0;
uint32_t dbg = KiPoolDebugFlags();
size_t size = requestedSize;
if (dbg & KE_DEBUG_POOL_CANARY) {
if (size > (size_t)-1 - sizeof(uint64_t))
return (void*)0;
size += sizeof(uint64_t);
}
size = (size + 7) & ~7ULL; // 8-byte align
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
// Best-fit search.
BLOCK_HDR* best = (BLOCK_HDR *)0;
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
KiPoolValidateFooterLocked(h, cur, "free list block corrupt");
if (cur->Free != BLOCK_STATE_FREE)
KiPoolPanic("free list contains non-free block", h, cur, KiBlockUserBase(cur));
if (cur->Size >= size) {
if (!best || cur->Size < best->Size) best = cur;
if (best->Size == size) break;
}
}
if (!best && h->QuarantineCount) {
KiHeapDrainQuarantineLocked(h);
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
KiPoolValidateFooterLocked(h, cur, "free list block corrupt after quarantine drain");
if (cur->Size >= size) {
if (!best || cur->Size < best->Size) best = cur;
if (best->Size == size) break;
}
}
}
if (!best) {
if (!KiHeapGrow(h, size + OVERHEAD)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
// Re-search; KiHeapGrow inserted at the front.
best = h->FreeList;
while (best && best->Size < size) best = best->NextFree;
if (!best) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
KiPoolValidateFooterLocked(h, best, "grown free block corrupt");
}
KiFreeListRemove(h, best);
// Split the block if the remainder is big enough.
if (best->Size >= size + OVERHEAD + sizeof(void*)) {
BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE);
rem->Magic = BLOCK_MAGIC;
rem->Free = BLOCK_STATE_FREE;
rem->Size = best->Size - size - OVERHEAD;
rem->PrevFree = (BLOCK_HDR *)0;
rem->NextFree = h->FreeList;
rem->RequestedSize = 0;
rem->Tag = 0;
rem->Flags = 0;
rem->Sequence = 0;
rem->AllocCaller = 0;
rem->FreeCaller = 0;
if (h->FreeList) h->FreeList->PrevFree = rem;
h->FreeList = rem;
KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
KiHdrToFtr(rem)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(rem)->Size = rem->Size;
best->Size = size;
}
best->Free = BLOCK_STATE_USED;
best->RequestedSize = requestedSize;
best->Tag = tag ? tag : POOL_TAG_DEFAULT;
best->Flags = 0;
best->Sequence = ++h->NextSequence;
best->AllocCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
best->FreeCaller = 0;
if (dbg & KE_DEBUG_POOL_CANARY)
best->Flags |= BLOCK_FLAG_CANARY;
KiHdrToFtr(best)->Free = BLOCK_STATE_USED;
KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
KiHdrToFtr(best)->Size = best->Size;
memset(KiBlockUserBase(best), 0, requestedSize);
if (best->Size > requestedSize)
memset(KiBlockUserBase(best) + requestedSize, 0, best->Size - requestedSize);
KiPoolWriteTailCanary(best);
KeReleaseSpinlock(&h->Lock, flags);
return KiBlockUserBase(best);
}
static void* KiHeapAlloc(HEAP* h, size_t size) {
return KiHeapAllocEx(h, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
}
// Implements the heap Free operation.
static void KiHeapFreeEx(HEAP* h, void* ptr, uintptr_t caller) {
if (!ptr) return;
uintptr_t u = (uintptr_t)ptr;
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid free pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic("invalid free pointer/bad header magic", h, blk, ptr);
KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on free");
if (blk->Free == BLOCK_STATE_FREE)
KiPoolPanic("double free", h, blk, ptr);
if (blk->Free == BLOCK_STATE_QUARANTINE)
KiPoolPanic("double free/quarantined UAF", h, blk, ptr);
if (blk->Free != BLOCK_STATE_USED)
KiPoolPanic("invalid block state on free", h, blk, ptr);
KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on free");
uint32_t dbg = KiPoolDebugFlags();
blk->FreeCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
if (dbg & KE_DEBUG_POOL_POISON)
KiPoolPoisonFreedPayload(blk);
if (dbg & KE_DEBUG_POOL_QUARANTINE) {
KiHeapQuarantineBlockLocked(h, blk);
} else {
KiHeapReleaseFreeBlockLocked(h, blk);
}
KeReleaseSpinlock(&h->Lock, flags);
}
static void KiHeapFree(HEAP* h, void* ptr) {
KiHeapFreeEx(h, ptr, (uintptr_t)_ReturnAddress());
}
// Implements the heap Realloc operation.
static void* KiHeapReallocEx(HEAP* h, void* ptr, size_t newSize, uintptr_t caller) {
// If ptr is NULL, realloc acts like malloc.
if (!ptr) {
return KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller);
}
// If size is 0, realloc acts like free.
if (newSize == 0) {
KiHeapFreeEx(h, ptr, caller);
return (void*)0;
}
uintptr_t u = (uintptr_t)ptr;
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid realloc pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic("invalid realloc pointer/bad header magic", h, blk, ptr);
KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on realloc");
if (blk->Free != BLOCK_STATE_USED)
KiPoolPanic("realloc of non-allocated block", h, blk, ptr);
KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on realloc");
size_t oldRequested = blk->RequestedSize;
size_t oldCopy = oldRequested < newSize ? oldRequested : newSize;
if (newSize <= oldRequested) {
blk->RequestedSize = newSize;
if (blk->Flags & BLOCK_FLAG_CANARY)
KiPoolWriteTailCanary(blk);
KeReleaseSpinlock(&h->Lock, flags);
return ptr;
}
KeReleaseSpinlock(&h->Lock, flags);
void* newPtr = KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller);
if (!newPtr)
return (void*)0;
memcpy(newPtr, ptr, oldCopy);
KiHeapFreeEx(h, ptr, caller);
return newPtr;
}
static void* KiHeapRealloc(HEAP* h, void* ptr, size_t newSize) {
return KiHeapReallocEx(h, ptr, newSize, (uintptr_t)_ReturnAddress());
}
// Kernel pool.
static HEAP gPoolHeap;
// VAD arena.
typedef enum MM_VAD_PAGE_STATE {
MmVadPageFree = 0,
@@ -803,22 +158,7 @@ static void KiVadInitialize(void) {
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
MmPteInitialize(bootPml4Phys);
gPoolHeap.FreeList = (BLOCK_HDR *)0;
gPoolHeap.Start = POOL_BASE;
gPoolHeap.Brk = POOL_BASE;
gPoolHeap.Limit = POOL_LIMIT;
gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
gPoolHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
gPoolHeap.DecommitTailOnFree = 0;
gPoolHeap.NextSequence = 0;
gPoolHeap.QuarantineCursor = 0;
gPoolHeap.QuarantineCount = 0;
memset(gPoolHeap.Quarantine, 0, sizeof(gPoolHeap.Quarantine));
if (poolSize) {
poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
KiHeapGrow(&gPoolHeap, poolSize);
}
MmPoolInitialize(poolSize);
KiVadInitialize();
@@ -830,26 +170,6 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
(void)MmPfnPin(gVadZeroPageFrame);
}
// Allocates zeroed kernel-pool memory from the non-user heap.
__declspec(dllexport) void* MmAllocatePool(size_t size) {
return KiHeapAllocEx(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
}
// Allocates zeroed kernel-pool memory with a diagnostic tag.
__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag) {
return KiHeapAllocEx(&gPoolHeap, size, tag, (uintptr_t)_ReturnAddress());
}
// Releases kernel-pool memory back to the non-user heap.
__declspec(dllexport) void MmFreePool(void* ptr) {
KiHeapFreeEx(&gPoolHeap, ptr, (uintptr_t)_ReturnAddress());
}
// Reallocates kernel-pool memory while preserving existing contents.
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) {
return KiHeapReallocEx(&gPoolHeap, ptr, newSize, (uintptr_t)_ReturnAddress());
}
// VAD utility helpers.
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
@@ -3122,72 +2442,6 @@ void MmVadDebugDumpProcess(PROCESS* process) {
if (!count) DbgPrint(" <none>\n");
}
// Implements the debug Describe Pool Address operation.
static void KiDebugDescribePoolAddress(uintptr_t addr) {
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
(unsigned long long)gPoolHeap.Start,
(unsigned long long)gPoolHeap.Brk,
(unsigned long long)gPoolHeap.Limit,
(unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
if (addr >= gPoolHeap.Brk) {
DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
(unsigned long long)addr,
(unsigned long long)(addr - gPoolHeap.Brk));
return;
}
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
(uintptr_t)cur < gPoolHeap.Brk;
cur = KiNextBlock(cur)) {
if (cur->Magic != BLOCK_MAGIC) {
DbgPrint("MM: pool walk corrupt at hdr=0x%016llX magic=%08x\n", cur, cur->Magic);
break;
}
uintptr_t hdr = (uintptr_t)cur;
uintptr_t data = hdr + HDR_SIZE;
uintptr_t end = data + cur->Size;
uintptr_t ftr = end;
BLOCK_FTR* footer = KiHdrToFtr(cur);
if ((uintptr_t)footer + FTR_SIZE > gPoolHeap.Brk ||
footer->Magic != BLOCK_MAGIC || footer->Size != cur->Size || footer->Free != cur->Free) {
DbgPrint("MM: pool walk corrupt footer hdr=0x%016llX state=%u size=%llx footer=0x%016llX fmagic=%08x fstate=%u fsize=%llx\n",
cur,
cur->Free,
(unsigned long long)cur->Size,
footer,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Magic : 0,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Free : 0,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? (unsigned long long)footer->Size : 0ULL);
break;
}
if (addr >= hdr && addr < ftr + FTR_SIZE) {
DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx req=%llx state=%u(%s) tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX\n",
(unsigned long long)addr,
(unsigned long long)hdr,
(unsigned long long)data,
(unsigned long long)end,
(unsigned long long)cur->Size,
(unsigned long long)cur->RequestedSize,
cur->Free,
KiPoolStateName(cur->Free),
cur->Tag,
cur->Flags,
(unsigned long long)cur->Sequence,
(void*)cur->AllocCaller,
(void*)cur->FreeCaller);
break;
}
if (KiNextBlock(cur) <= cur) {
DbgPrint("MM: pool walk stopped: non-advancing block hdr=0x%016llX size=%llx\n", cur, (unsigned long long)cur->Size);
break;
}
}
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
}
// Implements the debug Describe Address operation.
void MmDebugDescribeAddress(uintptr_t addr) {
PHYSICAL_ADDRESS phys = 0;
@@ -3200,7 +2454,7 @@ void MmDebugDescribeAddress(uintptr_t addr) {
(unsigned long long)flags);
if (addr >= POOL_BASE && addr < POOL_LIMIT) {
KiDebugDescribePoolAddress(addr);
MmPoolDebugDescribeAddress(addr);
return;
}
@@ -3334,12 +2588,6 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
MmFreeUserImageForProcess(process, ptr, size);
}
// Returns kernel Pool Usage.
size_t MmGetKernelPoolUsage(void) {
// Returns bytes committed to the kernel pool.
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
}
// Returns user Heap Usage.
size_t MmGetUserHeapUsage(void) {
size_t total = 0;