maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement proper IRQL, DPCs, kernel and user APCs, alertable waits, async file I/O with cancellation, trap frames, selftests, a real driver/device model, native async FAT32, VFAT dotfile fixes, syscall ABI cleanup, async_apc and async_cancel tests, POSIX multi-volume paths, and pimp the readme.

Browse Source
This commit is contained in:
maksw committed 2026-05-19 20:56:15 +02:00
1 parent f5fd7bdd75
commit 529f816166
75 files changed
+9969 -5675
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/mm_virt.c
+86 -74
View File
@@ -16,17 +16,18 @@
#include <string.h>
#include <mm_virt.h>
#include <ke_spinlock.h>
#include <ke_irql.h>
// ---------------------------------------------------------------------------
// Helpers - physical window access
// ---------------------------------------------------------------------------
static uint64_t gPhysBase = PHYS_MAP_BASE;
static inline void* PhysToVirt(PHYSICAL_ADDRESS p) {
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
return (void *)(p + gPhysBase);
}
static inline uint64_t* PtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)PhysToVirt(tablePhys) + idx;
static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
}
// ---------------------------------------------------------------------------
@@ -43,16 +44,16 @@ static PHYSICAL_ADDRESS gPml4Phys = 0;
// Ensure a page-table entry at 'parentPhys[index]' points to a child table.
// Allocates and zeros a new frame if the entry is not present.
// Returns the physical address of the child table, or 0 on OOM.
static PHYSICAL_ADDRESS EnsureTable(PHYSICAL_ADDRESS parentPhys,
static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
unsigned index, uint64_t extraFlags) {
uint64_t* entry = PtePtr(parentPhys, index);
uint64_t* entry = KiPtePtr(parentPhys, index);
if (*entry & PAGE_PRESENT) {
*entry |= extraFlags;
return* entry & ~0xFFFULL;
}
PHYSICAL_ADDRESS childPhys = MmAllocateFrame();
if (!childPhys) return 0;
memset(PhysToVirt(childPhys), 0, PAGE_SIZE);
memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE);
*entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags;
return childPhys;
}
@@ -63,27 +64,27 @@ static PHYSICAL_ADDRESS EnsureTable(PHYSICAL_ADDRESS parentPhys,
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags) {
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
PHYSICAL_ADDRESS pdpt = EnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
if (!pdpt) return STATUS_NO_MEMORY;
PHYSICAL_ADDRESS pdt = EnsureTable(pdpt, PDPT_IDX(virt), userFlag);
PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
if (!pdt) return STATUS_NO_MEMORY;
PHYSICAL_ADDRESS pt = EnsureTable(pdt, PDT_IDX(virt), userFlag);
PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag);
if (!pt) return STATUS_NO_MEMORY;
uint64_t* pte = PtePtr(pt, PT_IDX(virt));
uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
*pte = phys | flags;
__invlpg((void *)virt);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) {
uint64_t* e4 = PtePtr(gPml4Phys, PML4_IDX(virt));
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e3 = PtePtr(*e4 & ~0xFFFULL, PDPT_IDX(virt));
uint64_t* e3 = KiPtePtr(*e4 & ~0xFFFULL, PDPT_IDX(virt));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e2 = PtePtr(*e3 & ~0xFFFULL, PDT_IDX(virt));
uint64_t* e2 = KiPtePtr(*e3 & ~0xFFFULL, PDT_IDX(virt));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* pte = PtePtr(*e2 & ~0xFFFULL, PT_IDX(virt));
uint64_t* pte = KiPtePtr(*e2 & ~0xFFFULL, PT_IDX(virt));
*pte = 0;
__invlpg((void *)virt);
return STATUS_SUCCESS;
@@ -97,13 +98,13 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint
uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
uint64_t* e4 = PtePtr(gPml4Phys, PML4_IDX(va));
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e3 = PtePtr(*e4 & ~0xFFFULL, PDPT_IDX(va));
uint64_t* e3 = KiPtePtr(*e4 & ~0xFFFULL, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e2 = PtePtr(*e3 & ~0xFFFULL, PDT_IDX(va));
uint64_t* e2 = KiPtePtr(*e3 & ~0xFFFULL, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* pte = PtePtr(*e2 & ~0xFFFULL, PT_IDX(va));
uint64_t* pte = KiPtePtr(*e2 & ~0xFFFULL, PT_IDX(va));
if (!(*pte & PAGE_PRESENT)) return STATUS_NOT_FOUND;
// Preserve the physical address, replace everything else.
@@ -123,10 +124,10 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
uint64_t pd_idx = (virt >> 21) & 0x1FF;
uint64_t pt_idx = (virt >> 12) & 0x1FF;
uint64_t* pml4 = PhysToVirt(gPml4Phys);
uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
if (!(pml4[pml4_idx] & PAGE_PRESENT)) return 0;
uint64_t* pdpt = PhysToVirt(pml4[pml4_idx] & ~0xFFFULL);
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & ~0xFFFULL);
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) return 0;
// Check for 1GB huge pages (bit 7 - PS)
@@ -134,7 +135,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
return (pdpt[pdpt_idx] & 0x000FFFFFC0000000ULL) + (virt & 0x3FFFFFFF);
}
uint64_t* pd = PhysToVirt(pdpt[pdpt_idx] & ~0xFFFULL);
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & ~0xFFFULL);
if (!(pd[pd_idx] & PAGE_PRESENT)) return 0;
// Check for 2MB huge pages (bit 7 - PS)
@@ -142,7 +143,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
return (pd[pd_idx] & 0x000FFFFFFFE00000ULL) + (virt & 0x1FFFFF);
}
uint64_t* pt = PhysToVirt(pd[pd_idx] & ~0xFFFULL);
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & ~0xFFFULL);
if (!(pt[pt_idx] & PAGE_PRESENT)) return 0;
// Extract the physical frame and add the offset within the page
@@ -177,16 +178,16 @@ typedef struct BLOCK_FTR {
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
static inline BLOCK_FTR* HdrToFtr(BLOCK_HDR* h) {
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
}
static inline BLOCK_HDR* FtrToHdr(BLOCK_FTR* f) {
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
}
static inline BLOCK_HDR* NextBlock(BLOCK_HDR* h) {
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
}
static inline BLOCK_FTR* PrevFtr(BLOCK_HDR* h) {
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
}
@@ -200,7 +201,7 @@ typedef struct HEAP {
} HEAP;
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int HeapGrow(HEAP* h, size_t need) {
static int KiHeapGrow(HEAP* h, size_t need) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (h->Brk + grow > h->Limit) return 0;
@@ -223,23 +224,23 @@ static int HeapGrow(HEAP* h, size_t need) {
blk->NextFree = h->FreeList;
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
HdrToFtr(blk)->Free = 1;
HdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Size = blk->Size;
h->Brk += grow;
return 1;
}
// Unlink a free block from the free list.
static void FreeListRemove(HEAP* h, BLOCK_HDR* blk) {
static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
else h->FreeList = blk->NextFree;
if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
}
static void HeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
if (!blk || blk->Free == 0) return;
while (h->Brk > h->Start) {
@@ -264,13 +265,13 @@ static void HeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
}
if (blk->Free) {
HdrToFtr(blk)->Size = blk->Size;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
HdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
}
static void* HeapAlloc(HEAP* h, size_t size) {
static void* KiHeapAlloc(HEAP* h, size_t size) {
if (!size) return (void *)0;
size = (size + 7) & ~7ULL; // 8-byte align
@@ -286,11 +287,11 @@ static void* HeapAlloc(HEAP* h, size_t size) {
}
if (!best) {
if (!HeapGrow(h, size + OVERHEAD)) {
if (!KiHeapGrow(h, size + OVERHEAD)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
// Re-search; HeapGrow inserted at the front.
// Re-search; KiHeapGrow inserted at the front.
best = h->FreeList;
while (best && best->Size < size) best = best->NextFree;
if (!best) {
@@ -299,7 +300,7 @@ static void* HeapAlloc(HEAP* h, size_t size) {
}
}
FreeListRemove(h, best);
KiFreeListRemove(h, best);
// Split the block if the remainder is big enough.
if (best->Size >= size + OVERHEAD + sizeof(void*)) {
@@ -311,24 +312,24 @@ static void* HeapAlloc(HEAP* h, size_t size) {
rem->NextFree = h->FreeList;
if (h->FreeList) h->FreeList->PrevFree = rem;
h->FreeList = rem;
HdrToFtr(rem)->Magic = BLOCK_MAGIC;
HdrToFtr(rem)->Free = 1;
HdrToFtr(rem)->Size = rem->Size;
KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
KiHdrToFtr(rem)->Free = 1;
KiHdrToFtr(rem)->Size = rem->Size;
best->Size = size;
}
best->Free = 0;
HdrToFtr(best)->Free = 0;
HdrToFtr(best)->Magic = BLOCK_MAGIC;
HdrToFtr(best)->Size = best->Size;
KiHdrToFtr(best)->Free = 0;
KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
KiHdrToFtr(best)->Size = best->Size;
memset((char *)best + HDR_SIZE, 0, size);
KeReleaseSpinlock(&h->Lock, flags);
return (char *)best + HDR_SIZE;
}
static void HeapFree(HEAP* h, void* ptr) {
static void KiHeapFree(HEAP* h, void* ptr) {
if (!ptr) return;
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt
@@ -338,25 +339,25 @@ static void HeapFree(HEAP* h, void* ptr) {
blk->Free = 1;
// Coalesce with next block if it is free.
BLOCK_HDR* next = NextBlock(blk);
BLOCK_HDR* next = KiNextBlock(blk);
if ((uintptr_t)next < h->Brk && next->Magic == BLOCK_MAGIC && next->Free) {
FreeListRemove(h, next);
KiFreeListRemove(h, next);
blk->Size += OVERHEAD + next->Size;
HdrToFtr(blk)->Size = blk->Size;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
HdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
// Coalesce with previous block if it is free.
if ((uintptr_t)blk > h->Start) {
BLOCK_FTR* prevFtr = PrevFtr(blk);
BLOCK_FTR* prevFtr = KiPrevFtr(blk);
if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free) {
BLOCK_HDR* prev = FtrToHdr(prevFtr);
FreeListRemove(h, prev);
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
KiFreeListRemove(h, prev);
prev->Size += OVERHEAD + blk->Size;
HdrToFtr(prev)->Size = prev->Size;
HdrToFtr(prev)->Magic = BLOCK_MAGIC;
HdrToFtr(prev)->Free = 1;
KiHdrToFtr(prev)->Size = prev->Size;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = 1;
blk = prev;
}
}
@@ -367,23 +368,23 @@ static void HeapFree(HEAP* h, void* ptr) {
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
HdrToFtr(blk)->Free = 1;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
HeapTryDecommitTail(h, blk);
KiHeapTryDecommitTail(h, blk);
KeReleaseSpinlock(&h->Lock, flags);
}
static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
// If ptr is NULL, realloc acts like malloc
if (!ptr) {
return HeapAlloc(h, new_size);
return KiHeapAlloc(h, new_size);
}
// If size is 0, realloc acts like free
if (new_size == 0) {
HeapFree(h, ptr);
KiHeapFree(h, ptr);
return (void*)0;
}
@@ -395,18 +396,18 @@ static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
return (void*)0;
}
// Align the requested size (matches HeapAlloc 8-byte alignment)
// Align the requested size (matches KiHeapAlloc 8-byte alignment)
size_t aligned_size = (new_size + 7) & ~7ULL;
// If the current block is already big enough, we can reuse it in-place.
// (Note: To save space, you could optionally split the block here if
// the new size is significantly smaller, similar to HeapAlloc's logic).
// the new size is significantly smaller, similar to KiHeapAlloc's logic).
if (blk->Size >= aligned_size) {
return ptr;
}
// The block is too small. Allocate a new, larger block.
void* new_ptr = HeapAlloc(h, aligned_size);
void* new_ptr = KiHeapAlloc(h, aligned_size);
if (!new_ptr) {
return (void*)0; // Out of memory, original block is left untouched
}
@@ -417,7 +418,7 @@ static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
memcpy(new_ptr, ptr, blk->Size);
// Free the old block
HeapFree(h, ptr);
KiHeapFree(h, ptr);
return new_ptr;
}
@@ -449,7 +450,7 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
// Pre-commit the requested pool size.
if (poolSize) {
poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
HeapGrow(&gPoolHeap, poolSize);
KiHeapGrow(&gPoolHeap, poolSize);
}
// User heap.
@@ -468,16 +469,27 @@ void MmActivate(void) {
// ---------------------------------------------------------------------------
// Public pool API
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocatePool(size_t size) { return HeapAlloc(&gPoolHeap, size); }
__declspec(dllexport) void MmFreePool(void* ptr) { HeapFree(&gPoolHeap, ptr); }
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return HeapRealloc(&gPoolHeap, ptr, newSize); }
__declspec(dllexport) void* MmAllocatePool(size_t size) { return KiHeapAlloc(&gPoolHeap, size); }
__declspec(dllexport) void MmFreePool(void* ptr) { KiHeapFree(&gPoolHeap, ptr); }
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); }
// ---------------------------------------------------------------------------
// Public user heap API
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) { return HeapAlloc(&gUserHeap, size); }
__declspec(dllexport) void MmFreeUserMemory(void* ptr, size_t size) { (void)size; HeapFree(&gUserHeap, ptr); }
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { return HeapRealloc(&gUserHeap, ptr, newSize); }
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapAlloc(&gUserHeap, size);
}
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
KeAssertIrql(PASSIVE_LEVEL);
KiHeapFree(&gUserHeap, ptr);
}
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapRealloc(&gUserHeap, ptr, newSize);
}
size_t MmGetKernelPoolUsage(void) {
// Returns bytes committed to the kernel pool