add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

.gitignore

@@ -2,6 +2,9 @@ build.obj
 bin/
 obj/
 qemu.log
-*.vhd
+disk.vhd
+usb.vhd
+disk.vmdk
+code.zip
 code/build/build.exe
 code/build/build.exe.old

code/boot.ini

@@ -1,5 +1,5 @@
 [kernel]
 ; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file
-debug = 1
+debug = 0xffff
 [init]
 commandline = apps/shell_native.exe

code/boot/uefi/main.c

@@ -291,7 +291,7 @@ static int ParseBootIniInteger(const char* val) {
 static void BootIniCb(const char* section, const char* key,
                       const char* val, void* ud) {
     PARSE_STATE* s = (PARSE_STATE *)ud;
-    // [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log.
+    // [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log, 8+=kernel diagnostics.
     if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug"))
         s->debug = ParseBootIniInteger(val);
     // [init] commandline = hello.exe.
@@ -446,7 +446,7 @@ EFI_STATUS LoadPeImage(UINT8* rawFile, UINT64 virtDest, EFI_PHYSICAL_ADDRESS* ou
     
     IMAGE_DATA_DIRECTORY* dataDirs = (IMAGE_DATA_DIRECTORY*)(ntHeaders + 24 + 112);
     IMAGE_DATA_DIRECTORY* relocDir = &dataDirs[5]; 
-    //IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1];.
+    //IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1];
 
     // Allocate Physical Memory for the mapped image.
     UINTN pages = (sizeOfImage + PAGE_SIZE - 1) / PAGE_SIZE;
@@ -619,7 +619,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         return Status;
     }
 
-    //Print(L"1\n");.
+    Print(L"1\n");
 
     // boot.ini controls debugger and runtime policy before the kernel has a filesystem service.
     UINTN sz;
@@ -632,7 +632,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         FreePool(bootIniRaw);
     }
 
-    //Print(L"2\n");.
+    Print(L"2\n");
 
     // drivers.ini decides which boot and runtime drivers are loaded into the initial module list.
     UINT8* drvIniRaw = LoadFile(L"drivers\\drivers.ini", &sz);
@@ -643,7 +643,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         FreePool(drvIniRaw);
     }
 
-    //Print(L"3\n");.
+    Print(L"3\n");
 
     BOOT_INFO* bi = AllocateZeroPool(sizeof(BOOT_INFO));
     bi->Magic = BOOT_INFO_MAGIC;
@@ -675,7 +675,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
     // Current mapping cursor for upper half.
     UINT64 virtCursor = KERNEL_VIRT_BASE;
 
-    //Print(L"4\n");.
+    Print(L"4\n");
 
     // Core images are loaded first because driver import resolution depends on them.
     typedef struct { 
@@ -742,7 +742,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         while(true);
     }
 
-    //Print(L"5\n");.
+    Print(L"5\n");
 
     // Load driver .sys files into raw buffers.
     for (int i = 0; i < ps.driverCount; i++) {
@@ -752,8 +752,9 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         UINTN imgSz;
         UINT8* buf = LoadFile(path, &imgSz);
         if (!buf) {
-            Print(L"Driver not found: %s\n", ps.drivers[i]); 
+            Print(L"Fatal: boot driver not found: %s\n", ps.drivers[i]);
-            continue; 
+            while (true);
+            return EFI_NOT_FOUND;
         }
 
         // Store file info for the kernel to process later.
@@ -767,7 +768,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         mod->VirtBase  = 0;              // Kernel will map this later
     }
 
-    //Print(L"6\n");.
+    Print(L"6\n");
 
     // Query GOP FIRST, before the GetMemoryMap loop.
     EFI_GRAPHICS_OUTPUT_PROTOCOL* gop;
@@ -795,7 +796,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         }
     }
 
-    //Print(L"7\n");.
+    Print(L"7\n");
 
     bi->ApTrampolinePhys = (uintptr_t)AllocateApTrampolinePageBelow1M();
 
@@ -895,7 +896,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
         Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // Physical Window Map, P|W|PCD
     }
 
-    //Print(L"8\n");.
+    Print(L"8\n");
 
     // The kernel consumes a compact memory map so it does not need EFI descriptor semantics after boot.
     ConvertMemoryMap(&bi->MemoryMap);
@@ -940,12 +941,12 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
     }
 
     // Print it out to verify the bootloader actually found it before launching the kernel
-    //Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);.
+    Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);
 
     bi->PhysicalMapBase = PHYS_MAP_BASE;
     bi->Debug           = (UINT32)ps.debug;
 
-    //Print(L"9\n");.
+    Print(L"9\n");
 
     // ExitBootServices is the point of no return for firmware allocation, file I/O, and protocol access.
     gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer);

code/build/build.c

@@ -30,6 +30,7 @@ static const Driver_Target PCI_BUS_DRIVER =
 
 static const Driver_Target STANDALONE_DRIVERS[] = {
     DRIVER_TARGET("ps2",    "obj\\drivers\\ps2",    "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"),
+    DRIVER_TARGET("serial", "obj\\drivers\\serial", "code\\drivers\\input\\serial\\serial.c"),
     DRIVER_TARGET_LIBS("ata",    "obj\\drivers\\ata",    "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"),
     DRIVER_TARGET_LIBS("ahci",   "obj\\drivers\\ahci",   "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"),
     DRIVER_TARGET("fat32",  "obj\\drivers\\fat32",  "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
@@ -46,7 +47,10 @@ void create_layout() {
         "obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init",
         "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
         "obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
-        "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video"
+        "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop",
+        "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
+        "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
+        "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present"
     };
     for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) nob_mkdir_if_not_exists(dirs[i]);
     nob_mkdir_if_not_exists(PCI_BUS_DRIVER.obj_dir);
@@ -71,6 +75,41 @@ Nob_Proc run_async(const char *cmd_str) {
     return proc;
 }
 
+// Deletes stale object files from a target directory before compiling into it.
+// Link commands use directory wildcards, so removed translation units can otherwise
+// survive as old .obj files and keep unresolved references alive.
+bool clean_object_directory(const char *obj_dir) {
+    const char *cmd = nob_temp_sprintf("if exist %s\\*.obj del /q %s\\*.obj", obj_dir, obj_dir);
+    return run_sync(cmd);
+}
+
+// Cleans object directories that are linked through wildcards.
+// The build is intentionally simple and does not maintain per-file dependency state,
+// so stale objects are more dangerous than recompiling a few extra files.
+bool clean_build_object_directories(void) {
+    const char *dirs[] = {
+        "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
+        "obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix",
+        "obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
+        "obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
+        "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping",
+        "obj\\doom", "obj\\video", "obj\\memtop", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
+        "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
+        "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present"
+    };
+
+    for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) {
+        if (!clean_object_directory(dirs[i])) return false;
+    }
+
+    if (!clean_object_directory(PCI_BUS_DRIVER.obj_dir)) return false;
+    for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) {
+        if (!clean_object_directory(STANDALONE_DRIVERS[i].obj_dir)) return false;
+    }
+
+    return true;
+}
+
 // Creates an empty USB image the first time the build sees a missing usb.vhd.
 // The image is intentionally left unmounted and unpopulated after formatting so
 // callers can use it as removable FAT32 storage without the build overwriting it.
@@ -111,6 +150,7 @@ int main(int argc, char **argv) {
     }
     if (!create_usb_vhd_if_missing()) return 1;
     create_layout();
+    if (!clean_build_object_directories()) return 1;
 
     // Core images build sequentially because later images link against earlier import libraries.
     nob_log(NOB_INFO, "--- Building Core Subsystems ---");
@@ -196,6 +236,17 @@ int main(int argc, char **argv) {
         {"apps\\shell_native", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "shell_native", "/ENTRY:main"},
         {"tests\\async_apc", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_apc", "/ENTRY:main"},
         {"tests\\async_cancel", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_cancel", "/ENTRY:main"},
+        {"tests\\mmap_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "mmap_smoke", "/ENTRY:main"},
+        {"tests\\pcache_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_smoke", "/ENTRY:main"},
+        {"tests\\pcache_pressure", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_pressure", "/ENTRY:main"},
+        {"tests\\dirty_writeback", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "dirty_writeback", "/ENTRY:main"},
+        {"tests\\shared_mmap", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "shared_mmap", "/ENTRY:main"},
+        {"tests\\pcache_invalidate", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_invalidate", "/ENTRY:main"},
+        {"tests\\pcache_mmap_read_reuse", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_mmap_read_reuse", "/ENTRY:main"},
+        {"tests\\pcache_multithread", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_multithread", "/ENTRY:main"},
+        {"tests\\surface_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_smoke", "/ENTRY:main"},
+        {"tests\\surface_invalid_rect", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_invalid_rect", "/ENTRY:main"},
+        {"tests\\surface_multithread_present", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_multithread_present", "/ENTRY:main"},
         {"apps\\edit", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "edit", "/ENTRY:main"},
         {"apps\\httpget", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "httpget", "/ENTRY:main"},
         {"apps\\tcpserve", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "tcpserve", "/ENTRY:main"},
@@ -203,7 +254,8 @@ int main(int argc, char **argv) {
         {"apps\\resolve", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "resolve", "/ENTRY:main"},
         {"apps\\ping", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "ping", "/ENTRY:main"},
         {"apps\\doom", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "doom", "/ENTRY:main"},
-        {"apps\\video", "bin\\win32.lib obj\\libc.obj code\\user\\apps\\video\\bink2w64.lib", "video", "/ENTRY:main"}
+        {"apps\\video", "bin\\syscall.lib obj\\libc.obj code\\user\\apps\\video\\bink2w64.lib", "video", "/ENTRY:main"},
+        {"apps\\memtop", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "memtop", "/ENTRY:main"}
     };
 
     for (size_t i = 0; i < NOB_ARRAY_LEN(apps); i++) {
@@ -248,6 +300,7 @@ int main(int argc, char **argv) {
         {"code\\settings.ini", "Z:\\system\\settings.ini"}, {"bin\\init.exe", "Z:\\system\\init.exe"},
         
         {"bin\\pci.sys", "Z:\\drivers\\pci.sys"},           {"bin\\ps2.sys", "Z:\\drivers\\ps2.sys"},
+        {"bin\\serial.sys", "Z:\\drivers\\serial.sys"},
         {"bin\\ata.sys", "Z:\\drivers\\ata.sys"},           {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"},
         {"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"},       {"bin\\vmwaresvga.sys", "Z:\\drivers\\vmwaresvga.sys"},
         {"bin\\lwip.sys", "Z:\\drivers\\lwip.sys"},          {"bin\\usb.sys", "Z:\\drivers\\usb.sys"},
@@ -260,9 +313,19 @@ int main(int argc, char **argv) {
         {"bin\\httpget.exe", "Z:\\apps\\httpget.exe"},         {"bin\\tcpserve.exe", "Z:\\apps\\tcpserve.exe"},
         {"bin\\udp.exe", "Z:\\apps\\udp.exe"},                 {"bin\\resolve.exe", "Z:\\apps\\resolve.exe"},
         {"bin\\ping.exe", "Z:\\apps\\ping.exe"},               {"bin\\doom.exe", "Z:\\apps\\doom.exe"},
-        {"bin\\video.exe", "Z:\\apps\\video.exe"},
+        {"bin\\video.exe", "Z:\\apps\\video.exe"},             {"bin\\memtop.exe", "Z:\\apps\\memtop.exe"},
 
-        {"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"},     {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"}
+        {"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"},     {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"},
+        {"bin\\mmap_smoke.exe", "Y:\\apps\\mmap_smoke.exe"}, {"bin\\pcache_smoke.exe", "Y:\\apps\\pcache_smoke.exe"},
+        {"bin\\pcache_pressure.exe", "Y:\\apps\\pcache_pressure.exe"},
+        {"bin\\dirty_writeback.exe", "Y:\\apps\\dirty_writeback.exe"},
+        {"bin\\shared_mmap.exe", "Y:\\apps\\shared_mmap.exe"},
+        {"bin\\pcache_invalidate.exe", "Y:\\apps\\pcache_invalidate.exe"},
+        {"bin\\pcache_mmap_read_reuse.exe", "Y:\\apps\\pcache_mmap_read_reuse.exe"},
+        {"bin\\pcache_multithread.exe", "Y:\\apps\\pcache_multithread.exe"},
+        {"bin\\surface_smoke.exe", "Y:\\apps\\surface_smoke.exe"},
+        {"bin\\surface_invalid_rect.exe", "Y:\\apps\\surface_invalid_rect.exe"},
+        {"bin\\surface_multithread_present.exe", "Y:\\apps\\surface_multithread_present.exe"}
     };
 
     for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) nob_copy_file(copies[i][0], copies[i][1]);

code/drivers/bus/pci/pci.c

@@ -13,6 +13,7 @@
 #include <kernel/mm_virt.h>
 #include <kernel/ke_debug.h>
 #include <kernel/ke_irp.h>
+#include <kernel/ke_irq.h>
 #include <kernel/hal/hal_apic.h>
 
 int _fltused; // MSVC floating-point helper anchor required by printf.obj
@@ -540,7 +541,7 @@ __declspec(dllexport) KSTATUS PciMapBar(DEVICE_OBJECT* pdo, uint8_t index, uint6
         return st;
 
     for (uint64_t off = 0; off < mapLength; off += PAGE_SIZE) {
-        st = MmMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmMapFailIfPresent);
+        st = MmPteMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmPteMapFailIfPresent);
         if (!KSUCCESS(st))
             return st;
     }
@@ -659,6 +660,28 @@ __declspec(dllexport) KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot)
     return STATUS_SUCCESS;
 }
 
+// Enables a single MSI vector, allocating the nt-lite IRQ slot internally.
+__declspec(dllexport) KSTATUS PciEnableMsiAuto(DEVICE_OBJECT* pdo, uint8_t* outIrqSlot) {
+    PCI_PDO_EXTENSION* ext;
+    if (!outIrqSlot || !PciIsPdo(pdo, &ext))
+        return STATUS_INVALID_PARAMETER;
+
+    if (!PciFindCapability(ext, PCI_CAP_ID_MSI))
+        return STATUS_NOT_SUPPORTED;
+
+    uint8_t irq = 0xFFu;
+    KSTATUS st = KeIrqAllocateMsiVector(&irq);
+    if (!KSUCCESS(st))
+        return st;
+
+    st = PciEnableMsi(pdo, irq);
+    if (!KSUCCESS(st))
+        return st;
+
+    *outIrqSlot = irq;
+    return STATUS_SUCCESS;
+}
+
 // Implements the driver Entry operation.
 KSTATUS DriverEntry(const HAL_API* hal) {
     if (!hal) return STATUS_INVALID_PARAMETER;

code/drivers/bus/usb/usb.h

@@ -21,6 +21,7 @@
 #include <kernel/ke_semaphore.h>
 #include <kernel/ke_thread.h>
 #include <kernel/mm_phys.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/mm_virt.h>
 
 #ifndef USB_DEBUG_VERBOSE

code/drivers/bus/usb/usb_xhci.c

@@ -95,9 +95,9 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
     // hid this because it had simple low memory; laptop firmware does not.
     PHYSICAL_ADDRESS phys = 0;
     if (pages == 1) {
-        phys = MmAllocateFrame();
+        phys = MmPfnAllocate(MmPfnUseDma);
     } else {
-        phys = MmAllocateContiguous(pages);
+        phys = MmPfnAllocateContiguous(pages, MmPfnUseDma);
     }
 
     if (!phys) {
@@ -1930,21 +1930,31 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
         return STATUS_DEVICE_ERROR;
     }
 
-    st = PciGetInterrupt(xhci->Pdo, &xhci->Irq);
+    xhci->Irq = 0xFFu;
-    if (!KSUCCESS(st)) {
+    xhci->UsingMsi = false;
-        DbgPrint("usb: xHCI PciGetInterrupt failed st=0x%x\n", st);
+
-        return st;
+    st = PciEnableMsiAuto(xhci->Pdo, &xhci->Irq);
+    if (KSUCCESS(st)) {
+        xhci->UsingMsi = true;
+        DbgPrint("usb: xHCI using MSI IRQ slot %u\n", xhci->Irq);
+    } else {
+        KSTATUS intxStatus = PciGetInterrupt(xhci->Pdo, &xhci->Irq);
+        if (KSUCCESS(intxStatus) && xhci->Irq != 0xFFu) {
+            DbgPrint("usb: xHCI using legacy INTx IRQ %u\n", xhci->Irq);
+        } else {
+            xhci->Irq = 0xFFu;
+            DbgPrint("usb: xHCI no MSI/INTx interrupt available; using polled event-ring mode (msi=0x%x intx=0x%x)\n",
+                     st, intxStatus);
         }
-    if (xhci->Irq == 0xFFu) {
-        DbgPrint("usb: xHCI PCI %02x:%02x.%x has no INTx IRQ; MSI is not in pci.sys yet\n",
-                 xhci->Bus, xhci->Dev, xhci->Fn);
-        return STATUS_NOT_SUPPORTED;
     }
 
-    st = PciEnableDevice(xhci->Pdo,
+    uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER;
-                         PCI_ENABLE_MEMORY |
+    if (xhci->UsingMsi || xhci->Irq == 0xFFu)
-                         PCI_ENABLE_BUSMASTER |
+        enableFlags |= PCI_DISABLE_INTX;
-                         PCI_ENABLE_INTX);
+    else
+        enableFlags |= PCI_ENABLE_INTX;
+
+    st = PciEnableDevice(xhci->Pdo, enableFlags);
     if (!KSUCCESS(st)) {
         DbgPrint("usb: xHCI PciEnableDevice failed st=0x%x\n", st);
         return st;
@@ -2056,12 +2066,18 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
         return st;
     }
 
+    if (xhci->Irq != 0xFFu) {
         KeIrqRegister(xhci->Irq, XhciIsr);
         XhciEnableInterrupts(xhci);
         xhci->InterruptsEnabled = true;
-    xhci->Fdo->PnpState = DevicePnpStarted;
         KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
-    DbgPrint("usb: xHCI ISR registered on IRQ %u; event ring handled by DPC\n", xhci->Irq);
+        DbgPrint("usb: xHCI ISR registered on IRQ %u (%s); event ring handled by DPC\n",
+                 xhci->Irq, xhci->UsingMsi ? "MSI" : "INTx");
+    } else {
+        xhci->InterruptsEnabled = false;
+        DbgPrint("usb: xHCI started in polling mode; synchronous waits drain event ring\n");
+    }
+    xhci->Fdo->PnpState = DevicePnpStarted;
 
     st = UsbHubScheduleRootScan(&xhci->Hcd);
     if (!KSUCCESS(st))

code/drivers/display/vmwaresvga/vmwaresvga.c

@@ -890,19 +890,13 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT*
             result = syncSt;
     }
 
-    if (v->HardwareCursor && v->CursorVisible)
-        VmwSvgaCursorCommit(v);
-
     return result;
 }
 
 // Implements the display Sync operation.
 static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) {
     VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
-    KSTATUS st = VmwSvgaSyncDevice(v);
+    return VmwSvgaSyncDevice(v);
-    if (v && v->HardwareCursor && v->CursorVisible)
-        VmwSvgaCursorCommit(v);
-    return st;
 }
 
 // Implements the display Set Cursor Image operation.

code/drivers/drivers.ini

@@ -1,6 +1,7 @@
 [boot]
 pci.sys
 ps2.sys
+serial.sys
 ahci.sys
 ata.sys
 usb.sys

code/drivers/fs/fat32/fat32.c

@@ -128,12 +128,79 @@ typedef struct {
 
 static FAT_CACHE_ENTRY gFatCache[FAT_CACHE_SLOTS];
 
+#define FAT32_PATH_CACHE_SLOTS 64u
+#define FAT32_PATH_CACHE_MAX   512u
+#define FAT32_CHAIN_CACHE_SLOTS 16u
+#define FAT32_WRITE_RUN_MAX_SECTORS 128u
+#define FAT32_READ_RUN_MAX_SECTORS  512u
+
+typedef struct FAT32_PATH_CACHE_ENTRY {
+    bool     Valid;
+    uint64_t Generation;
+    uint32_t Hash;
+    size_t   PathLength;
+    char     Path[FAT32_PATH_CACHE_MAX];
+    uint32_t Cluster;
+    uint32_t Size;
+    uint8_t  Attr;
+    uint64_t DirentLba;
+    uint32_t DirentOffset;
+} FAT32_PATH_CACHE_ENTRY;
+
+typedef struct FAT32_CHAIN_CACHE_ENTRY {
+    bool     Valid;
+    uint32_t LogicalStart;
+    uint32_t PhysicalStart;
+    uint32_t Length;
+    uint32_t NextCluster;
+} FAT32_CHAIN_CACHE_ENTRY;
+
+// The FAT sector cache is global and direct-mapped.  mmap-backed video can fault
+// file pages from multiple Bink worker threads at the same time; without
+// serialization one thread can evict/reload a cache slot while another thread is
+// still reading a FAT entry from that slot, corrupting cluster-chain traversal.
+// Use a sleepable mutex rather than a spinlock because cache misses perform disk
+// I/O.  The static mutex avoids allocating an object from the paging path.
+static MUTEX    gFatCacheMutex;
+static SPINLOCK gFatCacheMutexInitLock = SPINLOCK_INIT;
+static int      gFatCacheMutexReady;
+
+static void FatCacheEnsureMutex(void) {
+    if (gFatCacheMutexReady)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&gFatCacheMutexInitLock);
+    if (!gFatCacheMutexReady) {
+        memset(&gFatCacheMutex, 0, sizeof(gFatCacheMutex));
+        gFatCacheMutex.Header.Type = ObjectTypeMutex;
+        gFatCacheMutex.Header.RefCount = 1;
+        gFatCacheMutex.Header.Destroy = (void*)0;
+        gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
+        gFatCacheMutex.Owner = (struct THREAD*)0;
+        LIST_INIT(&gFatCacheMutex.WaitList);
+        gFatCacheMutexReady = 1;
+    }
+    KeReleaseSpinlock(&gFatCacheMutexInitLock, flags);
+}
+
+static int FatCacheAcquire(void) {
+    if (!KeCanWait())
+        return 0;
+    FatCacheEnsureMutex();
+    return KSUCCESS(KeAcquireMutex(&gFatCacheMutex));
+}
+
+static void FatCacheRelease(int held) {
+    if (held)
+        KeReleaseMutex(&gFatCacheMutex);
+}
+
 // Raw (non-cached) sector I/O — forward declarations.
 static KSTATUS ReadSectorsRaw (FAT32_VOL *v, uint64_t lba, uint32_t count, void       *buf);
 static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, const void *buf);
 
 // Read one FAT sector through the cache.
-// Returns a pointer into the cache buffer (valid until the next eviction).
+// Caller must hold the FAT cache mutex while using the returned pointer.
 static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) {
     uint32_t        slot = (uint32_t)(lba % FAT_CACHE_SLOTS);
     FAT_CACHE_ENTRY *e   = &gFatCache[slot];
@@ -170,11 +237,13 @@ static void FatCacheMarkDirty(FAT32_VOL *v, uint64_t lba) {
 
 // Implements the fat Cache Invalidate operation.
 static void FatCacheInvalidate(void) {
+    int held = FatCacheAcquire();
     for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
         gFatCache[i].vol = (FAT32_VOL*)0;
         gFatCache[i].valid = false;
         gFatCache[i].dirty = false;
     }
+    FatCacheRelease(held);
 }
 
 // FlushFatCache — write every dirty slot to all FAT copies, clear dirty flag.
@@ -196,6 +265,10 @@ struct FAT32_VOL_S {
     uint64_t   DataStart;       // absolute LBA of cluster 2
     uint32_t   TotalClusters;   // usable data clusters
     uint32_t   NextFreeHint;    // fix #3: start cluster for next free-scan
+    SPINLOCK   PathCacheLock;
+    uint64_t   PathCacheGeneration;
+    IO_FILESYSTEM_STATS Stats;
+    FAT32_PATH_CACHE_ENTRY PathCache[FAT32_PATH_CACHE_SLOTS];
 };
 typedef struct {
     uint32_t   StartCluster;
@@ -203,6 +276,9 @@ typedef struct {
     FAT32_VOL *Vol;
     uint64_t   DirentLba;
     uint32_t   DirentOffset;
+    SPINLOCK   ChainCacheLock;
+    uint32_t   ChainCacheClock;
+    FAT32_CHAIN_CACHE_ENTRY ChainCache[FAT32_CHAIN_CACHE_SLOTS];
 } FAT32_FILE;
 
 // Raw sector I/O (bypasses FAT cache; used for file data and cache eviction).
@@ -231,6 +307,7 @@ static uint64_t ClusterToLba(FAT32_VOL *v, uint32_t cluster) {
 // FlushFatCache — needs full FAT32_VOL definition.
 static void FlushFatCache(FAT32_VOL *v) {
     uint64_t fat0Start = (uint64_t)v->PartitionLba + v->ReservedSectors;
+    int held = FatCacheAcquire();
 
     for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
         FAT_CACHE_ENTRY *e = &gFatCache[i];
@@ -246,6 +323,8 @@ static void FlushFatCache(FAT32_VOL *v) {
         }
         e->dirty = false;
     }
+
+    FatCacheRelease(held);
 }
 
 // FAT read / write  (through cache).
@@ -256,9 +335,14 @@ static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) {
                        + fatOffset / v->BytesPerSector;
     uint32_t offInSec  = fatOffset % v->BytesPerSector;
 
+    int held = FatCacheAcquire();
     uint8_t *sec;
-    if (!KSUCCESS(FatCacheRead(v, sector, &sec))) return 0;
+    KSTATUS s = FatCacheRead(v, sector, &sec);
-    return *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
+    uint32_t value = 0;
+    if (KSUCCESS(s))
+        value = *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
+    FatCacheRelease(held);
+    return value;
 }
 
 // Updates fat Entry.
@@ -268,19 +352,24 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) {
                        + fatOffset / v->BytesPerSector;
     uint32_t offInSec  = fatOffset % v->BytesPerSector;
 
+    int held = FatCacheAcquire();
     uint8_t *sec;
     KSTATUS s = FatCacheRead(v, sector, &sec);
-    if (!KSUCCESS(s)) return s;
+    if (!KSUCCESS(s)) {
+        FatCacheRelease(held);
+        return s;
+    }
 
     uint32_t existing = *(uint32_t *)(sec + offInSec);
     *(uint32_t *)(sec + offInSec) = (existing & 0xF0000000u)
                                   | (value    & 0x0FFFFFFFu);
     FatCacheMarkDirty(v, sector);
+    FatCacheRelease(held);
     return STATUS_SUCCESS;
 }
 
 // Cluster allocation  (fix #3: next-free hint + two-phase wrap-around scan).
-static uint32_t AllocCluster(FAT32_VOL *v) {
+static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) {
     if (v->TotalClusters == 0) return 0;
 
     uint32_t total = v->TotalClusters;
@@ -295,13 +384,19 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
                 if (!KSUCCESS(WriteFatEntry(v, c, FAT32_EOC_MARK))) return 0;
                 v->NextFreeHint = c + 1;
 
-                // Zero the cluster (required for directory correctness).
+                // Normal file growth zeroes newly allocated clusters so old
+                // disk contents are never exposed.  Swap/pagefile creation is
+                // the one exception: every allocated slot is overwritten before
+                // it can ever be read back, so skipping these writes avoids a
+                // slow boot-time full-pagefile zero pass.
+                if (zeroCluster) {
                     uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
                     if (zero) {
                         memset(zero, 0, v->BytesPerCluster);
                         WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero);
                         MmFreePool(zero);
                     }
+                }
                 return c;
             }
         }
@@ -309,14 +404,23 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
     return 0;
 }
 
-// Appends cluster.
+
-static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
+static uint32_t AllocCluster(FAT32_VOL *v) {
-    uint32_t newCluster = AllocCluster(v);
+    return AllocClusterEx(v, 1);
+}
+
+static uint32_t AppendClusterEx(FAT32_VOL *v, uint32_t lastCluster, int zeroCluster) {
+    uint32_t newCluster = AllocClusterEx(v, zeroCluster);
     if (!newCluster) return 0;
     if (!KSUCCESS(WriteFatEntry(v, lastCluster, newCluster))) return 0;
     return newCluster;
 }
 
+// Appends cluster.
+static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
+    return AppendClusterEx(v, lastCluster, 1);
+}
+
 // Returns chain End.
 static uint32_t GetChainEnd(FAT32_VOL *v, uint32_t start, uint32_t *outCount) {
     uint32_t cluster = start, prev = start, count = 0;
@@ -973,6 +1077,302 @@ static int MatchName83V(const char *dirName, const char *dirExt,
     return 1;
 }
 
+// Returns the path cache canonical byte used for hashes and comparisons.
+static char Fat32PathCacheChar(char c) {
+    if (c == '/') return '\\';
+    if (c >= 'A' && c <= 'Z') return (char)(c + 32);
+    return c;
+}
+
+// Hashes a drive-relative path using the same case/separator rules as lookup.
+static uint32_t Fat32PathCacheHash(SYS_STRING_VIEW path) {
+    uint32_t h = 2166136261u;
+    for (size_t i = 0; i < path.Length; i++) {
+        h ^= (uint8_t)Fat32PathCacheChar(path.Data[i]);
+        h *= 16777619u;
+    }
+    return h ? h : 1u;
+}
+
+// Tests whether the cached path names the same FAT path as the caller's view.
+static bool Fat32PathCachePathEquals(const FAT32_PATH_CACHE_ENTRY* e, SYS_STRING_VIEW path) {
+    if (!e || e->PathLength != path.Length) return false;
+    for (size_t i = 0; i < path.Length; i++) {
+        if (Fat32PathCacheChar(e->Path[i]) != Fat32PathCacheChar(path.Data[i]))
+            return false;
+    }
+    return true;
+}
+
+// Adds directory search counters without holding the lock during disk I/O.
+static void Fat32PathCacheAddDirectoryStats(FAT32_VOL* v, uint64_t clusters, uint64_t entries) {
+    if (!v) return;
+    uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
+    v->Stats.DirectorySearches++;
+    v->Stats.DirectoryClustersRead += clusters;
+    v->Stats.DirectoryEntriesScanned += entries;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+// Looks up a positive path-cache entry for a full drive-relative path.
+static bool Fat32PathCacheLookup(FAT32_VOL* v, SYS_STRING_VIEW path,
+                                 uint32_t* outCluster, uint32_t* outSize,
+                                 uint8_t* outAttr, uint64_t* outDirentLba,
+                                 uint32_t* outDirentOffset) {
+    if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
+        return false;
+
+    uint32_t hash = Fat32PathCacheHash(path);
+    uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
+    bool hit = false;
+
+    uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
+    v->Stats.PathLookups++;
+    FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
+    if (e->Valid && e->Generation == v->PathCacheGeneration &&
+        e->Hash == hash && Fat32PathCachePathEquals(e, path)) {
+        if (outCluster) *outCluster = e->Cluster;
+        if (outSize) *outSize = e->Size;
+        if (outAttr) *outAttr = e->Attr;
+        if (outDirentLba) *outDirentLba = e->DirentLba;
+        if (outDirentOffset) *outDirentOffset = e->DirentOffset;
+        v->Stats.PathCacheHits++;
+        hit = true;
+    } else {
+        v->Stats.PathCacheMisses++;
+    }
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+    return hit;
+}
+
+// Stores a positive path lookup in the volume-local path cache.
+static void Fat32PathCacheStore(FAT32_VOL* v, SYS_STRING_VIEW path,
+                                uint32_t cluster, uint32_t size, uint8_t attr,
+                                uint64_t direntLba, uint32_t direntOffset) {
+    if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
+        return;
+
+    uint32_t hash = Fat32PathCacheHash(path);
+    uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
+
+    uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
+    FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
+    memset(e, 0, sizeof(*e));
+    e->Valid = true;
+    e->Generation = v->PathCacheGeneration;
+    e->Hash = hash;
+    e->PathLength = path.Length;
+    memcpy(e->Path, path.Data, path.Length);
+    e->Path[path.Length] = '\0';
+    e->Cluster = cluster;
+    e->Size = size;
+    e->Attr = attr;
+    e->DirentLba = direntLba;
+    e->DirentOffset = direntOffset;
+    v->Stats.PathCacheStores++;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+// Invalidates every positive path-cache entry after a directory or size mutation.
+static void Fat32PathCacheInvalidate(FAT32_VOL* v) {
+    if (!v) return;
+    uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
+    v->PathCacheGeneration++;
+    if (v->PathCacheGeneration == 0)
+        v->PathCacheGeneration = 1;
+    v->Stats.PathCacheInvalidations++;
+    v->Stats.PathCacheGeneration = v->PathCacheGeneration;
+    for (uint32_t i = 0; i < FAT32_PATH_CACHE_SLOTS; i++)
+        v->PathCache[i].Valid = false;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+// Returns the current filesystem cache and traversal counters.
+void Fat32QueryStats(IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out) {
+    if (!fs || !out) return;
+    FAT32_VOL* v = (FAT32_VOL*)fs->DriverCtx;
+    if (!v) { memset(out, 0, sizeof(*out)); return; }
+
+    uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
+    *out = v->Stats;
+    out->PathCacheGeneration = v->PathCacheGeneration;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+
+// Adds FAT-chain traversal counters shared by the read path helpers.
+static void Fat32StatsAddChainWalk(FAT32_VOL* v, uint64_t steps) {
+    if (!v || !steps) return;
+    uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
+    v->Stats.FatChainWalkSteps += steps;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+// Records one data read span and its physically contiguous cluster merges.
+static void Fat32StatsAddReadRun(FAT32_VOL* v, uint32_t sectors, uint32_t merges) {
+    if (!v || !sectors) return;
+    uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
+    v->Stats.DataReadRuns++;
+    v->Stats.DataReadSectors += sectors;
+    v->Stats.DataReadRunMerges += merges;
+    if (v->Stats.MaxDataReadSpanSectors < sectors)
+        v->Stats.MaxDataReadSpanSectors = sectors;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+// Initializes the per-open cluster-chain cache for a FAT file object.
+static void Fat32FileInitChainCache(FAT32_FILE* f) {
+    if (!f) return;
+    f->ChainCacheLock = (SPINLOCK)SPINLOCK_INIT;
+    f->ChainCacheClock = 0;
+    memset(f->ChainCache, 0, sizeof(f->ChainCache));
+}
+
+// Invalidates cached FAT extents after the file's cluster chain changes.
+static void Fat32FileInvalidateChainCache(FAT32_FILE* f) {
+    if (!f) return;
+    uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
+    memset(f->ChainCache, 0, sizeof(f->ChainCache));
+    f->ChainCacheClock = 0;
+    KeReleaseSpinlock(&f->ChainCacheLock, flags);
+
+    FAT32_VOL* v = f->Vol;
+    if (!v) return;
+    flags = KeAcquireSpinlock(&v->PathCacheLock);
+    v->Stats.FatChainCacheInvalidations++;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+// Locates the best cached starting point for a logical cluster index.
+static bool Fat32ChainCacheLookupStart(FAT32_FILE* f, uint32_t logicalIndex,
+                                       uint32_t* outLogical, uint32_t* outCluster) {
+    if (!f || !outLogical || !outCluster)
+        return false;
+
+    bool hit = false;
+    uint32_t bestLogical = 0;
+    uint32_t bestCluster = 0;
+
+    uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
+    for (uint32_t i = 0; i < FAT32_CHAIN_CACHE_SLOTS; i++) {
+        FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[i];
+        if (!e->Valid || e->Length == 0)
+            continue;
+
+        uint32_t end = e->LogicalStart + e->Length;
+        if (logicalIndex >= e->LogicalStart && logicalIndex < end) {
+            *outLogical = logicalIndex;
+            *outCluster = e->PhysicalStart + (logicalIndex - e->LogicalStart);
+            hit = true;
+            break;
+        }
+
+        if (end <= logicalIndex && e->NextCluster >= 2 && e->NextCluster < FAT32_EOC) {
+            if (!hit || end > bestLogical) {
+                bestLogical = end;
+                bestCluster = e->NextCluster;
+                hit = true;
+            }
+        }
+    }
+
+    if (hit && bestCluster) {
+        *outLogical = bestLogical;
+        *outCluster = bestCluster;
+    }
+    KeReleaseSpinlock(&f->ChainCacheLock, flags);
+
+    FAT32_VOL* v = f->Vol;
+    if (v) {
+        flags = KeAcquireSpinlock(&v->PathCacheLock);
+        v->Stats.FatChainCacheLookups++;
+        if (hit) v->Stats.FatChainCacheHits++;
+        else v->Stats.FatChainCacheMisses++;
+        KeReleaseSpinlock(&v->PathCacheLock, flags);
+    }
+    return hit;
+}
+
+// Stores a physically contiguous FAT extent in the per-open chain cache.
+static void Fat32ChainCacheStore(FAT32_FILE* f, uint32_t logicalStart,
+                                 uint32_t physicalStart, uint32_t length,
+                                 uint32_t nextCluster) {
+    if (!f || physicalStart < 2 || physicalStart >= FAT32_EOC || length == 0)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
+    uint32_t slot = f->ChainCacheClock++ % FAT32_CHAIN_CACHE_SLOTS;
+    FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[slot];
+    e->Valid = true;
+    e->LogicalStart = logicalStart;
+    e->PhysicalStart = physicalStart;
+    e->Length = length;
+    e->NextCluster = nextCluster;
+    KeReleaseSpinlock(&f->ChainCacheLock, flags);
+
+    FAT32_VOL* v = f->Vol;
+    if (!v) return;
+    flags = KeAcquireSpinlock(&v->PathCacheLock);
+    v->Stats.FatChainCacheStores++;
+    KeReleaseSpinlock(&v->PathCacheLock, flags);
+}
+
+// Resolves a logical file cluster through the cached FAT-chain extent map.
+static uint32_t Fat32ResolveFileCluster(FAT32_FILE* f, uint32_t logicalIndex) {
+    if (!f || f->StartCluster < 2 || f->StartCluster >= FAT32_EOC)
+        return 0;
+    if (logicalIndex == 0)
+        return f->StartCluster;
+
+    uint32_t currentLogical = 0;
+    uint32_t cluster = f->StartCluster;
+    if (!Fat32ChainCacheLookupStart(f, logicalIndex, &currentLogical, &cluster)) {
+        currentLogical = 0;
+        cluster = f->StartCluster;
+    }
+
+    uint64_t steps = 0;
+    while (currentLogical < logicalIndex && cluster >= 2 && cluster < FAT32_EOC) {
+        cluster = FatEntry(f->Vol, cluster);
+        currentLogical++;
+        steps++;
+    }
+    Fat32StatsAddChainWalk(f->Vol, steps);
+    return cluster;
+}
+
+// Builds and caches one physically contiguous read run from a starting cluster.
+static bool Fat32BuildReadRun(FAT32_FILE* f, uint32_t logicalIndex,
+                              uint32_t startCluster, uint32_t maxSectors,
+                              uint32_t* outRunSectors, uint32_t* outNextCluster) {
+    if (!f || !outRunSectors || !outNextCluster)
+        return false;
+    FAT32_VOL* v = f->Vol;
+    if (!v || startCluster < 2 || startCluster >= FAT32_EOC)
+        return false;
+
+    uint32_t runSectors = v->SectorsPerCluster;
+    uint32_t runClusters = 1;
+    uint32_t last = startCluster;
+    uint32_t next = FatEntry(v, last);
+    uint64_t steps = 1;
+
+    while (next == last + 1 && next < FAT32_EOC &&
+           runSectors + v->SectorsPerCluster <= maxSectors) {
+        last = next;
+        runClusters++;
+        runSectors += v->SectorsPerCluster;
+        next = FatEntry(v, last);
+        steps++;
+    }
+
+    Fat32StatsAddChainWalk(v, steps);
+    Fat32ChainCacheStore(f, logicalIndex, startCluster, runClusters, next);
+    *outRunSectors = runSectors;
+    *outNextCluster = next;
+    return true;
+}
+
 static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
                       uint32_t *outCluster, uint32_t *outSize, uint8_t *outAttr,
                       uint64_t *outDirentLba, uint32_t *outDirentOffset) {
@@ -986,15 +1386,20 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
     int allowShortMatch = FatNameIsStructurallyShort83V(name);
     Make83NameV(name, wantName83, wantExt83);
     uint32_t cluster = dirCluster;
+    uint64_t clustersRead = 0;
+    uint64_t entriesScanned = 0;
+    int found = 0;
 
     while (cluster >= 2 && cluster < FAT32_EOC) {
         uint64_t clusterLba = ClusterToLba(v, cluster);
         if (!KSUCCESS(ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf))) break;
+        clustersRead++;
 
         uint32_t entries = v->BytesPerCluster / sizeof(DIRENT);
         DIRENT  *dir     = (DIRENT *)clusterBuf;
 
         for (uint32_t i = 0; i < entries; i++) {
+            entriesScanned++;
             if (dir[i].Name[0] == 0x00) goto done;
             if ((uint8_t)dir[i].Name[0] == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; }
             if (dir[i].Attr == ATTR_LFN) { LfnAppend(lfn, (LFN_ENTRY *)&dir[i]); hasLfn = 1; continue; }
@@ -1016,15 +1421,16 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
                     if (outDirentLba)    *outDirentLba    = clusterLba + eb / v->BytesPerSector;
                     if (outDirentOffset) *outDirentOffset = (uint32_t)(eb % v->BytesPerSector);
                 }
-                MmFreePool(clusterBuf);
+                found = 1;
-                return 1;
+                goto done;
             }
         }
         cluster = FatEntry(v, cluster);
     }
 done:
+    Fat32PathCacheAddDirectoryStats(v, clustersRead, entriesScanned);
     MmFreePool(clusterBuf);
-    return 0;
+    return found;
 }
 
 static int Fat32PathNextComponent(SYS_STRING_VIEW path, size_t* cursor,
@@ -1058,6 +1464,10 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
     if (outDirentLba)    *outDirentLba    = 0;
     if (outDirentOffset) *outDirentOffset = 0;
 
+    if (Fat32PathCacheLookup(v, path, outCluster, outSize, &attr,
+                             outDirentLba, outDirentOffset))
+        return STATUS_SUCCESS;
+
     size_t cursor = 0;
     SYS_STRING_VIEW component;
     while (Fat32PathNextComponent(path, &cursor, &component)) {
@@ -1078,6 +1488,9 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
 
     *outCluster = cluster;
     *outSize = size;
+    Fat32PathCacheStore(v, path, cluster, size, attr,
+                        outDirentLba ? *outDirentLba : 0,
+                        outDirentOffset ? *outDirentOffset : 0);
     return STATUS_SUCCESS;
 }
 
@@ -1220,6 +1633,7 @@ KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
     if (!f) return STATUS_NO_MEMORY;
     f->StartCluster = cluster; f->FileSize = size; f->Vol = v;
     f->DirentLba = direntLba; f->DirentOffset = direntOffset;
+    Fat32FileInitChainCache(f);
     *fileCtx = f;
     return STATUS_SUCCESS;
 }
@@ -1233,15 +1647,9 @@ void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
 }
 
 // Read  (fix #4: contiguous-run coalescing)
-// We peek ahead in the FAT chain: if cluster N+1 == cluster N physically,
+// Cluster-chain lookup is cached per open file, and data spans are allowed to
-// extend the current run instead of issuing a new ReadSectors call.
+// grow beyond the AHCI bounce size because the device layer now splits large
-// Run length is capped at MAX_RUN_SECTORS to stay within the AHCI bounce
+// sector requests for the concrete controller.
-// buffer (128 sectors = 64 KB).
-
-// FatEntry() calls in the peek loop are nearly free because they hit the
-// FAT sector cache (fix #1).
-
-#define MAX_RUN_SECTORS 128u
 
 KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
                           uint64_t offset, void *buf,
@@ -1257,14 +1665,12 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
     if (size > (size_t)(f->FileSize - offset)) size = (size_t)(f->FileSize - offset);
     size_t wanted = size;
 
-    // Allocate only as much bounce space as this read can actually use.
-    // The old code allocated 64 KiB and could read a 64 KiB contiguous run even
-    // for tiny random reads; Doom does lots of seek/read pairs for small lumps.
     uint32_t initialOffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
     uint64_t initialBytesCovered = (uint64_t)initialOffsetInCluster + (uint64_t)size;
     uint32_t maxUsefulSectors = (uint32_t)((initialBytesCovered + 511u) / 512u);
     if (maxUsefulSectors < v->SectorsPerCluster) maxUsefulSectors = v->SectorsPerCluster;
-    if (maxUsefulSectors > MAX_RUN_SECTORS) maxUsefulSectors = MAX_RUN_SECTORS;
+    if (maxUsefulSectors > FAT32_READ_RUN_MAX_SECTORS)
+        maxUsefulSectors = FAT32_READ_RUN_MAX_SECTORS;
 
     uint8_t *runBuf = (uint8_t *)MmAllocatePool((size_t)maxUsefulSectors * 512u);
     if (!runBuf) return STATUS_NO_MEMORY;
@@ -1272,35 +1678,27 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
     size_t   totalRead       = 0;
     uint8_t *out             = (uint8_t *)buf;
     KSTATUS readStatus       = STATUS_SUCCESS;
-    uint32_t skipClusters    = (uint32_t)(offset / v->BytesPerCluster);
+    uint32_t logicalCluster  = (uint32_t)(offset / v->BytesPerCluster);
     uint32_t offsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
-    uint32_t cluster         = f->StartCluster;
+    uint32_t cluster         = Fat32ResolveFileCluster(f, logicalCluster);
-
-    for (uint32_t i = 0; i < skipClusters && cluster < FAT32_EOC; i++)
-        cluster = FatEntry(v, cluster);
 
     while (size > 0 && cluster >= 2 && cluster < FAT32_EOC) {
-        // Build the longest physically-contiguous run that can contribute to
-        // this caller's remaining read. Do not prefetch an unrelated 64 KiB run
-        // for a tiny random read.
         uint64_t bytesCovered = (uint64_t)offsetInCluster + (uint64_t)size;
         uint32_t usefulSectors = (uint32_t)((bytesCovered + 511u) / 512u);
         if (usefulSectors < v->SectorsPerCluster) usefulSectors = v->SectorsPerCluster;
-        if (usefulSectors > MAX_RUN_SECTORS) usefulSectors = MAX_RUN_SECTORS;
+        if (usefulSectors > FAT32_READ_RUN_MAX_SECTORS)
+            usefulSectors = FAT32_READ_RUN_MAX_SECTORS;
         if (usefulSectors > maxUsefulSectors) usefulSectors = maxUsefulSectors;
 
-        uint32_t runStart   = cluster;
+        uint32_t runSectors = 0;
-        uint32_t runSectors = v->SectorsPerCluster;
+        uint32_t next = 0;
-        uint32_t next       = FatEntry(v, cluster);
+        if (!Fat32BuildReadRun(f, logicalCluster, cluster, usefulSectors,
-
+                               &runSectors, &next)) {
-        while (next == cluster + 1 && next < FAT32_EOC
+            readStatus = STATUS_DEVICE_ERROR;
-               && runSectors + v->SectorsPerCluster <= usefulSectors) {
+            break;
-            cluster     = next;
-            next        = FatEntry(v, cluster);
-            runSectors += v->SectorsPerCluster;
         }
 
-        KSTATUS rs = ReadSectors(v, ClusterToLba(v, runStart), runSectors, runBuf);
+        KSTATUS rs = ReadSectors(v, ClusterToLba(v, cluster), runSectors, runBuf);
         if (!KSUCCESS(rs)) {
             readStatus = rs;
             break;
@@ -1309,12 +1707,16 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
         uint32_t runBytes = runSectors * 512u;
         size_t   avail    = runBytes - offsetInCluster;
         size_t   copy     = avail < size ? avail : size;
+        uint32_t runClusters = runSectors / v->SectorsPerCluster;
+        uint32_t merges = runClusters ? runClusters - 1u : 0;
+        Fat32StatsAddReadRun(v, runSectors, merges);
 
         memcpy(out, runBuf + offsetInCluster, copy);
         out             += copy;
         totalRead       += copy;
         size            -= copy;
         offsetInCluster  = 0;
+        logicalCluster  += runClusters;
         cluster          = next;
     }
 
@@ -1345,6 +1747,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
     FAT32_FILE    *f   = (FAT32_FILE *)fileCtx;
     FAT32_VOL     *v   = f->Vol;
     const uint8_t *src = (const uint8_t *)buf;
+    Fat32FileInvalidateChainCache(f);
 
     // Ensure at least one cluster exists.
     if (f->StartCluster < 2) {
@@ -1368,7 +1771,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
         cluster = next;
     }
 
-    uint8_t *runBuf = (uint8_t *)MmAllocatePool(MAX_RUN_SECTORS * 512u);
+    uint8_t *runBuf = (uint8_t *)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
     if (!runBuf) { *bytesWritten = 0; return STATUS_NO_MEMORY; }
 
     size_t written = 0, remaining = size;
@@ -1382,7 +1785,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
 
         if (offsetInCluster == 0) {
             while (next == cluster + 1 && next < FAT32_EOC
-                   && runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
+                   && runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
                 cluster     = next;
                 next        = FatEntry(v, cluster);
                 runSectors += v->SectorsPerCluster;
@@ -1439,7 +1842,9 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
             u.Flags |= FAT32_DIRENT_UPDATE_SIZE;
             u.Size = f->FileSize;
         }
-        DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
+        KSTATUS ms = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
+        if (KSUCCESS(ms))
+            Fat32PathCacheInvalidate(v);
     }
 
     *bytesWritten = written;
@@ -1447,8 +1852,6 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
 }
 
 // Create.
-KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
-
 // Creates the requested object.
 KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
     if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
@@ -1483,7 +1886,7 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
             // Create means overwrite by default for regular files.  Reuse the
             // existing directory entry, drop the old cluster chain, and return
             // an open zero-length file instead of allocating a duplicate name.
-            KSTATUS ts = Fat32Truncate(fs, f, 0);
+            KSTATUS ts = Fat32Truncate(fs, f, 0, 0);
             if (!KSUCCESS(ts)) { MmFreePool(f); return ts; }
             *fileCtx = f;
             return STATUS_SUCCESS;
@@ -1558,21 +1961,27 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
     if (!f) return STATUS_NO_MEMORY;
     f->StartCluster = 0; f->FileSize = 0; f->Vol = v;
     f->DirentLba = direntLba; f->DirentOffset = direntOffset;
+    Fat32FileInitChainCache(f);
+    Fat32PathCacheInvalidate(v);
     *fileCtx = f;
     return STATUS_SUCCESS;
 }
 
 // Truncate.
-KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
+KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags) {
     (void)fs;
     FAT32_FILE *f = (FAT32_FILE *)fileCtx;
     FAT32_VOL  *v = f->Vol;
+    Fat32FileInvalidateChainCache(f);
+    int zeroNewClusters = (fileFlags & IO_FILE_FLAG_SKIP_ZERO_EXTEND) ? 0 : 1;
     if ((uint32_t)newSize == f->FileSize) {
         FAT32_DIRENT_UPDATE u;
         memset(&u, 0, sizeof(u));
         FatQueryEncodedTime(&u.Timestamp);
         u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME;
-        return DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
+        KSTATUS s = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
+        if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
+        return s;
     }
 
     if (newSize == 0) {
@@ -1584,6 +1993,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
         f->FileSize = 0;
         KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, 0);
         FlushFatCache(v);
+        if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
         return s;
     }
 
@@ -1604,7 +2014,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
             ? (f->FileSize + v->BytesPerCluster - 1) / v->BytesPerCluster : 0;
         uint32_t lastCluster = 0;
         if (f->StartCluster < 2) {
-            uint32_t first = AllocCluster(v);
+            uint32_t first = AllocClusterEx(v, zeroNewClusters);
             if (!first) return STATUS_NO_MEMORY;
             f->StartCluster = first;
             DirentUpdateFirstCluster(v, f->DirentLba, f->DirentOffset, first);
@@ -1613,7 +2023,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
             lastCluster = GetChainEnd(v, f->StartCluster, NULL);
         }
         for (uint32_t i = curClusters; i < newClusters; i++) {
-            uint32_t next = AppendCluster(v, lastCluster);
+            uint32_t next = AppendClusterEx(v, lastCluster, zeroNewClusters);
             if (!next) return STATUS_NO_MEMORY;
             lastCluster = next;
         }
@@ -1623,6 +2033,7 @@ update_size:
     f->FileSize = (uint32_t)newSize;
     { KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, (uint32_t)newSize);
       FlushFatCache(v);
+      if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
       return s; }
 }
 
@@ -1646,6 +2057,7 @@ KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
 
     if (cluster >= 2) FreeClusterChain(v, cluster);
     FlushFatCache(v);
+    Fat32PathCacheInvalidate(v);
     return STATUS_SUCCESS;
 }
 
@@ -1654,6 +2066,75 @@ uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
     (void)fs; return ((FAT32_FILE *)fileCtx)->FileSize;
 }
 
+// Mixes FAT directory metadata into a per-file content generation.
+static uint64_t Fat32DirentGeneration(const DIRENT *d) {
+    if (!d)
+        return 0;
+
+    uint32_t firstCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
+    uint64_t generation = ((uint64_t)d->WrtDate << 48) |
+                          ((uint64_t)d->WrtTime << 32) |
+                          ((uint64_t)d->LastAccDate << 16) |
+                          (uint64_t)d->Attr;
+    generation ^= ((uint64_t)d->CrtDate << 40);
+    generation ^= ((uint64_t)d->CrtTime << 24);
+    generation ^= ((uint64_t)d->CrtTimeTenth << 16);
+    generation ^= ((uint64_t)firstCluster << 1);
+    generation ^= ((uint64_t)d->FileSize << 17);
+    return generation ? generation : 1;
+}
+
+// Returns a stable FAT file identity for cross-open image-section caches.
+KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out) {
+    (void)fs;
+    if (!fileCtx || !out)
+        return STATUS_INVALID_PARAMETER;
+
+    FAT32_FILE *f = (FAT32_FILE *)fileCtx;
+    FAT32_VOL *v = f->Vol;
+    if (!v)
+        return STATUS_INVALID_PARAMETER;
+
+    uint32_t startCluster = f->StartCluster;
+    uint32_t fileSize = f->FileSize;
+    uint64_t generation = ((uint64_t)startCluster << 1) ^
+                          ((uint64_t)fileSize << 17);
+
+    if (f->DirentLba != 0) {
+        uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
+        if (!sec)
+            return STATUS_NO_MEMORY;
+
+        KSTATUS s = ReadSectors(v, f->DirentLba, 1, sec);
+        if (!KSUCCESS(s)) {
+            MmFreePool(sec);
+            return s;
+        }
+
+        DIRENT *d = (DIRENT *)(sec + f->DirentOffset);
+        if ((uint8_t)d->Name[0] == 0x00 || (uint8_t)d->Name[0] == 0xE5) {
+            MmFreePool(sec);
+            return STATUS_NOT_FOUND;
+        }
+
+        startCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
+        fileSize = d->FileSize;
+        generation = Fat32DirentGeneration(d);
+        MmFreePool(sec);
+    }
+
+    out->VolumeId = ((uint64_t)v->DriveIndex << 56) ^
+                    ((uint64_t)v->PartitionLba << 8) ^
+                    ((uint64_t)v->RootCluster << 1) ^
+                    0x4641543332494445ULL;
+    out->FileId = f->DirentLba ^
+                  ((uint64_t)f->DirentOffset << 32) ^
+                  ((uint64_t)startCluster << 1);
+    out->Generation = generation ? generation : 1;
+    out->Size = fileSize;
+    return STATUS_SUCCESS;
+}
+
 // Directory enumeration  (fix #2: cluster buffer cached in context)
 // Previously: allocate + ReadSectors + return 1 entry + free, every call.
 // Now: buffer lives in FAT32_DIR_CTX; ReadSectors fires only on cluster
@@ -1757,8 +2238,7 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
         }
 
         ctx->CurCluster = FatEntry(v, ctx->CurCluster);
-        ctx->EntryIndex = 0; ctx->HasLfn = 0;
+        ctx->EntryIndex = 0;
-        memset(ctx->LfnBuf, 0, sizeof(ctx->LfnBuf));
     }
 
     return STATUS_NO_MORE_FILES;
@@ -1962,6 +2442,7 @@ typedef struct FAT32_ASYNC_RW {
     uint64_t Offset;
     uint32_t OffsetInCluster;
     uint32_t Cluster;
+    uint32_t LogicalCluster;
     size_t Remaining;
     size_t Done;
     size_t Requested;
@@ -1972,6 +2453,7 @@ typedef struct FAT32_ASYNC_RW {
 
     uint32_t CurrentNext;
     uint32_t CurrentLast;
+    uint32_t CurrentClusters;
     uint32_t CurrentOffset;
     size_t CurrentCopy;
 
@@ -2088,7 +2570,13 @@ static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
     ctx->Done += ctx->CurrentCopy;
     ctx->Remaining -= ctx->CurrentCopy;
     ctx->OffsetInCluster = 0;
+    ctx->LogicalCluster += ctx->CurrentClusters;
     ctx->Cluster = ctx->CurrentNext;
+    if (ctx->CurrentClusters) {
+        uint32_t sectors = ctx->CurrentClusters * ctx->Vol->SectorsPerCluster;
+        uint32_t merges = ctx->CurrentClusters - 1u;
+        Fat32StatsAddReadRun(ctx->Vol, sectors, merges);
+    }
 
     Fat32AsyncQueueContinuation(ctx);
 }
@@ -2109,15 +2597,12 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
 
     FAT32_VOL* v = ctx->Vol;
     uint32_t runStart = ctx->Cluster;
-    uint32_t runSectors = v->SectorsPerCluster;
+    uint32_t runSectors = 0;
-    uint32_t last = ctx->Cluster;
+    uint32_t next = 0;
-    uint32_t next = FatEntry(v, last);
+    if (!Fat32BuildReadRun(ctx->File, ctx->LogicalCluster, ctx->Cluster,
-
+                           FAT32_READ_RUN_MAX_SECTORS, &runSectors, &next)) {
-    while (next == last + 1 && next < FAT32_EOC &&
+        Fat32AsyncFinish(ctx, STATUS_DEVICE_ERROR, ctx->Done);
-           runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
+        return;
-        last = next;
-        next = FatEntry(v, last);
-        runSectors += v->SectorsPerCluster;
     }
 
     uint32_t runBytes = runSectors * 512u;
@@ -2125,7 +2610,8 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
     size_t copy = avail < ctx->Remaining ? avail : ctx->Remaining;
 
     ctx->CurrentNext = next;
-    ctx->CurrentLast = last;
+    ctx->CurrentLast = runStart + (runSectors / v->SectorsPerCluster) - 1u;
+    ctx->CurrentClusters = runSectors / v->SectorsPerCluster;
     ctx->CurrentOffset = ctx->OffsetInCluster;
     ctx->CurrentCopy = copy;
 
@@ -2184,7 +2670,7 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
     memset(ctx, 0, sizeof(*ctx));
     LIST_INIT(&ctx->Link);
 
-    ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
+    ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_READ_RUN_MAX_SECTORS * 512u);
     if (!ctx->RunBuf) {
         MmFreePool(ctx);
         complete(completeContext, STATUS_NO_MEMORY, 0);
@@ -2196,17 +2682,14 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
     ctx->IsWrite = 0;
     ctx->Offset = offset;
     ctx->OffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
-    ctx->Cluster = f->StartCluster;
+    ctx->LogicalCluster = (uint32_t)(offset / v->BytesPerCluster);
+    ctx->Cluster = Fat32ResolveFileCluster(f, ctx->LogicalCluster);
     ctx->Remaining = size;
     ctx->Requested = size;
     ctx->Dst = (uint8_t*)buf;
     ctx->Complete = complete;
     ctx->CompleteContext = completeContext;
 
-    uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster);
-    for (uint32_t i = 0; i < skipClusters && ctx->Cluster < FAT32_EOC; i++)
-        ctx->Cluster = FatEntry(v, ctx->Cluster);
-
     Fat32AsyncQueueContinuation(ctx);
     return STATUS_PENDING;
 }
@@ -2276,7 +2759,7 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
 
     if (ctx->OffsetInCluster == 0) {
         while (next == last + 1 && next < FAT32_EOC &&
-               runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
+               runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
             last = next;
             next = FatEntry(v, last);
             runSectors += v->SectorsPerCluster;
@@ -2347,6 +2830,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
 
     FAT32_FILE* f = (FAT32_FILE*)fileCtx;
     FAT32_VOL* v = f->Vol;
+    Fat32FileInvalidateChainCache(f);
 
     if (f->StartCluster < 2) {
         uint32_t first = AllocCluster(v);
@@ -2380,7 +2864,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
     memset(ctx, 0, sizeof(*ctx));
     LIST_INIT(&ctx->Link);
 
-    ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
+    ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
     if (!ctx->RunBuf) {
         MmFreePool(ctx);
         complete(completeContext, STATUS_NO_MEMORY, 0);

code/drivers/fs/fat32/fat32.h

@@ -16,6 +16,8 @@
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
 #include <kernel/ke_time.h>
+#include <kernel/ke_mutex.h>
+#include <kernel/ke_irql.h>
 
 typedef struct FAT32_VOL_S FAT32_VOL;
 
@@ -27,10 +29,12 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *
 KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
 KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
 KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
-KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
+KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags);
 uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx);
+KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out);
 KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx);
 KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out);
 void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx);
+void Fat32QueryStats(IO_FILESYSTEM *fs, IO_FILESYSTEM_STATS *out);
 
 void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol);

code/drivers/fs/fat32/fat32_mount.c

@@ -14,9 +14,11 @@ void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
     fs->Delete     = Fat32Delete;
     fs->Truncate   = Fat32Truncate;
     fs->GetSize    = Fat32GetSize;
+    fs->QueryIdentity = Fat32QueryIdentity;
     fs->Close      = Fat32Close;
     fs->DriverCtx  = vol;
     fs->OpenDir    = Fat32OpenDir;
     fs->ReadDir    = Fat32ReadDir;
     fs->CloseDir   = Fat32CloseDir;
+    fs->QueryStats = Fat32QueryStats;
 }

code/drivers/input/serial/serial.c

@@ -0,0 +1,301 @@
+// 16550-compatible serial input driver. This file routes COM1 receive bytes into the canonical input core.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <intrin.h>
+
+#include <ntlite/status.h>
+#include <ntlite/ddi/hal_api.h>
+#include <ntlite/ddi/input_device.h>
+#include <kernel/ke_debug.h>
+#include <kernel/ke_irq.h>
+
+#define SERIAL_COM1_BASE 0x3F8u
+#define SERIAL_COM1_IRQ  4u
+
+#define SERIAL_REG_DATA       0u
+#define SERIAL_REG_IER        1u
+#define SERIAL_REG_IIR        2u
+#define SERIAL_REG_FCR        2u
+#define SERIAL_REG_LCR        3u
+#define SERIAL_REG_MCR        4u
+#define SERIAL_REG_LSR        5u
+
+#define SERIAL_LSR_DATA_READY 0x01u
+#define SERIAL_LSR_TX_EMPTY   0x20u
+#define SERIAL_LCR_DLAB       0x80u
+#define SERIAL_MCR_DTR        0x01u
+#define SERIAL_MCR_RTS        0x02u
+#define SERIAL_MCR_OUT2       0x08u
+
+#define SERIAL_IER_RX_READY   0x01u
+#define SERIAL_FCR_ENABLE     0x01u
+#define SERIAL_FCR_CLEAR_RX   0x02u
+#define SERIAL_FCR_CLEAR_TX   0x04u
+#define SERIAL_FCR_TRIGGER_14 0xC0u
+
+typedef enum SERIAL_ESCAPE_STATE {
+    SerialEscapeNone = 0,
+    SerialEscapeEsc,
+    SerialEscapeCsi,
+    SerialEscapeCsiNumber,
+} SERIAL_ESCAPE_STATE;
+
+static volatile SERIAL_ESCAPE_STATE gSerialEscapeState;
+static volatile uint8_t gSerialCsiNumber;
+static volatile bool gSerialDropNextLf;
+
+// Reads one byte from a COM register.
+static uint8_t SerialRead(uint16_t reg) {
+    return __inbyte((unsigned short)(SERIAL_COM1_BASE + reg));
+}
+
+// Writes one byte to a COM register.
+static void SerialWrite(uint16_t reg, uint8_t value) {
+    __outbyte((unsigned short)(SERIAL_COM1_BASE + reg), value);
+}
+
+// Submits one input key transition.
+static void SerialSubmitTransition(INPUT_KEY key, bool pressed) {
+    INPUT_KEY_EVENT event;
+    event.Key = key;
+    event.Pressed = pressed;
+    KeInputSubmitKey(&event);
+}
+
+// Submits one logical key press and release.
+static void SerialSubmitKey(INPUT_KEY key) {
+    if (key == INPUT_KEY_NONE) return;
+    SerialSubmitTransition(key, true);
+    SerialSubmitTransition(key, false);
+}
+
+// Submits one shifted key press and release.
+static void SerialSubmitShiftedKey(INPUT_KEY key) {
+    if (key == INPUT_KEY_NONE) return;
+    SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, true);
+    SerialSubmitKey(key);
+    SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, false);
+}
+
+// Submits one control-modified key press and release.
+static void SerialSubmitCtrlKey(INPUT_KEY key) {
+    if (key == INPUT_KEY_NONE) return;
+    SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, true);
+    SerialSubmitKey(key);
+    SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, false);
+}
+
+// Converts an ASCII printable byte into input events.
+static void SerialSubmitPrintable(uint8_t ch) {
+    if (ch >= 'a' && ch <= 'z') {
+        SerialSubmitKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'a')));
+        return;
+    }
+    if (ch >= 'A' && ch <= 'Z') {
+        SerialSubmitShiftedKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'A')));
+        return;
+    }
+    if (ch >= '1' && ch <= '9') {
+        SerialSubmitKey((INPUT_KEY)(INPUT_KEY_1 + (ch - '1')));
+        return;
+    }
+    if (ch == '0') {
+        SerialSubmitKey(INPUT_KEY_0);
+        return;
+    }
+
+    switch (ch) {
+        case ' ': SerialSubmitKey(INPUT_KEY_SPACE); return;
+        case '-': SerialSubmitKey(INPUT_KEY_MINUS); return;
+        case '=': SerialSubmitKey(INPUT_KEY_EQUAL); return;
+        case '[': SerialSubmitKey(INPUT_KEY_LEFT_BRACKET); return;
+        case ']': SerialSubmitKey(INPUT_KEY_RIGHT_BRACKET); return;
+        case '\\': SerialSubmitKey(INPUT_KEY_BACKSLASH); return;
+        case ';': SerialSubmitKey(INPUT_KEY_SEMICOLON); return;
+        case '\'': SerialSubmitKey(INPUT_KEY_APOSTROPHE); return;
+        case '`': SerialSubmitKey(INPUT_KEY_GRAVE); return;
+        case ',': SerialSubmitKey(INPUT_KEY_COMMA); return;
+        case '.': SerialSubmitKey(INPUT_KEY_PERIOD); return;
+        case '/': SerialSubmitKey(INPUT_KEY_SLASH); return;
+
+        case '!': SerialSubmitShiftedKey(INPUT_KEY_1); return;
+        case '@': SerialSubmitShiftedKey(INPUT_KEY_2); return;
+        case '#': SerialSubmitShiftedKey(INPUT_KEY_3); return;
+        case '$': SerialSubmitShiftedKey(INPUT_KEY_4); return;
+        case '%': SerialSubmitShiftedKey(INPUT_KEY_5); return;
+        case '^': SerialSubmitShiftedKey(INPUT_KEY_6); return;
+        case '&': SerialSubmitShiftedKey(INPUT_KEY_7); return;
+        case '*': SerialSubmitShiftedKey(INPUT_KEY_8); return;
+        case '(': SerialSubmitShiftedKey(INPUT_KEY_9); return;
+        case ')': SerialSubmitShiftedKey(INPUT_KEY_0); return;
+        case '_': SerialSubmitShiftedKey(INPUT_KEY_MINUS); return;
+        case '+': SerialSubmitShiftedKey(INPUT_KEY_EQUAL); return;
+        case '{': SerialSubmitShiftedKey(INPUT_KEY_LEFT_BRACKET); return;
+        case '}': SerialSubmitShiftedKey(INPUT_KEY_RIGHT_BRACKET); return;
+        case '|': SerialSubmitShiftedKey(INPUT_KEY_BACKSLASH); return;
+        case ':': SerialSubmitShiftedKey(INPUT_KEY_SEMICOLON); return;
+        case '"': SerialSubmitShiftedKey(INPUT_KEY_APOSTROPHE); return;
+        case '~': SerialSubmitShiftedKey(INPUT_KEY_GRAVE); return;
+        case '<': SerialSubmitShiftedKey(INPUT_KEY_COMMA); return;
+        case '>': SerialSubmitShiftedKey(INPUT_KEY_PERIOD); return;
+        case '?': SerialSubmitShiftedKey(INPUT_KEY_SLASH); return;
+        default: return;
+    }
+}
+
+// Submits a completed CSI escape sequence.
+static void SerialSubmitCsi(uint8_t finalByte) {
+    uint8_t number = gSerialCsiNumber;
+    gSerialEscapeState = SerialEscapeNone;
+    gSerialCsiNumber = 0;
+
+    switch (finalByte) {
+        case 'A': SerialSubmitKey(INPUT_KEY_UP); return;
+        case 'B': SerialSubmitKey(INPUT_KEY_DOWN); return;
+        case 'C': SerialSubmitKey(INPUT_KEY_RIGHT); return;
+        case 'D': SerialSubmitKey(INPUT_KEY_LEFT); return;
+        case 'H': SerialSubmitKey(INPUT_KEY_HOME); return;
+        case 'F': SerialSubmitKey(INPUT_KEY_END); return;
+        case '~':
+            switch (number) {
+                case 1: SerialSubmitKey(INPUT_KEY_HOME); return;
+                case 3: SerialSubmitKey(INPUT_KEY_DELETE); return;
+                case 4: SerialSubmitKey(INPUT_KEY_END); return;
+                case 5: SerialSubmitKey(INPUT_KEY_PAGE_UP); return;
+                case 6: SerialSubmitKey(INPUT_KEY_PAGE_DOWN); return;
+                default: return;
+            }
+        default: return;
+    }
+}
+
+// Handles one byte after an ESC prefix.
+static bool SerialHandleEscapeByte(uint8_t ch) {
+    if (gSerialEscapeState == SerialEscapeNone) return false;
+
+    if (gSerialEscapeState == SerialEscapeEsc) {
+        if (ch == '[') {
+            gSerialEscapeState = SerialEscapeCsi;
+            gSerialCsiNumber = 0;
+            return true;
+        }
+
+        gSerialEscapeState = SerialEscapeNone;
+        gSerialCsiNumber = 0;
+        SerialSubmitKey(INPUT_KEY_ESCAPE);
+        return false;
+    }
+
+    if (gSerialEscapeState == SerialEscapeCsi || gSerialEscapeState == SerialEscapeCsiNumber) {
+        if (ch >= '0' && ch <= '9') {
+            gSerialEscapeState = SerialEscapeCsiNumber;
+            if (gSerialCsiNumber < 25)
+                gSerialCsiNumber = (uint8_t)(gSerialCsiNumber * 10u + (ch - '0'));
+            return true;
+        }
+
+        if (ch == ';')
+            return true;
+
+        SerialSubmitCsi(ch);
+        return true;
+    }
+
+    gSerialEscapeState = SerialEscapeNone;
+    gSerialCsiNumber = 0;
+    return false;
+}
+
+// Routes one received serial byte into the input core.
+static void SerialReceiveByte(uint8_t ch) {
+    if (SerialHandleEscapeByte(ch)) return;
+
+    if (ch == 0x1Bu) {
+        gSerialEscapeState = SerialEscapeEsc;
+        gSerialCsiNumber = 0;
+        return;
+    }
+
+    if (ch == '\r') {
+        SerialSubmitKey(INPUT_KEY_ENTER);
+        gSerialDropNextLf = true;
+        return;
+    }
+
+    if (ch == '\n') {
+        if (gSerialDropNextLf) {
+            gSerialDropNextLf = false;
+            return;
+        }
+        SerialSubmitKey(INPUT_KEY_ENTER);
+        return;
+    }
+
+    gSerialDropNextLf = false;
+
+    if (ch == '\t') {
+        SerialSubmitKey(INPUT_KEY_TAB);
+        return;
+    }
+
+    if (ch == '\b' || ch == 0x7Fu) {
+        SerialSubmitKey(INPUT_KEY_BACKSPACE);
+        return;
+    }
+
+    if (ch == 0x03u) {
+        SerialSubmitCtrlKey(INPUT_KEY_C);
+        return;
+    }
+
+    if (ch >= 0x20u && ch <= 0x7Eu)
+        SerialSubmitPrintable(ch);
+}
+
+// Handles receive interrupts for COM1.
+static void SerialIsr(void) {
+    for (uint32_t i = 0; i < 256u; i++) {
+        if ((SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY) == 0)
+            break;
+        SerialReceiveByte(SerialRead(SERIAL_REG_DATA));
+    }
+}
+
+// Initializes COM1 as an 8N1 115200 baud input device.
+static void SerialInitializeCom1(void) {
+    SerialWrite(SERIAL_REG_IER, 0x00u);
+    SerialWrite(SERIAL_REG_LCR, SERIAL_LCR_DLAB);
+    SerialWrite(SERIAL_REG_DATA, 0x01u);
+    SerialWrite(SERIAL_REG_IER, 0x00u);
+    SerialWrite(SERIAL_REG_LCR, 0x03u);
+    SerialWrite(SERIAL_REG_FCR, SERIAL_FCR_ENABLE | SERIAL_FCR_CLEAR_RX |
+                            SERIAL_FCR_CLEAR_TX | SERIAL_FCR_TRIGGER_14);
+    SerialWrite(SERIAL_REG_MCR, SERIAL_MCR_DTR | SERIAL_MCR_RTS | SERIAL_MCR_OUT2);
+
+    while (SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY)
+        (void)SerialRead(SERIAL_REG_DATA);
+
+    SerialWrite(SERIAL_REG_IER, SERIAL_IER_RX_READY);
+}
+
+// DriverEntry registers COM1 as an input source.
+KSTATUS DriverEntry(const HAL_API* hal) {
+    (void)hal;
+
+    uint32_t deviceId = 0;
+    KSTATUS st = KeInputRegisterDevice("serial-com1", 0, &deviceId);
+    if (!KSUCCESS(st))
+        return st;
+
+    gSerialEscapeState = SerialEscapeNone;
+    gSerialCsiNumber = 0;
+    gSerialDropNextLf = false;
+
+    SerialInitializeCom1();
+    KeIrqRegister(SERIAL_COM1_IRQ, SerialIsr);
+
+    DbgPrint("serial: initialized COM1 input device=%u irq=%u\n", deviceId, SERIAL_COM1_IRQ);
+    return STATUS_SUCCESS;
+}

code/drivers/net/rtl8139/rtl8139.c

@@ -88,8 +88,8 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) {
 
 // Creates dma.
 static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
-    r->RxPhys = MmAllocateContiguous(RTL8139_RX_PAGES);
+    r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma);
-    r->TxPhys = MmAllocateContiguous(RTL8139_TX_PAGES);
+    r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma);
     if (!r->RxPhys || !r->TxPhys)
         return STATUS_NO_MEMORY;
 

code/drivers/net/rtl8139/rtl8139.h

@@ -17,6 +17,7 @@
 #include <kernel/ke_dpc.h>
 #include <kernel/ke_spinlock.h>
 #include <kernel/mm_phys.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/mm_virt.h>
 #include <ntlite/ddi/net_device.h>
 

code/drivers/net/rtl816x/rtl816x.c

@@ -85,10 +85,10 @@ static void RtlReadMac(RTL816X_DEVICE* r) {
 
 // Implements the rtl Allocate Dma operation.
 static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
-    r->RxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
+    r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
-    r->TxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
+    r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
-    r->RxBuffersPhys = MmAllocateContiguous(RTL_RXBUF_PAGES);
+    r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma);
-    r->TxBuffersPhys = MmAllocateContiguous(RTL_TXBUF_PAGES);
+    r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma);
 
     if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys)
         return STATUS_NO_MEMORY;

code/drivers/net/rtl816x/rtl816x.h

@@ -17,6 +17,7 @@
 #include <kernel/ke_dpc.h>
 #include <kernel/ke_spinlock.h>
 #include <kernel/mm_phys.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/mm_virt.h>
 #include <ntlite/ddi/net_device.h>
 

code/drivers/storage/ahci/ahci.c

@@ -58,20 +58,20 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
 static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
     uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
 
-    drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT);
+    drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma);
     if (!drv->structPhys) return STATUS_NO_MEMORY;
     drv->structVirt = g_hal->PhysToVirt(drv->structPhys);
     // Force NOCACHE on DMA descriptor memory.
-    MmSetPageProtect((uintptr_t)drv->structVirt,
+    MmPteProtectRange((uintptr_t)drv->structVirt,
                      PORT_PAGES_STRUCT * 4096, flags);
     memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096);
 
-    drv->bouncePhys = MmAllocateContiguous(BOUNCE_PAGES);
+    drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma);
     if (!drv->bouncePhys) return STATUS_NO_MEMORY;
     drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys);
     // Bounce buffer: NOCACHE to ensure CPU writes reach physical memory
     // before the HBA's DMA engine reads them.
-    MmSetPageProtect((uintptr_t)drv->bounceVirt,
+    MmPteProtectRange((uintptr_t)drv->bounceVirt,
                      BOUNCE_PAGES * 4096, flags);
     memset(drv->bounceVirt, 0, BOUNCE_BYTES);
 
@@ -203,6 +203,9 @@ static void AhciIsr(void) {
 // Blocks the current thread until the ISR signals completionEvent.
 
 static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
+    if (drv->controller && drv->controller->pollingMode)
+        return port_issue_cmd_poll(drv);
+
     HBA_PORT *port = drv->port;
 
     // Clear stale IS and snapshot before issuing so we don't mis-read a
@@ -647,12 +650,25 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
              ctl->pciInfo.VendorId,
              ctl->pciInfo.DeviceId);
 
-    st = PciGetInterrupt(ctl->pdo, &ctl->irq);
+    ctl->irq = 0xFFu;
-    if (!KSUCCESS(st) || ctl->irq == 0xFF) {
+    ctl->usingMsi = false;
-        DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n");
+    ctl->pollingMode = false;
-        return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
+
+    st = PciEnableMsiAuto(ctl->pdo, &ctl->irq);
+    if (KSUCCESS(st)) {
+        ctl->usingMsi = true;
+        DbgPrint("AHCI: using MSI IRQ slot %u\n", (unsigned)ctl->irq);
+    } else {
+        KSTATUS intxStatus = PciGetInterrupt(ctl->pdo, &ctl->irq);
+        if (KSUCCESS(intxStatus) && ctl->irq != 0xFFu) {
+            DbgPrint("AHCI: using legacy INTx IRQ %u\n", (unsigned)ctl->irq);
+        } else {
+            ctl->irq = 0xFFu;
+            ctl->pollingMode = true;
+            DbgPrint("AHCI: no MSI/INTx interrupt available; using polled command completion (msi=0x%x intx=0x%x)\n",
+                     st, intxStatus);
+        }
     }
-    DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)ctl->irq);
 
     st = PciGetBar(ctl->pdo, AHCI_PCI_BAR_ABAR, &ctl->abar);
     if (!KSUCCESS(st)) {
@@ -674,10 +690,13 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
         return st;
     }
 
-    st = PciEnableDevice(ctl->pdo,
+    uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER;
-                         PCI_ENABLE_MEMORY |
+    if (ctl->usingMsi || ctl->pollingMode)
-                         PCI_ENABLE_BUSMASTER |
+        enableFlags |= PCI_DISABLE_INTX;
-                         PCI_ENABLE_INTX);
+    else
+        enableFlags |= PCI_ENABLE_INTX;
+
+    st = PciEnableDevice(ctl->pdo, enableFlags);
     if (!KSUCCESS(st)) {
         DbgPrint("AHCI: PciEnableDevice failed: 0x%x\n", st);
         return st;
@@ -831,11 +850,16 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
                  ctl->pciInfo.Address.Device,
                  ctl->pciInfo.Address.Function);
 
+    if (!ctl->pollingMode) {
         KeIrqRegister(ctl->irq, AhciIsr);
+        DbgPrint("AHCI: ISR registered on IRQ %u (%s); controller drives=%d\n",
+                 (unsigned)ctl->irq, ctl->usingMsi ? "MSI" : "INTx", ctl->driveCount);
+    } else {
+        DbgPrint("AHCI: controller started in polling mode; controller drives=%d\n",
+                 ctl->driveCount);
+    }
     ctl->started = true;
     ctl->fdo->PnpState = DevicePnpStarted;
-    DbgPrint("AHCI: ISR registered on IRQ %u; controller drives=%d\n",
-             (unsigned)ctl->irq, ctl->driveCount);
 
     return STATUS_SUCCESS;
 }

code/drivers/storage/ahci/ahci.h

@@ -8,6 +8,7 @@
 #include <ntlite/ddi/io_driver.h>
 #include <ntlite/ddi/pci.h>
 #include <kernel/mm_phys.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ke_debug.h>
 #include <kernel/ke_event.h>
@@ -367,6 +368,8 @@ struct AHCI_CONTROLLER {
     HBA_MEM*        hba;
     uint64_t        abarLength;
     uint8_t         irq;
+    bool            usingMsi;
+    bool            pollingMode;
 
     AHCI_DRIVE      drives[MAX_AHCI_DRIVES];
     int             driveCount;

code/drivers/storage/ahci/ahci_port.c

@@ -12,15 +12,15 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
     for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) {
         uintptr_t   virt  = (uintptr_t)g_hal->PhysToVirt(phys);
         uint64_t    flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
-        KSTATUS     st    = MmMapPage(virt, phys, flags, MmMapFailIfPresent);
+        KSTATUS     st    = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent);
         // Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags
         // is possible for the first page if the bootloader was overly generous.
-        // Use MmSetPageProtect to force NOCACHE in that case.
+        // Use MmPteProtectRange to force NOCACHE in that case.
         if (st == STATUS_ALREADY_EXISTS) {
-            st = MmSetPageProtect(virt, 4096, flags);
+            st = MmPteProtectRange(virt, 4096, flags);
         }
         if (!KSUCCESS(st)) {
-            DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
+            DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
                      pageIndex, (unsigned long long)phys, st);
             return st;
         }

code/kernel/include/kernel/io_file.h

@@ -25,6 +25,39 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
                                      KSTATUS status,
                                      size_t information);
 
+// Filesystem-level cache and traversal statistics.
+typedef struct IO_FILESYSTEM_STATS {
+    uint64_t PathLookups;
+    uint64_t PathCacheHits;
+    uint64_t PathCacheMisses;
+    uint64_t PathCacheStores;
+    uint64_t PathCacheInvalidations;
+    uint64_t PathCacheGeneration;
+    uint64_t DirectorySearches;
+    uint64_t DirectoryClustersRead;
+    uint64_t DirectoryEntriesScanned;
+    uint64_t FatChainCacheLookups;
+    uint64_t FatChainCacheHits;
+    uint64_t FatChainCacheMisses;
+    uint64_t FatChainCacheStores;
+    uint64_t FatChainCacheInvalidations;
+    uint64_t FatChainWalkSteps;
+    uint64_t DataReadRuns;
+    uint64_t DataReadSectors;
+    uint64_t DataReadRunMerges;
+    uint64_t MaxDataReadSpanSectors;
+} IO_FILESYSTEM_STATS;
+
+// Stable identity for a file on a mounted filesystem.
+// Filesystems that can provide this let higher layers cache file-backed
+// objects by file identity instead of by one FILE_OBJECT instance.
+typedef struct IO_FILE_IDENTITY {
+    uint64_t VolumeId;
+    uint64_t FileId;
+    uint64_t Generation;
+    uint64_t Size;
+} IO_FILE_IDENTITY;
+
 // Filesystem driver vtable - filled in by a filesystem driver.
 typedef struct IO_FILESYSTEM {
     // Open a file by absolute path (e.g. "apps\\shell.exe").
@@ -69,14 +102,21 @@ typedef struct IO_FILESYSTEM {
 
     // Truncate or extend the file to exactly 'newSize' bytes.
     KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx,
-                        uint64_t newSize);
+                        uint64_t newSize, uint32_t fileFlags);
 
     // Return the file size in bytes.
     uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx);
 
+    // Return a stable file identity for caches that outlive one open handle.
+    KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx,
+                             IO_FILE_IDENTITY* out);
+
     // Release any resources the driver allocated for fileCtx.
     void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx);
 
+    // Query cache and traversal statistics exposed through kernel stats.
+    void (*QueryStats)(struct IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out);
+
     // Optional driver-private data.
     void* DriverCtx;
 
@@ -106,9 +146,13 @@ typedef struct IO_STATUS_BLOCK {
 
 struct EVENT;
 struct KAPC;
+typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
 typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
 
 // FILE_OBJECT - the kernel object returned to callers.
+#define IO_FILE_FLAG_NO_CACHE        0x00000001u
+#define IO_FILE_FLAG_SKIP_ZERO_EXTEND 0x00000002u
+
 typedef struct FILE_OBJECT {
     OBJECT        Header;          // must be first
     IO_FILESYSTEM* Fs;
@@ -116,6 +160,11 @@ typedef struct FILE_OBJECT {
     SPINLOCK       Lock;           // protects cached Size/Offset updates
     uint64_t       Size;
     uint64_t       Offset;         // current position for sequential reads
+    uint32_t       Flags;          // IO_FILE_FLAG_*
+    uint32_t       ReadaheadScore;
+    uint64_t       ReadaheadNextOffset;
+    uint64_t       ReadaheadIssuedUntil;
+    uint64_t       ReadaheadSuppressedUntil;
 } FILE_OBJECT;
 
 // DIR_OBJECT - kernel object for an open directory iterator.
@@ -137,6 +186,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESY
 __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
 __declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
 __declspec(dllexport) bool    IoFilesystemPresent(uint8_t driveIndex);
+__declspec(dllexport) void    IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out);
 
 // Kernel-side API.
 __declspec(dllexport) KSTATUS  IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out);
@@ -149,15 +199,23 @@ __declspec(dllexport) KSTATUS  IoWriteFile  (FILE_OBJECT* file, const void* buf,
 __declspec(dllexport) KSTATUS  IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
                                              const void* buf, size_t size,
                                              size_t* bytesWritten);
+__declspec(dllexport) KSTATUS  IoFlushFile(FILE_OBJECT* file);
 __declspec(dllexport) KSTATUS  IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf,
                     size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
                     struct EVENT* completionEvent, struct KAPC* completionApc,
+                    MM_LOCKED_USER_BUFFER* lockedBuffer,
+                    MM_LOCKED_USER_BUFFER* lockedIosb,
                     IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
 __declspec(dllexport) KSTATUS  IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
                     size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
                     struct EVENT* completionEvent, struct KAPC* completionApc,
+                    MM_LOCKED_USER_BUFFER* lockedBuffer,
+                    MM_LOCKED_USER_BUFFER* lockedIosb,
                     IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
 __declspec(dllexport) OBJECT*  IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* request);
+__declspec(dllexport) KSTATUS  IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
+                    uint32_t flags);
+__declspec(dllexport) struct EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* request);
 __declspec(dllexport) KSTATUS  IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request);
 __declspec(dllexport) KSTATUS  IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
                     IO_STATUS_BLOCK* iosbOut);
@@ -169,6 +227,7 @@ __declspec(dllexport) KSTATUS  IoRemoveDir  (SYS_STRING_VIEW path);
 __declspec(dllexport) KSTATUS  IoTruncateFile(FILE_OBJECT* file, uint64_t newSize);
 __declspec(dllexport) KSTATUS  IoSeekFile(FILE_OBJECT* file, uint64_t offset);
 __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
+__declspec(dllexport) KSTATUS  IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out);
 __declspec(dllexport) void     IoCloseFile(FILE_OBJECT* file);
 
 // Directory enumeration API.

code/kernel/include/kernel/ke_debug.h

@@ -12,6 +12,16 @@
 #define KE_DEBUG_SELFTEST  0x00000002u
 #define KE_DEBUG_FILE_LOG  0x00000004u
 
+#define KE_DEBUG_POOL_CANARY      0x00000008u
+#define KE_DEBUG_POOL_POISON      0x00000010u
+#define KE_DEBUG_POOL_QUARANTINE  0x00000020u
+#define KE_DEBUG_POOL_CALLERS     0x00000040u
+#define KE_DEBUG_DISPLAY_ASSERTS  0x00000080u
+#define KE_DEBUG_DISPLAY_TRACE    0x00000100u
+#define KE_DEBUG_INPUT_TRACE      0x00000200u
+#define KE_DEBUG_VM_ASSERTS       0x00000400u
+#define KE_DEBUG_SYSCALL_ASSERTS  0x00000800u
+
 void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
 void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t));
 uint32_t KeGetDebugFlags(void);
@@ -68,7 +78,7 @@ typedef struct KFAULT_FRAME {
     uint64_t R15;
 } KFAULT_FRAME;
 
-__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame);
+__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame);
 
 // Compatibility wrapper for any older assembly/object still calling this.
 __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,

code/kernel/include/kernel/ke_display.h

@@ -42,7 +42,9 @@ DISPLAY_API void KeDisplaySuspendConsole(bool suspend);
 
 DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode);
 DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight);
+typedef KSTATUS (*KE_DISPLAY_ROW_SOURCE)(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount);
 DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels);
+DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch);
 DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color);
 DISPLAY_API KSTATUS KeDisplayClear(uint32_t color);
 DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h);

code/kernel/include/kernel/ke_event.h

@@ -16,9 +16,12 @@ typedef struct EVENT {
     EVENT_TYPE   Type;
     uint32_t     Signaled;
     LIST_ENTRY   WaitList;
+    LIST_ENTRY   MultiWaitList;
 } EVENT;
 
 __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out);
 __declspec(dllexport) void    KeSignalEvent(EVENT* event);
 __declspec(dllexport) void    KeClearEvent(EVENT* event);
 __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable);
+__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event);
+__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count, uint8_t alertable, uint64_t timeoutMs, uint32_t* signaledIndexOut);

code/kernel/include/kernel/ke_input.h

@@ -6,6 +6,13 @@
 #include <ntlite/input.h>
 #include <ntlite/ddi/hal_api.h>
 #include <ntlite/status.h>
+#include <kernel/ob_object.h>
+#include <kernel/ke_event.h>
+
+typedef struct INPUT_QUEUE {
+    OBJECT Header;
+    EVENT* WaitEvent;
+} INPUT_QUEUE;
 
 void KeInputInitialize(const HAL_API* hal);
 
@@ -16,7 +23,12 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
 __declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons);
 
 INPUT_KEY KeInputKbPoll(void);
+KSTATUS   KeInputWaitKey(uint8_t alertable);
 bool      KeInputIsKeyDown(INPUT_KEY key);
+bool      KeInputPeekEvent(INPUT_KEY_RECORD* record);
+bool      KeInputReadEvent(INPUT_KEY_RECORD* record);
+KSTATUS   KeInputOpenQueue(INPUT_QUEUE** out);
+EVENT*    KeInputQueueWaitEvent(INPUT_QUEUE* queue);
 INPUT_KEY KeInputReadKey(void);
 char      KeInputReadChar(void);
 uint32_t  KeInputGetModifiers(void);

code/kernel/include/kernel/ke_mutex.h

@@ -14,7 +14,7 @@ typedef struct MUTEX {
     LIST_ENTRY   WaitList;
 } MUTEX;
 
-KSTATUS KeMutexCreate(MUTEX **out);
+__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out);
-KSTATUS KeAcquireMutex(MUTEX* mutex);
+__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex);
-KSTATUS KeTryAcquireMutex(MUTEX* mutex);
+__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex);
-void    KeReleaseMutex(MUTEX* mutex);
+__declspec(dllexport) void    KeReleaseMutex(MUTEX* mutex);

code/kernel/include/kernel/ke_process.h

@@ -62,6 +62,7 @@ typedef struct PROCESS {
     int          ExitCode;
     volatile long Terminating;
     volatile long Exited;
+    volatile long ResourcesReleased;
 
     // Main executable image. Used for cleanup.
     uintptr_t    ImageBase;
@@ -80,8 +81,8 @@ typedef struct PROCESS {
     // Per-process user virtual regions. The actual VA arena is global because
     // NT-Lite currently uses one shared address space; this list tracks
     // ownership so leaked user mappings can be reclaimed on process exit.
-    LIST_ENTRY   UserRegions;
+    LIST_ENTRY   VadList;
-    SPINLOCK     UserRegionLock;
+    SPINLOCK     VadLock;
 
     // Per-process loaded modules (user processes only).
     PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
@@ -102,6 +103,7 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* process,
                                         SYS_STRING_VIEW currentDirectory);
 void    KeExitProcess(PROCESS* process, int exitCode);
 KSTATUS KeTerminateProcess(PROCESS* process, int exitCode);
+void    KeCompleteProcessExitIfLastThread(PROCESS* process);
 int     KeIsProcessTerminating(PROCESS* process);
 KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut);
 
@@ -118,8 +120,12 @@ LIST_ENTRY* KeGetProcessList(void);
 PROCESS*    KeGetCurrentProcess(void);
 PROCESS*    KeGetSystemProcess(void);
 struct SYS_PROCESS_INFO;
+struct SYS_THREAD_INFO;
 
 // Copy a snapshot of rich process information into caller-provided storage.
 size_t      KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount);
+
+// Copy a snapshot of one process's threads into caller-provided storage.
+KSTATUS     KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut);
 // Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject.
 PROCESS*    KeGetProcessById(uint32_t pid);

code/kernel/include/kernel/ke_sched.h

@@ -58,6 +58,10 @@ void    KiSchedulerNoteVoluntaryBlock(THREAD* thread);
 void    KiSchedulerMarkThreadWaiting(THREAD* thread);
 void    KiSchedulerWakeThread(THREAD* thread, KSTATUS status);
 
+// Blocks the current thread until at least the requested delay has elapsed.
+__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds);
+KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus);
+
 // Voluntarily yield the CPU. Picks the next runnable thread immediately.
 __declspec(dllexport) void KeYield(void);
 

code/kernel/include/kernel/ke_system_root.h

@@ -0,0 +1,25 @@
+// System root volume declarations. This file exposes the one selected kernel root volume to loader and process code.
+
+#pragma once
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <ntlite/status.h>
+#include <kernel/io_file.h>
+
+#define KE_SYSTEM_ROOT_MAX_DRIVES 8
+
+// Selects the mounted volume that contains the kernel/system tree.
+KSTATUS KeInitializeSystemRootVolume(void);
+
+// Returns whether the system root volume has been selected.
+bool KeSystemRootReady(void);
+
+// Returns the selected system root drive, or zero before discovery.
+uint8_t KeGetSystemRootDrive(void);
+
+// Builds an absolute path rooted at the selected system volume.
+bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize);
+
+// Opens a file rooted at the selected system volume.
+KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out);

code/kernel/include/kernel/ke_thread.h

@@ -14,6 +14,8 @@
 #define KSTACK_SIZE   (64  * 1024)  // 64 KB kernel stack
 #define USTACK_SIZE   (512 * 1024)  // 512 KB user stack
 
+typedef struct THREAD THREAD;
+
 typedef enum THREAD_PRIORITY {
     PriorityLow    = 0,
     PriorityNormal = 1,
@@ -28,6 +30,10 @@ typedef enum THREAD_STATE {
     ThreadDead    = 3,
 } THREAD_STATE;
 
+typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread,
+                                        void* context,
+                                        KSTATUS status);
+
 // Saved register context - matches the layout pushed by KeSwitchContext.
 // The saved RSP points to this structure on the kernel stack.
 typedef struct KCONTEXT {
@@ -36,7 +42,7 @@ typedef struct KCONTEXT {
     uint64_t Rip;  // return address back into KeSwitchContext
 } KCONTEXT;
 
-typedef struct THREAD {
+struct THREAD {
     OBJECT          Header;  // must be first
 
     uint32_t        ThreadId;
@@ -82,7 +88,7 @@ typedef struct THREAD {
 
     // Signaled (manual-reset) when the thread exits.
     // Allocated in KeCreateThread, signaled in KeExitThread before teardown.
-    // Waiters block on this via SYS_WAIT_THREAD -> KeWaitForEvent.
+    // Waiters block on this through waitable thread handles.
     EVENT*          ExitEvent;
 
     void*   FpuContext;
@@ -113,6 +119,12 @@ typedef struct THREAD {
     KSTATUS WaitStatus;
     LIST_ENTRY* WaitListHead;
     uint8_t AlertableWait;
+    uint64_t SleepUntilTick;
+    KSTATUS TimeoutStatus;
+
+    // Optional cleanup hook for waits that use auxiliary wait blocks.
+    KE_WAIT_RUNDOWN_ROUTINE WaitRundownRoutine;
+    void* WaitRundownContext;
 
     // Kernel APC state. APCs are queued to a target thread and delivered
     // when that thread reaches PASSIVE_LEVEL with APC delivery enabled.
@@ -135,15 +147,22 @@ typedef struct THREAD {
     uintptr_t  UserApcDispatcher;
     uint8_t    UserApcDeliverable;
     uint8_t    UserApcDeliveryActive;
-} THREAD;
+};
 
+// Creates and queues a user thread.  On success, *out receives a referenced
+// THREAD pointer owned by the caller; release it with ObDereferenceObject()
+// after copying any needed IDs/state.  The live scheduler/thread lifetime is
+// separate and remains valid until the thread exits and is reaped.
 __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
                        THREAD_PRIORITY priority, THREAD** out);
 
 // Create a kernel-mode-only thread. fn is called directly in ring 0 with
 // arg; when fn returns the thread exits automatically. No user stack or TEB
 // is allocated. The thread is owned by the system process and runs at the
-// given priority.
+// given priority. On success, non-NULL out receives a referenced THREAD
+// pointer owned by the caller; release it with ObDereferenceObject() after
+// copying any needed IDs/state or after joining/waiting on caller-owned
+// completion state. The scheduler's live thread reference is separate.
 __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
                               THREAD_PRIORITY priority, THREAD** out);
 __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
@@ -167,9 +186,14 @@ __declspec(dllexport) void    KeExitThread(THREAD* thread, int code);
 // waiting. Alertable waits are explicit at every call site.
 void    KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
 void    KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead);
+void    KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context);
+void    KeClearThreadWaitRundown(THREAD* thread);
+void    KeRundownThreadWait(THREAD* thread, KSTATUS status);
 KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
 KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
 void    KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status);
+void    KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status);
+void    KeAbortThreadWait(THREAD* thread, KSTATUS status);
 void    KeUnblockThread(THREAD* thread);
 KSTATUS KeAlertThread(THREAD* thread);
 void    KiWakeThreadForKernelApc(THREAD* thread);

code/kernel/include/kernel/ke_work.h

@@ -0,0 +1,24 @@
+// Kernel passive-level work queue declarations.
+
+#pragma once
+#include <stdbool.h>
+#include <kernel/ke_list.h>
+#include <ntlite/status.h>
+
+// Work items are caller-owned embedded objects.  They may be queued from
+// interrupt/DPC/spinlock context and run later on a persistent system worker
+// thread at PASSIVE_LEVEL.  Queueing the same item while it is already queued
+// is coalesced.
+typedef void (*KWORK_ROUTINE)(void* context);
+
+typedef struct KWORK_ITEM {
+    LIST_ENTRY Link;
+    KWORK_ROUTINE Routine;
+    void* Context;
+    volatile long Queued;
+} KWORK_ITEM;
+
+void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context);
+KSTATUS KeStartWorkQueue(void);
+bool KeIsWorkQueueStarted(void);
+bool KeQueueWorkItem(KWORK_ITEM* item);

code/kernel/include/kernel/ldr_image.h

@@ -9,11 +9,12 @@
 #include <ntlite/status.h>
 
 struct PROCESS;
+struct FILE_OBJECT;
 
-// Map a PE32+ image from 'buffer' into the kernel virtual address space.
+// Map a PE32+ image from a bootloader/driver raw buffer into kernel space.
-// On success, *baseOut receives the virtual base address.
+// User images must be loaded from FILE_OBJECTs so image sections have stable identity.
-KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user);
+KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut);
-KSTATUS LdrLoadImageForProcess(struct PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut);
+KSTATUS LdrLoadImageFileForProcess(struct PROCESS* process, struct FILE_OBJECT* file, uintptr_t* baseOut);
 
 // Return the SizeOfImage recorded in the PE header for an already-mapped image.
 size_t LdrGetImageSize(uintptr_t base);

code/kernel/include/kernel/ldr_section.h

@@ -0,0 +1,67 @@
+// Kernel image section declarations shared by loader and memory-management code.
+
+#pragma once
+
+#include <stdint.h>
+#include <stddef.h>
+#include <ntlite/status.h>
+#include <kernel/mm_phys.h>
+
+struct PROCESS;
+struct FILE_OBJECT;
+typedef struct PROCESS PROCESS;
+typedef struct FILE_OBJECT FILE_OBJECT;
+typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT;
+
+// Describes one PE section in image-address-space coordinates.
+typedef struct IMAGE_SECTION_DESCRIPTOR {
+    char     Name[9];
+    uint32_t VirtualAddress;
+    uint32_t VirtualSize;
+    uint32_t RawDataOffset;
+    uint32_t RawDataSize;
+    uint32_t Characteristics;
+    uint64_t PageProtection;
+} IMAGE_SECTION_DESCRIPTOR;
+
+// Keeps the parsed identity and layout for a PE image section object.
+typedef struct IMAGE_SECTION_METADATA {
+    uint64_t ContentKey;
+    uint64_t VolumeId;
+    uint64_t FileId;
+    uint64_t FileGeneration;
+    uint64_t PreferredBase;
+    uint32_t EntryRva;
+    uint32_t HeaderSize;
+    size_t   RawSize;
+    size_t   ImageSize;
+    size_t   PageCount;
+    uint32_t SectionCount;
+} IMAGE_SECTION_METADATA;
+
+// Creates or references an image section object backed by an open file.
+KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file,
+                                      IMAGE_SECTION_OBJECT** sectionOut);
+
+// Adds a reference to an existing image section object.
+void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section);
+
+// Releases a reference returned by LdrCreateImageSectionFromFile or LdrReferenceImageSection.
+void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section);
+
+// Returns immutable metadata for an image section object.
+const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section);
+
+// Returns the immutable PE section table owned by the image section object.
+const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section,
+                                                          uint32_t* countOut);
+
+// Retains one prototype frame owned by an image section object.
+KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section,
+                                  size_t pageIndex,
+                                  PHYSICAL_ADDRESS* frameOut);
+
+// Maps an image section into a process as an image-backed VAD.
+KSTATUS LdrMapImageSection(PROCESS* process,
+                           IMAGE_SECTION_OBJECT* section,
+                           void** baseOut);

code/kernel/include/kernel/mm_file_cache.h

@@ -0,0 +1,99 @@
+// Kernel file-page cache declarations.
+// Caches PAGE_SIZE file pages shared by reads, mmap faults, and writeback.
+
+#pragma once
+
+#include <stdint.h>
+#include <stddef.h>
+#include <kernel/mm_phys.h>
+#include <ntlite/status.h>
+
+struct FILE_OBJECT;
+typedef struct FILE_OBJECT FILE_OBJECT;
+
+typedef struct MM_FILE_CACHE_PAGE {
+    FILE_OBJECT*      File;
+    uint64_t          Offset;
+    PHYSICAL_ADDRESS  Frame;
+    uint32_t          ValidBytes;
+    uint8_t           Dirty;
+} MM_FILE_CACHE_PAGE;
+
+typedef struct MM_FILE_CACHE_STATS {
+    uint64_t Lookups;
+    uint64_t Hits;
+    uint64_t Misses;
+    uint64_t Loads;
+    uint64_t LoadFailures;
+    uint64_t RaceDiscards;
+    uint64_t InflightWaits;
+    uint64_t Invalidations;
+    uint64_t DirtyPages;
+    uint64_t MaxDirtyPages;
+    uint64_t DirtyMarks;
+    uint64_t WritebackRequests;
+    uint64_t WritebackPages;
+    uint64_t WritebackBytes;
+    uint64_t WritebackFailures;
+    uint64_t FlushFileCalls;
+    uint64_t FlushFileFailures;
+    uint64_t MappedDirtySyncs;
+    uint64_t MappedDirtyPages;
+    uint64_t Drops;
+    uint64_t DropFailures;
+    uint64_t StandbyTransitions;
+    uint64_t StandbyHits;
+    uint64_t ReadAheadRequests;
+    uint64_t ReadAheadPagesRequested;
+    uint64_t ReadAheadPagesQueued;
+    uint64_t ReadAheadPagesLoaded;
+    uint64_t ReadAheadPagesSkipped;
+    uint64_t ReadAheadFailures;
+    uint64_t ReadAheadPagesThrottled;
+    uint64_t MaxReadAheadPagesPerRequest;
+    uint64_t BatchReads;
+    uint64_t BatchPagesRequested;
+    uint64_t BatchPagesLoaded;
+    uint64_t MaxBatchPagesRequested;
+    uint64_t MaxBatchPagesLoaded;
+    uint64_t ResidentPages;
+    uint64_t ActivePages;
+    uint64_t StandbyPages;
+    uint64_t MaxResidentPages;
+} MM_FILE_CACHE_STATS;
+
+// Returns a retained physical frame containing the file page at page-aligned
+// offset. The caller owns one PFN reference and must call
+// MmFileCacheReleasePage(). Bytes beyond ValidBytes are zero-filled.
+KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAGE* outPage);
+void    MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page);
+
+// Returns a retained coherent cache page suitable for a shared writable mapping.
+KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
+                                   uint64_t fileSizeBefore, int needExistingBytes,
+                                   MM_FILE_CACHE_PAGE* outPage);
+
+// Marks one coherent cache page dirty after a shared writable mapping is written.
+void MmFileCacheMarkDirty(FILE_OBJECT* file, uint64_t offset, uint32_t validBytes);
+
+// Writes into coherent cached file pages and marks them dirty.
+KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf,
+                         size_t size, uint64_t fileSizeBefore, size_t* writtenOut);
+
+// Flushes dirty cached pages for one file.
+KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file);
+
+// Drops cached clean pages overlapping the byte range. Dirty pages are flushed
+// by callers before destructive metadata operations.
+void    MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size);
+void    MmFileCacheInvalidateAll(FILE_OBJECT* file);
+
+// Reclaims one cache-owned page if possible. Safe to call under memory pressure;
+// it never writes back because v1 stores only clean file data.
+KSTATUS MmFileCacheDropOnePageForPressure(void);
+
+// Queues best-effort read-ahead for clean file pages. The queue owns a file
+// reference until the worker has consumed the request.
+KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages);
+
+void    MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out);

code/kernel/include/kernel/mm_pfn.h

@@ -0,0 +1,134 @@
+// Physical frame database declarations.
+// The PFN layer owns frame state, reference counts, pin counts, and allocation
+// policy. Higher VMM layers should ask for pages by purpose instead of treating
+// the physical allocator as an untyped bitmap.
+
+#pragma once
+
+#include <stdint.h>
+#include <stddef.h>
+#include <ntlite/status.h>
+#include <kernel/mm_phys.h>
+
+#define MM_PFN_INVALID_INDEX ((size_t)-1)
+
+typedef enum MM_PFN_STATE {
+    MmPfnStateInvalid = 0,
+    MmPfnStateReserved,
+    MmPfnStateFree,
+    MmPfnStateZeroed,
+    MmPfnStateActive,
+    MmPfnStateStandby,
+    MmPfnStateModified,
+} MM_PFN_STATE;
+
+typedef enum MM_PFN_USAGE {
+    MmPfnUseNone = 0,
+    MmPfnUseKernel,
+    MmPfnUsePageTable,
+    MmPfnUseUser,
+    MmPfnUseCleanFileCache,
+    MmPfnUseImage,
+    MmPfnUseMappedFile,
+    MmPfnUseAnonymous,
+    MmPfnUseCow,
+    MmPfnUseSurface,
+    MmPfnUseDriver,
+    MmPfnUseDma,
+    MmPfnUseBootloader,
+    MmPfnUsePfnDatabase,
+    MmPfnUseMmio,
+} MM_PFN_USAGE;
+
+typedef struct MM_PFN {
+    uint16_t State;
+    uint16_t Usage;
+    uint16_t RefCount;
+    uint16_t PinCount;
+    uint32_t Flags;
+    uint32_t MappedCount;
+    void*    Owner;
+    uint64_t BackingPage;
+} MM_PFN;
+
+typedef struct MM_PFN_STATS {
+    uint64_t TotalFrames;
+    uint64_t FreeFrames;
+    uint64_t UsedFrames;
+
+    uint64_t ReservedFrames;
+    uint64_t ZeroedFrames;
+    uint64_t ActiveFrames;
+    uint64_t StandbyFrames;
+    uint64_t ModifiedFrames;
+    uint64_t PinnedFrames;
+    uint64_t EvictableFrames;
+    uint64_t NonEvictableFrames;
+    uint64_t ReclaimableFrames;
+    uint64_t AvailableFrames;
+    uint64_t MappedFrames;
+
+    uint64_t KernelFrames;
+    uint64_t PageTableFrames;
+    uint64_t UserFrames;
+    uint64_t CleanFileCacheFrames;
+    uint64_t ActiveCleanFileCacheFrames;
+    uint64_t StandbyCleanFileCacheFrames;
+    uint64_t ImageFrames;
+    uint64_t ActiveImageFrames;
+    uint64_t StandbyImageFrames;
+    uint64_t MappedFileFrames;
+    uint64_t AnonymousFrames;
+    uint64_t CowFrames;
+    uint64_t SurfaceFrames;
+    uint64_t DriverFrames;
+    uint64_t DmaFrames;
+    uint64_t BootloaderFrames;
+    uint64_t PfnDatabaseFrames;
+    uint64_t MmioFrames;
+
+    uint64_t AllocationCount;
+    uint64_t FreeCount;
+    uint64_t RetainCount;
+    uint64_t ReleaseCount;
+    uint64_t PinCount;
+    uint64_t UnpinCount;
+    uint64_t FailedAllocations;
+    uint64_t FailedContiguousAllocations;
+
+    uint64_t FailedInvalidRange;
+    uint64_t FailedEmptyRange;
+    uint64_t FailedReserveDenied;
+    uint64_t FailedNoFrame;
+
+    uint64_t FailedContiguousInvalidRequest;
+    uint64_t FailedContiguousReserveDenied;
+    uint64_t FailedContiguousNoRun;
+
+    uint64_t ReclaimAttempts;
+    uint64_t ReclaimPagesRequested;
+    uint64_t ReclaimPagesSucceeded;
+    uint64_t ReclaimPagesFailed;
+    uint64_t ReclaimReentrantSkips;
+} MM_PFN_STATS;
+
+__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage);
+__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage);
+__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive,
+                                      PHYSICAL_ADDRESS maxExclusive,
+                                      MM_PFN_USAGE usage);
+__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage);
+__declspec(dllexport) KSTATUS          MmPfnFree(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnRetain(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnRelease(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnPin(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnUnpin(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnAddMapping(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnRemoveMapping(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnMoveToStandby(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) KSTATUS          MmPfnActivate(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) MM_PFN*          MmPfnGet(PHYSICAL_ADDRESS frame);
+__declspec(dllexport) void*           MmPfnToVirtual(PHYSICAL_ADDRESS frame);
+size_t           MmPfnIndex(PHYSICAL_ADDRESS frame);
+PHYSICAL_ADDRESS MmPfnAddress(size_t pfn);
+void             MmPfnQueryStats(MM_PFN_STATS* out);

code/kernel/include/kernel/mm_phys.h

@@ -1,8 +1,9 @@
-// Kernel mm phys declarations shared by implementation files and kernel clients.
+// Physical-memory declarations shared by the kernel and drivers.
+// The physical layer publishes page-size constants, boot memory-map ingestion,
+// and aggregate memory statistics. Frame ownership lives in the PFN database.
 
 #pragma once
-// mm_phys.h - physical memory manager (PMM)
+
-// Bitmap allocator over 4 KB frames, seeded from MEMORY_MAP.
 #include <stdint.h>
 #include <stddef.h>
 #include <stdbool.h>
@@ -22,22 +23,7 @@ typedef struct MM_PHYSICAL_STATS {
     uint64_t FreeRunHistogram[8];
 } MM_PHYSICAL_STATS;
 
-// Called once by KernelMain with the MEMORY_MAP from BOOT_INFO.
+bool   MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase);
-bool             MmInitializePhysical(const MEMORY_MAP* map,
-                                      uint64_t physicalWindowBase);
-
-// Allocate / free a single 4 KB frame.
-// MmAllocateFrame returns 0 on failure.
-__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void);
-PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive);
-PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
-                                        PHYSICAL_ADDRESS maxExclusive);
-void             MmFreeFrame(PHYSICAL_ADDRESS frame);
-
-// Allocate 'count' physically contiguous 4 KB frames.
-// Returns 0 on failure. Use sparingly (DMA, page tables).
-__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateContiguous(size_t count);
-
 size_t MmFreeFrameCount(void);
 size_t MmTotalFrameCount(void);
 void   MmQueryPhysicalStats(MM_PHYSICAL_STATS* out);

code/kernel/include/kernel/mm_pte.h

@@ -0,0 +1,105 @@
+// Kernel page-table manager declarations shared by the VMM and low-level kernel clients.
+#pragma once
+
+#include <stdint.h>
+#include <stddef.h>
+#include <kernel/mm_phys.h>
+#include <ntlite/status.h>
+
+// x86-64 PTE bits exposed to the rest of the kernel.
+#define PAGE_PRESENT      (1ULL << 0)
+#define PAGE_WRITE        (1ULL << 1)
+#define PAGE_USER         (1ULL << 2)
+#define PAGE_WRITETHROUGH (1ULL << 3)
+#define PAGE_NOCACHE      (1ULL << 4)
+#define PAGE_ACCESSED     (1ULL << 5)
+#define PAGE_DIRTY        (1ULL << 6)
+#define PAGE_NX           (1ULL << 63)
+
+// The physical direct map is a kernel-only alias used to reach page-table frames and early physical memory.
+#define PHYS_MAP_BASE     0xFFFF880000000000ULL
+
+
+// Page-table telemetry separates cumulative operations from the current number
+// of 4 KiB leaf mappings owned by this layer. Huge boot mappings may exist but
+// are not counted as leaves here.
+typedef enum MM_PTE_QUERY_PURPOSE {
+    MmPteQueryGeneric = 0,
+    MmPteQueryUserProbe,
+    MmPteQueryHeap,
+    MmPteQueryVad,
+    MmPteQueryDebug,
+    MmPteQuerySelfTest,
+    MmPteQueryPurposeCount,
+} MM_PTE_QUERY_PURPOSE;
+
+typedef struct MM_PTE_STATS {
+    uint64_t MapCalls;
+    uint64_t MapFailures;
+    uint64_t UnmapCalls;
+    uint64_t UnmapFailures;
+    uint64_t ProtectCalls;
+    uint64_t ProtectFailures;
+    uint64_t QueryCalls;
+    uint64_t QueryFailures;
+    uint64_t QueryByPurpose[MmPteQueryPurposeCount];
+    uint64_t QueryFailuresByPurpose[MmPteQueryPurposeCount];
+    uint64_t TranslateCalls;
+    uint64_t LocalFlushes;
+    uint64_t GlobalFlushes;
+    uint64_t ShootdownRequests;
+    uint64_t ShootdownTargetCpus;
+    uint64_t ShootdownSkippedNoTargets;
+    uint64_t ShootdownAcks;
+    uint64_t ShootdownTimeouts;
+    uint64_t NewMappingNoFlushes;
+    uint64_t ReplacedMappingFlushes;
+    uint64_t PageTablePagesAllocated;
+    uint64_t CurrentLeafMappings;
+} MM_PTE_STATS;
+
+typedef enum MM_PTE_MAP_MODE {
+    MmPteMapFailIfPresent = 0,
+    MmPteMapReplaceExisting = 1,
+} MM_PTE_MAP_MODE;
+
+// Initializes the page-table manager around the bootloader-created PML4.
+void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys);
+
+// Installs the shared PML4 into CR3.
+void MmPteActivate(void);
+
+// Maps a 4 KiB page into the shared address space.
+__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode);
+
+// Removes a 4 KiB mapping from the shared address space.
+__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt);
+
+// Replaces the flags on every mapped 4 KiB page in the range while preserving the physical frame.
+__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags);
+
+// Tests and clears the hardware accessed bit on one mapped 4 KiB leaf.
+// Returns nonzero when the page had been accessed and was given a second chance.
+__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt);
+
+// Tests and clears the hardware dirty bit on one mapped 4 KiB leaf.
+__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt);
+
+// Translates a virtual address through the current shared page tables.
+__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt);
+
+// Returns the physical address and effective flags for a mapped virtual address.
+__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
+KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose, uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
+
+// Flushes one address from the local TLB.
+void MmPteFlushLocal(uintptr_t virt);
+
+// Flushes a kernel mapping from every online CPU that shares the kernel page tables.
+void MmPteFlushKernelAddress(uintptr_t virt);
+
+// Handles the fixed-vector NMI used for kernel TLB shootdowns.
+int MmPteHandleShootdownNmi(void);
+
+// Returns page-table operation counters used by memory telemetry.
+void MmPteQueryStats(MM_PTE_STATS* out);

code/kernel/include/kernel/mm_swap.h

@@ -0,0 +1,31 @@
+// Kernel swap manager declarations.
+// Swap owns pagefile slot allocation and page-sized I/O for anonymous VAD pages.
+
+#pragma once
+#include <stdint.h>
+#include <stddef.h>
+#include <ntlite/status.h>
+#include <ntlite/string_view.h>
+#include <kernel/mm_phys.h>
+
+typedef struct MM_SWAP_STATS {
+    uint64_t Enabled;
+    uint64_t TotalSlots;
+    uint64_t UsedSlots;
+    uint64_t FreeSlots;
+    uint64_t HighWaterSlots;
+    uint64_t PageOuts;
+    uint64_t PageIns;
+    uint64_t PageOutFailures;
+    uint64_t PageInFailures;
+    uint64_t SlotAllocations;
+    uint64_t SlotFrees;
+    uint64_t SlotAllocationFailures;
+} MM_SWAP_STATS;
+
+KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes);
+int     MmSwapIsEnabled(void);
+KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut);
+KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame);
+void    MmSwapFreeSlot(uint32_t slot);
+void    MmSwapQueryStats(MM_SWAP_STATS* out);

code/kernel/include/kernel/mm_virt.h

@@ -2,50 +2,43 @@
 
 #pragma once
 // mm_virt.h - virtual memory manager (VMM)
-// Manages the single shared PML4, kernel pool, and page-granular user VM regions.
+// Manages the single shared PML4, kernel pool, and VAD-backed user VM ownership.
 #include <stdint.h>
 #include <stddef.h>
 #include <kernel/mm_phys.h>
+#include <kernel/mm_pte.h>
 #include <ntlite/status.h>
 
 struct PROCESS;
 typedef struct PROCESS PROCESS;
+struct FILE_OBJECT;
+typedef struct FILE_OBJECT FILE_OBJECT;
+struct IMAGE_SECTION_OBJECT;
+typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT;
 
-typedef enum MM_MAP_MODE {
+typedef enum MM_VAD_TYPE {
-    MmMapFailIfPresent = 0,
+    MmVadGeneric = 0,
-    MmMapReplaceExisting = 1,
+    MmVadHeap,
-} MM_MAP_MODE;
+    MmVadImage,
+    MmVadStack,
+    MmVadTeb,
+    MmVadPeb,
+    MmVadEnvironment,
+    MmVadMappedFile,
+    MmVadShared,
+} MM_VAD_TYPE;
 
-typedef enum USER_REGION_TYPE {
+#define MM_VAD_GUARD_LOW   0x00000001u
-    UserRegionGeneric = 0,
+#define MM_VAD_GUARD_HIGH  0x00000002u
-    UserRegionHeap,
+#define MM_VAD_ZERO        0x00000004u
-    UserRegionImage,
+#define MM_VAD_PREFETCH    0x00000008u
-    UserRegionStack,
+#define MM_VAD_SHARED_WRITE 0x00000010u
-    UserRegionTeb,
-    UserRegionPeb,
-    UserRegionEnvironment,
-    UserRegionMappedFile,
-    UserRegionShared,
-} USER_REGION_TYPE;
-
-#define USER_REGION_GUARD_LOW   0x00000001u
-#define USER_REGION_GUARD_HIGH  0x00000002u
-#define USER_REGION_ZERO        0x00000004u
-
-// Page flags (x86-64 PTE bits).
-#define PAGE_PRESENT   (1ULL << 0)
-#define PAGE_WRITE     (1ULL << 1)
-#define PAGE_USER      (1ULL << 2)
-#define PAGE_WRITETHROUGH (1ULL << 3)
-#define PAGE_NOCACHE   (1ULL << 4)
-#define PAGE_NX        (1ULL << 63)
 
 // Virtual address space constants.
 #define USER_BASE          0x0000000000001000ULL
 #define USER_LIMIT         0x00007FFFFFFFF000ULL
 
 #define KERNEL_BASE        0xFFFF800000000000ULL
-#define PHYS_MAP_BASE      0xFFFF880000000000ULL
 #define POOL_BASE          0xFFFFC00000000000ULL
 #define POOL_LIMIT         0xFFFFE00000000000ULL
 
@@ -55,37 +48,6 @@ typedef enum USER_REGION_TYPE {
 // poolSize: initial reservation for kernel pool (bytes, page-aligned).
 void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
 
-// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
-void MmActivate(void);
-
-// Page mapping.
-__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode);
-__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt);
-
-// Modify protection flags on an already-mapped virtual range.
-// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags
-// entirely (except the physical address, which is preserved).
-// Returns STATUS_NOT_FOUND if any page in the range is not mapped.
-__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags);
-
-// Walk the page tables to translate a virtual address to physical.
-// Returns 0 if the address is not mapped.
-__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt);
-
-// TLB/cache coherency hooks for the shared kernel address space. Local flushes
-// use INVLPG directly; kernel mapping changes broadcast an NMI shootdown to
-// other online CPUs and wait for acknowledgements.
-void MmFlushTlbLocal(uintptr_t virt);
-void MmFlushTlbForKernelMapping(uintptr_t virt);
-int MmHandleTlbShootdownNmi(void);
-
-
-// User/kernel copy boundary helpers
-// MmQueryPage returns the physical address and effective PAGE_* bits for a
-// mapped virtual address. The reported PAGE_USER/PAGE_WRITE bits account for
-// every page-table level, not just the leaf entry.
-__declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
-
 // Probe a user virtual range without copying it. These functions only accept
 // lower-half user mappings with PAGE_USER set; MmProbeUserWrite additionally
 // requires PAGE_WRITE.
@@ -97,31 +59,96 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size);
 __declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size);
 __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size);
 
+typedef struct MM_PINNED_USER_PAGE {
+    uintptr_t        UserPage;
+    size_t           PageOffset;
+    PHYSICAL_ADDRESS Frame;
+    void*            KernelAddress;
+    uint64_t         PageFlags;
+} MM_PINNED_USER_PAGE;
+
+typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
+
+// Pins one user page and returns a kernel direct-map alias for the requested
+// address. Callers may copy through KernelAddress + PageOffset while the page is
+// pinned. The pin is short-lived: never sleep, do file I/O, or hold it across
+// arbitrary callbacks.
+__declspec(dllexport) KSTATUS MmPinUserPageForRead(PROCESS* process, const void* userAddress, MM_PINNED_USER_PAGE* out);
+__declspec(dllexport) KSTATUS MmPinUserPageForWrite(PROCESS* process, void* userAddress, MM_PINNED_USER_PAGE* out);
+__declspec(dllexport) void    MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
+
+// Locks a user buffer for asynchronous kernel access. The containing VADs stay
+// alive and protected, and resident PFNs stay pinned until the buffer is
+// unlocked. WriteAccess describes kernel writes to the user buffer.
+__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
+__declspec(dllexport) void    MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
+
 // Kernel pool - ring 0 only, never PAGE_USER.
 // Best-fit free-list allocator, thread-safe via spinlock.
+#define MM_POOL_TAG(a, b, c, d) \
+    ((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
+     ((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24))
+
 __declspec(dllexport) void* MmAllocatePool(size_t size);
+__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag);
 __declspec(dllexport) void  MmFreePool(void* ptr);
 __declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
 
-// User virtual regions.
+// Virtual address descriptors.
-// The kernel has one shared user VA arena, but every user mapping is owned by
+// The single shared address space still has per-process VAD ownership so process
-// a PROCESS so process teardown can reclaim leaked memory. Guard pages are
+// teardown can reclaim leaked mappings. VADs track sparse per-page commitment,
-// reserved VA with no present PTE.
+// and committed pages are materialized by user faults or explicit prefaulting.
-KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
+KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment,
-                            uint64_t pageFlags, uint32_t regionFlags,
+                     uint64_t pageFlags, uint32_t vadFlags,
-                            USER_REGION_TYPE type, void** outBase);
+                     MM_VAD_TYPE type, void** outBase);
-KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
+KSTATUS MmVadCommit(PROCESS* process, void* base, size_t size,
-KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
+                    uint64_t pageFlags, uint32_t commitFlags);
-KSTATUS MmDestroyUserRegion(PROCESS* process, void* base);
+KSTATUS MmVadCreateCommitted(PROCESS* process, size_t size, size_t alignment,
-void    MmDestroyAllUserRegions(PROCESS* process);
+                             uint64_t pageFlags, uint32_t vadFlags,
-KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, USER_REGION_TYPE* typeOut, uint32_t* regionFlagsOut);
+                             MM_VAD_TYPE type, void** outBase);
-void    MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t* reservedBytesOut, size_t* committedBytesOut);
+KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file,
-size_t  MmGetUserRegionSize(PROCESS* process, const void* base);
+                               uint64_t fileOffset, size_t size,
-size_t  MmGetUserRegionSizeByBase(const void* base);
+                               uint64_t pageFlags, uint32_t vadFlags,
+                               void** outBase);
+KSTATUS MmVadCreateImageSectionMapping(PROCESS* process, IMAGE_SECTION_OBJECT* section,
+                                       uint64_t pageFlags, uint32_t vadFlags,
+                                       void** outBase);
+KSTATUS MmVadEnsureResident(PROCESS* process, void* base, size_t size, uint64_t accessFlags);
+KSTATUS MmVadMakeImageRangePrivate(PROCESS* process, void* base, size_t size);
+KSTATUS MmVadSyncFileMappings(FILE_OBJECT* file);
+KSTATUS MmVadCreateCowView(PROCESS* sourceProcess, void* sourceBase, size_t size,
+                           PROCESS* targetProcess, void** outBase);
+KSTATUS MmVadCreateCowViewFromFrames(PROCESS* targetProcess,
+                                     const PHYSICAL_ADDRESS* frames,
+                                     size_t pageCount,
+                                     size_t payloadSize,
+                                     uint64_t pageFlags,
+                                     uint32_t vadFlags,
+                                     MM_VAD_TYPE type,
+                                     void** outBase);
+KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess,
+                                        const PHYSICAL_ADDRESS* frames,
+                                        size_t pageCount,
+                                        size_t payloadSize,
+                                        uint64_t pageFlags,
+                                        uint32_t vadFlags,
+                                        MM_VAD_TYPE type,
+                                        void** outBase);
+KSTATUS MmVadResolveUserPageFault(PROCESS* process, uintptr_t faultAddress, uint64_t errorCode);
+KSTATUS MmVadResolvePageFaultByAddress(uintptr_t faultAddress, uint64_t errorCode);
+KSTATUS MmVadTrimOnePageForTest(PROCESS* process, void** baseOut);
+KSTATUS MmVadTrimOnePageForPressure(void);
+KSTATUS MmVadProtect(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
+KSTATUS MmVadDestroy(PROCESS* process, void* base);
+void    MmVadDestroyAllForProcess(PROCESS* process);
+KSTATUS MmVadQuery(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, MM_VAD_TYPE* typeOut, uint32_t* vadFlagsOut);
+void    MmVadQueryProcessMemory(PROCESS* process, size_t* vadCountOut, size_t* reservedBytesOut, size_t* committedBytesOut);
+size_t  MmVadGetSize(PROCESS* process, const void* base);
+size_t  MmVadGetSizeByBase(const void* base);
 
 // Debug-only ownership/address diagnostics used by fault and launch paths.
 void MmDebugDescribeAddress(uintptr_t addr);
-void MmDebugDumpUserRegions(PROCESS* process);
+void MmVadDebugDumpProcess(PROCESS* process);
 
 __declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size);
 __declspec(dllexport) void  MmFreeUserMemoryForProcess(PROCESS* process, void* ptr);
@@ -140,3 +167,40 @@ __declspec(dllexport) void  MmFreeUserImage(void* ptr, size_t size);
 
 size_t MmGetKernelPoolUsage(void);
 size_t MmGetUserHeapUsage(void);
+
+// Demand-paging telemetry used by the shell stats command.
+typedef struct MM_VAD_STATS {
+    uint64_t ResolveFaultCalls;
+    uint64_t ResolvedFaults;
+    uint64_t FailedFaults;
+    uint64_t ProtectionFaults;
+    uint64_t NoVadFaults;
+    uint64_t UncommittedFaults;
+    uint64_t NoAccessFaults;
+    uint64_t DemandZeroPages;
+    uint64_t ZeroPageMaps;
+    uint64_t ZeroPageCowSplits;
+    uint64_t CowFaults;
+    uint64_t CowPagesCopied;
+    uint64_t CowViewsCreated;
+    uint64_t CowPagesShared;
+    uint64_t EnsureResidentCalls;
+    uint64_t EnsureResidentPages;
+    uint64_t AlreadyResidentPages;
+    uint64_t FileBackedFaults;
+    uint64_t FileBackedReads;
+    uint64_t FileBackedReadFailures;
+    uint64_t FileBackedDiscards;
+    uint64_t FileBackedSharedWritableFaults;
+    uint64_t FileBackedDirtyFaults;
+    uint64_t FileBackedDirtySyncs;
+    uint64_t ImageBackedFaults;
+    uint64_t ImageBackedZeroFaults;
+    uint64_t ImageBackedPrivateFaults;
+    uint64_t ImageFixupPrivateCalls;
+    uint64_t ImageFixupPrivatePages;
+    uint64_t SwapPageOuts;
+    uint64_t SwapPageIns;
+} MM_VAD_STATS;
+
+void MmVadQueryStats(MM_VAD_STATS* out);

code/kernel/include/kernel/ob_object.h

@@ -23,6 +23,10 @@ typedef enum OBJECT_TYPE {
     ObjectTypeDriver    = 12,
     ObjectTypeUsbDevice = 13,
     ObjectTypeUsbPipe   = 14,
+    ObjectTypeFileMapping = 15,
+    ObjectTypeSurface   = 16,
+    ObjectTypeInputQueue = 17,
+    ObjectTypeImageSection = 18,
 } OBJECT_TYPE;
 
 typedef struct OBJECT {

code/kernel/include/kernel/syscall_nr.h

@@ -11,13 +11,13 @@
 #define SYS_EXIT_PROCESS               0
 #define SYS_EXIT_THREAD                1
 #define SYS_CREATE_PROCESS             2
-#define SYS_WAIT_FOR_PROCESS           3
+#define SYS_WAIT_SINGLE                3
 #define SYS_GET_PROCESS_EXIT_CODE      4
 #define SYS_KILL_PROCESS               5
 #define SYS_GET_PROCESS_LIST           6
 #define SYS_GET_PROCESS_PID            7
 #define SYS_CREATE_THREAD              8
-#define SYS_WAIT_THREAD                9
+#define SYS_RETIRED_WAIT_THREAD        9
 #define SYS_GET_CURRENT_TID            10
 #define SYS_GET_CURRENT_PID            11
 #define SYS_SET_THREAD_PRIORITY        12
@@ -31,17 +31,17 @@
 #define SYS_QUERY_VM                   20
 #define SYS_CLOSE_HANDLE               21
 #define SYS_CREATE_MUTEX               22
-#define SYS_ACQUIRE_MUTEX              23
+#define SYS_RETIRED_ACQUIRE_MUTEX      23
 #define SYS_TRY_ACQUIRE_MUTEX          24
 #define SYS_RELEASE_MUTEX              25
 #define SYS_CREATE_SEM                 26
-#define SYS_WAIT_SEM                   27
+#define SYS_RETIRED_WAIT_SEM           27
 #define SYS_TRY_WAIT_SEM               28
 #define SYS_SIGNAL_SEM                 29
 #define SYS_CREATE_EVENT               30
 #define SYS_SIGNAL_EVENT               31
 #define SYS_CLEAR_EVENT                32
-#define SYS_WAIT_EVENT                 33
+#define SYS_RETIRED_WAIT_EVENT         33
 #define SYS_CREATE_QUEUE               34
 #define SYS_SEND_QUEUE                 35
 #define SYS_RECV_QUEUE                 36
@@ -74,36 +74,47 @@
 #define SYS_FILL_CONSOLE_CHARACTER      63
 #define SYS_FILL_CONSOLE_ATTRIBUTE      64
 #define SYS_PUT_SCREEN_CHAR             65
-#define SYS_BLIT_SCREEN                 66
+#define SYS_CLEAR_SCREEN                66
-#define SYS_CLEAR_SCREEN                67
+#define SYS_GET_SCREEN_SIZE             67
-#define SYS_GET_SCREEN_SIZE             68
+#define SYS_QUERY_DISPLAY_INFO          68
-#define SYS_QUERY_DISPLAY_INFO          69
+#define SYS_POLL_KEYBOARD               69
-#define SYS_POLL_KEYBOARD               70
+#define SYS_IS_KEY_DOWN                 70
-#define SYS_IS_KEY_DOWN                 71
+#define SYS_READ_KEYBOARD_CHAR          71
-#define SYS_READ_KEYBOARD_CHAR          72
+#define SYS_READ_KEY                    72
-#define SYS_READ_KEY                    73
+#define SYS_GET_KEY_MODIFIERS           73
-#define SYS_GET_KEY_MODIFIERS           74
+#define SYS_POLL_MOUSE                  74
-#define SYS_POLL_MOUSE                  75
+#define SYS_READ_MOUSE                  75
-#define SYS_READ_MOUSE                  76
+#define SYS_GET_MOUSE_POS               76
-#define SYS_GET_MOUSE_POS               77
+#define SYS_GET_TICK                    77
-#define SYS_GET_TICK                    78
+#define SYS_PRINT_DEBUG                 78
-#define SYS_PRINT_DEBUG                 79
+#define SYS_QUERY_TRAP_FRAME            79
-#define SYS_QUERY_TRAP_FRAME            80
+#define SYS_CREATE_SOCKET               80
-#define SYS_CREATE_SOCKET               81
+#define SYS_BIND_SOCKET                 81
-#define SYS_BIND_SOCKET                 82
+#define SYS_LISTEN_SOCKET               82
-#define SYS_LISTEN_SOCKET               83
+#define SYS_ACCEPT_SOCKET               83
-#define SYS_ACCEPT_SOCKET               84
+#define SYS_CONNECT_SOCKET              84
-#define SYS_CONNECT_SOCKET              85
+#define SYS_SEND_SOCKET                 85
-#define SYS_SEND_SOCKET                 86
+#define SYS_RECV_SOCKET                 86
-#define SYS_RECV_SOCKET                 87
+#define SYS_SHUTDOWN_SOCKET             87
-#define SYS_SHUTDOWN_SOCKET             88
+#define SYS_SEND_TO_SOCKET              88
-#define SYS_SEND_TO_SOCKET              89
+#define SYS_RECV_FROM_SOCKET            89
-#define SYS_RECV_FROM_SOCKET            90
+#define SYS_RESOLVE_HOST                90
-#define SYS_RESOLVE_HOST                91
+#define SYS_SET_SOCKET_OPTION           91
-#define SYS_SET_SOCKET_OPTION           92
+#define SYS_GET_SOCKET_OPTION           92
-#define SYS_GET_SOCKET_OPTION           93
+#define SYS_POLL                        93
-#define SYS_POLL                        94
+#define SYS_GET_MEMORY_STATUS           94
-#define SYS_GET_MEMORY_STATUS           95
+#define SYS_QUERY_KERNEL_STATS          95
-#define SYS_QUERY_KERNEL_STATS          96
+#define SYS_GET_SYSTEM_TIME             96
-#define SYS_GET_SYSTEM_TIME             97
+#define SYS_CREATE_FILE_MAPPING          97
-#define SYS_MAX 98
+#define SYS_MAP_FILE                     98
+#define SYS_CREATE_SURFACE               99
+#define SYS_MAP_SURFACE                  100
+#define SYS_QUERY_SURFACE                101
+#define SYS_PRESENT_SURFACE              102
+#define SYS_GET_THREAD_LIST             103
+#define SYS_READ_INPUT_EVENT            104
+#define SYS_PEEK_INPUT_EVENT            105
+#define SYS_WAIT_MANY                  106
+#define SYS_OPEN_INPUT_QUEUE           107
+#define SYS_FLUSH_FILE                 108
+#define SYS_MAX 109

code/kernel/io_device.c

@@ -7,6 +7,7 @@
 #include <kernel/mm_virt.h>
 #include <kernel/ke_debug.h>
 #include <kernel/ob_object.h>
+#include <kernel/ke_sched.h>
 
 #define MAX_DRIVER_OBJECTS    64
 #define MAX_PDO_OBJECTS       128
@@ -443,6 +444,170 @@ static uint32_t gDiskChangeCallbackCount;
 static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
 static uint32_t gDiskArrivalCallbackCount;
 
+#define IO_DISK_READAHEAD_MAX_SECTORS 128u
+#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u
+#define IO_DISK_READAHEAD_MIN_SECTORS 8u
+#define IO_DISK_SECTOR_BYTES 512u
+
+typedef struct IO_DISK_READAHEAD_STATE {
+    SPINLOCK Lock;
+    uint8_t* Buffer;
+    uint8_t  Valid;
+    uint8_t  Loading;
+    uint64_t Generation;
+    uint64_t BaseLba;
+    uint32_t Count;
+    uint64_t NextLba;
+    uint32_t SequentialScore;
+    IO_DISK_STATS Stats;
+} IO_DISK_READAHEAD_STATE;
+
+static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES];
+
+// Updates a maximum statistic while holding the disk read-ahead lock.
+static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) {
+    if (field && *field < value) *field = value;
+}
+
+// Prepares the per-disk read-ahead buffer.
+static void IoDiskReadAheadPrepare(uint8_t driveIndex) {
+    if (driveIndex >= MAX_DISK_DEVICES)
+        return;
+
+    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    state->Lock = (SPINLOCK)SPINLOCK_INIT;
+    state->Valid = 0;
+    state->Loading = 0;
+    state->Generation = 0;
+    state->BaseLba = 0;
+    state->Count = 0;
+    state->NextLba = 0;
+    state->SequentialScore = 0;
+    memset(&state->Stats, 0, sizeof(state->Stats));
+
+    if (!state->Buffer) {
+        state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES,
+                                                    MM_POOL_TAG('d', 'r', 'a', 'h'));
+        if (!state->Buffer)
+            state->Stats.ReadaheadAllocFailures++;
+    }
+}
+
+// Releases the per-disk read-ahead buffer.
+static void IoDiskReadAheadRelease(uint8_t driveIndex) {
+    if (driveIndex >= MAX_DISK_DEVICES)
+        return;
+
+    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    uint8_t* buffer = (uint8_t*)0;
+    for (;;) {
+        uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+        if (!state->Loading) {
+            buffer = state->Buffer;
+            state->Buffer = (uint8_t*)0;
+            state->Valid = 0;
+            state->Generation++;
+            state->Count = 0;
+            state->SequentialScore = 0;
+            KeReleaseSpinlock(&state->Lock, flags);
+            break;
+        }
+        KeReleaseSpinlock(&state->Lock, flags);
+        KeYield();
+    }
+
+    if (buffer)
+        MmFreePool(buffer);
+}
+
+// Invalidates cached disk read-ahead data for one disk.
+static void IoDiskReadAheadInvalidate(uint8_t driveIndex) {
+    if (driveIndex >= MAX_DISK_DEVICES)
+        return;
+
+    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+    state->Valid = 0;
+    state->Generation++;
+    state->Count = 0;
+    state->Stats.ReadaheadInvalidations++;
+    KeReleaseSpinlock(&state->Lock, flags);
+}
+
+// Returns true when one range fully contains another range.
+static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount,
+                                uint64_t innerBase, uint32_t innerCount) {
+    uint64_t outerEnd = outerBase + outerCount;
+    uint64_t innerEnd = innerBase + innerCount;
+    if (outerEnd < outerBase || innerEnd < innerBase)
+        return false;
+    return innerBase >= outerBase && innerEnd <= outerEnd;
+}
+
+// Attempts to satisfy a disk read from the per-disk read-ahead buffer.
+static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) {
+    if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0)
+        return false;
+
+    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+    bool hit = state->Buffer && state->Valid && !state->Loading &&
+               IoDiskRangeContains(state->BaseLba, state->Count, lba, count);
+    if (hit) {
+        size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES;
+        size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES;
+        memcpy(buffer, state->Buffer + offset, bytes);
+        state->Stats.ReadaheadHits++;
+        state->Stats.ReadaheadBytesServed += bytes;
+    } else {
+        state->Stats.ReadaheadMisses++;
+    }
+    KeReleaseSpinlock(&state->Lock, flags);
+    return hit;
+}
+
+// Records request size and sequential access state for one disk read.
+static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) {
+    if (driveIndex >= MAX_DISK_DEVICES || count == 0)
+        return 0;
+
+    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+    state->Stats.ReadRequests++;
+    state->Stats.ReadSectors += count;
+    IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count);
+
+    if (lba == state->NextLba) {
+        if (state->SequentialScore < 32)
+            state->SequentialScore++;
+    } else {
+        state->SequentialScore = 1;
+    }
+    state->NextLba = lba + count;
+    uint32_t score = state->SequentialScore;
+    if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE)
+        state->Stats.ReadaheadSequentialDetections++;
+    KeReleaseSpinlock(&state->Lock, flags);
+    return score;
+}
+
+// Records one disk write and invalidates stale read-ahead data.
+static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) {
+    if (driveIndex >= MAX_DISK_DEVICES)
+        return;
+
+    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+    state->Stats.WriteRequests++;
+    state->Stats.WriteSectors += count;
+    IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count);
+    state->Valid = 0;
+    state->Generation++;
+    state->Count = 0;
+    state->Stats.ReadaheadInvalidations++;
+    KeReleaseSpinlock(&state->Lock, flags);
+}
+
 // Implements the notify Disk Change operation.
 static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
     for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
@@ -477,6 +642,8 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
     if (pnpStatus == STATUS_NOT_SUPPORTED)
         device->PnpState = DevicePnpStarted;
 
+    IoDiskReadAheadPrepare(driveIndex);
+
     IoReferenceDeviceObject(device);
     gDiskDevices[driveIndex] = device;
     IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
@@ -490,6 +657,8 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
     DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
     if (!dev) return STATUS_NOT_FOUND;
 
+    IoDiskReadAheadInvalidate(driveIndex);
+
     dev->PnpState = DevicePnpSurpriseRemoved;
     IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
     (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
@@ -518,6 +687,7 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
     (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
 
     gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
+    IoDiskReadAheadRelease(driveIndex);
     IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
     IoDereferenceDeviceObject(dev);
     return STATUS_SUCCESS;
@@ -632,6 +802,112 @@ __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
     return st;
 }
 
+// Validates that a sector range fits the disk and IRP byte length.
+static KSTATUS IoValidateDiskSectorRange(DEVICE_OBJECT* dev, uint64_t lba, uint32_t count) {
+    if (!dev || count == 0) return STATUS_INVALID_PARAMETER;
+    if (count > (0xffffffffu / IO_DISK_SECTOR_BYTES)) return STATUS_INVALID_PARAMETER;
+    if (lba > dev->u.Disk.SectorCount) return STATUS_INVALID_PARAMETER;
+    if ((uint64_t)count > dev->u.Disk.SectorCount - lba) return STATUS_INVALID_PARAMETER;
+    return STATUS_SUCCESS;
+}
+
+// Submits one raw disk read IRP.
+static KSTATUS IoReadDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
+                                    uint32_t count, void* buffer) {
+    if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
+
+    IRP irp;
+    IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
+    return IoSubmitIrpSync(dev, &irp);
+}
+
+// Submits one raw disk write IRP.
+static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
+                                     uint32_t count, void* buffer) {
+    if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
+
+    IRP irp;
+    IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, buffer);
+    return IoSubmitIrpSync(dev, &irp);
+}
+
+// Chooses a conservative read-ahead span from the current sequential score.
+static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) {
+    uint32_t sectors = currentCount;
+    if (sectors < IO_DISK_READAHEAD_MIN_SECTORS)
+        sectors = IO_DISK_READAHEAD_MIN_SECTORS;
+    if (score >= 8 && sectors < 128u)
+        sectors = 128u;
+    else if (score >= 4 && sectors < 64u)
+        sectors = 64u;
+    else if (score >= 2 && sectors < 32u)
+        sectors = 32u;
+    if (sectors > IO_DISK_READAHEAD_MAX_SECTORS)
+        sectors = IO_DISK_READAHEAD_MAX_SECTORS;
+    return sectors;
+}
+
+// Loads the next sequential disk range into the read-ahead buffer.
+static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev,
+                                 uint64_t nextLba, uint32_t currentCount, uint32_t score) {
+    if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE)
+        return;
+
+    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    if (!state->Buffer) {
+        uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+        state->Stats.ReadaheadSuppressed++;
+        KeReleaseSpinlock(&state->Lock, flags);
+        return;
+    }
+
+    uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score);
+    uint64_t diskSectors = dev->u.Disk.SectorCount;
+    if (nextLba >= diskSectors)
+        return;
+    if ((uint64_t)sectors > diskSectors - nextLba)
+        sectors = (uint32_t)(diskSectors - nextLba);
+    if (sectors == 0)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+    bool covered = state->Valid && !state->Loading &&
+                   IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors);
+    if (covered) {
+        KeReleaseSpinlock(&state->Lock, flags);
+        return;
+    }
+    if (state->Loading) {
+        state->Stats.ReadaheadSuppressed++;
+        KeReleaseSpinlock(&state->Lock, flags);
+        return;
+    }
+    uint64_t generation = state->Generation;
+    state->Loading = 1;
+    state->Valid = 0;
+    state->Count = 0;
+    KeReleaseSpinlock(&state->Lock, flags);
+
+    KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer);
+
+    flags = KeAcquireSpinlock(&state->Lock);
+    state->Loading = 0;
+    if (KSUCCESS(st) && generation == state->Generation) {
+        state->BaseLba = nextLba;
+        state->Count = sectors;
+        state->Valid = 1;
+        state->Stats.ReadaheadStores++;
+        state->Stats.ReadaheadSectorsLoaded += sectors;
+        IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors);
+    } else {
+        state->Valid = 0;
+        state->Count = 0;
+        if (!KSUCCESS(st))
+            state->Stats.ReadaheadReadFailures++;
+    }
+    KeReleaseSpinlock(&state->Lock, flags);
+}
+
 __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
                                                 uint64_t lba,
                                                 uint32_t count,
@@ -641,9 +917,23 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
     DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
     if (!dev) return STATUS_NOT_FOUND;
 
-    IRP irp;
+    KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
-    IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
+    if (!KSUCCESS(rangeStatus)) {
-    KSTATUS st = IoSubmitIrpSync(dev, &irp);
+        IoDereferenceDeviceObject(dev);
+        return rangeStatus;
+    }
+
+    if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) {
+        uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
+        IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
+        IoDereferenceDeviceObject(dev);
+        return STATUS_SUCCESS;
+    }
+
+    KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
+    uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
+    if (KSUCCESS(st))
+        IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
 
     IoDereferenceDeviceObject(dev);
     return st;
@@ -658,10 +948,50 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
     DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
     if (!dev) return STATUS_NOT_FOUND;
 
-    IRP irp;
+    KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
-    IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer);
+    if (!KSUCCESS(rangeStatus)) {
-    KSTATUS st = IoSubmitIrpSync(dev, &irp);
+        IoDereferenceDeviceObject(dev);
+        return rangeStatus;
+    }
+
+    KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
+    if (KSUCCESS(st))
+        IoDiskReadAheadNoteWrite(driveIndex, count);
 
     IoDereferenceDeviceObject(dev);
     return st;
 }
+
+// Queries aggregate disk I/O statistics.
+__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
+    if (!out) return;
+    memset(out, 0, sizeof(*out));
+
+    for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
+        IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i];
+        uintptr_t flags = KeAcquireSpinlock(&state->Lock);
+        if (gDiskDevices[i])
+            out->RegisteredDisks++;
+        out->ReadRequests += state->Stats.ReadRequests;
+        out->ReadSectors += state->Stats.ReadSectors;
+        out->WriteRequests += state->Stats.WriteRequests;
+        out->WriteSectors += state->Stats.WriteSectors;
+        if (out->MaxReadSectors < state->Stats.MaxReadSectors)
+            out->MaxReadSectors = state->Stats.MaxReadSectors;
+        if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
+            out->MaxWriteSectors = state->Stats.MaxWriteSectors;
+        out->ReadaheadHits += state->Stats.ReadaheadHits;
+        out->ReadaheadMisses += state->Stats.ReadaheadMisses;
+        out->ReadaheadStores += state->Stats.ReadaheadStores;
+        out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations;
+        out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed;
+        out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded;
+        out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections;
+        out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed;
+        out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures;
+        out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures;
+        if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors)
+            out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors;
+        KeReleaseSpinlock(&state->Lock, flags);
+    }
+}

code/kernel/io_file.c

@@ -4,6 +4,8 @@
 #include <kernel/ke_list.h>
 #include <kernel/ke_process.h>
 #include <kernel/io_file.h>
+#include <kernel/mm_file_cache.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_userenv.h>
@@ -13,6 +15,7 @@
 #include <kernel/ke_semaphore.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
+#include <ntlite/syscall.h>
 
 #define MAX_DRIVES  8
 
@@ -45,11 +48,15 @@ struct IO_ASYNC_FILE_REQUEST_OBJECT {
     uint8_t CancelRequested;
 
     FILE_OBJECT* File;                // referenced until object destruction
+    EVENT* WaitEvent;                 // internal completion event
     EVENT* CompletionEvent;           // optional, referenced until object destruction
     KAPC* CompletionApc;              // caller-owned APC storage
     IO_STATUS_BLOCK* Iosb;            // caller/user-owned completion block
+    IO_STATUS_BLOCK IosbSnapshot;
+    MM_LOCKED_USER_BUFFER* LockedBuffer;
+    MM_LOCKED_USER_BUFFER* LockedIosb;
 
-    void* Buffer;                     // caller/user-owned; must remain valid until completion/cancel
+    void* Buffer;                     // caller-owned; user buffers must be locked until completion/cancel
     size_t Size;
     uint64_t Offset;
 };
@@ -65,6 +72,10 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
     IO_ASYNC_FILE_REQUEST_OBJECT* req =
         CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header);
 
+    if (req->WaitEvent) {
+        ObDereferenceObject(&req->WaitEvent->Header);
+        req->WaitEvent = (EVENT*)0;
+    }
     if (req->CompletionEvent) {
         ObDereferenceObject(&req->CompletionEvent->Header);
         req->CompletionEvent = (EVENT*)0;
@@ -73,6 +84,14 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
         ObDereferenceObject(&req->File->Header);
         req->File = (FILE_OBJECT*)0;
     }
+    if (req->LockedBuffer) {
+        MmUnlockUserBuffer(req->LockedBuffer);
+        req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
+    }
+    if (req->LockedIosb) {
+        MmUnlockUserBuffer(req->LockedIosb);
+        req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
+    }
 }
 
 static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
@@ -80,6 +99,9 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
                                        size_t information) {
     if (!req) return;
 
+    req->IosbSnapshot.Status = status;
+    req->IosbSnapshot.Information = information;
+
     uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
     req->State = (status == STATUS_CANCELLED) ? IoAsyncStateCancelled : IoAsyncStateCompleted;
     KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
@@ -99,6 +121,18 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
 
     if (req->CompletionEvent)
         KeSignalEvent(req->CompletionEvent);
+
+    if (req->WaitEvent)
+        KeSignalEvent(req->WaitEvent);
+
+    if (req->LockedBuffer) {
+        MmUnlockUserBuffer(req->LockedBuffer);
+        req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
+    }
+    if (req->LockedIosb) {
+        MmUnlockUserBuffer(req->LockedIosb);
+        req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
+    }
 }
 
 // Implements the io Async File Dequeue operation.
@@ -226,10 +260,24 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
                                          IO_STATUS_BLOCK* iosb,
                                          EVENT* completionEvent,
                                          KAPC* completionApc,
+                                         MM_LOCKED_USER_BUFFER* lockedBuffer,
+                                         MM_LOCKED_USER_BUFFER* lockedIosb,
                                          IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
-    if (!file || !iosb) return STATUS_INVALID_PARAMETER;
+    if (!file || !iosb) {
-    if (!buffer && size) return STATUS_INVALID_PARAMETER;
+        MmUnlockUserBuffer(lockedBuffer);
-    if (!KeCanWait()) return STATUS_INVALID_IRQL;
+        MmUnlockUserBuffer(lockedIosb);
+        return STATUS_INVALID_PARAMETER;
+    }
+    if (!buffer && size) {
+        MmUnlockUserBuffer(lockedBuffer);
+        MmUnlockUserBuffer(lockedIosb);
+        return STATUS_INVALID_PARAMETER;
+    }
+    if (!KeCanWait()) {
+        MmUnlockUserBuffer(lockedBuffer);
+        MmUnlockUserBuffer(lockedIosb);
+        return STATUS_INVALID_IRQL;
+    }
 
     iosb->Status = STATUS_PENDING;
     iosb->Information = 0;
@@ -239,6 +287,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
                                                        sizeof(IO_ASYNC_FILE_REQUEST_OBJECT));
     if (!req) {
         iosb->Status = STATUS_NO_MEMORY;
+        MmUnlockUserBuffer(lockedBuffer);
+        MmUnlockUserBuffer(lockedIosb);
         return STATUS_NO_MEMORY;
     }
 
@@ -247,10 +297,23 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
     req->Op = op;
     req->State = IoAsyncStateQueued;
     req->CancelRequested = 0;
+    KSTATUS waitStatus = KeEventCreate(EventManualReset, 0, &req->WaitEvent);
+    if (!KSUCCESS(waitStatus)) {
+        iosb->Status = waitStatus;
+        MmUnlockUserBuffer(lockedBuffer);
+        MmUnlockUserBuffer(lockedIosb);
+        ObDereferenceObject(&req->Header);
+        return waitStatus;
+    }
+
     req->File = file;
     req->CompletionEvent = completionEvent;
     req->CompletionApc = completionApc;
     req->Iosb = iosb;
+    req->IosbSnapshot.Status = STATUS_PENDING;
+    req->IosbSnapshot.Information = 0;
+    req->LockedBuffer = lockedBuffer;
+    req->LockedIosb = lockedIosb;
     req->Buffer = buffer;
     req->Size = size;
     req->Offset = offset;
@@ -288,6 +351,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
 
         ObDereferenceObject(&req->Header); // provider rejected synchronously
         iosb->Status = ns;
+        req->IosbSnapshot.Status = ns;
+        req->IosbSnapshot.Information = 0;
         ObDereferenceObject(&req->Header); // creator reference
         return ns;
     }
@@ -295,6 +360,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
     KSTATUS s = KiIoEnsureAsyncFileIoStarted();
     if (!KSUCCESS(s)) {
         iosb->Status = s;
+        req->IosbSnapshot.Status = s;
+        req->IosbSnapshot.Information = 0;
         ObDereferenceObject(&req->Header);
         return s;
     }
@@ -323,10 +390,12 @@ __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf
                                                     IO_STATUS_BLOCK* iosb,
                                                     EVENT* completionEvent,
                                                     KAPC* completionApc,
+                                                    MM_LOCKED_USER_BUFFER* lockedBuffer,
+                                                    MM_LOCKED_USER_BUFFER* lockedIosb,
                                                     IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
     return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset,
                                     iosb, completionEvent, completionApc,
-                                    requestOut);
+                                    lockedBuffer, lockedIosb, requestOut);
 }
 
 __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
@@ -334,10 +403,12 @@ __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const vo
                                                      IO_STATUS_BLOCK* iosb,
                                                      EVENT* completionEvent,
                                                      KAPC* completionApc,
+                                                     MM_LOCKED_USER_BUFFER* lockedBuffer,
+                                                     MM_LOCKED_USER_BUFFER* lockedIosb,
                                                      IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
     return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset,
                                     iosb, completionEvent, completionApc,
-                                    requestOut);
+                                    lockedBuffer, lockedIosb, requestOut);
 }
 
 // Implements the async File Request Object operation.
@@ -345,6 +416,19 @@ __declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJ
     return req ? &req->Header : (OBJECT*)0;
 }
 
+
+// Returns the wait event owned by an async file request.
+__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
+    return req ? req->WaitEvent : (EVENT*)0;
+}
+
+// Waits for an async file request to complete according to wait flags.
+__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, uint32_t flags) {
+    if (!req || !req->WaitEvent) return STATUS_INVALID_PARAMETER;
+    if (flags & ~SYS_WAIT_FLAG_ALERTABLE) return STATUS_INVALID_PARAMETER;
+    return KeWaitForEvent(req->WaitEvent, (flags & SYS_WAIT_FLAG_ALERTABLE) ? 1 : 0);
+}
+
 // Cancels async File Request.
 __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
     if (!req) return STATUS_INVALID_PARAMETER;
@@ -377,13 +461,8 @@ __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJ
 __declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
                                                       IO_STATUS_BLOCK* iosbOut) {
     if (!req || !iosbOut) return STATUS_INVALID_PARAMETER;
-    if (req->Iosb) {
+    iosbOut->Status = req->IosbSnapshot.Status;
-        iosbOut->Status = req->Iosb->Status;
+    iosbOut->Information = req->IosbSnapshot.Information;
-        iosbOut->Information = req->Iosb->Information;
-    } else {
-        iosbOut->Status = STATUS_FAILURE;
-        iosbOut->Information = 0;
-    }
     return STATUS_SUCCESS;
 }
 
@@ -622,10 +701,51 @@ __declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
     return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
 }
 
+// Aggregates filesystem traversal statistics across mounted volumes.
+__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
+    if (!out) return;
+    memset(out, 0, sizeof(*out));
+
+    for (uint8_t i = 0; i < MAX_DRIVES; i++) {
+        IO_FILESYSTEM* fs = gVolumes[i].Fs;
+        if (!fs || !fs->QueryStats) continue;
+
+        IO_FILESYSTEM_STATS one;
+        memset(&one, 0, sizeof(one));
+        fs->QueryStats(fs, &one);
+
+        out->PathLookups += one.PathLookups;
+        out->PathCacheHits += one.PathCacheHits;
+        out->PathCacheMisses += one.PathCacheMisses;
+        out->PathCacheStores += one.PathCacheStores;
+        out->PathCacheInvalidations += one.PathCacheInvalidations;
+        if (one.PathCacheGeneration > out->PathCacheGeneration)
+            out->PathCacheGeneration = one.PathCacheGeneration;
+        out->DirectorySearches += one.DirectorySearches;
+        out->DirectoryClustersRead += one.DirectoryClustersRead;
+        out->DirectoryEntriesScanned += one.DirectoryEntriesScanned;
+        out->FatChainCacheLookups += one.FatChainCacheLookups;
+        out->FatChainCacheHits += one.FatChainCacheHits;
+        out->FatChainCacheMisses += one.FatChainCacheMisses;
+        out->FatChainCacheStores += one.FatChainCacheStores;
+        out->FatChainCacheInvalidations += one.FatChainCacheInvalidations;
+        out->FatChainWalkSteps += one.FatChainWalkSteps;
+        out->DataReadRuns += one.DataReadRuns;
+        out->DataReadSectors += one.DataReadSectors;
+        out->DataReadRunMerges += one.DataReadRunMerges;
+        if (one.MaxDataReadSpanSectors > out->MaxDataReadSpanSectors)
+            out->MaxDataReadSpanSectors = one.MaxDataReadSpanSectors;
+    }
+}
+
 // Destructor.
 static void KiFileDestroy(OBJECT* obj) {
     FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
-    if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx);
+    if (f->Fs && f->FileCtx) {
+        (void)IoFlushFile(f);
+        MmFileCacheInvalidateAll(f);
+        f->Fs->Close(f->Fs, f->FileCtx);
+    }
 }
 
 // IoOpenFile.
@@ -655,11 +775,73 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
     f->Lock    = (SPINLOCK)SPINLOCK_INIT;
     f->Size    = fs->GetSize(fs, fileCtx);
     f->Offset  = 0;
+    f->Flags   = 0;
+    f->ReadaheadScore = 0;
+    f->ReadaheadNextOffset = 0;
+    f->ReadaheadIssuedUntil = 0;
+    f->ReadaheadSuppressedUntil = 0;
 
     *out = f;
     return STATUS_SUCCESS;
 }
 
+
+static uint64_t KiAlignUpPage64(uint64_t value) {
+    return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
+}
+
+static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) {
+    if (!file || bytesRead < 512)
+        return;
+    if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0)
+        return;
+
+    uint64_t end = offset + bytesRead;
+    uint64_t start = 0;
+    uint32_t pages = 0;
+
+    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
+    if (offset == file->ReadaheadNextOffset) {
+        if (file->ReadaheadScore < 32)
+            file->ReadaheadScore++;
+    } else {
+        file->ReadaheadScore = 1;
+        file->ReadaheadIssuedUntil = end;
+    }
+    file->ReadaheadNextOffset = end;
+
+    if (end < file->ReadaheadSuppressedUntil) {
+        pages = 0;
+    } else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) {
+        pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u);
+        start = KiAlignUpPage64(end);
+        if (start < file->ReadaheadIssuedUntil)
+            start = KiAlignUpPage64(file->ReadaheadIssuedUntil);
+        if (start >= fileSize) {
+            pages = 0;
+        } else {
+            uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE);
+            if (maxEnd > fileSize)
+                pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE);
+            file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE);
+        }
+    }
+    KeReleaseSpinlock(&file->Lock, flags);
+
+    if (pages) {
+        KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages);
+        if (ras == STATUS_QUEUE_FULL) {
+            flags = KeAcquireSpinlock(&file->Lock);
+            uint64_t suppressUntil = end + (1024u * 1024u);
+            if (suppressUntil < end || suppressUntil > fileSize)
+                suppressUntil = fileSize;
+            if (file->ReadaheadSuppressedUntil < suppressUntil)
+                file->ReadaheadSuppressedUntil = suppressUntil;
+            KeReleaseSpinlock(&file->Lock, flags);
+        }
+    }
+}
+
 // Explicit-offset file I/O. Async I/O is built on these routines so it does
 // not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
 __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -679,7 +861,50 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
     size_t remaining = (size_t)(fileSize - offset);
     if (size > remaining) size = remaining;
 
+    if ((file->Flags & IO_FILE_FLAG_NO_CACHE) || size < 512)
         return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead);
+
+    size_t total = 0;
+    uint8_t* dst = (uint8_t*)buf;
+    while (total < size) {
+        uint64_t absolute = offset + total;
+        uint64_t pageOffset = absolute & ~(uint64_t)(PAGE_SIZE - 1);
+        size_t inPage = (size_t)(absolute - pageOffset);
+        size_t chunk = PAGE_SIZE - inPage;
+        if (chunk > size - total) chunk = size - total;
+
+        MM_FILE_CACHE_PAGE page;
+        KSTATUS s = MmFileCacheGetPage(file, pageOffset, &page);
+        if (!KSUCCESS(s)) {
+            if (total) {
+                *bytesRead = total;
+                return STATUS_SUCCESS;
+            }
+            return s;
+        }
+
+        if (inPage >= page.ValidBytes) {
+            MmFileCacheReleasePage(&page);
+            break;
+        }
+        size_t available = page.ValidBytes - inPage;
+        if (chunk > available) chunk = available;
+
+        void* srcPage = MmPfnToVirtual(page.Frame);
+        if (!srcPage) {
+            MmFileCacheReleasePage(&page);
+            return total ? STATUS_SUCCESS : STATUS_FAILURE;
+        }
+        memcpy(dst + total, (const uint8_t*)srcPage + inPage, chunk);
+        MmFileCacheReleasePage(&page);
+
+        total += chunk;
+        if (chunk == 0) break;
+    }
+
+    *bytesRead = total;
+    KiIoMaybeScheduleReadAhead(file, offset, total, fileSize);
+    return STATUS_SUCCESS;
 }
 
 __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -691,10 +916,21 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
     if (!file->Fs->Write) return STATUS_NOT_SUPPORTED;
     if (size == 0) return STATUS_SUCCESS;
 
-    size_t written = 0;
-    KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
-    if (KSUCCESS(s)) {
     uintptr_t flags = KeAcquireSpinlock(&file->Lock);
+    uint64_t oldSize = file->Size;
+    KeReleaseSpinlock(&file->Lock, flags);
+
+    size_t written = 0;
+    KSTATUS s;
+    if (file->Flags & IO_FILE_FLAG_NO_CACHE) {
+        s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
+        if (KSUCCESS(s) && written)
+            MmFileCacheInvalidate(file, offset, written);
+    } else {
+        s = MmFileCacheWrite(file, offset, buf, size, oldSize, &written);
+    }
+    if (KSUCCESS(s)) {
+        flags = KeAcquireSpinlock(&file->Lock);
         uint64_t end = offset + written;
         if (end > file->Size) file->Size = end;
         KeReleaseSpinlock(&file->Lock, flags);
@@ -704,6 +940,16 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
     return s;
 }
 
+// Flushes dirty mapped and cached pages for one file object.
+__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) {
+    if (!file) return STATUS_INVALID_PARAMETER;
+    if (!KeCanWait()) return STATUS_INVALID_IRQL;
+
+    KSTATUS sync = MmVadSyncFileMappings(file);
+    KSTATUS flush = MmFileCacheFlushFile(file);
+    return KSUCCESS(sync) ? flush : sync;
+}
+
 // IoReadFile - sequential read at current offset.
 __declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
                                          size_t size, size_t* bytesRead) {
@@ -768,6 +1014,11 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o
     f->Lock    = (SPINLOCK)SPINLOCK_INIT;
     f->Size    = 0;
     f->Offset  = 0;
+    f->Flags   = 0;
+    f->ReadaheadScore = 0;
+    f->ReadaheadNextOffset = 0;
+    f->ReadaheadIssuedUntil = 0;
+    f->ReadaheadSuppressedUntil = 0;
 
     *out = f;
     return STATUS_SUCCESS;
@@ -848,8 +1099,12 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize
     if (!KeCanWait()) return STATUS_INVALID_IRQL;
     if (!file->Fs->Truncate) return STATUS_NOT_SUPPORTED;
 
-    KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize);
+    KSTATUS flush = IoFlushFile(file);
+    if (!KSUCCESS(flush)) return flush;
+
+    KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize, file->Flags);
     if (KSUCCESS(s)) {
+        MmFileCacheInvalidate(file, newSize, (size_t)-1);
         file->Size = newSize;
         // Pull the file pointer back if it now sits past EOF.
         if (file->Offset > file->Size)
@@ -875,6 +1130,16 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
     return size;
 }
 
+// Queries the filesystem identity for a file object.
+__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) {
+    if (!file || !out)
+        return STATUS_INVALID_PARAMETER;
+    memset(out, 0, sizeof(*out));
+    if (!file->Fs || !file->Fs->QueryIdentity)
+        return STATUS_NOT_SUPPORTED;
+    return file->Fs->QueryIdentity(file->Fs, file->FileCtx, out);
+}
+
 // Releases file.
 __declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
     if (file) ObDereferenceObject(&file->Header);

code/kernel/ke_apc.c

@@ -204,7 +204,7 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
     if (thread->State == ThreadDead)
         return STATUS_OBJECT_DEAD;
 
-    KUSER_APC* apc = (KUSER_APC*)MmAllocatePool(sizeof(KUSER_APC));
+    KUSER_APC* apc = (KUSER_APC*)MmAllocatePoolTag(sizeof(KUSER_APC), MM_POOL_TAG('a', 'p', 'c', ' '));
     if (!apc)
         return STATUS_NO_MEMORY;
 

code/kernel/ke_debug.c

@@ -1,20 +1,24 @@
 // Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.
 
 #include <stdarg.h>
+#include <stdbool.h>
 #include <stdint.h>
 #include <string.h>
 #include <ntlite/ddi/hal_api.h>
 #include <kernel/hal/hal_apic.h>
 #include <intrin.h>
 #pragma intrinsic(_disable)
+#pragma intrinsic(_enable)
 #include <kernel/ke_debug.h>
 #include <kernel/ke_cpu.h>
 #include <kernel/ke_spinlock.h>
 #include <kernel/ke_process.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
+#include <kernel/ke_work.h>
 #include <kernel/mm_virt.h>
 #include <kernel/io_file.h>
+#include <kernel/ke_system_root.h>
 #include <common/printf.h>
 int _fltused; // MSVC floating-point helper anchor
 
@@ -23,14 +27,24 @@ static uint32_t gDebugFlags = 0;
 void (*PrintChar)(char, uint32_t, uint32_t);
 
 #define KE_DEBUG_LOG_BUFFER_SIZE    (50u * 1024u)
+#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u)
 #define KE_DEBUG_LOG_LINE_SIZE      1024u
+#define KE_DEBUG_SCREEN_DRAIN_SIZE  512u
 
 static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
 static char*    gDebugLogBuffer;
 static size_t   gDebugLogSize;
 static size_t   gDebugLogWrite;
 static size_t   gDebugLogUsed;
+static char*    gDebugScreenBuffer;
+static size_t   gDebugScreenSize;
+static size_t   gDebugScreenRead;
+static size_t   gDebugScreenWrite;
+static size_t   gDebugScreenUsed;
 static volatile long gDebugLogFlushing;
+static volatile long gDebugConsoleOwnsSerial;
+static volatile long gDebugConsoleWorkReady;
+static KWORK_ITEM gDebugConsoleWorkItem;
 
 // Implements the debug Log Append Locked operation.
 static void KiDebugLogAppendLocked(const char* text, size_t len) {
@@ -55,6 +69,95 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
         gDebugLogUsed += len;
 }
 
+// Implements the debug Screen Append Locked operation.
+static void KiDebugScreenAppendLocked(const char* text, size_t len) {
+    if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0)
+        return;
+
+    if (len >= gDebugScreenSize) {
+        text += len - (gDebugScreenSize - 1u);
+        len = gDebugScreenSize - 1u;
+    }
+
+    while (gDebugScreenUsed + len > gDebugScreenSize) {
+        gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize;
+        gDebugScreenUsed--;
+    }
+
+    size_t first = gDebugScreenSize - gDebugScreenWrite;
+    if (first > len) first = len;
+    memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first);
+    if (len > first)
+        memcpy(gDebugScreenBuffer, text + first, len - first);
+
+    gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize;
+    gDebugScreenUsed += len;
+}
+
+// Implements the debug Screen Drain Locked operation.
+static size_t KiDebugScreenDrainLocked(char* out, size_t max) {
+    if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0)
+        return 0;
+
+    size_t len = gDebugScreenUsed;
+    if (len > max) len = max;
+
+    size_t first = gDebugScreenSize - gDebugScreenRead;
+    if (first > len) first = len;
+    memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first);
+    if (len > first)
+        memcpy(out + first, gDebugScreenBuffer, len - first);
+
+    gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize;
+    gDebugScreenUsed -= len;
+    return len;
+}
+
+// Ensures the asynchronous screen ring exists before console output starts using it.
+static bool KiDebugEnsureScreenBuffer(void) {
+    if (gDebugScreenBuffer && gDebugScreenSize != 0)
+        return true;
+
+    char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r'));
+    if (!buffer)
+        return false;
+    memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE);
+
+    bool installed = false;
+    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    if (!gDebugScreenBuffer) {
+        gDebugScreenBuffer = buffer;
+        gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE;
+        gDebugScreenRead = 0;
+        gDebugScreenWrite = 0;
+        gDebugScreenUsed = 0;
+        installed = true;
+    }
+    KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+    if (!installed)
+        MmFreePool(buffer);
+    return true;
+}
+
+// Implements the debug Serial Write Char operation.
+static void KiDebugSerialWriteChar(char c) {
+    while ((__inbyte(0x3FD) & 0x20) == 0) { }
+    if (c == '\n') {
+        __outbyte(0x3F8, '\r');
+        while ((__inbyte(0x3FD) & 0x20) == 0) { }
+    }
+    __outbyte(0x3F8, (uint8_t)c);
+}
+
+// Implements the debug Serial Write Locked operation.
+static void KiDebugSerialWriteLocked(const char* text, size_t len) {
+    if (!text)
+        return;
+    for (size_t i = 0; i < len; i++)
+        KiDebugSerialWriteChar(text[i]);
+}
+
 // Implements the debug Console Write operation.
 static void KiDebugConsoleWrite(const char* text, size_t len) {
     if (!PrintChar || !text)
@@ -63,6 +166,28 @@ static void KiDebugConsoleWrite(const char* text, size_t len) {
         PrintChar(text[i], 0xCCCCCC, 0x012456);
 }
 
+// Implements the debug Console Work operation.
+static void KiDebugConsoleWork(void* context) {
+    (void)context;
+    char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE];
+
+    for (;;) {
+        uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+        size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk));
+        KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+        if (len == 0)
+            break;
+        KiDebugConsoleWrite(chunk, len);
+    }
+}
+
+// Queues the debug Console Work operation if the passive work queue is ready.
+static void KiDebugQueueConsoleWork(void) {
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted())
+        (void)KeQueueWorkItem(&gDebugConsoleWorkItem);
+}
+
 static volatile long gPanicInProgress = 0;
 static volatile uintptr_t gPanicOwnerToken = 0;
 static volatile long gPanicStoppedCpuCount = 0;
@@ -116,7 +241,7 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
     if (KiPanicIsActive())
         KiPanicSecondaryHalt();
 
-    if (MmHandleTlbShootdownNmi())
+    if (MmPteHandleShootdownNmi())
         return 1;
 
     return 0;
@@ -124,6 +249,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
 
 // Implements the kernel Put Char operation.
 void KiKernelPutChar(char c) {
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteChar(c);
     if (PrintChar)
         PrintChar(c, 0xCCCCCC, 0x012456);
 }
@@ -135,8 +262,11 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
     PrintChar = PrintPtr;
     printf_set_putchar(KiKernelPutChar);
 
+    if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0))
+        KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0);
+
     if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
-        gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE);
+        gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
         if (gDebugLogBuffer) {
             gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
             gDebugLogWrite = 0;
@@ -144,6 +274,9 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
             memset(gDebugLogBuffer, 0, gDebugLogSize);
         }
     }
+
+    if (flags & KE_DEBUG_CONSOLE)
+        (void)KiDebugEnsureScreenBuffer();
 }
 
 // Returns debug Flags.
@@ -157,6 +290,13 @@ void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
         return;
     PrintChar = PrintPtr;
     printf_set_putchar(KiKernelPutChar);
+
+    // After the display manager owns debug drawing, DbgPrint and user console
+    // writes mirror to COM1 themselves. The display writer is screen-only so
+    // asynchronous screen flushing does not duplicate serial bytes.
+    (void)KiDebugEnsureScreenBuffer();
+    _InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1);
+    KiDebugQueueConsoleWork();
 }
 
 // Implements the dbg Print operation.
@@ -179,17 +319,37 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
     if (KiPanicIsActive()) {
         if (!KiPanicCurrentCpuOwnsReport())
             KiPanicSecondaryHalt();
-        if (gDebugFlags & KE_DEBUG_CONSOLE)
+        if (gDebugFlags & KE_DEBUG_CONSOLE) {
+            if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+                KiDebugSerialWriteLocked(line, len);
             KiDebugConsoleWrite(line, len);
+        }
         return;
     }
 
+    bool queueConsole = false;
+    bool writeConsoleSync = false;
+
     uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteLocked(line, len);
     if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
         KiDebugLogAppendLocked(line, len);
-    if (gDebugFlags & KE_DEBUG_CONSOLE)
+
-        KiDebugConsoleWrite(line, len);
+    if (gDebugFlags & KE_DEBUG_CONSOLE) {
+        if (KeIsWorkQueueStarted() && gDebugScreenBuffer) {
+            KiDebugScreenAppendLocked(line, len);
+            queueConsole = true;
+        } else {
+            writeConsoleSync = true;
+        }
+    }
     KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+    if (queueConsole)
+        KiDebugQueueConsoleWork();
+    else if (writeConsoleSync)
+        KiDebugConsoleWrite(line, len);
 }
 
 // Flushes debug Log To File.
@@ -242,6 +402,23 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
     return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
 }
 
+
+// Attempts to persist the debug log during a panic when the filesystem is usable.
+static void KiTryFlushPanicDebugLog(void) {
+    if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
+        return;
+    if (!KeCanWait() || !KeSystemRootReady())
+        return;
+
+    char path[1024];
+    if (!KeBuildSystemRootPath("kernel-panic.log", path, sizeof(path)))
+        return;
+
+    KSTATUS st = KeFlushDebugLogToFile(SysSvFromCstr(path));
+    if (!KSUCCESS(st))
+        DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
+}
+
 // KePanic.
 __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
     KiBeginFatalReport();
@@ -279,15 +456,39 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
         va_end(va);
     }
 
+    KiTryFlushPanicDebugLog();
+
     for (;;) __halt();
 }
 
 // KePrintConsole - write user-visible console text using the default console palette.
-// Panic output keeps a distinct color so fatal paths remain visually obvious.
+// User text mirrors to COM1 immediately, then queues expensive screen drawing
+// through the passive debug-console worker when that path is available.
 __declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
-    if (!buf || !PrintChar) return;
+    if (!buf || len == 0) return;
-    for (size_t i = 0; i < len; i++)
+
-        PrintChar(buf[i], 0xCCCCCC, 0x012456);
+    bool queueConsole = false;
+    bool writeConsoleSync = false;
+    bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer();
+
+    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteLocked(buf, len);
+
+    if (PrintChar) {
+        if (!KiPanicIsActive() && canQueueScreen) {
+            KiDebugScreenAppendLocked(buf, len);
+            queueConsole = true;
+        } else {
+            writeConsoleSync = true;
+        }
+    }
+    KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+    if (queueConsole)
+        KiDebugQueueConsoleWork();
+    else if (writeConsoleSync)
+        KiDebugConsoleWrite(buf, len);
 }
 
 // KeBugCheckProcess - user-mode fault handler
@@ -613,8 +814,22 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
 
 // Prints crash Char.
 static void KiPrintCrashChar(char c) {
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteChar(c);
     if (PrintChar)
-        PrintChar(c, 0xFFAA00, 0x000000);   // amber - distinct from green debug / red panic
+        PrintChar(c, 0xFFAA00, 0x000000);
+}
+
+static KSTATUS KiFaultExitStatus(int vector) {
+    switch (vector) {
+        case 0:  return STATUS_PROCESS_DIVIDE_BY_ZERO;
+        case 3:  return STATUS_PROCESS_BREAKPOINT;
+        case 6:  return STATUS_PROCESS_ILLEGAL_OPCODE;
+        case 13: return STATUS_PROCESS_GENERAL_PROTECTION;
+        case 14: return STATUS_PROCESS_PAGE_FAULT;
+        case 17: return STATUS_PROCESS_ALIGNMENT_FAULT;
+        default: return STATUS_PROCESS_UNKNOWN_FAULT;
+    }
 }
 
 static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
@@ -648,13 +863,23 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
 }
 
 // Implements the finish Faulted Process operation.
-static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread) {
+static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
     printf_set_putchar(KiKernelPutChar);
 
-    if (proc)   KeExitProcess(proc, -1);
+    int exitCode = (int)exitStatus;
-    if (thread) KeExitThread(thread, -1);
+    if (proc)   (void)KeTerminateProcess(proc, exitCode);
+    if (thread) KeExitThread(thread, exitCode);
 
-    while (1) __halt();
+    // KeExitThread normally switches away and never resumes this context.  If
+    // a fault path re-enters here during scheduler bring-up or with no runnable
+    // peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
+    // dead thread to the scheduler with interrupts enabled so waiters such as
+    // the foreground shell can observe the process exit event.
+    while (true) {
+        _enable();
+        KeYield();
+        __halt();
+    }
 }
 
 _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
@@ -664,11 +889,11 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
     KiPrintCrashHeader(proc, vector, rip, errCode, cr2);
     KiDumpModules(proc);
 
-    KiFinishFaultedProcess(proc, thread);
+    KiFinishFaultedProcess(proc, thread, KiFaultExitStatus(vector));
 }
 
-// Implements the fault Dispatch Frame operation.
+// Dispatches a CPU fault frame and returns only when a user #PF was repaired.
-__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
+__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
     // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
     // the interrupted context on a secondary CPU once another CPU owns panic.
     if (KiPanicIsActive())
@@ -682,6 +907,12 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
                 (unsigned long long)(uintptr_t)frame);
     }
 
+    if (frame->Vector == 14) {
+        KSTATUS handled = MmVadResolvePageFaultByAddress((uintptr_t)frame->Cr2, frame->ErrorCode);
+        if (KSUCCESS(handled))
+            return STATUS_SUCCESS;
+    }
+
     if ((frame->Cs & 3) == 0) {
         KiBeginFatalReport();
 
@@ -702,7 +933,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
         if (vector == 14) {
             MmDebugDescribeAddress((uintptr_t)frame->Cr2);
             if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
-            MmDebugDumpUserRegions(proc);
+            MmVadDebugDumpProcess(proc);
         }
 
         if (vector == 14) {
@@ -743,7 +974,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
     KiDumpFaultFrame(proc, frame);
     KiDumpModules(proc);
 
-    KiFinishFaultedProcess(proc, thread);
+    KiFinishFaultedProcess(proc, thread, KiFaultExitStatus((int)frame->Vector));
 }
 
 // KeFaultDispatch - single C entry point for all three IDT fault stubs.
@@ -762,5 +993,6 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
     frame.Cr2 = cr2;
     frame.Rip = rip;
     frame.Cs = cs;
-    KeFaultDispatchFrame(&frame);
+    (void)KeFaultDispatchFrame(&frame);
+    while (1) __halt();
 }

code/kernel/ke_event.c

@@ -1,17 +1,100 @@
 // Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
 
 #include <stdbool.h>
+#include <intrin.h>
 #include <kernel/ke_event.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
 #include <kernel/ke_irql.h>
 #include <kernel/ke_apc.h>
 #include <kernel/ob_object.h>
+#include <kernel/mm_virt.h>
+#include <ntlite/syscall.h>
+
+// Multi-wait context shared by the stack wait blocks for one wait call.
+typedef struct KE_EVENT_MULTI_WAIT_CONTEXT KE_EVENT_MULTI_WAIT_CONTEXT;
+
+typedef struct KE_EVENT_WAIT_BLOCK {
+    LIST_ENTRY Link;
+    KE_EVENT_MULTI_WAIT_CONTEXT* Context;
+    EVENT* Event;
+    uint32_t Index;
+    uint8_t Armed;
+} KE_EVENT_WAIT_BLOCK;
+
+struct KE_EVENT_MULTI_WAIT_CONTEXT {
+    THREAD* Thread;
+    KE_EVENT_WAIT_BLOCK* Blocks;
+    uint32_t Count;
+    volatile long Completed;
+    KSTATUS Status;
+    uint32_t SelectedIndex;
+};
 
 // Implements the event Destroy operation.
 static void KiEventDestroy(OBJECT* obj) { (void)obj; }
 
-// Implements the event Create operation.
+// Completes a multi-wait block while the source event lock is held.
+static void KiCompleteMultiWaitBlockLocked(KE_EVENT_WAIT_BLOCK* block) {
+    if (!block || !block->Armed)
+        return;
+
+    ListRemove(&block->Link);
+    block->Armed = 0;
+
+    KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
+    if (!ctx)
+        return;
+
+    if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) != 0)
+        return;
+
+    ctx->Status = STATUS_SUCCESS;
+    ctx->SelectedIndex = block->Index;
+    KeUnblockThreadFromWaitListWithStatus(ctx->Thread, STATUS_SUCCESS);
+}
+
+// Removes any wait blocks that are still linked after the wait completed.
+static void KiCleanupMultiWaitBlocks(KE_EVENT_MULTI_WAIT_CONTEXT* ctx) {
+    if (!ctx || !ctx->Blocks)
+        return;
+
+    for (uint32_t i = 0; i < ctx->Count; i++) {
+        KE_EVENT_WAIT_BLOCK* block = &ctx->Blocks[i];
+        EVENT* event = block->Event;
+        if (!event)
+            continue;
+
+        uintptr_t flags = KeAcquireSpinlock(&event->Lock);
+        if (block->Armed) {
+            ListRemove(&block->Link);
+            block->Armed = 0;
+        }
+        KeReleaseSpinlock(&event->Lock, flags);
+    }
+}
+
+// Removes a multi-wait from every event list during abort or thread teardown.
+static void KiRundownMultiWait(THREAD* thread, void* context, KSTATUS status) {
+    KE_EVENT_MULTI_WAIT_CONTEXT* ctx = (KE_EVENT_MULTI_WAIT_CONTEXT*)context;
+    if (!ctx)
+        return;
+
+    if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) == 0) {
+        ctx->Status = status;
+        ctx->SelectedIndex = 0;
+    }
+
+    KiCleanupMultiWaitBlocks(ctx);
+
+    // Abort wakes a truly blocked waiter. Thread teardown may call this after a
+    // signal already made the thread ready; in that case only the list cleanup
+    // above is needed.
+    if (thread && thread->State == ThreadWaiting)
+        KeUnblockThreadFromWaitListWithStatus(thread, status);
+}
+
+// Creates event.
 __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
     if (!out) return STATUS_INVALID_PARAMETER;
     EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
@@ -21,6 +104,7 @@ __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialSta
     e->Type     = type;
     e->Signaled = initialState ? 1 : 0;
     LIST_INIT(&e->WaitList);
+    LIST_INIT(&e->MultiWaitList);
     *out = e;
     return STATUS_SUCCESS;
 }
@@ -35,22 +119,39 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
         while (!LIST_IS_EMPTY(&event->WaitList)) {
             LIST_ENTRY* node = ListPopHead(&event->WaitList);
             THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+            KeSetThreadWaitList(t, (LIST_ENTRY*)0);
             KeReleaseSpinlock(&event->Lock, flags);
             KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
             flags = KeAcquireSpinlock(&event->Lock);
         }
+        while (!LIST_IS_EMPTY(&event->MultiWaitList)) {
+            LIST_ENTRY* node = event->MultiWaitList.Fwd;
+            KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link);
+            KiCompleteMultiWaitBlockLocked(block);
+        }
     } else {
-        if (!LIST_IS_EMPTY(&event->WaitList)) {
+        while (!LIST_IS_EMPTY(&event->WaitList)) {
-            // Auto-reset handoff: transfer the signal directly to exactly one
-            // waiter. Do not leave Signaled=1, because a later waiter can steal
-            // the disk/APC completion before the selected thread resumes.
             event->Signaled = 0;
             LIST_ENTRY* node = ListPopHead(&event->WaitList);
             THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+            KeSetThreadWaitList(t, (LIST_ENTRY*)0);
             KeReleaseSpinlock(&event->Lock, flags);
             KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
             return;
         }
+
+        while (!LIST_IS_EMPTY(&event->MultiWaitList)) {
+            LIST_ENTRY* node = event->MultiWaitList.Fwd;
+            KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link);
+            KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
+            if (ctx && _InterlockedCompareExchange(&ctx->Completed, 0, 0) == 0) {
+                event->Signaled = 0;
+                KiCompleteMultiWaitBlockLocked(block);
+                KeReleaseSpinlock(&event->Lock, flags);
+                return;
+            }
+            KiCompleteMultiWaitBlockLocked(block);
+        }
         event->Signaled = 1;
     }
     KeReleaseSpinlock(&event->Lock, flags);
@@ -64,7 +165,22 @@ __declspec(dllexport) void KeClearEvent(EVENT* event) {
     KeReleaseSpinlock(&event->Lock, flags);
 }
 
-// Waits for for Event.
+// Attempts to consume an event signal without blocking.
+__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event) {
+    if (!event) return STATUS_INVALID_PARAMETER;
+    uintptr_t flags = KeAcquireSpinlock(&event->Lock);
+    if (!event->Signaled) {
+        KeReleaseSpinlock(&event->Lock, flags);
+        return STATUS_TIMEOUT;
+    }
+
+    if (event->Type == EventAutoReset)
+        event->Signaled = 0;
+    KeReleaseSpinlock(&event->Lock, flags);
+    return STATUS_SUCCESS;
+}
+
+// Waits for Event.
 __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
     if (!event) return STATUS_INVALID_PARAMETER;
     if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -85,3 +201,85 @@ __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
     return KeCommitBlockThread(self, alertable);
 }
 
+// Waits for any of a set of events, returning the selected index.
+__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count,
+                                                     uint8_t alertable, uint64_t timeoutMs,
+                                                     uint32_t* signaledIndexOut) {
+    if (!events || count == 0 || count > SYS_WAIT_MANY_MAX_HANDLES)
+        return STATUS_INVALID_PARAMETER;
+    if (!KeCanWait()) return STATUS_INVALID_IRQL;
+    if (alertable && KeAreKernelApcsDisabled()) return STATUS_INVALID_PARAMETER;
+
+    for (uint32_t i = 0; i < count; i++) {
+        if (!events[i]) return STATUS_INVALID_PARAMETER;
+        KSTATUS s = KeTryWaitEvent(events[i]);
+        if (KSUCCESS(s)) {
+            if (signaledIndexOut) *signaledIndexOut = i;
+            return STATUS_SUCCESS;
+        }
+    }
+
+    if (timeoutMs == 0)
+        return STATUS_TIMEOUT;
+
+    KE_EVENT_WAIT_BLOCK stackBlocks[SYS_WAIT_MANY_MAX_HANDLES];
+    KE_EVENT_MULTI_WAIT_CONTEXT ctx;
+    ctx.Thread = KeGetCurrentThread();
+    ctx.Blocks = stackBlocks;
+    ctx.Count = count;
+    ctx.Completed = 0;
+    ctx.Status = STATUS_PENDING;
+    ctx.SelectedIndex = 0;
+
+    for (uint32_t i = 0; i < count; i++) {
+        stackBlocks[i].Context = &ctx;
+        stackBlocks[i].Event = events[i];
+        stackBlocks[i].Index = i;
+        stackBlocks[i].Armed = 0;
+        LIST_INIT(&stackBlocks[i].Link);
+    }
+
+    KePrepareBlockThread(ctx.Thread, (OBJECT*)0, alertable);
+    KeSetThreadWaitRundown(ctx.Thread, KiRundownMultiWait, &ctx);
+
+    if (timeoutMs != SYS_WAIT_TIMEOUT_INFINITE) {
+        KSTATUS ts = KeArmCurrentThreadTimeout(timeoutMs, STATUS_TIMEOUT);
+        if (!KSUCCESS(ts)) {
+            KeClearThreadWaitRundown(ctx.Thread);
+            KeUnblockThreadFromWaitListWithStatus(ctx.Thread, ts);
+            return ts;
+        }
+    }
+
+    for (uint32_t i = 0; i < count; i++) {
+        EVENT* event = events[i];
+        uintptr_t flags = KeAcquireSpinlock(&event->Lock);
+        if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0 && event->Signaled) {
+            if (event->Type == EventAutoReset)
+                event->Signaled = 0;
+            ctx.Status = STATUS_SUCCESS;
+            ctx.SelectedIndex = i;
+            _InterlockedExchange(&ctx.Completed, 1);
+            KeReleaseSpinlock(&event->Lock, flags);
+            KeUnblockThreadFromWaitListWithStatus(ctx.Thread, STATUS_SUCCESS);
+            break;
+        }
+
+        if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0) {
+            stackBlocks[i].Armed = 1;
+            ListInsertTail(&event->MultiWaitList, &stackBlocks[i].Link);
+        }
+        KeReleaseSpinlock(&event->Lock, flags);
+    }
+
+    KSTATUS waitStatus = KeCommitBlockThread(ctx.Thread, alertable);
+    if (_InterlockedCompareExchange(&ctx.Completed, 1, 0) == 0)
+        ctx.Status = waitStatus;
+
+    KiCleanupMultiWaitBlocks(&ctx);
+    KeClearThreadWaitRundown(ctx.Thread);
+
+    if (ctx.Status == STATUS_SUCCESS && signaledIndexOut)
+        *signaledIndexOut = ctx.SelectedIndex;
+    return ctx.Status;
+}

code/kernel/ke_input.c

@@ -8,6 +8,8 @@
 #include <kernel/ke_input.h>
 #include <kernel/ke_debug.h>
 #include <kernel/ke_display.h>
+#include <kernel/ke_event.h>
+#include <kernel/ob_object.h>
 
 #define KEY_QUEUE_SIZE   256u
 #define KEY_QUEUE_MASK   (KEY_QUEUE_SIZE - 1u)
@@ -32,6 +34,7 @@ static volatile uint32_t gModifiers;
 typedef struct INPUT_KEY_QUEUE_ENTRY {
     INPUT_KEY key;
     char      ch;
+    uint32_t  modifiers;
 } INPUT_KEY_QUEUE_ENTRY;
 
 static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE];
@@ -50,6 +53,8 @@ static volatile int      gMouseBoundsH = 1080;
 static volatile uint8_t  gMouseButtons;
 
 static volatile uint32_t gNextInputDeviceId = 1;
+static EVENT* gKeyboardEvent;
+static EVENT* gInputStateEvent;
 
 static char KiInputKeyToChar(INPUT_KEY key);
 
@@ -77,14 +82,18 @@ static const KEY_CHAR_PAIR gCharMap[] = {
     { INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
 };
 
-// Implements the input Queue Key operation.
+// Queues a keyboard event and wakes one blocked reader when the event was accepted.
 static void KiInputQueueKey(INPUT_KEY key, char ch) {
     if (key == INPUT_KEY_NONE) return;
     int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
     if (next == gKeyHead) return;
     gKeyQueue[gKeyTail].key = key;
     gKeyQueue[gKeyTail].ch = ch;
+    gKeyQueue[gKeyTail].modifiers = gModifiers;
     gKeyTail = next;
+
+    if (gKeyboardEvent)
+        KeSignalEvent(gKeyboardEvent);
 }
 
 // Implements the input Queue Has Space operation.
@@ -211,6 +220,17 @@ void KeInputInitialize(const HAL_API* hal) {
     gMouseHead = gMouseTail = 0;
     gMouseButtons = 0;
 
+    if (!gKeyboardEvent) {
+        KSTATUS s = KeEventCreate(EventAutoReset, 0, &gKeyboardEvent);
+        if (!KSUCCESS(s))
+            DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
+    }
+    if (!gInputStateEvent) {
+        KSTATUS s = KeEventCreate(EventAutoReset, 0, &gInputStateEvent);
+        if (!KSUCCESS(s))
+            DbgPrint("Input: state wait event create failed st=0x%x\n", s);
+    }
+
     int w = 1920, h = 1080;
     DISPLAY_MODE mode;
     if (KSUCCESS(KeDisplayGetMode(&mode))) {
@@ -252,6 +272,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
 
         gKeyDown[key] = 1u;
         KiInputSetModifier(key, true);
+        if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
         KiInputQueueKey(key, KiInputKeyToChar(key));
         KiInputStartRepeat(key, KiInputNowTicks());
     } else {
@@ -260,6 +281,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
 
         gKeyDown[key] = 0u;
         KiInputSetModifier(key, false);
+        if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
         KiInputStopRepeat(key);
     }
 }
@@ -341,27 +363,60 @@ INPUT_KEY KeInputKbPoll(void) {
     return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
 }
 
+
+// Waits until at least one keyboard event is available.
+KSTATUS KeInputWaitKey(uint8_t alertable) {
+    if (KeInputKbPoll() != INPUT_KEY_NONE)
+        return STATUS_SUCCESS;
+    if (!gKeyboardEvent)
+        return STATUS_FAILURE;
+
+    for (;;) {
+        KSTATUS s = KeWaitForEvent(gKeyboardEvent, alertable);
+        if (!KSUCCESS(s))
+            return s;
+        if (KeInputKbPoll() != INPUT_KEY_NONE)
+            return STATUS_SUCCESS;
+    }
+}
+
 // Implements the input Is Key Down operation.
 bool KeInputIsKeyDown(INPUT_KEY key) {
     if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
     return gKeyDown[key] != 0;
 }
 
-INPUT_KEY KeInputReadKey(void) {
+// Implements the input Peek Event operation.
+bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
+    if (!record) return false;
     if (gKeyHead == gKeyTail) KiInputServiceRepeat();
-    if (gKeyHead == gKeyTail) return INPUT_KEY_NONE;
+    if (gKeyHead == gKeyTail) return false;
-    INPUT_KEY key = gKeyQueue[gKeyHead].key;
+
+    INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead];
+    record->Key = entry.key;
+    record->Char = entry.ch;
+    record->Reserved = 0;
+    record->Modifiers = entry.modifiers;
+    return true;
+}
+
+// Implements the input Read Event operation.
+bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
+    if (!KeInputPeekEvent(record)) return false;
     gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
-    return key;
+    return true;
+}
+
+// Implements the input Read Key operation.
+INPUT_KEY KeInputReadKey(void) {
+    INPUT_KEY_RECORD record;
+    return KeInputReadEvent(&record) ? record.Key : INPUT_KEY_NONE;
 }
 
 // Implements the input Read Char operation.
 char KeInputReadChar(void) {
-    if (gKeyHead == gKeyTail) KiInputServiceRepeat();
+    INPUT_KEY_RECORD record;
-    if (gKeyHead == gKeyTail) return 0;
+    return KeInputReadEvent(&record) ? record.Char : 0;
-    char ch = gKeyQueue[gKeyHead].ch;
-    gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
-    return ch;
 }
 
 // Implements the input Get Modifiers operation.
@@ -416,3 +471,32 @@ void KeInputMouseSetBounds(int w, int h) {
     gMouseBoundsH = h;
     KeInputMouseSetPosition(gMouseX, gMouseY);
 }
+
+// Destroys an input queue handle.
+static void KiInputQueueDestroy(OBJECT* object) {
+    INPUT_QUEUE* queue = CONTAINING_RECORD(object, INPUT_QUEUE, Header);
+    if (queue->WaitEvent) {
+        ObDereferenceObject(&queue->WaitEvent->Header);
+        queue->WaitEvent = (EVENT*)0;
+    }
+}
+
+// Opens the default keyboard input queue as a waitable object.
+KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
+    if (!out) return STATUS_INVALID_PARAMETER;
+    *out = (INPUT_QUEUE*)0;
+    if (!gInputStateEvent) return STATUS_FAILURE;
+
+    INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
+    if (!queue) return STATUS_NO_MEMORY;
+    queue->Header.Destroy = KiInputQueueDestroy;
+    ObReferenceObject(&gInputStateEvent->Header);
+    queue->WaitEvent = gInputStateEvent;
+    *out = queue;
+    return STATUS_SUCCESS;
+}
+
+// Returns the event that wakes when keyboard state changes.
+EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
+    return queue ? queue->WaitEvent : (EVENT*)0;
+}

code/kernel/ke_irp.c

@@ -138,7 +138,7 @@ static size_t IoIrpSize(uint8_t stackCount) {
 __declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
     if (!outIrp) return STATUS_INVALID_PARAMETER;
     if (stackCount == 0) stackCount = 1;
-    IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount));
+    IRP* irp = (IRP*)MmAllocatePoolTag(IoIrpSize(stackCount), MM_POOL_TAG('i', 'r', 'p', ' '));
     if (!irp) return STATUS_NO_MEMORY;
     IoInitializeIrp(irp, stackCount);
     *outIrp = irp;

code/kernel/ke_mutex.c

@@ -12,7 +12,7 @@
 static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
 
 // Implements the mutex Create operation.
-KSTATUS KeMutexCreate(MUTEX **out) {
+__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) {
     if (!out) return STATUS_INVALID_PARAMETER;
     MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
     if (!m)  return STATUS_NO_MEMORY;
@@ -25,7 +25,7 @@ KSTATUS KeMutexCreate(MUTEX **out) {
 }
 
 // Implements the acquire Mutex operation.
-KSTATUS KeAcquireMutex(MUTEX* mutex) {
+__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) {
     if (!mutex) return STATUS_INVALID_PARAMETER;
     if (!KeCanWait()) return STATUS_INVALID_IRQL;
     THREAD* self = KeGetCurrentThread();
@@ -45,7 +45,7 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) {
 }
 
 // Implements the try Acquire Mutex operation.
-KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
+__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
     if (!mutex) return STATUS_INVALID_PARAMETER;
     THREAD* self = KeGetCurrentThread();
     uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
@@ -59,13 +59,14 @@ KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
 }
 
 // Releases mutex.
-void KeReleaseMutex(MUTEX* mutex) {
+__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
     if (!mutex) return;
     uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
 
     if (!LIST_IS_EMPTY(&mutex->WaitList)) {
         LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
         THREAD* waiter   = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+        KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
         // Direct handoff: the selected waiter owns the mutex when its wait
         // returns. Do not drop Owner to NULL, or another thread can steal it.
         mutex->Owner = waiter;

code/kernel/ke_process.c

@@ -6,7 +6,9 @@
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
 #include <kernel/ke_event.h>
+#include <kernel/ke_system_root.h>
 #include <kernel/ke_userenv.h>
+#include <kernel/io_file.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_cpu.h>
@@ -140,7 +142,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
             if (p == system) info->Flags |= SYS_PROCESS_FLAG_SYSTEM;
             if (p == current) info->Flags |= SYS_PROCESS_FLAG_CURRENT;
 
-            MmQueryProcessUserMemory(p, &info->UserRegionCount,
+            MmVadQueryProcessMemory(p, &info->VadCount,
                                      &info->UserReservedBytes,
                                      &info->UserCommittedBytes);
 
@@ -158,27 +160,128 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
     return count;
 }
 
-// Destructor (called by ObDereferenceObject when RefCount hits zero).
+// Converts a kernel thread state to the public ABI state.
-static void KiProcessDestroy(OBJECT* obj) {
+static uint32_t KiThreadStateToSys(THREAD_STATE state) {
-    PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
+    switch (state) {
+        case ThreadReady:   return SYS_THREAD_STATE_READY;
+        case ThreadRunning: return SYS_THREAD_STATE_RUNNING;
+        case ThreadWaiting: return SYS_THREAD_STATE_WAITING;
+        case ThreadDead:    return SYS_THREAD_STATE_DEAD;
+        default:            return SYS_THREAD_STATE_DEAD;
+    }
+}
 
-    if (proc == gSystemProcess)
+// Converts a kernel object type to the public wait-object type.
-        gSystemProcess = (PROCESS*)0;
+static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
+    switch (type) {
+        case ObjectTypeProcess:     return SYS_THREAD_WAIT_PROCESS;
+        case ObjectTypeThread:      return SYS_THREAD_WAIT_THREAD;
+        case ObjectTypeMutex:       return SYS_THREAD_WAIT_MUTEX;
+        case ObjectTypeSemaphore:   return SYS_THREAD_WAIT_SEMAPHORE;
+        case ObjectTypeEvent:       return SYS_THREAD_WAIT_EVENT;
+        case ObjectTypeQueue:       return SYS_THREAD_WAIT_QUEUE;
+        case ObjectTypeFile:        return SYS_THREAD_WAIT_FILE;
+        case ObjectTypeDir:         return SYS_THREAD_WAIT_DIR;
+        case ObjectTypeAsyncIo:     return SYS_THREAD_WAIT_ASYNC_IO;
+        case ObjectTypeSocket:      return SYS_THREAD_WAIT_SOCKET;
+        case ObjectTypeDevice:      return SYS_THREAD_WAIT_DEVICE;
+        case ObjectTypeDriver:      return SYS_THREAD_WAIT_DRIVER;
+        case ObjectTypeUsbDevice:   return SYS_THREAD_WAIT_USB_DEVICE;
+        case ObjectTypeUsbPipe:     return SYS_THREAD_WAIT_USB_PIPE;
+        case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING;
+        case ObjectTypeSurface:     return SYS_THREAD_WAIT_SURFACE;
+        default:                    return SYS_THREAD_WAIT_UNKNOWN;
+    }
+}
 
-    // Release all remaining handles.
+// Copies one process's thread telemetry into caller-provided storage.
-    for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
+KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut) {
-        if (proc->Handles[i].Object) {
+    if (!countOut) return STATUS_INVALID_PARAMETER;
-            ObDereferenceObject(proc->Handles[i].Object);
+    if (maxCount && !infoOut) return STATUS_INVALID_PARAMETER;
+
+    PROCESS* process = KeGetProcessById(pid);
+    if (!process) return STATUS_NOT_FOUND;
+
+    THREAD* current = KeGetCurrentThread();
+    size_t count = 0;
+
+    uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
+    LIST_ENTRY* head = &process->ThreadList;
+    for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
+        THREAD* thread = CONTAINING_RECORD(node, THREAD, ProcessLink);
+        if (infoOut && count < maxCount) {
+            SYS_THREAD_INFO* info = &infoOut[count];
+            memset(info, 0, sizeof(*info));
+            info->Size = sizeof(*info);
+            info->ProcessId = process->ProcessId;
+            info->ThreadId = thread->ThreadId;
+            info->State = KiThreadStateToSys(thread->State);
+            info->Priority = (uint32_t)thread->Priority;
+            info->RunningCpu = thread->RunningCpu;
+            info->LastCpu = thread->LastCpu;
+            info->QuantumRemaining = thread->QuantumRemaining;
+            info->QuantumTicks = thread->QuantumTicks;
+            info->CpuBurstTicks = thread->CpuBurstTicks;
+            info->LastRunTicks = thread->LastRunTicks;
+            info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE;
+            info->WaitStatus = (uint32_t)thread->WaitStatus;
+            info->KernelApcDisable = thread->KernelApcDisable;
+
+            if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT;
+            if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE;
+            if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL;
+            if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED;
+            if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT;
+            if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING;
+            if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING;
+        }
+        count++;
+    }
+    KeReleaseSpinlock(&process->HandleLock, flags);
+
+    *countOut = count;
+    ObDereferenceObject(&process->Header);
+    return STATUS_SUCCESS;
+}
+
+// Releases all kernel objects still reachable from a process handle table.
+// This is part of process exit, not merely object destruction: a parent may keep
+// a handle to an exited process for wait/query, but the dead process must not
+// keep its own files, mappings, queues, sockets, or synchronization objects alive.
+static void KiProcessReleaseAllHandles(PROCESS* proc) {
+    if (!proc) return;
+
+    for (uint32_t i = 1; i < HANDLE_TABLE_SIZE; ++i) {
+        OBJECT* obj = (OBJECT*)0;
+
+        uintptr_t flags = KeAcquireSpinlock(&proc->HandleLock);
+        obj = proc->Handles[i].Object;
+        if (obj) {
             proc->Handles[i].Object = (OBJECT*)0;
             proc->Handles[i].Rights = 0;
             proc->Handles[i].Flags = 0;
         }
+        KeReleaseSpinlock(&proc->HandleLock, flags);
+
+        if (obj)
+            ObDereferenceObject(obj);
     }
+}
+
+// Releases the process-owned address space and per-process kernel resources.
+// Idempotent because the normal exit path and the final OBJECT destructor can
+// both observe a process whose threads are already gone.
+static void KiProcessRundownResources(PROCESS* proc) {
+    if (!proc) return;
+    if (_InterlockedExchange(&proc->ResourcesReleased, 1) != 0)
+        return;
+
+    KiProcessReleaseAllHandles(proc);
 
     // Release every user VA region owned by this process. This includes the
     // main image, DLLs, TEBs/stacks that survived thread teardown, process
-    // environment strings, and leaked user allocations.
+    // environment strings, mmap views, and leaked user allocations.
-    MmDestroyAllUserRegions(proc);
+    MmVadDestroyAllForProcess(proc);
     proc->ImageBase = 0;
     proc->ImageSize = 0;
     proc->ModuleCount = 0;
@@ -190,8 +293,21 @@ static void KiProcessDestroy(OBJECT* obj) {
     proc->UserCmdlineSize = 0;
     proc->UserCwdBase = 0;
     proc->UserCwdSize = 0;
+}
 
-    // Release the exit event.
+// Destructor (called by ObDereferenceObject when RefCount hits zero).
+static void KiProcessDestroy(OBJECT* obj) {
+    PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
+
+    if (proc == gSystemProcess)
+        gSystemProcess = (PROCESS*)0;
+
+    // If the process failed during launch, or no normal exit completion path
+    // ran, make sure its own handles and VADs are still reclaimed.
+    KiProcessRundownResources(proc);
+
+    // Release the exit event only when the process object itself dies. External
+    // process handles may still wait/query an exited process after rundown.
     if (proc->ExitEvent) {
         ObDereferenceObject(&proc->ExitEvent->Header);
         proc->ExitEvent = (EVENT *)0;
@@ -227,6 +343,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
     proc->ExitCode    = 0;
     proc->Terminating = 0;
     proc->Exited      = 0;
+    proc->ResourcesReleased = 0;
     proc->ImageBase   = 0;
     proc->ImageSize   = 0;
     proc->UserPebBase = 0;
@@ -240,10 +357,10 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
     proc->ModuleCount = 0;
     proc->CurrentDirectory[0] = '\0';
     proc->CurrentDirectoryLength = 0;
-    proc->CurrentDrive = 0;
+    proc->CurrentDrive = KeGetSystemRootDrive();
     proc->CwdLock = (SPINLOCK)SPINLOCK_INIT;
-    LIST_INIT(&proc->UserRegions);
+    LIST_INIT(&proc->VadList);
-    proc->UserRegionLock = (SPINLOCK)SPINLOCK_INIT;
+    proc->VadLock = (SPINLOCK)SPINLOCK_INIT;
 
     // Slot 0 is the invalid handle - leave it NULL.
     memset(proc->Handles, 0, sizeof(proc->Handles));
@@ -427,7 +544,7 @@ fail:
 }
 
 // Process exit / termination.
-static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
+void KeCompleteProcessExitIfLastThread(PROCESS* process) {
     if (!process) return;
 
     uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
@@ -435,6 +552,7 @@ static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
     KeReleaseSpinlock(&process->HandleLock, flags);
 
     if (threadCount == 0 && _InterlockedExchange(&process->Exited, 1) == 0) {
+        KiProcessRundownResources(process);
         if (process->ExitEvent) KeSignalEvent(process->ExitEvent);
     }
 }
@@ -448,7 +566,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
     // The calling thread exits immediately after this in SysExitProcess. Do
     // not drop the process object here: each thread owns one process reference,
     // and KiThreadDestroy releases that reference after unlinking the thread.
-    KiCompleteProcessExitIfLastThread(process);
+    KeCompleteProcessExitIfLastThread(process);
 }
 
 // Implements the terminate Process operation.
@@ -469,6 +587,14 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
         if (t == self)
             continue;
 
+        // A terminating process must not be held open by worker threads that
+        // are blocked in non-alertable waits.  User-mode DLLs such as Bink can
+        // leave IO/submit workers parked on semaphores; after the foreground
+        // thread crashes, those waiters would otherwise keep ThreadCount > 0
+        // and prevent the process ExitEvent from waking the shell.
+        if (t->State == ThreadWaiting)
+            KeAbortThreadWait(t, STATUS_ALERTED);
+        else
             (void)KeAlertThread(t);
 
         if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
@@ -481,7 +607,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
     }
     KeReleaseSpinlock(&process->HandleLock, flags);
 
-    KiCompleteProcessExitIfLastThread(process);
+    KeCompleteProcessExitIfLastThread(process);
     return STATUS_SUCCESS;
 }
 
@@ -535,6 +661,8 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) {
         process->Handles[handle].Rights = 0;
         process->Handles[handle].Flags = 0;
         KeReleaseSpinlock(&process->HandleLock, flags);
+        if (obj->Type == ObjectTypeFile)
+            (void)IoFlushFile((FILE_OBJECT*)obj);
         ObDereferenceObject(obj);
         return;
     }

code/kernel/ke_sched.c

@@ -17,6 +17,7 @@
 #include <kernel/hal/hal_apic.h>
 
 static LIST_ENTRY gRunQueue[PriorityCount];
+static LIST_ENTRY gSleepList;
 static SPINLOCK   gSchedLock;
 
 static uint32_t gAgingCounter = 0;
@@ -305,6 +306,7 @@ static void KiIdleLoop(void) {
 // Initializes scheduler Global.
 void KiInitializeSchedulerGlobal(void) {
     for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
+    LIST_INIT(&gSleepList);
     gSchedLock = (SPINLOCK)SPINLOCK_INIT;
 
     KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
@@ -349,6 +351,8 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
     idle->WaitStatus      = STATUS_SUCCESS;
     idle->WaitListHead    = (LIST_ENTRY*)0;
     idle->AlertableWait   = 0;
+    idle->SleepUntilTick  = 0;
+    idle->TimeoutStatus   = STATUS_SUCCESS;
     idle->UserStackBase   = (void*)0;
     idle->Teb             = (void*)0;
     idle->TebSize         = 0;
@@ -358,7 +362,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
     LIST_INIT(&idle->ProcessLink);
     LIST_INIT(&idle->SchedulerLink);
 
-    void* kstack = MmAllocatePool(KSTACK_SIZE);
+    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
     if (!kstack) {
         ObDereferenceObject(&idle->Header);
         return STATUS_NO_MEMORY;
@@ -621,12 +625,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
 }
 
 
-// Implements the scheduler Wake Thread operation.
+// Completes a wait while the scheduler lock is held.
-void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
+static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
     if (!thread) return;
 
-    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
-
     // Complete the wait while holding the scheduler lock. On SMP a wake can
     // race the prepare->commit window on another CPU: the target thread is
     // already on the object's wait list and has State=ThreadWaiting, but it is
@@ -634,7 +636,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
     // That case must complete in-place, not enqueue the still-running thread.
     if (thread->State != ThreadWaiting) {
         thread->WaitStatus = status;
-        KeReleaseSpinlock(&gSchedLock, flags);
         return;
     }
 
@@ -642,6 +643,8 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
     thread->WaitStatus    = status;
     thread->WaitListHead  = (LIST_ENTRY*)0;
     thread->AlertableWait = 0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
 
     thread->CpuBurstTicks = 0;
     thread->QuantaConsumed = 0;
@@ -652,7 +655,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
         // KeCommitBlockThread() will see State != ThreadWaiting and return the
         // completed wait status without ever entering the ready queue.
         thread->State = ThreadRunning;
-        KeReleaseSpinlock(&gSchedLock, flags);
         return;
     }
 
@@ -662,10 +664,49 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
 
     KiInsertReadyLocked(thread);
     KiPokeRunnableCpusLocked(thread);
+}
 
+
+// Wakes a thread after first removing its scheduler-link wait membership.
+void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
+    if (!thread) return;
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    if (thread->State == ThreadWaiting && thread->WaitListHead) {
+        ListRemove(&thread->SchedulerLink);
+        thread->WaitListHead = (LIST_ENTRY*)0;
+        thread->SleepUntilTick = 0;
+        thread->TimeoutStatus = STATUS_SUCCESS;
+    }
+    KiWakeThreadLocked(thread, status);
     KeReleaseSpinlock(&gSchedLock, flags);
 }
 
+// Implements the scheduler Wake Thread operation.
+void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    KiWakeThreadLocked(thread, status);
+    KeReleaseSpinlock(&gSchedLock, flags);
+}
+
+// Wakes threads whose scheduler-tick sleep deadline has expired.
+static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
+    LIST_ENTRY* node = gSleepList.Fwd;
+    while (node != &gSleepList) {
+        LIST_ENTRY* next = node->Fwd;
+        THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+
+        if (thread->State == ThreadWaiting &&
+            thread->WaitListHead == &gSleepList &&
+            thread->SleepUntilTick <= nowTick) {
+            ListRemove(node);
+            thread->WaitListHead = (LIST_ENTRY*)0;
+            KiWakeThreadLocked(thread, thread->TimeoutStatus);
+        }
+
+        node = next;
+    }
+}
+
 // Implements the scheduler Tick operation.
 void KeSchedulerTick(void) {
     KCPU* cpu = KeGetCurrentCpu();
@@ -677,6 +718,7 @@ void KeSchedulerTick(void) {
     KiArmPostSwitchUnlock(cur, flags);
 
     gSchedStats.Ticks++;
+    KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
 
     int requestReschedule = 0;
 
@@ -745,6 +787,90 @@ void KeSchedulerTick(void) {
         KiRequestReschedule();
 }
 
+
+// Arms a timeout for the current thread after a wait primitive has prepared it to block.
+KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) {
+    if (milliseconds == 0)
+        return STATUS_INVALID_PARAMETER;
+    if (!KeCanWait())
+        return STATUS_INVALID_IRQL;
+
+    THREAD* self = KeGetCurrentThread();
+    if (!self || self->IsIdleThread)
+        return STATUS_INVALID_PARAMETER;
+
+    uint64_t ticks = milliseconds / SCHED_TICK_MS;
+    if ((milliseconds % SCHED_TICK_MS) != 0)
+        ticks++;
+    if (ticks == 0)
+        ticks = 1;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    if (self->State != ThreadWaiting || self->WaitListHead != (LIST_ENTRY*)0 ||
+        self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
+        KeReleaseSpinlock(&gSchedLock, flags);
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    uint64_t now = gSchedStats.Ticks;
+    uint64_t wakeTick = now + ticks;
+    if (wakeTick < now)
+        wakeTick = UINT64_MAX;
+
+    self->WaitListHead = &gSleepList;
+    self->SleepUntilTick = wakeTick;
+    self->TimeoutStatus = timeoutStatus;
+    ListInsertTail(&gSleepList, &self->SchedulerLink);
+    KeReleaseSpinlock(&gSchedLock, flags);
+    return STATUS_SUCCESS;
+}
+
+// Implements the delay Thread operation.
+__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
+    if (milliseconds == 0) {
+        KeYield();
+        return STATUS_SUCCESS;
+    }
+
+    if (!KeCanWait())
+        return STATUS_INVALID_IRQL;
+
+    THREAD* self = KeGetCurrentThread();
+    if (!self || self->IsIdleThread)
+        return STATUS_INVALID_PARAMETER;
+
+    uint64_t ticks = milliseconds / SCHED_TICK_MS;
+    if ((milliseconds % SCHED_TICK_MS) != 0)
+        ticks++;
+    if (ticks == 0)
+        ticks = 1;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+
+    if (self->Queued || self->State != ThreadRunning ||
+        self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
+        KeReleaseSpinlock(&gSchedLock, flags);
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    uint64_t now = gSchedStats.Ticks;
+    uint64_t wakeTick = now + ticks;
+    if (wakeTick < now)
+        wakeTick = UINT64_MAX;
+
+    self->State = ThreadWaiting;
+    self->WaitObject = (OBJECT*)0;
+    self->WaitStatus = STATUS_PENDING;
+    self->WaitListHead = &gSleepList;
+    self->AlertableWait = 0;
+    self->SleepUntilTick = wakeTick;
+    self->TimeoutStatus = STATUS_SUCCESS;
+    ListInsertTail(&gSleepList, &self->SchedulerLink);
+
+    KeReleaseSpinlock(&gSchedLock, flags);
+    return KeCommitBlockThread(self, 0);
+}
+
 // Implements the yield operation.
 __declspec(dllexport) void KeYield(void) {
     KeAssertIrql(PASSIVE_LEVEL);

code/kernel/ke_selftest.c

@@ -14,10 +14,13 @@
 #include <kernel/ke_sched.h>
 #include <ntlite/status.h>
 #include <kernel/ke_thread.h>
+#include <kernel/ke_process.h>
 #include <kernel/io_file.h>
 #include <ntlite/ddi/io_driver.h>
 #include <kernel/ke_trap.h>
 #include <kernel/mm_virt.h>
+#include <kernel/mm_swap.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_smp.h>
 
@@ -478,11 +481,118 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
 
 
 
-// Virtual memory mapping invariants.
+
+// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
+static void KiRunPfnDatabaseSelfTest(void) {
+    MM_PFN_STATS before;
+    MM_PFN_STATS after;
+    MmPfnQueryStats(&before);
+
+    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
+    if (!frame)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n");
+
+    MM_PFN* pfn = MmPfnGet(frame);
+    if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "PFN selftest: allocated frame metadata invalid phys=%016llx\n",
+                (unsigned long long)frame);
+
+    if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount);
+    if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount);
+
+    if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount);
+    if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n");
+    if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount);
+    if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount);
+    if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n");
+    if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount);
+    if (!KSUCCESS(MmPfnFree(frame)))
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n");
+
+    PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma);
+    if (!run)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n");
+    MM_PFN* p0 = MmPfnGet(run);
+    MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE);
+    if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n");
+    MmPfnFree(run);
+    MmPfnFree(run + PAGE_SIZE);
+
+    // Accounting buckets are part of the VM contract. A missing increment here
+    // usually means a new allocation path is using an anonymous usage class.
+    PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache);
+    PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage);
+    PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile);
+    PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous);
+    PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow);
+    PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface);
+    if (!clean || !image || !mapped || !anon || !cow || !surface)
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n");
+
+    MM_PFN_STATS bucketStats;
+    MmPfnQueryStats(&bucketStats);
+    if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 ||
+        bucketStats.ImageFrames < before.ImageFrames + 1 ||
+        bucketStats.MappedFileFrames < before.MappedFileFrames + 1 ||
+        bucketStats.AnonymousFrames < before.AnonymousFrames + 1 ||
+        bucketStats.CowFrames < before.CowFrames + 1 ||
+        bucketStats.SurfaceFrames < before.SurfaceFrames + 1) {
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n");
+    }
+
+    // Standby is the clean-page holding state used by later file-cache and image
+    // reclaim. It must not change the free count until the owning cache drops it.
+    if (!KSUCCESS(MmPfnMoveToStandby(clean)))
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n");
+    if (!KSUCCESS(MmPfnMoveToStandby(image)))
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n");
+
+    MM_PFN_STATS standbyStats;
+    MmPfnQueryStats(&standbyStats);
+    if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 ||
+        standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 ||
+        standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 ||
+        standbyStats.FreeFrames != bucketStats.FreeFrames) {
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n");
+    }
+
+    if (!KSUCCESS(MmPfnActivate(clean)))
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n");
+    if (!KSUCCESS(MmPfnActivate(image)))
+        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n");
+
+    (void)MmPfnFree(clean);
+    (void)MmPfnFree(image);
+    (void)MmPfnFree(mapped);
+    (void)MmPfnFree(anon);
+    (void)MmPfnFree(cow);
+    (void)MmPfnFree(surface);
+
+    MmPfnQueryStats(&after);
+    if (before.FreeFrames != after.FreeFrames)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "PFN selftest: free count leak before=%llu after=%llu\n",
+                (unsigned long long)before.FreeFrames,
+                (unsigned long long)after.FreeFrames);
+
+    DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
+}
+
+// Verifies the page-table manager invariants that the higher VMM will depend on.
 static void KiRunMemoryMappingSelfTest(void) {
     PHYSICAL_ADDRESS phys = 0;
     uint64_t flags = 0;
-    KSTATUS s = MmQueryPage(0, &phys, &flags);
+    KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags);
     if (s != STATUS_NOT_FOUND) {
         KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n",
                 (unsigned long long)phys,
@@ -490,7 +600,329 @@ static void KiRunMemoryMappingSelfTest(void) {
                 (unsigned)s);
     }
 
-    DbgPrint("MM selftest: page zero unmapped\n");
+    uintptr_t testVa = 0xFFFFA00000000000ULL;
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
+    if (s != STATUS_NOT_FOUND) {
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n",
+                (unsigned long long)testVa,
+                (unsigned long long)phys,
+                (unsigned long long)flags);
+    }
+
+    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
+    if (!frame)
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n");
+
+    s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent);
+    if (!KSUCCESS(s)) {
+        MmPfnFree(frame);
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s);
+    }
+
+    volatile uint64_t* probe = (volatile uint64_t*)testVa;
+    *probe = 0x1122334455667788ULL;
+    if (*probe != 0x1122334455667788ULL)
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n");
+
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
+    if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE))
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n",
+                s,
+                (unsigned long long)phys,
+                (unsigned long long)frame,
+                (unsigned long long)flags);
+
+    s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s);
+
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
+    if (!KSUCCESS(s) || (flags & PAGE_WRITE))
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n",
+                s,
+                (unsigned long long)flags);
+
+    s = MmPteUnmapPage(testVa);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s);
+
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
+    if (s != STATUS_NOT_FOUND)
+        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s);
+
+    MmPfnFree(frame);
+    DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
+}
+
+
+// Verifies sparse VAD commitment and NOACCESS protection metadata.
+static void KiRunVadSparseSelfTest(void) {
+    PROCESS* process = KeGetCurrentProcess();
+    if (!process)
+        process = KeGetSystemProcess();
+    if (!process)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n");
+
+    size_t beforeVadCount = 0;
+    size_t beforeReserved = 0;
+    size_t beforeCommitted = 0;
+    MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted);
+
+    void* base = (void*)0;
+    KSTATUS s = MmVadReserve(process,
+                             PAGE_SIZE * 4,
+                             PAGE_SIZE,
+                             PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
+                             MM_VAD_ZERO,
+                             MmVadGeneric,
+                             &base);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s);
+
+    uint8_t* p = (uint8_t*)base;
+    s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s);
+
+    s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s);
+
+    void* queryBase = (void*)0;
+    size_t querySize = 0;
+    uint64_t queryFlags = 0;
+    s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
+    if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=%p size=%llu flags=%016llx\n",
+                s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags);
+
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
+    if (s != STATUS_NOT_FOUND)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s);
+
+    s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s);
+    s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s);
+
+    s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
+    if (s != STATUS_NOT_FOUND)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s);
+
+    p[0] = 0x5A;
+    p[PAGE_SIZE * 2] = 0xA5;
+
+    s = MmVadProtect(process, p, PAGE_SIZE, 0);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s);
+
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
+    if (s != STATUS_NOT_FOUND)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s);
+
+    s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
+    if (!KSUCCESS(s) || queryFlags != 0)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n",
+                s, (unsigned long long)queryFlags);
+
+    s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s);
+    if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n");
+
+    size_t afterVadCount = 0;
+    size_t afterReserved = 0;
+    size_t afterCommitted = 0;
+    MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
+    if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n",
+                (unsigned long long)beforeCommitted,
+                (unsigned long long)afterCommitted,
+                (unsigned long long)beforeVadCount,
+                (unsigned long long)afterVadCount);
+
+    s = MmVadDestroy(process, base);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s);
+
+    MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
+    if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n");
+
+    DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
+}
+
+
+// Verifies that writable copy-on-write views split before either alias can modify shared state.
+static void KiRunVadCopyOnWriteSelfTest(void) {
+    PROCESS* process = KeGetCurrentProcess();
+    if (!process)
+        process = KeGetSystemProcess();
+    if (!process)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n");
+
+    void* base = (void*)0;
+    KSTATUS s = MmVadCreateCommitted(process,
+                                     PAGE_SIZE,
+                                     PAGE_SIZE,
+                                     PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
+                                     MM_VAD_ZERO,
+                                     MmVadGeneric,
+                                     &base);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s);
+
+    s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s);
+
+    uint8_t* source = (uint8_t*)base;
+    source[0] = 0x31;
+
+    void* cowBase = (void*)0;
+    s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s);
+
+    PHYSICAL_ADDRESS sourcePhys = 0;
+    PHYSICAL_ADDRESS cowPhys = 0;
+    uint64_t sourceFlags = 0;
+    uint64_t cowFlags = 0;
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags);
+    if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n",
+                s, (unsigned long long)sourceFlags);
+
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags);
+    if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n",
+                s,
+                (unsigned long long)sourcePhys,
+                (unsigned long long)cowPhys,
+                (unsigned long long)cowFlags);
+
+    s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s);
+
+    uint8_t* copy = (uint8_t*)cowBase;
+    copy[0] = 0x62;
+    if (source[0] != 0x31 || copy[0] != 0x62)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n",
+                source[0], copy[0]);
+
+    PHYSICAL_ADDRESS splitPhys = 0;
+    uint64_t splitFlags = 0;
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags);
+    if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
+                s,
+                (unsigned long long)sourcePhys,
+                (unsigned long long)splitPhys,
+                (unsigned long long)splitFlags);
+
+    (void)MmVadDestroy(process, cowBase);
+    (void)MmVadDestroy(process, base);
+
+    void* zeroBase = (void*)0;
+    s = MmVadCreateCommitted(process,
+                             PAGE_SIZE,
+                             PAGE_SIZE,
+                             PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
+                             MM_VAD_ZERO,
+                             MmVadGeneric,
+                             &zeroBase);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s);
+
+    s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s);
+
+    PHYSICAL_ADDRESS zeroPhys = 0;
+    uint64_t zeroFlags = 0;
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags);
+    if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n",
+                s, (unsigned long long)zeroFlags);
+
+    s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s);
+
+    PHYSICAL_ADDRESS privatePhys = 0;
+    uint64_t privateFlags = 0;
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags);
+    if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
+                s,
+                (unsigned long long)zeroPhys,
+                (unsigned long long)privatePhys,
+                (unsigned long long)privateFlags);
+
+    ((uint8_t*)zeroBase)[0] = 0x9C;
+    if (((uint8_t*)zeroBase)[0] != 0x9C)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n");
+
+    (void)MmVadDestroy(process, zeroBase);
+    DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
+}
+
+
+// Verifies that a private resident VAD page can move to swap and fault back in.
+static void KiRunVadSwapSelfTest(void) {
+    if (!MmSwapIsEnabled()) {
+        DbgPrint("VAD swap selftest: SKIP, swap is disabled\n");
+        return;
+    }
+
+    PROCESS* process = KeGetCurrentProcess();
+    if (!process)
+        process = KeGetSystemProcess();
+    if (!process)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n");
+
+    void* base = (void*)0;
+    KSTATUS s = MmVadCreateCommitted(process,
+                                     PAGE_SIZE,
+                                     PAGE_SIZE,
+                                     PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
+                                     MM_VAD_ZERO,
+                                     MmVadGeneric,
+                                     &base);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s);
+
+    s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s);
+
+    uint8_t* p = (uint8_t*)base;
+    p[0] = 0xC1;
+    p[PAGE_SIZE - 1] = 0x5E;
+
+    void* trimmed = (void*)0;
+    s = MmVadTrimOnePageForTest(process, &trimmed);
+    if (!KSUCCESS(s) || trimmed != base)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=%p base=%p\n", s, trimmed, base);
+
+    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
+    if (s != STATUS_NOT_FOUND)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s);
+
+    s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s);
+
+    if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E)
+        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]);
+
+    (void)MmVadDestroy(process, base);
+    DbgPrint("VAD swap selftest: private pageout/pagein PASS\n");
 }
 
 // String-view ABI self-test.
@@ -614,6 +1046,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
                               &st.Iosb,
                               st.CompletionEvent,
                               &st.Apc,
+                              (MM_LOCKED_USER_BUFFER*)0,
+                              (MM_LOCKED_USER_BUFFER*)0,
                               &readReq);
     if (s != STATUS_PENDING)
         KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s);
@@ -683,6 +1117,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
                                    &wiosb,
                                    wev,
                                    (KAPC*)0,
+                                   (MM_LOCKED_USER_BUFFER*)0,
+                                   (MM_LOCKED_USER_BUFFER*)0,
                                    &writeReq);
         if (s != STATUS_PENDING)
             KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s);
@@ -832,7 +1268,11 @@ void KiRunEarlyKernelSelfTests(void) {
 void KiRunKernelSelfTests(const HAL_API* hal) {
     DbgPrint("Kernel selftests: begin\n");
     KiRunKernelApcSelfTest();
+    KiRunPfnDatabaseSelfTest();
     KiRunMemoryMappingSelfTest();
+    KiRunVadSparseSelfTest();
+    KiRunVadCopyOnWriteSelfTest();
+    KiRunVadSwapSelfTest();
     KiRunStringViewSelfTest();
     KiRunIrpKernelApcSelfTest(hal);
     KiRunAsyncFileIoSelfTest();

code/kernel/ke_semaphore.c

@@ -68,6 +68,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
         // consuming a count later.
         LIST_ENTRY* node = ListPopHead(&sem->WaitList);
         THREAD* waiter   = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+        KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
         KeReleaseSpinlock(&sem->Lock, flags);
         KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS);
         return;

code/kernel/ke_smp.c

@@ -20,6 +20,7 @@
 #include <kernel/ke_event.h>
 #include <kernel/ob_object.h>
 #include <kernel/mm_phys.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/mm_virt.h>
 #include <kernel/hal/hal_apic.h>
 
@@ -141,15 +142,17 @@ static KSTATUS KiBuildApTrampoline(void) {
         DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
                  (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
                  (uint64_t)AP_TRAMPOLINE_MAX_PHYS);
-        frame = MmAllocateFrameInRange(AP_TRAMPOLINE_PREFERRED_MIN,
+        frame = MmPfnAllocateInRange(AP_TRAMPOLINE_PREFERRED_MIN,
-                                       AP_TRAMPOLINE_MAX_PHYS);
+                                       AP_TRAMPOLINE_MAX_PHYS,
+                                       MmPfnUseKernel);
     }
     if (!frame) {
         DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
                  (uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
                  (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
-        frame = MmAllocateFrameInRange(AP_TRAMPOLINE_FALLBACK_MIN,
+        frame = MmPfnAllocateInRange(AP_TRAMPOLINE_FALLBACK_MIN,
-                                       AP_TRAMPOLINE_PREFERRED_MIN);
+                                       AP_TRAMPOLINE_PREFERRED_MIN,
+                                       MmPfnUseKernel);
     }
     if (!frame) {
         DbgPrint("SMP: no free PMM frame available below 0x%016llX for AP trampoline\n",
@@ -589,7 +592,7 @@ void KeBuildProcessorTopology(void) {
 // Creates ap Stack.
 static KSTATUS KiAllocateApStack(KCPU* cpu) {
     // AP kernel stacks do not need physically contiguous memory.  Requiring
-    // MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
+    // MmPfnAllocateContiguous(8) made SMP startup depend on a DMA-style run being
     // available, which is fragile on real machines and unrelated to AP startup.
     size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE;
     void* stack = MmAllocatePool(stackBytes);
@@ -967,12 +970,11 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
                     cpuIndex, st);
         }
 
-        THREAD* thread = (THREAD*)0;
         st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker,
                                               (uintptr_t)arg,
                                               PriorityNormal,
                                               (1ULL << cpuIndex),
-                                              &thread);
+                                              (THREAD**)0);
         if (!KSUCCESS(st)) {
             KePanic(KE_PANIC_CODE_SELFTEST,
                     "SMP scheduler selftest: AP%u worker create failed status=0x%x\n",

code/kernel/ke_syscall.asm

@@ -152,7 +152,7 @@ KeSyscallEntry ENDP
 ; also shows which register held the null target.
 ;
 ; KeFaultDispatchFrame signature (MS x64):
-;   void KeFaultDispatchFrame(const KFAULT_FRAME* frame)  ; rcx
+;   KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame)  ; rcx
 ; ---------------------------------------------------------------------------
 
 KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
@@ -379,8 +379,45 @@ pf_kernel_frame:
 pf_call:
     and  rsp, -16
     sub  rsp, FAULT_CALL_SHADOW
+    mov  rbx, rax               ; Keep the KFAULT_FRAME pointer across C.
     mov  rcx, rax
     call KeFaultDispatchFrame
+    test eax, eax
+    jnz  pf_unhandled_return
+
+    mov  rdx, rbx               ; rdx = KFAULT_FRAME*.
+    mov  rcx, [rdx + KF_Rcx]
+    mov  r8,  [rdx + KF_R8]
+    mov  r9,  [rdx + KF_R9]
+    mov  rsi, [rdx + KF_Rsi]
+    mov  rdi, [rdx + KF_Rdi]
+    mov  rbp, [rdx + KF_Rbp]
+    mov  r12, [rdx + KF_R12]
+    mov  r13, [rdx + KF_R13]
+    mov  r14, [rdx + KF_R14]
+    mov  r15, [rdx + KF_R15]
+    mov  rbx, [rdx + KF_Rbx]
+
+    lea  rsp, [rdx + KFAULT_SIZE + 24] ; Skip saved RAX/R11 and the CPU error code.
+    mov  rax, [rdx + KF_Rax]
+    mov  r11, [rdx + KF_R11]
+    mov  r10, [rdx + KF_Cs]
+    test r10, 3
+    jz   pf_resume_no_swapgs
+    mov  r10, [rdx + KF_R10]
+    mov  rdx, [rdx + KF_Rdx]
+    swapgs
+    iretq
+pf_resume_no_swapgs:
+    mov  r10, [rdx + KF_R10]
+    mov  rdx, [rdx + KF_Rdx]
+    iretq
+
+pf_unhandled_return:
+    cli
+pf_unhandled_halt:
+    hlt
+    jmp pf_unhandled_halt
 KeFaultPF ENDP
 
 

code/kernel/ke_system_root.c

@@ -0,0 +1,97 @@
+// System root volume selection. This file keeps boot-volume identity in one place so loaders do not guess.
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <kernel/ke_system_root.h>
+#include <kernel/ke_debug.h>
+#include <kernel/io_file.h>
+
+static uint8_t gSystemRootDrive;
+static bool gSystemRootReady;
+
+// Builds a drive-qualified path without depending on any process CWD.
+static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) {
+    if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES)
+        return false;
+
+    size_t relLen = strlen(relative);
+    if (relLen + 4 > outSize)
+        return false;
+
+    out[0] = (char)('0' + drive);
+    out[1] = ':';
+    out[2] = '/';
+    memcpy(out + 3, relative, relLen + 1);
+    return true;
+}
+
+// Returns true when the candidate volume looks like the volume that booted the kernel.
+static bool KiDriveLooksLikeSystemRoot(uint8_t drive) {
+    static const char* required[] = {
+        "system/colonel.exe",
+        "system/hal.dll",
+        "drivers/drivers.ini",
+    };
+
+    char path[1024];
+    for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) {
+        if (!KiBuildDrivePath(drive, required[i], path, sizeof(path)))
+            return false;
+
+        FILE_OBJECT* file = NULL;
+        KSTATUS s = IoOpenFile(SysSvFromCstr(path), &file);
+        if (!KSUCCESS(s))
+            return false;
+        IoCloseFile(file);
+    }
+
+    return true;
+}
+
+// Selects the mounted volume that contains the kernel/system tree.
+KSTATUS KeInitializeSystemRootVolume(void) {
+    for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) {
+        if (!IoFilesystemPresent(drive))
+            continue;
+        if (!KiDriveLooksLikeSystemRoot(drive))
+            continue;
+
+        gSystemRootDrive = drive;
+        gSystemRootReady = true;
+        DbgPrint("System root: drive=%u\n", drive);
+        return STATUS_SUCCESS;
+    }
+
+    return STATUS_NOT_FOUND;
+}
+
+// Returns whether the system root volume has been selected.
+bool KeSystemRootReady(void) {
+    return gSystemRootReady;
+}
+
+// Returns the selected system root drive, or zero before discovery.
+uint8_t KeGetSystemRootDrive(void) {
+    return gSystemRootReady ? gSystemRootDrive : 0;
+}
+
+// Builds an absolute path rooted at the selected system volume.
+bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize) {
+    if (!gSystemRootReady)
+        return false;
+    return KiBuildDrivePath(gSystemRootDrive, relative, out, outSize);
+}
+
+// Opens a file rooted at the selected system volume.
+KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out) {
+    if (!out)
+        return STATUS_INVALID_PARAMETER;
+    *out = NULL;
+
+    char path[1024];
+    if (!KeBuildSystemRootPath(relative, path, sizeof(path)))
+        return STATUS_NOT_FOUND;
+    return IoOpenFile(SysSvFromCstr(path), out);
+}

code/kernel/ke_thread.c

@@ -8,6 +8,8 @@
 #include <kernel/ke_cpu.h>
 #include <kernel/ke_irql.h>
 #include <kernel/ke_event.h>
+#include <kernel/ke_mutex.h>
+#include <kernel/ke_semaphore.h>
 #include <kernel/ke_apc.h>
 #include <kernel/ke_debug.h>
 #include <kernel/ke_userenv.h>
@@ -36,6 +38,11 @@ extern void KeThreadEntry(void);
 static void KiThreadDestroy(OBJECT* obj) {
     THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
 
+    // A wait that uses auxiliary stack wait blocks must be removed before the
+    // kernel stack is released. A terminated thread may never return to the
+    // wait syscall to run normal cleanup.
+    KeRundownThreadWait(thread, STATUS_ALERTED);
+
     KiRundownApcState(thread);
 
     if (thread->ExitEvent) {
@@ -43,16 +50,16 @@ static void KiThreadDestroy(OBJECT* obj) {
         thread->ExitEvent = (EVENT*)0;
     }
 
-    PROCESS* ownerForUserRegions = thread->Owner;
+    PROCESS* ownerForVadList = thread->Owner;
-    if (ownerForUserRegions) {
+    if (ownerForVadList) {
         if (thread->Teb && thread->TebSize) {
-            MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb);
+            MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb);
             thread->Teb = (void*)0;
             thread->TebSize = 0;
         }
 
         if (thread->UserStackBase) {
-            MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase);
+            MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase);
             thread->UserStackBase = (void *)0;
         }
     }
@@ -66,10 +73,13 @@ static void KiThreadDestroy(OBJECT* obj) {
         remaining = owner->ThreadCount;
         KeReleaseSpinlock(&owner->HandleLock, flags);
 
-        if (remaining == 0 && owner->Terminating &&
+        if (remaining == 0) {
-            // Implements the interlocked Exchange operation.
+            // The last thread leaving a process is process exit even if user
-            _InterlockedExchange(&owner->Exited, 1) == 0) {
+            // code called ExitThread instead of ExitProcess.  KeExitProcess /
-            if (owner->ExitEvent) KeSignalEvent(owner->ExitEvent);
+            // KeTerminateProcess set Terminating earlier; this covers the last
+            // thread voluntarily exiting without an explicit process-exit call.
+            _InterlockedExchange(&owner->Terminating, 1);
+            KeCompleteProcessExitIfLastThread(owner);
         }
 
         ObDereferenceObject(&owner->Header);
@@ -111,11 +121,13 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     thread->WaitStatus       = STATUS_SUCCESS;
     thread->WaitListHead     = (LIST_ENTRY*)0;
     thread->AlertableWait    = 0;
+    thread->SleepUntilTick   = 0;
+    thread->TimeoutStatus    = STATUS_SUCCESS;
     thread->SwitchLockCookie = 0x202ULL;
     KiInitializeApcState(thread);
     thread->TrapFrame = (KTRAP_FRAME*)0;
 
-    void* kstack = MmAllocatePool(KSTACK_SIZE);
+    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
     if (!kstack) {
         ObDereferenceObject(&thread->Header);
         return STATUS_NO_MEMORY;
@@ -127,10 +139,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
 
     KSTATUS s;
     void* ustack = (void*)0;
-    s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE,
+    s = MmVadCreateCommitted(owner, USTACK_SIZE, PAGE_SIZE,
                            PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
-                           USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
+                           MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
-                           UserRegionStack,
+                           MmVadStack,
                            &ustack);
     if (!KSUCCESS(s) || !ustack) {
         MmFreePool(kstack);
@@ -139,11 +151,19 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
         return STATUS_NO_MEMORY;
     }
     thread->UserStackBase = ustack;
-    // Ring-3 starts at the image entry via IRET, not CALL, so there is no
+    // Ring-3 starts at the image entry via IRET, not CALL.  The MS x64 ABI
-    // pushed return address. The x64 ABI expects RSP % 16 == 8 at function
+    // nevertheless lets callees immediately use the caller-provided 32-byte
-    // entry; bias the initial stack as if a call had happened so compiler
+    // home/shadow area above RSP, and MSVC commonly emits a first instruction
-    // generated aligned SSE spills (movaps) are safe.
+    // like "mov [rsp+8], rcx" in unoptimized test code.  Bias the synthetic
-    uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8;
+    // entry stack as if a CALL with a normal home area had happened:
+    //
+    //   RSP % 16 == 8 at function entry
+    //   [RSP + 0]     fake return-address slot
+    //   [RSP + 8..39] home/shadow space
+    //
+    // Without the 32-byte home area, the first spill lands in the high stack
+    // guard page and every freshly-created MSVC thread faults immediately.
+    uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8 - 32;
 
     SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
     if (!teb) {
@@ -211,8 +231,16 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     KeReleaseSpinlock(&owner->HandleLock, flags);
 
     KeInitThreadFpuContext(thread);
-    KeReadyThread(thread);
+
+    // Hand the creator a short-lived reference before making the thread
+    // runnable.  On SMP a new user thread can run and exit before
+    // KeCreateThread() returns; without this reference, callers that merely
+    // read ThreadId for logging or syscall output can race the scheduler's
+    // dead-thread dereference and touch freed pool.
+    ObReferenceObject(&thread->Header);
     *out = thread;
+
+    KeReadyThread(thread);
     return STATUS_SUCCESS;
 }
 
@@ -283,6 +311,8 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     thread->WaitStatus      = STATUS_SUCCESS;
     thread->WaitListHead    = (LIST_ENTRY*)0;
     thread->AlertableWait   = 0;
+    thread->SleepUntilTick  = 0;
+    thread->TimeoutStatus   = STATUS_SUCCESS;
     thread->SwitchLockCookie = 0x202ULL;
     KiInitializeApcState(thread);
     thread->TrapFrame       = (KTRAP_FRAME*)0;
@@ -290,7 +320,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     thread->Teb             = (void*)0;   // no TEB; GS_BASE will be 0 in kernel ctx
     thread->TebSize         = 0;
 
-    void* kstack = MmAllocatePool(KSTACK_SIZE);
+    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
     if (!kstack) {
         ObDereferenceObject(&thread->Header);
         return STATUS_NO_MEMORY;
@@ -320,7 +350,16 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     owner->ThreadCount++;
     KeReleaseSpinlock(&owner->HandleLock, flags);
 
-    if (out) *out = thread;
+    // Hand the creator a short-lived reference before making the kernel
+    // thread runnable.  Kernel workers can run, signal their completion event,
+    // exit, and be reaped on another CPU before KeCreateKernelThread()'s
+    // caller drops its returned pointer.  The live scheduler reference remains
+    // separate and is released by KeSchedulerPostSwitch() after the dead
+    // thread has switched away.
+    if (out) {
+        ObReferenceObject(&thread->Header);
+        *out = thread;
+    }
 
     KeInitThreadFpuContext(thread);
     KeReadyThread(thread);
@@ -352,6 +391,8 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable)
     thread->WaitStatus    = STATUS_PENDING;
     thread->WaitListHead  = (LIST_ENTRY*)0;
     thread->AlertableWait = alertable ? 1 : 0;
+    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
+    thread->WaitRundownContext = (void*)0;
 }
 
 
@@ -362,12 +403,41 @@ void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
     thread->WaitListHead = waitListHead;
 }
 
+// Registers cleanup for waits with auxiliary wait blocks.
+void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context) {
+    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
+    if (!thread) return;
+    thread->WaitRundownRoutine = routine;
+    thread->WaitRundownContext = context;
+}
+
+// Clears auxiliary wait cleanup after the owning wait has removed its blocks.
+void KeClearThreadWaitRundown(THREAD* thread) {
+    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
+    if (!thread) return;
+    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
+    thread->WaitRundownContext = (void*)0;
+}
+
+// Runs auxiliary wait cleanup for abort and thread teardown paths.
+void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
+    if (!thread || !thread->WaitRundownRoutine) return;
+
+    KE_WAIT_RUNDOWN_ROUTINE routine = thread->WaitRundownRoutine;
+    void* context = thread->WaitRundownContext;
+    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
+    thread->WaitRundownContext = (void*)0;
+    routine(thread, context, status);
+}
+
 // Implements the finish Wait operation.
 static void KiFinishWait(THREAD* thread, KSTATUS status) {
     thread->WaitObject    = (OBJECT*)0;
     thread->WaitStatus    = status;
     thread->WaitListHead  = (LIST_ENTRY*)0;
     thread->AlertableWait = 0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
 }
 
 // Implements the commit Block Thread operation.
@@ -385,6 +455,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     if (thread->State != ThreadWaiting) {
         KSTATUS status = thread->WaitStatus;
         thread->AlertableWait = 0;
+        thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
         return status;
     }
 
@@ -406,6 +478,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     KiDeliverKernelApcs();
 
     thread->AlertableWait = 0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
     return thread->WaitStatus;
 }
 
@@ -423,6 +497,92 @@ void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
     KiSchedulerWakeThread(thread, status);
 }
 
+// Returns whether the scheduler link still belongs to an object wait list.
+static int KiThreadWaitLinkIsLinked(const THREAD* thread) {
+    if (!thread) return 0;
+    return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
+           thread->SchedulerLink.Bwd != &thread->SchedulerLink;
+}
+
+// Removes a thread from one wait list while that wait object's lock is held.
+static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
+    if (!thread || !waitListHead) return 0;
+    if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead)
+        return 0;
+
+    // The signaling side pops the wait-list entry before it wakes the thread.
+    // In that handoff window the scheduler link is already self-linked; abort
+    // must not remove it again or turn a successful mutex/semaphore handoff
+    // into an aborted wait for a thread that may already own the object.
+    if (!KiThreadWaitLinkIsLinked(thread))
+        return 0;
+
+    ListRemove(&thread->SchedulerLink);
+    thread->WaitListHead = (LIST_ENTRY*)0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
+    return 1;
+}
+
+// Removes a thread from an object-owned wait queue using that object's lock.
+static int KiAbortObjectWait(THREAD* thread) {
+    if (!thread || !thread->WaitObject || !thread->WaitListHead)
+        return 0;
+
+    OBJECT* object = thread->WaitObject;
+    switch (object->Type) {
+        case ObjectTypeEvent: {
+            EVENT* event = CONTAINING_RECORD(object, EVENT, Header);
+            uintptr_t flags = KeAcquireSpinlock(&event->Lock);
+            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &event->WaitList);
+            KeReleaseSpinlock(&event->Lock, flags);
+            return removed;
+        }
+        case ObjectTypeMutex: {
+            MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
+            uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
+            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList);
+            KeReleaseSpinlock(&mutex->Lock, flags);
+            return removed;
+        }
+        case ObjectTypeSemaphore: {
+            SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
+            uintptr_t flags = KeAcquireSpinlock(&semaphore->Lock);
+            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &semaphore->WaitList);
+            KeReleaseSpinlock(&semaphore->Lock, flags);
+            return removed;
+        }
+        default:
+            return 0;
+    }
+}
+
+// Aborts a thread's current wait regardless of whether the wait was marked
+// alertable. Process teardown uses this for crash/kill paths: a background
+// worker blocked forever on a semaphore must not keep the process exit event
+// unsignaled after the foreground thread has faulted. Object wait-list removal
+// must use the object's own lock; otherwise signal/abort races can leave stale
+// scheduler links in an event, mutex, or semaphore wait queue.
+void KeAbortThreadWait(THREAD* thread, KSTATUS status) {
+    if (!thread) return;
+    if (thread->State != ThreadWaiting) return;
+
+    if (thread->WaitRundownRoutine) {
+        KeRundownThreadWait(thread, status);
+        return;
+    }
+
+    if (KiAbortObjectWait(thread)) {
+        KeUnblockThreadWithStatus(thread, status);
+        return;
+    }
+
+    // Sleep/timeouts and already-dequeued handoffs are owned by the scheduler.
+    // This path removes from the scheduler sleep list when present and otherwise
+    // completes the wait in place.
+    KeUnblockThreadFromWaitListWithStatus(thread, status);
+}
+
 // Implements the unblock Thread operation.
 void KeUnblockThread(THREAD* thread) {
     KeUnblockThreadWithStatus(thread, STATUS_SUCCESS);

code/kernel/ke_work.c

@@ -0,0 +1,117 @@
+// Kernel passive-level work queue. This file owns the generic mechanism used
+// to move ISR/DPC-originated work to a sleepable system thread context.
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <intrin.h>
+
+#include <kernel/ke_work.h>
+#include <kernel/ke_event.h>
+#include <kernel/ke_spinlock.h>
+#include <kernel/ke_thread.h>
+#include <kernel/ke_sched.h>
+#include <kernel/ke_irql.h>
+#include <kernel/ke_debug.h>
+
+static SPINLOCK gWorkQueueLock = SPINLOCK_INIT;
+static LIST_ENTRY gWorkQueueList;
+static EVENT* gWorkQueueEvent;
+static volatile long gWorkQueueListReady;
+static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started.
+
+static void KiWorkQueueWorker(void* arg) {
+    (void)arg;
+    KeAssertIrql(PASSIVE_LEVEL);
+
+    for (;;) {
+        (void)KeWaitForEvent(gWorkQueueEvent, 0);
+
+        for (;;) {
+            KWORK_ITEM* item = (KWORK_ITEM*)0;
+            uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
+            if (!LIST_IS_EMPTY(&gWorkQueueList)) {
+                LIST_ENTRY* node = ListPopHead(&gWorkQueueList);
+                item = CONTAINING_RECORD(node, KWORK_ITEM, Link);
+            }
+            KeReleaseSpinlock(&gWorkQueueLock, flags);
+
+            if (!item)
+                break;
+
+            // Allow the item to be queued again while it is running. This keeps
+            // level-triggered users such as cursor updates naturally coalesced
+            // but does not lose an update that arrives during the callback.
+            _InterlockedExchange(&item->Queued, 0);
+            if (item->Routine)
+                item->Routine(item->Context);
+        }
+    }
+}
+
+void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context) {
+    if (!item)
+        return;
+    LIST_INIT(&item->Link);
+    item->Routine = routine;
+    item->Context = context;
+    item->Queued = 0;
+}
+
+bool KeIsWorkQueueStarted(void) {
+    return _InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 2;
+}
+
+bool KeQueueWorkItem(KWORK_ITEM* item) {
+    if (!item || !item->Routine)
+        return false;
+
+    if (_InterlockedCompareExchange(&item->Queued, 1, 0) != 0)
+        return false;
+
+    if (!_InterlockedCompareExchange(&gWorkQueueListReady, 1, 1)) {
+        _InterlockedExchange(&item->Queued, 0);
+        return false;
+    }
+
+    uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
+    ListInsertTail(&gWorkQueueList, &item->Link);
+    EVENT* event = gWorkQueueEvent;
+    KeReleaseSpinlock(&gWorkQueueLock, flags);
+
+    if (event)
+        KeSignalEvent(event);
+    return true;
+}
+
+KSTATUS KeStartWorkQueue(void) {
+    long state = _InterlockedCompareExchange(&gWorkQueueStartState, 1, 0);
+    if (state == 2)
+        return STATUS_SUCCESS;
+    if (state == 1) {
+        while (_InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 1)
+            KeYield();
+        return KeIsWorkQueueStarted() ? STATUS_SUCCESS : STATUS_FAILURE;
+    }
+
+    LIST_INIT(&gWorkQueueList);
+    _InterlockedExchange(&gWorkQueueListReady, 1);
+
+    KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
+    if (KSUCCESS(s)) {
+        THREAD* worker = (THREAD*)0;
+        s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker,
+                                 0,
+                                 PriorityNormal,
+                                 &worker);
+        // Persistent system worker; keep the creator reference for kernel life.
+        (void)worker;
+    }
+
+    if (KSUCCESS(s)) {
+        _InterlockedExchange(&gWorkQueueStartState, 2);
+        KeSignalEvent(gWorkQueueEvent);
+    } else {
+        _InterlockedExchange(&gWorkQueueStartState, 0);
+    }
+    return s;
+}

code/kernel/kmain.c

@@ -7,7 +7,9 @@
 #pragma intrinsic(_enable)
 #include <ntlite/boot_info.h>
 #include <kernel/mm_phys.h>
+#include <kernel/mm_pfn.h>
 #include <kernel/mm_virt.h>
+#include <kernel/mm_swap.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_process.h>
 #include <kernel/ke_thread.h>
@@ -28,6 +30,9 @@
 #include <kernel/ke_input.h>
 #include <kernel/ke_time.h>
 #include <kernel/ke_display.h>
+#include <kernel/ke_settings.h>
+#include <kernel/ke_work.h>
+#include <kernel/ke_system_root.h>
 #include <common/ini_parser.h>
 
 typedef struct {
@@ -35,6 +40,90 @@ typedef struct {
     const HAL_API* hal;
 } SYSTEM_INIT_ARGS;
 
+#define KI_SWAP_MIN_MB 4ull
+#define KI_SWAP_MAX_MB 8192ull
+
+// Parses a required unsigned decimal integer from settings.ini.
+// Configuration errors are treated as boot-time corruption instead of guessed defaults.
+static uint64_t KiParseRequiredUInt64Setting(const char* section, const char* key) {
+    char value[64];
+    KSTATUS s = KeGetSettingString(section, key, value, sizeof(value));
+    if (!KSUCCESS(s)) {
+        KePanic(KE_PANIC_CODE_INIT,
+                "settings: missing required [%s] %s entry, status=0x%x\n",
+                section,
+                key,
+                s);
+    }
+
+    uint64_t result = 0;
+    const char* p = value;
+    if (!*p) {
+        KePanic(KE_PANIC_CODE_INIT,
+                "settings: empty numeric value for [%s] %s\n",
+                section,
+                key);
+    }
+
+    while (*p) {
+        if (*p < '0' || *p > '9') {
+            KePanic(KE_PANIC_CODE_INIT,
+                    "settings: invalid numeric value for [%s] %s: %s\n",
+                    section,
+                    key,
+                    value);
+        }
+
+        uint64_t digit = (uint64_t)(*p - '0');
+        if (result > (UINT64_MAX - digit) / 10ull) {
+            KePanic(KE_PANIC_CODE_INIT,
+                    "settings: numeric value for [%s] %s is too large: %s\n",
+                    section,
+                    key,
+                    value);
+        }
+
+        result = result * 10ull + digit;
+        ++p;
+    }
+
+    return result;
+}
+
+// Initializes swap according to settings.ini.
+// A size of zero is the explicit opt-out. Any missing or invalid size is a broken configuration.
+static void KiInitializeSwapFromSettings(void) {
+    uint64_t swapMb = KiParseRequiredUInt64Setting("vm", "swap_size_mb");
+    if (swapMb == 0) {
+        DbgPrint("swap: disabled by settings\n");
+        return;
+    }
+
+    if (swapMb < KI_SWAP_MIN_MB || swapMb > KI_SWAP_MAX_MB) {
+        KePanic(KE_PANIC_CODE_INIT,
+                "settings: [vm] swap_size_mb=%llu is outside the allowed range %llu..%llu MB\n",
+                (unsigned long long)swapMb,
+                (unsigned long long)KI_SWAP_MIN_MB,
+                (unsigned long long)KI_SWAP_MAX_MB);
+    }
+
+    char pagefilePath[1024];
+    if (!KeBuildSystemRootPath("pagefile.sys", pagefilePath, sizeof(pagefilePath))) {
+        KePanic(KE_PANIC_CODE_INIT,
+                "swap: failed to build system-root pagefile path\n");
+    }
+
+    uint64_t swapBytes = swapMb * 1024ull * 1024ull;
+    KSTATUS s = MmSwapInitialize(SysSvFromCstr(pagefilePath), swapBytes);
+    if (!KSUCCESS(s)) {
+        KePanic(KE_PANIC_CODE_INIT,
+                "swap: failed to initialize %llu MB pagefile at %s, status=0x%x\n",
+                (unsigned long long)swapMb,
+                pagefilePath,
+                s);
+    }
+}
+
 typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *);
 
 // HAL pointer cached at boot for subsystems that still need platform services.
@@ -68,7 +157,7 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
 
         for (uintptr_t page = base; page < end; page += PAGE_SIZE) {
             if (!KiPageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue;
-            MmFreeFrame((PHYSICAL_ADDRESS)page);
+            MmPfnFree((PHYSICAL_ADDRESS)page);
         }
     }
 }
@@ -86,6 +175,7 @@ static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
     }
 }
 
+
 // Phase Helper: Initialize dynamically loaded .sys drivers.
 static void KiDriversIniCb(const char* section, const char* key,
                           const char* val, void* ud) {
@@ -120,7 +210,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
 
         // Map the PE image into kernel space.
         uintptr_t mappedBase = 0;
-        KSTATUS st = LdrLoadImage(rawPeBuffer, mod->ImageSize, &mappedBase, false);
+        KSTATUS st = LdrLoadKernelImage(rawPeBuffer, mod->ImageSize, &mappedBase);
         if (!KSUCCESS(st)) {
             DbgPrint("ERROR: Failed to map driver %s (0x%X)\n", mod->Name, st);
             continue;
@@ -129,7 +219,9 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
         // Update the struct so other modules can find it.
         mod->VirtBase = mappedBase;
 
-        // Resolve imports (can now see Kernel, HAL, and previous drivers).
+        // Resolve imports while the image is still writable. Kernel driver images
+        // stay writable until the loader grows real section-backed mappings; the
+        // loader still clears NX for the whole image before returning it.
         if (!KSUCCESS(LdrResolveImports(mappedBase))) {
             DbgPrint("ERROR: Failed to resolve imports for %s\n", mod->Name);
             continue;
@@ -152,9 +244,24 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
     KSTATUS s;
     FILE_OBJECT* f;
 
-    s = IoOpenFile(SYS_SV_LITERAL("drivers\\drivers.ini"), &f);
+    s = KeInitializeSystemRootVolume();
     if (s != STATUS_SUCCESS) {
-        KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini (status %d)\n", s);
+        KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to find system root volume (status %d)\n", s);
+    }
+
+    char root[4];
+    root[0] = (char)('0' + KeGetSystemRootDrive());
+    root[1] = ':';
+    root[2] = '/';
+    root[3] = '\0';
+    s = IoSetCwd(SysSvFromCstr(root));
+    if (s != STATUS_SUCCESS) {
+        KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to set system CWD to %s (status %d)\n", root, s);
+    }
+
+    s = KeOpenSystemRootFile("drivers\\drivers.ini", &f);
+    if (s != STATUS_SUCCESS) {
+        KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini on system root (status %d)\n", s);
     }
 
     uint64_t size = IoGetFileSize(f) + 1;
@@ -174,26 +281,27 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
     for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
         DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]);
 
-        // Load the driver.
+        char path[1024];
+        char relative[512];
         size_t len = strlen(runtimeKeys[i]);
-        char* path = MmAllocatePool(len + 10);
+        if (len + 9 > sizeof(relative)) {
-        if (!path) {
+            DbgPrint("KiInitializeDrivers: runtime driver path too long: %s\n", runtimeKeys[i]);
-            DbgPrint("KernelMain: Failed to allocate memory for driver path. How?\n");
             continue;
         }
+        memcpy(relative, "drivers\\", 8);
+        memcpy(relative + 8, runtimeKeys[i], len + 1);
 
-        // memcpy maneuvers.
+        if (!KeBuildSystemRootPath(relative, path, sizeof(path))) {
-        memcpy(path, "drivers\\", 8);
+            DbgPrint("KiInitializeDrivers: failed to build runtime driver path: %s\n", runtimeKeys[i]);
-        memcpy(path + 8, runtimeKeys[i], len + 1);
+            continue;
+        }
 
         s = IoOpenFile(SysSvFromCstr(path), &f);
         if (s != STATUS_SUCCESS) {
             DbgPrint("KiInitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path);
-            continue; // Skip this driver but keep going with others
+            continue;
         }
 
-        MmFreePool(path);
-
         uint64_t drvSize = IoGetFileSize(f);
         void* drvMem = MmAllocatePool(drvSize);
         if (!KSUCCESS(IoReadFile(f, drvMem, drvSize, &got))) {
@@ -204,7 +312,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
         IoCloseFile(f);
 
         uintptr_t mappedBase = 0;
-        if (!KSUCCESS(LdrLoadImage(drvMem, drvSize, &mappedBase, false))) { // Load into kernel space
+        if (!KSUCCESS(LdrLoadKernelImage(drvMem, drvSize, &mappedBase))) { // Load into kernel space
             DbgPrint("KiInitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]);
             MmFreePool(drvMem);
             continue;
@@ -215,7 +323,6 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
             DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
             continue;
         }
-
         // Runtime modules may also export DDIs to drivers loaded later.  Boot
         // drivers were already registered above; keep the module model uniform.
         LdrRegisterModule(runtimeKeys[i], mappedBase);
@@ -250,28 +357,15 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
     KSTATUS s;
     FILE_OBJECT* f;
 
-    // Read the executable file into a raw buffer.
+    // Kernel-launched user images are rooted at the selected system volume.
-    s = IoOpenFile(SysSvFromCstr(filePath), &f);
+    s = KeOpenSystemRootFile(filePath, &f);
     if (s != STATUS_SUCCESS) {
-        DbgPrint("Failed to load file: \"%s\"!\n", filePath);
+        DbgPrint("Failed to load system file: \"%s\" (status %d)!\n", filePath, s);
         return 0;
     }
 
     uint64_t size = IoGetFileSize(f);
-
+    (void)size;
-    void* mem = MmAllocatePool(size);
-    if (!mem) {
-        IoCloseFile(f);
-        return 0;
-    }
-
-    size_t got = 0;
-    s = IoReadFile(f, mem, size, &got);
-    IoCloseFile(f);
-    if (!KSUCCESS(s)) {
-        MmFreePool(mem);
-        return 0;
-    }
 
     // Create the process object.
     PROCESS* proc;
@@ -280,7 +374,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
     s = KeCreateProcess(filePath, &proc);
     if (!KSUCCESS(s)) {
         DbgPrint("Failed to create process. System halted.\n");
-        MmFreePool(mem);
+        IoCloseFile(f);
         return 0;
     }
     DbgPrint("Created process (PID %u) with command line: %s\n", proc->ProcessId, (args && args[0]) ? args : filePath);
@@ -291,17 +385,17 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
                                       SysSvFromCstr(proc->CurrentDirectory));
     if (!KSUCCESS(s)) {
         DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s);
-        MmFreePool(mem);
+        IoCloseFile(f);
         ObDereferenceObject(&proc->Header);
         return 0;
     }
 
-    // Parse the raw PE file and map it into executable memory.
+    // Map the executable through an image section backed by the open file.
     uintptr_t imageBase = 0;
-    s = LdrLoadImageForProcess(proc, mem, size, &imageBase);
+    s = LdrLoadImageFileForProcess(proc, f, &imageBase);
-    MmFreePool(mem);
+    IoCloseFile(f);
     if (!KSUCCESS(s)) {
-        DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s);
+        DbgPrint("KiLaunchUserProcess: LdrLoadImageFileForProcess failed with status: %d\n", s);
         ObDereferenceObject(&proc->Header);
         return 0;
     }
@@ -313,6 +407,13 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
         return 0;
     }
 
+    s = LdrProtectImageForProcess(proc, imageBase, true);
+    if (!KSUCCESS(s)) {
+        DbgPrint("KiLaunchUserProcess: Failed to protect EXE image for %s (status %d)\n", filePath, s);
+        ObDereferenceObject(&proc->Header);
+        return 0;
+    }
+
     proc->ImageBase = imageBase;
     proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
 
@@ -327,7 +428,9 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
         ObDereferenceObject(&proc->Header);
         return 0;
     }
-    DbgPrint("KiLaunchUserProcess: Thread created and queued!\n");
+    uint32_t tid = thread->ThreadId;
+    DbgPrint("KiLaunchUserProcess: Thread created and queued! TID=%u\n", tid);
+    ObDereferenceObject(&thread->Header);
 
     // Caller owns the reference returned by KeCreateProcess.
     return proc;
@@ -344,35 +447,39 @@ void SystemInitThread(void* arg) {
     KiInitializeDrivers(args->bootInfo, args->hal);
     KeTimeReloadSettings();
 
+    KiInitializeSwapFromSettings();
+
     KiResolveSystemImports(args->bootInfo);
     KiReclaimBootloaderMemory(args->bootInfo);
 
     if (args->bootInfo->Debug & KE_DEBUG_SELFTEST)
         KiRunKernelSelfTests(args->hal);
 
-    // Launch User-Mode init.exe and restart it until it exits cleanly.
+    // Launch User-Mode init.exe.
-    while (true) {
     PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
     if (!shellProc) {
-            DbgPrint("SystemInitThread: Failed to start system/init.exe; retrying.\n");
+        DbgPrint("SystemInitThread: Failed to start system/init.exe.\n");
-            KeYield();
+        KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n");
-            continue;
     }
 
     KeWaitForEvent(shellProc->ExitEvent, 0);
     int exitCode = shellProc->ExitCode;
+    if (exitCode != 0) {
+        KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init.exe exited with code %d.\n", exitCode);
+    }
     DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
     ObDereferenceObject(&shellProc->Header);
 
-        if (exitCode == 0) break;
-        DbgPrint("SystemInitThread: Restarting init.exe.\n");
-    }
-
-    DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n");
     if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
-        KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log"));
+        char logPath[1024];
+        KSTATUS logStatus = STATUS_NOT_FOUND;
+        if (KeBuildSystemRootPath("kernel.log", logPath, sizeof(logPath))) {
+            logStatus = KeFlushDebugLogToFile(SysSvFromCstr(logPath));
             if (!KSUCCESS(logStatus))
-            DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus);
+                DbgPrint("debuglog: shutdown flush to %s failed st=0x%x\n", logPath, logStatus);
+        } else {
+            DbgPrint("debuglog: shutdown flush skipped; system root path unavailable\n");
+        }
     }
     gKmainHal->Power.Shutdown();
     MmFreePool(args);
@@ -459,6 +566,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
 
     KeInitializeScheduler();
 
+    KSTATUS workStatus = KeStartWorkQueue();
+    if (!KSUCCESS(workStatus)) {
+        KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus);
+    }
+
     KSTATUS smpSchedStatus = KeAdmitApplicationProcessorsToScheduler();
     if (!KSUCCESS(smpSchedStatus)) {
         KePanic(KE_PANIC_CODE_INIT, "SMP: failed to admit APs to scheduler, status=0x%x\n", smpSchedStatus);
@@ -471,11 +583,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
     initArgs->bootInfo = bootInfo;
     initArgs->hal      = hal;
 
-    // Spawn the Initialization Thread.
+    // Spawn the Initialization Thread. No creator reference is needed; the
-    THREAD* initThread = (THREAD*)0;
+    // scheduler owns the live thread lifetime.
     KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
                                       (uintptr_t)initArgs,
-                                      PriorityNormal, &initThread);
+                                      PriorityNormal, (THREAD**)0);
     
     if (!KSUCCESS(st)) {
         KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st);

code/kernel/ldr_image.c

@@ -3,11 +3,13 @@
 #include <stdint.h>
 #include <string.h>
 #include <kernel/ldr_image.h>
+#include <kernel/ldr_section.h>
 #include <ntlite/boot_info.h>
 #include <kernel/ke_process.h>
 #include <kernel/io_file.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ke_debug.h>
+#include <kernel/ke_system_root.h>
 
 #ifndef LDR_DEBUG_VERBOSE
 #define LDR_DEBUG_VERBOSE 0
@@ -155,6 +157,8 @@ typedef struct {
 
 #define IMAGE_REL_BASED_DIR64 10
 #define IMAGE_SCN_CNT_CODE               0x00000020u
+#define IMAGE_SCN_CNT_INITIALIZED_DATA   0x00000040u
+#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u
 #define IMAGE_SCN_MEM_EXECUTE            0x20000000u
 #define IMAGE_SCN_MEM_READ               0x40000000u
 #define IMAGE_SCN_MEM_WRITE              0x80000000u
@@ -171,6 +175,11 @@ typedef struct {
 static MODULE_ENTRY gModules[MAX_MODULES];
 static uint32_t     gModuleCount = 0;
 
+// Ensures a user image page is privately writable before the kernel patches it.
+static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t address, size_t size) {
+    if (!user) return STATUS_SUCCESS;
+    return MmVadMakeImageRangePrivate(process, (void*)address, size);
+}
 
 // Implements the canonical Module Name operation.
 static const char* KiCanonicalModuleName(const char* name) {
@@ -310,10 +319,10 @@ size_t LdrGetImageSize(uintptr_t base) {
 size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
     if (!base) return 0;
     if (process) {
-        size_t regionSize = MmGetUserRegionSize(process, (const void*)base);
+        size_t regionSize = MmVadGetSize(process, (const void*)base);
         if (regionSize) return regionSize;
     }
-    size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base);
+    size_t globalRegionSize = MmVadGetSizeByBase((const void*)base);
     if (globalRegionSize) return globalRegionSize;
     return LdrGetImageSize(base);
 }
@@ -327,13 +336,26 @@ static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, b
     else MmFreePool((void*)imageBase);
 }
 
-// Releases loaded Image.
+// Returns the final page flags for a PE section.
-static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
+static uint64_t KiSectionPageProtection(const SECTION_HEADER* section, bool user) {
-    KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user);
+    uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
+
+    // Some old toolchains mark BSS-like sections as uninitialized data without
+    // setting MEM_WRITE. Treating those pages as read-only breaks programs that
+    // legitimately write their zero-fill storage.
+    if ((section->Characteristics & IMAGE_SCN_MEM_WRITE) ||
+        (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)) {
+        prot |= PAGE_WRITE;
     }
 
+    if (!((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
+          (section->Characteristics & IMAGE_SCN_CNT_CODE))) {
+        prot |= PAGE_NX;
+    }
+    return prot;
+}
 
-// Implements the ldr Protect Image For Process operation.
+// Applies final page protections to a mapped PE image.
 KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
     if (!imageBase) return STATUS_INVALID_PARAMETER;
     if (user && !process) return STATUS_INVALID_PARAMETER;
@@ -353,6 +375,14 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
     if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset)
         return STATUS_FAILURE;
 
+    size_t pageCount = (mappedSize + PAGE_SIZE - 1) / PAGE_SIZE;
+    uint64_t* pageProt = (uint64_t*)MmAllocatePoolTag(pageCount * sizeof(uint64_t), MM_POOL_TAG('i', 'p', 'r', 't'));
+    if (!pageProt) return STATUS_NO_MEMORY;
+
+    uint64_t headerProt = PAGE_PRESENT | (user ? PAGE_USER : 0) | PAGE_NX;
+    for (size_t i = 0; i < pageCount; ++i)
+        pageProt[i] = headerProt;
+
     SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset);
     for (int i = 0; i < pe->NumberOfSections; i++) {
         size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData;
@@ -368,36 +398,57 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
                    (unsigned long long)imageBase, (unsigned long long)mappedSize);
             continue;
         }
-        if (secEnd < secBase) return STATUS_NO_MEMORY;
+        if (secEnd < secBase) {
+            MmFreePool(pageProt);
+            return STATUS_NO_MEMORY;
+        }
         if (secEnd > mapEnd) {
             LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n",
                    i, (unsigned long long)secEnd, (unsigned long long)mapEnd);
-            protectSize = (size_t)(mapEnd - secBase);
+            secEnd = mapEnd;
         }
 
-        uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
+        size_t firstPage = (size_t)((secBase - imageBase) / PAGE_SIZE);
-        if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE)
+        size_t lastPage = (size_t)((secEnd - 1 - imageBase) / PAGE_SIZE);
-            prot |= PAGE_WRITE;
+        uint64_t prot = KiSectionPageProtection(&sec[i], user);
-        if (!((sec[i].Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
+        for (size_t page = firstPage; page <= lastPage && page < pageCount; ++page) {
-              (sec[i].Characteristics & IMAGE_SCN_CNT_CODE)))
+            // Protection is page-granular. If sections ever share a page, the
-            prot |= PAGE_NX;
+            // hardware page must allow the union of their access requirements.
-
+            if (prot & PAGE_WRITE)
-        KSTATUS s;
+                pageProt[page] |= PAGE_WRITE;
-        if (user) {
+            if ((prot & PAGE_NX) == 0)
-            s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot);
+                pageProt[page] &= ~PAGE_NX;
-        } else {
-            s = MmSetPageProtect(secBase, protectSize, prot);
         }
+    }
+
+    size_t runStart = 0;
+    KSTATUS s = STATUS_SUCCESS;
+    while (runStart < pageCount) {
+        uint64_t prot = pageProt[runStart];
+        size_t runEnd = runStart + 1;
+        while (runEnd < pageCount && pageProt[runEnd] == prot)
+            runEnd++;
+
+        uintptr_t va = imageBase + runStart * PAGE_SIZE;
+        size_t bytes = (runEnd - runStart) * PAGE_SIZE;
+        if (user)
+            s = MmVadProtect(process, (void*)va, bytes, prot);
+        else
+            s = MmPteProtectRange(va, bytes, prot);
+
         if (!KSUCCESS(s)) {
-            LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
+            LdrDbg("LDR: protect failed va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
-                   i, (unsigned long long)secBase, (unsigned long long)protectSize,
+                   (unsigned long long)va, (unsigned long long)bytes,
                    (unsigned long long)prot, s,
                    (unsigned long long)imageBase, (unsigned long long)mappedSize);
+            break;
+        }
+        runStart = runEnd;
+    }
+
+    MmFreePool(pageProt);
     return s;
 }
-    }
-    return STATUS_SUCCESS;
-}
 
 
 // Entry point lookup.
@@ -408,7 +459,7 @@ uintptr_t LdrGetEntryPoint(uintptr_t base) {
 }
 
 // Base Relocation.
-static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
+static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, uintptr_t delta, bool user) {
     uint32_t peOffset = *(uint32_t *)(base + 0x3C);
     PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
     OPT_HEADER64* opt = &pe->OptionalHeader;
@@ -432,6 +483,8 @@ static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
             // For 64-bit PEs, we mostly care about DIR64 (Type 10).
             if (type == IMAGE_REL_BASED_DIR64) {
                 uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset);
+                KSTATUS ws = KiEnsureUserImageWrite(process, user, (uintptr_t)ptr, sizeof(*ptr));
+                if (!KSUCCESS(ws)) return ws;
                 *ptr += delta;
             }
         }
@@ -534,6 +587,8 @@ static KSTATUS LdrResolveImportsCommon(
                 return STATUS_NOT_FOUND;
             }
 
+            KSTATUS ws = KiEnsureUserImageWrite(process, process != (PROCESS*)0, (uintptr_t)thunk, sizeof(*thunk));
+            if (!KSUCCESS(ws)) return ws;
             *thunk = (uint64_t)addr;
 
             thunk++;
@@ -566,6 +621,41 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
     );
 }
 
+// Loads a user image from an open file through the image-section object model.
+KSTATUS LdrLoadImageFileForProcess(PROCESS* process, FILE_OBJECT* file, uintptr_t* baseOut) {
+    if (!process || !file || !baseOut)
+        return STATUS_INVALID_PARAMETER;
+    *baseOut = 0;
+
+    IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)0;
+    KSTATUS s = LdrCreateImageSectionFromFile(file, &section);
+    if (!KSUCCESS(s))
+        return s;
+
+    void* view = (void*)0;
+    s = LdrMapImageSection(process, section, &view);
+    if (!KSUCCESS(s)) {
+        LdrDereferenceImageSection(section);
+        return s;
+    }
+
+    const IMAGE_SECTION_METADATA* metadata = LdrImageSectionMetadata(section);
+    uintptr_t imageBase = (uintptr_t)view;
+    uintptr_t delta = imageBase - metadata->PreferredBase;
+    if (delta != 0) {
+        s = LdrApplyRelocationsForProcess(process, imageBase, delta, true);
+        if (!KSUCCESS(s)) {
+            KiFreeLoadedImageForProcess(process, imageBase, true);
+            LdrDereferenceImageSection(section);
+            return s;
+        }
+    }
+
+    LdrDereferenceImageSection(section);
+    *baseOut = imageBase;
+    return STATUS_SUCCESS;
+}
+
 // Implements the ldr Register Process Module operation.
 static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) {
     if (!process || !name || !base) return STATUS_INVALID_PARAMETER;
@@ -602,36 +692,27 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
     FILE_OBJECT* f;
     KSTATUS s = IoOpenFile(SysSvFromCstr(filename), &f);
     if (!KSUCCESS(s)) {
-        char path[260];
+        char relative[260];
-        memcpy(path, "0:/system/", 10);
+        size_t nameLen = strnlen(filename, sizeof(relative) - 8);
-        memcpy(path + 10, filename, strnlen(filename, 248) + 1);
+        if (nameLen + 8 >= sizeof(relative))
-        path[259] = '\0';
+            return STATUS_BUFFER_TOO_SMALL;
-        s = IoOpenFile(SysSvFromCstr(path), &f);
+        memcpy(relative, "system/", 7);
+        memcpy(relative + 7, filename, nameLen);
+        relative[7 + nameLen] = '\0';
+
+        s = KeOpenSystemRootFile(relative, &f);
         if (!KSUCCESS(s)) {
-            DbgPrint("LdrLoadModuleForProcess: Failed to open %s (%d)\n", filename, s);
+            DbgPrint("LdrLoadModuleForProcess: Failed to open %s on system root (%d)\n", filename, s);
             return s;
         }
     }
 
     uint64_t fileSize = IoGetFileSize(f);
     LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize);
-    void* mem = MmAllocatePool((size_t)fileSize);
-    if (!mem) {
-        IoCloseFile(f);
-        return STATUS_NO_MEMORY;
-    }
-
-    size_t got = 0;
-    s = IoReadFile(f, mem, (size_t)fileSize, &got);
-    IoCloseFile(f);
-    if (!KSUCCESS(s)) {
-        MmFreePool(mem);
-        return s;
-    }
 
     uintptr_t imageBase = 0;
-    s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase);
+    s = LdrLoadImageFileForProcess(process, f, &imageBase);
-    MmFreePool(mem);
+    IoCloseFile(f);
     if (!KSUCCESS(s)) {
         DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s);
         return s;
@@ -665,10 +746,9 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
     return STATUS_SUCCESS;
 }
 
-// LdrLoadImage - load a PE from a raw buffer into virtual space.
+// Loads a boot/kernel PE image from a raw memory buffer into kernel space.
-static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
+static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, uintptr_t* baseOut) {
     if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER;
-    if (user && !process) return STATUS_INVALID_PARAMETER;
     if (size < 0x40) return STATUS_FAILURE;
 
     const uint8_t* bytes = (const uint8_t*)buffer;
@@ -687,13 +767,13 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
 
     uint16_t optionalMagic = *(const uint16_t *)(bytes + peOffset + 24);
     if (machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
-        DbgPrint("LdrLoadImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
+        DbgPrint("LdrLoadKernelImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
                  machine, optionalMagic);
         return STATUS_NOT_SUPPORTED;
     }
 
     if (machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
-        DbgPrint("LdrLoadImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
+        DbgPrint("LdrLoadKernelImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
                  machine, optionalMagic);
         return STATUS_NOT_SUPPORTED;
     }
@@ -704,17 +784,16 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
     KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize);
     if (!KSUCCESS(sizeStatus)) return sizeStatus;
 
-    LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n",
+    LdrDbg("LDR: kernel image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u\n",
            opt->SizeOfImage,
            (unsigned long long)imageSize,
            (unsigned long long)opt->ImageBase,
-           pe->NumberOfSections,
+           pe->NumberOfSections);
-           user ? 1 : 0,
+
-           process ? process->ProcessId : 0);
+    void* mem = MmAllocatePoolTag(imageSize, MM_POOL_TAG('i', 'm', 'g', ' '));
-    void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize);
     if (!mem) {
-        DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n",
+        DbgPrint("LDR: kernel image allocation failed required=%llx\n",
-                 (unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0);
+                 (unsigned long long)imageSize);
         return STATUS_NO_MEMORY;
     }
     memset(mem, 0, imageSize);
@@ -730,27 +809,37 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
             memcpy(dst, src, copySize);
         }
 
-        uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0);
+        MmPteProtectRange((uintptr_t)mem + sec[i].VirtualAddress,
-        MmSetPageProtect((uintptr_t)mem + sec[i].VirtualAddress,
                          sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData,
-                         prot);
+                         PAGE_PRESENT | PAGE_WRITE);
     }
 
     uintptr_t delta = (uintptr_t)mem - opt->ImageBase;
     if (delta != 0) {
-        LdrApplyRelocations((uintptr_t)mem, delta);
+        KSTATUS rs = LdrApplyRelocationsForProcess((PROCESS*)0, (uintptr_t)mem, delta, false);
+        if (!KSUCCESS(rs)) {
+            MmFreePool(mem);
+            return rs;
+        }
+    }
+
+    // Kernel drivers are still pool-backed images. Section protections cannot
+    // safely make pages read-only yet because imports and driver-owned writable
+    // image data are not separated from callback code by a section object. What
+    // must be true is simpler: every page in the loaded image must be executable
+    // enough for driver callbacks, including section-alignment slack near the
+    // end of the image.
+    KSTATUS xs = MmPteProtectRange((uintptr_t)mem, imageSize, PAGE_PRESENT | PAGE_WRITE);
+    if (!KSUCCESS(xs)) {
+        MmFreePool(mem);
+        return xs;
     }
 
     *baseOut = (uintptr_t)mem;
     return STATUS_SUCCESS;
 }
 
-// Implements the ldr Load Image For Process operation.
+// Loads a kernel image from a bootloader or driver raw buffer.
-KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) {
+KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut) {
-    return KiLdrLoadImageCommon(process, buffer, size, baseOut, true);
+    return KiLdrLoadKernelImageFromBuffer(buffer, size, baseOut);
-}
-
-// Implements the ldr Load Image operation.
-KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
-    return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user);
 }

code/kernel/ldr_section.c

@@ -0,0 +1,736 @@
+// Image section objects. This file owns parsed PE image identity, sparse prototype pages, and image-section caching.
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <string.h>
+#include <kernel/ldr_section.h>
+#include <kernel/mm_virt.h>
+#include <kernel/mm_pfn.h>
+#include <kernel/mm_pte.h>
+#include <kernel/ob_object.h>
+#include <kernel/io_file.h>
+#include <kernel/ke_spinlock.h>
+#include <kernel/ke_debug.h>
+
+#define PE_SIGNATURE                      0x00004550UL
+#define IMAGE_FILE_MACHINE_AMD64          0x8664
+#define IMAGE_FILE_MACHINE_I386           0x014c
+#define IMAGE_NT_OPTIONAL_HDR32_MAGIC     0x10b
+#define IMAGE_NT_OPTIONAL_HDR64_MAGIC     0x20b
+#define IMAGE_SCN_CNT_CODE                0x00000020u
+#define IMAGE_SCN_CNT_INITIALIZED_DATA    0x00000040u
+#define IMAGE_SCN_CNT_UNINITIALIZED_DATA  0x00000080u
+#define IMAGE_SCN_MEM_EXECUTE             0x20000000u
+#define IMAGE_SCN_MEM_READ                0x40000000u
+#define IMAGE_SCN_MEM_WRITE               0x80000000u
+
+#define LDR_IMAGE_SECTION_CACHE_MAX       32u
+
+#pragma pack(push, 1)
+typedef struct LDR_OPT_HEADER64 {
+    uint16_t Magic;
+    uint8_t  MajorLinkerVersion;
+    uint8_t  MinorLinkerVersion;
+    uint32_t SizeOfCode;
+    uint32_t SizeOfInitializedData;
+    uint32_t SizeOfUninitializedData;
+    uint32_t AddressOfEntryPoint;
+    uint32_t BaseOfCode;
+    uint64_t ImageBase;
+    uint32_t SectionAlignment;
+    uint32_t FileAlignment;
+    uint16_t MajorOSVersion;
+    uint16_t MinorOSVersion;
+    uint16_t MajorImageVersion;
+    uint16_t MinorImageVersion;
+    uint16_t MajorSubsystemVersion;
+    uint16_t MinorSubsystemVersion;
+    uint32_t Win32VersionValue;
+    uint32_t SizeOfImage;
+    uint32_t SizeOfHeaders;
+    uint32_t CheckSum;
+    uint16_t Subsystem;
+    uint16_t DllCharacteristics;
+    uint64_t SizeOfStackReserve;
+    uint64_t SizeOfStackCommit;
+    uint64_t SizeOfHeapReserve;
+    uint64_t SizeOfHeapCommit;
+    uint32_t LoaderFlags;
+    uint32_t NumberOfRvaAndSizes;
+    struct { uint32_t VirtualAddress; uint32_t Size; } DataDirectory[16];
+} LDR_OPT_HEADER64;
+
+typedef struct LDR_PE_HEADER {
+    uint32_t Signature;
+    uint16_t Machine;
+    uint16_t NumberOfSections;
+    uint32_t TimeDateStamp;
+    uint32_t PointerToSymbolTable;
+    uint32_t NumberOfSymbols;
+    uint16_t SizeOfOptionalHeader;
+    uint16_t Characteristics;
+    LDR_OPT_HEADER64 OptionalHeader;
+} LDR_PE_HEADER;
+
+typedef struct LDR_SECTION_HEADER {
+    char     Name[8];
+    uint32_t VirtualSize;
+    uint32_t VirtualAddress;
+    uint32_t SizeOfRawData;
+    uint32_t PointerToRawData;
+    uint32_t PointerToRelocations;
+    uint32_t PointerToLinenumbers;
+    uint16_t NumberOfRelocations;
+    uint16_t NumberOfLinenumbers;
+    uint32_t Characteristics;
+} LDR_SECTION_HEADER;
+#pragma pack(pop)
+
+struct IMAGE_SECTION_OBJECT {
+    OBJECT Header;
+    IMAGE_SECTION_METADATA Metadata;
+    IMAGE_SECTION_DESCRIPTOR* Sections;
+    PHYSICAL_ADDRESS* Frames;
+    FILE_OBJECT* BackingFile;
+    size_t HeadersFileSize;
+    SPINLOCK Lock;
+};
+
+static IMAGE_SECTION_OBJECT* gImageSectionCache[LDR_IMAGE_SECTION_CACHE_MAX];
+static SPINLOCK gImageSectionCacheLock = SPINLOCK_INIT;
+
+// Aligns a PE image size to the hardware page size.
+static size_t KiImageSectionAlignSize(size_t value) {
+    return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
+}
+
+// Mixes filesystem identity into a compact cache key.
+static uint64_t KiImageSectionIdentityKey(const IO_FILE_IDENTITY* identity) {
+    if (!identity)
+        return 0;
+
+    uint64_t h = 1469598103934665603ULL;
+    h ^= identity->VolumeId;
+    h *= 1099511628211ULL;
+    h ^= identity->FileId;
+    h *= 1099511628211ULL;
+    h ^= identity->Generation;
+    h *= 1099511628211ULL;
+    h ^= identity->Size;
+    h *= 1099511628211ULL;
+    return h ? h : 1;
+}
+
+// Converts PE section characteristics to the final user page protection union for that section.
+static uint64_t KiImageSectionProtection(uint32_t characteristics) {
+    uint64_t prot = PAGE_PRESENT | PAGE_USER | PAGE_NX;
+    if (characteristics & IMAGE_SCN_MEM_WRITE)
+        prot |= PAGE_WRITE;
+    if (characteristics & IMAGE_SCN_MEM_EXECUTE)
+        prot &= ~PAGE_NX;
+    return prot;
+}
+
+// Computes the required image mapping size after validating raw section extents.
+static KSTATUS KiImageSectionComputeSize(const LDR_PE_HEADER* pe, size_t fileSize, size_t* outSize) {
+    if (!pe || !outSize) return STATUS_INVALID_PARAMETER;
+
+    const LDR_OPT_HEADER64* opt = &pe->OptionalHeader;
+    size_t required = KiImageSectionAlignSize(opt->SizeOfImage);
+    size_t headers = KiImageSectionAlignSize(opt->SizeOfHeaders);
+    if (headers > required) required = headers;
+
+    const LDR_SECTION_HEADER* sec = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
+    for (uint32_t i = 0; i < pe->NumberOfSections; i++) {
+        size_t va = (size_t)sec[i].VirtualAddress;
+        size_t virt = (size_t)sec[i].VirtualSize;
+        size_t raw = (size_t)sec[i].SizeOfRawData;
+        size_t span = virt ? virt : raw;
+        if (raw > span) span = raw;
+        if (!span) continue;
+
+        if (va > (size_t)-1 - span) return STATUS_NO_MEMORY;
+        size_t end = KiImageSectionAlignSize(va + span);
+        if (end > required) required = end;
+
+        if (raw) {
+            size_t rawBase = (size_t)sec[i].PointerToRawData;
+            if (rawBase > fileSize || raw > fileSize - rawBase)
+                return STATUS_FAILURE;
+        }
+    }
+
+    if (!required) return STATUS_FAILURE;
+    *outSize = required;
+    return STATUS_SUCCESS;
+}
+
+// Parses and validates the PE header that backs an image section object.
+static KSTATUS KiImageSectionParseHeader(const void* buffer,
+                                         size_t bufferSize,
+                                         size_t fileSize,
+                                         const LDR_PE_HEADER** peOut,
+                                         const LDR_SECTION_HEADER** sectionsOut,
+                                         size_t* imageSizeOut) {
+    if (!buffer || !bufferSize || !peOut || !sectionsOut || !imageSizeOut)
+        return STATUS_INVALID_PARAMETER;
+    if (bufferSize < 0x40) return STATUS_FAILURE;
+
+    const uint8_t* bytes = (const uint8_t*)buffer;
+    uint32_t peOffset = *(const uint32_t*)(bytes + 0x3C);
+    if ((size_t)peOffset > bufferSize || bufferSize - (size_t)peOffset < 24)
+        return STATUS_FAILURE;
+
+    const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)(bytes + peOffset);
+    if (pe->Signature != PE_SIGNATURE)
+        return STATUS_FAILURE;
+
+    uint16_t optionalSize = pe->SizeOfOptionalHeader;
+    if (optionalSize < sizeof(LDR_OPT_HEADER64) ||
+        (size_t)peOffset + 24 + optionalSize > bufferSize) {
+        return STATUS_FAILURE;
+    }
+
+    uint16_t optionalMagic = pe->OptionalHeader.Magic;
+    if (pe->Machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
+        DbgPrint("LdrImageSection: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
+                 pe->Machine, optionalMagic);
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    if (pe->Machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
+        DbgPrint("LdrImageSection: unsupported PE image (machine=0x%X magic=0x%X)\n",
+                 pe->Machine, optionalMagic);
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    if (!pe->NumberOfSections)
+        return STATUS_FAILURE;
+
+    const LDR_SECTION_HEADER* sections = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
+    size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader;
+    size_t sectionTableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER);
+    if (sectionTableOffset > bufferSize || sectionTableBytes > bufferSize - sectionTableOffset)
+        return STATUS_FAILURE;
+
+    size_t imageSize = 0;
+    KSTATUS s = KiImageSectionComputeSize(pe, fileSize, &imageSize);
+    if (!KSUCCESS(s)) return s;
+
+    *peOut = pe;
+    *sectionsOut = sections;
+    *imageSizeOut = imageSize;
+    return STATUS_SUCCESS;
+}
+
+// Releases all resources owned by an image section object.
+static void KiDestroyImageSectionObject(OBJECT* object) {
+    IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)object;
+    if (!section) return;
+
+    if (section->Frames) {
+        for (size_t i = 0; i < section->Metadata.PageCount; ++i) {
+            if (section->Frames[i])
+                (void)MmPfnFree(section->Frames[i]);
+        }
+        MmFreePool(section->Frames);
+        section->Frames = (PHYSICAL_ADDRESS*)0;
+    }
+
+    if (section->Sections) {
+        MmFreePool(section->Sections);
+        section->Sections = (IMAGE_SECTION_DESCRIPTOR*)0;
+    }
+
+    if (section->BackingFile) {
+        IoCloseFile(section->BackingFile);
+        section->BackingFile = (FILE_OBJECT*)0;
+    }
+}
+
+// Finds a cached image section while the image-section cache lock is held.
+static IMAGE_SECTION_OBJECT* KiFindImageSectionLocked(const IO_FILE_IDENTITY* identity, uint64_t key) {
+    if (!identity || !key)
+        return (IMAGE_SECTION_OBJECT*)0;
+
+    for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) {
+        IMAGE_SECTION_OBJECT* section = gImageSectionCache[i];
+        if (section &&
+            section->Metadata.ContentKey == key &&
+            section->Metadata.VolumeId == identity->VolumeId &&
+            section->Metadata.FileId == identity->FileId &&
+            section->Metadata.FileGeneration == identity->Generation &&
+            section->Metadata.RawSize == identity->Size) {
+            return section;
+        }
+    }
+    return (IMAGE_SECTION_OBJECT*)0;
+}
+
+
+// Verifies that a cached image section is still backed by the same file generation.
+static KSTATUS KiValidateImageSectionBackingFile(IMAGE_SECTION_OBJECT* section) {
+    if (!section || !section->BackingFile)
+        return STATUS_INVALID_PARAMETER;
+
+    IO_FILE_IDENTITY identity;
+    KSTATUS s = IoQueryFileIdentity(section->BackingFile, &identity);
+    if (!KSUCCESS(s))
+        return s;
+
+    if (identity.VolumeId != section->Metadata.VolumeId ||
+        identity.FileId != section->Metadata.FileId ||
+        identity.Generation != section->Metadata.FileGeneration ||
+        identity.Size != section->Metadata.RawSize) {
+        return STATUS_CHECKSUM_MISMATCH;
+    }
+
+    return STATUS_SUCCESS;
+}
+
+// Returns true when a virtual image page has initialized bytes in the backing file.
+static bool KiImageSectionPageHasFileBytes(const IMAGE_SECTION_OBJECT* section, size_t pageIndex) {
+    if (!section || pageIndex >= section->Metadata.PageCount)
+        return false;
+
+    size_t pageStart = pageIndex * PAGE_SIZE;
+    size_t pageEnd = pageStart + PAGE_SIZE;
+    if (pageStart < section->HeadersFileSize && pageEnd > 0)
+        return true;
+
+    for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) {
+        const IMAGE_SECTION_DESCRIPTOR* desc = &section->Sections[i];
+        size_t raw = desc->RawDataSize;
+        if (desc->VirtualSize && desc->VirtualSize < raw)
+            raw = desc->VirtualSize;
+        if (!raw)
+            continue;
+
+        size_t start = desc->VirtualAddress;
+        size_t end = start + raw;
+        if (end < start)
+            continue;
+        if (pageStart < end && pageEnd > start)
+            return true;
+    }
+
+    return false;
+}
+
+// Copies a file range into an image prototype page, leaving absent bytes zero-filled.
+static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section,
+                                           uint64_t fileOffset,
+                                           void* dst,
+                                           size_t size) {
+    if (!section || !section->BackingFile || (!dst && size))
+        return STATUS_INVALID_PARAMETER;
+    if (!size)
+        return STATUS_SUCCESS;
+
+    size_t got = 0;
+    KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got);
+    if (!KSUCCESS(s))
+        return s;
+    if (got != size)
+        return STATUS_FAILURE;
+    return STATUS_SUCCESS;
+}
+
+// Fills one prototype frame from the image backing file.
+static KSTATUS KiFillImageSectionFrameFromFile(IMAGE_SECTION_OBJECT* section,
+                                               size_t pageIndex,
+                                               PHYSICAL_ADDRESS frame) {
+    if (!section || !frame || pageIndex >= section->Metadata.PageCount)
+        return STATUS_INVALID_PARAMETER;
+
+    uint8_t* page = (uint8_t*)MmPfnToVirtual(frame);
+    if (!page)
+        return STATUS_FAILURE;
+    memset(page, 0, PAGE_SIZE);
+
+    size_t pageStart = pageIndex * PAGE_SIZE;
+    size_t pageEnd = pageStart + PAGE_SIZE;
+
+    if (pageStart < section->HeadersFileSize) {
+        size_t copyStart = pageStart;
+        size_t copyEnd = pageEnd;
+        if (copyEnd > section->HeadersFileSize)
+            copyEnd = section->HeadersFileSize;
+        KSTATUS s = KiReadImageSectionFileBytes(section,
+                                                (uint64_t)copyStart,
+                                                page,
+                                                copyEnd - copyStart);
+        if (!KSUCCESS(s))
+            return s;
+    }
+
+    for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) {
+        const IMAGE_SECTION_DESCRIPTOR* desc = &section->Sections[i];
+        size_t raw = desc->RawDataSize;
+        if (desc->VirtualSize && desc->VirtualSize < raw)
+            raw = desc->VirtualSize;
+        if (!raw)
+            continue;
+
+        size_t sectionStart = desc->VirtualAddress;
+        size_t sectionEnd = sectionStart + raw;
+        if (sectionEnd < sectionStart)
+            return STATUS_FAILURE;
+        if (pageStart >= sectionEnd || pageEnd <= sectionStart)
+            continue;
+
+        size_t copyStart = pageStart > sectionStart ? pageStart : sectionStart;
+        size_t copyEnd = pageEnd < sectionEnd ? pageEnd : sectionEnd;
+        size_t dstOffset = copyStart - pageStart;
+        uint64_t srcOffset = (uint64_t)desc->RawDataOffset + (uint64_t)(copyStart - sectionStart);
+
+        KSTATUS s = KiReadImageSectionFileBytes(section,
+                                                srcOffset,
+                                                page + dstOffset,
+                                                copyEnd - copyStart);
+        if (!KSUCCESS(s))
+            return s;
+    }
+
+    return STATUS_SUCCESS;
+}
+
+// Ensures a prototype page exists for initialized image bytes.
+static KSTATUS KiEnsureImageSectionFrame(IMAGE_SECTION_OBJECT* section, size_t pageIndex) {
+    if (!section || pageIndex >= section->Metadata.PageCount)
+        return STATUS_INVALID_PARAMETER;
+
+    uintptr_t flags = KeAcquireSpinlock(&section->Lock);
+    PHYSICAL_ADDRESS existing = section->Frames[pageIndex];
+    KeReleaseSpinlock(&section->Lock, flags);
+    if (existing)
+        return STATUS_SUCCESS;
+
+    if (section->BackingFile && !KiImageSectionPageHasFileBytes(section, pageIndex))
+        return STATUS_SUCCESS;
+
+    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseImage);
+    if (!frame)
+        return STATUS_NO_MEMORY;
+
+    void* va = MmPfnToVirtual(frame);
+    if (!va) {
+        MmPfnFree(frame);
+        return STATUS_FAILURE;
+    }
+
+    KSTATUS s = STATUS_SUCCESS;
+    if (section->BackingFile) {
+        s = KiValidateImageSectionBackingFile(section);
+        if (KSUCCESS(s))
+            s = KiFillImageSectionFrameFromFile(section, pageIndex, frame);
+    } else {
+        memset(va, 0, PAGE_SIZE);
+    }
+    if (!KSUCCESS(s)) {
+        MmPfnFree(frame);
+        return s;
+    }
+
+    flags = KeAcquireSpinlock(&section->Lock);
+    if (section->Frames[pageIndex]) {
+        KeReleaseSpinlock(&section->Lock, flags);
+        MmPfnFree(frame);
+        return STATUS_SUCCESS;
+    }
+    section->Frames[pageIndex] = frame;
+    KeReleaseSpinlock(&section->Lock, flags);
+    return STATUS_SUCCESS;
+}
+
+// Builds an uncached image section object backed directly by an open file.
+static KSTATUS KiBuildImageSectionObjectFromFile(FILE_OBJECT* file,
+                                                 const void* headerBuffer,
+                                                 size_t headerBufferSize,
+                                                 const IO_FILE_IDENTITY* identity,
+                                                 uint64_t key,
+                                                 IMAGE_SECTION_OBJECT** sectionOut) {
+    if (!file || !headerBuffer || !headerBufferSize || !identity || !sectionOut)
+        return STATUS_INVALID_PARAMETER;
+    *sectionOut = (IMAGE_SECTION_OBJECT*)0;
+
+    size_t fileSize = (size_t)identity->Size;
+    const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)0;
+    const LDR_SECTION_HEADER* rawSections = (const LDR_SECTION_HEADER*)0;
+    size_t imageSize = 0;
+    KSTATUS s = KiImageSectionParseHeader(headerBuffer,
+                                          headerBufferSize,
+                                          fileSize,
+                                          &pe,
+                                          &rawSections,
+                                          &imageSize);
+    if (!KSUCCESS(s))
+        return s;
+
+    size_t pageCount = (imageSize + PAGE_SIZE - 1) / PAGE_SIZE;
+    if (!pageCount || pageCount > ((size_t)-1 / sizeof(PHYSICAL_ADDRESS)))
+        return STATUS_FAILURE;
+
+    IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)ObAllocateObject(ObjectTypeImageSection,
+                                                                            sizeof(IMAGE_SECTION_OBJECT));
+    if (!section)
+        return STATUS_NO_MEMORY;
+    section->Header.Destroy = KiDestroyImageSectionObject;
+
+    PHYSICAL_ADDRESS* frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS),
+                                                                    MM_POOL_TAG('i', 's', 'f', 'r'));
+    if (!frames) {
+        ObDereferenceObject(&section->Header);
+        return STATUS_NO_MEMORY;
+    }
+    memset(frames, 0, pageCount * sizeof(PHYSICAL_ADDRESS));
+
+    IMAGE_SECTION_DESCRIPTOR* descriptors = (IMAGE_SECTION_DESCRIPTOR*)MmAllocatePoolTag(
+        (size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR),
+        MM_POOL_TAG('i', 's', 'e', 'c'));
+    if (!descriptors) {
+        MmFreePool(frames);
+        ObDereferenceObject(&section->Header);
+        return STATUS_NO_MEMORY;
+    }
+    memset(descriptors, 0, (size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR));
+
+    section->Frames = frames;
+    section->Sections = descriptors;
+    section->BackingFile = file;
+    section->HeadersFileSize = pe->OptionalHeader.SizeOfHeaders;
+    section->Lock = (SPINLOCK)SPINLOCK_INIT;
+    ObReferenceObject(&file->Header);
+
+    section->Metadata.ContentKey = key;
+    section->Metadata.VolumeId = identity->VolumeId;
+    section->Metadata.FileId = identity->FileId;
+    section->Metadata.FileGeneration = identity->Generation;
+    section->Metadata.PreferredBase = pe->OptionalHeader.ImageBase;
+    section->Metadata.EntryRva = pe->OptionalHeader.AddressOfEntryPoint;
+    section->Metadata.HeaderSize = pe->OptionalHeader.SizeOfHeaders;
+    section->Metadata.RawSize = fileSize;
+    section->Metadata.ImageSize = imageSize;
+    section->Metadata.PageCount = pageCount;
+    section->Metadata.SectionCount = pe->NumberOfSections;
+
+    for (uint32_t i = 0; i < pe->NumberOfSections; ++i) {
+        IMAGE_SECTION_DESCRIPTOR* dst = &section->Sections[i];
+        memcpy(dst->Name, rawSections[i].Name, sizeof(rawSections[i].Name));
+        dst->Name[8] = '\0';
+        dst->VirtualAddress = rawSections[i].VirtualAddress;
+        dst->VirtualSize = rawSections[i].VirtualSize;
+        dst->RawDataOffset = rawSections[i].PointerToRawData;
+        dst->RawDataSize = rawSections[i].SizeOfRawData;
+        dst->Characteristics = rawSections[i].Characteristics;
+        dst->PageProtection = KiImageSectionProtection(rawSections[i].Characteristics);
+    }
+
+    *sectionOut = section;
+    return STATUS_SUCCESS;
+}
+
+// Creates or references an image section object backed by an open file.
+KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file,
+                                      IMAGE_SECTION_OBJECT** sectionOut) {
+    if (!file || !sectionOut)
+        return STATUS_INVALID_PARAMETER;
+    *sectionOut = (IMAGE_SECTION_OBJECT*)0;
+
+    IO_FILE_IDENTITY identity;
+    KSTATUS s = IoQueryFileIdentity(file, &identity);
+    if (!KSUCCESS(s))
+        return s;
+
+    size_t fileSize = (size_t)identity.Size;
+    if (!fileSize)
+        return STATUS_FAILURE;
+
+    size_t firstRead = fileSize < 4096u ? fileSize : 4096u;
+    uint8_t stackHeader[4096];
+    size_t got = 0;
+    s = IoReadFileAt(file, 0, stackHeader, firstRead, &got);
+    if (!KSUCCESS(s) || got != firstRead)
+        return KSUCCESS(s) ? STATUS_FAILURE : s;
+    if (firstRead < 0x40)
+        return STATUS_FAILURE;
+
+    uint32_t peOffset = *(const uint32_t*)(stackHeader + 0x3C);
+    if ((size_t)peOffset > fileSize || fileSize - (size_t)peOffset < 24)
+        return STATUS_FAILURE;
+
+    size_t minimumHeader = (size_t)peOffset + 24u;
+    if (minimumHeader > firstRead) {
+        if (minimumHeader > 65536u || minimumHeader > fileSize)
+            return STATUS_FAILURE;
+        void* tmp = MmAllocatePoolTag(minimumHeader, MM_POOL_TAG('i', 's', 'h', 'd'));
+        if (!tmp)
+            return STATUS_NO_MEMORY;
+        s = IoReadFileAt(file, 0, tmp, minimumHeader, &got);
+        if (!KSUCCESS(s) || got != minimumHeader) {
+            MmFreePool(tmp);
+            return KSUCCESS(s) ? STATUS_FAILURE : s;
+        }
+        uint16_t optSize = *(const uint16_t*)((const uint8_t*)tmp + peOffset + 20u);
+        size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize;
+        MmFreePool(tmp);
+        if (fullHeader > firstRead) {
+            firstRead = fullHeader;
+        }
+    } else {
+        uint16_t optSize = *(const uint16_t*)(stackHeader + peOffset + 20u);
+        size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize;
+        if (fullHeader > firstRead)
+            firstRead = fullHeader;
+    }
+
+    if (firstRead > fileSize || firstRead > 65536u)
+        return STATUS_FAILURE;
+
+    void* header = stackHeader;
+    bool heapHeader = false;
+    if (firstRead > sizeof(stackHeader)) {
+        header = MmAllocatePoolTag(firstRead, MM_POOL_TAG('i', 's', 'h', 'd'));
+        if (!header)
+            return STATUS_NO_MEMORY;
+        heapHeader = true;
+        s = IoReadFileAt(file, 0, header, firstRead, &got);
+        if (!KSUCCESS(s) || got != firstRead) {
+            MmFreePool(header);
+            return KSUCCESS(s) ? STATUS_FAILURE : s;
+        }
+    }
+
+    const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)((const uint8_t*)header + peOffset);
+    size_t tableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER);
+    size_t tableEnd = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader + tableBytes;
+    if (tableEnd > firstRead) {
+        if (tableEnd > fileSize || tableEnd > 65536u) {
+            if (heapHeader) MmFreePool(header);
+            return STATUS_FAILURE;
+        }
+        void* bigger = MmAllocatePoolTag(tableEnd, MM_POOL_TAG('i', 's', 'h', 'd'));
+        if (!bigger) {
+            if (heapHeader) MmFreePool(header);
+            return STATUS_NO_MEMORY;
+        }
+        s = IoReadFileAt(file, 0, bigger, tableEnd, &got);
+        if (heapHeader) MmFreePool(header);
+        if (!KSUCCESS(s) || got != tableEnd) {
+            MmFreePool(bigger);
+            return KSUCCESS(s) ? STATUS_FAILURE : s;
+        }
+        header = bigger;
+        firstRead = tableEnd;
+        heapHeader = true;
+    }
+
+    uint64_t key = KiImageSectionIdentityKey(&identity);
+
+    uintptr_t flags = KeAcquireSpinlock(&gImageSectionCacheLock);
+    IMAGE_SECTION_OBJECT* cached = KiFindImageSectionLocked(&identity, key);
+    if (cached) {
+        ObReferenceObject(&cached->Header);
+        KeReleaseSpinlock(&gImageSectionCacheLock, flags);
+        if (heapHeader) MmFreePool(header);
+        *sectionOut = cached;
+        return STATUS_SUCCESS;
+    }
+    KeReleaseSpinlock(&gImageSectionCacheLock, flags);
+
+    IMAGE_SECTION_OBJECT* built = (IMAGE_SECTION_OBJECT*)0;
+    s = KiBuildImageSectionObjectFromFile(file, header, firstRead, &identity, key, &built);
+    if (heapHeader) MmFreePool(header);
+    if (!KSUCCESS(s))
+        return s;
+
+    flags = KeAcquireSpinlock(&gImageSectionCacheLock);
+    cached = KiFindImageSectionLocked(&identity, key);
+    if (cached) {
+        ObReferenceObject(&cached->Header);
+        KeReleaseSpinlock(&gImageSectionCacheLock, flags);
+        ObDereferenceObject(&built->Header);
+        *sectionOut = cached;
+        return STATUS_SUCCESS;
+    }
+
+    for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) {
+        if (!gImageSectionCache[i]) {
+            ObReferenceObject(&built->Header);
+            gImageSectionCache[i] = built;
+            break;
+        }
+    }
+    KeReleaseSpinlock(&gImageSectionCacheLock, flags);
+
+    *sectionOut = built;
+    return STATUS_SUCCESS;
+}
+
+// Adds a reference to an existing image section object.
+void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section) {
+    if (!section) return;
+    ObReferenceObject(&section->Header);
+}
+
+// Releases a reference returned by LdrCreateImageSectionFromFile.
+void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section) {
+    if (!section) return;
+    ObDereferenceObject(&section->Header);
+}
+
+// Returns immutable metadata for an image section object.
+const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section) {
+    if (!section) return (const IMAGE_SECTION_METADATA*)0;
+    return &section->Metadata;
+}
+
+// Returns the immutable PE section table owned by the image section object.
+const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section,
+                                                          uint32_t* countOut) {
+    if (countOut) *countOut = section ? section->Metadata.SectionCount : 0;
+    if (!section) return (const IMAGE_SECTION_DESCRIPTOR*)0;
+    return section->Sections;
+}
+
+// Retains one prototype page frame owned by an image section object.
+KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section,
+                                  size_t pageIndex,
+                                  PHYSICAL_ADDRESS* frameOut) {
+    if (!section || !frameOut) return STATUS_INVALID_PARAMETER;
+    *frameOut = 0;
+    if (pageIndex >= section->Metadata.PageCount)
+        return STATUS_INVALID_PARAMETER;
+
+    KSTATUS es = KiEnsureImageSectionFrame(section, pageIndex);
+    if (!KSUCCESS(es))
+        return es;
+
+    uintptr_t flags = KeAcquireSpinlock(&section->Lock);
+    PHYSICAL_ADDRESS frame = section->Frames[pageIndex];
+    KeReleaseSpinlock(&section->Lock, flags);
+    if (!frame)
+        return STATUS_SUCCESS;
+
+    KSTATUS s = MmPfnRetain(frame);
+    if (!KSUCCESS(s))
+        return s;
+
+    *frameOut = frame;
+    return STATUS_SUCCESS;
+}
+
+// Maps an image section into a process as an image-backed VAD.
+KSTATUS LdrMapImageSection(PROCESS* process,
+                           IMAGE_SECTION_OBJECT* section,
+                           void** baseOut) {
+    if (!process || !section || !baseOut) return STATUS_INVALID_PARAMETER;
+    *baseOut = (void*)0;
+
+    return MmVadCreateImageSectionMapping(process,
+                                          section,
+                                          PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
+                                          MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
+                                          baseOut);
+}

code/kernel/mm_pte.c

@@ -0,0 +1,575 @@
+// Kernel page-table manager. This file owns x86-64 PTE manipulation, TLB invalidation, and shared-CR3 activation.
+
+#include <stddef.h>
+#include <string.h>
+#include <intrin.h>
+
+#include <kernel/mm_pte.h>
+#include <kernel/mm_pfn.h>
+#include <kernel/ke_cpu.h>
+#include <kernel/ke_debug.h>
+#include <kernel/ke_spinlock.h>
+#include <kernel/hal/hal_apic.h>
+
+// Page-table frames are reached through the physical direct map while the kernel still owns one shared CR3.
+static uint64_t gPhysBase = PHYS_MAP_BASE;
+
+// Converts a physical address to the kernel physical-window alias used for page-table editing.
+static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
+    return (void *)(p + gPhysBase);
+}
+// Returns the entry pointer for a page-table frame that is already mapped in the physical window.
+static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
+    return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
+}
+
+// The bootloader builds the initial PML4, and this layer owns all later edits to it.
+static PHYSICAL_ADDRESS gPml4Phys = 0;
+static SPINLOCK gPageTableLock = SPINLOCK_INIT;
+
+static volatile long gTlbShootdownLock = 0;
+static volatile long gTlbShootdownActive = 0;
+static volatile long gTlbShootdownAcks = 0;
+static uintptr_t gTlbShootdownVirt = 0;
+static MM_PTE_STATS gPteStats;
+
+#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u
+
+// These macros extract the hardware page-table index for each level.
+#define PML4_IDX(va)  (((va) >> 39) & 0x1FF)
+#define PDPT_IDX(va)  (((va) >> 30) & 0x1FF)
+#define PDT_IDX(va)   (((va) >> 21) & 0x1FF)
+#define PT_IDX(va)    (((va) >> 12) & 0x1FF)
+
+#define PAGE_PS              (1ULL << 7)
+#define PAGE_FLAGS_MASK      (0xFFFULL | PAGE_NX)
+#define PAGE_FRAME_4K_MASK   0x000FFFFFFFFFF000ULL
+#define PAGE_FRAME_2M_MASK   0x000FFFFFFFE00000ULL
+#define PAGE_FRAME_1G_MASK   0x000FFFFFC0000000ULL
+
+// Bumps a telemetry counter. These counters are diagnostic only; exactness is
+// less important than avoiding extra locks in hot page-table paths.
+static inline void KiPteStatInc(uint64_t* counter) {
+    (*counter)++;
+}
+
+// Clamps query attribution so corrupt callers cannot index outside telemetry arrays.
+static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose) {
+    if ((uint32_t)purpose >= (uint32_t)MmPteQueryPurposeCount)
+        return MmPteQueryGeneric;
+    return purpose;
+}
+
+// Ensures that one parent entry points to a child page table.
+// New child tables are zeroed before publication so no stale mappings become visible.
+static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
+                                    unsigned index, uint64_t extraFlags) {
+    uint64_t* entry = KiPtePtr(parentPhys, index);
+    if (*entry & PAGE_PRESENT) {
+        *entry |= extraFlags;
+        return *entry & PAGE_FRAME_4K_MASK;
+    }
+    PHYSICAL_ADDRESS childPhys = MmPfnAllocate(MmPfnUsePageTable);
+    if (!childPhys) return 0;
+    KiPteStatInc(&gPteStats.PageTablePagesAllocated);
+    memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE);
+    *entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags;
+    return childPhys;
+}
+
+
+// Initializes the page-table manager around the bootloader-created PML4.
+void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys) {
+    gPml4Phys = bootPml4Phys;
+    gPageTableLock = (SPINLOCK)SPINLOCK_INIT;
+    gTlbShootdownLock = 0;
+    gTlbShootdownActive = 0;
+    gTlbShootdownAcks = 0;
+    gTlbShootdownVirt = 0;
+    memset(&gPteStats, 0, sizeof(gPteStats));
+}
+
+// Installs the shared PML4 into CR3.
+void MmPteActivate(void) {
+    __writecr3(gPml4Phys);
+}
+
+// Flushes one virtual address from the local TLB.
+void MmPteFlushLocal(uintptr_t virt) {
+    KiPteStatInc(&gPteStats.LocalFlushes);
+    __invlpg((void*)virt);
+}
+
+// Acquires the shootdown state lock without disabling interrupts.
+static void KiAcquireTlbShootdownLock(void) {
+    // Do not use the normal spinlock here: it masks interrupts while spinning.
+    // A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
+    // concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is
+    // meant to avoid.
+    while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0)
+        _mm_pause();
+    _ReadWriteBarrier();
+}
+
+// Releases the shootdown state lock after all shared fields are stable.
+static void KiReleaseTlbShootdownLock(void) {
+    _ReadWriteBarrier();
+    _InterlockedExchange(&gTlbShootdownLock, 0);
+}
+
+// Flushes one kernel mapping on every online CPU that shares this PML4.
+void MmPteFlushKernelAddress(uintptr_t virt) {
+    KiPteStatInc(&gPteStats.GlobalFlushes);
+    MmPteFlushLocal(virt);
+
+    if (gCpuCount <= 1) {
+        KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets);
+        return;
+    }
+
+    KCPU* self = KeGetCurrentCpu();
+    uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU;
+
+    KiAcquireTlbShootdownLock();
+
+    long targets = 0;
+    gTlbShootdownVirt = virt;
+    _InterlockedExchange(&gTlbShootdownAcks, 0);
+    _ReadWriteBarrier();
+    _InterlockedExchange(&gTlbShootdownActive, 1);
+    _ReadWriteBarrier();
+
+    for (uint32_t i = 0; i < gCpuCount; i++) {
+        KCPU* cpu = &gCpus[i];
+        if (i == selfIndex)
+            continue;
+        if (!cpu->Present || !cpu->Started || !cpu->Online)
+            continue;
+        targets++;
+        KiPteStatInc(&gPteStats.ShootdownTargetCpus);
+        HalApicSendTlbShootdownIpi(cpu->LapicId);
+    }
+
+    if (!targets) {
+        KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets);
+        _InterlockedExchange(&gTlbShootdownActive, 0);
+        _ReadWriteBarrier();
+        KiReleaseTlbShootdownLock();
+        return;
+    }
+
+    KiPteStatInc(&gPteStats.ShootdownRequests);
+
+    for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) {
+        if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets)
+            break;
+        _mm_pause();
+    }
+
+    long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0);
+    _InterlockedExchange(&gTlbShootdownActive, 0);
+    _ReadWriteBarrier();
+    KiReleaseTlbShootdownLock();
+
+    gPteStats.ShootdownAcks += (uint64_t)acks;
+    if (acks < targets) {
+        KiPteStatInc(&gPteStats.ShootdownTimeouts);
+        KePanic(KE_PANIC_CODE_GENERIC,
+                "TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n",
+                (uint64_t)virt, targets, acks, selfIndex);
+    }
+}
+
+// Handles an incoming shootdown request on the current CPU.
+static int KiHandleTlbShootdown(void) {
+    if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
+        return 0;
+
+    uintptr_t va = gTlbShootdownVirt;
+    _ReadWriteBarrier();
+    MmPteFlushLocal(va);
+    _ReadWriteBarrier();
+    _InterlockedIncrement(&gTlbShootdownAcks);
+    return 1;
+}
+
+// Handles the NMI vector reserved for TLB shootdowns.
+int MmPteHandleShootdownNmi(void) {
+    return KiHandleTlbShootdown();
+}
+
+// Maps or replaces one 4 KiB leaf PTE in the shared page tables.
+__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) {
+    KiPteStatInc(&gPteStats.MapCalls);
+    unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
+
+    uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
+
+    PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
+    if (!pdpt) {
+        KiPteStatInc(&gPteStats.MapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_NO_MEMORY;
+    }
+    PHYSICAL_ADDRESS pdt  = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
+    if (!pdt) {
+        KiPteStatInc(&gPteStats.MapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_NO_MEMORY;
+    }
+    PHYSICAL_ADDRESS pt   = KiEnsureTable(pdt,  PDT_IDX(virt),  userFlag);
+    if (!pt) {
+        KiPteStatInc(&gPteStats.MapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_NO_MEMORY;
+    }
+
+    uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
+    if ((*pte & PAGE_PRESENT) && mode == MmPteMapFailIfPresent) {
+        KiPteStatInc(&gPteStats.MapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_ALREADY_EXISTS;
+    }
+    int wasPresent = ((*pte & PAGE_PRESENT) != 0);
+    *pte = phys | flags;
+    if (!wasPresent) {
+        KiPteStatInc(&gPteStats.CurrentLeafMappings);
+        KiPteStatInc(&gPteStats.NewMappingNoFlushes);
+    }
+
+    KeReleaseSpinlock(&gPageTableLock, lockFlags);
+
+    if (wasPresent) {
+        KiPteStatInc(&gPteStats.ReplacedMappingFlushes);
+        MmPteFlushKernelAddress(virt);
+    }
+    return STATUS_SUCCESS;
+}
+
+
+// Removes one 4 KiB leaf PTE from the shared page tables.
+__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
+    KiPteStatInc(&gPteStats.UnmapCalls);
+    uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
+
+    uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
+    if (!(*e4 & PAGE_PRESENT)) {
+        KiPteStatInc(&gPteStats.UnmapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_NOT_FOUND;
+    }
+    uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt));
+    if (!(*e3 & PAGE_PRESENT)) {
+        KiPteStatInc(&gPteStats.UnmapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_NOT_FOUND;
+    }
+    uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt));
+    if (!(*e2 & PAGE_PRESENT)) {
+        KiPteStatInc(&gPteStats.UnmapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_NOT_FOUND;
+    }
+    uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt));
+    if (!(*pte & PAGE_PRESENT)) {
+        KiPteStatInc(&gPteStats.UnmapFailures);
+        KeReleaseSpinlock(&gPageTableLock, lockFlags);
+        return STATUS_NOT_FOUND;
+    }
+    *pte = 0;
+    if (gPteStats.CurrentLeafMappings)
+        gPteStats.CurrentLeafMappings--;
+
+    KeReleaseSpinlock(&gPageTableLock, lockFlags);
+    MmPteFlushKernelAddress(virt);
+    return STATUS_SUCCESS;
+}
+
+
+// Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them.
+__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) {
+    KiPteStatInc(&gPteStats.ProtectCalls);
+    if (!size) return STATUS_SUCCESS;
+
+    // The protection operation is page-granular even when callers pass an interior address.
+    uintptr_t base = virt & PAGE_MASK;
+    uintptr_t end  = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
+
+    uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
+    KSTATUS status = STATUS_SUCCESS;
+    uintptr_t flushEnd = base;
+
+    for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
+        uint64_t* e4 = KiPtePtr(gPml4Phys,          PML4_IDX(va));
+        if (!(*e4 & PAGE_PRESENT)) {
+            status = STATUS_NOT_FOUND;
+            break;
+        }
+        uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK,    PDPT_IDX(va));
+        if (!(*e3 & PAGE_PRESENT)) {
+            status = STATUS_NOT_FOUND;
+            break;
+        }
+        if (*e3 & PAGE_PS) {
+            status = STATUS_NOT_SUPPORTED;
+            break;
+        }
+        uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK,    PDT_IDX(va));
+        if (!(*e2 & PAGE_PRESENT)) {
+            status = STATUS_NOT_FOUND;
+            break;
+        }
+        if (*e2 & PAGE_PS) {
+            status = STATUS_NOT_SUPPORTED;
+            break;
+        }
+        uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK,   PT_IDX(va));
+        if (!(*pte & PAGE_PRESENT)) {
+            status = STATUS_NOT_FOUND;
+            break;
+        }
+
+        // The physical frame stays attached while access and cache attributes change.
+        PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK;
+        *pte = phys | flags;
+        flushEnd = va + PAGE_SIZE;
+    }
+
+    if (!KSUCCESS(status))
+        KiPteStatInc(&gPteStats.ProtectFailures);
+
+    KeReleaseSpinlock(&gPageTableLock, lockFlags);
+
+    for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE)
+        MmPteFlushKernelAddress(va);
+
+    return status;
+}
+
+
+// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf.  This is
+// used by the pressure trimmer as a second-chance test for clean file-backed
+// pages.  If the page was accessed recently, reclaim skips it once and flushes
+// the TLB entry so future accesses can set PAGE_ACCESSED again.
+__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt) {
+    if (!gPml4Phys)
+        return 0;
+
+    uintptr_t va = virt & PAGE_MASK;
+    int wasAccessed = 0;
+
+    uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
+
+    uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
+    if (!(*e4 & PAGE_PRESENT))
+        goto out;
+    uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
+    if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS))
+        goto out;
+    uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
+    if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS))
+        goto out;
+    uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
+    if (!(*pte & PAGE_PRESENT))
+        goto out;
+
+    if (*pte & PAGE_ACCESSED) {
+        *pte &= ~PAGE_ACCESSED;
+        wasAccessed = 1;
+    }
+
+out:
+    KeReleaseSpinlock(&gPageTableLock, lockFlags);
+
+    if (wasAccessed)
+        MmPteFlushKernelAddress(va);
+    return wasAccessed;
+}
+
+// Tests and clears the hardware dirty bit on a mapped 4 KiB leaf.
+__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt) {
+    if (!gPml4Phys)
+        return 0;
+
+    uintptr_t va = virt & PAGE_MASK;
+    int wasDirty = 0;
+
+    uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
+
+    uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
+    if (!(*e4 & PAGE_PRESENT))
+        goto out;
+    uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
+    if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS))
+        goto out;
+    uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
+    if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS))
+        goto out;
+    uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
+    if (!(*pte & PAGE_PRESENT))
+        goto out;
+
+    if (*pte & PAGE_DIRTY) {
+        *pte &= ~PAGE_DIRTY;
+        wasDirty = 1;
+    }
+
+out:
+    KeReleaseSpinlock(&gPageTableLock, lockFlags);
+
+    if (wasDirty)
+        MmPteFlushKernelAddress(va);
+    return wasDirty;
+}
+
+// Translates a mapped virtual address to a physical address.
+__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt) {
+    KiPteStatInc(&gPteStats.TranslateCalls);
+    if (!gPml4Phys) return 0;
+
+    PHYSICAL_ADDRESS result = 0;
+    uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
+
+    uint64_t pml4_idx = (virt >> 39) & 0x1FF;
+    uint64_t pdpt_idx = (virt >> 30) & 0x1FF;
+    uint64_t pd_idx   = (virt >> 21) & 0x1FF;
+    uint64_t pt_idx   = (virt >> 12) & 0x1FF;
+
+    uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
+    if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out;
+
+    uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK);
+    if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out;
+
+    // A present PDPT leaf maps a 1 GiB huge page.
+    if (pdpt[pdpt_idx] & PAGE_PS) {
+        result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF);
+        goto out;
+    }
+
+    uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK);
+    if (!(pd[pd_idx] & PAGE_PRESENT)) goto out;
+    
+    // A present PDT leaf maps a 2 MiB huge page.
+    if (pd[pd_idx] & PAGE_PS) {
+        result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF);
+        goto out;
+    }
+
+    uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK);
+    if (!(pt[pt_idx] & PAGE_PRESENT)) goto out;
+
+    // A normal PT leaf maps a 4 KiB page.
+    result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
+
+out:
+    KeReleaseSpinlock(&gPageTableLock, lockFlags);
+    return result;
+}
+
+// Queries one virtual address through the shared page-table hierarchy.
+KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose,
+                           uintptr_t virt,
+                           PHYSICAL_ADDRESS* physOut,
+                           uint64_t* flagsOut) {
+    purpose = KiNormalizeQueryPurpose(purpose);
+    KiPteStatInc(&gPteStats.QueryCalls);
+    KiPteStatInc(&gPteStats.QueryByPurpose[purpose]);
+    if (!gPml4Phys) {
+        KiPteStatInc(&gPteStats.QueryFailures);
+        KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]);
+        return STATUS_NOT_FOUND;
+    }
+
+    KSTATUS status = STATUS_NOT_FOUND;
+    uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
+
+    uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys);
+    uint64_t e4 = pml4[PML4_IDX(virt)];
+    if (!(e4 & PAGE_PRESENT)) goto out;
+
+    uint64_t effectiveWrite = e4 & PAGE_WRITE;
+    uint64_t effectiveUser  = e4 & PAGE_USER;
+    uint64_t effectiveNx    = e4 & PAGE_NX;
+
+    uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK);
+    uint64_t e3 = pdpt[PDPT_IDX(virt)];
+    if (!(e3 & PAGE_PRESENT)) goto out;
+    effectiveWrite &= e3 & PAGE_WRITE;
+    effectiveUser  &= e3 & PAGE_USER;
+    effectiveNx    |= e3 & PAGE_NX;
+
+    if (e3 & PAGE_PS) {
+        if (physOut)
+            *physOut = (e3 & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFFULL);
+        if (flagsOut) {
+            uint64_t flags = (e3 & PAGE_FLAGS_MASK) | effectiveNx;
+            if (!effectiveWrite) flags &= ~PAGE_WRITE;
+            if (!effectiveUser)  flags &= ~PAGE_USER;
+            *flagsOut = flags;
+        }
+        status = STATUS_SUCCESS;
+        goto out;
+    }
+
+    uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK);
+    uint64_t e2 = pd[PDT_IDX(virt)];
+    if (!(e2 & PAGE_PRESENT)) goto out;
+    effectiveWrite &= e2 & PAGE_WRITE;
+    effectiveUser  &= e2 & PAGE_USER;
+    effectiveNx    |= e2 & PAGE_NX;
+
+    if (e2 & PAGE_PS) {
+        if (physOut)
+            *physOut = (e2 & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFFULL);
+        if (flagsOut) {
+            uint64_t flags = (e2 & PAGE_FLAGS_MASK) | effectiveNx;
+            if (!effectiveWrite) flags &= ~PAGE_WRITE;
+            if (!effectiveUser)  flags &= ~PAGE_USER;
+            *flagsOut = flags;
+        }
+        status = STATUS_SUCCESS;
+        goto out;
+    }
+
+    uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK);
+    uint64_t e1 = pt[PT_IDX(virt)];
+    if (!(e1 & PAGE_PRESENT)) goto out;
+    effectiveWrite &= e1 & PAGE_WRITE;
+    effectiveUser  &= e1 & PAGE_USER;
+    effectiveNx    |= e1 & PAGE_NX;
+
+    if (physOut)
+        *physOut = (e1 & PAGE_FRAME_4K_MASK) + (virt & 0xFFFULL);
+    if (flagsOut) {
+        uint64_t flags = (e1 & PAGE_FLAGS_MASK) | effectiveNx;
+        if (!effectiveWrite) flags &= ~PAGE_WRITE;
+        if (!effectiveUser)  flags &= ~PAGE_USER;
+        *flagsOut = flags;
+    }
+    status = STATUS_SUCCESS;
+
+out:
+    if (!KSUCCESS(status)) {
+        KiPteStatInc(&gPteStats.QueryFailures);
+        KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]);
+    }
+    KeReleaseSpinlock(&gPageTableLock, lockFlags);
+    return status;
+}
+
+// Queries one virtual address without caller-specific attribution.
+__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt,
+                                          PHYSICAL_ADDRESS* physOut,
+                                          uint64_t* flagsOut) {
+    return MmPteQueryPageFor(MmPteQueryGeneric, virt, physOut, flagsOut);
+}
+
+// Returns page-table telemetry for diagnostics.
+void MmPteQueryStats(MM_PTE_STATS* out) {
+    if (!out)
+        return;
+    *out = gPteStats;
+}
+
+

code/kernel/mm_swap.c

@@ -0,0 +1,243 @@
+// Kernel swap manager.
+// The first backend is a fixed-size pagefile accessed through VFS offset I/O.
+// Kernel memory and filesystem code remain non-pageable, so this layer is safe
+// to use before the VMM has direct block-extents for pageout.
+
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include <kernel/mm_swap.h>
+#include <kernel/mm_virt.h>
+#include <kernel/mm_pfn.h>
+#include <kernel/io_file.h>
+#include <kernel/ke_spinlock.h>
+#include <kernel/ke_debug.h>
+
+typedef struct MM_SWAP_STATE {
+    SPINLOCK Lock;
+    FILE_OBJECT* File;
+    uint8_t* Bitmap;
+    uint32_t SlotCount;
+    uint32_t UsedSlots;
+    uint32_t HighWaterSlots;
+    int Enabled;
+    MM_SWAP_STATS Stats;
+} MM_SWAP_STATE;
+
+static MM_SWAP_STATE gSwap = { .Lock = SPINLOCK_INIT };
+
+// Returns whether a swap slot is marked in use.
+static int KiSwapTestSlot(uint32_t slot) {
+    return (gSwap.Bitmap[slot >> 3] & (uint8_t)(1u << (slot & 7))) != 0;
+}
+
+// Marks a swap slot as allocated or free.
+static void KiSwapSetSlot(uint32_t slot, int used) {
+    uint8_t mask = (uint8_t)(1u << (slot & 7));
+    if (used)
+        gSwap.Bitmap[slot >> 3] |= mask;
+    else
+        gSwap.Bitmap[slot >> 3] &= (uint8_t)~mask;
+}
+
+// Allocates one pagefile slot.
+static KSTATUS KiSwapAllocateSlot(uint32_t* slotOut) {
+    if (!slotOut) return STATUS_INVALID_PARAMETER;
+    *slotOut = 0;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
+    if (!gSwap.Enabled || !gSwap.Bitmap || !gSwap.SlotCount) {
+        KeReleaseSpinlock(&gSwap.Lock, flags);
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    for (uint32_t slot = 0; slot < gSwap.SlotCount; ++slot) {
+        if (!KiSwapTestSlot(slot)) {
+            KiSwapSetSlot(slot, 1);
+            gSwap.UsedSlots++;
+            if (gSwap.UsedSlots > gSwap.HighWaterSlots)
+                gSwap.HighWaterSlots = gSwap.UsedSlots;
+            gSwap.Stats.SlotAllocations++;
+            *slotOut = slot;
+            KeReleaseSpinlock(&gSwap.Lock, flags);
+            return STATUS_SUCCESS;
+        }
+    }
+
+    gSwap.Stats.SlotAllocationFailures++;
+    KeReleaseSpinlock(&gSwap.Lock, flags);
+    return STATUS_NO_MEMORY;
+}
+
+// Initializes the fixed pagefile backend.
+KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
+    if (!path.Data || !path.Length || sizeBytes < PAGE_SIZE)
+        return STATUS_INVALID_PARAMETER;
+    if (sizeBytes & (PAGE_SIZE - 1))
+        sizeBytes = (sizeBytes + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1);
+
+    uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
+    if (gSwap.Enabled) {
+        KeReleaseSpinlock(&gSwap.Lock, flags);
+        return STATUS_SUCCESS;
+    }
+    KeReleaseSpinlock(&gSwap.Lock, flags);
+
+    FILE_OBJECT* file = (FILE_OBJECT*)0;
+    KSTATUS s = IoOpenFile(path, &file);
+    if (!KSUCCESS(s)) {
+        s = IoCreateFile(path, &file);
+        if (!KSUCCESS(s))
+            return s;
+    }
+
+    // Swap slots are never read before MmSwapPageOut overwrites the whole
+    // page, so the filesystem may allocate backing clusters without zeroing
+    // them.  Normal files still get zero-on-extend semantics.
+    file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;
+
+    s = IoTruncateFile(file, sizeBytes);
+    if (!KSUCCESS(s)) {
+        IoCloseFile(file);
+        return s;
+    }
+
+    uint64_t slots64 = sizeBytes / PAGE_SIZE;
+    if (!slots64 || slots64 > 0xFFFFFFFFull) {
+        IoCloseFile(file);
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    uint32_t slotCount = (uint32_t)slots64;
+    size_t bitmapBytes = (slotCount + 7u) / 8u;
+    uint8_t* bitmap = (uint8_t*)MmAllocatePoolTag(bitmapBytes, MM_POOL_TAG('s', 'w', 'a', 'p'));
+    if (!bitmap) {
+        IoCloseFile(file);
+        return STATUS_NO_MEMORY;
+    }
+    memset(bitmap, 0, bitmapBytes);
+
+    flags = KeAcquireSpinlock(&gSwap.Lock);
+    if (gSwap.Enabled) {
+        KeReleaseSpinlock(&gSwap.Lock, flags);
+        MmFreePool(bitmap);
+        IoCloseFile(file);
+        return STATUS_SUCCESS;
+    }
+
+    gSwap.File = file;
+    gSwap.Bitmap = bitmap;
+    gSwap.SlotCount = slotCount;
+    gSwap.UsedSlots = 0;
+    gSwap.HighWaterSlots = 0;
+    gSwap.Enabled = 1;
+    memset(&gSwap.Stats, 0, sizeof(gSwap.Stats));
+    gSwap.Stats.Enabled = 1;
+    gSwap.Stats.TotalSlots = slotCount;
+    gSwap.Stats.FreeSlots = slotCount;
+    KeReleaseSpinlock(&gSwap.Lock, flags);
+
+    DbgPrint("swap: initialized %u slots (%llu KiB) at %v\n",
+             slotCount,
+             (unsigned long long)(sizeBytes / 1024),
+             path);
+    return STATUS_SUCCESS;
+}
+
+// Returns whether swap is available for pageout.
+int MmSwapIsEnabled(void) {
+    return gSwap.Enabled;
+}
+
+// Writes one physical page to a newly allocated swap slot.
+KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut) {
+    if (!slotOut || !frame)
+        return STATUS_INVALID_PARAMETER;
+    *slotOut = 0;
+    if (!gSwap.Enabled || !gSwap.File)
+        return STATUS_NOT_SUPPORTED;
+
+    uint32_t slot = 0;
+    KSTATUS s = KiSwapAllocateSlot(&slot);
+    if (!KSUCCESS(s)) {
+        gSwap.Stats.PageOutFailures++;
+        return s;
+    }
+
+    void* page = MmPfnToVirtual(frame);
+    if (!page) {
+        MmSwapFreeSlot(slot);
+        gSwap.Stats.PageOutFailures++;
+        return STATUS_FAILURE;
+    }
+
+    size_t written = 0;
+    s = IoWriteFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &written);
+    if (!KSUCCESS(s) || written != PAGE_SIZE) {
+        MmSwapFreeSlot(slot);
+        gSwap.Stats.PageOutFailures++;
+        return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s;
+    }
+
+    gSwap.Stats.PageOuts++;
+    *slotOut = slot;
+    return STATUS_SUCCESS;
+}
+
+// Reads one swap slot into an already allocated physical page.
+KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame) {
+    if (!frame)
+        return STATUS_INVALID_PARAMETER;
+    if (!gSwap.Enabled || !gSwap.File)
+        return STATUS_NOT_SUPPORTED;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
+    int valid = slot < gSwap.SlotCount && KiSwapTestSlot(slot);
+    KeReleaseSpinlock(&gSwap.Lock, flags);
+    if (!valid) {
+        gSwap.Stats.PageInFailures++;
+        return STATUS_NOT_FOUND;
+    }
+
+    void* page = MmPfnToVirtual(frame);
+    if (!page) {
+        gSwap.Stats.PageInFailures++;
+        return STATUS_FAILURE;
+    }
+
+    size_t read = 0;
+    KSTATUS s = IoReadFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &read);
+    if (!KSUCCESS(s) || read != PAGE_SIZE) {
+        gSwap.Stats.PageInFailures++;
+        return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s;
+    }
+
+    gSwap.Stats.PageIns++;
+    return STATUS_SUCCESS;
+}
+
+// Releases a swap slot after the owning VAD page no longer references it.
+void MmSwapFreeSlot(uint32_t slot) {
+    uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
+    if (gSwap.Enabled && slot < gSwap.SlotCount && KiSwapTestSlot(slot)) {
+        KiSwapSetSlot(slot, 0);
+        if (gSwap.UsedSlots)
+            gSwap.UsedSlots--;
+        gSwap.Stats.SlotFrees++;
+    }
+    KeReleaseSpinlock(&gSwap.Lock, flags);
+}
+
+// Copies swap telemetry for user diagnostics.
+void MmSwapQueryStats(MM_SWAP_STATS* out) {
+    if (!out) return;
+    uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
+    MM_SWAP_STATS stats = gSwap.Stats;
+    stats.Enabled = gSwap.Enabled ? 1 : 0;
+    stats.TotalSlots = gSwap.SlotCount;
+    stats.UsedSlots = gSwap.UsedSlots;
+    stats.FreeSlots = gSwap.SlotCount >= gSwap.UsedSlots ? gSwap.SlotCount - gSwap.UsedSlots : 0;
+    stats.HighWaterSlots = gSwap.HighWaterSlots;
+    KeReleaseSpinlock(&gSwap.Lock, flags);
+    *out = stats;
+}

code/kernel/ob_object.c

@@ -5,11 +5,36 @@
 #include <kernel/ob_object.h>
 #include <kernel/mm_virt.h>
 
-// Creates object.
+// Returns a diagnostic pool tag for the object type.
+static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
+    switch (type) {
+    case ObjectTypeProcess:     return MM_POOL_TAG('p', 'r', 'o', 'c');
+    case ObjectTypeThread:      return MM_POOL_TAG('t', 'h', 'r', 'd');
+    case ObjectTypeMutex:       return MM_POOL_TAG('m', 't', 'x', ' ');
+    case ObjectTypeSemaphore:   return MM_POOL_TAG('s', 'e', 'm', ' ');
+    case ObjectTypeEvent:       return MM_POOL_TAG('e', 'v', 'n', 't');
+    case ObjectTypeQueue:       return MM_POOL_TAG('q', 'u', 'e', 'u');
+    case ObjectTypeFile:        return MM_POOL_TAG('f', 'i', 'l', 'e');
+    case ObjectTypeDir:         return MM_POOL_TAG('d', 'i', 'r', ' ');
+    case ObjectTypeAsyncIo:     return MM_POOL_TAG('a', 'i', 'o', ' ');
+    case ObjectTypeSocket:      return MM_POOL_TAG('s', 'o', 'c', 'k');
+    case ObjectTypeDevice:      return MM_POOL_TAG('d', 'e', 'v', ' ');
+    case ObjectTypeDriver:      return MM_POOL_TAG('d', 'r', 'v', 'r');
+    case ObjectTypeUsbDevice:   return MM_POOL_TAG('u', 's', 'b', 'd');
+    case ObjectTypeUsbPipe:     return MM_POOL_TAG('u', 's', 'b', 'p');
+    case ObjectTypeFileMapping: return MM_POOL_TAG('m', 'a', 'p', 'f');
+    case ObjectTypeSurface:     return MM_POOL_TAG('s', 'u', 'r', 'f');
+    case ObjectTypeInputQueue:  return MM_POOL_TAG('i', 'n', 'p', 'q');
+    case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
+    default:                    return MM_POOL_TAG('o', 'b', 'j', ' ');
+    }
+}
+
+// Creates a zeroed, tagged object with one initial reference.
 __declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) {
     if (size < sizeof(OBJECT)) size = sizeof(OBJECT);
 
-    OBJECT* obj = (OBJECT *)MmAllocatePool(size);
+    OBJECT* obj = (OBJECT *)MmAllocatePoolTag(size, KiObjectPoolTag(type));
     if (!obj) return (OBJECT *)0;
 
     memset(obj, 0, size);

code/public/ntlite/ddi/hal_api.h

@@ -120,7 +120,7 @@ typedef struct HAL_INIT_PARAMS {
     void*      (*AllocatePool)(size_t size);
 } HAL_INIT_PARAMS;
 
-#define HAL_API_VERSION  5
+#define HAL_API_VERSION // do not change during development
 
 __declspec(dllexport) const HAL_API* HalGetApi(void);
 __declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void));

code/public/ntlite/ddi/io_driver.h

@@ -179,6 +179,27 @@ typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex,
                                           DEVICE_OBJECT* device,
                                           void* context);
 
+typedef struct IO_DISK_STATS {
+    uint64_t RegisteredDisks;
+    uint64_t ReadRequests;
+    uint64_t ReadSectors;
+    uint64_t WriteRequests;
+    uint64_t WriteSectors;
+    uint64_t MaxReadSectors;
+    uint64_t MaxWriteSectors;
+    uint64_t ReadaheadHits;
+    uint64_t ReadaheadMisses;
+    uint64_t ReadaheadStores;
+    uint64_t ReadaheadInvalidations;
+    uint64_t ReadaheadBytesServed;
+    uint64_t ReadaheadSectorsLoaded;
+    uint64_t ReadaheadSequentialDetections;
+    uint64_t ReadaheadSuppressed;
+    uint64_t ReadaheadAllocFailures;
+    uint64_t ReadaheadReadFailures;
+    uint64_t MaxReadaheadSectors;
+} IO_DISK_STATS;
+
 __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
                                                    DEVICE_OBJECT* device);
 __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex);
@@ -203,3 +224,4 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
                                                  uint64_t lba,
                                                  uint32_t count,
                                                  const void* buffer);
+__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out);

code/public/ntlite/ddi/pci.h

@@ -118,3 +118,6 @@ PCI_API KSTATUS PciGetInterrupt(DEVICE_OBJECT* pdo,
 
 PCI_API KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo,
                                            uint8_t irqSlot);
+
+PCI_API KSTATUS PciEnableMsiAuto(DEVICE_OBJECT* pdo,
+                                               uint8_t* outIrqSlot);

code/public/ntlite/display.h

@@ -32,6 +32,43 @@ typedef struct DISPLAY_RECT {
     int32_t H;
 } DISPLAY_RECT;
 
+
+#define SYS_SURFACE_FORMAT_XRGB8888 1u
+#define SYS_SURFACE_FLAG_NONE       0u
+#define SYS_SURFACE_MAX_RECTS       64u
+
+typedef struct SYS_SURFACE_DESC {
+    uint32_t Width;
+    uint32_t Height;
+    uint32_t PitchBytes;
+    uint32_t Format;
+    void*    Base;
+    uint64_t PayloadBytes;
+    uint64_t PageCount;
+    uint64_t MapCount;
+    uint64_t PresentCount;
+    uint64_t PresentRectCount;
+    uint64_t PresentClippedRectCount;
+    uint64_t PresentEmptyClipCount;
+    uint64_t PresentRowsCopied;
+    uint64_t PresentBytesCopied;
+} SYS_SURFACE_DESC;
+
+typedef struct SYS_SURFACE_STATS {
+    uint64_t CreatedSurfaces;
+    uint64_t DestroyedSurfaces;
+    uint64_t LiveSurfaces;
+    uint64_t LivePages;
+    uint64_t LiveBytes;
+    uint64_t MapCalls;
+    uint64_t PresentCalls;
+    uint64_t PresentRectInputs;
+    uint64_t PresentClippedRects;
+    uint64_t PresentEmptyClips;
+    uint64_t PresentRowsCopied;
+    uint64_t PresentBytesCopied;
+} SYS_SURFACE_STATS;
+
 typedef struct DISPLAY_STATS {
     uint64_t BlitCount;
     uint64_t FillRectCount;

code/public/ntlite/input.h

@@ -136,6 +136,13 @@ typedef struct INPUT_KEY_EVENT {
     bool      Pressed;
 } INPUT_KEY_EVENT;
 
+typedef struct INPUT_KEY_RECORD {
+    INPUT_KEY Key;
+    char      Char;
+    uint8_t   Reserved;
+    uint32_t  Modifiers;
+} INPUT_KEY_RECORD;
+
 typedef struct INPUT_MOUSE_EVENT {
     int     dx;
     int     dy;

code/public/ntlite/status.h

@@ -45,4 +45,26 @@ typedef uint32_t KSTATUS;
 // --- Execution level (0x0060) ---.
 #define STATUS_INVALID_IRQL       ((KSTATUS)0x0060)
 
+// --- Process termination reasons ---.
+// Low values are reserved for ordinary KSTATUS returns. Abnormal process
+// termination uses a high-bit private range so exit statuses cannot collide
+// with normal kernel/API result codes.
+//
+// Layout:
+//   31        : set => abnormal/private status
+//   30..24    : subsystem group
+//   23..0     : group-local reason
+//
+// Group 0x61 is process termination/fault status.
+#define STATUS_PRIVATE_BASE               ((KSTATUS)0x80000000u)
+#define STATUS_PROCESS_BASE               ((KSTATUS)0xE1000000u)
+#define STATUS_PROCESS_KILLED             ((KSTATUS)(STATUS_PROCESS_BASE | 0x0000u))
+#define STATUS_PROCESS_PAGE_FAULT         ((KSTATUS)(STATUS_PROCESS_BASE | 0x0010u))
+#define STATUS_PROCESS_GENERAL_PROTECTION ((KSTATUS)(STATUS_PROCESS_BASE | 0x0011u))
+#define STATUS_PROCESS_ILLEGAL_OPCODE     ((KSTATUS)(STATUS_PROCESS_BASE | 0x0012u))
+#define STATUS_PROCESS_DIVIDE_BY_ZERO     ((KSTATUS)(STATUS_PROCESS_BASE | 0x0013u))
+#define STATUS_PROCESS_BREAKPOINT         ((KSTATUS)(STATUS_PROCESS_BASE | 0x0014u))
+#define STATUS_PROCESS_ALIGNMENT_FAULT    ((KSTATUS)(STATUS_PROCESS_BASE | 0x0015u))
+#define STATUS_PROCESS_UNKNOWN_FAULT      ((KSTATUS)(STATUS_PROCESS_BASE | 0x001Fu))
+
 #define KSUCCESS(s)  ((s) == STATUS_SUCCESS)

code/public/ntlite/syscall.h

@@ -18,7 +18,11 @@
 #define SYS_STATUS_PENDING       4u
 #define SYS_STATUS_ALERTED       5u
 #define SYS_STATUS_INVALID_PARAMETER 0x10u
+#define SYS_STATUS_INVALID_HANDLE 0x11u
+#define SYS_STATUS_BUFFER_TOO_SMALL 0x12u
+#define SYS_STATUS_NOT_SUPPORTED 0x02u
 #define SYS_STATUS_NO_MEMORY     0x20u
+#define SYS_STATUS_ACCESS_DENIED 0x21u
 #define SYS_STATUS_KERNEL_APC    6u
 #define SYS_STATUS_USER_APC      7u
 #define SYS_STATUS_CANCELLED     0x33u
@@ -30,6 +34,7 @@
 #define SYS_TEB_MAGIC   0x30424554u  // "TEB0"
 #define SYS_ENV_VERSION 1u
 #define SYS_CWD_MAX     520u
+#define SYS_PATH_MAX    1024u
 
 #ifndef SYS_USER_ENV_TYPES_DEFINED
 #define SYS_USER_ENV_TYPES_DEFINED 1
@@ -52,9 +57,7 @@ typedef struct SYS_TEB {
     uint32_t        ThreadId;
     uint32_t        Magic;
 
-    // Native per-thread extension pointer. The kernel leaves this NULL;
+    // =The kernel leaves this NULL, userspace can modify and use.
-    // user-mode personalities/libraries may hang their own thread-local
-    // state here without baking subsystem semantics into SYS_TEB.
     void*           SubsystemData;
 } SYS_TEB;
 
@@ -80,6 +83,27 @@ typedef void (*SYS_USER_APC_ROUTINE)(void* normalContext,
                                      SYS_IO_STATUS_BLOCK* iosb,
                                      uint64_t information);
 
+typedef struct SYS_FILE_IO_REQUEST {
+    uint32_t FileHandle;
+    uint32_t Flags;
+    void*    Buffer;
+    size_t   Size;
+    size_t*  TransferredOut;
+} SYS_FILE_IO_REQUEST;
+
+static inline SYS_FILE_IO_REQUEST SysMakeFileIoRequest(uint32_t fileHandle,
+                                                       const void* buffer,
+                                                       size_t size,
+                                                       size_t* transferredOut) {
+    SYS_FILE_IO_REQUEST request;
+    request.FileHandle = fileHandle;
+    request.Flags = 0;
+    request.Buffer = (void*)(uintptr_t)buffer;
+    request.Size = size;
+    request.TransferredOut = transferredOut;
+    return request;
+}
+
 typedef struct SYS_ASYNC_FILE_REQUEST {
     uint32_t FileHandle;
     uint32_t EventHandle;       // optional; 0 means no event signal
@@ -181,11 +205,40 @@ typedef struct SYS_NAMED_QUEUE_OWNER_REQUEST {
     uint32_t*       PidOut;
 } SYS_NAMED_QUEUE_OWNER_REQUEST;
 
+#define SYS_WAIT_FLAG_NONE      0x00000000u
+#define SYS_WAIT_FLAG_ALERTABLE 0x00000001u
+#define SYS_WAIT_TIMEOUT_INFINITE UINT64_MAX
+#define SYS_WAIT_MANY_MAX_HANDLES 32u
+
+typedef struct SYS_WAIT_MANY_REQUEST {
+    uint32_t Size;
+    uint32_t Flags;
+    const uint32_t* Handles;
+    uint32_t Count;
+    uint32_t* SignaledIndexOut;
+    uint64_t TimeoutMilliseconds;
+} SYS_WAIT_MANY_REQUEST;
+
+typedef struct SYS_CREATE_THREAD_REQUEST {
+    uintptr_t Entry;
+    uintptr_t Argument;
+    uint32_t  Priority;
+    uint32_t  Flags;
+    uint32_t* ThreadHandleOut;
+    uint32_t* ThreadIdOut;
+} SYS_CREATE_THREAD_REQUEST;
+
 // Process & thread lifecycle (0-14).
 void     SysExitProcess(int exitCode);
 void     SysExitThread(int exitCode);
 uint32_t SysCreateProcess(const SYS_CREATE_PROCESS_REQUEST* request);
-uint32_t SysWaitForProcess(uint32_t handle);
+uint32_t SysWaitSingle(uint32_t handle, uint32_t flags, uint64_t timeoutMilliseconds);
+uint32_t SysWaitMany(const SYS_WAIT_MANY_REQUEST* request);
+uint32_t SysOpenInputQueue(uint32_t flags, uint32_t* handleOut);
+
+static inline uint32_t SysWait(uint32_t handle, uint32_t flags) {
+    return SysWaitSingle(handle, flags, SYS_WAIT_TIMEOUT_INFINITE);
+}
 uint32_t SysGetProcessExitCode(uint32_t handle, int* exitCodeOut);
 uint32_t SysKillProcess(uint32_t pid, int exitCode);
 
@@ -200,6 +253,61 @@ uint32_t SysKillProcess(uint32_t pid, int exitCode);
 #define SYS_PROCESS_FLAG_CURRENT     0x00000002u
 #define SYS_PROCESS_FLAG_TERMINATING 0x00000004u
 #define SYS_PROCESS_FLAG_EXITED      0x00000008u
+#define SYS_PROCESS_FLAG_IDLE        0x00000010u
+
+#define SYS_THREAD_STATE_READY       0u
+#define SYS_THREAD_STATE_RUNNING     1u
+#define SYS_THREAD_STATE_WAITING     2u
+#define SYS_THREAD_STATE_DEAD        3u
+
+#define SYS_THREAD_FLAG_CURRENT            0x00000001u
+#define SYS_THREAD_FLAG_IDLE               0x00000002u
+#define SYS_THREAD_FLAG_KERNEL             0x00000004u
+#define SYS_THREAD_FLAG_QUEUED             0x00000008u
+#define SYS_THREAD_FLAG_ALERTABLE_WAIT     0x00000010u
+#define SYS_THREAD_FLAG_KERNEL_APC_PENDING 0x00000020u
+#define SYS_THREAD_FLAG_USER_APC_PENDING   0x00000040u
+
+#define SYS_THREAD_WAIT_NONE         0u
+#define SYS_THREAD_WAIT_PROCESS      1u
+#define SYS_THREAD_WAIT_THREAD       2u
+#define SYS_THREAD_WAIT_MUTEX        3u
+#define SYS_THREAD_WAIT_SEMAPHORE    4u
+#define SYS_THREAD_WAIT_EVENT        5u
+#define SYS_THREAD_WAIT_QUEUE        6u
+#define SYS_THREAD_WAIT_FILE         7u
+#define SYS_THREAD_WAIT_DIR          8u
+#define SYS_THREAD_WAIT_ASYNC_IO     9u
+#define SYS_THREAD_WAIT_SOCKET       10u
+#define SYS_THREAD_WAIT_DEVICE       11u
+#define SYS_THREAD_WAIT_DRIVER       12u
+#define SYS_THREAD_WAIT_USB_DEVICE   13u
+#define SYS_THREAD_WAIT_USB_PIPE     14u
+#define SYS_THREAD_WAIT_FILE_MAPPING 15u
+#define SYS_THREAD_WAIT_SURFACE      16u
+#define SYS_THREAD_WAIT_UNKNOWN      0xffffffffu
+
+typedef struct SYS_THREAD_INFO {
+    uint32_t Size;
+    uint32_t ProcessId;
+    uint32_t ThreadId;
+    uint32_t State;
+
+    uint32_t Priority;
+    uint32_t Flags;
+    uint32_t RunningCpu;
+    uint32_t LastCpu;
+
+    uint32_t QuantumRemaining;
+    uint32_t QuantumTicks;
+    uint32_t CpuBurstTicks;
+    uint32_t LastRunTicks;
+
+    uint32_t WaitObjectType;
+    uint32_t WaitStatus;
+    int32_t  KernelApcDisable;
+    uint32_t Reserved0;
+} SYS_THREAD_INFO;
 
 typedef struct SYS_PROCESS_INFO {
     uint32_t Size;
@@ -219,7 +327,7 @@ typedef struct SYS_PROCESS_INFO {
     size_t    ImageSize;
     uintptr_t PebBase;
 
-    size_t UserRegionCount;
+    size_t VadCount;
     size_t UserReservedBytes;
     size_t UserCommittedBytes;
 
@@ -229,9 +337,9 @@ typedef struct SYS_PROCESS_INFO {
 
 uint32_t SysGetProcessList(SYS_PROCESS_INFO* processesOut, size_t maxCount, size_t* countOut);
 uint32_t SysGetProcessPid(uint32_t handle, uint32_t* pidOut);
+uint32_t SysGetThreadList(uint32_t pid, SYS_THREAD_INFO* threadsOut, size_t maxCount, size_t* countOut);
 
-uint32_t SysCreateThread(uintptr_t entry, uintptr_t arg, uint32_t priority, uint32_t* tidOut);
+uint32_t SysCreateThread(const SYS_CREATE_THREAD_REQUEST* request);
-uint32_t SysWaitThread(uint32_t tid);
 uint32_t SysGetCurrentTid(void);
 uint32_t SysGetCurrentPid(void);
 uint32_t SysSetThreadPriority(uint32_t priority);
@@ -266,23 +374,32 @@ uint32_t SysAllocVm(void* preferredBase, size_t pageCount, uint32_t protect, uin
 uint32_t SysFreeVm(void* base);
 uint32_t SysProtectVm(void* base, size_t pageCount, uint32_t protect, uint32_t* oldProtectOut);
 uint32_t SysQueryVm(void* address, SYS_VM_REGION_INFO* out);
+
+#define SYS_MAP_ACCESS_READ     0x00000001u
+#define SYS_MAP_ACCESS_WRITE    0x00000002u
+#define SYS_MAP_ACCESS_COPY     0x00000004u
+#define SYS_MAP_ACCESS_EXECUTE  0x00000008u
+#define SYS_MAP_FLAG_NONE       0x00000000u
+
+uint32_t SysCreateFileMapping(uint32_t fileHandle, uint32_t protect, uint32_t flags,
+                              uint64_t maximumSize, uint32_t* mappingHandleOut);
+uint32_t SysMapFile(uint32_t mappingHandle, uint64_t offset, size_t size,
+                    uint32_t access, uint32_t flags, void** baseOut);
 uint32_t SysCloseHandle(uint32_t handle);
 
 // Synchronization primitives (19-31).
 uint32_t SysCreateMutex(uint32_t* handleOut);
-uint32_t SysAcquireMutex(uint32_t handle);
 uint32_t SysTryAcquireMutex(uint32_t handle);
 uint32_t SysReleaseMutex(uint32_t handle);
 
 uint32_t SysCreateSem(int32_t initial, int32_t max, uint32_t* handleOut);
-uint32_t SysWaitSem(uint32_t handle);
 uint32_t SysTryWaitSem(uint32_t handle);
 uint32_t SysSignalSem(uint32_t handle);
 
 uint32_t SysCreateEvent(uint32_t type, uint32_t initialState, uint32_t* handleOut);
 uint32_t SysSignalEvent(uint32_t handle);
 uint32_t SysClearEvent(uint32_t handle);
-uint32_t SysWaitEvent(uint32_t handle, uint32_t alertable);
+
 
 // Queues and named queues (32-38).
 uint32_t SysCreateQueue(uint32_t* handleOut);
@@ -296,8 +413,8 @@ uint32_t SysGetNamedQueueOwnerPid(const SYS_NAMED_QUEUE_OWNER_REQUEST* request);
 // Files, directories, and current directory (39-56).
 uint32_t SysOpenFile(const SYS_PATH_HANDLE_REQUEST* request);
 uint32_t SysCreateFile(const SYS_PATH_HANDLE_REQUEST* request);
-uint32_t SysReadFile(uint32_t handle, void* buf, size_t size, size_t* bytesRead);
+uint32_t SysReadFile(const SYS_FILE_IO_REQUEST* request);
-uint32_t SysWriteFile(uint32_t handle, const void* buf, size_t size, size_t* written);
+uint32_t SysWriteFile(const SYS_FILE_IO_REQUEST* request);
 uint32_t SysReadFileAsync(const SYS_ASYNC_FILE_REQUEST* request);
 uint32_t SysWriteFileAsync(const SYS_ASYNC_FILE_REQUEST* request);
 uint32_t SysCancelIo(uint32_t asyncRequestHandle);
@@ -305,6 +422,7 @@ uint32_t SysSeekFile(uint32_t handle, uint64_t offset);
 uint64_t SysGetFilePos(uint32_t handle);
 uint64_t SysGetFileSize(uint32_t handle);
 uint32_t SysTruncateFile(uint32_t handle, uint64_t newSize);
+uint32_t SysFlushFile(uint32_t handle);
 uint32_t SysDeleteFile(const SYS_PATH_REQUEST* request);
 uint32_t SysRenameFile(const SYS_RENAME_FILE_REQUEST* request);
 
@@ -330,7 +448,10 @@ uint32_t SysSetConsoleCursor(int x, int y);
 uint32_t SysFillConsoleCharacter(char c, int x, int y, size_t length, size_t* writtenOut);
 uint32_t SysFillConsoleAttribute(uint16_t attr, int x, int y, size_t length, size_t* writtenOut);
 uint32_t SysPutScreenChar(char c, int x, int y, uint32_t fg, uint32_t bg);
-uint32_t SysBlitScreen(int x, int y, int w, int h, const uint32_t* pixels);
+uint32_t SysCreateSurface(uint32_t width, uint32_t height, uint32_t format, uint32_t flags, uint32_t* handleOut);
+uint32_t SysMapSurface(uint32_t surfaceHandle, void** baseOut, uint32_t* pitchBytesOut);
+uint32_t SysQuerySurface(uint32_t surfaceHandle, SYS_SURFACE_DESC* descOut);
+uint32_t SysPresentSurface(uint32_t surfaceHandle, const DISPLAY_RECT* rects, uint32_t rectCount);
 uint32_t SysClearScreen(uint32_t color);
 uint32_t SysGetScreenSize(int* widthOut, int* heightOut);
 uint32_t SysQueryDisplayInfo(SYS_DISPLAY_INFO* infoOut);
@@ -341,6 +462,8 @@ uint64_t SysIsKeyDown(INPUT_KEY key);
 char     SysReadKeyboardChar(void);
 INPUT_KEY SysReadKey(void);
 uint32_t SysGetKeyModifiers(void);
+uint32_t SysReadInputEvent(INPUT_KEY_RECORD* recordOut);
+uint32_t SysPeekInputEvent(INPUT_KEY_RECORD* recordOut);
 
 uint64_t SysPollMouse(void);
 uint64_t SysReadMouse(int* dx, int* dy, uint8_t* buttons);
@@ -437,11 +560,25 @@ typedef struct SYS_MEMORY_STATUS {
     uint64_t FreePhysicalBytes;
     uint64_t KernelPoolBytes;
     uint64_t UserHeapBytes;
+    uint64_t ReclaimablePhysicalBytes;
+    uint64_t AvailablePhysicalBytes;
+    uint64_t PinnedPhysicalBytes;
+    uint64_t NonEvictablePhysicalBytes;
+    uint64_t UserPhysicalBytes;
+    uint64_t PageTablePhysicalBytes;
 } SYS_MEMORY_STATUS;
 
 #define SYS_KERNEL_STATS_SCHEDULER 1u
 #define SYS_KERNEL_STATS_PMM       2u
 #define SYS_KERNEL_STATS_CPU       3u
+#define SYS_KERNEL_STATS_PFN       4u
+#define SYS_KERNEL_STATS_PTE       5u
+#define SYS_KERNEL_STATS_VAD       6u
+#define SYS_KERNEL_STATS_SWAP      7u
+#define SYS_KERNEL_STATS_FILE_CACHE 8u
+#define SYS_KERNEL_STATS_SURFACE    9u
+#define SYS_KERNEL_STATS_FILESYSTEM 10u
+#define SYS_KERNEL_STATS_DISK      11u
 
 #define SYS_KERNEL_STATS_MAX_CPUS  32u
 
@@ -469,11 +606,15 @@ typedef struct SYS_PMM_STATS {
     uint64_t FreeRunHistogram[8];
 } SYS_PMM_STATS;
 
+#define SYS_CPU_THREAD_FLAG_IDLE 0x00000001u
+
 typedef struct SYS_CPU_STAT {
     uint32_t CpuIndex;
     uint32_t LapicId;
     uint32_t Flags;
     uint32_t CurrentTid;
+    uint32_t CurrentThreadFlags;
+    uint32_t Reserved0;
     uint64_t SchedulerTicks;
     uint64_t ContextSwitches;
 } SYS_CPU_STAT;
@@ -484,6 +625,228 @@ typedef struct SYS_CPU_STATS {
     SYS_CPU_STAT Cpu[SYS_KERNEL_STATS_MAX_CPUS];
 } SYS_CPU_STATS;
 
+typedef struct SYS_PFN_STATS {
+    uint64_t TotalFrames;
+    uint64_t FreeFrames;
+    uint64_t UsedFrames;
+
+    uint64_t ReservedFrames;
+    uint64_t ZeroedFrames;
+    uint64_t ActiveFrames;
+    uint64_t StandbyFrames;
+    uint64_t ModifiedFrames;
+    uint64_t PinnedFrames;
+    uint64_t EvictableFrames;
+    uint64_t NonEvictableFrames;
+    uint64_t ReclaimableFrames;
+    uint64_t AvailableFrames;
+    uint64_t MappedFrames;
+
+    uint64_t KernelFrames;
+    uint64_t PageTableFrames;
+    uint64_t UserFrames;
+    uint64_t CleanFileCacheFrames;
+    uint64_t ActiveCleanFileCacheFrames;
+    uint64_t StandbyCleanFileCacheFrames;
+    uint64_t ImageFrames;
+    uint64_t ActiveImageFrames;
+    uint64_t StandbyImageFrames;
+    uint64_t MappedFileFrames;
+    uint64_t AnonymousFrames;
+    uint64_t CowFrames;
+    uint64_t SurfaceFrames;
+    uint64_t DriverFrames;
+    uint64_t DmaFrames;
+    uint64_t BootloaderFrames;
+    uint64_t PfnDatabaseFrames;
+    uint64_t MmioFrames;
+
+    uint64_t AllocationCount;
+    uint64_t FreeCount;
+    uint64_t RetainCount;
+    uint64_t ReleaseCount;
+    uint64_t PinCount;
+    uint64_t UnpinCount;
+    uint64_t FailedAllocations;
+    uint64_t FailedContiguousAllocations;
+
+    uint64_t FailedInvalidRange;
+    uint64_t FailedEmptyRange;
+    uint64_t FailedReserveDenied;
+    uint64_t FailedNoFrame;
+
+    uint64_t FailedContiguousInvalidRequest;
+    uint64_t FailedContiguousReserveDenied;
+    uint64_t FailedContiguousNoRun;
+
+    uint64_t ReclaimAttempts;
+    uint64_t ReclaimPagesRequested;
+    uint64_t ReclaimPagesSucceeded;
+    uint64_t ReclaimPagesFailed;
+    uint64_t ReclaimReentrantSkips;
+} SYS_PFN_STATS;
+
+typedef struct SYS_VAD_STATS {
+    uint64_t ResolveFaultCalls;
+    uint64_t ResolvedFaults;
+    uint64_t FailedFaults;
+    uint64_t ProtectionFaults;
+    uint64_t NoVadFaults;
+    uint64_t UncommittedFaults;
+    uint64_t NoAccessFaults;
+    uint64_t DemandZeroPages;
+    uint64_t ZeroPageMaps;
+    uint64_t ZeroPageCowSplits;
+    uint64_t CowFaults;
+    uint64_t CowPagesCopied;
+    uint64_t CowViewsCreated;
+    uint64_t CowPagesShared;
+    uint64_t EnsureResidentCalls;
+    uint64_t EnsureResidentPages;
+    uint64_t AlreadyResidentPages;
+    uint64_t FileBackedFaults;
+    uint64_t FileBackedReads;
+    uint64_t FileBackedReadFailures;
+    uint64_t FileBackedDiscards;
+    uint64_t FileBackedSharedWritableFaults;
+    uint64_t FileBackedDirtyFaults;
+    uint64_t FileBackedDirtySyncs;
+    uint64_t ImageBackedFaults;
+    uint64_t ImageBackedZeroFaults;
+    uint64_t ImageBackedPrivateFaults;
+    uint64_t ImageFixupPrivateCalls;
+    uint64_t ImageFixupPrivatePages;
+    uint64_t SwapPageOuts;
+    uint64_t SwapPageIns;
+} SYS_VAD_STATS;
+
+
+typedef struct SYS_FILE_CACHE_STATS {
+    uint64_t Lookups;
+    uint64_t Hits;
+    uint64_t Misses;
+    uint64_t Loads;
+    uint64_t LoadFailures;
+    uint64_t RaceDiscards;
+    uint64_t InflightWaits;
+    uint64_t Invalidations;
+    uint64_t DirtyPages;
+    uint64_t MaxDirtyPages;
+    uint64_t DirtyMarks;
+    uint64_t WritebackRequests;
+    uint64_t WritebackPages;
+    uint64_t WritebackBytes;
+    uint64_t WritebackFailures;
+    uint64_t FlushFileCalls;
+    uint64_t FlushFileFailures;
+    uint64_t MappedDirtySyncs;
+    uint64_t MappedDirtyPages;
+    uint64_t Drops;
+    uint64_t DropFailures;
+    uint64_t StandbyTransitions;
+    uint64_t StandbyHits;
+    uint64_t ReadAheadRequests;
+    uint64_t ReadAheadPagesRequested;
+    uint64_t ReadAheadPagesQueued;
+    uint64_t ReadAheadPagesLoaded;
+    uint64_t ReadAheadPagesSkipped;
+    uint64_t ReadAheadFailures;
+    uint64_t ReadAheadPagesThrottled;
+    uint64_t MaxReadAheadPagesPerRequest;
+    uint64_t BatchReads;
+    uint64_t BatchPagesRequested;
+    uint64_t BatchPagesLoaded;
+    uint64_t MaxBatchPagesRequested;
+    uint64_t MaxBatchPagesLoaded;
+    uint64_t ResidentPages;
+    uint64_t ActivePages;
+    uint64_t StandbyPages;
+    uint64_t MaxResidentPages;
+} SYS_FILE_CACHE_STATS;
+
+typedef struct SYS_FILESYSTEM_STATS {
+    uint64_t PathLookups;
+    uint64_t PathCacheHits;
+    uint64_t PathCacheMisses;
+    uint64_t PathCacheStores;
+    uint64_t PathCacheInvalidations;
+    uint64_t PathCacheGeneration;
+    uint64_t DirectorySearches;
+    uint64_t DirectoryClustersRead;
+    uint64_t DirectoryEntriesScanned;
+    uint64_t FatChainCacheLookups;
+    uint64_t FatChainCacheHits;
+    uint64_t FatChainCacheMisses;
+    uint64_t FatChainCacheStores;
+    uint64_t FatChainCacheInvalidations;
+    uint64_t FatChainWalkSteps;
+    uint64_t DataReadRuns;
+    uint64_t DataReadSectors;
+    uint64_t DataReadRunMerges;
+    uint64_t MaxDataReadSpanSectors;
+} SYS_FILESYSTEM_STATS;
+
+typedef struct SYS_DISK_STATS {
+    uint64_t RegisteredDisks;
+    uint64_t ReadRequests;
+    uint64_t ReadSectors;
+    uint64_t WriteRequests;
+    uint64_t WriteSectors;
+    uint64_t MaxReadSectors;
+    uint64_t MaxWriteSectors;
+    uint64_t ReadaheadHits;
+    uint64_t ReadaheadMisses;
+    uint64_t ReadaheadStores;
+    uint64_t ReadaheadInvalidations;
+    uint64_t ReadaheadBytesServed;
+    uint64_t ReadaheadSectorsLoaded;
+    uint64_t ReadaheadSequentialDetections;
+    uint64_t ReadaheadSuppressed;
+    uint64_t ReadaheadAllocFailures;
+    uint64_t ReadaheadReadFailures;
+    uint64_t MaxReadaheadSectors;
+} SYS_DISK_STATS;
+
+typedef struct SYS_SWAP_STATS {
+    uint64_t Enabled;
+    uint64_t TotalSlots;
+    uint64_t UsedSlots;
+    uint64_t FreeSlots;
+    uint64_t HighWaterSlots;
+    uint64_t PageOuts;
+    uint64_t PageIns;
+    uint64_t PageOutFailures;
+    uint64_t PageInFailures;
+    uint64_t SlotAllocations;
+    uint64_t SlotFrees;
+    uint64_t SlotAllocationFailures;
+} SYS_SWAP_STATS;
+
+typedef struct SYS_PTE_STATS {
+    uint64_t MapCalls;
+    uint64_t MapFailures;
+    uint64_t UnmapCalls;
+    uint64_t UnmapFailures;
+    uint64_t ProtectCalls;
+    uint64_t ProtectFailures;
+    uint64_t QueryCalls;
+    uint64_t QueryFailures;
+    uint64_t QueryByPurpose[6];
+    uint64_t QueryFailuresByPurpose[6];
+    uint64_t TranslateCalls;
+    uint64_t LocalFlushes;
+    uint64_t GlobalFlushes;
+    uint64_t ShootdownRequests;
+    uint64_t ShootdownTargetCpus;
+    uint64_t ShootdownSkippedNoTargets;
+    uint64_t ShootdownAcks;
+    uint64_t ShootdownTimeouts;
+    uint64_t NewMappingNoFlushes;
+    uint64_t ReplacedMappingFlushes;
+    uint64_t PageTablePagesAllocated;
+    uint64_t CurrentLeafMappings;
+} SYS_PTE_STATS;
+
 #define SYS_CPU_FLAG_PRESENT  0x00000001u
 #define SYS_CPU_FLAG_STARTED  0x00000002u
 #define SYS_CPU_FLAG_PARKED   0x00000004u

code/settings.ini

@@ -24,3 +24,7 @@ daylight_saving=auto
 hardware_clock=utc
 ; Used when timezone is a custom/fixed policy; minutes east of UTC.
 standard_offset_minutes=60
+
+[vm]
+; 0 disables swap. Nonzero swap must be between 4 MB and 8192 MB.
+swap_size_mb=32

code/user/apps/doom/doom.c

@@ -88,7 +88,8 @@ static int doom_read_wrapper(void* handle, void* buf, int count)
     if (!handle || count <= 0) return 0;
     DOOM_FILE_HANDLE* f = (DOOM_FILE_HANDLE*)handle;
     size_t read = 0;
-    SysReadFile(f->Handle, buf, (size_t)count, &read);
+    SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(f->Handle, buf, (size_t)count, &read);
+    SysReadFile(&req);
     f->Pos += read;
     if (f->Pos > f->Size) f->Size = f->Pos;
     return (int)read;
@@ -99,7 +100,8 @@ static int doom_write_wrapper(void* handle, const void* buf, int count)
     if (!handle || count <= 0) return 0;
     DOOM_FILE_HANDLE* f = (DOOM_FILE_HANDLE*)handle;
     size_t written = 0;
-    SysWriteFile(f->Handle, buf, (size_t)count, &written);
+    SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(f->Handle, buf, (size_t)count, &written);
+    SysWriteFile(&req);
     f->Pos += written;
     if (f->Pos > f->Size) f->Size = f->Pos;
     return (int)written;
@@ -262,7 +264,7 @@ static void poll_input(void)
 }
 
 // ---------------------------------------------------------------------------
-// Render — Performs lightweight software upscale and blits to the screen
+// Render — writes directly into a mapped kernel surface and presents it.
 // ---------------------------------------------------------------------------
 #define DOOM_W  320
 #define DOOM_H  200
@@ -270,32 +272,65 @@ static void poll_input(void)
 #define DISP_W  (DOOM_W * SCALE)
 #define DISP_H  (DOOM_H * SCALE)
 
-static uint32_t g_blit_buf[DISP_W * DISP_H]; 
+static uint32_t  g_surface_handle;
+static uint32_t* g_surface_pixels;
+static uint32_t  g_surface_pitch_pixels;
+
+static void init_surface(void)
+{
+    uint32_t s = SysCreateSurface(DISP_W, DISP_H, SYS_SURFACE_FORMAT_XRGB8888,
+                                  SYS_SURFACE_FLAG_NONE, &g_surface_handle);
+    if (s != SYS_STATUS_SUCCESS || !g_surface_handle) {
+        doom_print_wrapper("doom: SysCreateSurface failed\n");
+        SysExitProcess(1);
+    }
+
+    void* base = 0;
+    uint32_t pitchBytes = 0;
+    s = SysMapSurface(g_surface_handle, &base, &pitchBytes);
+    if (s != SYS_STATUS_SUCCESS || !base || pitchBytes < DISP_W * sizeof(uint32_t)) {
+        doom_print_wrapper("doom: SysMapSurface failed\n");
+        SysExitProcess(1);
+    }
+
+    g_surface_pixels = (uint32_t*)base;
+    g_surface_pitch_pixels = pitchBytes / sizeof(uint32_t);
+}
 
 static void render_frame(const uint8_t* rgba)
 {
     const uint32_t* src = (const uint32_t*)rgba;
+    uint32_t* dstBase = g_surface_pixels;
+    uint32_t dstPitch = g_surface_pitch_pixels;
+
+    if (!dstBase || dstPitch < DISP_W) {
+        doom_print_wrapper("doom: surface not initialized\n");
+        SysExitProcess(1);
+    }
 
-    // Software upscale loop mapping 320x200 buffer directly to scaled window buffer
     for (int y = 0; y < DOOM_H; y++) {
         for (int x = 0; x < DOOM_W; x++) {
             uint32_t p = src[y * DOOM_W + x];
 
-            // RGBA -> BGRA conversion (standard for linear x86/64 framebuffers)
+            // RGBA -> BGRA/XRGB8888 conversion for the current display path.
             uint32_t bgra = (p & 0xFF00FF00u)
                           | ((p & 0x00FF0000u) >> 16)
                           | ((p & 0x000000FFu) << 16);
 
-            // Populate SCALE * SCALE pixel block
+            uint32_t* block = dstBase + (size_t)(y * SCALE) * dstPitch + (size_t)(x * SCALE);
             for (int sy = 0; sy < SCALE; sy++) {
                 for (int sx = 0; sx < SCALE; sx++) {
-                    g_blit_buf[(y * SCALE + sy) * DISP_W + (x * SCALE + sx)] = bgra;
+                    block[(size_t)sy * dstPitch + sx] = bgra;
                 }
             }
         }
     }
 
-    SysBlitScreen(0, 0, DISP_W, DISP_H, g_blit_buf);
+    DISPLAY_RECT rect = { 0, 0, DISP_W, DISP_H };
+    uint32_t code = SysPresentSurface(g_surface_handle, &rect, 1);
+    if (code != SYS_STATUS_SUCCESS) {
+        SysExitProcess(code);
+    }
 }
 
 static int build_argv_from_command_line(const char* cmdline, char* storage, size_t storageSize,
@@ -373,6 +408,7 @@ int main(void)
     char argvStorage[512];
     const char* argv[32];
     int argc = build_argv_from_command_line(cmdline, argvStorage, sizeof(argvStorage), argv, 32);
+    init_surface();
     doom_init(argc, argv, 0);
 
     do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: entering game loop\n") }; SysWriteConsole(&req); } while (0);

code/user/apps/edit/edit.c

@@ -380,12 +380,14 @@ static void save_file(void) {
     for (int i = 0; i < gLineCount; i++) {
         size_t written = 0;
         if (gLines[i].len) {
-            st = SysWriteFile(h, gLines[i].text, (size_t)gLines[i].len, &written);
+            SYS_FILE_IO_REQUEST writeReq = SysMakeFileIoRequest(h, gLines[i].text, (size_t)gLines[i].len, &written);
+            st = SysWriteFile(&writeReq);
             if (st != 0) { SysCloseHandle(h); msg_set("save failed: write error"); return; }
         }
         if (i + 1 < gLineCount) {
             const char nl = '\n';
-            st = SysWriteFile(h, &nl, 1, &written);
+            SYS_FILE_IO_REQUEST newlineReq = SysMakeFileIoRequest(h, &nl, 1, &written);
+            st = SysWriteFile(&newlineReq);
             if (st != 0) { SysCloseHandle(h); msg_set("save failed: write error"); return; }
         }
     }
@@ -418,7 +420,9 @@ static void load_file(void) {
     char buf[512];
     size_t got = 0;
     bool truncated = false;
-    while (SysReadFile(h, buf, sizeof(buf), &got) == 0 && got > 0) {
+    for (;;) {
+        SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(h, buf, sizeof(buf), &got);
+        if (SysReadFile(&readReq) != 0 || got == 0) break;
         for (size_t i = 0; i < got; i++) {
             char c = buf[i];
             if (c == '\r') continue;

code/user/apps/memtop/memtop.c

@@ -0,0 +1,71 @@
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <string.h>
+#include <ntlite/syscall.h>
+
+static int u64_to_string(uint64_t n, char* buf) {
+    int i = 0;
+    if (n == 0) {
+        buf[i++] = '0';
+        buf[i] = '\0';
+        return i;
+    }
+    char rev[24];
+    int r_idx = 0;
+    while (n > 0) {
+        rev[r_idx++] = (char)('0' + (n % 10));
+        n /= 10;
+    }
+    while (r_idx > 0) {
+        buf[i++] = rev[--r_idx];
+    }
+    buf[i] = '\0';
+    return i;
+}
+
+int main(void) {
+    SYS_CONSOLE_INFO console;
+    SYS_MEMORY_STATUS mem;
+    char metricsBuf[128];
+
+    while (true) {
+        if (SysGetConsoleInfo(&console) == SYS_STATUS_SUCCESS &&
+            SysGetMemoryStatus(&mem) == SYS_STATUS_SUCCESS) {
+
+            // If the system isn't reporting columns correctly, default to a safe standard width
+            int columns = (console.Columns > 0) ? (int)console.Columns : 80;
+
+            char freeStr[24], poolStr[24], heapStr[24];
+            u64_to_string(mem.FreePhysicalBytes / 1024u, freeStr);
+            u64_to_string(mem.KernelPoolBytes / 1024u, poolStr);
+            u64_to_string(mem.UserHeapBytes / 1024u, heapStr);
+
+            metricsBuf[0] = '\0';
+            strcat(metricsBuf, "[ Free: ");
+            strcat(metricsBuf, freeStr);
+            strcat(metricsBuf, "K | Pool: ");
+            strcat(metricsBuf, poolStr);
+            strcat(metricsBuf, "K | Heap: ");
+            strcat(metricsBuf, heapStr);
+            strcat(metricsBuf, "K ]");
+
+            size_t textLen = strlen(metricsBuf);
+            int targetX = columns - (int)textLen;
+            int targetY = 0; 
+
+            if (targetX >= 0) {
+                size_t written = 0;
+                
+                // Loop through the string and use the console's dedicated text filling API
+                // This utilizes the console's native font configuration instead of drawing raw screen bits
+                for (size_t i = 0; i < textLen; i++) {
+                    SysFillConsoleCharacter(metricsBuf[i], targetX + (int)i, targetY, 1, &written);
+                }
+            }
+        }
+
+        SysSleep(500);
+    }
+    return 0;
+}

code/user/apps/shell_win32/shell_win32.c

@@ -147,6 +147,8 @@ static int runCommand(const char* cmd) {
         WaitForSingleObject(pi.hProcess, INFINITE);
         if (pi.hThread && pi.hThread != INVALID_HANDLE_VALUE)
             CloseHandle(pi.hThread);
+        if (pi.hProcess && pi.hProcess != INVALID_HANDLE_VALUE)
+            CloseHandle(pi.hProcess);
         FlushConsoleInputBuffer(hIn);
     } else {
         print("shell: launch failed\n");

code/user/apps/video/video.c

@@ -1,110 +1,231 @@
-// User application entry points for video.
+// Native NTLite Bink video player.
+//
+// This application intentionally uses the native syscall ABI directly.  Do not
+// include <windows.h> here; Win32 compatibility belongs in the Win32 personality
+// layer, not in native apps.
 
-#include <windows.h>
 #include <stdint.h>
 #include <stddef.h>
 #include <string.h>
 #include <intrin.h>
+#include <ntlite/syscall.h>
 #include "bink.h"
 
-// Implements the command Line Second Token operation.
+#define VIDEO_DEFAULT_PATH "video.bk2"
-static const char* commandLineSecondToken(const char* cmdline, char* out, size_t outSize) {
-    if (!out || outSize == 0) return "video.bk2";
-    strcpy(out, "video.bk2");
-    if (!cmdline) return out;
 
-    while (*cmdline == ' ' || *cmdline == '\t') cmdline++;
+static void print_span(const char* s, size_t len) {
-    if (*cmdline == '\0') return out;
+    if (!s || !len) return;
-
+    SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(s, len) };
-    if (*cmdline == '"') {
+    SysWriteConsole(&req);
-        cmdline++;
-        while (*cmdline && *cmdline != '"') cmdline++;
-        if (*cmdline == '"') cmdline++;
-    } else {
-        while (*cmdline && *cmdline != ' ' && *cmdline != '\t') cmdline++;
 }
 
-    while (*cmdline == ' ' || *cmdline == '\t') cmdline++;
+static void print(const char* s) {
-    if (*cmdline == '\0') return out;
+    if (!s) return;
-
+    SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) };
-    size_t pos = 0;
+    SysWriteConsole(&req);
-    if (*cmdline == '"') {
-        cmdline++;
-        while (*cmdline && *cmdline != '"' && pos + 1 < outSize)
-            out[pos++] = *cmdline++;
-    } else {
-        while (*cmdline && *cmdline != ' ' && *cmdline != '\t' && pos + 1 < outSize)
-            out[pos++] = *cmdline++;
 }
-    out[pos] = '\0';
+
+static void print_u64(uint64_t v) {
+    char buf[32];
+    int i = 0;
+    if (v == 0) {
+        print("0");
+        return;
+    }
+    while (v && i < (int)sizeof(buf)) {
+        buf[i++] = (char)('0' + (v % 10));
+        v /= 10;
+    }
+    while (i) {
+        char c = buf[--i];
+        print_span(&c, 1);
+    }
+}
+
+static void fail_status(const char* what, uint32_t status) {
+    print("video: ");
+    print(what);
+    print(" failed status=");
+    print_u64(status);
+    print("\n");
+    for (;;) SysExitProcess(1);
+}
+
+static void exit_now(int code) {
+    for (;;) SysExitProcess(code);
+}
+
+static SYS_STRING_VIEW command_line_view(void) {
+    SYS_PEB* peb = SysGetPeb();
+    if (!peb || peb->Magic != SYS_PEB_MAGIC)
+        return SYS_SV_NULL;
+    return peb->CommandLine;
+}
+
+static int sv_is_space(char c) {
+    return c == ' ' || c == '\t';
+}
+
+static SYS_STRING_VIEW skip_one_token(SYS_STRING_VIEW s) {
+    size_t i = 0;
+    while (i < s.Length && sv_is_space(s.Data[i]))
+        i++;
+
+    if (i < s.Length && s.Data[i] == '"') {
+        i++;
+        while (i < s.Length && s.Data[i] != '"')
+            i++;
+        if (i < s.Length && s.Data[i] == '"')
+            i++;
+    } else {
+        while (i < s.Length && !sv_is_space(s.Data[i]))
+            i++;
+    }
+
+    while (i < s.Length && sv_is_space(s.Data[i]))
+        i++;
+
+    return SysSvFromParts(s.Data + i, s.Length - i);
+}
+
+static SYS_STRING_VIEW command_line_second_token(SYS_STRING_VIEW cmdline) {
+    SYS_STRING_VIEW rest = skip_one_token(cmdline);
+    if (!rest.Data || rest.Length == 0)
+        return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
+
+    size_t begin = 0;
+    size_t end = 0;
+    if (rest.Data[0] == '"') {
+        begin = 1;
+        end = begin;
+        while (end < rest.Length && rest.Data[end] != '"')
+            end++;
+    } else {
+        begin = 0;
+        end = 0;
+        while (end < rest.Length && !sv_is_space(rest.Data[end]))
+            end++;
+    }
+
+    if (end <= begin)
+        return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
+
+    return SysSvFromParts(rest.Data + begin, end - begin);
+}
+
+static const char* path_to_cstr(SYS_STRING_VIEW path, char* out, size_t outSize) {
+    if (!out || outSize == 0)
+        return VIDEO_DEFAULT_PATH;
+
+    if (!path.Data || path.Length == 0)
+        path = SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
+
+    size_t n = path.Length;
+    if (n + 1 > outSize)
+        n = outSize - 1;
+
+    memcpy(out, path.Data, n);
+    out[n] = '\0';
     return out;
 }
 
-// Runs the requested operation.
+static uint32_t open_file_view(SYS_STRING_VIEW path, uint32_t* handleOut) {
+    SYS_PATH_HANDLE_REQUEST req = { path, handleOut };
+    return SysOpenFile(&req);
+}
+
 int main(void) {
-    const char* cmdline = GetCommandLineA();
+    SYS_STRING_VIEW cmdline = command_line_view();
-    char videoPath[260];
+    SYS_STRING_VIEW pathView = command_line_second_token(cmdline);
-    const char* path = commandLineSecondToken(cmdline, videoPath, sizeof(videoPath));
 
-    void* h = GetStdHandle(STD_OUTPUT_HANDLE);
+    print("video: starting\n");
-    WriteConsoleA(h, "video: starting\n", 16, 0, 0);
+    if (cmdline.Data && cmdline.Length) {
-
+        print_span(cmdline.Data, cmdline.Length);
-    WriteConsoleA(h, cmdline, (unsigned int)strlen(cmdline), 0, 0);
+        print("\n");
-    WriteConsoleA(h, "\n", 1, 0, 0);
-
-    void* f = CreateFileA(path, 0, 0, 0, 3, 0, 0);
-    if (!f || f == (void*)(unsigned long long)0xFFFFFFFF) { // fuckass
-        WriteConsoleA(h, "video: failed to open input file\n", 33, 0, 0);
-        ExitProcess(1);
     }
 
-    unsigned long s = GetFileSize(f, 0);
+    uint32_t file = 0;
-    if (!s || s == 0xFFFFFFFF) {
+    uint32_t status = open_file_view(pathView, &file);
-        WriteConsoleA(h, "video: GetFileSize failed!\n", 27, 0, 0);
+    if (status != SYS_STATUS_SUCCESS) {
-        ExitProcess(1);
+        char pathBuf[260];
-    }
+        print("video: failed to open input file: ");
-    void* buf = HeapAlloc(0, 0, s);
+        print(path_to_cstr(pathView, pathBuf, sizeof(pathBuf)));
-    if (!buf) {
+        print("\n");
-        WriteConsoleA(h, "video: HeapAlloc failed\n", 24, 0, 0);
+        fail_status("SysOpenFile", status);
-        ExitProcess(1);
     }
 
-    int r = ReadFile(f, buf, s, 0, 0);
+    uint64_t fileSize = SysGetFileSize(file);
-    if (!r) {
+    if (fileSize == 0) {
-        WriteConsoleA(h, "video: ReadFile failed\n", 23, 0, 0);
+        SysCloseHandle(file);
-        ExitProcess(1);
+        print("video: input file is empty\n");
+        exit_now(1);
     }
-    WriteConsoleA(h, "video: input file loaded\n", 25, 0, 0);
 
-    CloseHandle(h);
+    uint32_t mapping = 0;
+    status = SysCreateFileMapping(file, SYS_VM_READ, SYS_MAP_FLAG_NONE, fileSize, &mapping);
+    if (status != SYS_STATUS_SUCCESS || !mapping) {
+        SysCloseHandle(file);
+        fail_status("SysCreateFileMapping", status);
+    }
 
-    HBINK b = BinkOpen(buf, BINKFROMMEMORY);
+    void* buf = 0;
+    status = SysMapFile(mapping, 0, (size_t)fileSize,
+                        SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &buf);
+    if (status != SYS_STATUS_SUCCESS || !buf) {
+        SysCloseHandle(mapping);
+        SysCloseHandle(file);
+        fail_status("SysMapFile", status);
+    }
+
+    // The mapped VAD owns the file reference now; keep only the mapped pointer
+    // alive while Bink is open. This avoids a second whole-file heap allocation
+    // and lets the VM fault file pages in on demand.
+    SysCloseHandle(mapping);
+    SysCloseHandle(file);
+
+    print("video: input file mapped\n");
+
+    HBINK b = BinkOpen((const char*)buf, BINKFROMMEMORY);
     if (!b) {
-        WriteConsoleA(h, "video: BinkOpen failed: ", 24, 0, 0);
+        print("video: BinkOpen failed: ");
         const char* err = BinkGetError();
-        WriteConsoleA(h, err, (unsigned int)strlen(err), 0, 0);
+        print(err ? err : "unknown");
-        WriteConsoleA(h, "\n", 1, 0, 0);
+        print("\n");
-        ExitProcess(1);
+        SysFreeVm(buf);
+        exit_now(1);
     }
 
     uint32_t width = b->Width;
     uint32_t height = b->Height;
-
+    if (!width || !height) {
-    uint32_t* fb = HeapAlloc(0, 0, width * height * sizeof(uint32_t));
+        print("video: invalid video dimensions\n");
-    if (!fb) {
+        BinkClose(b);
-        WriteConsoleA(h, "video: framebuffer alloc failed\n", 32, 0, 0);
+        SysFreeVm(buf);
-        ExitProcess(1);
+        exit_now(1);
     }
 
-    // Prepare the Win32 BITMAPINFO structure.
+    uint32_t surface = 0;
-    BITMAPINFO bmi;
+    status = SysCreateSurface(width, height, SYS_SURFACE_FORMAT_XRGB8888,
-    memset(&bmi, 0, sizeof(bmi));
+                              SYS_SURFACE_FLAG_NONE, &surface);
-    bmi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
+    if (status != SYS_STATUS_SUCCESS || !surface) {
-    bmi.bmiHeader.biWidth = width;
+        BinkClose(b);
-    bmi.bmiHeader.biHeight = -(LONG)height; // Negative means top-down drawing!
+        SysFreeVm(buf);
-    bmi.bmiHeader.biPlanes = 1;
+        fail_status("SysCreateSurface", status);
-    bmi.bmiHeader.biBitCount = 32;
+    }
-    bmi.bmiHeader.biCompression = BI_RGB;
+
+    void* surfaceBase = 0;
+    uint32_t pitchBytes = 0;
+    status = SysMapSurface(surface, &surfaceBase, &pitchBytes);
+    if (status != SYS_STATUS_SUCCESS || !surfaceBase || pitchBytes < width * sizeof(uint32_t)) {
+        SysCloseHandle(surface);
+        BinkClose(b);
+        SysFreeVm(buf);
+        fail_status("SysMapSurface", status);
+    }
+
+    print("video: native surface ready\n");
+
+    DISPLAY_RECT fullFrame = { 0, 0, (int32_t)width, (int32_t)height };
 
     while (1) {
         if (BinkWait(b)) {
@@ -120,25 +241,15 @@ int main(void) {
             continue;
         }
 
-        BinkCopyToBuffer(b, fb, width * 4, height, 0, 0, 
+        BinkCopyToBuffer(b, surfaceBase, pitchBytes, height, 0, 0, BINKSURFACE32);
-            BINKSURFACE32);
 
-        // Welcome to the GDI pipeline!
+        status = SysPresentSurface(surface, &fullFrame, 1);
-        PAINTSTRUCT ps;
+        if (status != SYS_STATUS_SUCCESS) {
-        // Using NULL/0 as the HWND since we don't have a real window handle yet.
+            SysCloseHandle(surface);
-        HDC hdc = BeginPaint((HWND)0, &ps);
+            BinkClose(b);
-
+            SysFreeVm(buf);
-        StretchDIBits(
+            fail_status("SysPresentSurface", status);
-            hdc,
+        }
-            0, 0, width, height, // Destination rect
-            0, 0, width, height, // Source rect
-            fb,
-            &bmi,
-            DIB_RGB_COLORS,
-            SRCCOPY
-        );
-
-        EndPaint((HWND)0, &ps);
 
         if (b->FrameNum == b->Frames)
             break;
@@ -146,7 +257,10 @@ int main(void) {
         BinkNextFrame(b);
     }
 
-    WriteConsoleA(h, "video: goodbye\n", 15, 0, 0);
+    SysCloseHandle(surface);
+    BinkClose(b);
+    SysFreeVm(buf);
 
-    ExitProcess(0);
+    print("video: goodbye\n");
+    exit_now(0);
 }

code/user/personalities/posix/posix.h

@@ -72,12 +72,27 @@ extern int errno;   // defined in posix_fd.c; single global in v1
 #define SEEK_CUR 1
 #define SEEK_END 2
 
+// mmap() protection and mapping flags.
+#define PROT_NONE  0x0
+#define PROT_READ  0x1
+#define PROT_WRITE 0x2
+#define PROT_EXEC  0x4
+#define MAP_SHARED  0x01
+#define MAP_PRIVATE 0x02
+#define MAP_FAILED ((void*)(intptr_t)-1)
+#define MS_SYNC 0x04
+#define MS_ASYNC 0x01
+#define MS_INVALIDATE 0x02
+
 // Scalar types.
 typedef int32_t  pid_t;
 typedef int32_t  off_t;
 typedef uint32_t mode_t;
 typedef int64_t ssize_t;
 
+__declspec(dllexport) void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset);
+__declspec(dllexport) int   munmap(void* addr, size_t length);
+__declspec(dllexport) int   msync(void* addr, size_t length, int flags);
 
 // Sockets.
 typedef uint16_t sa_family_t;
@@ -188,7 +203,7 @@ __declspec(dllexport) int getaddrinfo(const char* node, const char* service, con
 __declspec(dllexport) void freeaddrinfo(struct addrinfo* res);
 
 // pthread types.
-typedef struct { uint32_t Tid; }     pthread_t;
+typedef struct { uint32_t Handle; uint32_t Tid; } pthread_t;
 typedef struct { uint32_t Handle; }  pthread_mutex_t;
 typedef void*                        pthread_attr_t;     // unused in v1
 typedef void*                        pthread_mutexattr_t;

code/user/personalities/posix/posix_io.c

@@ -81,7 +81,8 @@ __declspec(dllexport) ssize_t read(int fd, void* buf, size_t count) {
             return (ssize_t)count;
         case FD_FILE: {
             size_t got = 0;
-            uint32_t s = SysReadFile(e->KHandle, buf, count, &got);
+            SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(e->KHandle, buf, count, &got);
+            uint32_t s = SysReadFile(&req);
             if (s != 0 && got == 0) { errno = PosixStatusToErrno(s); return -1; }
             return (ssize_t)got;
         }
@@ -115,7 +116,8 @@ __declspec(dllexport) ssize_t write(int fd, const void* buf, size_t count) {
             return (ssize_t)count;
         case FD_FILE: {
             size_t written = 0;
-            uint32_t s = SysWriteFile(e->KHandle, buf, count, &written);
+            SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(e->KHandle, buf, count, &written);
+            uint32_t s = SysWriteFile(&req);
             if (s != 0 && written == 0) { errno = PosixStatusToErrno(s); return -1; }
             return (ssize_t)written;
         }

code/user/personalities/posix/posix_mman.c

@@ -0,0 +1,85 @@
+// POSIX compatibility support for mmap-style file mappings.
+
+#include <stdint.h>
+#include <stddef.h>
+#include <ntlite/syscall.h>
+#include "posix.h"
+#include "posix_fd.h"
+
+static uint32_t PosixProtToSysMapAccess(int prot, int flags) {
+    uint32_t access = 0;
+    if (prot & PROT_EXEC) access |= SYS_MAP_ACCESS_EXECUTE;
+    if (prot & PROT_WRITE) {
+        access |= (flags & MAP_SHARED) ? SYS_MAP_ACCESS_WRITE : SYS_MAP_ACCESS_COPY;
+    } else if (flags & MAP_PRIVATE) {
+        access |= SYS_MAP_ACCESS_COPY;
+    }
+    if (prot & PROT_READ) access |= SYS_MAP_ACCESS_READ;
+    if (!access) access = SYS_MAP_ACCESS_READ;
+    return access;
+}
+
+__declspec(dllexport) void* mmap(void* addr, size_t length, int prot,
+                                  int flags, int fd, off_t offset) {
+    if (addr || length == 0 || offset < 0 || ((uint64_t)offset & 0xFFFu)) {
+        errno = EINVAL;
+        return MAP_FAILED;
+    }
+
+    if (!(flags & MAP_PRIVATE) && !(flags & MAP_SHARED)) {
+        errno = EINVAL;
+        return MAP_FAILED;
+    }
+
+    uint32_t pid = SysGetCurrentPid();
+    FD_ENTRY* e = FdGet(pid, fd);
+    if (!e) return MAP_FAILED;
+    if (e->Type != FD_FILE) {
+        errno = EBADF;
+        return MAP_FAILED;
+    }
+
+    uint64_t maxSize = (uint64_t)offset + (uint64_t)length;
+    if (maxSize < (uint64_t)offset) {
+        errno = EINVAL;
+        return MAP_FAILED;
+    }
+
+    uint32_t protect = (prot & PROT_WRITE) ? SYS_VM_READWRITE : SYS_VM_READ;
+    if (prot & PROT_EXEC)
+        protect = (prot & PROT_WRITE) ? SYS_VM_EXECUTE_READWRITE : SYS_VM_EXECUTE_READ;
+
+    uint32_t mapping = 0;
+    uint32_t s = SysCreateFileMapping(e->KHandle, protect, SYS_MAP_FLAG_NONE,
+                                      maxSize, &mapping);
+    if (s != SYS_STATUS_SUCCESS) {
+        errno = PosixStatusToErrno(s);
+        return MAP_FAILED;
+    }
+
+    void* base = (void*)0;
+    s = SysMapFile(mapping, (uint64_t)offset, length,
+                   PosixProtToSysMapAccess(prot, flags),
+                   SYS_MAP_FLAG_NONE, &base);
+    SysCloseHandle(mapping);
+    if (s != SYS_STATUS_SUCCESS) {
+        errno = PosixStatusToErrno(s);
+        return MAP_FAILED;
+    }
+    return base;
+}
+
+__declspec(dllexport) int munmap(void* addr, size_t length) {
+    (void)length;
+    if (!addr) { errno = EINVAL; return -1; }
+    uint32_t s = SysFreeVm(addr);
+    if (s != SYS_STATUS_SUCCESS) { errno = PosixStatusToErrno(s); return -1; }
+    return 0;
+}
+
+__declspec(dllexport) int msync(void* addr, size_t length, int flags) {
+    (void)addr;
+    (void)length;
+    (void)flags;
+    return 0;
+}

code/user/personalities/posix/posix_proc.c

@@ -126,7 +126,7 @@ __declspec(dllexport) pid_t waitpid(pid_t pid, int* status, int options) {
     uint32_t handle = ProcTableFind(pid);
     if (!handle) { errno = ESRCH; return -1; }
 
-    SysWaitForProcess(handle);
+    SysWait(handle, SYS_WAIT_FLAG_NONE);
 
     int exitCode = 0;
     SysGetProcessExitCode(handle, &exitCode);

code/user/personalities/posix/posix_thread.c

@@ -84,28 +84,41 @@ __declspec(dllexport) int pthread_create(pthread_t* t, const pthread_attr_t* att
     tr->Pid = pid;
 
     uint32_t tid = 0;
-    uint32_t s = SysCreateThread((uint64_t)PthreadTrampoline,
+    uint32_t handle = 0;
-                                  (uint64_t)tr, 1, &tid);
+    SYS_CREATE_THREAD_REQUEST req = {
+        (uintptr_t)PthreadTrampoline,
+        (uintptr_t)tr,
+        1,
+        0,
+        &handle,
+        &tid
+    };
+    uint32_t s = SysCreateThread(&req);
     if (s != 0) {
         free(tr);
         return EAGAIN;
     }
     // The trampoline allocates the retval slot itself after it knows its real
     // TID, avoiding the creator/new-thread race.
-    if (t) t->Tid = tid;
+    if (t) {
+        t->Handle = handle;
+        t->Tid = tid;
+    } else {
+        SysCloseHandle(handle);
+    }
     return 0;
 }
 
 // Implements the pthread join operation.
 __declspec(dllexport) int pthread_join(pthread_t t, void** retval) {
     uint32_t pid = SysGetCurrentPid();
-    uint32_t s = SysWaitThread(t.Tid);
+    uint32_t s = SysWait(t.Handle, SYS_WAIT_FLAG_NONE);
-    // STATUS_NOT_FOUND (0x22) means already dead - treat as success.
+    if (s != 0) return ESRCH;
-    if (s != 0 && s != 0x0022) return ESRCH;
 
     PTHREAD_SLOT* slot = PthreadSlotFind(pid, t.Tid);
     if (retval) *retval = slot ? slot->RetVal : (void*)0;
     if (slot) slot->Used = false;
+    SysCloseHandle(t.Handle);
     return 0;
 }
 
@@ -138,7 +151,7 @@ __declspec(dllexport) int pthread_mutex_destroy(pthread_mutex_t* m) {
 
 // Implements the pthread mutex lock operation.
 __declspec(dllexport) int pthread_mutex_lock(pthread_mutex_t* m) {
-    uint32_t s = SysAcquireMutex(m->Handle);
+    uint32_t s = SysWait(m->Handle, SYS_WAIT_FLAG_NONE);
     return s == 0 ? 0 : EINVAL;
 }
 

code/user/personalities/win32/gdi32.c

@@ -63,101 +63,16 @@ static int* srcX_map = NULL;
 static int currentDestCapacity = 0;
 static int currentXMapCapacity = 0;
 
-__declspec(dllexport) int StretchDIBits(
+__declspec(dllexport) int StretchDIBits(HDC hdc, int xDest, int yDest, int DestWidth, int DestHeight,
-    HDC hdc,
-    int xDest, int yDest, int DestWidth, int DestHeight,
     int xSrc, int ySrc, int SrcWidth, int SrcHeight,
     const void *lpBits, const BITMAPINFO *lpbmi,
     UINT iUsage, DWORD rop) 
 {
-    (void)hdc; (void)lpbmi; (void)iUsage; (void)rop;
+    (void)hdc; (void)xDest; (void)yDest; (void)DestWidth; (void)DestHeight;
+    (void)xSrc; (void)ySrc; (void)SrcWidth; (void)SrcHeight;
+    (void)lpBits; (void)lpbmi; (void)iUsage; (void)rop;
 
-    // Safety check to prevent division by zero.
+    // There is no implicit display blit ABI. Native graphics clients own a
-    if (DestWidth <= 0 || DestHeight <= 0 || SrcWidth <= 0 || SrcHeight <= 0) {
+    // surface and present it explicitly.
     return 0;
 }
-
-    // Handle trivial case (1:1 copy) to bypass allocation and scaling overhead.
-    if (DestWidth == SrcWidth && DestHeight == SrcHeight && xSrc == 0 && ySrc == 0) {
-        SysBlitScreen(xDest, yDest, DestWidth, DestHeight, (const uint32_t*)lpBits);
-        return DestHeight;
-    }
-
-    const uint32_t* srcPixels = (const uint32_t*)lpBits;
-
-    // Only reallocate if the new size is LARGER than what we already have.
-    int requiredPixels = DestWidth * DestHeight;
-    if (requiredPixels > currentDestCapacity) {
-        free(destBuffer);
-        destBuffer = (uint32_t*)malloc(requiredPixels * sizeof(uint32_t));
-        currentDestCapacity = requiredPixels;
-    }
-
-    if (DestWidth > currentXMapCapacity) {
-        free(srcX_map);
-        srcX_map = (int*)malloc(DestWidth * sizeof(int));
-        currentXMapCapacity = DestWidth;
-    }
-
-    for (int dx = 0; dx < DestWidth; ++dx) {
-        // Nearest neighbor coordinate mapping.
-        srcX_map[dx] = xSrc + (dx * SrcWidth) / DestWidth;
-        
-        // Clamp to safe bounds.
-        if (srcX_map[dx] < 0) srcX_map[dx] = 0;
-        if (srcX_map[dx] >= SrcWidth) srcX_map[dx] = SrcWidth - 1;
-    }
-
-    // Process the image row by row.
-    for (int dy = 0; dy < DestHeight; ++dy) {
-        // Map destination Y to source Y.
-        int sy = ySrc + (dy * SrcHeight) / DestHeight;
-        if (sy < 0) sy = 0;
-        if (sy >= SrcHeight) sy = SrcHeight - 1;
-
-        // Assumes stride == SrcWidth (tightly packed 32bpp).
-        const uint32_t* srcRow = srcPixels + (sy * SrcWidth); 
-        uint32_t* dstRow = destBuffer + (dy * DestWidth);
-
-        int dx = 0;
-
-#if defined(__SSE2__) || defined(_M_X64)
-        // Process 4 pixels (16 bytes) per loop iteration using SSE2.
-        for (; dx <= DestWidth - 4; dx += 4) {
-            // Gather 4 source pixels using our precomputed X map.
-            // Note: _mm_set_epi32 takes arguments in reverse order (3, 2, 1, 0).
-            __m128i pixels = _mm_set_epi32(
-                srcRow[srcX_map[dx + 3]],
-                srcRow[srcX_map[dx + 2]],
-                srcRow[srcX_map[dx + 1]],
-                srcRow[srcX_map[dx + 0]]
-            );
-            // Issue a single 128-bit unaligned store.
-            _mm_storeu_si128((__m128i*)&dstRow[dx], pixels);
-        }
-#elif defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM64)
-        // Process 4 pixels per loop iteration using ARM NEON.
-        for (; dx <= DestWidth - 4; dx += 4) {
-            // Initialize a vector and populate lanes (avoids stack-memory hop).
-            uint32x4_t pixels = vdupq_n_u32(0);
-            pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 0]], pixels, 0);
-            pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 1]], pixels, 1);
-            pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 2]], pixels, 2);
-            pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 3]], pixels, 3);
-            
-            // Issue a single 128-bit store.
-            vst1q_u32(&dstRow[dx], pixels);
-        }
-#endif
-
-        // Epilogue: Handle remaining pixels if DestWidth isn't a multiple of 4.
-        for (; dx < DestWidth; ++dx) {
-            dstRow[dx] = srcRow[srcX_map[dx]];
-        }
-    }
-
-    // Direct mapping to kernel screen blit using the stretched buffer.
-    SysBlitScreen(xDest, yDest, DestWidth, DestHeight, destBuffer);
-
-    return DestHeight; 
-}

code/user/personalities/win32/kernel32.c

@@ -27,6 +27,8 @@
 #include <intrin.h>
 #include <ntlite/syscall.h>
 
+extern size_t _malloc_usable_size(const void* ptr);
+
 // Types.
 typedef void*          HANDLE;
 typedef unsigned long  DWORD;
@@ -58,6 +60,7 @@ typedef struct _CONSOLE_SCREEN_BUFFER_INFO {
 #define MEM_RELEASE     0x00008000UL
 #define MEM_FREE        0x00010000UL
 #define MEM_PRIVATE     0x00020000UL
+#define MEM_MAPPED      0x00040000UL
 #define PAGE_NOACCESS          0x01UL
 #define PAGE_READONLY          0x02UL
 #define PAGE_READWRITE         0x04UL
@@ -65,6 +68,13 @@ typedef struct _CONSOLE_SCREEN_BUFFER_INFO {
 #define PAGE_EXECUTE           0x10UL
 #define PAGE_EXECUTE_READ      0x20UL
 #define PAGE_EXECUTE_READWRITE 0x40UL
+#define SEC_COMMIT             0x08000000UL
+
+#define FILE_MAP_COPY          0x00000001UL
+#define FILE_MAP_WRITE         0x00000002UL
+#define FILE_MAP_READ          0x00000004UL
+#define FILE_MAP_EXECUTE       0x00000020UL
+#define FILE_MAP_ALL_ACCESS    0x000F001FUL
 
 typedef struct _MEMORY_BASIC_INFORMATION {
     void*  BaseAddress;
@@ -90,6 +100,26 @@ static uint32_t WinProtectToSysVm(DWORD protect) {
     }
 }
 
+static uint32_t WinFileMappingProtectToSysVm(DWORD protect) {
+    // Strip section attributes; the low byte carries PAGE_* protection.
+    DWORD pageProtect = protect & 0xFFu;
+    if (pageProtect == PAGE_WRITECOPY)
+        return SYS_VM_READ; // MapViewOfFile(FILE_MAP_COPY) asks for the private writable view.
+    return WinProtectToSysVm(pageProtect ? pageProtect : PAGE_READONLY);
+}
+
+static uint32_t WinFileMapAccessToSys(DWORD desiredAccess) {
+    uint32_t access = 0;
+    if (desiredAccess & FILE_MAP_EXECUTE) access |= SYS_MAP_ACCESS_EXECUTE;
+    if (desiredAccess & FILE_MAP_COPY)    access |= SYS_MAP_ACCESS_COPY;
+    if (desiredAccess & FILE_MAP_WRITE)   access |= SYS_MAP_ACCESS_WRITE;
+    if (desiredAccess & FILE_MAP_READ)    access |= SYS_MAP_ACCESS_READ;
+    if ((desiredAccess & FILE_MAP_ALL_ACCESS) == FILE_MAP_ALL_ACCESS)
+        access |= SYS_MAP_ACCESS_READ | SYS_MAP_ACCESS_COPY;
+    if (!access) access = SYS_MAP_ACCESS_READ;
+    return access;
+}
+
 // Implements the protect Vm To Win operation.
 static DWORD SysProtectVmToWin(uint32_t protect) {
     switch (protect) {
@@ -153,24 +183,9 @@ typedef struct _WIN32_FIND_DATAA {
 static inline HANDLE   KH_TO_HANDLE(uint32_t kh) { return (HANDLE)(uintptr_t)kh; }
 static inline uint32_t HANDLE_TO_KH(HANDLE h)    { return (uint32_t)(uintptr_t)h; }
 
-// Thread IDs are not kernel handles. Keep Win32 thread pseudo-handles out of
+static inline BOOL IS_VALID_KERNEL_HANDLE(HANDLE h) {
-// the low numeric handle namespace so CloseHandle/WaitForSingleObject cannot
-// accidentally operate on an unrelated file/process handle with the same value.
-#define WIN32_THREAD_HANDLE_TAG ((uintptr_t)1ULL << 63)
-#define WIN32_MUTEX_SENTINEL    ((uintptr_t)0x4D555458UL)  // 'MUTX'
-#define WIN32_SEMA_SENTINEL     ((uintptr_t)0x53454D41UL)  // 'SEMA'
-static inline HANDLE TID_TO_THREAD_HANDLE(uint32_t tid) {
-    return (HANDLE)(WIN32_THREAD_HANDLE_TAG | (uintptr_t)tid);
-}
-static inline BOOL IS_THREAD_HANDLE(HANDLE h) {
-    return h && h != INVALID_HANDLE_VALUE && (((uintptr_t)h & WIN32_THREAD_HANDLE_TAG) != 0);
-}
-static inline uint32_t THREAD_HANDLE_TO_TID(HANDLE h) {
-    return (uint32_t)((uintptr_t)h & 0xFFFFFFFFu);
-}
-static inline BOOL IS_NOOP_SYNC_HANDLE(HANDLE h) {
     uintptr_t v = (uintptr_t)h;
-    return v == WIN32_MUTEX_SENTINEL || v == WIN32_SEMA_SENTINEL;
+    return v != 0 && v != (uintptr_t)INVALID_HANDLE_VALUE;
 }
 
 // Path translator.
@@ -268,11 +283,14 @@ static void K32DebugPrintA(const char* s) {
 }
 
 #define WIN32_TLS_SLOTS 64
+#define FLS_MAX_SLOTS   64
+#define FLS_OUT_OF_INDEXES 0xFFFFFFFFUL
 
 typedef struct WIN32_THREAD_ENV {
     DWORD LastError;
     DWORD Reserved0;
     void* TlsSlots[WIN32_TLS_SLOTS];
+    void* FlsSlots[FLS_MAX_SLOTS];
 } WIN32_THREAD_ENV;
 
 static WIN32_THREAD_ENV gFallbackThreadEnv;
@@ -418,11 +436,27 @@ __declspec(dllexport) uint64_t GetTickCount64(void) {
 }
 
 // Memory  (heap argument is ignored - single user-mode heap).
+#define HEAP_ZERO_MEMORY             0x00000008UL
+#define HEAP_REALLOC_IN_PLACE_ONLY   0x00000010UL
+
 __declspec(dllexport) void* HeapAlloc(void* heap, DWORD flags, size_t size) {
-    (void)heap; (void)flags;
+    (void)heap;
     if (!size) return NULL;
+
     void* ptr = malloc(size);
-    if (!ptr) gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY
+    if (!ptr) {
+        gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY
+        return NULL;
+    }
+
+    // Bink and the Microsoft CRT routinely allocate internal state with
+    // HEAP_ZERO_MEMORY. Ignoring this flag only works accidentally while the
+    // heap keeps receiving fresh demand-zero pages; under low memory / swap the
+    // allocator reuses dirty freed blocks and stale pitch/pointer fields can
+    // corrupt the decoder.
+    if (flags & HEAP_ZERO_MEMORY)
+        memset(ptr, 0, size);
+
     return ptr;
 }
 
@@ -436,9 +470,28 @@ __declspec(dllexport) BOOL HeapFree(void* heap, DWORD flags, void* ptr) {
 
 __declspec(dllexport) void* HeapReAlloc(void* heap, DWORD flags,
                                          void* ptr, size_t size) {
-    (void)heap; (void)flags;
+    (void)heap;
+
+    if (flags & HEAP_REALLOC_IN_PLACE_ONLY) {
+        // The user heap does not expose an in-place grow primitive. Preserve
+        // Windows semantics by failing instead of silently moving the block.
+        size_t oldSize = _malloc_usable_size(ptr);
+        if (oldSize != (size_t)-1 && oldSize >= size)
+            return ptr;
+        gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY
+        return NULL;
+    }
+
+    size_t oldSize = ptr ? _malloc_usable_size(ptr) : 0;
     void* newptr = realloc(ptr, size);
-    if (!newptr && size) gLastError = 8;
+    if (!newptr && size) {
+        gLastError = 8;
+        return NULL;
+    }
+
+    if (newptr && (flags & HEAP_ZERO_MEMORY) && oldSize != (size_t)-1 && size > oldSize)
+        memset((unsigned char*)newptr + oldSize, 0, size - oldSize);
+
     return newptr;
 }
 
@@ -508,10 +561,88 @@ __declspec(dllexport) size_t VirtualQuery(const void* lpAddress,
     lpBuffer->RegionSize = info.PageCount * 4096u;
     lpBuffer->State = MEM_COMMIT;
     lpBuffer->Protect = protect;
-    lpBuffer->Type = MEM_PRIVATE;
+    lpBuffer->Type = (info.Type == 7u) ? MEM_MAPPED : MEM_PRIVATE; // 7 == MmVadMappedFile
     return sizeof(MEMORY_BASIC_INFORMATION);
 }
 
+__declspec(dllexport) HANDLE CreateFileMappingA(HANDLE hFile,
+                                                void* lpFileMappingAttributes,
+                                                DWORD flProtect,
+                                                DWORD dwMaximumSizeHigh,
+                                                DWORD dwMaximumSizeLow,
+                                                const char* lpName) {
+    (void)lpFileMappingAttributes;
+    (void)lpName;
+    if (!hFile || hFile == INVALID_HANDLE_VALUE) {
+        gLastError = 6; // ERROR_INVALID_HANDLE
+        return NULL;
+    }
+
+    uint64_t maximumSize = ((uint64_t)dwMaximumSizeHigh << 32) | (uint64_t)dwMaximumSizeLow;
+    uint32_t kh = 0;
+    uint32_t s = SysCreateFileMapping(HANDLE_TO_KH(hFile),
+                                      WinFileMappingProtectToSysVm(flProtect),
+                                      SYS_MAP_FLAG_NONE,
+                                      maximumSize,
+                                      &kh);
+    if (s != SYS_STATUS_SUCCESS) {
+        gLastError = (s == SYS_STATUS_NOT_SUPPORTED) ? 120 : ((s == SYS_STATUS_NO_MEMORY) ? 8 : 87);
+        return NULL;
+    }
+    return KH_TO_HANDLE(kh);
+}
+
+__declspec(dllexport) HANDLE CreateFileMappingW(HANDLE hFile,
+                                                void* lpFileMappingAttributes,
+                                                DWORD flProtect,
+                                                DWORD dwMaximumSizeHigh,
+                                                DWORD dwMaximumSizeLow,
+                                                const unsigned short* lpName) {
+    (void)lpName;
+    return CreateFileMappingA(hFile, lpFileMappingAttributes, flProtect,
+                              dwMaximumSizeHigh, dwMaximumSizeLow, NULL);
+}
+
+__declspec(dllexport) void* MapViewOfFile(HANDLE hFileMappingObject,
+                                           DWORD dwDesiredAccess,
+                                           DWORD dwFileOffsetHigh,
+                                           DWORD dwFileOffsetLow,
+                                           size_t dwNumberOfBytesToMap) {
+    if (!hFileMappingObject || hFileMappingObject == INVALID_HANDLE_VALUE) {
+        gLastError = 6;
+        return NULL;
+    }
+
+    uint64_t offset = ((uint64_t)dwFileOffsetHigh << 32) | (uint64_t)dwFileOffsetLow;
+    void* base = NULL;
+    uint32_t s = SysMapFile(HANDLE_TO_KH(hFileMappingObject),
+                            offset,
+                            dwNumberOfBytesToMap,
+                            WinFileMapAccessToSys(dwDesiredAccess),
+                            SYS_MAP_FLAG_NONE,
+                            &base);
+    if (s != SYS_STATUS_SUCCESS) {
+        gLastError = (s == SYS_STATUS_NOT_SUPPORTED) ? 120 : ((s == SYS_STATUS_NO_MEMORY) ? 8 : 87);
+        return NULL;
+    }
+    return base;
+}
+
+__declspec(dllexport) BOOL UnmapViewOfFile(const void* lpBaseAddress) {
+    if (!lpBaseAddress) { gLastError = 87; return FALSE; }
+    uint32_t s = SysFreeVm((void*)lpBaseAddress);
+    if (s != SYS_STATUS_SUCCESS) { gLastError = 87; return FALSE; }
+    return TRUE;
+}
+
+__declspec(dllexport) BOOL FlushViewOfFile(const void* lpBaseAddress, size_t dwNumberOfBytesToFlush) {
+    (void)dwNumberOfBytesToFlush;
+    if (!lpBaseAddress) { gLastError = 87; return FALSE; }
+    // Native flush is file-handle based for now; unmap/close also harvests dirty
+    // PTEs, so FlushViewOfFile remains a successful cache-coherency hint.
+    return TRUE;
+}
+
 // --- Win32 API ---.
 
 // GetAsyncKeyState implementation over the canonical input key namespace.
@@ -720,7 +851,7 @@ __declspec(dllexport) HANDLE CreateFileA(const char* name,
 // GetFileSize - fills optional high-DWORD pointer, returns low DWORD.
 // Returns 0xFFFFFFFF on error, matching Win32 behaviour.
 __declspec(dllexport) DWORD GetFileSize(HANDLE hFile, DWORD* fileSizeHigh) {
-    if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) { gLastError = 6; return 0xFFFFFFFFUL; }
+    if (!hFile || hFile == INVALID_HANDLE_VALUE) { gLastError = 6; return 0xFFFFFFFFUL; }
     uint64_t sz = SysGetFileSize(HANDLE_TO_KH(hFile));
     if (fileSizeHigh) *fileSizeHigh = (DWORD)(sz >> 32);
     return (DWORD)(sz & 0xFFFFFFFFULL);
@@ -731,25 +862,27 @@ __declspec(dllexport) BOOL ReadFile(HANDLE hFile, void* buf, DWORD toRead,
                                      DWORD* bytesRead, void* overlapped) {
     (void)overlapped;
     if (bytesRead) *bytesRead = 0;
-    if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile) || !buf) {
+    if (!hFile || hFile == INVALID_HANDLE_VALUE || !buf) {
         gLastError = 6;
         return FALSE;
     }
 
     size_t got = 0;
-    uint32_t s = SysReadFile(HANDLE_TO_KH(hFile), buf, (size_t)toRead, &got);
+    SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(HANDLE_TO_KH(hFile), buf, (size_t)toRead, &got);
+    uint32_t s = SysReadFile(&req);
     if (bytesRead) *bytesRead = (DWORD)got;
 
-    // Harden callers that accidentally look past *bytesRead. EOF/short reads
-    // are legal for files, but consuming stale stack/heap bytes after them makes
-    // the failure mode look like random game-data corruption.
-    if (got < (size_t)toRead)
-        memset((unsigned char*)buf + got, 0, (size_t)toRead - got);
-
     if (s != 0) {
         gLastError = 1;
         return FALSE;
     }
+
+    // Harden callers that accidentally look past *bytesRead. EOF/short reads
+    // are legal for files, but do not touch the caller's remaining buffer after
+    // a failed read because that can fault in megabytes under low memory.
+    if (got < (size_t)toRead)
+        memset((unsigned char*)buf + got, 0, (size_t)toRead - got);
+
     return TRUE;
 }
 
@@ -758,9 +891,10 @@ __declspec(dllexport) BOOL ReadFile(HANDLE hFile, void* buf, DWORD toRead,
 __declspec(dllexport) BOOL WriteFile(HANDLE hFile, const void* buf, DWORD toWrite,
                                       DWORD* bytesWritten, void* overlapped) {
     (void)overlapped;
-    if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile) || !buf) { gLastError = 6; return FALSE; }
+    if (!hFile || hFile == INVALID_HANDLE_VALUE || !buf) { gLastError = 6; return FALSE; }
     size_t written = 0;
-    uint32_t s = SysWriteFile(HANDLE_TO_KH(hFile), buf, (size_t)toWrite, &written);
+    SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(HANDLE_TO_KH(hFile), buf, (size_t)toWrite, &written);
+    uint32_t s = SysWriteFile(&req);
     if (bytesWritten) *bytesWritten = (DWORD)written;
     if (s != 0) { gLastError = 29; return FALSE; } // ERROR_WRITE_FAULT
     return TRUE;
@@ -770,7 +904,7 @@ __declspec(dllexport) BOOL WriteFile(HANDLE hFile, const void* buf, DWORD toWrit
 // Returns new position low DWORD, or 0xFFFFFFFF on error.
 __declspec(dllexport) DWORD SetFilePointer(HANDLE hFile, long dist,
                                             long* distHigh, DWORD method) {
-    if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) {
+    if (!hFile || hFile == INVALID_HANDLE_VALUE) {
         gLastError = 6;
         return 0xFFFFFFFFUL;
     }
@@ -813,7 +947,7 @@ __declspec(dllexport) DWORD SetFilePointer(HANDLE hFile, long dist,
 
 // SetEndOfFile - truncate or extend the file to the current seek position.
 __declspec(dllexport) BOOL SetEndOfFile(HANDLE hFile) {
-    if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) { gLastError = 6; return FALSE; }
+    if (!hFile || hFile == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; }
     uint32_t kh  = HANDLE_TO_KH(hFile);
     uint64_t pos = SysGetFilePos(kh);
     if (pos == (uint64_t)-1) { gLastError = 6; return FALSE; }
@@ -834,14 +968,10 @@ __declspec(dllexport) BOOL DeleteFileA(const char* lpFileName) {
     return TRUE;
 }
 
-// CloseHandle - unified close for files, mutexes, semaphores, events, queues.
+// CloseHandle - unified close for files, threads, mutexes, semaphores, events, queues, and processes.
 __declspec(dllexport) BOOL CloseHandle(HANDLE hObject) {
     if (!hObject || hObject == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; }
 
-    // Our thread/sentinel sync handles are not entries in the kernel handle table.
-    // There is nothing to close; the thread object is reaped by the scheduler.
-    if (IS_THREAD_HANDLE(hObject) || IS_NOOP_SYNC_HANDLE(hObject)) return TRUE;
-
     SysCloseHandle(HANDLE_TO_KH(hObject));
     return TRUE;
 }
@@ -870,8 +1000,11 @@ __declspec(dllexport) BOOL HeapDestroy(HANDLE hHeap) {
 
 // Implements the heap Size operation.
 __declspec(dllexport) size_t HeapSize(HANDLE heap, DWORD flags, const void* ptr) {
-    (void)heap; (void)flags; (void)ptr;
+    (void)heap; (void)flags;
-    return (size_t)-1;  // Win32 returns (SIZE_T)-1 on error / unsupported
+    size_t size = _malloc_usable_size(ptr);
+    if (size == (size_t)-1)
+        gLastError = 87; // ERROR_INVALID_PARAMETER
+    return size;
 }
 
 __declspec(dllexport) BOOL HeapSetInformation(HANDLE HeapHandle,
@@ -1050,11 +1183,31 @@ __declspec(dllexport) DWORD GetModuleFileNameA(HANDLE hModule,
 
 // Implements the terminate Process operation.
 __declspec(dllexport) BOOL TerminateProcess(HANDLE hProcess, unsigned int uExitCode) {
-    (void)hProcess;
+    if (hProcess == GetCurrentProcess()) {
         SysExitProcess((int)uExitCode);
         while (true) ;
     }
 
+    if (!hProcess) {
+        gLastError = 6; // ERROR_INVALID_HANDLE
+        return FALSE;
+    }
+
+    uint32_t pid = 0;
+    uint32_t s = SysGetProcessPid(HANDLE_TO_KH(hProcess), &pid);
+    if (s != 0 || pid == 0) {
+        gLastError = 6; // ERROR_INVALID_HANDLE
+        return FALSE;
+    }
+
+    s = SysKillProcess(pid, (int)uExitCode);
+    if (s != 0) {
+        gLastError = 5; // ERROR_ACCESS_DENIED / generic failure for now
+        return FALSE;
+    }
+    return TRUE;
+}
+
 // STARTUPINFOA - only cb matters for our stub.
 typedef struct {
     DWORD  cb;
@@ -1099,35 +1252,42 @@ __declspec(dllexport) HANDLE CreateThread(void* lpThreadAttributes,
                                            DWORD* lpThreadId) {
     (void)lpThreadAttributes; (void)dwStackSize; (void)dwCreationFlags;
     uint32_t tid = 0;
-    uint32_t s = SysCreateThread((uint64_t)(uintptr_t)lpStartAddress,
+    uint32_t kh = 0;
-                                  (uint64_t)(uintptr_t)lpParameter,
+    SYS_CREATE_THREAD_REQUEST req = {
-                                  0, &tid);
+        (uintptr_t)lpStartAddress,
+        (uintptr_t)lpParameter,
+        0,
+        0,
+        &kh,
+        &tid
+    };
+    uint32_t s = SysCreateThread(&req);
     if (s != 0) { gLastError = 11; return INVALID_HANDLE_VALUE; }
     if (lpThreadId) *lpThreadId = (DWORD)tid;
-    return TID_TO_THREAD_HANDLE(tid);
+    return KH_TO_HANDLE(kh);
 }
 
-// WaitForSingleObject - blocks on process handles via SysWaitForProcess;
+// WaitForSingleObject - blocks on native waitable handles. A zero timeout is
-// thread and other handle types are not yet waitable so return immediately.
+// only a probe for mutexes/semaphores until native timeout waits exist.
 __declspec(dllexport) DWORD WaitForSingleObject(HANDLE hHandle, DWORD dwMilliseconds) {
-    (void)dwMilliseconds;
+    if (!IS_VALID_KERNEL_HANDLE(hHandle)) { gLastError = 6; return 0xFFFFFFFFUL; }
-    if (!hHandle || hHandle == INVALID_HANDLE_VALUE) { gLastError = 6; return 0xFFFFFFFFUL; }
-
-    if (IS_THREAD_HANDLE(hHandle)) {
-        uint32_t tid = THREAD_HANDLE_TO_TID(hHandle);
-        uint32_t s = SysWaitThread(tid);
-        if (s != 0 && s != 0x0022) { gLastError = 6; return 0xFFFFFFFFUL; }
-        return WAIT_OBJECT_0;
-    }
-
-    if (IS_NOOP_SYNC_HANDLE(hHandle)) return WAIT_OBJECT_0;
 
     uint32_t kh = HANDLE_TO_KH(hHandle);
-    if (kh) {
+    if (dwMilliseconds == 0) {
-        uint32_t s = SysWaitForProcess(kh);
+        uint32_t s = SysTryAcquireMutex(kh);
-        if (s != 0) { gLastError = 6; return 0xFFFFFFFFUL; }
+        if (s == 0) return WAIT_OBJECT_0;
+
+        s = SysTryWaitSem(kh);
+        if (s == 0) return WAIT_OBJECT_0;
+
+        return WAIT_TIMEOUT;
     }
-    return WAIT_OBJECT_0;
+
+    uint32_t s = SysWait(kh, SYS_WAIT_FLAG_NONE);
+    if (s == 0) return WAIT_OBJECT_0;
+
+    gLastError = 6;
+    return 0xFFFFFFFFUL;
 }
 
 
@@ -1250,7 +1410,7 @@ __declspec(dllexport) BOOL CreateProcessA(
 __declspec(dllexport) BOOL GetExitCodeProcess(HANDLE hProcess, DWORD* lpExitCode) {
     if (!lpExitCode) { gLastError = 87; return FALSE; }
     uint32_t kh = HANDLE_TO_KH(hProcess);
-    if (!kh || IS_THREAD_HANDLE(hProcess)) { gLastError = 6; return FALSE; } // ERROR_INVALID_HANDLE
+    if (!kh) { gLastError = 6; return FALSE; } // ERROR_INVALID_HANDLE
     int code = 0;
     uint32_t s = SysGetProcessExitCode(kh, &code);
     if (s != 0) { gLastError = 6; return FALSE; }
@@ -1791,57 +1951,123 @@ __declspec(dllexport) DWORD GetCurrentDirectoryA(DWORD nBufferLength, char* lpBu
 
 __declspec(dllexport) HANDLE CreateMutexA(void* lpMutexAttributes, BOOL bInitialOwner,
                                            const char* lpName) {
-    (void)lpMutexAttributes; (void)bInitialOwner; (void)lpName;
+    (void)lpMutexAttributes; (void)lpName;
-    return (HANDLE)WIN32_MUTEX_SENTINEL;
+    uint32_t kh = 0;
+    uint32_t s = SysCreateMutex(&kh);
+    if (s != 0 || !kh) { gLastError = 8; return NULL; }
+    if (bInitialOwner) {
+        s = SysWait(kh, SYS_WAIT_FLAG_NONE);
+        if (s != 0) {
+            SysCloseHandle(kh);
+            gLastError = 1;
+            return NULL;
+        }
+    }
+    return KH_TO_HANDLE(kh);
 }
 
 // Releases mutex.
 __declspec(dllexport) BOOL ReleaseMutex(HANDLE hMutex) {
-    (void)hMutex;
+    if (!IS_VALID_KERNEL_HANDLE(hMutex)) { gLastError = 6; return FALSE; }
+    uint32_t s = SysReleaseMutex(HANDLE_TO_KH(hMutex));
+    if (s != 0) { gLastError = 6; return FALSE; }
     return TRUE;
 }
 
 __declspec(dllexport) HANDLE CreateSemaphoreA(void* lpSemaphoreAttributes,
                                                long lInitialCount, long lMaximumCount,
                                                const char* lpName) {
-    (void)lpSemaphoreAttributes; (void)lInitialCount;
+    (void)lpSemaphoreAttributes; (void)lpName;
-    (void)lMaximumCount; (void)lpName;
+    uint32_t kh = 0;
-    return (HANDLE)WIN32_SEMA_SENTINEL;
+    uint32_t s = SysCreateSem((int32_t)lInitialCount, (int32_t)lMaximumCount, &kh);
+    if (s != 0 || !kh) { gLastError = 8; return NULL; }
+    return KH_TO_HANDLE(kh);
 }
 
 __declspec(dllexport) BOOL ReleaseSemaphore(HANDLE hSemaphore, long lReleaseCount,
                                              long* lpPreviousCount) {
-    (void)hSemaphore; (void)lReleaseCount;
+    if (!IS_VALID_KERNEL_HANDLE(hSemaphore) || lReleaseCount < 0) { gLastError = 6; return FALSE; }
     if (lpPreviousCount) *lpPreviousCount = 0;
+    for (long i = 0; i < lReleaseCount; ++i) {
+        uint32_t s = SysSignalSem(HANDLE_TO_KH(hSemaphore));
+        if (s != 0) { gLastError = 6; return FALSE; }
+    }
     return TRUE;
 }
 
-// Critical Sections & Condition Variables
+// Critical Sections & Condition Variables.
-// Single-threaded: all locks are no-ops.
+// Match the size of Win64 RTL_CRITICAL_SECTION (5 pointer-sized fields) while
-
+// using the storage as a small recursive spin lock. Bink uses these from its
-// Minimal opaque types; callers allocate these from their Windows headers -
+// worker/copy paths, so the old no-op implementation allowed internal state
-// we just need the pointer to be accepted by our stubs.
+// races as soon as CreateThread started working.
 typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION;
 typedef struct { void* _ptr; }           CONDITION_VARIABLE;
 
+#define K32_CS_LOCK      0
+#define K32_CS_OWNER     1
+#define K32_CS_RECURSION 2
+
+static volatile long* K32CsLockWord(CRITICAL_SECTION* cs) {
+    return (volatile long*)&cs->_opaque[K32_CS_LOCK];
+}
+
+static DWORD K32CurrentTidOrOne(void) {
+    DWORD tid = (DWORD)SysGetCurrentTid();
+    return tid ? tid : 1;
+}
+
 // Initializes critical Section.
 __declspec(dllexport) void InitializeCriticalSection(CRITICAL_SECTION* lpCS) {
-    (void)lpCS;
+    if (!lpCS) return;
+    memset(lpCS, 0, sizeof(*lpCS));
 }
 __declspec(dllexport) BOOL InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION* lpCS,
                                                                    DWORD dwSpinCount) {
-    (void)lpCS; (void)dwSpinCount;
+    (void)dwSpinCount;
+    InitializeCriticalSection(lpCS);
     return TRUE;
 }
 // Releases critical Section.
 __declspec(dllexport) void DeleteCriticalSection(CRITICAL_SECTION* lpCS)  { (void)lpCS; }
 // Implements the enter Critical Section operation.
-__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS)   { (void)lpCS; }
+__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS) {
+    if (!lpCS) return;
+    DWORD tid = K32CurrentTidOrOne();
+    if ((DWORD)lpCS->_opaque[K32_CS_OWNER] == tid) {
+        lpCS->_opaque[K32_CS_RECURSION]++;
+        return;
+    }
+    while (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0)
+        _mm_pause();
+    lpCS->_opaque[K32_CS_OWNER] = tid;
+    lpCS->_opaque[K32_CS_RECURSION] = 1;
+}
 // Implements the leave Critical Section operation.
-__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION* lpCS)   { (void)lpCS; }
+__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION* lpCS) {
+    if (!lpCS) return;
+    DWORD tid = K32CurrentTidOrOne();
+    if ((DWORD)lpCS->_opaque[K32_CS_OWNER] != tid || lpCS->_opaque[K32_CS_RECURSION] == 0)
+        return;
+    lpCS->_opaque[K32_CS_RECURSION]--;
+    if (lpCS->_opaque[K32_CS_RECURSION] == 0) {
+        lpCS->_opaque[K32_CS_OWNER] = 0;
+        _ReadWriteBarrier();
+        _InterlockedExchange(K32CsLockWord(lpCS), 0);
+    }
+}
 // Implements the try Enter Critical Section operation.
 __declspec(dllexport) BOOL TryEnterCriticalSection(CRITICAL_SECTION* lpCS) {
-    (void)lpCS; return TRUE;
+    if (!lpCS) return FALSE;
+    DWORD tid = K32CurrentTidOrOne();
+    if ((DWORD)lpCS->_opaque[K32_CS_OWNER] == tid) {
+        lpCS->_opaque[K32_CS_RECURSION]++;
+        return TRUE;
+    }
+    if (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0)
+        return FALSE;
+    lpCS->_opaque[K32_CS_OWNER] = tid;
+    lpCS->_opaque[K32_CS_RECURSION] = 1;
+    return TRUE;
 }
 
 // Initializes condition Variable.
@@ -2357,14 +2583,12 @@ __declspec(dllexport) int GetLocaleInfoA(unsigned int Locale, DWORD LCType,
     return 0;
 }
 
-// Fiber Local Storage (FLS)
+// Fiber Local Storage (FLS).
-// We implement a minimal 64-slot static table so callers that use FLS for
+// Allocation state is process-global, but the values themselves are per-thread,
-// thread-local-ish state don't immediately fail.
+// matching the contract that Bink and the CRT expect when worker threads are
+// enabled. The old single global value array let unrelated threads overwrite
+// each other's decoder-local state.
 
-#define FLS_MAX_SLOTS 64
-#define FLS_OUT_OF_INDEXES 0xFFFFFFFFUL
-
-static void* gFlsSlots[FLS_MAX_SLOTS];
 static int   gFlsUsed[FLS_MAX_SLOTS];  // 0 = free, 1 = allocated
 
 // Implements the fls Alloc operation.
@@ -2373,7 +2597,7 @@ __declspec(dllexport) DWORD FlsAlloc(void* lpCallback) {
     for (int i = 0; i < FLS_MAX_SLOTS; i++) {
         if (!gFlsUsed[i]) {
             gFlsUsed[i] = 1;
-            gFlsSlots[i] = NULL;
+            K32ThreadEnv()->FlsSlots[i] = NULL;
             return (DWORD)i;
         }
     }
@@ -2387,7 +2611,7 @@ __declspec(dllexport) void* FlsGetValue(DWORD dwFlsIndex) {
         gLastError = 87; // ERROR_INVALID_PARAMETER
         return NULL;
     }
-    return gFlsSlots[dwFlsIndex];
+    return K32ThreadEnv()->FlsSlots[dwFlsIndex];
 }
 
 // Implements the fls Set Value operation.
@@ -2396,7 +2620,7 @@ __declspec(dllexport) BOOL FlsSetValue(DWORD dwFlsIndex, void* lpFlsData) {
         gLastError = 87;
         return FALSE;
     }
-    gFlsSlots[dwFlsIndex] = lpFlsData;
+    K32ThreadEnv()->FlsSlots[dwFlsIndex] = lpFlsData;
     return TRUE;
 }
 
@@ -2407,7 +2631,7 @@ __declspec(dllexport) BOOL FlsFree(DWORD dwFlsIndex) {
         return FALSE;
     }
     gFlsUsed[dwFlsIndex] = 0;
-    gFlsSlots[dwFlsIndex] = NULL;
+    K32ThreadEnv()->FlsSlots[dwFlsIndex] = NULL;
     return TRUE;
 }
 

code/user/runtime/heap.c

@@ -210,6 +210,29 @@ void* _calloc(size_t count, size_t size) {
     return ptr;
 }
 
+
+// Returns the usable payload size of an allocation made by this heap.
+size_t _malloc_usable_size(const void* ptr) {
+    if (!ptr) return (size_t)-1;
+
+    HeapLock();
+    HEAP_SEGMENT* seg = SegmentForPointer((void*)ptr);
+    if (!seg) {
+        HeapUnlock();
+        return (size_t)-1;
+    }
+
+    HEAP_BLOCK* block = PayloadBlock((void*)ptr);
+    if (block->Magic != USER_HEAP_MAGIC || block->Free) {
+        HeapUnlock();
+        return (size_t)-1;
+    }
+
+    size_t size = block->Size;
+    HeapUnlock();
+    return size;
+}
+
 // Implements the realloc operation.
 void* _realloc(void* ptr, size_t newSize) {
     if (!ptr) return _malloc(newSize);

code/user/runtime/heap_exports.c

@@ -12,6 +12,7 @@ void* _malloc(size_t size);
 void  _free(void* ptr);
 void* _calloc(size_t count, size_t size);
 void* _realloc(void* ptr, size_t size);
+size_t _malloc_usable_size(const void* ptr);
 
 HEAP_EXPORT void* malloc(size_t size) {
     return _malloc(size);
@@ -28,3 +29,7 @@ HEAP_EXPORT void* calloc(size_t count, size_t size) {
 HEAP_EXPORT void* realloc(void* ptr, size_t size) {
     return _realloc(ptr, size);
 }
+
+HEAP_EXPORT size_t malloc_usable_size(const void* ptr) {
+    return _malloc_usable_size(ptr);
+}

code/user/syscall/syscall_nr.inc

@@ -3,13 +3,13 @@
 SYS_EXIT_PROCESS               EQU 0
 SYS_EXIT_THREAD                EQU 1
 SYS_CREATE_PROCESS             EQU 2
-SYS_WAIT_FOR_PROCESS           EQU 3
+SYS_WAIT_SINGLE                EQU 3
 SYS_GET_PROCESS_EXIT_CODE      EQU 4
 SYS_KILL_PROCESS               EQU 5
 SYS_GET_PROCESS_LIST           EQU 6
 SYS_GET_PROCESS_PID            EQU 7
 SYS_CREATE_THREAD              EQU 8
-SYS_WAIT_THREAD                EQU 9
+SYS_RETIRED_WAIT_THREAD        EQU 9
 SYS_GET_CURRENT_TID            EQU 10
 SYS_GET_CURRENT_PID            EQU 11
 SYS_SET_THREAD_PRIORITY        EQU 12
@@ -23,17 +23,17 @@ SYS_PROTECT_VM                 EQU 19
 SYS_QUERY_VM                   EQU 20
 SYS_CLOSE_HANDLE               EQU 21
 SYS_CREATE_MUTEX               EQU 22
-SYS_ACQUIRE_MUTEX              EQU 23
+SYS_RETIRED_ACQUIRE_MUTEX      EQU 23
 SYS_TRY_ACQUIRE_MUTEX          EQU 24
 SYS_RELEASE_MUTEX              EQU 25
 SYS_CREATE_SEM                 EQU 26
-SYS_WAIT_SEM                   EQU 27
+SYS_RETIRED_WAIT_SEM           EQU 27
 SYS_TRY_WAIT_SEM               EQU 28
 SYS_SIGNAL_SEM                 EQU 29
 SYS_CREATE_EVENT               EQU 30
 SYS_SIGNAL_EVENT               EQU 31
 SYS_CLEAR_EVENT                EQU 32
-SYS_WAIT_EVENT                 EQU 33
+SYS_RETIRED_WAIT_EVENT         EQU 33
 SYS_CREATE_QUEUE               EQU 34
 SYS_SEND_QUEUE                 EQU 35
 SYS_RECV_QUEUE                 EQU 36
@@ -66,36 +66,47 @@ SYS_SET_CONSOLE_CURSOR          EQU 62
 SYS_FILL_CONSOLE_CHARACTER      EQU 63
 SYS_FILL_CONSOLE_ATTRIBUTE      EQU 64
 SYS_PUT_SCREEN_CHAR             EQU 65
-SYS_BLIT_SCREEN                 EQU 66
+SYS_CLEAR_SCREEN                EQU 66
-SYS_CLEAR_SCREEN                EQU 67
+SYS_GET_SCREEN_SIZE             EQU 67
-SYS_GET_SCREEN_SIZE             EQU 68
+SYS_QUERY_DISPLAY_INFO          EQU 68
-SYS_QUERY_DISPLAY_INFO          EQU 69
+SYS_POLL_KEYBOARD               EQU 69
-SYS_POLL_KEYBOARD               EQU 70
+SYS_IS_KEY_DOWN                 EQU 70
-SYS_IS_KEY_DOWN                 EQU 71
+SYS_READ_KEYBOARD_CHAR          EQU 71
-SYS_READ_KEYBOARD_CHAR          EQU 72
+SYS_READ_KEY                    EQU 72
-SYS_READ_KEY                    EQU 73
+SYS_GET_KEY_MODIFIERS           EQU 73
-SYS_GET_KEY_MODIFIERS           EQU 74
+SYS_POLL_MOUSE                  EQU 74
-SYS_POLL_MOUSE                  EQU 75
+SYS_READ_MOUSE                  EQU 75
-SYS_READ_MOUSE                  EQU 76
+SYS_GET_MOUSE_POS               EQU 76
-SYS_GET_MOUSE_POS               EQU 77
+SYS_GET_TICK                    EQU 77
-SYS_GET_TICK                    EQU 78
+SYS_PRINT_DEBUG                 EQU 78
-SYS_PRINT_DEBUG                 EQU 79
+SYS_QUERY_TRAP_FRAME            EQU 79
-SYS_QUERY_TRAP_FRAME            EQU 80
+SYS_CREATE_SOCKET               EQU 80
-SYS_CREATE_SOCKET               EQU 81
+SYS_BIND_SOCKET                 EQU 81
-SYS_BIND_SOCKET                 EQU 82
+SYS_LISTEN_SOCKET               EQU 82
-SYS_LISTEN_SOCKET               EQU 83
+SYS_ACCEPT_SOCKET               EQU 83
-SYS_ACCEPT_SOCKET               EQU 84
+SYS_CONNECT_SOCKET              EQU 84
-SYS_CONNECT_SOCKET              EQU 85
+SYS_SEND_SOCKET                 EQU 85
-SYS_SEND_SOCKET                 EQU 86
+SYS_RECV_SOCKET                 EQU 86
-SYS_RECV_SOCKET                 EQU 87
+SYS_SHUTDOWN_SOCKET             EQU 87
-SYS_SHUTDOWN_SOCKET             EQU 88
+SYS_SEND_TO_SOCKET              EQU 88
-SYS_SEND_TO_SOCKET              EQU 89
+SYS_RECV_FROM_SOCKET            EQU 89
-SYS_RECV_FROM_SOCKET            EQU 90
+SYS_RESOLVE_HOST                EQU 90
-SYS_RESOLVE_HOST                EQU 91
+SYS_SET_SOCKET_OPTION           EQU 91
-SYS_SET_SOCKET_OPTION           EQU 92
+SYS_GET_SOCKET_OPTION           EQU 92
-SYS_GET_SOCKET_OPTION           EQU 93
+SYS_POLL                        EQU 93
-SYS_POLL                        EQU 94
+SYS_GET_MEMORY_STATUS           EQU 94
-SYS_GET_MEMORY_STATUS           EQU 95
+SYS_QUERY_KERNEL_STATS          EQU 95
-SYS_QUERY_KERNEL_STATS          EQU 96
+SYS_GET_SYSTEM_TIME             EQU 96
-SYS_GET_SYSTEM_TIME             EQU 97
+SYS_CREATE_FILE_MAPPING          EQU 97
-SYS_MAX                         EQU 98
+SYS_MAP_FILE                     EQU 98
+SYS_CREATE_SURFACE               EQU 99
+SYS_MAP_SURFACE                  EQU 100
+SYS_QUERY_SURFACE                EQU 101
+SYS_PRESENT_SURFACE              EQU 102
+SYS_GET_THREAD_LIST             EQU 103
+SYS_READ_INPUT_EVENT            EQU 104
+SYS_PEEK_INPUT_EVENT            EQU 105
+SYS_WAIT_MANY                  EQU 106
+SYS_OPEN_INPUT_QUEUE           EQU 107
+SYS_FLUSH_FILE                 EQU 108
+SYS_MAX                         EQU 109

code/user/syscall/syscall_stubs.asm

@@ -12,7 +12,8 @@
 ; User-mode callers follow standard MS x64: args 1-4 in RCX/RDX/R8/R9,
 ; args 5+ on the stack above the home area ([rsp+28h], [rsp+30h], ...).
 ;
-; SYSCALL_STUB2  - 4 or fewer args: pure register passing, no stack read.
+; SYSCALL_STUB2/3 - 4 or fewer args: pure register passing, no stack read.
+; SYSCALL_STUB5  - exactly 5 args: loads [rsp+28h] -> R10 only.
 ; SYSCALL_STUB6  - 5 or 6 args: loads [rsp+28h] -> R10 and [rsp+30h] -> R11.
 ;
 ; Handle lifecycle:
@@ -44,6 +45,27 @@ Sys&name PROC EXPORT
 Sys&name ENDP
 ENDM
 
+; ---------------------------------------------------------------------------
+SYSCALL_STUB3 MACRO name, nr
+PUBLIC Sys&name
+Sys&name PROC EXPORT
+    mov     rax, nr
+    int     40h
+    ret
+Sys&name ENDP
+ENDM
+
+; ---------------------------------------------------------------------------
+SYSCALL_STUB5 MACRO name, nr
+PUBLIC Sys&name
+Sys&name PROC EXPORT
+    mov     r10, [rsp+28h]      ; 5th argument
+    mov     rax, nr
+    int     40h
+    ret
+Sys&name ENDP
+ENDM
+
 ; ---------------------------------------------------------------------------
 SYSCALL_STUB6 MACRO name, nr
 PUBLIC Sys&name
@@ -62,14 +84,16 @@ ENDM
 SYSCALL_STUB2 ExitProcess,          SYS_EXIT_PROCESS
 SYSCALL_STUB2 ExitThread,           SYS_EXIT_THREAD
 SYSCALL_STUB2 CreateProcess,        SYS_CREATE_PROCESS
-SYSCALL_STUB2 WaitForProcess,       SYS_WAIT_FOR_PROCESS
+SYSCALL_STUB3 WaitSingle,           SYS_WAIT_SINGLE
+SYSCALL_STUB2 WaitMany,             SYS_WAIT_MANY
+SYSCALL_STUB2 OpenInputQueue,       SYS_OPEN_INPUT_QUEUE
 SYSCALL_STUB2 GetProcessExitCode,   SYS_GET_PROCESS_EXIT_CODE
 SYSCALL_STUB2 KillProcess,          SYS_KILL_PROCESS
 SYSCALL_STUB2 GetProcessList,       SYS_GET_PROCESS_LIST
 SYSCALL_STUB2 GetProcessPid,        SYS_GET_PROCESS_PID
+SYSCALL_STUB2 GetThreadList,       SYS_GET_THREAD_LIST
 
 SYSCALL_STUB2 CreateThread,         SYS_CREATE_THREAD
-SYSCALL_STUB2 WaitThread,           SYS_WAIT_THREAD
 SYSCALL_STUB2 GetCurrentTid,        SYS_GET_CURRENT_TID
 SYSCALL_STUB2 GetCurrentPid,        SYS_GET_CURRENT_PID
 SYSCALL_STUB2 SetThreadPriority,    SYS_SET_THREAD_PRIORITY
@@ -120,19 +144,16 @@ SYSCALL_STUB2 CloseHandle,          SYS_CLOSE_HANDLE
 ; Synchronization objects
 ; ---------------------------------------------------------------------------
 SYSCALL_STUB2 CreateMutex,          SYS_CREATE_MUTEX
-SYSCALL_STUB2 AcquireMutex,         SYS_ACQUIRE_MUTEX
 SYSCALL_STUB2 TryAcquireMutex,      SYS_TRY_ACQUIRE_MUTEX
 SYSCALL_STUB2 ReleaseMutex,         SYS_RELEASE_MUTEX
 
 SYSCALL_STUB2 CreateSem,            SYS_CREATE_SEM
-SYSCALL_STUB2 WaitSem,              SYS_WAIT_SEM
 SYSCALL_STUB2 TryWaitSem,           SYS_TRY_WAIT_SEM
 SYSCALL_STUB2 SignalSem,            SYS_SIGNAL_SEM
 
 SYSCALL_STUB2 CreateEvent,          SYS_CREATE_EVENT
 SYSCALL_STUB2 SignalEvent,          SYS_SIGNAL_EVENT
 SYSCALL_STUB2 ClearEvent,           SYS_CLEAR_EVENT
-SYSCALL_STUB2 WaitEvent,            SYS_WAIT_EVENT
 
 ; ---------------------------------------------------------------------------
 ; Queues and named queues
@@ -159,6 +180,7 @@ SYSCALL_STUB2 SeekFile,             SYS_SEEK_FILE
 SYSCALL_STUB2 GetFilePos,           SYS_GET_FILE_POS
 SYSCALL_STUB2 GetFileSize,          SYS_GET_FILE_SIZE
 SYSCALL_STUB2 TruncateFile,         SYS_TRUNCATE_FILE
+SYSCALL_STUB2 FlushFile,            SYS_FLUSH_FILE
 SYSCALL_STUB2 DeleteFile,           SYS_DELETE_FILE
 SYSCALL_STUB2 RenameFile,           SYS_RENAME_FILE
 
@@ -178,7 +200,10 @@ SYSCALL_STUB2 SetConsoleCursor,     SYS_SET_CONSOLE_CURSOR
 SYSCALL_STUB6 FillConsoleCharacter, SYS_FILL_CONSOLE_CHARACTER
 SYSCALL_STUB6 FillConsoleAttribute, SYS_FILL_CONSOLE_ATTRIBUTE
 SYSCALL_STUB6 PutScreenChar,        SYS_PUT_SCREEN_CHAR
-SYSCALL_STUB6 BlitScreen,           SYS_BLIT_SCREEN
+SYSCALL_STUB5 CreateSurface,        SYS_CREATE_SURFACE
+SYSCALL_STUB2 MapSurface,           SYS_MAP_SURFACE
+SYSCALL_STUB2 QuerySurface,         SYS_QUERY_SURFACE
+SYSCALL_STUB2 PresentSurface,       SYS_PRESENT_SURFACE
 SYSCALL_STUB2 ClearScreen,          SYS_CLEAR_SCREEN
 SYSCALL_STUB2 GetScreenSize,        SYS_GET_SCREEN_SIZE
 SYSCALL_STUB2 QueryDisplayInfo,     SYS_QUERY_DISPLAY_INFO
@@ -191,6 +216,8 @@ SYSCALL_STUB2 IsKeyDown,          SYS_IS_KEY_DOWN
 SYSCALL_STUB2 ReadKeyboardChar,           SYS_READ_KEYBOARD_CHAR
 SYSCALL_STUB2 ReadKey,            SYS_READ_KEY
 SYSCALL_STUB2 GetKeyModifiers,       SYS_GET_KEY_MODIFIERS
+SYSCALL_STUB2 ReadInputEvent,      SYS_READ_INPUT_EVENT
+SYSCALL_STUB2 PeekInputEvent,      SYS_PEEK_INPUT_EVENT
 
 SYSCALL_STUB2 PollMouse,            SYS_POLL_MOUSE
 SYSCALL_STUB2 ReadMouse,            SYS_READ_MOUSE
@@ -224,5 +251,7 @@ SYSCALL_STUB2 Poll,                 SYS_POLL
 SYSCALL_STUB2 GetMemoryStatus,       SYS_GET_MEMORY_STATUS
 SYSCALL_STUB2 QueryKernelStats,     SYS_QUERY_KERNEL_STATS
 SYSCALL_STUB2 GetSystemTime,        SYS_GET_SYSTEM_TIME
+SYSCALL_STUB5 CreateFileMapping,    SYS_CREATE_FILE_MAPPING
+SYSCALL_STUB6 MapFile,              SYS_MAP_FILE
 
 END

code/user/system/init/init.c

@@ -76,9 +76,9 @@ void _start(void)
         }
 
         debug_print("[init] waiting\n");
-        s = SysWaitForProcess(h);
+        s = SysWait(h, SYS_WAIT_FLAG_NONE);
         if (s != STATUS_SUCCESS)
-            debug_print("[init] WARNING: SysWaitForProcess returned error\n");
+            debug_print("[init] WARNING: SysWait returned error\n");
 
         int exitCode = 0;
         if (SysGetProcessExitCode(h, &exitCode) != STATUS_SUCCESS) {

code/user/tests/async_apc/async_apc.c

@@ -15,10 +15,35 @@ static void print(const char* s) {
     SysWriteConsole(&req);
 }
 
-// Opens file.
+// Opens the first available executable payload for async I/O.
-static uint32_t open_file(const char* path, uint32_t* handleOut) {
+// Tests are usually launched from /apps, so a bare "system/init.exe"
-    SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), handleOut };
+// resolves as /apps/system/init.exe and fails. Prefer the test binary itself,
-    return SysOpenFile(&req);
+// then fall back to absolute/drive-qualified paths.
+static uint32_t open_test_file(uint32_t* handleOut, const char** pathOut) {
+    static const char* paths[] = {
+        "async_apc.exe",
+        "/apps/async_apc.exe",
+        "1:/apps/async_apc.exe",
+        "0:/apps/async_apc.exe",
+        "/system/init.exe",
+        "0:/system/init.exe",
+        "1:/system/init.exe",
+        "system/init.exe",
+    };
+
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
+        uint32_t h = 0;
+        SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h };
+        uint32_t s = SysOpenFile(&req);
+        if (s == SYS_STATUS_SUCCESS) {
+            *handleOut = h;
+            if (pathOut)
+                *pathOut = paths[i];
+            return SYS_STATUS_SUCCESS;
+        }
+    }
+
+    return SYS_STATUS_NOT_FOUND;
 }
 
 // Prints hex8.
@@ -74,7 +99,8 @@ int main(void) {
     }
 
     uint32_t file = 0;
-    s = open_file("system/init.exe", &file);
+    const char* openedPath = 0;
+    s = open_test_file(&file, &openedPath);
     if (s != SYS_STATUS_SUCCESS) {
         print("async_apc: open failed status=");
         print_u64(s);
@@ -82,6 +108,10 @@ int main(void) {
         SysExitProcess(1);
     }
 
+    print("async_apc: using ");
+    print(openedPath ? openedPath : "<unknown>");
+    print("\n");
+
     uint32_t event = 0;
     s = SysCreateEvent(EVENT_AUTO_RESET, 0, &event);
     if (s != SYS_STATUS_SUCCESS) {
@@ -125,7 +155,7 @@ int main(void) {
     }
 
     while (!state.Delivered) {
-        s = SysWaitEvent(event, 1);
+        s = SysWait(event, SYS_WAIT_FLAG_ALERTABLE);
         print("async_apc: alertable wait returned ");
         print_u64(s);
         print("\n");

code/user/tests/async_cancel/async_cancel.c

@@ -14,10 +14,35 @@ static void print(const char* s) {
     SysWriteConsole(&req);
 }
 
-// Opens file.
+// Opens the first available executable payload for async I/O.
-static uint32_t open_file(const char* path, uint32_t* handleOut) {
+// Tests are usually launched from /apps, so a bare "system/init.exe"
-    SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), handleOut };
+// resolves as /apps/system/init.exe and fails. Prefer the test binary itself,
-    return SysOpenFile(&req);
+// then fall back to absolute/drive-qualified paths.
+static uint32_t open_test_file(uint32_t* handleOut, const char** pathOut) {
+    static const char* paths[] = {
+        "async_cancel.exe",
+        "/apps/async_cancel.exe",
+        "1:/apps/async_cancel.exe",
+        "0:/apps/async_cancel.exe",
+        "/system/init.exe",
+        "0:/system/init.exe",
+        "1:/system/init.exe",
+        "system/init.exe",
+    };
+
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
+        uint32_t h = 0;
+        SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h };
+        uint32_t s = SysOpenFile(&req);
+        if (s == SYS_STATUS_SUCCESS) {
+            *handleOut = h;
+            if (pathOut)
+                *pathOut = paths[i];
+            return SYS_STATUS_SUCCESS;
+        }
+    }
+
+    return SYS_STATUS_NOT_FOUND;
 }
 
 // Prints u64.
@@ -46,12 +71,17 @@ int main(void) {
     uint32_t event = 0;
     uint32_t request = 0;
 
-    uint32_t s = open_file("system/init.exe", &file);
+    const char* openedPath = 0;
+    uint32_t s = open_test_file(&file, &openedPath);
     if (s != SYS_STATUS_SUCCESS) {
         print("async_cancel: open failed status="); print_u64(s); print("\n");
         SysExitProcess(1);
     }
 
+    print("async_cancel: using ");
+    print(openedPath ? openedPath : "<unknown>");
+    print("\n");
+
     s = SysCreateEvent(EVENT_AUTO_RESET, 0, &event);
     if (s != SYS_STATUS_SUCCESS) {
         print("async_cancel: event create failed status="); print_u64(s); print("\n");
@@ -90,7 +120,7 @@ int main(void) {
     print("async_cancel: cancel returned "); print_u64(cs); print("\n");
 
     while (iosb.Status == SYS_STATUS_PENDING) {
-        s = SysWaitEvent(event, 1);
+        s = SysWait(event, SYS_WAIT_FLAG_ALERTABLE);
         print("async_cancel: wait returned "); print_u64(s); print("\n");
         if (s != SYS_STATUS_SUCCESS && s != SYS_STATUS_KERNEL_APC && s != SYS_STATUS_USER_APC && s != SYS_STATUS_ALERTED)
             break;

code/user/tests/dirty_writeback/dirty_writeback.c

@@ -0,0 +1,227 @@
+// Dirty file-cache writeback regression test.
+// Verifies buffered writes dirty cache pages, explicit flush writes them back, and close flushes synchronously.
+
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include <ntlite/syscall.h>
+
+#define PAGE_SIZE_BYTES 4096u
+#define TEST_SIZE       (PAGE_SIZE_BYTES * 4u)
+#define REWRITE_SIZE    (PAGE_SIZE_BYTES * 2u)
+#define CLOSE_SIZE      (PAGE_SIZE_BYTES * 3u)
+
+static uint8_t gPatternA[TEST_SIZE];
+static uint8_t gPatternB[TEST_SIZE];
+static uint8_t gReadback[TEST_SIZE];
+
+static void print(const char* s) {
+    SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) };
+    SysWriteConsole(&req);
+}
+
+static void print_u64(uint64_t v) {
+    char buf[32];
+    int i = 0;
+    if (v == 0) {
+        print("0");
+        return;
+    }
+    while (v && i < (int)sizeof(buf)) {
+        buf[i++] = (char)('0' + (v % 10));
+        v /= 10;
+    }
+    while (i--) {
+        char c[2] = { buf[i], 0 };
+        print(c);
+    }
+}
+
+static void exit_now(int code) {
+    for (;;) SysExitProcess(code);
+}
+
+static void fail(const char* msg) {
+    print("dirty_writeback: ");
+    print(msg);
+    print("\n");
+    exit_now(1);
+}
+
+static void fail_status(const char* what, uint32_t status) {
+    print("dirty_writeback: ");
+    print(what);
+    print(" failed status=");
+    print_u64(status);
+    print("\n");
+    exit_now(1);
+}
+
+static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) {
+    memset(out, 0, sizeof(*out));
+    return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0);
+}
+
+static void delete_path(const char* path) {
+    SYS_PATH_REQUEST req = { SysSvFromCstr(path) };
+    (void)SysDeleteFile(&req);
+}
+
+static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) {
+    static const char* paths[] = {
+        "dirty_writeback.tmp",
+        "/dirty_writeback.tmp",
+        "1:/dirty_writeback.tmp",
+        "0:/dirty_writeback.tmp",
+    };
+
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
+        uint32_t h = 0;
+        delete_path(paths[i]);
+        SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h };
+        uint32_t s = SysCreateFile(&req);
+        if (s == SYS_STATUS_SUCCESS) {
+            *fileOut = h;
+            if (pathOut) *pathOut = paths[i];
+            return SYS_STATUS_SUCCESS;
+        }
+    }
+
+    return SYS_STATUS_NOT_FOUND;
+}
+
+static uint32_t open_path(const char* path, uint32_t* fileOut) {
+    uint32_t h = 0;
+    SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), &h };
+    uint32_t s = SysOpenFile(&req);
+    if (s == SYS_STATUS_SUCCESS)
+        *fileOut = h;
+    return s;
+}
+
+static void fill_patterns(void) {
+    for (size_t i = 0; i < sizeof(gPatternA); ++i) {
+        gPatternA[i] = (uint8_t)(0x11u + (uint8_t)(i * 13u + (i >> 6)));
+        gPatternB[i] = (uint8_t)(0x80u + (uint8_t)(i * 31u + (i >> 4)));
+    }
+}
+
+static void checked_write(uint32_t file, const void* buffer, size_t size, const char* what) {
+    size_t done = 0;
+    SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &done);
+    uint32_t s = SysWriteFile(&req);
+    if (s != SYS_STATUS_SUCCESS) fail_status(what, s);
+    if (done != size) fail("short write");
+}
+
+static void checked_read(uint32_t file, void* buffer, size_t size, size_t* gotOut, const char* what) {
+    size_t got = 0;
+    memset(buffer, 0xCC, size);
+    SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &got);
+    uint32_t s = SysReadFile(&req);
+    if (s != SYS_STATUS_SUCCESS) fail_status(what, s);
+    if (gotOut) *gotOut = got;
+}
+
+static void checked_seek(uint32_t file, uint64_t offset, const char* what) {
+    uint32_t s = SysSeekFile(file, offset);
+    if (s != SYS_STATUS_SUCCESS) fail_status(what, s);
+}
+
+static void expect_writeback_progress(const SYS_FILE_CACHE_STATS* before,
+                                      const SYS_FILE_CACHE_STATS* after,
+                                      const char* what) {
+    if (after->FlushFileFailures != before->FlushFileFailures)
+        fail("flush failure counter increased");
+    if (after->WritebackFailures != before->WritebackFailures)
+        fail("writeback failure counter increased");
+    if (after->WritebackBytes <= before->WritebackBytes) {
+        print("dirty_writeback: ");
+        print(what);
+        print(" did not write back bytes\n");
+        exit_now(1);
+    }
+}
+
+int main(void) {
+    print("dirty_writeback: start\n");
+    fill_patterns();
+
+    uint32_t file = 0;
+    const char* path = 0;
+    uint32_t s = create_temp(&file, &path);
+    if (s != SYS_STATUS_SUCCESS) fail_status("create temp", s);
+
+    print("dirty_writeback: using ");
+    print(path ? path : "<unknown>");
+    print("\n");
+
+    SYS_FILE_CACHE_STATS before;
+    SYS_FILE_CACHE_STATS afterWrite;
+    SYS_FILE_CACHE_STATS afterFlush;
+    SYS_FILE_CACHE_STATS beforeClose;
+    SYS_FILE_CACHE_STATS afterClose;
+
+    if (query_cache(&before) != SYS_STATUS_SUCCESS) fail("initial stats query failed");
+
+    checked_write(file, gPatternA, sizeof(gPatternA), "write pattern A");
+
+    if (query_cache(&afterWrite) != SYS_STATUS_SUCCESS) fail("post-write stats query failed");
+    if (afterWrite.DirtyMarks <= before.DirtyMarks) fail("write did not mark dirty cache pages");
+    if (afterWrite.DirtyPages <= before.DirtyPages) fail("write did not leave dirty pages resident");
+
+    checked_seek(file, 0, "seek readback before flush");
+    size_t got = 0;
+    checked_read(file, gReadback, sizeof(gReadback), &got, "read before flush");
+    if (got != sizeof(gPatternA)) fail("read before flush size mismatch");
+    if (memcmp(gReadback, gPatternA, sizeof(gPatternA)) != 0) fail("cached read before flush mismatch");
+
+    s = SysFlushFile(file);
+    if (s != SYS_STATUS_SUCCESS) fail_status("explicit flush", s);
+
+    if (query_cache(&afterFlush) != SYS_STATUS_SUCCESS) fail("post-flush stats query failed");
+    if (afterFlush.FlushFileCalls <= afterWrite.FlushFileCalls) fail("flush call counter did not increase");
+    expect_writeback_progress(&afterWrite, &afterFlush, "explicit flush");
+
+    checked_seek(file, 0, "seek rewrite");
+    checked_write(file, gPatternB, REWRITE_SIZE, "rewrite pattern B");
+    checked_seek(file, REWRITE_SIZE, "seek close-tail");
+    checked_write(file, gPatternA + REWRITE_SIZE, CLOSE_SIZE - REWRITE_SIZE, "write close tail");
+
+    if (query_cache(&beforeClose) != SYS_STATUS_SUCCESS) fail("pre-close stats query failed");
+    if (beforeClose.DirtyMarks <= afterFlush.DirtyMarks) fail("rewrite did not mark dirty pages");
+
+    s = SysCloseHandle(file);
+    file = 0;
+    if (s != SYS_STATUS_SUCCESS) fail_status("close", s);
+
+    if (query_cache(&afterClose) != SYS_STATUS_SUCCESS) fail("post-close stats query failed");
+    if (afterClose.FlushFileCalls <= beforeClose.FlushFileCalls) fail("close did not flush file");
+    expect_writeback_progress(&beforeClose, &afterClose, "close flush");
+
+    s = open_path(path, &file);
+    if (s != SYS_STATUS_SUCCESS) fail_status("reopen", s);
+
+    checked_read(file, gReadback, CLOSE_SIZE, &got, "read after reopen");
+    if (got != CLOSE_SIZE) fail("read after reopen size mismatch");
+    if (memcmp(gReadback, gPatternB, REWRITE_SIZE) != 0) fail("rewrite data not durable after close");
+    if (memcmp(gReadback + REWRITE_SIZE, gPatternA + REWRITE_SIZE,
+               CLOSE_SIZE - REWRITE_SIZE) != 0) fail("tail data not durable after close");
+
+    s = SysCloseHandle(file);
+    file = 0;
+    if (s != SYS_STATUS_SUCCESS) fail_status("close reopened", s);
+    delete_path(path);
+
+    print("dirty_writeback: dirty-marks+=");
+    print_u64(afterClose.DirtyMarks - before.DirtyMarks);
+    print(" writeback-bytes+=");
+    print_u64(afterClose.WritebackBytes - before.WritebackBytes);
+    print(" flushes+=");
+    print_u64(afterClose.FlushFileCalls - before.FlushFileCalls);
+    print("\n");
+
+    print("dirty_writeback: PASS\n");
+    exit_now(0);
+    return 0;
+}