diff --git a/.gitignore b/.gitignore index 0075d3e..5be5314 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ build.obj bin/ obj/ qemu.log -*.vhd +disk.vhd +usb.vhd +disk.vmdk +code.zip code/build/build.exe code/build/build.exe.old \ No newline at end of file diff --git a/OVMF/OVMF_VARS_4M.fd b/OVMF/OVMF_VARS_4M.fd index 1ee7a92..a943fd5 100644 Binary files a/OVMF/OVMF_VARS_4M.fd and b/OVMF/OVMF_VARS_4M.fd differ diff --git a/OVMF/OVMF_VARS_4M_d.fd b/OVMF/OVMF_VARS_4M_d.fd index 8037eed..36b1d57 100644 Binary files a/OVMF/OVMF_VARS_4M_d.fd and b/OVMF/OVMF_VARS_4M_d.fd differ diff --git a/code/boot.ini b/code/boot.ini index 5cc5468..e6994fb 100644 --- a/code/boot.ini +++ b/code/boot.ini @@ -1,5 +1,5 @@ [kernel] ; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file -debug = 1 +debug = 0xffff [init] commandline = apps/shell_native.exe diff --git a/code/boot/uefi/main.c b/code/boot/uefi/main.c index 7bf1f07..c25e124 100644 --- a/code/boot/uefi/main.c +++ b/code/boot/uefi/main.c @@ -291,7 +291,7 @@ static int ParseBootIniInteger(const char* val) { static void BootIniCb(const char* section, const char* key, const char* val, void* ud) { PARSE_STATE* s = (PARSE_STATE *)ud; - // [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log. + // [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log, 8+=kernel diagnostics. if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug")) s->debug = ParseBootIniInteger(val); // [init] commandline = hello.exe. @@ -446,7 +446,7 @@ EFI_STATUS LoadPeImage(UINT8* rawFile, UINT64 virtDest, EFI_PHYSICAL_ADDRESS* ou IMAGE_DATA_DIRECTORY* dataDirs = (IMAGE_DATA_DIRECTORY*)(ntHeaders + 24 + 112); IMAGE_DATA_DIRECTORY* relocDir = &dataDirs[5]; - //IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1];. + //IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1]; // Allocate Physical Memory for the mapped image. UINTN pages = (sizeOfImage + PAGE_SIZE - 1) / PAGE_SIZE; @@ -619,7 +619,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable return Status; } - //Print(L"1\n");. + Print(L"1\n"); // boot.ini controls debugger and runtime policy before the kernel has a filesystem service. UINTN sz; @@ -632,7 +632,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable FreePool(bootIniRaw); } - //Print(L"2\n");. + Print(L"2\n"); // drivers.ini decides which boot and runtime drivers are loaded into the initial module list. UINT8* drvIniRaw = LoadFile(L"drivers\\drivers.ini", &sz); @@ -643,7 +643,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable FreePool(drvIniRaw); } - //Print(L"3\n");. + Print(L"3\n"); BOOT_INFO* bi = AllocateZeroPool(sizeof(BOOT_INFO)); bi->Magic = BOOT_INFO_MAGIC; @@ -675,7 +675,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable // Current mapping cursor for upper half. UINT64 virtCursor = KERNEL_VIRT_BASE; - //Print(L"4\n");. + Print(L"4\n"); // Core images are loaded first because driver import resolution depends on them. typedef struct { @@ -742,7 +742,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable while(true); } - //Print(L"5\n");. + Print(L"5\n"); // Load driver .sys files into raw buffers. for (int i = 0; i < ps.driverCount; i++) { @@ -751,9 +751,10 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable UINTN imgSz; UINT8* buf = LoadFile(path, &imgSz); - if (!buf) { - Print(L"Driver not found: %s\n", ps.drivers[i]); - continue; + if (!buf) { + Print(L"Fatal: boot driver not found: %s\n", ps.drivers[i]); + while (true); + return EFI_NOT_FOUND; } // Store file info for the kernel to process later. @@ -767,7 +768,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable mod->VirtBase = 0; // Kernel will map this later } - //Print(L"6\n");. + Print(L"6\n"); // Query GOP FIRST, before the GetMemoryMap loop. EFI_GRAPHICS_OUTPUT_PROTOCOL* gop; @@ -795,7 +796,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable } } - //Print(L"7\n");. + Print(L"7\n"); bi->ApTrampolinePhys = (uintptr_t)AllocateApTrampolinePageBelow1M(); @@ -895,7 +896,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // Physical Window Map, P|W|PCD } - //Print(L"8\n");. + Print(L"8\n"); // The kernel consumes a compact memory map so it does not need EFI descriptor semantics after boot. ConvertMemoryMap(&bi->MemoryMap); @@ -940,12 +941,12 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable } // Print it out to verify the bootloader actually found it before launching the kernel - //Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);. + Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical); bi->PhysicalMapBase = PHYS_MAP_BASE; bi->Debug = (UINT32)ps.debug; - //Print(L"9\n");. + Print(L"9\n"); // ExitBootServices is the point of no return for firmware allocation, file I/O, and protocol access. gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer); diff --git a/code/build/build.c b/code/build/build.c index deae697..6fecfb1 100644 --- a/code/build/build.c +++ b/code/build/build.c @@ -30,6 +30,7 @@ static const Driver_Target PCI_BUS_DRIVER = static const Driver_Target STANDALONE_DRIVERS[] = { DRIVER_TARGET("ps2", "obj\\drivers\\ps2", "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"), + DRIVER_TARGET("serial", "obj\\drivers\\serial", "code\\drivers\\input\\serial\\serial.c"), DRIVER_TARGET_LIBS("ata", "obj\\drivers\\ata", "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"), DRIVER_TARGET_LIBS("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"), DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"), @@ -46,7 +47,10 @@ void create_layout() { "obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", "obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget", - "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video" + "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop", + "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", + "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread", + "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present" }; for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) nob_mkdir_if_not_exists(dirs[i]); nob_mkdir_if_not_exists(PCI_BUS_DRIVER.obj_dir); @@ -71,6 +75,41 @@ Nob_Proc run_async(const char *cmd_str) { return proc; } +// Deletes stale object files from a target directory before compiling into it. +// Link commands use directory wildcards, so removed translation units can otherwise +// survive as old .obj files and keep unresolved references alive. +bool clean_object_directory(const char *obj_dir) { + const char *cmd = nob_temp_sprintf("if exist %s\\*.obj del /q %s\\*.obj", obj_dir, obj_dir); + return run_sync(cmd); +} + +// Cleans object directories that are linked through wildcards. +// The build is intentionally simple and does not maintain per-file dependency state, +// so stale objects are more dangerous than recompiling a few extra files. +bool clean_build_object_directories(void) { + const char *dirs[] = { + "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal", + "obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix", + "obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", + "obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget", + "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", + "obj\\doom", "obj\\video", "obj\\memtop", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", + "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread", + "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present" + }; + + for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) { + if (!clean_object_directory(dirs[i])) return false; + } + + if (!clean_object_directory(PCI_BUS_DRIVER.obj_dir)) return false; + for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) { + if (!clean_object_directory(STANDALONE_DRIVERS[i].obj_dir)) return false; + } + + return true; +} + // Creates an empty USB image the first time the build sees a missing usb.vhd. // The image is intentionally left unmounted and unpopulated after formatting so // callers can use it as removable FAT32 storage without the build overwriting it. @@ -111,6 +150,7 @@ int main(int argc, char **argv) { } if (!create_usb_vhd_if_missing()) return 1; create_layout(); + if (!clean_build_object_directories()) return 1; // Core images build sequentially because later images link against earlier import libraries. nob_log(NOB_INFO, "--- Building Core Subsystems ---"); @@ -196,6 +236,17 @@ int main(int argc, char **argv) { {"apps\\shell_native", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "shell_native", "/ENTRY:main"}, {"tests\\async_apc", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_apc", "/ENTRY:main"}, {"tests\\async_cancel", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_cancel", "/ENTRY:main"}, + {"tests\\mmap_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "mmap_smoke", "/ENTRY:main"}, + {"tests\\pcache_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_smoke", "/ENTRY:main"}, + {"tests\\pcache_pressure", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_pressure", "/ENTRY:main"}, + {"tests\\dirty_writeback", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "dirty_writeback", "/ENTRY:main"}, + {"tests\\shared_mmap", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "shared_mmap", "/ENTRY:main"}, + {"tests\\pcache_invalidate", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_invalidate", "/ENTRY:main"}, + {"tests\\pcache_mmap_read_reuse", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_mmap_read_reuse", "/ENTRY:main"}, + {"tests\\pcache_multithread", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_multithread", "/ENTRY:main"}, + {"tests\\surface_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_smoke", "/ENTRY:main"}, + {"tests\\surface_invalid_rect", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_invalid_rect", "/ENTRY:main"}, + {"tests\\surface_multithread_present", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_multithread_present", "/ENTRY:main"}, {"apps\\edit", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "edit", "/ENTRY:main"}, {"apps\\httpget", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "httpget", "/ENTRY:main"}, {"apps\\tcpserve", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "tcpserve", "/ENTRY:main"}, @@ -203,7 +254,8 @@ int main(int argc, char **argv) { {"apps\\resolve", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "resolve", "/ENTRY:main"}, {"apps\\ping", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "ping", "/ENTRY:main"}, {"apps\\doom", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "doom", "/ENTRY:main"}, - {"apps\\video", "bin\\win32.lib obj\\libc.obj code\\user\\apps\\video\\bink2w64.lib", "video", "/ENTRY:main"} + {"apps\\video", "bin\\syscall.lib obj\\libc.obj code\\user\\apps\\video\\bink2w64.lib", "video", "/ENTRY:main"}, + {"apps\\memtop", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "memtop", "/ENTRY:main"} }; for (size_t i = 0; i < NOB_ARRAY_LEN(apps); i++) { @@ -248,6 +300,7 @@ int main(int argc, char **argv) { {"code\\settings.ini", "Z:\\system\\settings.ini"}, {"bin\\init.exe", "Z:\\system\\init.exe"}, {"bin\\pci.sys", "Z:\\drivers\\pci.sys"}, {"bin\\ps2.sys", "Z:\\drivers\\ps2.sys"}, + {"bin\\serial.sys", "Z:\\drivers\\serial.sys"}, {"bin\\ata.sys", "Z:\\drivers\\ata.sys"}, {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"}, {"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"}, {"bin\\vmwaresvga.sys", "Z:\\drivers\\vmwaresvga.sys"}, {"bin\\lwip.sys", "Z:\\drivers\\lwip.sys"}, {"bin\\usb.sys", "Z:\\drivers\\usb.sys"}, @@ -260,9 +313,19 @@ int main(int argc, char **argv) { {"bin\\httpget.exe", "Z:\\apps\\httpget.exe"}, {"bin\\tcpserve.exe", "Z:\\apps\\tcpserve.exe"}, {"bin\\udp.exe", "Z:\\apps\\udp.exe"}, {"bin\\resolve.exe", "Z:\\apps\\resolve.exe"}, {"bin\\ping.exe", "Z:\\apps\\ping.exe"}, {"bin\\doom.exe", "Z:\\apps\\doom.exe"}, - {"bin\\video.exe", "Z:\\apps\\video.exe"}, + {"bin\\video.exe", "Z:\\apps\\video.exe"}, {"bin\\memtop.exe", "Z:\\apps\\memtop.exe"}, - {"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"}, {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"} + {"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"}, {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"}, + {"bin\\mmap_smoke.exe", "Y:\\apps\\mmap_smoke.exe"}, {"bin\\pcache_smoke.exe", "Y:\\apps\\pcache_smoke.exe"}, + {"bin\\pcache_pressure.exe", "Y:\\apps\\pcache_pressure.exe"}, + {"bin\\dirty_writeback.exe", "Y:\\apps\\dirty_writeback.exe"}, + {"bin\\shared_mmap.exe", "Y:\\apps\\shared_mmap.exe"}, + {"bin\\pcache_invalidate.exe", "Y:\\apps\\pcache_invalidate.exe"}, + {"bin\\pcache_mmap_read_reuse.exe", "Y:\\apps\\pcache_mmap_read_reuse.exe"}, + {"bin\\pcache_multithread.exe", "Y:\\apps\\pcache_multithread.exe"}, + {"bin\\surface_smoke.exe", "Y:\\apps\\surface_smoke.exe"}, + {"bin\\surface_invalid_rect.exe", "Y:\\apps\\surface_invalid_rect.exe"}, + {"bin\\surface_multithread_present.exe", "Y:\\apps\\surface_multithread_present.exe"} }; for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) nob_copy_file(copies[i][0], copies[i][1]); diff --git a/code/drivers/bus/pci/pci.c b/code/drivers/bus/pci/pci.c index 0f24a20..c8df4b5 100644 --- a/code/drivers/bus/pci/pci.c +++ b/code/drivers/bus/pci/pci.c @@ -13,6 +13,7 @@ #include #include #include +#include #include int _fltused; // MSVC floating-point helper anchor required by printf.obj @@ -540,7 +541,7 @@ __declspec(dllexport) KSTATUS PciMapBar(DEVICE_OBJECT* pdo, uint8_t index, uint6 return st; for (uint64_t off = 0; off < mapLength; off += PAGE_SIZE) { - st = MmMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmMapFailIfPresent); + st = MmPteMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmPteMapFailIfPresent); if (!KSUCCESS(st)) return st; } @@ -659,6 +660,28 @@ __declspec(dllexport) KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot) return STATUS_SUCCESS; } +// Enables a single MSI vector, allocating the nt-lite IRQ slot internally. +__declspec(dllexport) KSTATUS PciEnableMsiAuto(DEVICE_OBJECT* pdo, uint8_t* outIrqSlot) { + PCI_PDO_EXTENSION* ext; + if (!outIrqSlot || !PciIsPdo(pdo, &ext)) + return STATUS_INVALID_PARAMETER; + + if (!PciFindCapability(ext, PCI_CAP_ID_MSI)) + return STATUS_NOT_SUPPORTED; + + uint8_t irq = 0xFFu; + KSTATUS st = KeIrqAllocateMsiVector(&irq); + if (!KSUCCESS(st)) + return st; + + st = PciEnableMsi(pdo, irq); + if (!KSUCCESS(st)) + return st; + + *outIrqSlot = irq; + return STATUS_SUCCESS; +} + // Implements the driver Entry operation. KSTATUS DriverEntry(const HAL_API* hal) { if (!hal) return STATUS_INVALID_PARAMETER; diff --git a/code/drivers/bus/usb/usb.h b/code/drivers/bus/usb/usb.h index 3746a89..089db1f 100644 --- a/code/drivers/bus/usb/usb.h +++ b/code/drivers/bus/usb/usb.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #ifndef USB_DEBUG_VERBOSE diff --git a/code/drivers/bus/usb/usb_xhci.c b/code/drivers/bus/usb/usb_xhci.c index 0c394eb..847cab8 100644 --- a/code/drivers/bus/usb/usb_xhci.c +++ b/code/drivers/bus/usb/usb_xhci.c @@ -95,9 +95,9 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, // hid this because it had simple low memory; laptop firmware does not. PHYSICAL_ADDRESS phys = 0; if (pages == 1) { - phys = MmAllocateFrame(); + phys = MmPfnAllocate(MmPfnUseDma); } else { - phys = MmAllocateContiguous(pages); + phys = MmPfnAllocateContiguous(pages, MmPfnUseDma); } if (!phys) { @@ -1930,21 +1930,31 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { return STATUS_DEVICE_ERROR; } - st = PciGetInterrupt(xhci->Pdo, &xhci->Irq); - if (!KSUCCESS(st)) { - DbgPrint("usb: xHCI PciGetInterrupt failed st=0x%x\n", st); - return st; - } - if (xhci->Irq == 0xFFu) { - DbgPrint("usb: xHCI PCI %02x:%02x.%x has no INTx IRQ; MSI is not in pci.sys yet\n", - xhci->Bus, xhci->Dev, xhci->Fn); - return STATUS_NOT_SUPPORTED; + xhci->Irq = 0xFFu; + xhci->UsingMsi = false; + + st = PciEnableMsiAuto(xhci->Pdo, &xhci->Irq); + if (KSUCCESS(st)) { + xhci->UsingMsi = true; + DbgPrint("usb: xHCI using MSI IRQ slot %u\n", xhci->Irq); + } else { + KSTATUS intxStatus = PciGetInterrupt(xhci->Pdo, &xhci->Irq); + if (KSUCCESS(intxStatus) && xhci->Irq != 0xFFu) { + DbgPrint("usb: xHCI using legacy INTx IRQ %u\n", xhci->Irq); + } else { + xhci->Irq = 0xFFu; + DbgPrint("usb: xHCI no MSI/INTx interrupt available; using polled event-ring mode (msi=0x%x intx=0x%x)\n", + st, intxStatus); + } } - st = PciEnableDevice(xhci->Pdo, - PCI_ENABLE_MEMORY | - PCI_ENABLE_BUSMASTER | - PCI_ENABLE_INTX); + uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER; + if (xhci->UsingMsi || xhci->Irq == 0xFFu) + enableFlags |= PCI_DISABLE_INTX; + else + enableFlags |= PCI_ENABLE_INTX; + + st = PciEnableDevice(xhci->Pdo, enableFlags); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI PciEnableDevice failed st=0x%x\n", st); return st; @@ -2056,12 +2066,18 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { return st; } - KeIrqRegister(xhci->Irq, XhciIsr); - XhciEnableInterrupts(xhci); - xhci->InterruptsEnabled = true; + if (xhci->Irq != 0xFFu) { + KeIrqRegister(xhci->Irq, XhciIsr); + XhciEnableInterrupts(xhci); + xhci->InterruptsEnabled = true; + KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); + DbgPrint("usb: xHCI ISR registered on IRQ %u (%s); event ring handled by DPC\n", + xhci->Irq, xhci->UsingMsi ? "MSI" : "INTx"); + } else { + xhci->InterruptsEnabled = false; + DbgPrint("usb: xHCI started in polling mode; synchronous waits drain event ring\n"); + } xhci->Fdo->PnpState = DevicePnpStarted; - KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); - DbgPrint("usb: xHCI ISR registered on IRQ %u; event ring handled by DPC\n", xhci->Irq); st = UsbHubScheduleRootScan(&xhci->Hcd); if (!KSUCCESS(st)) diff --git a/code/drivers/display/vmwaresvga/vmwaresvga.c b/code/drivers/display/vmwaresvga/vmwaresvga.c index 180b8c2..3060af8 100644 --- a/code/drivers/display/vmwaresvga/vmwaresvga.c +++ b/code/drivers/display/vmwaresvga/vmwaresvga.c @@ -890,19 +890,13 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* result = syncSt; } - if (v->HardwareCursor && v->CursorVisible) - VmwSvgaCursorCommit(v); - return result; } // Implements the display Sync operation. static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) { VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device); - KSTATUS st = VmwSvgaSyncDevice(v); - if (v && v->HardwareCursor && v->CursorVisible) - VmwSvgaCursorCommit(v); - return st; + return VmwSvgaSyncDevice(v); } // Implements the display Set Cursor Image operation. diff --git a/code/drivers/drivers.ini b/code/drivers/drivers.ini index a183c58..57ef400 100644 --- a/code/drivers/drivers.ini +++ b/code/drivers/drivers.ini @@ -1,6 +1,7 @@ [boot] pci.sys ps2.sys +serial.sys ahci.sys ata.sys usb.sys diff --git a/code/drivers/fs/fat32/fat32.c b/code/drivers/fs/fat32/fat32.c index ed99abc..836901c 100644 --- a/code/drivers/fs/fat32/fat32.c +++ b/code/drivers/fs/fat32/fat32.c @@ -128,12 +128,79 @@ typedef struct { static FAT_CACHE_ENTRY gFatCache[FAT_CACHE_SLOTS]; +#define FAT32_PATH_CACHE_SLOTS 64u +#define FAT32_PATH_CACHE_MAX 512u +#define FAT32_CHAIN_CACHE_SLOTS 16u +#define FAT32_WRITE_RUN_MAX_SECTORS 128u +#define FAT32_READ_RUN_MAX_SECTORS 512u + +typedef struct FAT32_PATH_CACHE_ENTRY { + bool Valid; + uint64_t Generation; + uint32_t Hash; + size_t PathLength; + char Path[FAT32_PATH_CACHE_MAX]; + uint32_t Cluster; + uint32_t Size; + uint8_t Attr; + uint64_t DirentLba; + uint32_t DirentOffset; +} FAT32_PATH_CACHE_ENTRY; + +typedef struct FAT32_CHAIN_CACHE_ENTRY { + bool Valid; + uint32_t LogicalStart; + uint32_t PhysicalStart; + uint32_t Length; + uint32_t NextCluster; +} FAT32_CHAIN_CACHE_ENTRY; + +// The FAT sector cache is global and direct-mapped. mmap-backed video can fault +// file pages from multiple Bink worker threads at the same time; without +// serialization one thread can evict/reload a cache slot while another thread is +// still reading a FAT entry from that slot, corrupting cluster-chain traversal. +// Use a sleepable mutex rather than a spinlock because cache misses perform disk +// I/O. The static mutex avoids allocating an object from the paging path. +static MUTEX gFatCacheMutex; +static SPINLOCK gFatCacheMutexInitLock = SPINLOCK_INIT; +static int gFatCacheMutexReady; + +static void FatCacheEnsureMutex(void) { + if (gFatCacheMutexReady) + return; + + uintptr_t flags = KeAcquireSpinlock(&gFatCacheMutexInitLock); + if (!gFatCacheMutexReady) { + memset(&gFatCacheMutex, 0, sizeof(gFatCacheMutex)); + gFatCacheMutex.Header.Type = ObjectTypeMutex; + gFatCacheMutex.Header.RefCount = 1; + gFatCacheMutex.Header.Destroy = (void*)0; + gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT; + gFatCacheMutex.Owner = (struct THREAD*)0; + LIST_INIT(&gFatCacheMutex.WaitList); + gFatCacheMutexReady = 1; + } + KeReleaseSpinlock(&gFatCacheMutexInitLock, flags); +} + +static int FatCacheAcquire(void) { + if (!KeCanWait()) + return 0; + FatCacheEnsureMutex(); + return KSUCCESS(KeAcquireMutex(&gFatCacheMutex)); +} + +static void FatCacheRelease(int held) { + if (held) + KeReleaseMutex(&gFatCacheMutex); +} + // Raw (non-cached) sector I/O — forward declarations. static KSTATUS ReadSectorsRaw (FAT32_VOL *v, uint64_t lba, uint32_t count, void *buf); static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, const void *buf); // Read one FAT sector through the cache. -// Returns a pointer into the cache buffer (valid until the next eviction). +// Caller must hold the FAT cache mutex while using the returned pointer. static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) { uint32_t slot = (uint32_t)(lba % FAT_CACHE_SLOTS); FAT_CACHE_ENTRY *e = &gFatCache[slot]; @@ -170,11 +237,13 @@ static void FatCacheMarkDirty(FAT32_VOL *v, uint64_t lba) { // Implements the fat Cache Invalidate operation. static void FatCacheInvalidate(void) { + int held = FatCacheAcquire(); for (int i = 0; i < FAT_CACHE_SLOTS; i++) { gFatCache[i].vol = (FAT32_VOL*)0; gFatCache[i].valid = false; gFatCache[i].dirty = false; } + FatCacheRelease(held); } // FlushFatCache — write every dirty slot to all FAT copies, clear dirty flag. @@ -196,6 +265,10 @@ struct FAT32_VOL_S { uint64_t DataStart; // absolute LBA of cluster 2 uint32_t TotalClusters; // usable data clusters uint32_t NextFreeHint; // fix #3: start cluster for next free-scan + SPINLOCK PathCacheLock; + uint64_t PathCacheGeneration; + IO_FILESYSTEM_STATS Stats; + FAT32_PATH_CACHE_ENTRY PathCache[FAT32_PATH_CACHE_SLOTS]; }; typedef struct { uint32_t StartCluster; @@ -203,6 +276,9 @@ typedef struct { FAT32_VOL *Vol; uint64_t DirentLba; uint32_t DirentOffset; + SPINLOCK ChainCacheLock; + uint32_t ChainCacheClock; + FAT32_CHAIN_CACHE_ENTRY ChainCache[FAT32_CHAIN_CACHE_SLOTS]; } FAT32_FILE; // Raw sector I/O (bypasses FAT cache; used for file data and cache eviction). @@ -231,6 +307,7 @@ static uint64_t ClusterToLba(FAT32_VOL *v, uint32_t cluster) { // FlushFatCache — needs full FAT32_VOL definition. static void FlushFatCache(FAT32_VOL *v) { uint64_t fat0Start = (uint64_t)v->PartitionLba + v->ReservedSectors; + int held = FatCacheAcquire(); for (int i = 0; i < FAT_CACHE_SLOTS; i++) { FAT_CACHE_ENTRY *e = &gFatCache[i]; @@ -246,6 +323,8 @@ static void FlushFatCache(FAT32_VOL *v) { } e->dirty = false; } + + FatCacheRelease(held); } // FAT read / write (through cache). @@ -256,9 +335,14 @@ static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) { + fatOffset / v->BytesPerSector; uint32_t offInSec = fatOffset % v->BytesPerSector; + int held = FatCacheAcquire(); uint8_t *sec; - if (!KSUCCESS(FatCacheRead(v, sector, &sec))) return 0; - return *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF; + KSTATUS s = FatCacheRead(v, sector, &sec); + uint32_t value = 0; + if (KSUCCESS(s)) + value = *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF; + FatCacheRelease(held); + return value; } // Updates fat Entry. @@ -268,19 +352,24 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) { + fatOffset / v->BytesPerSector; uint32_t offInSec = fatOffset % v->BytesPerSector; + int held = FatCacheAcquire(); uint8_t *sec; KSTATUS s = FatCacheRead(v, sector, &sec); - if (!KSUCCESS(s)) return s; + if (!KSUCCESS(s)) { + FatCacheRelease(held); + return s; + } uint32_t existing = *(uint32_t *)(sec + offInSec); *(uint32_t *)(sec + offInSec) = (existing & 0xF0000000u) | (value & 0x0FFFFFFFu); FatCacheMarkDirty(v, sector); + FatCacheRelease(held); return STATUS_SUCCESS; } // Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan). -static uint32_t AllocCluster(FAT32_VOL *v) { +static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) { if (v->TotalClusters == 0) return 0; uint32_t total = v->TotalClusters; @@ -295,12 +384,18 @@ static uint32_t AllocCluster(FAT32_VOL *v) { if (!KSUCCESS(WriteFatEntry(v, c, FAT32_EOC_MARK))) return 0; v->NextFreeHint = c + 1; - // Zero the cluster (required for directory correctness). - uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster); - if (zero) { - memset(zero, 0, v->BytesPerCluster); - WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero); - MmFreePool(zero); + // Normal file growth zeroes newly allocated clusters so old + // disk contents are never exposed. Swap/pagefile creation is + // the one exception: every allocated slot is overwritten before + // it can ever be read back, so skipping these writes avoids a + // slow boot-time full-pagefile zero pass. + if (zeroCluster) { + uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster); + if (zero) { + memset(zero, 0, v->BytesPerCluster); + WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero); + MmFreePool(zero); + } } return c; } @@ -309,14 +404,23 @@ static uint32_t AllocCluster(FAT32_VOL *v) { return 0; } -// Appends cluster. -static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) { - uint32_t newCluster = AllocCluster(v); + +static uint32_t AllocCluster(FAT32_VOL *v) { + return AllocClusterEx(v, 1); +} + +static uint32_t AppendClusterEx(FAT32_VOL *v, uint32_t lastCluster, int zeroCluster) { + uint32_t newCluster = AllocClusterEx(v, zeroCluster); if (!newCluster) return 0; if (!KSUCCESS(WriteFatEntry(v, lastCluster, newCluster))) return 0; return newCluster; } +// Appends cluster. +static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) { + return AppendClusterEx(v, lastCluster, 1); +} + // Returns chain End. static uint32_t GetChainEnd(FAT32_VOL *v, uint32_t start, uint32_t *outCount) { uint32_t cluster = start, prev = start, count = 0; @@ -973,6 +1077,302 @@ static int MatchName83V(const char *dirName, const char *dirExt, return 1; } +// Returns the path cache canonical byte used for hashes and comparisons. +static char Fat32PathCacheChar(char c) { + if (c == '/') return '\\'; + if (c >= 'A' && c <= 'Z') return (char)(c + 32); + return c; +} + +// Hashes a drive-relative path using the same case/separator rules as lookup. +static uint32_t Fat32PathCacheHash(SYS_STRING_VIEW path) { + uint32_t h = 2166136261u; + for (size_t i = 0; i < path.Length; i++) { + h ^= (uint8_t)Fat32PathCacheChar(path.Data[i]); + h *= 16777619u; + } + return h ? h : 1u; +} + +// Tests whether the cached path names the same FAT path as the caller's view. +static bool Fat32PathCachePathEquals(const FAT32_PATH_CACHE_ENTRY* e, SYS_STRING_VIEW path) { + if (!e || e->PathLength != path.Length) return false; + for (size_t i = 0; i < path.Length; i++) { + if (Fat32PathCacheChar(e->Path[i]) != Fat32PathCacheChar(path.Data[i])) + return false; + } + return true; +} + +// Adds directory search counters without holding the lock during disk I/O. +static void Fat32PathCacheAddDirectoryStats(FAT32_VOL* v, uint64_t clusters, uint64_t entries) { + if (!v) return; + uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock); + v->Stats.DirectorySearches++; + v->Stats.DirectoryClustersRead += clusters; + v->Stats.DirectoryEntriesScanned += entries; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + +// Looks up a positive path-cache entry for a full drive-relative path. +static bool Fat32PathCacheLookup(FAT32_VOL* v, SYS_STRING_VIEW path, + uint32_t* outCluster, uint32_t* outSize, + uint8_t* outAttr, uint64_t* outDirentLba, + uint32_t* outDirentOffset) { + if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX) + return false; + + uint32_t hash = Fat32PathCacheHash(path); + uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS; + bool hit = false; + + uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock); + v->Stats.PathLookups++; + FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot]; + if (e->Valid && e->Generation == v->PathCacheGeneration && + e->Hash == hash && Fat32PathCachePathEquals(e, path)) { + if (outCluster) *outCluster = e->Cluster; + if (outSize) *outSize = e->Size; + if (outAttr) *outAttr = e->Attr; + if (outDirentLba) *outDirentLba = e->DirentLba; + if (outDirentOffset) *outDirentOffset = e->DirentOffset; + v->Stats.PathCacheHits++; + hit = true; + } else { + v->Stats.PathCacheMisses++; + } + KeReleaseSpinlock(&v->PathCacheLock, flags); + return hit; +} + +// Stores a positive path lookup in the volume-local path cache. +static void Fat32PathCacheStore(FAT32_VOL* v, SYS_STRING_VIEW path, + uint32_t cluster, uint32_t size, uint8_t attr, + uint64_t direntLba, uint32_t direntOffset) { + if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX) + return; + + uint32_t hash = Fat32PathCacheHash(path); + uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS; + + uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock); + FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot]; + memset(e, 0, sizeof(*e)); + e->Valid = true; + e->Generation = v->PathCacheGeneration; + e->Hash = hash; + e->PathLength = path.Length; + memcpy(e->Path, path.Data, path.Length); + e->Path[path.Length] = '\0'; + e->Cluster = cluster; + e->Size = size; + e->Attr = attr; + e->DirentLba = direntLba; + e->DirentOffset = direntOffset; + v->Stats.PathCacheStores++; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + +// Invalidates every positive path-cache entry after a directory or size mutation. +static void Fat32PathCacheInvalidate(FAT32_VOL* v) { + if (!v) return; + uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock); + v->PathCacheGeneration++; + if (v->PathCacheGeneration == 0) + v->PathCacheGeneration = 1; + v->Stats.PathCacheInvalidations++; + v->Stats.PathCacheGeneration = v->PathCacheGeneration; + for (uint32_t i = 0; i < FAT32_PATH_CACHE_SLOTS; i++) + v->PathCache[i].Valid = false; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + +// Returns the current filesystem cache and traversal counters. +void Fat32QueryStats(IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out) { + if (!fs || !out) return; + FAT32_VOL* v = (FAT32_VOL*)fs->DriverCtx; + if (!v) { memset(out, 0, sizeof(*out)); return; } + + uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock); + *out = v->Stats; + out->PathCacheGeneration = v->PathCacheGeneration; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + + +// Adds FAT-chain traversal counters shared by the read path helpers. +static void Fat32StatsAddChainWalk(FAT32_VOL* v, uint64_t steps) { + if (!v || !steps) return; + uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock); + v->Stats.FatChainWalkSteps += steps; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + +// Records one data read span and its physically contiguous cluster merges. +static void Fat32StatsAddReadRun(FAT32_VOL* v, uint32_t sectors, uint32_t merges) { + if (!v || !sectors) return; + uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock); + v->Stats.DataReadRuns++; + v->Stats.DataReadSectors += sectors; + v->Stats.DataReadRunMerges += merges; + if (v->Stats.MaxDataReadSpanSectors < sectors) + v->Stats.MaxDataReadSpanSectors = sectors; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + +// Initializes the per-open cluster-chain cache for a FAT file object. +static void Fat32FileInitChainCache(FAT32_FILE* f) { + if (!f) return; + f->ChainCacheLock = (SPINLOCK)SPINLOCK_INIT; + f->ChainCacheClock = 0; + memset(f->ChainCache, 0, sizeof(f->ChainCache)); +} + +// Invalidates cached FAT extents after the file's cluster chain changes. +static void Fat32FileInvalidateChainCache(FAT32_FILE* f) { + if (!f) return; + uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock); + memset(f->ChainCache, 0, sizeof(f->ChainCache)); + f->ChainCacheClock = 0; + KeReleaseSpinlock(&f->ChainCacheLock, flags); + + FAT32_VOL* v = f->Vol; + if (!v) return; + flags = KeAcquireSpinlock(&v->PathCacheLock); + v->Stats.FatChainCacheInvalidations++; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + +// Locates the best cached starting point for a logical cluster index. +static bool Fat32ChainCacheLookupStart(FAT32_FILE* f, uint32_t logicalIndex, + uint32_t* outLogical, uint32_t* outCluster) { + if (!f || !outLogical || !outCluster) + return false; + + bool hit = false; + uint32_t bestLogical = 0; + uint32_t bestCluster = 0; + + uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock); + for (uint32_t i = 0; i < FAT32_CHAIN_CACHE_SLOTS; i++) { + FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[i]; + if (!e->Valid || e->Length == 0) + continue; + + uint32_t end = e->LogicalStart + e->Length; + if (logicalIndex >= e->LogicalStart && logicalIndex < end) { + *outLogical = logicalIndex; + *outCluster = e->PhysicalStart + (logicalIndex - e->LogicalStart); + hit = true; + break; + } + + if (end <= logicalIndex && e->NextCluster >= 2 && e->NextCluster < FAT32_EOC) { + if (!hit || end > bestLogical) { + bestLogical = end; + bestCluster = e->NextCluster; + hit = true; + } + } + } + + if (hit && bestCluster) { + *outLogical = bestLogical; + *outCluster = bestCluster; + } + KeReleaseSpinlock(&f->ChainCacheLock, flags); + + FAT32_VOL* v = f->Vol; + if (v) { + flags = KeAcquireSpinlock(&v->PathCacheLock); + v->Stats.FatChainCacheLookups++; + if (hit) v->Stats.FatChainCacheHits++; + else v->Stats.FatChainCacheMisses++; + KeReleaseSpinlock(&v->PathCacheLock, flags); + } + return hit; +} + +// Stores a physically contiguous FAT extent in the per-open chain cache. +static void Fat32ChainCacheStore(FAT32_FILE* f, uint32_t logicalStart, + uint32_t physicalStart, uint32_t length, + uint32_t nextCluster) { + if (!f || physicalStart < 2 || physicalStart >= FAT32_EOC || length == 0) + return; + + uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock); + uint32_t slot = f->ChainCacheClock++ % FAT32_CHAIN_CACHE_SLOTS; + FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[slot]; + e->Valid = true; + e->LogicalStart = logicalStart; + e->PhysicalStart = physicalStart; + e->Length = length; + e->NextCluster = nextCluster; + KeReleaseSpinlock(&f->ChainCacheLock, flags); + + FAT32_VOL* v = f->Vol; + if (!v) return; + flags = KeAcquireSpinlock(&v->PathCacheLock); + v->Stats.FatChainCacheStores++; + KeReleaseSpinlock(&v->PathCacheLock, flags); +} + +// Resolves a logical file cluster through the cached FAT-chain extent map. +static uint32_t Fat32ResolveFileCluster(FAT32_FILE* f, uint32_t logicalIndex) { + if (!f || f->StartCluster < 2 || f->StartCluster >= FAT32_EOC) + return 0; + if (logicalIndex == 0) + return f->StartCluster; + + uint32_t currentLogical = 0; + uint32_t cluster = f->StartCluster; + if (!Fat32ChainCacheLookupStart(f, logicalIndex, ¤tLogical, &cluster)) { + currentLogical = 0; + cluster = f->StartCluster; + } + + uint64_t steps = 0; + while (currentLogical < logicalIndex && cluster >= 2 && cluster < FAT32_EOC) { + cluster = FatEntry(f->Vol, cluster); + currentLogical++; + steps++; + } + Fat32StatsAddChainWalk(f->Vol, steps); + return cluster; +} + +// Builds and caches one physically contiguous read run from a starting cluster. +static bool Fat32BuildReadRun(FAT32_FILE* f, uint32_t logicalIndex, + uint32_t startCluster, uint32_t maxSectors, + uint32_t* outRunSectors, uint32_t* outNextCluster) { + if (!f || !outRunSectors || !outNextCluster) + return false; + FAT32_VOL* v = f->Vol; + if (!v || startCluster < 2 || startCluster >= FAT32_EOC) + return false; + + uint32_t runSectors = v->SectorsPerCluster; + uint32_t runClusters = 1; + uint32_t last = startCluster; + uint32_t next = FatEntry(v, last); + uint64_t steps = 1; + + while (next == last + 1 && next < FAT32_EOC && + runSectors + v->SectorsPerCluster <= maxSectors) { + last = next; + runClusters++; + runSectors += v->SectorsPerCluster; + next = FatEntry(v, last); + steps++; + } + + Fat32StatsAddChainWalk(v, steps); + Fat32ChainCacheStore(f, logicalIndex, startCluster, runClusters, next); + *outRunSectors = runSectors; + *outNextCluster = next; + return true; +} + static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name, uint32_t *outCluster, uint32_t *outSize, uint8_t *outAttr, uint64_t *outDirentLba, uint32_t *outDirentOffset) { @@ -986,15 +1386,20 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name, int allowShortMatch = FatNameIsStructurallyShort83V(name); Make83NameV(name, wantName83, wantExt83); uint32_t cluster = dirCluster; + uint64_t clustersRead = 0; + uint64_t entriesScanned = 0; + int found = 0; while (cluster >= 2 && cluster < FAT32_EOC) { uint64_t clusterLba = ClusterToLba(v, cluster); if (!KSUCCESS(ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf))) break; + clustersRead++; uint32_t entries = v->BytesPerCluster / sizeof(DIRENT); DIRENT *dir = (DIRENT *)clusterBuf; for (uint32_t i = 0; i < entries; i++) { + entriesScanned++; if (dir[i].Name[0] == 0x00) goto done; if ((uint8_t)dir[i].Name[0] == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; } if (dir[i].Attr == ATTR_LFN) { LfnAppend(lfn, (LFN_ENTRY *)&dir[i]); hasLfn = 1; continue; } @@ -1016,15 +1421,16 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name, if (outDirentLba) *outDirentLba = clusterLba + eb / v->BytesPerSector; if (outDirentOffset) *outDirentOffset = (uint32_t)(eb % v->BytesPerSector); } - MmFreePool(clusterBuf); - return 1; + found = 1; + goto done; } } cluster = FatEntry(v, cluster); } done: + Fat32PathCacheAddDirectoryStats(v, clustersRead, entriesScanned); MmFreePool(clusterBuf); - return 0; + return found; } static int Fat32PathNextComponent(SYS_STRING_VIEW path, size_t* cursor, @@ -1058,6 +1464,10 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path, if (outDirentLba) *outDirentLba = 0; if (outDirentOffset) *outDirentOffset = 0; + if (Fat32PathCacheLookup(v, path, outCluster, outSize, &attr, + outDirentLba, outDirentOffset)) + return STATUS_SUCCESS; + size_t cursor = 0; SYS_STRING_VIEW component; while (Fat32PathNextComponent(path, &cursor, &component)) { @@ -1078,6 +1488,9 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path, *outCluster = cluster; *outSize = size; + Fat32PathCacheStore(v, path, cluster, size, attr, + outDirentLba ? *outDirentLba : 0, + outDirentOffset ? *outDirentOffset : 0); return STATUS_SUCCESS; } @@ -1220,6 +1633,7 @@ KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { if (!f) return STATUS_NO_MEMORY; f->StartCluster = cluster; f->FileSize = size; f->Vol = v; f->DirentLba = direntLba; f->DirentOffset = direntOffset; + Fat32FileInitChainCache(f); *fileCtx = f; return STATUS_SUCCESS; } @@ -1233,15 +1647,9 @@ void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) { } // Read (fix #4: contiguous-run coalescing) -// We peek ahead in the FAT chain: if cluster N+1 == cluster N physically, -// extend the current run instead of issuing a new ReadSectors call. -// Run length is capped at MAX_RUN_SECTORS to stay within the AHCI bounce -// buffer (128 sectors = 64 KB). - -// FatEntry() calls in the peek loop are nearly free because they hit the -// FAT sector cache (fix #1). - -#define MAX_RUN_SECTORS 128u +// Cluster-chain lookup is cached per open file, and data spans are allowed to +// grow beyond the AHCI bounce size because the device layer now splits large +// sector requests for the concrete controller. KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, @@ -1257,14 +1665,12 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, if (size > (size_t)(f->FileSize - offset)) size = (size_t)(f->FileSize - offset); size_t wanted = size; - // Allocate only as much bounce space as this read can actually use. - // The old code allocated 64 KiB and could read a 64 KiB contiguous run even - // for tiny random reads; Doom does lots of seek/read pairs for small lumps. uint32_t initialOffsetInCluster = (uint32_t)(offset % v->BytesPerCluster); uint64_t initialBytesCovered = (uint64_t)initialOffsetInCluster + (uint64_t)size; uint32_t maxUsefulSectors = (uint32_t)((initialBytesCovered + 511u) / 512u); if (maxUsefulSectors < v->SectorsPerCluster) maxUsefulSectors = v->SectorsPerCluster; - if (maxUsefulSectors > MAX_RUN_SECTORS) maxUsefulSectors = MAX_RUN_SECTORS; + if (maxUsefulSectors > FAT32_READ_RUN_MAX_SECTORS) + maxUsefulSectors = FAT32_READ_RUN_MAX_SECTORS; uint8_t *runBuf = (uint8_t *)MmAllocatePool((size_t)maxUsefulSectors * 512u); if (!runBuf) return STATUS_NO_MEMORY; @@ -1272,35 +1678,27 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, size_t totalRead = 0; uint8_t *out = (uint8_t *)buf; KSTATUS readStatus = STATUS_SUCCESS; - uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster); + uint32_t logicalCluster = (uint32_t)(offset / v->BytesPerCluster); uint32_t offsetInCluster = (uint32_t)(offset % v->BytesPerCluster); - uint32_t cluster = f->StartCluster; - - for (uint32_t i = 0; i < skipClusters && cluster < FAT32_EOC; i++) - cluster = FatEntry(v, cluster); + uint32_t cluster = Fat32ResolveFileCluster(f, logicalCluster); while (size > 0 && cluster >= 2 && cluster < FAT32_EOC) { - // Build the longest physically-contiguous run that can contribute to - // this caller's remaining read. Do not prefetch an unrelated 64 KiB run - // for a tiny random read. uint64_t bytesCovered = (uint64_t)offsetInCluster + (uint64_t)size; uint32_t usefulSectors = (uint32_t)((bytesCovered + 511u) / 512u); if (usefulSectors < v->SectorsPerCluster) usefulSectors = v->SectorsPerCluster; - if (usefulSectors > MAX_RUN_SECTORS) usefulSectors = MAX_RUN_SECTORS; + if (usefulSectors > FAT32_READ_RUN_MAX_SECTORS) + usefulSectors = FAT32_READ_RUN_MAX_SECTORS; if (usefulSectors > maxUsefulSectors) usefulSectors = maxUsefulSectors; - uint32_t runStart = cluster; - uint32_t runSectors = v->SectorsPerCluster; - uint32_t next = FatEntry(v, cluster); - - while (next == cluster + 1 && next < FAT32_EOC - && runSectors + v->SectorsPerCluster <= usefulSectors) { - cluster = next; - next = FatEntry(v, cluster); - runSectors += v->SectorsPerCluster; + uint32_t runSectors = 0; + uint32_t next = 0; + if (!Fat32BuildReadRun(f, logicalCluster, cluster, usefulSectors, + &runSectors, &next)) { + readStatus = STATUS_DEVICE_ERROR; + break; } - KSTATUS rs = ReadSectors(v, ClusterToLba(v, runStart), runSectors, runBuf); + KSTATUS rs = ReadSectors(v, ClusterToLba(v, cluster), runSectors, runBuf); if (!KSUCCESS(rs)) { readStatus = rs; break; @@ -1309,12 +1707,16 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, uint32_t runBytes = runSectors * 512u; size_t avail = runBytes - offsetInCluster; size_t copy = avail < size ? avail : size; + uint32_t runClusters = runSectors / v->SectorsPerCluster; + uint32_t merges = runClusters ? runClusters - 1u : 0; + Fat32StatsAddReadRun(v, runSectors, merges); memcpy(out, runBuf + offsetInCluster, copy); out += copy; totalRead += copy; size -= copy; offsetInCluster = 0; + logicalCluster += runClusters; cluster = next; } @@ -1345,6 +1747,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, FAT32_FILE *f = (FAT32_FILE *)fileCtx; FAT32_VOL *v = f->Vol; const uint8_t *src = (const uint8_t *)buf; + Fat32FileInvalidateChainCache(f); // Ensure at least one cluster exists. if (f->StartCluster < 2) { @@ -1368,7 +1771,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, cluster = next; } - uint8_t *runBuf = (uint8_t *)MmAllocatePool(MAX_RUN_SECTORS * 512u); + uint8_t *runBuf = (uint8_t *)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u); if (!runBuf) { *bytesWritten = 0; return STATUS_NO_MEMORY; } size_t written = 0, remaining = size; @@ -1382,7 +1785,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, if (offsetInCluster == 0) { while (next == cluster + 1 && next < FAT32_EOC - && runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) { + && runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) { cluster = next; next = FatEntry(v, cluster); runSectors += v->SectorsPerCluster; @@ -1439,7 +1842,9 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, u.Flags |= FAT32_DIRENT_UPDATE_SIZE; u.Size = f->FileSize; } - DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u); + KSTATUS ms = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u); + if (KSUCCESS(ms)) + Fat32PathCacheInvalidate(v); } *bytesWritten = written; @@ -1447,8 +1852,6 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, } // Create. -KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize); - // Creates the requested object. KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0) @@ -1483,7 +1886,7 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { // Create means overwrite by default for regular files. Reuse the // existing directory entry, drop the old cluster chain, and return // an open zero-length file instead of allocating a duplicate name. - KSTATUS ts = Fat32Truncate(fs, f, 0); + KSTATUS ts = Fat32Truncate(fs, f, 0, 0); if (!KSUCCESS(ts)) { MmFreePool(f); return ts; } *fileCtx = f; return STATUS_SUCCESS; @@ -1558,21 +1961,27 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { if (!f) return STATUS_NO_MEMORY; f->StartCluster = 0; f->FileSize = 0; f->Vol = v; f->DirentLba = direntLba; f->DirentOffset = direntOffset; + Fat32FileInitChainCache(f); + Fat32PathCacheInvalidate(v); *fileCtx = f; return STATUS_SUCCESS; } // Truncate. -KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) { +KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags) { (void)fs; FAT32_FILE *f = (FAT32_FILE *)fileCtx; FAT32_VOL *v = f->Vol; + Fat32FileInvalidateChainCache(f); + int zeroNewClusters = (fileFlags & IO_FILE_FLAG_SKIP_ZERO_EXTEND) ? 0 : 1; if ((uint32_t)newSize == f->FileSize) { FAT32_DIRENT_UPDATE u; memset(&u, 0, sizeof(u)); FatQueryEncodedTime(&u.Timestamp); u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME; - return DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u); + KSTATUS s = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u); + if (KSUCCESS(s)) Fat32PathCacheInvalidate(v); + return s; } if (newSize == 0) { @@ -1584,6 +1993,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) { f->FileSize = 0; KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, 0); FlushFatCache(v); + if (KSUCCESS(s)) Fat32PathCacheInvalidate(v); return s; } @@ -1604,7 +2014,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) { ? (f->FileSize + v->BytesPerCluster - 1) / v->BytesPerCluster : 0; uint32_t lastCluster = 0; if (f->StartCluster < 2) { - uint32_t first = AllocCluster(v); + uint32_t first = AllocClusterEx(v, zeroNewClusters); if (!first) return STATUS_NO_MEMORY; f->StartCluster = first; DirentUpdateFirstCluster(v, f->DirentLba, f->DirentOffset, first); @@ -1613,7 +2023,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) { lastCluster = GetChainEnd(v, f->StartCluster, NULL); } for (uint32_t i = curClusters; i < newClusters; i++) { - uint32_t next = AppendCluster(v, lastCluster); + uint32_t next = AppendClusterEx(v, lastCluster, zeroNewClusters); if (!next) return STATUS_NO_MEMORY; lastCluster = next; } @@ -1623,6 +2033,7 @@ update_size: f->FileSize = (uint32_t)newSize; { KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, (uint32_t)newSize); FlushFatCache(v); + if (KSUCCESS(s)) Fat32PathCacheInvalidate(v); return s; } } @@ -1646,6 +2057,7 @@ KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) { if (cluster >= 2) FreeClusterChain(v, cluster); FlushFatCache(v); + Fat32PathCacheInvalidate(v); return STATUS_SUCCESS; } @@ -1654,6 +2066,75 @@ uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) { (void)fs; return ((FAT32_FILE *)fileCtx)->FileSize; } +// Mixes FAT directory metadata into a per-file content generation. +static uint64_t Fat32DirentGeneration(const DIRENT *d) { + if (!d) + return 0; + + uint32_t firstCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo; + uint64_t generation = ((uint64_t)d->WrtDate << 48) | + ((uint64_t)d->WrtTime << 32) | + ((uint64_t)d->LastAccDate << 16) | + (uint64_t)d->Attr; + generation ^= ((uint64_t)d->CrtDate << 40); + generation ^= ((uint64_t)d->CrtTime << 24); + generation ^= ((uint64_t)d->CrtTimeTenth << 16); + generation ^= ((uint64_t)firstCluster << 1); + generation ^= ((uint64_t)d->FileSize << 17); + return generation ? generation : 1; +} + +// Returns a stable FAT file identity for cross-open image-section caches. +KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out) { + (void)fs; + if (!fileCtx || !out) + return STATUS_INVALID_PARAMETER; + + FAT32_FILE *f = (FAT32_FILE *)fileCtx; + FAT32_VOL *v = f->Vol; + if (!v) + return STATUS_INVALID_PARAMETER; + + uint32_t startCluster = f->StartCluster; + uint32_t fileSize = f->FileSize; + uint64_t generation = ((uint64_t)startCluster << 1) ^ + ((uint64_t)fileSize << 17); + + if (f->DirentLba != 0) { + uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector); + if (!sec) + return STATUS_NO_MEMORY; + + KSTATUS s = ReadSectors(v, f->DirentLba, 1, sec); + if (!KSUCCESS(s)) { + MmFreePool(sec); + return s; + } + + DIRENT *d = (DIRENT *)(sec + f->DirentOffset); + if ((uint8_t)d->Name[0] == 0x00 || (uint8_t)d->Name[0] == 0xE5) { + MmFreePool(sec); + return STATUS_NOT_FOUND; + } + + startCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo; + fileSize = d->FileSize; + generation = Fat32DirentGeneration(d); + MmFreePool(sec); + } + + out->VolumeId = ((uint64_t)v->DriveIndex << 56) ^ + ((uint64_t)v->PartitionLba << 8) ^ + ((uint64_t)v->RootCluster << 1) ^ + 0x4641543332494445ULL; + out->FileId = f->DirentLba ^ + ((uint64_t)f->DirentOffset << 32) ^ + ((uint64_t)startCluster << 1); + out->Generation = generation ? generation : 1; + out->Size = fileSize; + return STATUS_SUCCESS; +} + // Directory enumeration (fix #2: cluster buffer cached in context) // Previously: allocate + ReadSectors + return 1 entry + free, every call. // Now: buffer lives in FAT32_DIR_CTX; ReadSectors fires only on cluster @@ -1757,8 +2238,7 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) { } ctx->CurCluster = FatEntry(v, ctx->CurCluster); - ctx->EntryIndex = 0; ctx->HasLfn = 0; - memset(ctx->LfnBuf, 0, sizeof(ctx->LfnBuf)); + ctx->EntryIndex = 0; } return STATUS_NO_MORE_FILES; @@ -1962,6 +2442,7 @@ typedef struct FAT32_ASYNC_RW { uint64_t Offset; uint32_t OffsetInCluster; uint32_t Cluster; + uint32_t LogicalCluster; size_t Remaining; size_t Done; size_t Requested; @@ -1972,6 +2453,7 @@ typedef struct FAT32_ASYNC_RW { uint32_t CurrentNext; uint32_t CurrentLast; + uint32_t CurrentClusters; uint32_t CurrentOffset; size_t CurrentCopy; @@ -2088,7 +2570,13 @@ static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) { ctx->Done += ctx->CurrentCopy; ctx->Remaining -= ctx->CurrentCopy; ctx->OffsetInCluster = 0; + ctx->LogicalCluster += ctx->CurrentClusters; ctx->Cluster = ctx->CurrentNext; + if (ctx->CurrentClusters) { + uint32_t sectors = ctx->CurrentClusters * ctx->Vol->SectorsPerCluster; + uint32_t merges = ctx->CurrentClusters - 1u; + Fat32StatsAddReadRun(ctx->Vol, sectors, merges); + } Fat32AsyncQueueContinuation(ctx); } @@ -2109,15 +2597,12 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) { FAT32_VOL* v = ctx->Vol; uint32_t runStart = ctx->Cluster; - uint32_t runSectors = v->SectorsPerCluster; - uint32_t last = ctx->Cluster; - uint32_t next = FatEntry(v, last); - - while (next == last + 1 && next < FAT32_EOC && - runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) { - last = next; - next = FatEntry(v, last); - runSectors += v->SectorsPerCluster; + uint32_t runSectors = 0; + uint32_t next = 0; + if (!Fat32BuildReadRun(ctx->File, ctx->LogicalCluster, ctx->Cluster, + FAT32_READ_RUN_MAX_SECTORS, &runSectors, &next)) { + Fat32AsyncFinish(ctx, STATUS_DEVICE_ERROR, ctx->Done); + return; } uint32_t runBytes = runSectors * 512u; @@ -2125,7 +2610,8 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) { size_t copy = avail < ctx->Remaining ? avail : ctx->Remaining; ctx->CurrentNext = next; - ctx->CurrentLast = last; + ctx->CurrentLast = runStart + (runSectors / v->SectorsPerCluster) - 1u; + ctx->CurrentClusters = runSectors / v->SectorsPerCluster; ctx->CurrentOffset = ctx->OffsetInCluster; ctx->CurrentCopy = copy; @@ -2184,7 +2670,7 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, memset(ctx, 0, sizeof(*ctx)); LIST_INIT(&ctx->Link); - ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u); + ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_READ_RUN_MAX_SECTORS * 512u); if (!ctx->RunBuf) { MmFreePool(ctx); complete(completeContext, STATUS_NO_MEMORY, 0); @@ -2196,17 +2682,14 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, ctx->IsWrite = 0; ctx->Offset = offset; ctx->OffsetInCluster = (uint32_t)(offset % v->BytesPerCluster); - ctx->Cluster = f->StartCluster; + ctx->LogicalCluster = (uint32_t)(offset / v->BytesPerCluster); + ctx->Cluster = Fat32ResolveFileCluster(f, ctx->LogicalCluster); ctx->Remaining = size; ctx->Requested = size; ctx->Dst = (uint8_t*)buf; ctx->Complete = complete; ctx->CompleteContext = completeContext; - uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster); - for (uint32_t i = 0; i < skipClusters && ctx->Cluster < FAT32_EOC; i++) - ctx->Cluster = FatEntry(v, ctx->Cluster); - Fat32AsyncQueueContinuation(ctx); return STATUS_PENDING; } @@ -2276,7 +2759,7 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) { if (ctx->OffsetInCluster == 0) { while (next == last + 1 && next < FAT32_EOC && - runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) { + runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) { last = next; next = FatEntry(v, last); runSectors += v->SectorsPerCluster; @@ -2347,6 +2830,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, FAT32_FILE* f = (FAT32_FILE*)fileCtx; FAT32_VOL* v = f->Vol; + Fat32FileInvalidateChainCache(f); if (f->StartCluster < 2) { uint32_t first = AllocCluster(v); @@ -2380,7 +2864,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, memset(ctx, 0, sizeof(*ctx)); LIST_INIT(&ctx->Link); - ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u); + ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u); if (!ctx->RunBuf) { MmFreePool(ctx); complete(completeContext, STATUS_NO_MEMORY, 0); diff --git a/code/drivers/fs/fat32/fat32.h b/code/drivers/fs/fat32/fat32.h index ccd039f..702ee48 100644 --- a/code/drivers/fs/fat32/fat32.h +++ b/code/drivers/fs/fat32/fat32.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include typedef struct FAT32_VOL_S FAT32_VOL; @@ -27,10 +29,12 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void * KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext); KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx); KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path); -KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize); +KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags); uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx); +KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out); KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx); KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out); void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx); +void Fat32QueryStats(IO_FILESYSTEM *fs, IO_FILESYSTEM_STATS *out); void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol); diff --git a/code/drivers/fs/fat32/fat32_mount.c b/code/drivers/fs/fat32/fat32_mount.c index cd598a2..ce1137e 100644 --- a/code/drivers/fs/fat32/fat32_mount.c +++ b/code/drivers/fs/fat32/fat32_mount.c @@ -14,9 +14,11 @@ void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) { fs->Delete = Fat32Delete; fs->Truncate = Fat32Truncate; fs->GetSize = Fat32GetSize; + fs->QueryIdentity = Fat32QueryIdentity; fs->Close = Fat32Close; fs->DriverCtx = vol; fs->OpenDir = Fat32OpenDir; fs->ReadDir = Fat32ReadDir; fs->CloseDir = Fat32CloseDir; + fs->QueryStats = Fat32QueryStats; } diff --git a/code/drivers/input/serial/serial.c b/code/drivers/input/serial/serial.c new file mode 100644 index 0000000..e051893 --- /dev/null +++ b/code/drivers/input/serial/serial.c @@ -0,0 +1,301 @@ +// 16550-compatible serial input driver. This file routes COM1 receive bytes into the canonical input core. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define SERIAL_COM1_BASE 0x3F8u +#define SERIAL_COM1_IRQ 4u + +#define SERIAL_REG_DATA 0u +#define SERIAL_REG_IER 1u +#define SERIAL_REG_IIR 2u +#define SERIAL_REG_FCR 2u +#define SERIAL_REG_LCR 3u +#define SERIAL_REG_MCR 4u +#define SERIAL_REG_LSR 5u + +#define SERIAL_LSR_DATA_READY 0x01u +#define SERIAL_LSR_TX_EMPTY 0x20u +#define SERIAL_LCR_DLAB 0x80u +#define SERIAL_MCR_DTR 0x01u +#define SERIAL_MCR_RTS 0x02u +#define SERIAL_MCR_OUT2 0x08u + +#define SERIAL_IER_RX_READY 0x01u +#define SERIAL_FCR_ENABLE 0x01u +#define SERIAL_FCR_CLEAR_RX 0x02u +#define SERIAL_FCR_CLEAR_TX 0x04u +#define SERIAL_FCR_TRIGGER_14 0xC0u + +typedef enum SERIAL_ESCAPE_STATE { + SerialEscapeNone = 0, + SerialEscapeEsc, + SerialEscapeCsi, + SerialEscapeCsiNumber, +} SERIAL_ESCAPE_STATE; + +static volatile SERIAL_ESCAPE_STATE gSerialEscapeState; +static volatile uint8_t gSerialCsiNumber; +static volatile bool gSerialDropNextLf; + +// Reads one byte from a COM register. +static uint8_t SerialRead(uint16_t reg) { + return __inbyte((unsigned short)(SERIAL_COM1_BASE + reg)); +} + +// Writes one byte to a COM register. +static void SerialWrite(uint16_t reg, uint8_t value) { + __outbyte((unsigned short)(SERIAL_COM1_BASE + reg), value); +} + +// Submits one input key transition. +static void SerialSubmitTransition(INPUT_KEY key, bool pressed) { + INPUT_KEY_EVENT event; + event.Key = key; + event.Pressed = pressed; + KeInputSubmitKey(&event); +} + +// Submits one logical key press and release. +static void SerialSubmitKey(INPUT_KEY key) { + if (key == INPUT_KEY_NONE) return; + SerialSubmitTransition(key, true); + SerialSubmitTransition(key, false); +} + +// Submits one shifted key press and release. +static void SerialSubmitShiftedKey(INPUT_KEY key) { + if (key == INPUT_KEY_NONE) return; + SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, true); + SerialSubmitKey(key); + SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, false); +} + +// Submits one control-modified key press and release. +static void SerialSubmitCtrlKey(INPUT_KEY key) { + if (key == INPUT_KEY_NONE) return; + SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, true); + SerialSubmitKey(key); + SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, false); +} + +// Converts an ASCII printable byte into input events. +static void SerialSubmitPrintable(uint8_t ch) { + if (ch >= 'a' && ch <= 'z') { + SerialSubmitKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'a'))); + return; + } + if (ch >= 'A' && ch <= 'Z') { + SerialSubmitShiftedKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'A'))); + return; + } + if (ch >= '1' && ch <= '9') { + SerialSubmitKey((INPUT_KEY)(INPUT_KEY_1 + (ch - '1'))); + return; + } + if (ch == '0') { + SerialSubmitKey(INPUT_KEY_0); + return; + } + + switch (ch) { + case ' ': SerialSubmitKey(INPUT_KEY_SPACE); return; + case '-': SerialSubmitKey(INPUT_KEY_MINUS); return; + case '=': SerialSubmitKey(INPUT_KEY_EQUAL); return; + case '[': SerialSubmitKey(INPUT_KEY_LEFT_BRACKET); return; + case ']': SerialSubmitKey(INPUT_KEY_RIGHT_BRACKET); return; + case '\\': SerialSubmitKey(INPUT_KEY_BACKSLASH); return; + case ';': SerialSubmitKey(INPUT_KEY_SEMICOLON); return; + case '\'': SerialSubmitKey(INPUT_KEY_APOSTROPHE); return; + case '`': SerialSubmitKey(INPUT_KEY_GRAVE); return; + case ',': SerialSubmitKey(INPUT_KEY_COMMA); return; + case '.': SerialSubmitKey(INPUT_KEY_PERIOD); return; + case '/': SerialSubmitKey(INPUT_KEY_SLASH); return; + + case '!': SerialSubmitShiftedKey(INPUT_KEY_1); return; + case '@': SerialSubmitShiftedKey(INPUT_KEY_2); return; + case '#': SerialSubmitShiftedKey(INPUT_KEY_3); return; + case '$': SerialSubmitShiftedKey(INPUT_KEY_4); return; + case '%': SerialSubmitShiftedKey(INPUT_KEY_5); return; + case '^': SerialSubmitShiftedKey(INPUT_KEY_6); return; + case '&': SerialSubmitShiftedKey(INPUT_KEY_7); return; + case '*': SerialSubmitShiftedKey(INPUT_KEY_8); return; + case '(': SerialSubmitShiftedKey(INPUT_KEY_9); return; + case ')': SerialSubmitShiftedKey(INPUT_KEY_0); return; + case '_': SerialSubmitShiftedKey(INPUT_KEY_MINUS); return; + case '+': SerialSubmitShiftedKey(INPUT_KEY_EQUAL); return; + case '{': SerialSubmitShiftedKey(INPUT_KEY_LEFT_BRACKET); return; + case '}': SerialSubmitShiftedKey(INPUT_KEY_RIGHT_BRACKET); return; + case '|': SerialSubmitShiftedKey(INPUT_KEY_BACKSLASH); return; + case ':': SerialSubmitShiftedKey(INPUT_KEY_SEMICOLON); return; + case '"': SerialSubmitShiftedKey(INPUT_KEY_APOSTROPHE); return; + case '~': SerialSubmitShiftedKey(INPUT_KEY_GRAVE); return; + case '<': SerialSubmitShiftedKey(INPUT_KEY_COMMA); return; + case '>': SerialSubmitShiftedKey(INPUT_KEY_PERIOD); return; + case '?': SerialSubmitShiftedKey(INPUT_KEY_SLASH); return; + default: return; + } +} + +// Submits a completed CSI escape sequence. +static void SerialSubmitCsi(uint8_t finalByte) { + uint8_t number = gSerialCsiNumber; + gSerialEscapeState = SerialEscapeNone; + gSerialCsiNumber = 0; + + switch (finalByte) { + case 'A': SerialSubmitKey(INPUT_KEY_UP); return; + case 'B': SerialSubmitKey(INPUT_KEY_DOWN); return; + case 'C': SerialSubmitKey(INPUT_KEY_RIGHT); return; + case 'D': SerialSubmitKey(INPUT_KEY_LEFT); return; + case 'H': SerialSubmitKey(INPUT_KEY_HOME); return; + case 'F': SerialSubmitKey(INPUT_KEY_END); return; + case '~': + switch (number) { + case 1: SerialSubmitKey(INPUT_KEY_HOME); return; + case 3: SerialSubmitKey(INPUT_KEY_DELETE); return; + case 4: SerialSubmitKey(INPUT_KEY_END); return; + case 5: SerialSubmitKey(INPUT_KEY_PAGE_UP); return; + case 6: SerialSubmitKey(INPUT_KEY_PAGE_DOWN); return; + default: return; + } + default: return; + } +} + +// Handles one byte after an ESC prefix. +static bool SerialHandleEscapeByte(uint8_t ch) { + if (gSerialEscapeState == SerialEscapeNone) return false; + + if (gSerialEscapeState == SerialEscapeEsc) { + if (ch == '[') { + gSerialEscapeState = SerialEscapeCsi; + gSerialCsiNumber = 0; + return true; + } + + gSerialEscapeState = SerialEscapeNone; + gSerialCsiNumber = 0; + SerialSubmitKey(INPUT_KEY_ESCAPE); + return false; + } + + if (gSerialEscapeState == SerialEscapeCsi || gSerialEscapeState == SerialEscapeCsiNumber) { + if (ch >= '0' && ch <= '9') { + gSerialEscapeState = SerialEscapeCsiNumber; + if (gSerialCsiNumber < 25) + gSerialCsiNumber = (uint8_t)(gSerialCsiNumber * 10u + (ch - '0')); + return true; + } + + if (ch == ';') + return true; + + SerialSubmitCsi(ch); + return true; + } + + gSerialEscapeState = SerialEscapeNone; + gSerialCsiNumber = 0; + return false; +} + +// Routes one received serial byte into the input core. +static void SerialReceiveByte(uint8_t ch) { + if (SerialHandleEscapeByte(ch)) return; + + if (ch == 0x1Bu) { + gSerialEscapeState = SerialEscapeEsc; + gSerialCsiNumber = 0; + return; + } + + if (ch == '\r') { + SerialSubmitKey(INPUT_KEY_ENTER); + gSerialDropNextLf = true; + return; + } + + if (ch == '\n') { + if (gSerialDropNextLf) { + gSerialDropNextLf = false; + return; + } + SerialSubmitKey(INPUT_KEY_ENTER); + return; + } + + gSerialDropNextLf = false; + + if (ch == '\t') { + SerialSubmitKey(INPUT_KEY_TAB); + return; + } + + if (ch == '\b' || ch == 0x7Fu) { + SerialSubmitKey(INPUT_KEY_BACKSPACE); + return; + } + + if (ch == 0x03u) { + SerialSubmitCtrlKey(INPUT_KEY_C); + return; + } + + if (ch >= 0x20u && ch <= 0x7Eu) + SerialSubmitPrintable(ch); +} + +// Handles receive interrupts for COM1. +static void SerialIsr(void) { + for (uint32_t i = 0; i < 256u; i++) { + if ((SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY) == 0) + break; + SerialReceiveByte(SerialRead(SERIAL_REG_DATA)); + } +} + +// Initializes COM1 as an 8N1 115200 baud input device. +static void SerialInitializeCom1(void) { + SerialWrite(SERIAL_REG_IER, 0x00u); + SerialWrite(SERIAL_REG_LCR, SERIAL_LCR_DLAB); + SerialWrite(SERIAL_REG_DATA, 0x01u); + SerialWrite(SERIAL_REG_IER, 0x00u); + SerialWrite(SERIAL_REG_LCR, 0x03u); + SerialWrite(SERIAL_REG_FCR, SERIAL_FCR_ENABLE | SERIAL_FCR_CLEAR_RX | + SERIAL_FCR_CLEAR_TX | SERIAL_FCR_TRIGGER_14); + SerialWrite(SERIAL_REG_MCR, SERIAL_MCR_DTR | SERIAL_MCR_RTS | SERIAL_MCR_OUT2); + + while (SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY) + (void)SerialRead(SERIAL_REG_DATA); + + SerialWrite(SERIAL_REG_IER, SERIAL_IER_RX_READY); +} + +// DriverEntry registers COM1 as an input source. +KSTATUS DriverEntry(const HAL_API* hal) { + (void)hal; + + uint32_t deviceId = 0; + KSTATUS st = KeInputRegisterDevice("serial-com1", 0, &deviceId); + if (!KSUCCESS(st)) + return st; + + gSerialEscapeState = SerialEscapeNone; + gSerialCsiNumber = 0; + gSerialDropNextLf = false; + + SerialInitializeCom1(); + KeIrqRegister(SERIAL_COM1_IRQ, SerialIsr); + + DbgPrint("serial: initialized COM1 input device=%u irq=%u\n", deviceId, SERIAL_COM1_IRQ); + return STATUS_SUCCESS; +} diff --git a/code/drivers/net/rtl8139/rtl8139.c b/code/drivers/net/rtl8139/rtl8139.c index d9c07e3..a733277 100644 --- a/code/drivers/net/rtl8139/rtl8139.c +++ b/code/drivers/net/rtl8139/rtl8139.c @@ -88,8 +88,8 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) { // Creates dma. static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) { - r->RxPhys = MmAllocateContiguous(RTL8139_RX_PAGES); - r->TxPhys = MmAllocateContiguous(RTL8139_TX_PAGES); + r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma); + r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma); if (!r->RxPhys || !r->TxPhys) return STATUS_NO_MEMORY; diff --git a/code/drivers/net/rtl8139/rtl8139.h b/code/drivers/net/rtl8139/rtl8139.h index fb0d102..f00b366 100644 --- a/code/drivers/net/rtl8139/rtl8139.h +++ b/code/drivers/net/rtl8139/rtl8139.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/code/drivers/net/rtl816x/rtl816x.c b/code/drivers/net/rtl816x/rtl816x.c index 4ff78a3..cd8d27b 100644 --- a/code/drivers/net/rtl816x/rtl816x.c +++ b/code/drivers/net/rtl816x/rtl816x.c @@ -85,10 +85,10 @@ static void RtlReadMac(RTL816X_DEVICE* r) { // Implements the rtl Allocate Dma operation. static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) { - r->RxRingPhys = MmAllocateContiguous(RTL_RING_PAGES); - r->TxRingPhys = MmAllocateContiguous(RTL_RING_PAGES); - r->RxBuffersPhys = MmAllocateContiguous(RTL_RXBUF_PAGES); - r->TxBuffersPhys = MmAllocateContiguous(RTL_TXBUF_PAGES); + r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma); + r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma); + r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma); + r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma); if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys) return STATUS_NO_MEMORY; diff --git a/code/drivers/net/rtl816x/rtl816x.h b/code/drivers/net/rtl816x/rtl816x.h index e89e60d..d6e19eb 100644 --- a/code/drivers/net/rtl816x/rtl816x.h +++ b/code/drivers/net/rtl816x/rtl816x.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/code/drivers/storage/ahci/ahci.c b/code/drivers/storage/ahci/ahci.c index b4d4b70..b2b6318 100644 --- a/code/drivers/storage/ahci/ahci.c +++ b/code/drivers/storage/ahci/ahci.c @@ -58,20 +58,20 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { static KSTATUS port_init_memory(AHCI_DRIVE *drv) { uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; - drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT); + drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma); if (!drv->structPhys) return STATUS_NO_MEMORY; drv->structVirt = g_hal->PhysToVirt(drv->structPhys); // Force NOCACHE on DMA descriptor memory. - MmSetPageProtect((uintptr_t)drv->structVirt, + MmPteProtectRange((uintptr_t)drv->structVirt, PORT_PAGES_STRUCT * 4096, flags); memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096); - drv->bouncePhys = MmAllocateContiguous(BOUNCE_PAGES); + drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma); if (!drv->bouncePhys) return STATUS_NO_MEMORY; drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys); // Bounce buffer: NOCACHE to ensure CPU writes reach physical memory // before the HBA's DMA engine reads them. - MmSetPageProtect((uintptr_t)drv->bounceVirt, + MmPteProtectRange((uintptr_t)drv->bounceVirt, BOUNCE_PAGES * 4096, flags); memset(drv->bounceVirt, 0, BOUNCE_BYTES); @@ -203,6 +203,9 @@ static void AhciIsr(void) { // Blocks the current thread until the ISR signals completionEvent. static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { + if (drv->controller && drv->controller->pollingMode) + return port_issue_cmd_poll(drv); + HBA_PORT *port = drv->port; // Clear stale IS and snapshot before issuing so we don't mis-read a @@ -647,12 +650,25 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { ctl->pciInfo.VendorId, ctl->pciInfo.DeviceId); - st = PciGetInterrupt(ctl->pdo, &ctl->irq); - if (!KSUCCESS(st) || ctl->irq == 0xFF) { - DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n"); - return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st; + ctl->irq = 0xFFu; + ctl->usingMsi = false; + ctl->pollingMode = false; + + st = PciEnableMsiAuto(ctl->pdo, &ctl->irq); + if (KSUCCESS(st)) { + ctl->usingMsi = true; + DbgPrint("AHCI: using MSI IRQ slot %u\n", (unsigned)ctl->irq); + } else { + KSTATUS intxStatus = PciGetInterrupt(ctl->pdo, &ctl->irq); + if (KSUCCESS(intxStatus) && ctl->irq != 0xFFu) { + DbgPrint("AHCI: using legacy INTx IRQ %u\n", (unsigned)ctl->irq); + } else { + ctl->irq = 0xFFu; + ctl->pollingMode = true; + DbgPrint("AHCI: no MSI/INTx interrupt available; using polled command completion (msi=0x%x intx=0x%x)\n", + st, intxStatus); + } } - DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)ctl->irq); st = PciGetBar(ctl->pdo, AHCI_PCI_BAR_ABAR, &ctl->abar); if (!KSUCCESS(st)) { @@ -674,10 +690,13 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { return st; } - st = PciEnableDevice(ctl->pdo, - PCI_ENABLE_MEMORY | - PCI_ENABLE_BUSMASTER | - PCI_ENABLE_INTX); + uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER; + if (ctl->usingMsi || ctl->pollingMode) + enableFlags |= PCI_DISABLE_INTX; + else + enableFlags |= PCI_ENABLE_INTX; + + st = PciEnableDevice(ctl->pdo, enableFlags); if (!KSUCCESS(st)) { DbgPrint("AHCI: PciEnableDevice failed: 0x%x\n", st); return st; @@ -831,11 +850,16 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { ctl->pciInfo.Address.Device, ctl->pciInfo.Address.Function); - KeIrqRegister(ctl->irq, AhciIsr); + if (!ctl->pollingMode) { + KeIrqRegister(ctl->irq, AhciIsr); + DbgPrint("AHCI: ISR registered on IRQ %u (%s); controller drives=%d\n", + (unsigned)ctl->irq, ctl->usingMsi ? "MSI" : "INTx", ctl->driveCount); + } else { + DbgPrint("AHCI: controller started in polling mode; controller drives=%d\n", + ctl->driveCount); + } ctl->started = true; ctl->fdo->PnpState = DevicePnpStarted; - DbgPrint("AHCI: ISR registered on IRQ %u; controller drives=%d\n", - (unsigned)ctl->irq, ctl->driveCount); return STATUS_SUCCESS; } diff --git a/code/drivers/storage/ahci/ahci.h b/code/drivers/storage/ahci/ahci.h index c6ff2ec..2fa29fc 100644 --- a/code/drivers/storage/ahci/ahci.h +++ b/code/drivers/storage/ahci/ahci.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -367,6 +368,8 @@ struct AHCI_CONTROLLER { HBA_MEM* hba; uint64_t abarLength; uint8_t irq; + bool usingMsi; + bool pollingMode; AHCI_DRIVE drives[MAX_AHCI_DRIVES]; int driveCount; diff --git a/code/drivers/storage/ahci/ahci_port.c b/code/drivers/storage/ahci/ahci_port.c index 56ca872..67c00e2 100644 --- a/code/drivers/storage/ahci/ahci_port.c +++ b/code/drivers/storage/ahci/ahci_port.c @@ -12,15 +12,15 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) { for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) { uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys); uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; - KSTATUS st = MmMapPage(virt, phys, flags, MmMapFailIfPresent); + KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent); // Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags // is possible for the first page if the bootloader was overly generous. - // Use MmSetPageProtect to force NOCACHE in that case. + // Use MmPteProtectRange to force NOCACHE in that case. if (st == STATUS_ALREADY_EXISTS) { - st = MmSetPageProtect(virt, 4096, flags); + st = MmPteProtectRange(virt, 4096, flags); } if (!KSUCCESS(st)) { - DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n", + DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n", pageIndex, (unsigned long long)phys, st); return st; } diff --git a/code/kernel/include/kernel/io_file.h b/code/kernel/include/kernel/io_file.h index 291803c..2947603 100644 --- a/code/kernel/include/kernel/io_file.h +++ b/code/kernel/include/kernel/io_file.h @@ -25,6 +25,39 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext, KSTATUS status, size_t information); +// Filesystem-level cache and traversal statistics. +typedef struct IO_FILESYSTEM_STATS { + uint64_t PathLookups; + uint64_t PathCacheHits; + uint64_t PathCacheMisses; + uint64_t PathCacheStores; + uint64_t PathCacheInvalidations; + uint64_t PathCacheGeneration; + uint64_t DirectorySearches; + uint64_t DirectoryClustersRead; + uint64_t DirectoryEntriesScanned; + uint64_t FatChainCacheLookups; + uint64_t FatChainCacheHits; + uint64_t FatChainCacheMisses; + uint64_t FatChainCacheStores; + uint64_t FatChainCacheInvalidations; + uint64_t FatChainWalkSteps; + uint64_t DataReadRuns; + uint64_t DataReadSectors; + uint64_t DataReadRunMerges; + uint64_t MaxDataReadSpanSectors; +} IO_FILESYSTEM_STATS; + +// Stable identity for a file on a mounted filesystem. +// Filesystems that can provide this let higher layers cache file-backed +// objects by file identity instead of by one FILE_OBJECT instance. +typedef struct IO_FILE_IDENTITY { + uint64_t VolumeId; + uint64_t FileId; + uint64_t Generation; + uint64_t Size; +} IO_FILE_IDENTITY; + // Filesystem driver vtable - filled in by a filesystem driver. typedef struct IO_FILESYSTEM { // Open a file by absolute path (e.g. "apps\\shell.exe"). @@ -69,14 +102,21 @@ typedef struct IO_FILESYSTEM { // Truncate or extend the file to exactly 'newSize' bytes. KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx, - uint64_t newSize); + uint64_t newSize, uint32_t fileFlags); // Return the file size in bytes. uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx); + // Return a stable file identity for caches that outlive one open handle. + KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx, + IO_FILE_IDENTITY* out); + // Release any resources the driver allocated for fileCtx. void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx); + // Query cache and traversal statistics exposed through kernel stats. + void (*QueryStats)(struct IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out); + // Optional driver-private data. void* DriverCtx; @@ -106,9 +146,13 @@ typedef struct IO_STATUS_BLOCK { struct EVENT; struct KAPC; +typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER; typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT; // FILE_OBJECT - the kernel object returned to callers. +#define IO_FILE_FLAG_NO_CACHE 0x00000001u +#define IO_FILE_FLAG_SKIP_ZERO_EXTEND 0x00000002u + typedef struct FILE_OBJECT { OBJECT Header; // must be first IO_FILESYSTEM* Fs; @@ -116,6 +160,11 @@ typedef struct FILE_OBJECT { SPINLOCK Lock; // protects cached Size/Offset updates uint64_t Size; uint64_t Offset; // current position for sequential reads + uint32_t Flags; // IO_FILE_FLAG_* + uint32_t ReadaheadScore; + uint64_t ReadaheadNextOffset; + uint64_t ReadaheadIssuedUntil; + uint64_t ReadaheadSuppressedUntil; } FILE_OBJECT; // DIR_OBJECT - kernel object for an open directory iterator. @@ -137,6 +186,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESY __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex); __declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex); __declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex); +__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out); // Kernel-side API. __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out); @@ -149,15 +199,23 @@ __declspec(dllexport) KSTATUS IoWriteFile (FILE_OBJECT* file, const void* buf, __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset, const void* buf, size_t size, size_t* bytesWritten); +__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file); __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf, size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb, struct EVENT* completionEvent, struct KAPC* completionApc, + MM_LOCKED_USER_BUFFER* lockedBuffer, + MM_LOCKED_USER_BUFFER* lockedIosb, IO_ASYNC_FILE_REQUEST_OBJECT** requestOut); __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf, size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb, struct EVENT* completionEvent, struct KAPC* completionApc, + MM_LOCKED_USER_BUFFER* lockedBuffer, + MM_LOCKED_USER_BUFFER* lockedIosb, IO_ASYNC_FILE_REQUEST_OBJECT** requestOut); __declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* request); +__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request, + uint32_t flags); +__declspec(dllexport) struct EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* request); __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request); __declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request, IO_STATUS_BLOCK* iosbOut); @@ -169,6 +227,7 @@ __declspec(dllexport) KSTATUS IoRemoveDir (SYS_STRING_VIEW path); __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize); __declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset); __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file); +__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out); __declspec(dllexport) void IoCloseFile(FILE_OBJECT* file); // Directory enumeration API. diff --git a/code/kernel/include/kernel/ke_debug.h b/code/kernel/include/kernel/ke_debug.h index a8d5d25..e899776 100644 --- a/code/kernel/include/kernel/ke_debug.h +++ b/code/kernel/include/kernel/ke_debug.h @@ -12,6 +12,16 @@ #define KE_DEBUG_SELFTEST 0x00000002u #define KE_DEBUG_FILE_LOG 0x00000004u +#define KE_DEBUG_POOL_CANARY 0x00000008u +#define KE_DEBUG_POOL_POISON 0x00000010u +#define KE_DEBUG_POOL_QUARANTINE 0x00000020u +#define KE_DEBUG_POOL_CALLERS 0x00000040u +#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u +#define KE_DEBUG_DISPLAY_TRACE 0x00000100u +#define KE_DEBUG_INPUT_TRACE 0x00000200u +#define KE_DEBUG_VM_ASSERTS 0x00000400u +#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u + void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags); void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)); uint32_t KeGetDebugFlags(void); @@ -68,7 +78,7 @@ typedef struct KFAULT_FRAME { uint64_t R15; } KFAULT_FRAME; -__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame); +__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame); // Compatibility wrapper for any older assembly/object still calling this. __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip, diff --git a/code/kernel/include/kernel/ke_display.h b/code/kernel/include/kernel/ke_display.h index e226715..24e871c 100644 --- a/code/kernel/include/kernel/ke_display.h +++ b/code/kernel/include/kernel/ke_display.h @@ -42,7 +42,9 @@ DISPLAY_API void KeDisplaySuspendConsole(bool suspend); DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode); DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight); +typedef KSTATUS (*KE_DISPLAY_ROW_SOURCE)(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount); DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels); +DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch); DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color); DISPLAY_API KSTATUS KeDisplayClear(uint32_t color); DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h); diff --git a/code/kernel/include/kernel/ke_event.h b/code/kernel/include/kernel/ke_event.h index 0573d36..9577a92 100644 --- a/code/kernel/include/kernel/ke_event.h +++ b/code/kernel/include/kernel/ke_event.h @@ -16,9 +16,12 @@ typedef struct EVENT { EVENT_TYPE Type; uint32_t Signaled; LIST_ENTRY WaitList; + LIST_ENTRY MultiWaitList; } EVENT; __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out); __declspec(dllexport) void KeSignalEvent(EVENT* event); __declspec(dllexport) void KeClearEvent(EVENT* event); __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable); +__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event); +__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count, uint8_t alertable, uint64_t timeoutMs, uint32_t* signaledIndexOut); diff --git a/code/kernel/include/kernel/ke_input.h b/code/kernel/include/kernel/ke_input.h index f695390..e8bc586 100644 --- a/code/kernel/include/kernel/ke_input.h +++ b/code/kernel/include/kernel/ke_input.h @@ -6,6 +6,13 @@ #include #include #include +#include +#include + +typedef struct INPUT_QUEUE { + OBJECT Header; + EVENT* WaitEvent; +} INPUT_QUEUE; void KeInputInitialize(const HAL_API* hal); @@ -16,7 +23,12 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE __declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons); INPUT_KEY KeInputKbPoll(void); +KSTATUS KeInputWaitKey(uint8_t alertable); bool KeInputIsKeyDown(INPUT_KEY key); +bool KeInputPeekEvent(INPUT_KEY_RECORD* record); +bool KeInputReadEvent(INPUT_KEY_RECORD* record); +KSTATUS KeInputOpenQueue(INPUT_QUEUE** out); +EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue); INPUT_KEY KeInputReadKey(void); char KeInputReadChar(void); uint32_t KeInputGetModifiers(void); diff --git a/code/kernel/include/kernel/ke_mutex.h b/code/kernel/include/kernel/ke_mutex.h index 7db2b60..6a88659 100644 --- a/code/kernel/include/kernel/ke_mutex.h +++ b/code/kernel/include/kernel/ke_mutex.h @@ -14,7 +14,7 @@ typedef struct MUTEX { LIST_ENTRY WaitList; } MUTEX; -KSTATUS KeMutexCreate(MUTEX **out); -KSTATUS KeAcquireMutex(MUTEX* mutex); -KSTATUS KeTryAcquireMutex(MUTEX* mutex); -void KeReleaseMutex(MUTEX* mutex); +__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out); +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex); +__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex); +__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex); diff --git a/code/kernel/include/kernel/ke_process.h b/code/kernel/include/kernel/ke_process.h index 759201d..8097df4 100644 --- a/code/kernel/include/kernel/ke_process.h +++ b/code/kernel/include/kernel/ke_process.h @@ -62,6 +62,7 @@ typedef struct PROCESS { int ExitCode; volatile long Terminating; volatile long Exited; + volatile long ResourcesReleased; // Main executable image. Used for cleanup. uintptr_t ImageBase; @@ -80,8 +81,8 @@ typedef struct PROCESS { // Per-process user virtual regions. The actual VA arena is global because // NT-Lite currently uses one shared address space; this list tracks // ownership so leaked user mappings can be reclaimed on process exit. - LIST_ENTRY UserRegions; - SPINLOCK UserRegionLock; + LIST_ENTRY VadList; + SPINLOCK VadLock; // Per-process loaded modules (user processes only). PROCESS_MODULE Modules[PROCESS_MODULE_MAX]; @@ -102,6 +103,7 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* process, SYS_STRING_VIEW currentDirectory); void KeExitProcess(PROCESS* process, int exitCode); KSTATUS KeTerminateProcess(PROCESS* process, int exitCode); +void KeCompleteProcessExitIfLastThread(PROCESS* process); int KeIsProcessTerminating(PROCESS* process); KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut); @@ -118,8 +120,12 @@ LIST_ENTRY* KeGetProcessList(void); PROCESS* KeGetCurrentProcess(void); PROCESS* KeGetSystemProcess(void); struct SYS_PROCESS_INFO; +struct SYS_THREAD_INFO; // Copy a snapshot of rich process information into caller-provided storage. size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount); + +// Copy a snapshot of one process's threads into caller-provided storage. +KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut); // Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject. PROCESS* KeGetProcessById(uint32_t pid); diff --git a/code/kernel/include/kernel/ke_sched.h b/code/kernel/include/kernel/ke_sched.h index bd27cca..3abb3b3 100644 --- a/code/kernel/include/kernel/ke_sched.h +++ b/code/kernel/include/kernel/ke_sched.h @@ -58,6 +58,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread); void KiSchedulerMarkThreadWaiting(THREAD* thread); void KiSchedulerWakeThread(THREAD* thread, KSTATUS status); +// Blocks the current thread until at least the requested delay has elapsed. +__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds); +KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus); + // Voluntarily yield the CPU. Picks the next runnable thread immediately. __declspec(dllexport) void KeYield(void); diff --git a/code/kernel/include/kernel/ke_system_root.h b/code/kernel/include/kernel/ke_system_root.h new file mode 100644 index 0000000..729fab1 --- /dev/null +++ b/code/kernel/include/kernel/ke_system_root.h @@ -0,0 +1,25 @@ +// System root volume declarations. This file exposes the one selected kernel root volume to loader and process code. + +#pragma once +#include +#include +#include +#include +#include + +#define KE_SYSTEM_ROOT_MAX_DRIVES 8 + +// Selects the mounted volume that contains the kernel/system tree. +KSTATUS KeInitializeSystemRootVolume(void); + +// Returns whether the system root volume has been selected. +bool KeSystemRootReady(void); + +// Returns the selected system root drive, or zero before discovery. +uint8_t KeGetSystemRootDrive(void); + +// Builds an absolute path rooted at the selected system volume. +bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize); + +// Opens a file rooted at the selected system volume. +KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out); diff --git a/code/kernel/include/kernel/ke_thread.h b/code/kernel/include/kernel/ke_thread.h index 5acde67..b742384 100644 --- a/code/kernel/include/kernel/ke_thread.h +++ b/code/kernel/include/kernel/ke_thread.h @@ -14,6 +14,8 @@ #define KSTACK_SIZE (64 * 1024) // 64 KB kernel stack #define USTACK_SIZE (512 * 1024) // 512 KB user stack +typedef struct THREAD THREAD; + typedef enum THREAD_PRIORITY { PriorityLow = 0, PriorityNormal = 1, @@ -28,6 +30,10 @@ typedef enum THREAD_STATE { ThreadDead = 3, } THREAD_STATE; +typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread, + void* context, + KSTATUS status); + // Saved register context - matches the layout pushed by KeSwitchContext. // The saved RSP points to this structure on the kernel stack. typedef struct KCONTEXT { @@ -36,7 +42,7 @@ typedef struct KCONTEXT { uint64_t Rip; // return address back into KeSwitchContext } KCONTEXT; -typedef struct THREAD { +struct THREAD { OBJECT Header; // must be first uint32_t ThreadId; @@ -82,7 +88,7 @@ typedef struct THREAD { // Signaled (manual-reset) when the thread exits. // Allocated in KeCreateThread, signaled in KeExitThread before teardown. - // Waiters block on this via SYS_WAIT_THREAD -> KeWaitForEvent. + // Waiters block on this through waitable thread handles. EVENT* ExitEvent; void* FpuContext; @@ -113,6 +119,12 @@ typedef struct THREAD { KSTATUS WaitStatus; LIST_ENTRY* WaitListHead; uint8_t AlertableWait; + uint64_t SleepUntilTick; + KSTATUS TimeoutStatus; + + // Optional cleanup hook for waits that use auxiliary wait blocks. + KE_WAIT_RUNDOWN_ROUTINE WaitRundownRoutine; + void* WaitRundownContext; // Kernel APC state. APCs are queued to a target thread and delivered // when that thread reaches PASSIVE_LEVEL with APC delivery enabled. @@ -135,15 +147,22 @@ typedef struct THREAD { uintptr_t UserApcDispatcher; uint8_t UserApcDeliverable; uint8_t UserApcDeliveryActive; -} THREAD; +}; +// Creates and queues a user thread. On success, *out receives a referenced +// THREAD pointer owned by the caller; release it with ObDereferenceObject() +// after copying any needed IDs/state. The live scheduler/thread lifetime is +// separate and remains valid until the thread exits and is reaped. __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg, THREAD_PRIORITY priority, THREAD** out); // Create a kernel-mode-only thread. fn is called directly in ring 0 with // arg; when fn returns the thread exits automatically. No user stack or TEB // is allocated. The thread is owned by the system process and runs at the -// given priority. +// given priority. On success, non-NULL out receives a referenced THREAD +// pointer owned by the caller; release it with ObDereferenceObject() after +// copying any needed IDs/state or after joining/waiting on caller-owned +// completion state. The scheduler's live thread reference is separate. __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, THREAD_PRIORITY priority, THREAD** out); __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, @@ -167,9 +186,14 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code); // waiting. Alertable waits are explicit at every call site. void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable); void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead); +void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context); +void KeClearThreadWaitRundown(THREAD* thread); +void KeRundownThreadWait(THREAD* thread, KSTATUS status); KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable); KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable); void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status); +void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status); +void KeAbortThreadWait(THREAD* thread, KSTATUS status); void KeUnblockThread(THREAD* thread); KSTATUS KeAlertThread(THREAD* thread); void KiWakeThreadForKernelApc(THREAD* thread); diff --git a/code/kernel/include/kernel/ke_work.h b/code/kernel/include/kernel/ke_work.h new file mode 100644 index 0000000..ef75164 --- /dev/null +++ b/code/kernel/include/kernel/ke_work.h @@ -0,0 +1,24 @@ +// Kernel passive-level work queue declarations. + +#pragma once +#include +#include +#include + +// Work items are caller-owned embedded objects. They may be queued from +// interrupt/DPC/spinlock context and run later on a persistent system worker +// thread at PASSIVE_LEVEL. Queueing the same item while it is already queued +// is coalesced. +typedef void (*KWORK_ROUTINE)(void* context); + +typedef struct KWORK_ITEM { + LIST_ENTRY Link; + KWORK_ROUTINE Routine; + void* Context; + volatile long Queued; +} KWORK_ITEM; + +void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context); +KSTATUS KeStartWorkQueue(void); +bool KeIsWorkQueueStarted(void); +bool KeQueueWorkItem(KWORK_ITEM* item); diff --git a/code/kernel/include/kernel/ldr_image.h b/code/kernel/include/kernel/ldr_image.h index e2b816d..fffcda2 100644 --- a/code/kernel/include/kernel/ldr_image.h +++ b/code/kernel/include/kernel/ldr_image.h @@ -9,11 +9,12 @@ #include struct PROCESS; +struct FILE_OBJECT; -// Map a PE32+ image from 'buffer' into the kernel virtual address space. -// On success, *baseOut receives the virtual base address. -KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user); -KSTATUS LdrLoadImageForProcess(struct PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut); +// Map a PE32+ image from a bootloader/driver raw buffer into kernel space. +// User images must be loaded from FILE_OBJECTs so image sections have stable identity. +KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut); +KSTATUS LdrLoadImageFileForProcess(struct PROCESS* process, struct FILE_OBJECT* file, uintptr_t* baseOut); // Return the SizeOfImage recorded in the PE header for an already-mapped image. size_t LdrGetImageSize(uintptr_t base); diff --git a/code/kernel/include/kernel/ldr_section.h b/code/kernel/include/kernel/ldr_section.h new file mode 100644 index 0000000..a49e202 --- /dev/null +++ b/code/kernel/include/kernel/ldr_section.h @@ -0,0 +1,67 @@ +// Kernel image section declarations shared by loader and memory-management code. + +#pragma once + +#include +#include +#include +#include + +struct PROCESS; +struct FILE_OBJECT; +typedef struct PROCESS PROCESS; +typedef struct FILE_OBJECT FILE_OBJECT; +typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT; + +// Describes one PE section in image-address-space coordinates. +typedef struct IMAGE_SECTION_DESCRIPTOR { + char Name[9]; + uint32_t VirtualAddress; + uint32_t VirtualSize; + uint32_t RawDataOffset; + uint32_t RawDataSize; + uint32_t Characteristics; + uint64_t PageProtection; +} IMAGE_SECTION_DESCRIPTOR; + +// Keeps the parsed identity and layout for a PE image section object. +typedef struct IMAGE_SECTION_METADATA { + uint64_t ContentKey; + uint64_t VolumeId; + uint64_t FileId; + uint64_t FileGeneration; + uint64_t PreferredBase; + uint32_t EntryRva; + uint32_t HeaderSize; + size_t RawSize; + size_t ImageSize; + size_t PageCount; + uint32_t SectionCount; +} IMAGE_SECTION_METADATA; + +// Creates or references an image section object backed by an open file. +KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file, + IMAGE_SECTION_OBJECT** sectionOut); + +// Adds a reference to an existing image section object. +void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section); + +// Releases a reference returned by LdrCreateImageSectionFromFile or LdrReferenceImageSection. +void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section); + +// Returns immutable metadata for an image section object. +const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section); + +// Returns the immutable PE section table owned by the image section object. +const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section, + uint32_t* countOut); + +// Retains one prototype frame owned by an image section object. +KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section, + size_t pageIndex, + PHYSICAL_ADDRESS* frameOut); + +// Maps an image section into a process as an image-backed VAD. +KSTATUS LdrMapImageSection(PROCESS* process, + IMAGE_SECTION_OBJECT* section, + void** baseOut); diff --git a/code/kernel/include/kernel/mm_file_cache.h b/code/kernel/include/kernel/mm_file_cache.h new file mode 100644 index 0000000..603bb7c --- /dev/null +++ b/code/kernel/include/kernel/mm_file_cache.h @@ -0,0 +1,99 @@ +// Kernel file-page cache declarations. +// Caches PAGE_SIZE file pages shared by reads, mmap faults, and writeback. + +#pragma once + +#include +#include +#include +#include + +struct FILE_OBJECT; +typedef struct FILE_OBJECT FILE_OBJECT; + +typedef struct MM_FILE_CACHE_PAGE { + FILE_OBJECT* File; + uint64_t Offset; + PHYSICAL_ADDRESS Frame; + uint32_t ValidBytes; + uint8_t Dirty; +} MM_FILE_CACHE_PAGE; + +typedef struct MM_FILE_CACHE_STATS { + uint64_t Lookups; + uint64_t Hits; + uint64_t Misses; + uint64_t Loads; + uint64_t LoadFailures; + uint64_t RaceDiscards; + uint64_t InflightWaits; + uint64_t Invalidations; + uint64_t DirtyPages; + uint64_t MaxDirtyPages; + uint64_t DirtyMarks; + uint64_t WritebackRequests; + uint64_t WritebackPages; + uint64_t WritebackBytes; + uint64_t WritebackFailures; + uint64_t FlushFileCalls; + uint64_t FlushFileFailures; + uint64_t MappedDirtySyncs; + uint64_t MappedDirtyPages; + uint64_t Drops; + uint64_t DropFailures; + uint64_t StandbyTransitions; + uint64_t StandbyHits; + uint64_t ReadAheadRequests; + uint64_t ReadAheadPagesRequested; + uint64_t ReadAheadPagesQueued; + uint64_t ReadAheadPagesLoaded; + uint64_t ReadAheadPagesSkipped; + uint64_t ReadAheadFailures; + uint64_t ReadAheadPagesThrottled; + uint64_t MaxReadAheadPagesPerRequest; + uint64_t BatchReads; + uint64_t BatchPagesRequested; + uint64_t BatchPagesLoaded; + uint64_t MaxBatchPagesRequested; + uint64_t MaxBatchPagesLoaded; + uint64_t ResidentPages; + uint64_t ActivePages; + uint64_t StandbyPages; + uint64_t MaxResidentPages; +} MM_FILE_CACHE_STATS; + +// Returns a retained physical frame containing the file page at page-aligned +// offset. The caller owns one PFN reference and must call +// MmFileCacheReleasePage(). Bytes beyond ValidBytes are zero-filled. +KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAGE* outPage); +void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page); + +// Returns a retained coherent cache page suitable for a shared writable mapping. +KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset, + uint64_t fileSizeBefore, int needExistingBytes, + MM_FILE_CACHE_PAGE* outPage); + +// Marks one coherent cache page dirty after a shared writable mapping is written. +void MmFileCacheMarkDirty(FILE_OBJECT* file, uint64_t offset, uint32_t validBytes); + +// Writes into coherent cached file pages and marks them dirty. +KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf, + size_t size, uint64_t fileSizeBefore, size_t* writtenOut); + +// Flushes dirty cached pages for one file. +KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file); + +// Drops cached clean pages overlapping the byte range. Dirty pages are flushed +// by callers before destructive metadata operations. +void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size); +void MmFileCacheInvalidateAll(FILE_OBJECT* file); + +// Reclaims one cache-owned page if possible. Safe to call under memory pressure; +// it never writes back because v1 stores only clean file data. +KSTATUS MmFileCacheDropOnePageForPressure(void); + +// Queues best-effort read-ahead for clean file pages. The queue owns a file +// reference until the worker has consumed the request. +KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages); + +void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out); diff --git a/code/kernel/include/kernel/mm_pfn.h b/code/kernel/include/kernel/mm_pfn.h new file mode 100644 index 0000000..acbccc1 --- /dev/null +++ b/code/kernel/include/kernel/mm_pfn.h @@ -0,0 +1,134 @@ +// Physical frame database declarations. +// The PFN layer owns frame state, reference counts, pin counts, and allocation +// policy. Higher VMM layers should ask for pages by purpose instead of treating +// the physical allocator as an untyped bitmap. + +#pragma once + +#include +#include +#include +#include + +#define MM_PFN_INVALID_INDEX ((size_t)-1) + +typedef enum MM_PFN_STATE { + MmPfnStateInvalid = 0, + MmPfnStateReserved, + MmPfnStateFree, + MmPfnStateZeroed, + MmPfnStateActive, + MmPfnStateStandby, + MmPfnStateModified, +} MM_PFN_STATE; + +typedef enum MM_PFN_USAGE { + MmPfnUseNone = 0, + MmPfnUseKernel, + MmPfnUsePageTable, + MmPfnUseUser, + MmPfnUseCleanFileCache, + MmPfnUseImage, + MmPfnUseMappedFile, + MmPfnUseAnonymous, + MmPfnUseCow, + MmPfnUseSurface, + MmPfnUseDriver, + MmPfnUseDma, + MmPfnUseBootloader, + MmPfnUsePfnDatabase, + MmPfnUseMmio, +} MM_PFN_USAGE; + +typedef struct MM_PFN { + uint16_t State; + uint16_t Usage; + uint16_t RefCount; + uint16_t PinCount; + uint32_t Flags; + uint32_t MappedCount; + void* Owner; + uint64_t BackingPage; +} MM_PFN; + +typedef struct MM_PFN_STATS { + uint64_t TotalFrames; + uint64_t FreeFrames; + uint64_t UsedFrames; + + uint64_t ReservedFrames; + uint64_t ZeroedFrames; + uint64_t ActiveFrames; + uint64_t StandbyFrames; + uint64_t ModifiedFrames; + uint64_t PinnedFrames; + uint64_t EvictableFrames; + uint64_t NonEvictableFrames; + uint64_t ReclaimableFrames; + uint64_t AvailableFrames; + uint64_t MappedFrames; + + uint64_t KernelFrames; + uint64_t PageTableFrames; + uint64_t UserFrames; + uint64_t CleanFileCacheFrames; + uint64_t ActiveCleanFileCacheFrames; + uint64_t StandbyCleanFileCacheFrames; + uint64_t ImageFrames; + uint64_t ActiveImageFrames; + uint64_t StandbyImageFrames; + uint64_t MappedFileFrames; + uint64_t AnonymousFrames; + uint64_t CowFrames; + uint64_t SurfaceFrames; + uint64_t DriverFrames; + uint64_t DmaFrames; + uint64_t BootloaderFrames; + uint64_t PfnDatabaseFrames; + uint64_t MmioFrames; + + uint64_t AllocationCount; + uint64_t FreeCount; + uint64_t RetainCount; + uint64_t ReleaseCount; + uint64_t PinCount; + uint64_t UnpinCount; + uint64_t FailedAllocations; + uint64_t FailedContiguousAllocations; + + uint64_t FailedInvalidRange; + uint64_t FailedEmptyRange; + uint64_t FailedReserveDenied; + uint64_t FailedNoFrame; + + uint64_t FailedContiguousInvalidRequest; + uint64_t FailedContiguousReserveDenied; + uint64_t FailedContiguousNoRun; + + uint64_t ReclaimAttempts; + uint64_t ReclaimPagesRequested; + uint64_t ReclaimPagesSucceeded; + uint64_t ReclaimPagesFailed; + uint64_t ReclaimReentrantSkips; +} MM_PFN_STATS; + +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage); +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage); +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive, + MM_PFN_USAGE usage); +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage); +__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnAddMapping(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnRemoveMapping(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnMoveToStandby(PHYSICAL_ADDRESS frame); +__declspec(dllexport) KSTATUS MmPfnActivate(PHYSICAL_ADDRESS frame); +__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame); +__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame); +size_t MmPfnIndex(PHYSICAL_ADDRESS frame); +PHYSICAL_ADDRESS MmPfnAddress(size_t pfn); +void MmPfnQueryStats(MM_PFN_STATS* out); diff --git a/code/kernel/include/kernel/mm_phys.h b/code/kernel/include/kernel/mm_phys.h index 998a843..cc44ff8 100644 --- a/code/kernel/include/kernel/mm_phys.h +++ b/code/kernel/include/kernel/mm_phys.h @@ -1,8 +1,9 @@ -// Kernel mm phys declarations shared by implementation files and kernel clients. +// Physical-memory declarations shared by the kernel and drivers. +// The physical layer publishes page-size constants, boot memory-map ingestion, +// and aggregate memory statistics. Frame ownership lives in the PFN database. #pragma once -// mm_phys.h - physical memory manager (PMM) -// Bitmap allocator over 4 KB frames, seeded from MEMORY_MAP. + #include #include #include @@ -22,22 +23,7 @@ typedef struct MM_PHYSICAL_STATS { uint64_t FreeRunHistogram[8]; } MM_PHYSICAL_STATS; -// Called once by KernelMain with the MEMORY_MAP from BOOT_INFO. -bool MmInitializePhysical(const MEMORY_MAP* map, - uint64_t physicalWindowBase); - -// Allocate / free a single 4 KB frame. -// MmAllocateFrame returns 0 on failure. -__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void); -PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive); -PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive, - PHYSICAL_ADDRESS maxExclusive); -void MmFreeFrame(PHYSICAL_ADDRESS frame); - -// Allocate 'count' physically contiguous 4 KB frames. -// Returns 0 on failure. Use sparingly (DMA, page tables). -__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateContiguous(size_t count); - -size_t MmFreeFrameCount(void); -size_t MmTotalFrameCount(void); -void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out); +bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase); +size_t MmFreeFrameCount(void); +size_t MmTotalFrameCount(void); +void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out); diff --git a/code/kernel/include/kernel/mm_pte.h b/code/kernel/include/kernel/mm_pte.h new file mode 100644 index 0000000..f357884 --- /dev/null +++ b/code/kernel/include/kernel/mm_pte.h @@ -0,0 +1,105 @@ +// Kernel page-table manager declarations shared by the VMM and low-level kernel clients. +#pragma once + +#include +#include +#include +#include + +// x86-64 PTE bits exposed to the rest of the kernel. +#define PAGE_PRESENT (1ULL << 0) +#define PAGE_WRITE (1ULL << 1) +#define PAGE_USER (1ULL << 2) +#define PAGE_WRITETHROUGH (1ULL << 3) +#define PAGE_NOCACHE (1ULL << 4) +#define PAGE_ACCESSED (1ULL << 5) +#define PAGE_DIRTY (1ULL << 6) +#define PAGE_NX (1ULL << 63) + +// The physical direct map is a kernel-only alias used to reach page-table frames and early physical memory. +#define PHYS_MAP_BASE 0xFFFF880000000000ULL + + +// Page-table telemetry separates cumulative operations from the current number +// of 4 KiB leaf mappings owned by this layer. Huge boot mappings may exist but +// are not counted as leaves here. +typedef enum MM_PTE_QUERY_PURPOSE { + MmPteQueryGeneric = 0, + MmPteQueryUserProbe, + MmPteQueryHeap, + MmPteQueryVad, + MmPteQueryDebug, + MmPteQuerySelfTest, + MmPteQueryPurposeCount, +} MM_PTE_QUERY_PURPOSE; + +typedef struct MM_PTE_STATS { + uint64_t MapCalls; + uint64_t MapFailures; + uint64_t UnmapCalls; + uint64_t UnmapFailures; + uint64_t ProtectCalls; + uint64_t ProtectFailures; + uint64_t QueryCalls; + uint64_t QueryFailures; + uint64_t QueryByPurpose[MmPteQueryPurposeCount]; + uint64_t QueryFailuresByPurpose[MmPteQueryPurposeCount]; + uint64_t TranslateCalls; + uint64_t LocalFlushes; + uint64_t GlobalFlushes; + uint64_t ShootdownRequests; + uint64_t ShootdownTargetCpus; + uint64_t ShootdownSkippedNoTargets; + uint64_t ShootdownAcks; + uint64_t ShootdownTimeouts; + uint64_t NewMappingNoFlushes; + uint64_t ReplacedMappingFlushes; + uint64_t PageTablePagesAllocated; + uint64_t CurrentLeafMappings; +} MM_PTE_STATS; + +typedef enum MM_PTE_MAP_MODE { + MmPteMapFailIfPresent = 0, + MmPteMapReplaceExisting = 1, +} MM_PTE_MAP_MODE; + +// Initializes the page-table manager around the bootloader-created PML4. +void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys); + +// Installs the shared PML4 into CR3. +void MmPteActivate(void); + +// Maps a 4 KiB page into the shared address space. +__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode); + +// Removes a 4 KiB mapping from the shared address space. +__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt); + +// Replaces the flags on every mapped 4 KiB page in the range while preserving the physical frame. +__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags); + +// Tests and clears the hardware accessed bit on one mapped 4 KiB leaf. +// Returns nonzero when the page had been accessed and was given a second chance. +__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt); + +// Tests and clears the hardware dirty bit on one mapped 4 KiB leaf. +__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt); + +// Translates a virtual address through the current shared page tables. +__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt); + +// Returns the physical address and effective flags for a mapped virtual address. +__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut); +KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose, uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut); + +// Flushes one address from the local TLB. +void MmPteFlushLocal(uintptr_t virt); + +// Flushes a kernel mapping from every online CPU that shares the kernel page tables. +void MmPteFlushKernelAddress(uintptr_t virt); + +// Handles the fixed-vector NMI used for kernel TLB shootdowns. +int MmPteHandleShootdownNmi(void); + +// Returns page-table operation counters used by memory telemetry. +void MmPteQueryStats(MM_PTE_STATS* out); diff --git a/code/kernel/include/kernel/mm_swap.h b/code/kernel/include/kernel/mm_swap.h new file mode 100644 index 0000000..0b61c29 --- /dev/null +++ b/code/kernel/include/kernel/mm_swap.h @@ -0,0 +1,31 @@ +// Kernel swap manager declarations. +// Swap owns pagefile slot allocation and page-sized I/O for anonymous VAD pages. + +#pragma once +#include +#include +#include +#include +#include + +typedef struct MM_SWAP_STATS { + uint64_t Enabled; + uint64_t TotalSlots; + uint64_t UsedSlots; + uint64_t FreeSlots; + uint64_t HighWaterSlots; + uint64_t PageOuts; + uint64_t PageIns; + uint64_t PageOutFailures; + uint64_t PageInFailures; + uint64_t SlotAllocations; + uint64_t SlotFrees; + uint64_t SlotAllocationFailures; +} MM_SWAP_STATS; + +KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes); +int MmSwapIsEnabled(void); +KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut); +KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame); +void MmSwapFreeSlot(uint32_t slot); +void MmSwapQueryStats(MM_SWAP_STATS* out); diff --git a/code/kernel/include/kernel/mm_virt.h b/code/kernel/include/kernel/mm_virt.h index 389d934..3fc229b 100644 --- a/code/kernel/include/kernel/mm_virt.h +++ b/code/kernel/include/kernel/mm_virt.h @@ -2,50 +2,43 @@ #pragma once // mm_virt.h - virtual memory manager (VMM) -// Manages the single shared PML4, kernel pool, and page-granular user VM regions. +// Manages the single shared PML4, kernel pool, and VAD-backed user VM ownership. #include #include #include +#include #include struct PROCESS; typedef struct PROCESS PROCESS; +struct FILE_OBJECT; +typedef struct FILE_OBJECT FILE_OBJECT; +struct IMAGE_SECTION_OBJECT; +typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT; -typedef enum MM_MAP_MODE { - MmMapFailIfPresent = 0, - MmMapReplaceExisting = 1, -} MM_MAP_MODE; +typedef enum MM_VAD_TYPE { + MmVadGeneric = 0, + MmVadHeap, + MmVadImage, + MmVadStack, + MmVadTeb, + MmVadPeb, + MmVadEnvironment, + MmVadMappedFile, + MmVadShared, +} MM_VAD_TYPE; -typedef enum USER_REGION_TYPE { - UserRegionGeneric = 0, - UserRegionHeap, - UserRegionImage, - UserRegionStack, - UserRegionTeb, - UserRegionPeb, - UserRegionEnvironment, - UserRegionMappedFile, - UserRegionShared, -} USER_REGION_TYPE; - -#define USER_REGION_GUARD_LOW 0x00000001u -#define USER_REGION_GUARD_HIGH 0x00000002u -#define USER_REGION_ZERO 0x00000004u - -// Page flags (x86-64 PTE bits). -#define PAGE_PRESENT (1ULL << 0) -#define PAGE_WRITE (1ULL << 1) -#define PAGE_USER (1ULL << 2) -#define PAGE_WRITETHROUGH (1ULL << 3) -#define PAGE_NOCACHE (1ULL << 4) -#define PAGE_NX (1ULL << 63) +#define MM_VAD_GUARD_LOW 0x00000001u +#define MM_VAD_GUARD_HIGH 0x00000002u +#define MM_VAD_ZERO 0x00000004u +#define MM_VAD_PREFETCH 0x00000008u +#define MM_VAD_SHARED_WRITE 0x00000010u // Virtual address space constants. #define USER_BASE 0x0000000000001000ULL #define USER_LIMIT 0x00007FFFFFFFF000ULL #define KERNEL_BASE 0xFFFF800000000000ULL -#define PHYS_MAP_BASE 0xFFFF880000000000ULL #define POOL_BASE 0xFFFFC00000000000ULL #define POOL_LIMIT 0xFFFFE00000000000ULL @@ -55,37 +48,6 @@ typedef enum USER_REGION_TYPE { // poolSize: initial reservation for kernel pool (bytes, page-aligned). void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize); -// Install the shared PML4 into CR3. Called once at the end of KernelMain init. -void MmActivate(void); - -// Page mapping. -__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode); -__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt); - -// Modify protection flags on an already-mapped virtual range. -// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags -// entirely (except the physical address, which is preserved). -// Returns STATUS_NOT_FOUND if any page in the range is not mapped. -__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags); - -// Walk the page tables to translate a virtual address to physical. -// Returns 0 if the address is not mapped. -__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt); - -// TLB/cache coherency hooks for the shared kernel address space. Local flushes -// use INVLPG directly; kernel mapping changes broadcast an NMI shootdown to -// other online CPUs and wait for acknowledgements. -void MmFlushTlbLocal(uintptr_t virt); -void MmFlushTlbForKernelMapping(uintptr_t virt); -int MmHandleTlbShootdownNmi(void); - - -// User/kernel copy boundary helpers -// MmQueryPage returns the physical address and effective PAGE_* bits for a -// mapped virtual address. The reported PAGE_USER/PAGE_WRITE bits account for -// every page-table level, not just the leaf entry. -__declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut); - // Probe a user virtual range without copying it. These functions only accept // lower-half user mappings with PAGE_USER set; MmProbeUserWrite additionally // requires PAGE_WRITE. @@ -97,31 +59,96 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size); __declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size); __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size); +typedef struct MM_PINNED_USER_PAGE { + uintptr_t UserPage; + size_t PageOffset; + PHYSICAL_ADDRESS Frame; + void* KernelAddress; + uint64_t PageFlags; +} MM_PINNED_USER_PAGE; + +typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER; + +// Pins one user page and returns a kernel direct-map alias for the requested +// address. Callers may copy through KernelAddress + PageOffset while the page is +// pinned. The pin is short-lived: never sleep, do file I/O, or hold it across +// arbitrary callbacks. +__declspec(dllexport) KSTATUS MmPinUserPageForRead(PROCESS* process, const void* userAddress, MM_PINNED_USER_PAGE* out); +__declspec(dllexport) KSTATUS MmPinUserPageForWrite(PROCESS* process, void* userAddress, MM_PINNED_USER_PAGE* out); +__declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page); + +// Locks a user buffer for asynchronous kernel access. The containing VADs stay +// alive and protected, and resident PFNs stay pinned until the buffer is +// unlocked. WriteAccess describes kernel writes to the user buffer. +__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out); +__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer); + // Kernel pool - ring 0 only, never PAGE_USER. // Best-fit free-list allocator, thread-safe via spinlock. +#define MM_POOL_TAG(a, b, c, d) \ + ((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \ + ((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24)) + __declspec(dllexport) void* MmAllocatePool(size_t size); +__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag); __declspec(dllexport) void MmFreePool(void* ptr); __declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize); -// User virtual regions. -// The kernel has one shared user VA arena, but every user mapping is owned by -// a PROCESS so process teardown can reclaim leaked memory. Guard pages are -// reserved VA with no present PTE. -KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment, - uint64_t pageFlags, uint32_t regionFlags, - USER_REGION_TYPE type, void** outBase); -KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags); -KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags); -KSTATUS MmDestroyUserRegion(PROCESS* process, void* base); -void MmDestroyAllUserRegions(PROCESS* process); -KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, USER_REGION_TYPE* typeOut, uint32_t* regionFlagsOut); -void MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t* reservedBytesOut, size_t* committedBytesOut); -size_t MmGetUserRegionSize(PROCESS* process, const void* base); -size_t MmGetUserRegionSizeByBase(const void* base); +// Virtual address descriptors. +// The single shared address space still has per-process VAD ownership so process +// teardown can reclaim leaked mappings. VADs track sparse per-page commitment, +// and committed pages are materialized by user faults or explicit prefaulting. +KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment, + uint64_t pageFlags, uint32_t vadFlags, + MM_VAD_TYPE type, void** outBase); +KSTATUS MmVadCommit(PROCESS* process, void* base, size_t size, + uint64_t pageFlags, uint32_t commitFlags); +KSTATUS MmVadCreateCommitted(PROCESS* process, size_t size, size_t alignment, + uint64_t pageFlags, uint32_t vadFlags, + MM_VAD_TYPE type, void** outBase); +KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file, + uint64_t fileOffset, size_t size, + uint64_t pageFlags, uint32_t vadFlags, + void** outBase); +KSTATUS MmVadCreateImageSectionMapping(PROCESS* process, IMAGE_SECTION_OBJECT* section, + uint64_t pageFlags, uint32_t vadFlags, + void** outBase); +KSTATUS MmVadEnsureResident(PROCESS* process, void* base, size_t size, uint64_t accessFlags); +KSTATUS MmVadMakeImageRangePrivate(PROCESS* process, void* base, size_t size); +KSTATUS MmVadSyncFileMappings(FILE_OBJECT* file); +KSTATUS MmVadCreateCowView(PROCESS* sourceProcess, void* sourceBase, size_t size, + PROCESS* targetProcess, void** outBase); +KSTATUS MmVadCreateCowViewFromFrames(PROCESS* targetProcess, + const PHYSICAL_ADDRESS* frames, + size_t pageCount, + size_t payloadSize, + uint64_t pageFlags, + uint32_t vadFlags, + MM_VAD_TYPE type, + void** outBase); +KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess, + const PHYSICAL_ADDRESS* frames, + size_t pageCount, + size_t payloadSize, + uint64_t pageFlags, + uint32_t vadFlags, + MM_VAD_TYPE type, + void** outBase); +KSTATUS MmVadResolveUserPageFault(PROCESS* process, uintptr_t faultAddress, uint64_t errorCode); +KSTATUS MmVadResolvePageFaultByAddress(uintptr_t faultAddress, uint64_t errorCode); +KSTATUS MmVadTrimOnePageForTest(PROCESS* process, void** baseOut); +KSTATUS MmVadTrimOnePageForPressure(void); +KSTATUS MmVadProtect(PROCESS* process, void* base, size_t size, uint64_t pageFlags); +KSTATUS MmVadDestroy(PROCESS* process, void* base); +void MmVadDestroyAllForProcess(PROCESS* process); +KSTATUS MmVadQuery(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, MM_VAD_TYPE* typeOut, uint32_t* vadFlagsOut); +void MmVadQueryProcessMemory(PROCESS* process, size_t* vadCountOut, size_t* reservedBytesOut, size_t* committedBytesOut); +size_t MmVadGetSize(PROCESS* process, const void* base); +size_t MmVadGetSizeByBase(const void* base); // Debug-only ownership/address diagnostics used by fault and launch paths. void MmDebugDescribeAddress(uintptr_t addr); -void MmDebugDumpUserRegions(PROCESS* process); +void MmVadDebugDumpProcess(PROCESS* process); __declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size); __declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr); @@ -140,3 +167,40 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size); size_t MmGetKernelPoolUsage(void); size_t MmGetUserHeapUsage(void); + +// Demand-paging telemetry used by the shell stats command. +typedef struct MM_VAD_STATS { + uint64_t ResolveFaultCalls; + uint64_t ResolvedFaults; + uint64_t FailedFaults; + uint64_t ProtectionFaults; + uint64_t NoVadFaults; + uint64_t UncommittedFaults; + uint64_t NoAccessFaults; + uint64_t DemandZeroPages; + uint64_t ZeroPageMaps; + uint64_t ZeroPageCowSplits; + uint64_t CowFaults; + uint64_t CowPagesCopied; + uint64_t CowViewsCreated; + uint64_t CowPagesShared; + uint64_t EnsureResidentCalls; + uint64_t EnsureResidentPages; + uint64_t AlreadyResidentPages; + uint64_t FileBackedFaults; + uint64_t FileBackedReads; + uint64_t FileBackedReadFailures; + uint64_t FileBackedDiscards; + uint64_t FileBackedSharedWritableFaults; + uint64_t FileBackedDirtyFaults; + uint64_t FileBackedDirtySyncs; + uint64_t ImageBackedFaults; + uint64_t ImageBackedZeroFaults; + uint64_t ImageBackedPrivateFaults; + uint64_t ImageFixupPrivateCalls; + uint64_t ImageFixupPrivatePages; + uint64_t SwapPageOuts; + uint64_t SwapPageIns; +} MM_VAD_STATS; + +void MmVadQueryStats(MM_VAD_STATS* out); diff --git a/code/kernel/include/kernel/ob_object.h b/code/kernel/include/kernel/ob_object.h index 67b8bfc..ab09519 100644 --- a/code/kernel/include/kernel/ob_object.h +++ b/code/kernel/include/kernel/ob_object.h @@ -23,6 +23,10 @@ typedef enum OBJECT_TYPE { ObjectTypeDriver = 12, ObjectTypeUsbDevice = 13, ObjectTypeUsbPipe = 14, + ObjectTypeFileMapping = 15, + ObjectTypeSurface = 16, + ObjectTypeInputQueue = 17, + ObjectTypeImageSection = 18, } OBJECT_TYPE; typedef struct OBJECT { diff --git a/code/kernel/include/kernel/syscall_nr.h b/code/kernel/include/kernel/syscall_nr.h index a05a408..ba67364 100644 --- a/code/kernel/include/kernel/syscall_nr.h +++ b/code/kernel/include/kernel/syscall_nr.h @@ -11,13 +11,13 @@ #define SYS_EXIT_PROCESS 0 #define SYS_EXIT_THREAD 1 #define SYS_CREATE_PROCESS 2 -#define SYS_WAIT_FOR_PROCESS 3 +#define SYS_WAIT_SINGLE 3 #define SYS_GET_PROCESS_EXIT_CODE 4 #define SYS_KILL_PROCESS 5 #define SYS_GET_PROCESS_LIST 6 #define SYS_GET_PROCESS_PID 7 #define SYS_CREATE_THREAD 8 -#define SYS_WAIT_THREAD 9 +#define SYS_RETIRED_WAIT_THREAD 9 #define SYS_GET_CURRENT_TID 10 #define SYS_GET_CURRENT_PID 11 #define SYS_SET_THREAD_PRIORITY 12 @@ -31,17 +31,17 @@ #define SYS_QUERY_VM 20 #define SYS_CLOSE_HANDLE 21 #define SYS_CREATE_MUTEX 22 -#define SYS_ACQUIRE_MUTEX 23 +#define SYS_RETIRED_ACQUIRE_MUTEX 23 #define SYS_TRY_ACQUIRE_MUTEX 24 #define SYS_RELEASE_MUTEX 25 #define SYS_CREATE_SEM 26 -#define SYS_WAIT_SEM 27 +#define SYS_RETIRED_WAIT_SEM 27 #define SYS_TRY_WAIT_SEM 28 #define SYS_SIGNAL_SEM 29 #define SYS_CREATE_EVENT 30 #define SYS_SIGNAL_EVENT 31 #define SYS_CLEAR_EVENT 32 -#define SYS_WAIT_EVENT 33 +#define SYS_RETIRED_WAIT_EVENT 33 #define SYS_CREATE_QUEUE 34 #define SYS_SEND_QUEUE 35 #define SYS_RECV_QUEUE 36 @@ -74,36 +74,47 @@ #define SYS_FILL_CONSOLE_CHARACTER 63 #define SYS_FILL_CONSOLE_ATTRIBUTE 64 #define SYS_PUT_SCREEN_CHAR 65 -#define SYS_BLIT_SCREEN 66 -#define SYS_CLEAR_SCREEN 67 -#define SYS_GET_SCREEN_SIZE 68 -#define SYS_QUERY_DISPLAY_INFO 69 -#define SYS_POLL_KEYBOARD 70 -#define SYS_IS_KEY_DOWN 71 -#define SYS_READ_KEYBOARD_CHAR 72 -#define SYS_READ_KEY 73 -#define SYS_GET_KEY_MODIFIERS 74 -#define SYS_POLL_MOUSE 75 -#define SYS_READ_MOUSE 76 -#define SYS_GET_MOUSE_POS 77 -#define SYS_GET_TICK 78 -#define SYS_PRINT_DEBUG 79 -#define SYS_QUERY_TRAP_FRAME 80 -#define SYS_CREATE_SOCKET 81 -#define SYS_BIND_SOCKET 82 -#define SYS_LISTEN_SOCKET 83 -#define SYS_ACCEPT_SOCKET 84 -#define SYS_CONNECT_SOCKET 85 -#define SYS_SEND_SOCKET 86 -#define SYS_RECV_SOCKET 87 -#define SYS_SHUTDOWN_SOCKET 88 -#define SYS_SEND_TO_SOCKET 89 -#define SYS_RECV_FROM_SOCKET 90 -#define SYS_RESOLVE_HOST 91 -#define SYS_SET_SOCKET_OPTION 92 -#define SYS_GET_SOCKET_OPTION 93 -#define SYS_POLL 94 -#define SYS_GET_MEMORY_STATUS 95 -#define SYS_QUERY_KERNEL_STATS 96 -#define SYS_GET_SYSTEM_TIME 97 -#define SYS_MAX 98 +#define SYS_CLEAR_SCREEN 66 +#define SYS_GET_SCREEN_SIZE 67 +#define SYS_QUERY_DISPLAY_INFO 68 +#define SYS_POLL_KEYBOARD 69 +#define SYS_IS_KEY_DOWN 70 +#define SYS_READ_KEYBOARD_CHAR 71 +#define SYS_READ_KEY 72 +#define SYS_GET_KEY_MODIFIERS 73 +#define SYS_POLL_MOUSE 74 +#define SYS_READ_MOUSE 75 +#define SYS_GET_MOUSE_POS 76 +#define SYS_GET_TICK 77 +#define SYS_PRINT_DEBUG 78 +#define SYS_QUERY_TRAP_FRAME 79 +#define SYS_CREATE_SOCKET 80 +#define SYS_BIND_SOCKET 81 +#define SYS_LISTEN_SOCKET 82 +#define SYS_ACCEPT_SOCKET 83 +#define SYS_CONNECT_SOCKET 84 +#define SYS_SEND_SOCKET 85 +#define SYS_RECV_SOCKET 86 +#define SYS_SHUTDOWN_SOCKET 87 +#define SYS_SEND_TO_SOCKET 88 +#define SYS_RECV_FROM_SOCKET 89 +#define SYS_RESOLVE_HOST 90 +#define SYS_SET_SOCKET_OPTION 91 +#define SYS_GET_SOCKET_OPTION 92 +#define SYS_POLL 93 +#define SYS_GET_MEMORY_STATUS 94 +#define SYS_QUERY_KERNEL_STATS 95 +#define SYS_GET_SYSTEM_TIME 96 +#define SYS_CREATE_FILE_MAPPING 97 +#define SYS_MAP_FILE 98 +#define SYS_CREATE_SURFACE 99 +#define SYS_MAP_SURFACE 100 +#define SYS_QUERY_SURFACE 101 +#define SYS_PRESENT_SURFACE 102 +#define SYS_GET_THREAD_LIST 103 +#define SYS_READ_INPUT_EVENT 104 +#define SYS_PEEK_INPUT_EVENT 105 +#define SYS_WAIT_MANY 106 +#define SYS_OPEN_INPUT_QUEUE 107 +#define SYS_FLUSH_FILE 108 +#define SYS_MAX 109 diff --git a/code/kernel/io_device.c b/code/kernel/io_device.c index c44cf91..89aeb71 100644 --- a/code/kernel/io_device.c +++ b/code/kernel/io_device.c @@ -7,6 +7,7 @@ #include #include #include +#include #define MAX_DRIVER_OBJECTS 64 #define MAX_PDO_OBJECTS 128 @@ -443,6 +444,170 @@ static uint32_t gDiskChangeCallbackCount; static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS]; static uint32_t gDiskArrivalCallbackCount; +#define IO_DISK_READAHEAD_MAX_SECTORS 128u +#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u +#define IO_DISK_READAHEAD_MIN_SECTORS 8u +#define IO_DISK_SECTOR_BYTES 512u + +typedef struct IO_DISK_READAHEAD_STATE { + SPINLOCK Lock; + uint8_t* Buffer; + uint8_t Valid; + uint8_t Loading; + uint64_t Generation; + uint64_t BaseLba; + uint32_t Count; + uint64_t NextLba; + uint32_t SequentialScore; + IO_DISK_STATS Stats; +} IO_DISK_READAHEAD_STATE; + +static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES]; + +// Updates a maximum statistic while holding the disk read-ahead lock. +static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) { + if (field && *field < value) *field = value; +} + +// Prepares the per-disk read-ahead buffer. +static void IoDiskReadAheadPrepare(uint8_t driveIndex) { + if (driveIndex >= MAX_DISK_DEVICES) + return; + + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + state->Lock = (SPINLOCK)SPINLOCK_INIT; + state->Valid = 0; + state->Loading = 0; + state->Generation = 0; + state->BaseLba = 0; + state->Count = 0; + state->NextLba = 0; + state->SequentialScore = 0; + memset(&state->Stats, 0, sizeof(state->Stats)); + + if (!state->Buffer) { + state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES, + MM_POOL_TAG('d', 'r', 'a', 'h')); + if (!state->Buffer) + state->Stats.ReadaheadAllocFailures++; + } +} + +// Releases the per-disk read-ahead buffer. +static void IoDiskReadAheadRelease(uint8_t driveIndex) { + if (driveIndex >= MAX_DISK_DEVICES) + return; + + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + uint8_t* buffer = (uint8_t*)0; + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + if (!state->Loading) { + buffer = state->Buffer; + state->Buffer = (uint8_t*)0; + state->Valid = 0; + state->Generation++; + state->Count = 0; + state->SequentialScore = 0; + KeReleaseSpinlock(&state->Lock, flags); + break; + } + KeReleaseSpinlock(&state->Lock, flags); + KeYield(); + } + + if (buffer) + MmFreePool(buffer); +} + +// Invalidates cached disk read-ahead data for one disk. +static void IoDiskReadAheadInvalidate(uint8_t driveIndex) { + if (driveIndex >= MAX_DISK_DEVICES) + return; + + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + state->Valid = 0; + state->Generation++; + state->Count = 0; + state->Stats.ReadaheadInvalidations++; + KeReleaseSpinlock(&state->Lock, flags); +} + +// Returns true when one range fully contains another range. +static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount, + uint64_t innerBase, uint32_t innerCount) { + uint64_t outerEnd = outerBase + outerCount; + uint64_t innerEnd = innerBase + innerCount; + if (outerEnd < outerBase || innerEnd < innerBase) + return false; + return innerBase >= outerBase && innerEnd <= outerEnd; +} + +// Attempts to satisfy a disk read from the per-disk read-ahead buffer. +static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) { + if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0) + return false; + + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + bool hit = state->Buffer && state->Valid && !state->Loading && + IoDiskRangeContains(state->BaseLba, state->Count, lba, count); + if (hit) { + size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES; + size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES; + memcpy(buffer, state->Buffer + offset, bytes); + state->Stats.ReadaheadHits++; + state->Stats.ReadaheadBytesServed += bytes; + } else { + state->Stats.ReadaheadMisses++; + } + KeReleaseSpinlock(&state->Lock, flags); + return hit; +} + +// Records request size and sequential access state for one disk read. +static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) { + if (driveIndex >= MAX_DISK_DEVICES || count == 0) + return 0; + + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + state->Stats.ReadRequests++; + state->Stats.ReadSectors += count; + IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count); + + if (lba == state->NextLba) { + if (state->SequentialScore < 32) + state->SequentialScore++; + } else { + state->SequentialScore = 1; + } + state->NextLba = lba + count; + uint32_t score = state->SequentialScore; + if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE) + state->Stats.ReadaheadSequentialDetections++; + KeReleaseSpinlock(&state->Lock, flags); + return score; +} + +// Records one disk write and invalidates stale read-ahead data. +static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) { + if (driveIndex >= MAX_DISK_DEVICES) + return; + + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + state->Stats.WriteRequests++; + state->Stats.WriteSectors += count; + IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count); + state->Valid = 0; + state->Generation++; + state->Count = 0; + state->Stats.ReadaheadInvalidations++; + KeReleaseSpinlock(&state->Lock, flags); +} + // Implements the notify Disk Change operation. static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) { for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) { @@ -477,6 +642,8 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, if (pnpStatus == STATUS_NOT_SUPPORTED) device->PnpState = DevicePnpStarted; + IoDiskReadAheadPrepare(driveIndex); + IoReferenceDeviceObject(device); gDiskDevices[driveIndex] = device; IoNotifyDiskChange(driveIndex, device, IoDeviceArrived); @@ -490,6 +657,8 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) { DEVICE_OBJECT* dev = gDiskDevices[driveIndex]; if (!dev) return STATUS_NOT_FOUND; + IoDiskReadAheadInvalidate(driveIndex); + dev->PnpState = DevicePnpSurpriseRemoved; IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved); (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL); @@ -518,6 +687,7 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) { (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE); gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0; + IoDiskReadAheadRelease(driveIndex); IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved); IoDereferenceDeviceObject(dev); return STATUS_SUCCESS; @@ -632,6 +802,112 @@ __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) { return st; } +// Validates that a sector range fits the disk and IRP byte length. +static KSTATUS IoValidateDiskSectorRange(DEVICE_OBJECT* dev, uint64_t lba, uint32_t count) { + if (!dev || count == 0) return STATUS_INVALID_PARAMETER; + if (count > (0xffffffffu / IO_DISK_SECTOR_BYTES)) return STATUS_INVALID_PARAMETER; + if (lba > dev->u.Disk.SectorCount) return STATUS_INVALID_PARAMETER; + if ((uint64_t)count > dev->u.Disk.SectorCount - lba) return STATUS_INVALID_PARAMETER; + return STATUS_SUCCESS; +} + +// Submits one raw disk read IRP. +static KSTATUS IoReadDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba, + uint32_t count, void* buffer) { + if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER; + + IRP irp; + IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer); + return IoSubmitIrpSync(dev, &irp); +} + +// Submits one raw disk write IRP. +static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba, + uint32_t count, void* buffer) { + if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER; + + IRP irp; + IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, buffer); + return IoSubmitIrpSync(dev, &irp); +} + +// Chooses a conservative read-ahead span from the current sequential score. +static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) { + uint32_t sectors = currentCount; + if (sectors < IO_DISK_READAHEAD_MIN_SECTORS) + sectors = IO_DISK_READAHEAD_MIN_SECTORS; + if (score >= 8 && sectors < 128u) + sectors = 128u; + else if (score >= 4 && sectors < 64u) + sectors = 64u; + else if (score >= 2 && sectors < 32u) + sectors = 32u; + if (sectors > IO_DISK_READAHEAD_MAX_SECTORS) + sectors = IO_DISK_READAHEAD_MAX_SECTORS; + return sectors; +} + +// Loads the next sequential disk range into the read-ahead buffer. +static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev, + uint64_t nextLba, uint32_t currentCount, uint32_t score) { + if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE) + return; + + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + if (!state->Buffer) { + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + state->Stats.ReadaheadSuppressed++; + KeReleaseSpinlock(&state->Lock, flags); + return; + } + + uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score); + uint64_t diskSectors = dev->u.Disk.SectorCount; + if (nextLba >= diskSectors) + return; + if ((uint64_t)sectors > diskSectors - nextLba) + sectors = (uint32_t)(diskSectors - nextLba); + if (sectors == 0) + return; + + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + bool covered = state->Valid && !state->Loading && + IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors); + if (covered) { + KeReleaseSpinlock(&state->Lock, flags); + return; + } + if (state->Loading) { + state->Stats.ReadaheadSuppressed++; + KeReleaseSpinlock(&state->Lock, flags); + return; + } + uint64_t generation = state->Generation; + state->Loading = 1; + state->Valid = 0; + state->Count = 0; + KeReleaseSpinlock(&state->Lock, flags); + + KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer); + + flags = KeAcquireSpinlock(&state->Lock); + state->Loading = 0; + if (KSUCCESS(st) && generation == state->Generation) { + state->BaseLba = nextLba; + state->Count = sectors; + state->Valid = 1; + state->Stats.ReadaheadStores++; + state->Stats.ReadaheadSectorsLoaded += sectors; + IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors); + } else { + state->Valid = 0; + state->Count = 0; + if (!KSUCCESS(st)) + state->Stats.ReadaheadReadFailures++; + } + KeReleaseSpinlock(&state->Lock, flags); +} + __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, uint64_t lba, uint32_t count, @@ -641,9 +917,23 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex); if (!dev) return STATUS_NOT_FOUND; - IRP irp; - IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer); - KSTATUS st = IoSubmitIrpSync(dev, &irp); + KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count); + if (!KSUCCESS(rangeStatus)) { + IoDereferenceDeviceObject(dev); + return rangeStatus; + } + + if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) { + uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count); + IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score); + IoDereferenceDeviceObject(dev); + return STATUS_SUCCESS; + } + + KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer); + uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count); + if (KSUCCESS(st)) + IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score); IoDereferenceDeviceObject(dev); return st; @@ -658,10 +948,50 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex); if (!dev) return STATUS_NOT_FOUND; - IRP irp; - IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer); - KSTATUS st = IoSubmitIrpSync(dev, &irp); + KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count); + if (!KSUCCESS(rangeStatus)) { + IoDereferenceDeviceObject(dev); + return rangeStatus; + } + + KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer); + if (KSUCCESS(st)) + IoDiskReadAheadNoteWrite(driveIndex, count); IoDereferenceDeviceObject(dev); return st; } + +// Queries aggregate disk I/O statistics. +__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) { + if (!out) return; + memset(out, 0, sizeof(*out)); + + for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) { + IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i]; + uintptr_t flags = KeAcquireSpinlock(&state->Lock); + if (gDiskDevices[i]) + out->RegisteredDisks++; + out->ReadRequests += state->Stats.ReadRequests; + out->ReadSectors += state->Stats.ReadSectors; + out->WriteRequests += state->Stats.WriteRequests; + out->WriteSectors += state->Stats.WriteSectors; + if (out->MaxReadSectors < state->Stats.MaxReadSectors) + out->MaxReadSectors = state->Stats.MaxReadSectors; + if (out->MaxWriteSectors < state->Stats.MaxWriteSectors) + out->MaxWriteSectors = state->Stats.MaxWriteSectors; + out->ReadaheadHits += state->Stats.ReadaheadHits; + out->ReadaheadMisses += state->Stats.ReadaheadMisses; + out->ReadaheadStores += state->Stats.ReadaheadStores; + out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations; + out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed; + out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded; + out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections; + out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed; + out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures; + out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures; + if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors) + out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors; + KeReleaseSpinlock(&state->Lock, flags); + } +} diff --git a/code/kernel/io_file.c b/code/kernel/io_file.c index a608942..e53c878 100644 --- a/code/kernel/io_file.c +++ b/code/kernel/io_file.c @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -13,6 +15,7 @@ #include #include #include +#include #define MAX_DRIVES 8 @@ -45,11 +48,15 @@ struct IO_ASYNC_FILE_REQUEST_OBJECT { uint8_t CancelRequested; FILE_OBJECT* File; // referenced until object destruction + EVENT* WaitEvent; // internal completion event EVENT* CompletionEvent; // optional, referenced until object destruction KAPC* CompletionApc; // caller-owned APC storage IO_STATUS_BLOCK* Iosb; // caller/user-owned completion block + IO_STATUS_BLOCK IosbSnapshot; + MM_LOCKED_USER_BUFFER* LockedBuffer; + MM_LOCKED_USER_BUFFER* LockedIosb; - void* Buffer; // caller/user-owned; must remain valid until completion/cancel + void* Buffer; // caller-owned; user buffers must be locked until completion/cancel size_t Size; uint64_t Offset; }; @@ -65,6 +72,10 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) { IO_ASYNC_FILE_REQUEST_OBJECT* req = CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header); + if (req->WaitEvent) { + ObDereferenceObject(&req->WaitEvent->Header); + req->WaitEvent = (EVENT*)0; + } if (req->CompletionEvent) { ObDereferenceObject(&req->CompletionEvent->Header); req->CompletionEvent = (EVENT*)0; @@ -73,6 +84,14 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) { ObDereferenceObject(&req->File->Header); req->File = (FILE_OBJECT*)0; } + if (req->LockedBuffer) { + MmUnlockUserBuffer(req->LockedBuffer); + req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0; + } + if (req->LockedIosb) { + MmUnlockUserBuffer(req->LockedIosb); + req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0; + } } static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, @@ -80,6 +99,9 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, size_t information) { if (!req) return; + req->IosbSnapshot.Status = status; + req->IosbSnapshot.Information = information; + uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock); req->State = (status == STATUS_CANCELLED) ? IoAsyncStateCancelled : IoAsyncStateCompleted; KeReleaseSpinlock(&gAsyncFileQueueLock, flags); @@ -99,6 +121,18 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, if (req->CompletionEvent) KeSignalEvent(req->CompletionEvent); + + if (req->WaitEvent) + KeSignalEvent(req->WaitEvent); + + if (req->LockedBuffer) { + MmUnlockUserBuffer(req->LockedBuffer); + req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0; + } + if (req->LockedIosb) { + MmUnlockUserBuffer(req->LockedIosb); + req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0; + } } // Implements the io Async File Dequeue operation. @@ -226,10 +260,24 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op, IO_STATUS_BLOCK* iosb, EVENT* completionEvent, KAPC* completionApc, + MM_LOCKED_USER_BUFFER* lockedBuffer, + MM_LOCKED_USER_BUFFER* lockedIosb, IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) { - if (!file || !iosb) return STATUS_INVALID_PARAMETER; - if (!buffer && size) return STATUS_INVALID_PARAMETER; - if (!KeCanWait()) return STATUS_INVALID_IRQL; + if (!file || !iosb) { + MmUnlockUserBuffer(lockedBuffer); + MmUnlockUserBuffer(lockedIosb); + return STATUS_INVALID_PARAMETER; + } + if (!buffer && size) { + MmUnlockUserBuffer(lockedBuffer); + MmUnlockUserBuffer(lockedIosb); + return STATUS_INVALID_PARAMETER; + } + if (!KeCanWait()) { + MmUnlockUserBuffer(lockedBuffer); + MmUnlockUserBuffer(lockedIosb); + return STATUS_INVALID_IRQL; + } iosb->Status = STATUS_PENDING; iosb->Information = 0; @@ -239,6 +287,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op, sizeof(IO_ASYNC_FILE_REQUEST_OBJECT)); if (!req) { iosb->Status = STATUS_NO_MEMORY; + MmUnlockUserBuffer(lockedBuffer); + MmUnlockUserBuffer(lockedIosb); return STATUS_NO_MEMORY; } @@ -247,10 +297,23 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op, req->Op = op; req->State = IoAsyncStateQueued; req->CancelRequested = 0; + KSTATUS waitStatus = KeEventCreate(EventManualReset, 0, &req->WaitEvent); + if (!KSUCCESS(waitStatus)) { + iosb->Status = waitStatus; + MmUnlockUserBuffer(lockedBuffer); + MmUnlockUserBuffer(lockedIosb); + ObDereferenceObject(&req->Header); + return waitStatus; + } + req->File = file; req->CompletionEvent = completionEvent; req->CompletionApc = completionApc; req->Iosb = iosb; + req->IosbSnapshot.Status = STATUS_PENDING; + req->IosbSnapshot.Information = 0; + req->LockedBuffer = lockedBuffer; + req->LockedIosb = lockedIosb; req->Buffer = buffer; req->Size = size; req->Offset = offset; @@ -288,6 +351,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op, ObDereferenceObject(&req->Header); // provider rejected synchronously iosb->Status = ns; + req->IosbSnapshot.Status = ns; + req->IosbSnapshot.Information = 0; ObDereferenceObject(&req->Header); // creator reference return ns; } @@ -295,6 +360,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op, KSTATUS s = KiIoEnsureAsyncFileIoStarted(); if (!KSUCCESS(s)) { iosb->Status = s; + req->IosbSnapshot.Status = s; + req->IosbSnapshot.Information = 0; ObDereferenceObject(&req->Header); return s; } @@ -323,10 +390,12 @@ __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf IO_STATUS_BLOCK* iosb, EVENT* completionEvent, KAPC* completionApc, + MM_LOCKED_USER_BUFFER* lockedBuffer, + MM_LOCKED_USER_BUFFER* lockedIosb, IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) { return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset, iosb, completionEvent, completionApc, - requestOut); + lockedBuffer, lockedIosb, requestOut); } __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf, @@ -334,10 +403,12 @@ __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const vo IO_STATUS_BLOCK* iosb, EVENT* completionEvent, KAPC* completionApc, + MM_LOCKED_USER_BUFFER* lockedBuffer, + MM_LOCKED_USER_BUFFER* lockedIosb, IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) { return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset, iosb, completionEvent, completionApc, - requestOut); + lockedBuffer, lockedIosb, requestOut); } // Implements the async File Request Object operation. @@ -345,6 +416,19 @@ __declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJ return req ? &req->Header : (OBJECT*)0; } + +// Returns the wait event owned by an async file request. +__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) { + return req ? req->WaitEvent : (EVENT*)0; +} + +// Waits for an async file request to complete according to wait flags. +__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, uint32_t flags) { + if (!req || !req->WaitEvent) return STATUS_INVALID_PARAMETER; + if (flags & ~SYS_WAIT_FLAG_ALERTABLE) return STATUS_INVALID_PARAMETER; + return KeWaitForEvent(req->WaitEvent, (flags & SYS_WAIT_FLAG_ALERTABLE) ? 1 : 0); +} + // Cancels async File Request. __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) { if (!req) return STATUS_INVALID_PARAMETER; @@ -377,13 +461,8 @@ __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJ __declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, IO_STATUS_BLOCK* iosbOut) { if (!req || !iosbOut) return STATUS_INVALID_PARAMETER; - if (req->Iosb) { - iosbOut->Status = req->Iosb->Status; - iosbOut->Information = req->Iosb->Information; - } else { - iosbOut->Status = STATUS_FAILURE; - iosbOut->Information = 0; - } + iosbOut->Status = req->IosbSnapshot.Status; + iosbOut->Information = req->IosbSnapshot.Information; return STATUS_SUCCESS; } @@ -622,10 +701,51 @@ __declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) { return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0; } +// Aggregates filesystem traversal statistics across mounted volumes. +__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) { + if (!out) return; + memset(out, 0, sizeof(*out)); + + for (uint8_t i = 0; i < MAX_DRIVES; i++) { + IO_FILESYSTEM* fs = gVolumes[i].Fs; + if (!fs || !fs->QueryStats) continue; + + IO_FILESYSTEM_STATS one; + memset(&one, 0, sizeof(one)); + fs->QueryStats(fs, &one); + + out->PathLookups += one.PathLookups; + out->PathCacheHits += one.PathCacheHits; + out->PathCacheMisses += one.PathCacheMisses; + out->PathCacheStores += one.PathCacheStores; + out->PathCacheInvalidations += one.PathCacheInvalidations; + if (one.PathCacheGeneration > out->PathCacheGeneration) + out->PathCacheGeneration = one.PathCacheGeneration; + out->DirectorySearches += one.DirectorySearches; + out->DirectoryClustersRead += one.DirectoryClustersRead; + out->DirectoryEntriesScanned += one.DirectoryEntriesScanned; + out->FatChainCacheLookups += one.FatChainCacheLookups; + out->FatChainCacheHits += one.FatChainCacheHits; + out->FatChainCacheMisses += one.FatChainCacheMisses; + out->FatChainCacheStores += one.FatChainCacheStores; + out->FatChainCacheInvalidations += one.FatChainCacheInvalidations; + out->FatChainWalkSteps += one.FatChainWalkSteps; + out->DataReadRuns += one.DataReadRuns; + out->DataReadSectors += one.DataReadSectors; + out->DataReadRunMerges += one.DataReadRunMerges; + if (one.MaxDataReadSpanSectors > out->MaxDataReadSpanSectors) + out->MaxDataReadSpanSectors = one.MaxDataReadSpanSectors; + } +} + // Destructor. static void KiFileDestroy(OBJECT* obj) { FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header); - if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx); + if (f->Fs && f->FileCtx) { + (void)IoFlushFile(f); + MmFileCacheInvalidateAll(f); + f->Fs->Close(f->Fs, f->FileCtx); + } } // IoOpenFile. @@ -655,11 +775,73 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out f->Lock = (SPINLOCK)SPINLOCK_INIT; f->Size = fs->GetSize(fs, fileCtx); f->Offset = 0; + f->Flags = 0; + f->ReadaheadScore = 0; + f->ReadaheadNextOffset = 0; + f->ReadaheadIssuedUntil = 0; + f->ReadaheadSuppressedUntil = 0; *out = f; return STATUS_SUCCESS; } + +static uint64_t KiAlignUpPage64(uint64_t value) { + return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u); +} + +static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) { + if (!file || bytesRead < 512) + return; + if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0) + return; + + uint64_t end = offset + bytesRead; + uint64_t start = 0; + uint32_t pages = 0; + + uintptr_t flags = KeAcquireSpinlock(&file->Lock); + if (offset == file->ReadaheadNextOffset) { + if (file->ReadaheadScore < 32) + file->ReadaheadScore++; + } else { + file->ReadaheadScore = 1; + file->ReadaheadIssuedUntil = end; + } + file->ReadaheadNextOffset = end; + + if (end < file->ReadaheadSuppressedUntil) { + pages = 0; + } else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) { + pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u); + start = KiAlignUpPage64(end); + if (start < file->ReadaheadIssuedUntil) + start = KiAlignUpPage64(file->ReadaheadIssuedUntil); + if (start >= fileSize) { + pages = 0; + } else { + uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE); + if (maxEnd > fileSize) + pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE); + file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE); + } + } + KeReleaseSpinlock(&file->Lock, flags); + + if (pages) { + KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages); + if (ras == STATUS_QUEUE_FULL) { + flags = KeAcquireSpinlock(&file->Lock); + uint64_t suppressUntil = end + (1024u * 1024u); + if (suppressUntil < end || suppressUntil > fileSize) + suppressUntil = fileSize; + if (file->ReadaheadSuppressedUntil < suppressUntil) + file->ReadaheadSuppressedUntil = suppressUntil; + KeReleaseSpinlock(&file->Lock, flags); + } + } +} + // Explicit-offset file I/O. Async I/O is built on these routines so it does // not race the sequential FILE_OBJECT.Offset used by legacy read/write calls. __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, @@ -679,7 +861,50 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, size_t remaining = (size_t)(fileSize - offset); if (size > remaining) size = remaining; - return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead); + if ((file->Flags & IO_FILE_FLAG_NO_CACHE) || size < 512) + return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead); + + size_t total = 0; + uint8_t* dst = (uint8_t*)buf; + while (total < size) { + uint64_t absolute = offset + total; + uint64_t pageOffset = absolute & ~(uint64_t)(PAGE_SIZE - 1); + size_t inPage = (size_t)(absolute - pageOffset); + size_t chunk = PAGE_SIZE - inPage; + if (chunk > size - total) chunk = size - total; + + MM_FILE_CACHE_PAGE page; + KSTATUS s = MmFileCacheGetPage(file, pageOffset, &page); + if (!KSUCCESS(s)) { + if (total) { + *bytesRead = total; + return STATUS_SUCCESS; + } + return s; + } + + if (inPage >= page.ValidBytes) { + MmFileCacheReleasePage(&page); + break; + } + size_t available = page.ValidBytes - inPage; + if (chunk > available) chunk = available; + + void* srcPage = MmPfnToVirtual(page.Frame); + if (!srcPage) { + MmFileCacheReleasePage(&page); + return total ? STATUS_SUCCESS : STATUS_FAILURE; + } + memcpy(dst + total, (const uint8_t*)srcPage + inPage, chunk); + MmFileCacheReleasePage(&page); + + total += chunk; + if (chunk == 0) break; + } + + *bytesRead = total; + KiIoMaybeScheduleReadAhead(file, offset, total, fileSize); + return STATUS_SUCCESS; } __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset, @@ -691,10 +916,21 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset, if (!file->Fs->Write) return STATUS_NOT_SUPPORTED; if (size == 0) return STATUS_SUCCESS; + uintptr_t flags = KeAcquireSpinlock(&file->Lock); + uint64_t oldSize = file->Size; + KeReleaseSpinlock(&file->Lock, flags); + size_t written = 0; - KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written); + KSTATUS s; + if (file->Flags & IO_FILE_FLAG_NO_CACHE) { + s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written); + if (KSUCCESS(s) && written) + MmFileCacheInvalidate(file, offset, written); + } else { + s = MmFileCacheWrite(file, offset, buf, size, oldSize, &written); + } if (KSUCCESS(s)) { - uintptr_t flags = KeAcquireSpinlock(&file->Lock); + flags = KeAcquireSpinlock(&file->Lock); uint64_t end = offset + written; if (end > file->Size) file->Size = end; KeReleaseSpinlock(&file->Lock, flags); @@ -704,6 +940,16 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset, return s; } +// Flushes dirty mapped and cached pages for one file object. +__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) { + if (!file) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + + KSTATUS sync = MmVadSyncFileMappings(file); + KSTATUS flush = MmFileCacheFlushFile(file); + return KSUCCESS(sync) ? flush : sync; +} + // IoReadFile - sequential read at current offset. __declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, size_t size, size_t* bytesRead) { @@ -768,6 +1014,11 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o f->Lock = (SPINLOCK)SPINLOCK_INIT; f->Size = 0; f->Offset = 0; + f->Flags = 0; + f->ReadaheadScore = 0; + f->ReadaheadNextOffset = 0; + f->ReadaheadIssuedUntil = 0; + f->ReadaheadSuppressedUntil = 0; *out = f; return STATUS_SUCCESS; @@ -848,8 +1099,12 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize if (!KeCanWait()) return STATUS_INVALID_IRQL; if (!file->Fs->Truncate) return STATUS_NOT_SUPPORTED; - KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize); + KSTATUS flush = IoFlushFile(file); + if (!KSUCCESS(flush)) return flush; + + KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize, file->Flags); if (KSUCCESS(s)) { + MmFileCacheInvalidate(file, newSize, (size_t)-1); file->Size = newSize; // Pull the file pointer back if it now sits past EOF. if (file->Offset > file->Size) @@ -875,6 +1130,16 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) { return size; } +// Queries the filesystem identity for a file object. +__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) { + if (!file || !out) + return STATUS_INVALID_PARAMETER; + memset(out, 0, sizeof(*out)); + if (!file->Fs || !file->Fs->QueryIdentity) + return STATUS_NOT_SUPPORTED; + return file->Fs->QueryIdentity(file->Fs, file->FileCtx, out); +} + // Releases file. __declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) { if (file) ObDereferenceObject(&file->Header); diff --git a/code/kernel/ke_apc.c b/code/kernel/ke_apc.c index 026dcc0..48f81d9 100644 --- a/code/kernel/ke_apc.c +++ b/code/kernel/ke_apc.c @@ -204,7 +204,7 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread, if (thread->State == ThreadDead) return STATUS_OBJECT_DEAD; - KUSER_APC* apc = (KUSER_APC*)MmAllocatePool(sizeof(KUSER_APC)); + KUSER_APC* apc = (KUSER_APC*)MmAllocatePoolTag(sizeof(KUSER_APC), MM_POOL_TAG('a', 'p', 'c', ' ')); if (!apc) return STATUS_NO_MEMORY; diff --git a/code/kernel/ke_debug.c b/code/kernel/ke_debug.c index 092fecb..38b0c66 100644 --- a/code/kernel/ke_debug.c +++ b/code/kernel/ke_debug.c @@ -1,20 +1,24 @@ // Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries. #include +#include #include #include #include #include #include #pragma intrinsic(_disable) +#pragma intrinsic(_enable) #include #include #include #include #include #include +#include #include #include +#include #include int _fltused; // MSVC floating-point helper anchor @@ -22,15 +26,25 @@ int gDebugEnabled = 0; static uint32_t gDebugFlags = 0; void (*PrintChar)(char, uint32_t, uint32_t); -#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u) -#define KE_DEBUG_LOG_LINE_SIZE 1024u +#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u) +#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u) +#define KE_DEBUG_LOG_LINE_SIZE 1024u +#define KE_DEBUG_SCREEN_DRAIN_SIZE 512u static SPINLOCK gDebugPrintLock = SPINLOCK_INIT; static char* gDebugLogBuffer; static size_t gDebugLogSize; static size_t gDebugLogWrite; static size_t gDebugLogUsed; +static char* gDebugScreenBuffer; +static size_t gDebugScreenSize; +static size_t gDebugScreenRead; +static size_t gDebugScreenWrite; +static size_t gDebugScreenUsed; static volatile long gDebugLogFlushing; +static volatile long gDebugConsoleOwnsSerial; +static volatile long gDebugConsoleWorkReady; +static KWORK_ITEM gDebugConsoleWorkItem; // Implements the debug Log Append Locked operation. static void KiDebugLogAppendLocked(const char* text, size_t len) { @@ -55,6 +69,95 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) { gDebugLogUsed += len; } +// Implements the debug Screen Append Locked operation. +static void KiDebugScreenAppendLocked(const char* text, size_t len) { + if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0) + return; + + if (len >= gDebugScreenSize) { + text += len - (gDebugScreenSize - 1u); + len = gDebugScreenSize - 1u; + } + + while (gDebugScreenUsed + len > gDebugScreenSize) { + gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize; + gDebugScreenUsed--; + } + + size_t first = gDebugScreenSize - gDebugScreenWrite; + if (first > len) first = len; + memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first); + if (len > first) + memcpy(gDebugScreenBuffer, text + first, len - first); + + gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize; + gDebugScreenUsed += len; +} + +// Implements the debug Screen Drain Locked operation. +static size_t KiDebugScreenDrainLocked(char* out, size_t max) { + if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0) + return 0; + + size_t len = gDebugScreenUsed; + if (len > max) len = max; + + size_t first = gDebugScreenSize - gDebugScreenRead; + if (first > len) first = len; + memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first); + if (len > first) + memcpy(out + first, gDebugScreenBuffer, len - first); + + gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize; + gDebugScreenUsed -= len; + return len; +} + +// Ensures the asynchronous screen ring exists before console output starts using it. +static bool KiDebugEnsureScreenBuffer(void) { + if (gDebugScreenBuffer && gDebugScreenSize != 0) + return true; + + char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r')); + if (!buffer) + return false; + memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE); + + bool installed = false; + uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock); + if (!gDebugScreenBuffer) { + gDebugScreenBuffer = buffer; + gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE; + gDebugScreenRead = 0; + gDebugScreenWrite = 0; + gDebugScreenUsed = 0; + installed = true; + } + KeReleaseSpinlock(&gDebugPrintLock, flags); + + if (!installed) + MmFreePool(buffer); + return true; +} + +// Implements the debug Serial Write Char operation. +static void KiDebugSerialWriteChar(char c) { + while ((__inbyte(0x3FD) & 0x20) == 0) { } + if (c == '\n') { + __outbyte(0x3F8, '\r'); + while ((__inbyte(0x3FD) & 0x20) == 0) { } + } + __outbyte(0x3F8, (uint8_t)c); +} + +// Implements the debug Serial Write Locked operation. +static void KiDebugSerialWriteLocked(const char* text, size_t len) { + if (!text) + return; + for (size_t i = 0; i < len; i++) + KiDebugSerialWriteChar(text[i]); +} + // Implements the debug Console Write operation. static void KiDebugConsoleWrite(const char* text, size_t len) { if (!PrintChar || !text) @@ -63,6 +166,28 @@ static void KiDebugConsoleWrite(const char* text, size_t len) { PrintChar(text[i], 0xCCCCCC, 0x012456); } +// Implements the debug Console Work operation. +static void KiDebugConsoleWork(void* context) { + (void)context; + char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE]; + + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock); + size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk)); + KeReleaseSpinlock(&gDebugPrintLock, flags); + + if (len == 0) + break; + KiDebugConsoleWrite(chunk, len); + } +} + +// Queues the debug Console Work operation if the passive work queue is ready. +static void KiDebugQueueConsoleWork(void) { + if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted()) + (void)KeQueueWorkItem(&gDebugConsoleWorkItem); +} + static volatile long gPanicInProgress = 0; static volatile uintptr_t gPanicOwnerToken = 0; static volatile long gPanicStoppedCpuCount = 0; @@ -116,7 +241,7 @@ __declspec(dllexport) int KeHandleNmiIpi(void) { if (KiPanicIsActive()) KiPanicSecondaryHalt(); - if (MmHandleTlbShootdownNmi()) + if (MmPteHandleShootdownNmi()) return 1; return 0; @@ -124,6 +249,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void) { // Implements the kernel Put Char operation. void KiKernelPutChar(char c) { + if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1)) + KiDebugSerialWriteChar(c); if (PrintChar) PrintChar(c, 0xCCCCCC, 0x012456); } @@ -135,8 +262,11 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) { PrintChar = PrintPtr; printf_set_putchar(KiKernelPutChar); + if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0)) + KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0); + if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) { - gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE); + gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g')); if (gDebugLogBuffer) { gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE; gDebugLogWrite = 0; @@ -144,6 +274,9 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) { memset(gDebugLogBuffer, 0, gDebugLogSize); } } + + if (flags & KE_DEBUG_CONSOLE) + (void)KiDebugEnsureScreenBuffer(); } // Returns debug Flags. @@ -157,6 +290,13 @@ void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) { return; PrintChar = PrintPtr; printf_set_putchar(KiKernelPutChar); + + // After the display manager owns debug drawing, DbgPrint and user console + // writes mirror to COM1 themselves. The display writer is screen-only so + // asynchronous screen flushing does not duplicate serial bytes. + (void)KiDebugEnsureScreenBuffer(); + _InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1); + KiDebugQueueConsoleWork(); } // Implements the dbg Print operation. @@ -179,17 +319,37 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) { if (KiPanicIsActive()) { if (!KiPanicCurrentCpuOwnsReport()) KiPanicSecondaryHalt(); - if (gDebugFlags & KE_DEBUG_CONSOLE) + if (gDebugFlags & KE_DEBUG_CONSOLE) { + if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1)) + KiDebugSerialWriteLocked(line, len); KiDebugConsoleWrite(line, len); + } return; } + bool queueConsole = false; + bool writeConsoleSync = false; + uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock); + if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1)) + KiDebugSerialWriteLocked(line, len); if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0)) KiDebugLogAppendLocked(line, len); - if (gDebugFlags & KE_DEBUG_CONSOLE) - KiDebugConsoleWrite(line, len); + + if (gDebugFlags & KE_DEBUG_CONSOLE) { + if (KeIsWorkQueueStarted() && gDebugScreenBuffer) { + KiDebugScreenAppendLocked(line, len); + queueConsole = true; + } else { + writeConsoleSync = true; + } + } KeReleaseSpinlock(&gDebugPrintLock, flags); + + if (queueConsole) + KiDebugQueueConsoleWork(); + else if (writeConsoleSync) + KiDebugConsoleWrite(line, len); } // Flushes debug Log To File. @@ -242,6 +402,23 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st; } + +// Attempts to persist the debug log during a panic when the filesystem is usable. +static void KiTryFlushPanicDebugLog(void) { + if (!(gDebugFlags & KE_DEBUG_FILE_LOG)) + return; + if (!KeCanWait() || !KeSystemRootReady()) + return; + + char path[1024]; + if (!KeBuildSystemRootPath("kernel-panic.log", path, sizeof(path))) + return; + + KSTATUS st = KeFlushDebugLogToFile(SysSvFromCstr(path)); + if (!KSUCCESS(st)) + DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st); +} + // KePanic. __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) { KiBeginFatalReport(); @@ -279,15 +456,39 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) { va_end(va); } + KiTryFlushPanicDebugLog(); + for (;;) __halt(); } // KePrintConsole - write user-visible console text using the default console palette. -// Panic output keeps a distinct color so fatal paths remain visually obvious. +// User text mirrors to COM1 immediately, then queues expensive screen drawing +// through the passive debug-console worker when that path is available. __declspec(dllexport) void KePrintConsole(const char* buf, size_t len) { - if (!buf || !PrintChar) return; - for (size_t i = 0; i < len; i++) - PrintChar(buf[i], 0xCCCCCC, 0x012456); + if (!buf || len == 0) return; + + bool queueConsole = false; + bool writeConsoleSync = false; + bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer(); + + uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock); + if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1)) + KiDebugSerialWriteLocked(buf, len); + + if (PrintChar) { + if (!KiPanicIsActive() && canQueueScreen) { + KiDebugScreenAppendLocked(buf, len); + queueConsole = true; + } else { + writeConsoleSync = true; + } + } + KeReleaseSpinlock(&gDebugPrintLock, flags); + + if (queueConsole) + KiDebugQueueConsoleWork(); + else if (writeConsoleSync) + KiDebugConsoleWrite(buf, len); } // KeBugCheckProcess - user-mode fault handler @@ -613,8 +814,22 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) { // Prints crash Char. static void KiPrintCrashChar(char c) { + if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1)) + KiDebugSerialWriteChar(c); if (PrintChar) - PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic + PrintChar(c, 0xFFAA00, 0x000000); +} + +static KSTATUS KiFaultExitStatus(int vector) { + switch (vector) { + case 0: return STATUS_PROCESS_DIVIDE_BY_ZERO; + case 3: return STATUS_PROCESS_BREAKPOINT; + case 6: return STATUS_PROCESS_ILLEGAL_OPCODE; + case 13: return STATUS_PROCESS_GENERAL_PROTECTION; + case 14: return STATUS_PROCESS_PAGE_FAULT; + case 17: return STATUS_PROCESS_ALIGNMENT_FAULT; + default: return STATUS_PROCESS_UNKNOWN_FAULT; + } } static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip, @@ -648,13 +863,23 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip, } // Implements the finish Faulted Process operation. -static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread) { +static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) { printf_set_putchar(KiKernelPutChar); - if (proc) KeExitProcess(proc, -1); - if (thread) KeExitThread(thread, -1); + int exitCode = (int)exitStatus; + if (proc) (void)KeTerminateProcess(proc, exitCode); + if (thread) KeExitThread(thread, exitCode); - while (1) __halt(); + // KeExitThread normally switches away and never resumes this context. If + // a fault path re-enters here during scheduler bring-up or with no runnable + // peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the + // dead thread to the scheduler with interrupts enabled so waiters such as + // the foreground shell can observe the process exit event. + while (true) { + _enable(); + KeYield(); + __halt(); + } } _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) { @@ -664,11 +889,11 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u KiPrintCrashHeader(proc, vector, rip, errCode, cr2); KiDumpModules(proc); - KiFinishFaultedProcess(proc, thread); + KiFinishFaultedProcess(proc, thread, KiFaultExitStatus(vector)); } -// Implements the fault Dispatch Frame operation. -__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) { +// Dispatches a CPU fault frame and returns only when a user #PF was repaired. +__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) { // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode // the interrupted context on a secondary CPU once another CPU owns panic. if (KiPanicIsActive()) @@ -682,6 +907,12 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr (unsigned long long)(uintptr_t)frame); } + if (frame->Vector == 14) { + KSTATUS handled = MmVadResolvePageFaultByAddress((uintptr_t)frame->Cr2, frame->ErrorCode); + if (KSUCCESS(handled)) + return STATUS_SUCCESS; + } + if ((frame->Cs & 3) == 0) { KiBeginFatalReport(); @@ -702,7 +933,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr if (vector == 14) { MmDebugDescribeAddress((uintptr_t)frame->Cr2); if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp); - MmDebugDumpUserRegions(proc); + MmVadDebugDumpProcess(proc); } if (vector == 14) { @@ -743,7 +974,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr KiDumpFaultFrame(proc, frame); KiDumpModules(proc); - KiFinishFaultedProcess(proc, thread); + KiFinishFaultedProcess(proc, thread, KiFaultExitStatus((int)frame->Vector)); } // KeFaultDispatch - single C entry point for all three IDT fault stubs. @@ -762,5 +993,6 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip, frame.Cr2 = cr2; frame.Rip = rip; frame.Cs = cs; - KeFaultDispatchFrame(&frame); + (void)KeFaultDispatchFrame(&frame); + while (1) __halt(); } diff --git a/code/kernel/ke_display.c b/code/kernel/ke_display.c index afc0cd7..bd211e6 100644 --- a/code/kernel/ke_display.c +++ b/code/kernel/ke_display.c @@ -12,6 +12,10 @@ #include #include #include +#include +#include +#include +#include #define DISPLAY_DEFAULT_FG 0x00CCCCCCu #define DISPLAY_DEFAULT_BG 0x00012456u @@ -25,6 +29,7 @@ #define DISPLAY_CONSOLE_MAX_ROWS 128 #define DISPLAY_CONSOLE_MAX_CELLS (DISPLAY_CONSOLE_MAX_COLS * DISPLAY_CONSOLE_MAX_ROWS) #define DISPLAY_CONSOLE_DEFAULT_ATTR 0x07u +#define DISPLAY_CURSOR_POS_INVALID (-2147483647 - 1) typedef struct BOOTFB_CONTEXT { const HAL_API* Hal; @@ -40,6 +45,11 @@ static const HAL_API* gDisplayHal; static DISPLAY_DEVICE* gDisplayList; static DISPLAY_DEVICE* gPrimaryDisplay; static SPINLOCK gDisplayLock = SPINLOCK_INIT; +static MUTEX* gDisplayProviderMutex; +static KWORK_ITEM gDisplayCursorWorkItem; +static volatile long gDisplayCursorWorkInitialized; +static bool gDisplayTransitionInProgress; +static uint64_t gDisplayGeneration; static DISPLAY_DEVICE gBootFbDevice; static BOOTFB_CONTEXT gBootFb; @@ -56,6 +66,11 @@ static int gCursorX; static int gCursorY; static int gCursorDrawX; static int gCursorDrawY; +static bool gCursorUpdatePending; +static uint64_t gCursorRequestGeneration; +static uint64_t gCursorAppliedGeneration; +static int gCursorCommittedX; +static int gCursorCommittedY; static int gConsoleX; static int gConsoleY; @@ -91,6 +106,69 @@ static const DISPLAY_DEVICE_OPS gBootFbOps = { }; +static bool KiDisplayOutputSuspended(void); +static void KiDisplayCursorWorker(void* context); +static void KiDisplayDirtyAll(void); + +static bool KiDisplayHasCurrentThread(void) { + return KeGetCurrentThread() != (THREAD*)0; +} + +static bool KiDisplayProviderCallbacksRequirePassive(void) { + return KiDisplayHasCurrentThread() && KeIsWorkQueueStarted(); +} + +static KSTATUS KiDisplayAcquireProviderMutex(bool* acquired) { + if (acquired) + *acquired = false; + + if (!gDisplayProviderMutex || !KiDisplayProviderCallbacksRequirePassive()) + return STATUS_SUCCESS; + + if (!KeCanWait()) + return STATUS_INVALID_IRQL; + + if (KeGetDebugFlags() & KE_DEBUG_DISPLAY_ASSERTS) + KeAssertIrql(PASSIVE_LEVEL); + + KSTATUS s = KeAcquireMutex(gDisplayProviderMutex); + if (KSUCCESS(s) && acquired) + *acquired = true; + return s; +} + +static void KiDisplayReleaseProviderMutex(bool acquired) { + if (acquired && gDisplayProviderMutex) + KeReleaseMutex(gDisplayProviderMutex); +} + +// Writes one character through the boot HAL framebuffer path when the display +// manager cannot safely call provider callbacks, such as during a panic. +static KSTATUS KiDisplayFallbackPrintChar(char c, uint32_t fg, uint32_t bg) { + const HAL_API* hal = gDisplayHal; + if (hal && hal->Screen.PrintChar) { + hal->Screen.PrintChar(c, fg, bg); + return STATUS_SUCCESS; + } + return STATUS_INVALID_IRQL; +} + +static void KiDisplayQueueCursorWork(void) { + if (!_InterlockedCompareExchange(&gDisplayCursorWorkInitialized, 1, 1)) + return; + if (KeIsWorkQueueStarted()) { + (void)KeQueueWorkItem(&gDisplayCursorWorkItem); + return; + } + + // During single-threaded early boot there is no worker thread yet. The + // boot framebuffer console still needs synchronous cursor setup; once the + // scheduler work queue exists all cursor commits are asynchronous. + if (!KiDisplayProviderCallbacksRequirePassive()) + KiDisplayCursorWorker(0); +} + + // Checks whether a primary display provider is available. static bool KiDisplayHasDevice(void) { return gPrimaryDisplay && gPrimaryDisplay->Ops; @@ -117,6 +195,136 @@ static bool KiDisplayHasCursorOps(void) { gPrimaryDisplay->Ops->ShowCursor; } + +static void KiDisplayCursorWorker(void* context) { + (void)context; + + bool providerHeld = false; + if (!KSUCCESS(KiDisplayAcquireProviderMutex(&providerHeld))) + return; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + int x = 0; + int y = 0; + bool visible = false; + bool drawn = false; + bool suspended = false; + uint64_t displayGeneration = 0; + uint64_t requestGeneration = 0; + int committedX = 0; + int committedY = 0; + bool hadPendingWork = false; + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + hadPendingWork = gCursorUpdatePending || (gCursorRequestGeneration != gCursorAppliedGeneration); + if (!hadPendingWork) { + KeReleaseSpinlock(&gDisplayLock, flags); + KiDisplayReleaseProviderMutex(providerHeld); + return; + } + + gCursorUpdatePending = false; + if (KiDisplayHasDevice()) { + device = gPrimaryDisplay; + ops = gPrimaryDisplay->Ops; + caps = gPrimaryDisplay->Caps; + x = gCursorX; + y = gCursorY; + visible = gCursorVisible; + drawn = gCursorDrawn; + suspended = KiDisplayOutputSuspended() || gDisplayTransitionInProgress; + displayGeneration = gDisplayGeneration; + requestGeneration = gCursorRequestGeneration; + committedX = gCursorCommittedX; + committedY = gCursorCommittedY; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + bool newDrawn = false; + bool positionCommitted = false; + int newCommittedX = committedX; + int newCommittedY = committedY; + KSTATUS status = STATUS_SUCCESS; + bool hasCursorOps = device && ops && ops->ShowCursor && + (caps & (DISPLAY_CAP_SOFTWARE_CURSOR | DISPLAY_CAP_HARDWARE_CURSOR)); + bool softwareCursor = hasCursorOps && (caps & DISPLAY_CAP_SOFTWARE_CURSOR) != 0; + bool hardwareCursor = hasCursorOps && (caps & DISPLAY_CAP_HARDWARE_CURSOR) != 0; + + if (hasCursorOps) { + if (!visible || suspended) { + if (drawn) + status = ops->ShowCursor(device, false); + newDrawn = false; + if (KSUCCESS(status)) { + positionCommitted = true; + newCommittedX = DISPLAY_CURSOR_POS_INVALID; + newCommittedY = DISPLAY_CURSOR_POS_INVALID; + } + } else if (hardwareCursor) { + // Hardware cursors are independent of framebuffer drawing. Commit + // position only when the logical cursor request changed or the + // provider has not yet been synchronized. This prevents stale or + // duplicate work items from hammering device cursor registers. + bool needPosition = !drawn || x != committedX || y != committedY; + if (needPosition && ops->SetCursorPosition) { + status = ops->SetCursorPosition(device, x, y); + if (KSUCCESS(status)) { + positionCommitted = true; + newCommittedX = x; + newCommittedY = y; + } + } else { + positionCommitted = true; + newCommittedX = committedX; + newCommittedY = committedY; + } + + if (KSUCCESS(status)) { + if (!drawn) + status = ops->ShowCursor(device, true); + newDrawn = KSUCCESS(status); + } + } else if (softwareCursor) { + if (drawn) + (void)ops->ShowCursor(device, false); + if (ops->SetCursorPosition) + (void)ops->SetCursorPosition(device, x, y); + status = ops->ShowCursor(device, true); + newDrawn = KSUCCESS(status); + if (newDrawn) { + positionCommitted = true; + newCommittedX = x; + newCommittedY = y; + } + } + } + + flags = KeAcquireSpinlock(&gDisplayLock); + if (displayGeneration == gDisplayGeneration && device == gPrimaryDisplay) { + gCursorDrawn = newDrawn; + if (positionCommitted) { + gCursorCommittedX = newCommittedX; + gCursorCommittedY = newCommittedY; + } + // Do not spin forever on provider failure. A later explicit cursor + // request, show/hide request, or display-generation change will retry. + gCursorAppliedGeneration = requestGeneration; + if (gCursorRequestGeneration != requestGeneration) + gCursorUpdatePending = true; + } else { + gCursorUpdatePending = true; + } + bool requeue = gCursorUpdatePending && !gDisplayTransitionInProgress; + KeReleaseSpinlock(&gDisplayLock, flags); + + KiDisplayReleaseProviderMutex(providerHeld); + + if (requeue) + KiDisplayQueueCursorWork(); +} + // Clips a rectangle to a display mode so providers never see negative or out-of-bounds regions. static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) { if (!mode || !rect || rect->W <= 0 || rect->H <= 0) @@ -141,6 +349,350 @@ static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) { return true; } + +// Snapshots the current primary under the manager spinlock. The caller must +// hold the provider mutex before invoking callbacks through the returned ops; +// KeDisplaySetPrimary takes the same mutex, so the provider object cannot be +// switched while the callback sequence is in progress. +static bool KiDisplaySnapshotPrimary(DISPLAY_DEVICE** outDevice, + const DISPLAY_DEVICE_OPS** outOps, + uint32_t* outCaps, + uint64_t* outGeneration) { + if (outDevice) *outDevice = (DISPLAY_DEVICE*)0; + if (outOps) *outOps = (const DISPLAY_DEVICE_OPS*)0; + if (outCaps) *outCaps = 0; + if (outGeneration) *outGeneration = 0; + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0; + if (device) { + if (outDevice) *outDevice = device; + if (outOps) *outOps = device->Ops; + if (outCaps) *outCaps = device->Caps; + if (outGeneration) *outGeneration = gDisplayGeneration; + } + KeReleaseSpinlock(&gDisplayLock, flags); + return device != (DISPLAY_DEVICE*)0; +} + +static bool KiDisplayConsoleGridFromMode(const DISPLAY_MODE* mode, int* outCols, int* outRows) { + if (!mode) + return false; + + int cols = (int)mode->Width / FONT_W; + int rows = (int)mode->Height / FONT_H; + if (cols > DISPLAY_CONSOLE_MAX_COLS) cols = DISPLAY_CONSOLE_MAX_COLS; + if (rows > DISPLAY_CONSOLE_MAX_ROWS) rows = DISPLAY_CONSOLE_MAX_ROWS; + if (cols <= 0 || rows <= 0) + return false; + + if (outCols) *outCols = cols; + if (outRows) *outRows = rows; + return true; +} + +static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { + if (!mode || gDirtyAll || w <= 0 || h <= 0) + return; + + DISPLAY_RECT rect = { x, y, w, h }; + if (!KiDisplayClipRect(mode, &rect)) + return; + + if (rect.X == 0 && rect.Y == 0 && + rect.W == (int32_t)mode->Width && rect.H == (int32_t)mode->Height) { + KiDisplayDirtyAll(); + return; + } + + if (gDirtyRectCount >= DISPLAY_MAX_DIRTY_RECTS) { + KiDisplayDirtyAll(); + return; + } + + gDisplayStats.DirtyRectCount++; + gDirtyRects[gDirtyRectCount++] = rect; +} + +static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { + gDisplayStats.BlitCount++; + gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t); + KiDisplayAddDirtyForModeLocked(mode, x, y, w, h); +} + +static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { + gDisplayStats.FillRectCount++; + gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t); + KiDisplayAddDirtyForModeLocked(mode, x, y, w, h); +} + +static void KiDisplayRecordCopyLocked(const DISPLAY_MODE* mode, int dstX, int dstY, int w, int h) { + gDisplayStats.CopyRectCount++; + gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t); + KiDisplayAddDirtyForModeLocked(mode, dstX, dstY, w, h); +} + +static KSTATUS KiDisplayFlushDirtyWithSnapshot(DISPLAY_DEVICE* device, const DISPLAY_DEVICE_OPS* ops) { + if (!device || !ops || !ops->Present) + return STATUS_SUCCESS; + + DISPLAY_RECT localRects[DISPLAY_MAX_DIRTY_RECTS]; + DISPLAY_RECT fullRect; + const DISPLAY_RECT* rects = localRects; + size_t count = 0; + bool dirtyAll = false; + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device != gPrimaryDisplay || (!gDirtyAll && gDirtyRectCount == 0)) { + KeReleaseSpinlock(&gDisplayLock, flags); + return STATUS_SUCCESS; + } + + dirtyAll = gDirtyAll; + if (!dirtyAll) { + count = gDirtyRectCount; + if (count > DISPLAY_MAX_DIRTY_RECTS) + count = DISPLAY_MAX_DIRTY_RECTS; + for (size_t i = 0; i < count; i++) + localRects[i] = gDirtyRects[i]; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + if (dirtyAll) { + DISPLAY_MODE mode; + if (!ops->GetMode) + return STATUS_NOT_SUPPORTED; + KSTATUS ms = ops->GetMode(device, &mode); + if (!KSUCCESS(ms)) + return ms; + fullRect.X = 0; + fullRect.Y = 0; + fullRect.W = (int32_t)mode.Width; + fullRect.H = (int32_t)mode.Height; + rects = &fullRect; + count = 1; + } + + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + gDisplayStats.PresentCount++; + gDisplayStats.PresentRectCount += count; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + KSTATUS s = ops->Present(device, rects, count); + if (KSUCCESS(s)) { + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + gDirtyAll = false; + gDirtyRectCount = 0; + } + KeReleaseSpinlock(&gDisplayLock, flags); + } + return s; +} + +static bool KiDisplayHideCursorForDrawSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + uint32_t caps) { + if (!device || !ops || !ops->ShowCursor || !(caps & DISPLAY_CAP_SOFTWARE_CURSOR)) + return false; + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + bool hide = (device == gPrimaryDisplay) && gCursorDrawn; + if (hide) + gCursorDrawn = false; + KeReleaseSpinlock(&gDisplayLock, flags); + + if (hide) + (void)ops->ShowCursor(device, false); + return hide; +} + +static void KiDisplayRestoreCursorAfterDrawSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + uint32_t caps) { + // Only software cursors are hidden/restored around framebuffer drawing. + // Hardware cursors are owned by the provider/device and must not be + // re-positioned after every blit/present; the coalesced cursor worker is + // the only path that commits hardware cursor movement. + if (!device || !ops || !ops->ShowCursor || !(caps & DISPLAY_CAP_SOFTWARE_CURSOR)) + return; + + int x = 0; + int y = 0; + bool shouldShow = false; + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + shouldShow = (device == gPrimaryDisplay) && + gCursorVisible && !gCursorDrawn && + !KiDisplayOutputSuspended() && + !gDisplayTransitionInProgress; + if (shouldShow) { + x = gCursorX; + y = gCursorY; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + if (!shouldShow) + return; + + if (ops->SetCursorPosition) + (void)ops->SetCursorPosition(device, x, y); + KSTATUS s = ops->ShowCursor(device, true); + + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay && KSUCCESS(s)) { + gCursorDrawn = true; + gCursorCommittedX = x; + gCursorCommittedY = y; + gCursorAppliedGeneration = gCursorRequestGeneration; + gCursorUpdatePending = false; + } else if (device != gPrimaryDisplay) { + gCursorUpdatePending = true; + } + bool requeue = gCursorUpdatePending && !gDisplayTransitionInProgress; + KeReleaseSpinlock(&gDisplayLock, flags); + + if (requeue) + KiDisplayQueueCursorWork(); +} + +static KSTATUS KiDisplayGetProviderSnapshot(DISPLAY_DEVICE** outDevice, + const DISPLAY_DEVICE_OPS** outOps, + uint32_t* outCaps, + DISPLAY_MODE* outMode) { + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + + if (!KiDisplaySnapshotPrimary(&device, &ops, &caps, 0) || !device || !ops) + return STATUS_NOT_SUPPORTED; + if (!ops->GetMode) + return STATUS_NOT_SUPPORTED; + + KSTATUS s = ops->GetMode(device, outMode); + if (!KSUCCESS(s)) + return s; + + if (outDevice) *outDevice = device; + if (outOps) *outOps = ops; + if (outCaps) *outCaps = caps; + return STATUS_SUCCESS; +} + +static void KiDisplayBuildGlyphPixels(char c, uint32_t fg, uint32_t bg, uint32_t* outPixels) { + const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u); + for (int row = 0; row < FONT_H; row++) { + uint8_t bits = glyph[row]; + for (int col = 0; col < FONT_W; col++) + outPixels[row * FONT_W + col] = (bits & (0x80u >> col)) ? fg : bg; + } +} + +static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + const DISPLAY_MODE* mode, + char c, + int x, + int y, + uint32_t fg, + uint32_t bg) { + if (!device || !ops || !ops->Blit || !mode) + return STATUS_NOT_SUPPORTED; + + uint32_t glyphPixels[FONT_W * FONT_H]; + KiDisplayBuildGlyphPixels(c, fg, bg, glyphPixels); + KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) + KiDisplayRecordBlitLocked(mode, x, y, FONT_W, FONT_H); + KeReleaseSpinlock(&gDisplayLock, flags); + } + return s; +} + +static KSTATUS KiDisplayDrawBlitSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + const DISPLAY_MODE* mode, + int x, + int y, + int w, + int h, + const uint32_t* pixels) { + if (!pixels || w <= 0 || h <= 0) + return STATUS_INVALID_PARAMETER; + if (!device || !ops || !ops->Blit || !mode) + return STATUS_NOT_SUPPORTED; + + KSTATUS s = ops->Blit(device, x, y, w, h, pixels); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) + KiDisplayRecordBlitLocked(mode, x, y, w, h); + KeReleaseSpinlock(&gDisplayLock, flags); + } + return s; +} + +static KSTATUS KiDisplayFillRectSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + const DISPLAY_MODE* mode, + int x, + int y, + int w, + int h, + uint32_t color) { + if (w <= 0 || h <= 0) + return STATUS_INVALID_PARAMETER; + if (!device || !ops || !ops->FillRect || !mode) + return STATUS_NOT_SUPPORTED; + + KSTATUS s = ops->FillRect(device, x, y, w, h, color); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) + KiDisplayRecordFillLocked(mode, x, y, w, h); + KeReleaseSpinlock(&gDisplayLock, flags); + } + return s; +} + +static KSTATUS KiDisplayCopyRectSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + const DISPLAY_MODE* mode, + int dstX, + int dstY, + int srcX, + int srcY, + int w, + int h) { + if (w <= 0 || h <= 0) + return STATUS_INVALID_PARAMETER; + if (!device || !ops || !ops->CopyRect || !mode) + return STATUS_NOT_SUPPORTED; + + KSTATUS s = ops->CopyRect(device, dstX, dstY, srcX, srcY, w, h); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) + KiDisplayRecordCopyLocked(mode, dstX, dstY, w, h); + KeReleaseSpinlock(&gDisplayLock, flags); + } + return s; +} + +static KSTATUS KiDisplayFinishProviderDraw(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + uint32_t caps, + KSTATUS drawStatus) { + KSTATUS presentStatus = STATUS_SUCCESS; + if (KSUCCESS(drawStatus)) + presentStatus = KiDisplayFlushDirtyWithSnapshot(device, ops); + KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps); + return KSUCCESS(drawStatus) ? presentStatus : drawStatus; +} + // Implements the display Dirty All operation. static void KiDisplayDirtyAll(void) { gDisplayStats.DirtyAllCount++; @@ -437,6 +989,17 @@ void KeDisplayInitialize(const HAL_API* hal) { gCursorY = 0; gCursorDrawX = -1; gCursorDrawY = -1; + gCursorUpdatePending = false; + gCursorRequestGeneration = 0; + gCursorAppliedGeneration = 0; + gCursorCommittedX = DISPLAY_CURSOR_POS_INVALID; + gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID; + gDisplayTransitionInProgress = false; + gDisplayGeneration = 1; + if (!gDisplayProviderMutex) + (void)KeMutexCreate(&gDisplayProviderMutex); + KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0); + _InterlockedExchange(&gDisplayCursorWorkInitialized, 1); gConsoleX = 0; gConsoleY = 0; KiDisplayConsoleResetCellsLocked(); @@ -572,12 +1135,8 @@ DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) { return STATUS_INVALID_PARAMETER; const char* name = device->Name ? device->Name : "?"; - KSTATUS result = STATUS_SUCCESS; - KSTATUS seedResult = STATUS_SUCCESS; - const char* seedOldName = NULL; uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - bool found = false; for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) { if (it == device) { @@ -591,51 +1150,96 @@ DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) { } DISPLAY_DEVICE* oldDevice = gPrimaryDisplay; - bool cursorWasVisible = gCursorVisible; + bool oldCursorDrawn = gCursorDrawn; + gDisplayTransitionInProgress = true; + gDisplayGeneration++; + gCursorUpdatePending = gCursorVisible; + KeReleaseSpinlock(&gDisplayLock, flags); - if (oldDevice && gCursorDrawn && oldDevice->Ops && oldDevice->Ops->ShowCursor) - (void)oldDevice->Ops->ShowCursor(oldDevice, false); - - if (oldDevice && oldDevice != device) { - seedOldName = oldDevice->Name; - seedResult = KiDisplaySeedNewPrimary(oldDevice, device); + bool providerHeld = false; + KSTATUS result = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(result)) { + flags = KeAcquireSpinlock(&gDisplayLock); + gDisplayTransitionInProgress = false; + gDisplayGeneration++; + KeReleaseSpinlock(&gDisplayLock, flags); + KiDisplayQueueCursorWork(); + return result; } - gPrimaryDisplay = device; - gCursorDrawn = false; - gDirtyAll = false; - gDirtyRectCount = 0; + if (oldDevice && oldCursorDrawn && oldDevice->Ops && oldDevice->Ops->ShowCursor) + (void)oldDevice->Ops->ShowCursor(oldDevice, false); - if (device->Ops->Present) { + // Clean architecture policy: handoff does not read back/copy the old boot + // framebuffer. The new primary is cleared to the kernel console blue and + // user/session code redraws state above the provider boundary. + if (device->Ops->GetMode && device->Ops->FillRect) { DISPLAY_MODE mode; - if (KSUCCESS(device->Ops->GetMode(device, &mode))) { - DISPLAY_RECT full = { 0, 0, (int32_t)mode.Width, (int32_t)mode.Height }; - (void)device->Ops->Present(device, &full, 1); + KSTATUS modeStatus = device->Ops->GetMode(device, &mode); + if (KSUCCESS(modeStatus)) { + (void)device->Ops->FillRect(device, 0, 0, + (int)mode.Width, + (int)mode.Height, + DISPLAY_DEFAULT_BG); + if (device->Ops->Present) { + DISPLAY_RECT full = { 0, 0, (int32_t)mode.Width, (int32_t)mode.Height }; + (void)device->Ops->Present(device, &full, 1); + } } } - if (cursorWasVisible) - KiDisplayRestoreCursorAfterDraw(); + flags = KeAcquireSpinlock(&gDisplayLock); + gPrimaryDisplay = device; + gCursorDrawn = false; + gCursorCommittedX = DISPLAY_CURSOR_POS_INVALID; + gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID; + gDirtyAll = false; + gDirtyRectCount = 0; + gDrawDepth = 0; + // The provider handoff clears the framebuffer instead of preserving boot + // text. Keep the console model coherent with that clear so subsequent + // kernel/debug output restarts at the top-left of the new primary. + KiDisplayConsoleResetCellsLocked(); + gConsoleX = 0; + gConsoleY = 0; + + gDisplayTransitionInProgress = false; + gDisplayGeneration++; + bool needCursor = gCursorVisible; + gCursorUpdatePending = needCursor; + if (needCursor) + gCursorRequestGeneration++; KeReleaseSpinlock(&gDisplayLock, flags); - if (!KSUCCESS(seedResult) && seedResult != STATUS_NOT_SUPPORTED) { - DbgPrint("Display: primary handoff seed %s -> %s failed st=0x%x\n", - seedOldName ? seedOldName : "?", name, seedResult); - } - if (KSUCCESS(result)) - DbgPrint("Display: primary is %s\n", name); + KiDisplayReleaseProviderMutex(providerHeld); + + if (needCursor) + KiDisplayQueueCursorWork(); + + DbgPrint("Display: primary is %s\n", name); return result; } DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode) { if (!outMode) return STATUS_INVALID_PARAMETER; + + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KSTATUS s = (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode) - ? STATUS_INVALID_PARAMETER - : gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, outMode); + DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0; KeReleaseSpinlock(&gDisplayLock, flags); + + s = (!device || !ops || !ops->GetMode) + ? STATUS_INVALID_PARAMETER + : ops->GetMode(device, outMode); + + KiDisplayReleaseProviderMutex(providerHeld); return s; } @@ -649,120 +1253,308 @@ DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight) { return STATUS_SUCCESS; } -DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels) { +DISPLAY_API KSTATUS KeDisplaySync(void) { + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KiDisplayBeginDraw(); - KSTATUS s = KiDisplayDrawBlitNoCursor(x, y, w, h, pixels); - KSTATUS ps = KiDisplayEndDraw(); + DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0; KeReleaseSpinlock(&gDisplayLock, flags); - return KSUCCESS(s) ? ps : s; + + if (!device || !ops || !ops->Sync) { + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_NOT_SUPPORTED; + } + + s = ops->Sync(device); + + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) + gDisplayStats.SyncCount++; + KeReleaseSpinlock(&gDisplayLock, flags); + + KiDisplayReleaseProviderMutex(providerHeld); + return s; +} + +DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) { + if (!outInfo) + return STATUS_INVALID_PARAMETER; + + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + KE_DISPLAY_INFO info; + memset(&info, 0, sizeof(info)); + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + + uint32_t count = 0; + for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) + count++; + info.DeviceCount = count; + info.ManagerStats = gDisplayStats; + + DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0; + if (device) { + const char* name = device->Name ? device->Name : "?"; + size_t len = strlen(name); + if (len >= sizeof(info.PrimaryName)) + len = sizeof(info.PrimaryName) - 1; + memcpy(info.PrimaryName, name, len); + info.PrimaryName[len] = '\0'; + info.Caps = device->Caps; + } + + KeReleaseSpinlock(&gDisplayLock, flags); + + if (!device || !ops) { + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_NOT_FOUND; + } + + if (ops->GetMode) { + s = ops->GetMode(device, &info.Mode); + if (!KSUCCESS(s)) { + KiDisplayReleaseProviderMutex(providerHeld); + return s; + } + } + + if (ops->QueryStats) + (void)ops->QueryStats(device, &info.DeviceStats); + + *outInfo = info; + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_SUCCESS; +} + +DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels) { + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (KSUCCESS(s)) { + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y, w, h, pixels); + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + } + + KiDisplayReleaseProviderMutex(providerHeld); + return s; +} + +DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch) { + if (!source || !scratch || w <= 0 || h <= 0) + return STATUS_INVALID_PARAMETER; + + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (KSUCCESS(s)) { + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + for (int row = 0; row < h; ++row) { + s = source(context, row, scratch, (size_t)w); + if (!KSUCCESS(s)) + break; + s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y + row, w, 1, scratch); + if (!KSUCCESS(s)) + break; + } + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + } + + KiDisplayReleaseProviderMutex(providerHeld); + return s; } DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KiDisplayBeginDraw(); - KSTATUS s = KiDisplayFillRectNoCursor(x, y, w, h, color); - KSTATUS ps = KiDisplayEndDraw(); - KeReleaseSpinlock(&gDisplayLock, flags); - return KSUCCESS(s) ? ps : s; -} - -// Implements the display Clear Locked operation. -static KSTATUS KiDisplayClearLocked(uint32_t color) { - DISPLAY_MODE mode; - if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode) - return STATUS_NOT_SUPPORTED; - KSTATUS s = gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, &mode); + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); if (!KSUCCESS(s)) return s; - KiDisplayBeginDraw(); - s = KiDisplayFillRectNoCursor(0, 0, (int)mode.Width, (int)mode.Height, color); + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); if (KSUCCESS(s)) { - KiDisplayConsoleResetCellsLocked(); - gConsoleX = 0; - gConsoleY = 0; + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + s = KiDisplayFillRectSnapshot(device, ops, &mode, x, y, w, h, color); + s = KiDisplayFinishProviderDraw(device, ops, caps, s); } - KSTATUS ps = KiDisplayEndDraw(); - return KSUCCESS(s) ? ps : s; + + KiDisplayReleaseProviderMutex(providerHeld); + return s; } DISPLAY_API KSTATUS KeDisplayClear(uint32_t color) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KSTATUS s = KiDisplayClearLocked(color); - KeReleaseSpinlock(&gDisplayLock, flags); + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (KSUCCESS(s)) { + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0, (int)mode.Width, (int)mode.Height, color); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + KiDisplayConsoleResetCellsLocked(); + gConsoleX = 0; + gConsoleY = 0; + } + KeReleaseSpinlock(&gDisplayLock, flags); + } + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + } + + KiDisplayReleaseProviderMutex(providerHeld); return s; } DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KiDisplayBeginDraw(); - KSTATUS s = KiDisplayCopyRectNoCursor(dstX, dstY, srcX, srcY, w, h); - KSTATUS ps = KiDisplayEndDraw(); - KeReleaseSpinlock(&gDisplayLock, flags); - return KSUCCESS(s) ? ps : s; -} - -// Implements the display Scroll Locked operation. -static KSTATUS KiDisplayScrollLocked(int lines) { - if (lines <= 0) - return STATUS_SUCCESS; - - if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode) - return STATUS_NOT_SUPPORTED; - DISPLAY_MODE mode; - KSTATUS s = gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, &mode); + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); if (!KSUCCESS(s)) return s; - if ((uint32_t)lines >= mode.Height) - return KiDisplayClearLocked(DISPLAY_DEFAULT_BG); - KiDisplayBeginDraw(); - s = KiDisplayCopyRectNoCursor(0, 0, 0, lines, (int)mode.Width, (int)(mode.Height - (uint32_t)lines)); + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); if (KSUCCESS(s)) { - s = KiDisplayFillRectNoCursor(0, (int)(mode.Height - (uint32_t)lines), - (int)mode.Width, lines, DISPLAY_DEFAULT_BG); + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + s = KiDisplayCopyRectSnapshot(device, ops, &mode, dstX, dstY, srcX, srcY, w, h); + s = KiDisplayFinishProviderDraw(device, ops, caps, s); } - KSTATUS ps = KiDisplayEndDraw(); - return KSUCCESS(s) ? ps : s; + + KiDisplayReleaseProviderMutex(providerHeld); + return s; } DISPLAY_API KSTATUS KeDisplayScroll(int lines) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KSTATUS s = KiDisplayScrollLocked(lines); - KeReleaseSpinlock(&gDisplayLock, flags); - return s; -} + if (lines <= 0) + return STATUS_SUCCESS; -// Implements the display Put Char Locked operation. -static KSTATUS KiDisplayPutCharLocked(char c, int x, int y, uint32_t fg, uint32_t bg) { - KiDisplayBeginDraw(); - KSTATUS s = KiDisplayDrawCharPixelsLocked(c, x, y, fg, bg); - KSTATUS ps = KiDisplayEndDraw(); - return KSUCCESS(s) ? ps : s; + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (KSUCCESS(s)) { + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + if ((uint32_t)lines >= mode.Height) { + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0, (int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + KiDisplayConsoleResetCellsLocked(); + gConsoleX = 0; + gConsoleY = 0; + } + KeReleaseSpinlock(&gDisplayLock, flags); + } + } else { + s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, lines, + (int)mode.Width, (int)(mode.Height - (uint32_t)lines)); + if (KSUCCESS(s)) { + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, + (int)(mode.Height - (uint32_t)lines), + (int)mode.Width, lines, DISPLAY_DEFAULT_BG); + } + } + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + } + + KiDisplayReleaseProviderMutex(providerHeld); + return s; } DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KSTATUS s = KiDisplayPutCharLocked(c, x, y, fg, bg); - KeReleaseSpinlock(&gDisplayLock, flags); + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (KSUCCESS(s)) { + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, x, y, fg, bg); + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + } + + KiDisplayReleaseProviderMutex(providerHeld); return s; } -DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) { - KiDisplaySerialWriteChar(c); +static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg, bool mirrorSerial) { + if (mirrorSerial) + KiDisplaySerialWriteChar(c); - if (KiDisplayOutputSuspended()) + if (KiDisplayOutputSuspended()) { + if (!KeCanWait()) + return KiDisplayFallbackPrintChar(c, fg, bg); return STATUS_SUCCESS; + } - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + // Once the scheduler/work-queue era begins, display provider callbacks are + // PASSIVE_LEVEL-only. DbgPrint queues screen output through the work queue; + // panic and explicit console paths may still use the serial mirroring entry. + // A panic has interrupts disabled and cannot take the provider mutex, so it + // falls back to the boot framebuffer writer instead of disappearing from + // the screen while still printing to COM1. + if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) + return KiDisplayFallbackPrintChar(c, fg, bg); + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; DISPLAY_MODE mode; - KSTATUS s = (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode) - ? STATUS_NOT_SUPPORTED - : gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, &mode); + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); if (!KSUCCESS(s)) { + KiDisplayReleaseProviderMutex(providerHeld); const HAL_API* hal = gDisplayHal; - KeReleaseSpinlock(&gDisplayLock, flags); if (hal && hal->Screen.PrintChar) hal->Screen.PrintChar(c, fg, bg); return s; @@ -770,11 +1562,19 @@ DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) { int cols = 0; int rows = 0; - if (!KiDisplayGetConsoleGridLocked(&cols, &rows)) { - KeReleaseSpinlock(&gDisplayLock, flags); + if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) { + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_DEVICE_ERROR; } + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + + bool drawChar = false; + bool needScroll = false; + int drawCol = 0; + int drawRow = 0; + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); if (c == '\n') { gConsoleX = 0; gConsoleY++; @@ -788,243 +1588,339 @@ DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) { gConsoleX = cols - 1; } } else { - s = KiDisplayPutCharLocked(c, gConsoleX * FONT_W, gConsoleY * FONT_H, fg, bg); - if (!KSUCCESS(s)) { - KeReleaseSpinlock(&gDisplayLock, flags); - return s; - } - if (gConsoleX >= 0 && gConsoleX < cols && gConsoleY >= 0 && gConsoleY < rows) { - size_t i = KiDisplayConsoleIndex(gConsoleX, gConsoleY); - gConsoleCells[i] = c; - gConsoleAttrs[i] = (uint8_t)gConsoleAttribute; - } + drawChar = true; + drawCol = gConsoleX; + drawRow = gConsoleY; if (++gConsoleX >= cols) { gConsoleX = 0; gConsoleY++; } } - if (gConsoleY >= rows) { - s = KiDisplayScrollLocked(FONT_H); - if (!KSUCCESS(s)) { + if (gConsoleY >= rows) + needScroll = true; + KeReleaseSpinlock(&gDisplayLock, flags); + + if (drawChar) { + s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, drawCol * FONT_W, drawRow * FONT_H, fg, bg); + if (KSUCCESS(s)) { + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay && drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) { + size_t i = KiDisplayConsoleIndex(drawCol, drawRow); + gConsoleCells[i] = c; + gConsoleAttrs[i] = (uint8_t)gConsoleAttribute; + } KeReleaseSpinlock(&gDisplayLock, flags); - return s; } - KiDisplayConsoleScrollCellsLocked(cols, rows, 1); - gConsoleY = rows - 1; + } else { + s = STATUS_SUCCESS; } - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; + if (KSUCCESS(s) && needScroll) { + s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, FONT_H, + (int)mode.Width, (int)(mode.Height - FONT_H)); + if (KSUCCESS(s)) { + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, + (int)(mode.Height - FONT_H), + (int)mode.Width, FONT_H, DISPLAY_DEFAULT_BG); + } + if (KSUCCESS(s)) { + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + KiDisplayConsoleScrollCellsLocked(cols, rows, 1); + gConsoleY = rows - 1; + if (gConsoleX < 0) gConsoleX = 0; + if (gConsoleX >= cols) gConsoleX = cols - 1; + } + KeReleaseSpinlock(&gDisplayLock, flags); + } + } + + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + KiDisplayReleaseProviderMutex(providerHeld); + return s; +} + +DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) { + return KiDisplayPrintCharInternal(c, fg, bg, true); } // Implements the display Debug Print Char operation. void KeDisplayDebugPrintChar(char c, uint32_t fg, uint32_t bg) { - (void)KeDisplayPrintChar(c, fg, bg); + (void)KiDisplayPrintCharInternal(c, fg, bg, false); } DISPLAY_API KSTATUS KeDisplayGetConsoleInfo(KE_CONSOLE_INFO* outInfo) { if (!outInfo) return STATUS_INVALID_PARAMETER; - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - int cols = 0; - int rows = 0; - if (!KiDisplayGetConsoleGridLocked(&cols, &rows)) { - KeReleaseSpinlock(&gDisplayLock, flags); + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + (void)caps; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (!KSUCCESS(s)) { + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_NOT_SUPPORTED; } + int cols = 0; + int rows = 0; + if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) { + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_NOT_SUPPORTED; + } + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); outInfo->Columns = (uint32_t)cols; outInfo->Rows = (uint32_t)rows; outInfo->CursorX = (uint32_t)gConsoleX; outInfo->CursorY = (uint32_t)gConsoleY; outInfo->Attribute = gConsoleAttribute; KeReleaseSpinlock(&gDisplayLock, flags); + + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_SUCCESS; } DISPLAY_API KSTATUS KeDisplaySetConsoleCursor(int col, int row) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + (void)caps; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (!KSUCCESS(s)) { + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_NOT_SUPPORTED; + } + int cols = 0; int rows = 0; - if (!KiDisplayGetConsoleGridLocked(&cols, &rows)) { - KeReleaseSpinlock(&gDisplayLock, flags); + if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) { + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_NOT_SUPPORTED; } if (col < 0 || row < 0 || col >= cols || row >= rows) { - KeReleaseSpinlock(&gDisplayLock, flags); + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_INVALID_PARAMETER; } + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); gConsoleX = col; gConsoleY = row; KeReleaseSpinlock(&gDisplayLock, flags); + + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_SUCCESS; } DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int col, int row, size_t length, size_t* writtenOut) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (writtenOut) *writtenOut = 0; + + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (!KSUCCESS(s)) { + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_NOT_SUPPORTED; + } + int cols = 0; int rows = 0; - if (!KiDisplayGetConsoleGridLocked(&cols, &rows)) { - KeReleaseSpinlock(&gDisplayLock, flags); + if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) { + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_NOT_SUPPORTED; } if (col < 0 || row < 0 || col >= cols || row >= rows) { - KeReleaseSpinlock(&gDisplayLock, flags); - if (writtenOut) *writtenOut = 0; + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_INVALID_PARAMETER; } + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + size_t written = 0; - KiDisplayBeginDraw(); - KSTATUS s = STATUS_SUCCESS; int x = col; int y = row; while (written < length && y < rows) { - size_t i = KiDisplayConsoleIndex(x, y); - uint16_t attr = gConsoleAttrs[i]; - s = KiDisplayDrawConsoleCellLocked(x, y, c, attr, true); + uint16_t attr = DISPLAY_CONSOLE_DEFAULT_ATTR; + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + size_t i = KiDisplayConsoleIndex(x, y); + attr = gConsoleAttrs[i]; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, x * FONT_W, y * FONT_H, + KiDisplayConsoleFg(attr), KiDisplayConsoleBg(attr)); if (!KSUCCESS(s)) break; + + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + size_t i = KiDisplayConsoleIndex(x, y); + gConsoleCells[i] = c; + gConsoleAttrs[i] = (uint8_t)attr; + } + KeReleaseSpinlock(&gDisplayLock, flags); + written++; if (++x >= cols) { x = 0; y++; } } - KSTATUS ps = KiDisplayEndDraw(); - KeReleaseSpinlock(&gDisplayLock, flags); + KSTATUS fs = KiDisplayFinishProviderDraw(device, ops, caps, s); + KiDisplayReleaseProviderMutex(providerHeld); if (writtenOut) *writtenOut = written; - return KSUCCESS(s) ? ps : s; + return KSUCCESS(s) ? fs : s; } DISPLAY_API KSTATUS KeDisplayFillConsoleAttrs(uint16_t attr, int col, int row, size_t length, size_t* writtenOut) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (writtenOut) *writtenOut = 0; + + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (!KSUCCESS(s)) { + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_NOT_SUPPORTED; + } + int cols = 0; int rows = 0; - if (!KiDisplayGetConsoleGridLocked(&cols, &rows)) { - KeReleaseSpinlock(&gDisplayLock, flags); + if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) { + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_NOT_SUPPORTED; } if (col < 0 || row < 0 || col >= cols || row >= rows) { - KeReleaseSpinlock(&gDisplayLock, flags); - if (writtenOut) *writtenOut = 0; + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_INVALID_PARAMETER; } + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + size_t written = 0; - KiDisplayBeginDraw(); - KSTATUS s = STATUS_SUCCESS; int x = col; int y = row; while (written < length && y < rows) { - size_t i = KiDisplayConsoleIndex(x, y); - char c = gConsoleCells[i]; - s = KiDisplayDrawConsoleCellLocked(x, y, c, attr, true); + char ch = ' '; + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + size_t i = KiDisplayConsoleIndex(x, y); + ch = gConsoleCells[i]; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, ch, x * FONT_W, y * FONT_H, + KiDisplayConsoleFg(attr), KiDisplayConsoleBg(attr)); if (!KSUCCESS(s)) break; + + flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + size_t i = KiDisplayConsoleIndex(x, y); + gConsoleCells[i] = ch; + gConsoleAttrs[i] = (uint8_t)attr; + } + KeReleaseSpinlock(&gDisplayLock, flags); + written++; if (++x >= cols) { x = 0; y++; } } - KSTATUS ps = KiDisplayEndDraw(); - KeReleaseSpinlock(&gDisplayLock, flags); + KSTATUS fs = KiDisplayFinishProviderDraw(device, ops, caps, s); + KiDisplayReleaseProviderMutex(providerHeld); if (writtenOut) *writtenOut = written; - return KSUCCESS(s) ? ps : s; + return KSUCCESS(s) ? fs : s; } DISPLAY_API KSTATUS KeDisplayPresent(const DISPLAY_RECT* rects, size_t rectCount) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Present) { - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_NOT_SUPPORTED; - } + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; - if (!rects || rectCount == 0) { - KSTATUS s = KiDisplayFlushDirty(); - KeReleaseSpinlock(&gDisplayLock, flags); + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (!KSUCCESS(s)) { + KiDisplayReleaseProviderMutex(providerHeld); return s; } - - bool hideCursor = gCursorDrawn && KiDisplayUsesSoftwareCursor(); - if (hideCursor) - KiDisplayHideCursorForDraw(); - - gDisplayStats.PresentCount++; - gDisplayStats.PresentRectCount += rectCount; - KSTATUS s = gPrimaryDisplay->Ops->Present(gPrimaryDisplay, rects, rectCount); - - if (hideCursor) - KiDisplayRestoreCursorAfterDraw(); - KeReleaseSpinlock(&gDisplayLock, flags); - return s; -} - -DISPLAY_API KSTATUS KeDisplaySync(void) { - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Sync) { - KeReleaseSpinlock(&gDisplayLock, flags); + if (!ops->Present) { + KiDisplayReleaseProviderMutex(providerHeld); return STATUS_NOT_SUPPORTED; } - gDisplayStats.SyncCount++; - KSTATUS s = gPrimaryDisplay->Ops->Sync(gPrimaryDisplay); - KeReleaseSpinlock(&gDisplayLock, flags); - return s; -} -DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) { - if (!outInfo) - return STATUS_INVALID_PARAMETER; + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - memset(outInfo, 0, sizeof(*outInfo)); - - uint32_t count = 0; - for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) - count++; - outInfo->DeviceCount = count; - outInfo->ManagerStats = gDisplayStats; - - if (!KiDisplayHasDevice()) { - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_NOT_FOUND; - } - - const char* name = gPrimaryDisplay->Name ? gPrimaryDisplay->Name : "?"; - size_t len = strlen(name); - if (len >= sizeof(outInfo->PrimaryName)) - len = sizeof(outInfo->PrimaryName) - 1; - memcpy(outInfo->PrimaryName, name, len); - outInfo->PrimaryName[len] = '\0'; - - outInfo->Caps = gPrimaryDisplay->Caps; - if (gPrimaryDisplay->Ops->GetMode) { - KSTATUS s = gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, &outInfo->Mode); - if (!KSUCCESS(s)) { - KeReleaseSpinlock(&gDisplayLock, flags); - return s; + if (!rects || rectCount == 0) { + s = KiDisplayFlushDirtyWithSnapshot(device, ops); + } else { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) { + gDisplayStats.PresentCount++; + gDisplayStats.PresentRectCount += rectCount; } + KeReleaseSpinlock(&gDisplayLock, flags); + s = ops->Present(device, rects, rectCount); } - if (gPrimaryDisplay->Ops->QueryStats) - (void)gPrimaryDisplay->Ops->QueryStats(gPrimaryDisplay, &outInfo->DeviceStats); - - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; + KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps); + KiDisplayReleaseProviderMutex(providerHeld); + return s; } DISPLAY_API KSTATUS KeDisplaySetCursorImage(int width, int height, int hotX, int hotY, const uint32_t* argbPixels) { + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - KSTATUS s = (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->SetCursorImage) - ? STATUS_NOT_SUPPORTED - : gPrimaryDisplay->Ops->SetCursorImage(gPrimaryDisplay, width, height, hotX, hotY, argbPixels); + DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0; KeReleaseSpinlock(&gDisplayLock, flags); + + if (!device || !ops || !ops->SetCursorImage) { + KiDisplayReleaseProviderMutex(providerHeld); + return STATUS_NOT_SUPPORTED; + } + + s = ops->SetCursorImage(device, width, height, hotX, hotY, argbPixels); + KiDisplayReleaseProviderMutex(providerHeld); return s; } @@ -1034,31 +1930,18 @@ DISPLAY_API KSTATUS KeDisplaySetCursorPosition(int x, int y) { gCursorY = y; gDisplayStats.CursorMoveCount++; - if (KiDisplayOutputSuspended()) { - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; + bool shouldQueue = !KiDisplayOutputSuspended() && + gCursorVisible && + KiDisplayHasCursorOps(); + if (shouldQueue) { + gCursorUpdatePending = true; + gCursorRequestGeneration++; } - - if (!gCursorVisible || !KiDisplayHasCursorOps()) { - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; - } - - if (gCursorDrawn && KiDisplayUsesSoftwareCursor()) - (void)gPrimaryDisplay->Ops->ShowCursor(gPrimaryDisplay, false); - - if (gPrimaryDisplay->Ops->SetCursorPosition) - (void)gPrimaryDisplay->Ops->SetCursorPosition(gPrimaryDisplay, x, y); - - if (gCursorDrawn && KiDisplayUsesHardwareCursor()) { - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; - } - - KSTATUS s = gPrimaryDisplay->Ops->ShowCursor(gPrimaryDisplay, true); - gCursorDrawn = KSUCCESS(s); KeReleaseSpinlock(&gDisplayLock, flags); - return s; + + if (shouldQueue) + KiDisplayQueueCursorWork(); + return STATUS_SUCCESS; } DISPLAY_API KSTATUS KeDisplayShowCursor(bool show) { @@ -1066,37 +1949,21 @@ DISPLAY_API KSTATUS KeDisplayShowCursor(bool show) { gCursorVisible = show; if (show) gDisplayStats.CursorShowCount++; else gDisplayStats.CursorHideCount++; - if (KiDisplayOutputSuspended()) { - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; - } - - if (!KiDisplayHasCursorOps()) { + bool shouldQueue = !KiDisplayOutputSuspended() && KiDisplayHasCursorOps(); + if (!shouldQueue && !show) { gCursorDrawn = false; - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; + gCursorCommittedX = DISPLAY_CURSOR_POS_INVALID; + gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID; } - - if (!show) { - if (gCursorDrawn) - (void)gPrimaryDisplay->Ops->ShowCursor(gPrimaryDisplay, false); - gCursorDrawn = false; - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; + if (shouldQueue) { + gCursorUpdatePending = true; + gCursorRequestGeneration++; } - - if (gCursorDrawn) { - KeReleaseSpinlock(&gDisplayLock, flags); - return STATUS_SUCCESS; - } - - if (gPrimaryDisplay->Ops->SetCursorPosition) - (void)gPrimaryDisplay->Ops->SetCursorPosition(gPrimaryDisplay, gCursorX, gCursorY); - - KSTATUS s = gPrimaryDisplay->Ops->ShowCursor(gPrimaryDisplay, true); - gCursorDrawn = KSUCCESS(s); KeReleaseSpinlock(&gDisplayLock, flags); - return s; + + if (shouldQueue) + KiDisplayQueueCursorWork(); + return STATUS_SUCCESS; } // Implements the boot Fb Context operation. @@ -1331,6 +2198,7 @@ static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) { gCursorDrawX = -1; gCursorDrawY = -1; + gCursorUpdatePending = false; } // Implements the boot Fb Draw Cursor operation. diff --git a/code/kernel/ke_event.c b/code/kernel/ke_event.c index 458d7f6..179e10e 100644 --- a/code/kernel/ke_event.c +++ b/code/kernel/ke_event.c @@ -1,17 +1,100 @@ // Kernel event primitive. This file owns manual-reset event state and waiter wakeups. #include +#include #include #include #include #include #include #include +#include +#include + +// Multi-wait context shared by the stack wait blocks for one wait call. +typedef struct KE_EVENT_MULTI_WAIT_CONTEXT KE_EVENT_MULTI_WAIT_CONTEXT; + +typedef struct KE_EVENT_WAIT_BLOCK { + LIST_ENTRY Link; + KE_EVENT_MULTI_WAIT_CONTEXT* Context; + EVENT* Event; + uint32_t Index; + uint8_t Armed; +} KE_EVENT_WAIT_BLOCK; + +struct KE_EVENT_MULTI_WAIT_CONTEXT { + THREAD* Thread; + KE_EVENT_WAIT_BLOCK* Blocks; + uint32_t Count; + volatile long Completed; + KSTATUS Status; + uint32_t SelectedIndex; +}; // Implements the event Destroy operation. static void KiEventDestroy(OBJECT* obj) { (void)obj; } -// Implements the event Create operation. +// Completes a multi-wait block while the source event lock is held. +static void KiCompleteMultiWaitBlockLocked(KE_EVENT_WAIT_BLOCK* block) { + if (!block || !block->Armed) + return; + + ListRemove(&block->Link); + block->Armed = 0; + + KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context; + if (!ctx) + return; + + if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) != 0) + return; + + ctx->Status = STATUS_SUCCESS; + ctx->SelectedIndex = block->Index; + KeUnblockThreadFromWaitListWithStatus(ctx->Thread, STATUS_SUCCESS); +} + +// Removes any wait blocks that are still linked after the wait completed. +static void KiCleanupMultiWaitBlocks(KE_EVENT_MULTI_WAIT_CONTEXT* ctx) { + if (!ctx || !ctx->Blocks) + return; + + for (uint32_t i = 0; i < ctx->Count; i++) { + KE_EVENT_WAIT_BLOCK* block = &ctx->Blocks[i]; + EVENT* event = block->Event; + if (!event) + continue; + + uintptr_t flags = KeAcquireSpinlock(&event->Lock); + if (block->Armed) { + ListRemove(&block->Link); + block->Armed = 0; + } + KeReleaseSpinlock(&event->Lock, flags); + } +} + +// Removes a multi-wait from every event list during abort or thread teardown. +static void KiRundownMultiWait(THREAD* thread, void* context, KSTATUS status) { + KE_EVENT_MULTI_WAIT_CONTEXT* ctx = (KE_EVENT_MULTI_WAIT_CONTEXT*)context; + if (!ctx) + return; + + if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) == 0) { + ctx->Status = status; + ctx->SelectedIndex = 0; + } + + KiCleanupMultiWaitBlocks(ctx); + + // Abort wakes a truly blocked waiter. Thread teardown may call this after a + // signal already made the thread ready; in that case only the list cleanup + // above is needed. + if (thread && thread->State == ThreadWaiting) + KeUnblockThreadFromWaitListWithStatus(thread, status); +} + +// Creates event. __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) { if (!out) return STATUS_INVALID_PARAMETER; EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT)); @@ -21,6 +104,7 @@ __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialSta e->Type = type; e->Signaled = initialState ? 1 : 0; LIST_INIT(&e->WaitList); + LIST_INIT(&e->MultiWaitList); *out = e; return STATUS_SUCCESS; } @@ -35,22 +119,39 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) { while (!LIST_IS_EMPTY(&event->WaitList)) { LIST_ENTRY* node = ListPopHead(&event->WaitList); THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeSetThreadWaitList(t, (LIST_ENTRY*)0); KeReleaseSpinlock(&event->Lock, flags); KeUnblockThreadWithStatus(t, STATUS_SUCCESS); flags = KeAcquireSpinlock(&event->Lock); } + while (!LIST_IS_EMPTY(&event->MultiWaitList)) { + LIST_ENTRY* node = event->MultiWaitList.Fwd; + KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link); + KiCompleteMultiWaitBlockLocked(block); + } } else { - if (!LIST_IS_EMPTY(&event->WaitList)) { - // Auto-reset handoff: transfer the signal directly to exactly one - // waiter. Do not leave Signaled=1, because a later waiter can steal - // the disk/APC completion before the selected thread resumes. + while (!LIST_IS_EMPTY(&event->WaitList)) { event->Signaled = 0; LIST_ENTRY* node = ListPopHead(&event->WaitList); THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeSetThreadWaitList(t, (LIST_ENTRY*)0); KeReleaseSpinlock(&event->Lock, flags); KeUnblockThreadWithStatus(t, STATUS_SUCCESS); return; } + + while (!LIST_IS_EMPTY(&event->MultiWaitList)) { + LIST_ENTRY* node = event->MultiWaitList.Fwd; + KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link); + KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context; + if (ctx && _InterlockedCompareExchange(&ctx->Completed, 0, 0) == 0) { + event->Signaled = 0; + KiCompleteMultiWaitBlockLocked(block); + KeReleaseSpinlock(&event->Lock, flags); + return; + } + KiCompleteMultiWaitBlockLocked(block); + } event->Signaled = 1; } KeReleaseSpinlock(&event->Lock, flags); @@ -64,7 +165,22 @@ __declspec(dllexport) void KeClearEvent(EVENT* event) { KeReleaseSpinlock(&event->Lock, flags); } -// Waits for for Event. +// Attempts to consume an event signal without blocking. +__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event) { + if (!event) return STATUS_INVALID_PARAMETER; + uintptr_t flags = KeAcquireSpinlock(&event->Lock); + if (!event->Signaled) { + KeReleaseSpinlock(&event->Lock, flags); + return STATUS_TIMEOUT; + } + + if (event->Type == EventAutoReset) + event->Signaled = 0; + KeReleaseSpinlock(&event->Lock, flags); + return STATUS_SUCCESS; +} + +// Waits for Event. __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) { if (!event) return STATUS_INVALID_PARAMETER; if (!KeCanWait()) return STATUS_INVALID_IRQL; @@ -85,3 +201,85 @@ __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) { return KeCommitBlockThread(self, alertable); } +// Waits for any of a set of events, returning the selected index. +__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count, + uint8_t alertable, uint64_t timeoutMs, + uint32_t* signaledIndexOut) { + if (!events || count == 0 || count > SYS_WAIT_MANY_MAX_HANDLES) + return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + if (alertable && KeAreKernelApcsDisabled()) return STATUS_INVALID_PARAMETER; + + for (uint32_t i = 0; i < count; i++) { + if (!events[i]) return STATUS_INVALID_PARAMETER; + KSTATUS s = KeTryWaitEvent(events[i]); + if (KSUCCESS(s)) { + if (signaledIndexOut) *signaledIndexOut = i; + return STATUS_SUCCESS; + } + } + + if (timeoutMs == 0) + return STATUS_TIMEOUT; + + KE_EVENT_WAIT_BLOCK stackBlocks[SYS_WAIT_MANY_MAX_HANDLES]; + KE_EVENT_MULTI_WAIT_CONTEXT ctx; + ctx.Thread = KeGetCurrentThread(); + ctx.Blocks = stackBlocks; + ctx.Count = count; + ctx.Completed = 0; + ctx.Status = STATUS_PENDING; + ctx.SelectedIndex = 0; + + for (uint32_t i = 0; i < count; i++) { + stackBlocks[i].Context = &ctx; + stackBlocks[i].Event = events[i]; + stackBlocks[i].Index = i; + stackBlocks[i].Armed = 0; + LIST_INIT(&stackBlocks[i].Link); + } + + KePrepareBlockThread(ctx.Thread, (OBJECT*)0, alertable); + KeSetThreadWaitRundown(ctx.Thread, KiRundownMultiWait, &ctx); + + if (timeoutMs != SYS_WAIT_TIMEOUT_INFINITE) { + KSTATUS ts = KeArmCurrentThreadTimeout(timeoutMs, STATUS_TIMEOUT); + if (!KSUCCESS(ts)) { + KeClearThreadWaitRundown(ctx.Thread); + KeUnblockThreadFromWaitListWithStatus(ctx.Thread, ts); + return ts; + } + } + + for (uint32_t i = 0; i < count; i++) { + EVENT* event = events[i]; + uintptr_t flags = KeAcquireSpinlock(&event->Lock); + if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0 && event->Signaled) { + if (event->Type == EventAutoReset) + event->Signaled = 0; + ctx.Status = STATUS_SUCCESS; + ctx.SelectedIndex = i; + _InterlockedExchange(&ctx.Completed, 1); + KeReleaseSpinlock(&event->Lock, flags); + KeUnblockThreadFromWaitListWithStatus(ctx.Thread, STATUS_SUCCESS); + break; + } + + if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0) { + stackBlocks[i].Armed = 1; + ListInsertTail(&event->MultiWaitList, &stackBlocks[i].Link); + } + KeReleaseSpinlock(&event->Lock, flags); + } + + KSTATUS waitStatus = KeCommitBlockThread(ctx.Thread, alertable); + if (_InterlockedCompareExchange(&ctx.Completed, 1, 0) == 0) + ctx.Status = waitStatus; + + KiCleanupMultiWaitBlocks(&ctx); + KeClearThreadWaitRundown(ctx.Thread); + + if (ctx.Status == STATUS_SUCCESS && signaledIndexOut) + *signaledIndexOut = ctx.SelectedIndex; + return ctx.Status; +} diff --git a/code/kernel/ke_input.c b/code/kernel/ke_input.c index b1bebc5..9201faf 100644 --- a/code/kernel/ke_input.c +++ b/code/kernel/ke_input.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #define KEY_QUEUE_SIZE 256u #define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u) @@ -32,6 +34,7 @@ static volatile uint32_t gModifiers; typedef struct INPUT_KEY_QUEUE_ENTRY { INPUT_KEY key; char ch; + uint32_t modifiers; } INPUT_KEY_QUEUE_ENTRY; static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE]; @@ -50,6 +53,8 @@ static volatile int gMouseBoundsH = 1080; static volatile uint8_t gMouseButtons; static volatile uint32_t gNextInputDeviceId = 1; +static EVENT* gKeyboardEvent; +static EVENT* gInputStateEvent; static char KiInputKeyToChar(INPUT_KEY key); @@ -77,14 +82,18 @@ static const KEY_CHAR_PAIR gCharMap[] = { { INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' }, }; -// Implements the input Queue Key operation. +// Queues a keyboard event and wakes one blocked reader when the event was accepted. static void KiInputQueueKey(INPUT_KEY key, char ch) { if (key == INPUT_KEY_NONE) return; int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK; if (next == gKeyHead) return; gKeyQueue[gKeyTail].key = key; gKeyQueue[gKeyTail].ch = ch; + gKeyQueue[gKeyTail].modifiers = gModifiers; gKeyTail = next; + + if (gKeyboardEvent) + KeSignalEvent(gKeyboardEvent); } // Implements the input Queue Has Space operation. @@ -211,6 +220,17 @@ void KeInputInitialize(const HAL_API* hal) { gMouseHead = gMouseTail = 0; gMouseButtons = 0; + if (!gKeyboardEvent) { + KSTATUS s = KeEventCreate(EventAutoReset, 0, &gKeyboardEvent); + if (!KSUCCESS(s)) + DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s); + } + if (!gInputStateEvent) { + KSTATUS s = KeEventCreate(EventAutoReset, 0, &gInputStateEvent); + if (!KSUCCESS(s)) + DbgPrint("Input: state wait event create failed st=0x%x\n", s); + } + int w = 1920, h = 1080; DISPLAY_MODE mode; if (KSUCCESS(KeDisplayGetMode(&mode))) { @@ -252,6 +272,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) { gKeyDown[key] = 1u; KiInputSetModifier(key, true); + if (gInputStateEvent) KeSignalEvent(gInputStateEvent); KiInputQueueKey(key, KiInputKeyToChar(key)); KiInputStartRepeat(key, KiInputNowTicks()); } else { @@ -260,6 +281,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) { gKeyDown[key] = 0u; KiInputSetModifier(key, false); + if (gInputStateEvent) KeSignalEvent(gInputStateEvent); KiInputStopRepeat(key); } } @@ -341,27 +363,60 @@ INPUT_KEY KeInputKbPoll(void) { return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key; } + +// Waits until at least one keyboard event is available. +KSTATUS KeInputWaitKey(uint8_t alertable) { + if (KeInputKbPoll() != INPUT_KEY_NONE) + return STATUS_SUCCESS; + if (!gKeyboardEvent) + return STATUS_FAILURE; + + for (;;) { + KSTATUS s = KeWaitForEvent(gKeyboardEvent, alertable); + if (!KSUCCESS(s)) + return s; + if (KeInputKbPoll() != INPUT_KEY_NONE) + return STATUS_SUCCESS; + } +} + // Implements the input Is Key Down operation. bool KeInputIsKeyDown(INPUT_KEY key) { if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false; return gKeyDown[key] != 0; } -INPUT_KEY KeInputReadKey(void) { +// Implements the input Peek Event operation. +bool KeInputPeekEvent(INPUT_KEY_RECORD* record) { + if (!record) return false; if (gKeyHead == gKeyTail) KiInputServiceRepeat(); - if (gKeyHead == gKeyTail) return INPUT_KEY_NONE; - INPUT_KEY key = gKeyQueue[gKeyHead].key; + if (gKeyHead == gKeyTail) return false; + + INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead]; + record->Key = entry.key; + record->Char = entry.ch; + record->Reserved = 0; + record->Modifiers = entry.modifiers; + return true; +} + +// Implements the input Read Event operation. +bool KeInputReadEvent(INPUT_KEY_RECORD* record) { + if (!KeInputPeekEvent(record)) return false; gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK; - return key; + return true; +} + +// Implements the input Read Key operation. +INPUT_KEY KeInputReadKey(void) { + INPUT_KEY_RECORD record; + return KeInputReadEvent(&record) ? record.Key : INPUT_KEY_NONE; } // Implements the input Read Char operation. char KeInputReadChar(void) { - if (gKeyHead == gKeyTail) KiInputServiceRepeat(); - if (gKeyHead == gKeyTail) return 0; - char ch = gKeyQueue[gKeyHead].ch; - gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK; - return ch; + INPUT_KEY_RECORD record; + return KeInputReadEvent(&record) ? record.Char : 0; } // Implements the input Get Modifiers operation. @@ -416,3 +471,32 @@ void KeInputMouseSetBounds(int w, int h) { gMouseBoundsH = h; KeInputMouseSetPosition(gMouseX, gMouseY); } + +// Destroys an input queue handle. +static void KiInputQueueDestroy(OBJECT* object) { + INPUT_QUEUE* queue = CONTAINING_RECORD(object, INPUT_QUEUE, Header); + if (queue->WaitEvent) { + ObDereferenceObject(&queue->WaitEvent->Header); + queue->WaitEvent = (EVENT*)0; + } +} + +// Opens the default keyboard input queue as a waitable object. +KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) { + if (!out) return STATUS_INVALID_PARAMETER; + *out = (INPUT_QUEUE*)0; + if (!gInputStateEvent) return STATUS_FAILURE; + + INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE)); + if (!queue) return STATUS_NO_MEMORY; + queue->Header.Destroy = KiInputQueueDestroy; + ObReferenceObject(&gInputStateEvent->Header); + queue->WaitEvent = gInputStateEvent; + *out = queue; + return STATUS_SUCCESS; +} + +// Returns the event that wakes when keyboard state changes. +EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) { + return queue ? queue->WaitEvent : (EVENT*)0; +} diff --git a/code/kernel/ke_irp.c b/code/kernel/ke_irp.c index b565caa..33ced20 100644 --- a/code/kernel/ke_irp.c +++ b/code/kernel/ke_irp.c @@ -138,7 +138,7 @@ static size_t IoIrpSize(uint8_t stackCount) { __declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) { if (!outIrp) return STATUS_INVALID_PARAMETER; if (stackCount == 0) stackCount = 1; - IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount)); + IRP* irp = (IRP*)MmAllocatePoolTag(IoIrpSize(stackCount), MM_POOL_TAG('i', 'r', 'p', ' ')); if (!irp) return STATUS_NO_MEMORY; IoInitializeIrp(irp, stackCount); *outIrp = irp; diff --git a/code/kernel/ke_mutex.c b/code/kernel/ke_mutex.c index 1d1b034..fe50c5e 100644 --- a/code/kernel/ke_mutex.c +++ b/code/kernel/ke_mutex.c @@ -12,7 +12,7 @@ static void KiMutexDestroy(OBJECT* obj) { (void)obj; } // Implements the mutex Create operation. -KSTATUS KeMutexCreate(MUTEX **out) { +__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) { if (!out) return STATUS_INVALID_PARAMETER; MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX)); if (!m) return STATUS_NO_MEMORY; @@ -25,7 +25,7 @@ KSTATUS KeMutexCreate(MUTEX **out) { } // Implements the acquire Mutex operation. -KSTATUS KeAcquireMutex(MUTEX* mutex) { +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) { if (!mutex) return STATUS_INVALID_PARAMETER; if (!KeCanWait()) return STATUS_INVALID_IRQL; THREAD* self = KeGetCurrentThread(); @@ -45,7 +45,7 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) { } // Implements the try Acquire Mutex operation. -KSTATUS KeTryAcquireMutex(MUTEX* mutex) { +__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) { if (!mutex) return STATUS_INVALID_PARAMETER; THREAD* self = KeGetCurrentThread(); uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); @@ -59,13 +59,14 @@ KSTATUS KeTryAcquireMutex(MUTEX* mutex) { } // Releases mutex. -void KeReleaseMutex(MUTEX* mutex) { +__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) { if (!mutex) return; uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); if (!LIST_IS_EMPTY(&mutex->WaitList)) { LIST_ENTRY* node = ListPopHead(&mutex->WaitList); THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeSetThreadWaitList(waiter, (LIST_ENTRY*)0); // Direct handoff: the selected waiter owns the mutex when its wait // returns. Do not drop Owner to NULL, or another thread can steal it. mutex->Owner = waiter; diff --git a/code/kernel/ke_process.c b/code/kernel/ke_process.c index 4911578..4bfa1cc 100644 --- a/code/kernel/ke_process.c +++ b/code/kernel/ke_process.c @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -140,7 +142,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) { if (p == system) info->Flags |= SYS_PROCESS_FLAG_SYSTEM; if (p == current) info->Flags |= SYS_PROCESS_FLAG_CURRENT; - MmQueryProcessUserMemory(p, &info->UserRegionCount, + MmVadQueryProcessMemory(p, &info->VadCount, &info->UserReservedBytes, &info->UserCommittedBytes); @@ -158,27 +160,128 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) { return count; } -// Destructor (called by ObDereferenceObject when RefCount hits zero). -static void KiProcessDestroy(OBJECT* obj) { - PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header); +// Converts a kernel thread state to the public ABI state. +static uint32_t KiThreadStateToSys(THREAD_STATE state) { + switch (state) { + case ThreadReady: return SYS_THREAD_STATE_READY; + case ThreadRunning: return SYS_THREAD_STATE_RUNNING; + case ThreadWaiting: return SYS_THREAD_STATE_WAITING; + case ThreadDead: return SYS_THREAD_STATE_DEAD; + default: return SYS_THREAD_STATE_DEAD; + } +} - if (proc == gSystemProcess) - gSystemProcess = (PROCESS*)0; +// Converts a kernel object type to the public wait-object type. +static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) { + switch (type) { + case ObjectTypeProcess: return SYS_THREAD_WAIT_PROCESS; + case ObjectTypeThread: return SYS_THREAD_WAIT_THREAD; + case ObjectTypeMutex: return SYS_THREAD_WAIT_MUTEX; + case ObjectTypeSemaphore: return SYS_THREAD_WAIT_SEMAPHORE; + case ObjectTypeEvent: return SYS_THREAD_WAIT_EVENT; + case ObjectTypeQueue: return SYS_THREAD_WAIT_QUEUE; + case ObjectTypeFile: return SYS_THREAD_WAIT_FILE; + case ObjectTypeDir: return SYS_THREAD_WAIT_DIR; + case ObjectTypeAsyncIo: return SYS_THREAD_WAIT_ASYNC_IO; + case ObjectTypeSocket: return SYS_THREAD_WAIT_SOCKET; + case ObjectTypeDevice: return SYS_THREAD_WAIT_DEVICE; + case ObjectTypeDriver: return SYS_THREAD_WAIT_DRIVER; + case ObjectTypeUsbDevice: return SYS_THREAD_WAIT_USB_DEVICE; + case ObjectTypeUsbPipe: return SYS_THREAD_WAIT_USB_PIPE; + case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING; + case ObjectTypeSurface: return SYS_THREAD_WAIT_SURFACE; + default: return SYS_THREAD_WAIT_UNKNOWN; + } +} - // Release all remaining handles. - for (int i = 1; i < HANDLE_TABLE_SIZE; i++) { - if (proc->Handles[i].Object) { - ObDereferenceObject(proc->Handles[i].Object); +// Copies one process's thread telemetry into caller-provided storage. +KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut) { + if (!countOut) return STATUS_INVALID_PARAMETER; + if (maxCount && !infoOut) return STATUS_INVALID_PARAMETER; + + PROCESS* process = KeGetProcessById(pid); + if (!process) return STATUS_NOT_FOUND; + + THREAD* current = KeGetCurrentThread(); + size_t count = 0; + + uintptr_t flags = KeAcquireSpinlock(&process->HandleLock); + LIST_ENTRY* head = &process->ThreadList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, ProcessLink); + if (infoOut && count < maxCount) { + SYS_THREAD_INFO* info = &infoOut[count]; + memset(info, 0, sizeof(*info)); + info->Size = sizeof(*info); + info->ProcessId = process->ProcessId; + info->ThreadId = thread->ThreadId; + info->State = KiThreadStateToSys(thread->State); + info->Priority = (uint32_t)thread->Priority; + info->RunningCpu = thread->RunningCpu; + info->LastCpu = thread->LastCpu; + info->QuantumRemaining = thread->QuantumRemaining; + info->QuantumTicks = thread->QuantumTicks; + info->CpuBurstTicks = thread->CpuBurstTicks; + info->LastRunTicks = thread->LastRunTicks; + info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE; + info->WaitStatus = (uint32_t)thread->WaitStatus; + info->KernelApcDisable = thread->KernelApcDisable; + + if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT; + if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE; + if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL; + if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED; + if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT; + if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING; + if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING; + } + count++; + } + KeReleaseSpinlock(&process->HandleLock, flags); + + *countOut = count; + ObDereferenceObject(&process->Header); + return STATUS_SUCCESS; +} + +// Releases all kernel objects still reachable from a process handle table. +// This is part of process exit, not merely object destruction: a parent may keep +// a handle to an exited process for wait/query, but the dead process must not +// keep its own files, mappings, queues, sockets, or synchronization objects alive. +static void KiProcessReleaseAllHandles(PROCESS* proc) { + if (!proc) return; + + for (uint32_t i = 1; i < HANDLE_TABLE_SIZE; ++i) { + OBJECT* obj = (OBJECT*)0; + + uintptr_t flags = KeAcquireSpinlock(&proc->HandleLock); + obj = proc->Handles[i].Object; + if (obj) { proc->Handles[i].Object = (OBJECT*)0; proc->Handles[i].Rights = 0; proc->Handles[i].Flags = 0; } + KeReleaseSpinlock(&proc->HandleLock, flags); + + if (obj) + ObDereferenceObject(obj); } +} + +// Releases the process-owned address space and per-process kernel resources. +// Idempotent because the normal exit path and the final OBJECT destructor can +// both observe a process whose threads are already gone. +static void KiProcessRundownResources(PROCESS* proc) { + if (!proc) return; + if (_InterlockedExchange(&proc->ResourcesReleased, 1) != 0) + return; + + KiProcessReleaseAllHandles(proc); // Release every user VA region owned by this process. This includes the // main image, DLLs, TEBs/stacks that survived thread teardown, process - // environment strings, and leaked user allocations. - MmDestroyAllUserRegions(proc); + // environment strings, mmap views, and leaked user allocations. + MmVadDestroyAllForProcess(proc); proc->ImageBase = 0; proc->ImageSize = 0; proc->ModuleCount = 0; @@ -190,8 +293,21 @@ static void KiProcessDestroy(OBJECT* obj) { proc->UserCmdlineSize = 0; proc->UserCwdBase = 0; proc->UserCwdSize = 0; +} - // Release the exit event. +// Destructor (called by ObDereferenceObject when RefCount hits zero). +static void KiProcessDestroy(OBJECT* obj) { + PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header); + + if (proc == gSystemProcess) + gSystemProcess = (PROCESS*)0; + + // If the process failed during launch, or no normal exit completion path + // ran, make sure its own handles and VADs are still reclaimed. + KiProcessRundownResources(proc); + + // Release the exit event only when the process object itself dies. External + // process handles may still wait/query an exited process after rundown. if (proc->ExitEvent) { ObDereferenceObject(&proc->ExitEvent->Header); proc->ExitEvent = (EVENT *)0; @@ -227,6 +343,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) { proc->ExitCode = 0; proc->Terminating = 0; proc->Exited = 0; + proc->ResourcesReleased = 0; proc->ImageBase = 0; proc->ImageSize = 0; proc->UserPebBase = 0; @@ -240,10 +357,10 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) { proc->ModuleCount = 0; proc->CurrentDirectory[0] = '\0'; proc->CurrentDirectoryLength = 0; - proc->CurrentDrive = 0; + proc->CurrentDrive = KeGetSystemRootDrive(); proc->CwdLock = (SPINLOCK)SPINLOCK_INIT; - LIST_INIT(&proc->UserRegions); - proc->UserRegionLock = (SPINLOCK)SPINLOCK_INIT; + LIST_INIT(&proc->VadList); + proc->VadLock = (SPINLOCK)SPINLOCK_INIT; // Slot 0 is the invalid handle - leave it NULL. memset(proc->Handles, 0, sizeof(proc->Handles)); @@ -427,7 +544,7 @@ fail: } // Process exit / termination. -static void KiCompleteProcessExitIfLastThread(PROCESS* process) { +void KeCompleteProcessExitIfLastThread(PROCESS* process) { if (!process) return; uintptr_t flags = KeAcquireSpinlock(&process->HandleLock); @@ -435,6 +552,7 @@ static void KiCompleteProcessExitIfLastThread(PROCESS* process) { KeReleaseSpinlock(&process->HandleLock, flags); if (threadCount == 0 && _InterlockedExchange(&process->Exited, 1) == 0) { + KiProcessRundownResources(process); if (process->ExitEvent) KeSignalEvent(process->ExitEvent); } } @@ -448,7 +566,7 @@ void KeExitProcess(PROCESS* process, int exitCode) { // The calling thread exits immediately after this in SysExitProcess. Do // not drop the process object here: each thread owns one process reference, // and KiThreadDestroy releases that reference after unlinking the thread. - KiCompleteProcessExitIfLastThread(process); + KeCompleteProcessExitIfLastThread(process); } // Implements the terminate Process operation. @@ -469,7 +587,15 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) { if (t == self) continue; - (void)KeAlertThread(t); + // A terminating process must not be held open by worker threads that + // are blocked in non-alertable waits. User-mode DLLs such as Bink can + // leave IO/submit workers parked on semaphores; after the foreground + // thread crashes, those waiters would otherwise keep ThreadCount > 0 + // and prevent the process ExitEvent from waking the shell. + if (t->State == ThreadWaiting) + KeAbortThreadWait(t, STATUS_ALERTED); + else + (void)KeAlertThread(t); if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) { KCPU* cpu = &gCpus[t->RunningCpu]; @@ -481,7 +607,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) { } KeReleaseSpinlock(&process->HandleLock, flags); - KiCompleteProcessExitIfLastThread(process); + KeCompleteProcessExitIfLastThread(process); return STATUS_SUCCESS; } @@ -535,6 +661,8 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) { process->Handles[handle].Rights = 0; process->Handles[handle].Flags = 0; KeReleaseSpinlock(&process->HandleLock, flags); + if (obj->Type == ObjectTypeFile) + (void)IoFlushFile((FILE_OBJECT*)obj); ObDereferenceObject(obj); return; } diff --git a/code/kernel/ke_sched.c b/code/kernel/ke_sched.c index 1504df4..67a64b4 100644 --- a/code/kernel/ke_sched.c +++ b/code/kernel/ke_sched.c @@ -17,6 +17,7 @@ #include static LIST_ENTRY gRunQueue[PriorityCount]; +static LIST_ENTRY gSleepList; static SPINLOCK gSchedLock; static uint32_t gAgingCounter = 0; @@ -305,6 +306,7 @@ static void KiIdleLoop(void) { // Initializes scheduler Global. void KiInitializeSchedulerGlobal(void) { for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]); + LIST_INIT(&gSleepList); gSchedLock = (SPINLOCK)SPINLOCK_INIT; KSTATUS s = KeCreateProcess("idle", &gIdleProcess); @@ -349,6 +351,8 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { idle->WaitStatus = STATUS_SUCCESS; idle->WaitListHead = (LIST_ENTRY*)0; idle->AlertableWait = 0; + idle->SleepUntilTick = 0; + idle->TimeoutStatus = STATUS_SUCCESS; idle->UserStackBase = (void*)0; idle->Teb = (void*)0; idle->TebSize = 0; @@ -358,7 +362,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { LIST_INIT(&idle->ProcessLink); LIST_INIT(&idle->SchedulerLink); - void* kstack = MmAllocatePool(KSTACK_SIZE); + void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k')); if (!kstack) { ObDereferenceObject(&idle->Header); return STATUS_NO_MEMORY; @@ -621,12 +625,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) { } -// Implements the scheduler Wake Thread operation. -void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { +// Completes a wait while the scheduler lock is held. +static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) { if (!thread) return; - uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - // Complete the wait while holding the scheduler lock. On SMP a wake can // race the prepare->commit window on another CPU: the target thread is // already on the object's wait list and has State=ThreadWaiting, but it is @@ -634,7 +636,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { // That case must complete in-place, not enqueue the still-running thread. if (thread->State != ThreadWaiting) { thread->WaitStatus = status; - KeReleaseSpinlock(&gSchedLock, flags); return; } @@ -642,6 +643,8 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { thread->WaitStatus = status; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; thread->CpuBurstTicks = 0; thread->QuantaConsumed = 0; @@ -652,7 +655,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { // KeCommitBlockThread() will see State != ThreadWaiting and return the // completed wait status without ever entering the ready queue. thread->State = ThreadRunning; - KeReleaseSpinlock(&gSchedLock, flags); return; } @@ -662,10 +664,49 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { KiInsertReadyLocked(thread); KiPokeRunnableCpusLocked(thread); +} + +// Wakes a thread after first removing its scheduler-link wait membership. +void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) { + if (!thread) return; + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->State == ThreadWaiting && thread->WaitListHead) { + ListRemove(&thread->SchedulerLink); + thread->WaitListHead = (LIST_ENTRY*)0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; + } + KiWakeThreadLocked(thread, status); KeReleaseSpinlock(&gSchedLock, flags); } +// Implements the scheduler Wake Thread operation. +void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + KiWakeThreadLocked(thread, status); + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Wakes threads whose scheduler-tick sleep deadline has expired. +static void KiWakeExpiredSleepersLocked(uint64_t nowTick) { + LIST_ENTRY* node = gSleepList.Fwd; + while (node != &gSleepList) { + LIST_ENTRY* next = node->Fwd; + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + + if (thread->State == ThreadWaiting && + thread->WaitListHead == &gSleepList && + thread->SleepUntilTick <= nowTick) { + ListRemove(node); + thread->WaitListHead = (LIST_ENTRY*)0; + KiWakeThreadLocked(thread, thread->TimeoutStatus); + } + + node = next; + } +} + // Implements the scheduler Tick operation. void KeSchedulerTick(void) { KCPU* cpu = KeGetCurrentCpu(); @@ -677,6 +718,7 @@ void KeSchedulerTick(void) { KiArmPostSwitchUnlock(cur, flags); gSchedStats.Ticks++; + KiWakeExpiredSleepersLocked(gSchedStats.Ticks); int requestReschedule = 0; @@ -745,6 +787,90 @@ void KeSchedulerTick(void) { KiRequestReschedule(); } + +// Arms a timeout for the current thread after a wait primitive has prepared it to block. +KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) { + if (milliseconds == 0) + return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) + return STATUS_INVALID_IRQL; + + THREAD* self = KeGetCurrentThread(); + if (!self || self->IsIdleThread) + return STATUS_INVALID_PARAMETER; + + uint64_t ticks = milliseconds / SCHED_TICK_MS; + if ((milliseconds % SCHED_TICK_MS) != 0) + ticks++; + if (ticks == 0) + ticks = 1; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (self->State != ThreadWaiting || self->WaitListHead != (LIST_ENTRY*)0 || + self->RunningCpu != KeGetCurrentCpu()->CpuIndex) { + KeReleaseSpinlock(&gSchedLock, flags); + return STATUS_INVALID_PARAMETER; + } + + uint64_t now = gSchedStats.Ticks; + uint64_t wakeTick = now + ticks; + if (wakeTick < now) + wakeTick = UINT64_MAX; + + self->WaitListHead = &gSleepList; + self->SleepUntilTick = wakeTick; + self->TimeoutStatus = timeoutStatus; + ListInsertTail(&gSleepList, &self->SchedulerLink); + KeReleaseSpinlock(&gSchedLock, flags); + return STATUS_SUCCESS; +} + +// Implements the delay Thread operation. +__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) { + if (milliseconds == 0) { + KeYield(); + return STATUS_SUCCESS; + } + + if (!KeCanWait()) + return STATUS_INVALID_IRQL; + + THREAD* self = KeGetCurrentThread(); + if (!self || self->IsIdleThread) + return STATUS_INVALID_PARAMETER; + + uint64_t ticks = milliseconds / SCHED_TICK_MS; + if ((milliseconds % SCHED_TICK_MS) != 0) + ticks++; + if (ticks == 0) + ticks = 1; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + + if (self->Queued || self->State != ThreadRunning || + self->RunningCpu != KeGetCurrentCpu()->CpuIndex) { + KeReleaseSpinlock(&gSchedLock, flags); + return STATUS_INVALID_PARAMETER; + } + + uint64_t now = gSchedStats.Ticks; + uint64_t wakeTick = now + ticks; + if (wakeTick < now) + wakeTick = UINT64_MAX; + + self->State = ThreadWaiting; + self->WaitObject = (OBJECT*)0; + self->WaitStatus = STATUS_PENDING; + self->WaitListHead = &gSleepList; + self->AlertableWait = 0; + self->SleepUntilTick = wakeTick; + self->TimeoutStatus = STATUS_SUCCESS; + ListInsertTail(&gSleepList, &self->SchedulerLink); + + KeReleaseSpinlock(&gSchedLock, flags); + return KeCommitBlockThread(self, 0); +} + // Implements the yield operation. __declspec(dllexport) void KeYield(void) { KeAssertIrql(PASSIVE_LEVEL); diff --git a/code/kernel/ke_selftest.c b/code/kernel/ke_selftest.c index c898f52..49061e3 100644 --- a/code/kernel/ke_selftest.c +++ b/code/kernel/ke_selftest.c @@ -14,10 +14,13 @@ #include #include #include +#include #include #include #include #include +#include +#include #include #include @@ -478,11 +481,118 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) { -// Virtual memory mapping invariants. + +// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs. +static void KiRunPfnDatabaseSelfTest(void) { + MM_PFN_STATS before; + MM_PFN_STATS after; + MmPfnQueryStats(&before); + + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel); + if (!frame) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n"); + + MM_PFN* pfn = MmPfnGet(frame); + if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1) + KePanic(KE_PANIC_CODE_SELFTEST, + "PFN selftest: allocated frame metadata invalid phys=%016llx\n", + (unsigned long long)frame); + + if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount); + if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount); + + if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount); + if (MmPfnFree(frame) != STATUS_ACCESS_DENIED) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n"); + if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount); + if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount); + if (MmPfnFree(frame) != STATUS_ACCESS_DENIED) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n"); + if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount); + if (!KSUCCESS(MmPfnFree(frame))) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n"); + + PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma); + if (!run) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n"); + MM_PFN* p0 = MmPfnGet(run); + MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE); + if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n"); + MmPfnFree(run); + MmPfnFree(run + PAGE_SIZE); + + // Accounting buckets are part of the VM contract. A missing increment here + // usually means a new allocation path is using an anonymous usage class. + PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache); + PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage); + PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile); + PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous); + PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow); + PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface); + if (!clean || !image || !mapped || !anon || !cow || !surface) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n"); + + MM_PFN_STATS bucketStats; + MmPfnQueryStats(&bucketStats); + if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 || + bucketStats.ImageFrames < before.ImageFrames + 1 || + bucketStats.MappedFileFrames < before.MappedFileFrames + 1 || + bucketStats.AnonymousFrames < before.AnonymousFrames + 1 || + bucketStats.CowFrames < before.CowFrames + 1 || + bucketStats.SurfaceFrames < before.SurfaceFrames + 1) { + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n"); + } + + // Standby is the clean-page holding state used by later file-cache and image + // reclaim. It must not change the free count until the owning cache drops it. + if (!KSUCCESS(MmPfnMoveToStandby(clean))) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n"); + if (!KSUCCESS(MmPfnMoveToStandby(image))) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n"); + + MM_PFN_STATS standbyStats; + MmPfnQueryStats(&standbyStats); + if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 || + standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 || + standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 || + standbyStats.FreeFrames != bucketStats.FreeFrames) { + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n"); + } + + if (!KSUCCESS(MmPfnActivate(clean))) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n"); + if (!KSUCCESS(MmPfnActivate(image))) + KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n"); + + (void)MmPfnFree(clean); + (void)MmPfnFree(image); + (void)MmPfnFree(mapped); + (void)MmPfnFree(anon); + (void)MmPfnFree(cow); + (void)MmPfnFree(surface); + + MmPfnQueryStats(&after); + if (before.FreeFrames != after.FreeFrames) + KePanic(KE_PANIC_CODE_SELFTEST, + "PFN selftest: free count leak before=%llu after=%llu\n", + (unsigned long long)before.FreeFrames, + (unsigned long long)after.FreeFrames); + + DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n"); +} + +// Verifies the page-table manager invariants that the higher VMM will depend on. static void KiRunMemoryMappingSelfTest(void) { PHYSICAL_ADDRESS phys = 0; uint64_t flags = 0; - KSTATUS s = MmQueryPage(0, &phys, &flags); + KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags); if (s != STATUS_NOT_FOUND) { KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n", (unsigned long long)phys, @@ -490,7 +600,329 @@ static void KiRunMemoryMappingSelfTest(void) { (unsigned)s); } - DbgPrint("MM selftest: page zero unmapped\n"); + uintptr_t testVa = 0xFFFFA00000000000ULL; + s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags); + if (s != STATUS_NOT_FOUND) { + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n", + (unsigned long long)testVa, + (unsigned long long)phys, + (unsigned long long)flags); + } + + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel); + if (!frame) + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n"); + + s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + MmPfnFree(frame); + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s); + } + + volatile uint64_t* probe = (volatile uint64_t*)testVa; + *probe = 0x1122334455667788ULL; + if (*probe != 0x1122334455667788ULL) + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n"); + + s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags); + if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE)) + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n", + s, + (unsigned long long)phys, + (unsigned long long)frame, + (unsigned long long)flags); + + s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s); + + s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags); + if (!KSUCCESS(s) || (flags & PAGE_WRITE)) + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n", + s, + (unsigned long long)flags); + + s = MmPteUnmapPage(testVa); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s); + + s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags); + if (s != STATUS_NOT_FOUND) + KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s); + + MmPfnFree(frame); + DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n"); +} + + +// Verifies sparse VAD commitment and NOACCESS protection metadata. +static void KiRunVadSparseSelfTest(void) { + PROCESS* process = KeGetCurrentProcess(); + if (!process) + process = KeGetSystemProcess(); + if (!process) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n"); + + size_t beforeVadCount = 0; + size_t beforeReserved = 0; + size_t beforeCommitted = 0; + MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted); + + void* base = (void*)0; + KSTATUS s = MmVadReserve(process, + PAGE_SIZE * 4, + PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, + MmVadGeneric, + &base); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s); + + uint8_t* p = (uint8_t*)base; + s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s); + + s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s); + + void* queryBase = (void*)0; + size_t querySize = 0; + uint64_t queryFlags = 0; + s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0); + if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=%p size=%llu flags=%016llx\n", + s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags); + + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0); + if (s != STATUS_NOT_FOUND) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s); + + s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s); + s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s); + + s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0); + if (s != STATUS_NOT_FOUND) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s); + + p[0] = 0x5A; + p[PAGE_SIZE * 2] = 0xA5; + + s = MmVadProtect(process, p, PAGE_SIZE, 0); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s); + + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0); + if (s != STATUS_NOT_FOUND) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s); + + s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0); + if (!KSUCCESS(s) || queryFlags != 0) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n", + s, (unsigned long long)queryFlags); + + s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s); + if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n"); + + size_t afterVadCount = 0; + size_t afterReserved = 0; + size_t afterCommitted = 0; + MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted); + if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2) + KePanic(KE_PANIC_CODE_SELFTEST, + "VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n", + (unsigned long long)beforeCommitted, + (unsigned long long)afterCommitted, + (unsigned long long)beforeVadCount, + (unsigned long long)afterVadCount); + + s = MmVadDestroy(process, base); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s); + + MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted); + if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n"); + + DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n"); +} + + +// Verifies that writable copy-on-write views split before either alias can modify shared state. +static void KiRunVadCopyOnWriteSelfTest(void) { + PROCESS* process = KeGetCurrentProcess(); + if (!process) + process = KeGetSystemProcess(); + if (!process) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n"); + + void* base = (void*)0; + KSTATUS s = MmVadCreateCommitted(process, + PAGE_SIZE, + PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, + MmVadGeneric, + &base); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s); + + s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s); + + uint8_t* source = (uint8_t*)base; + source[0] = 0x31; + + void* cowBase = (void*)0; + s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s); + + PHYSICAL_ADDRESS sourcePhys = 0; + PHYSICAL_ADDRESS cowPhys = 0; + uint64_t sourceFlags = 0; + uint64_t cowFlags = 0; + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags); + if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n", + s, (unsigned long long)sourceFlags); + + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags); + if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n", + s, + (unsigned long long)sourcePhys, + (unsigned long long)cowPhys, + (unsigned long long)cowFlags); + + s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s); + + uint8_t* copy = (uint8_t*)cowBase; + copy[0] = 0x62; + if (source[0] != 0x31 || copy[0] != 0x62) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n", + source[0], copy[0]); + + PHYSICAL_ADDRESS splitPhys = 0; + uint64_t splitFlags = 0; + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags); + if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n", + s, + (unsigned long long)sourcePhys, + (unsigned long long)splitPhys, + (unsigned long long)splitFlags); + + (void)MmVadDestroy(process, cowBase); + (void)MmVadDestroy(process, base); + + void* zeroBase = (void*)0; + s = MmVadCreateCommitted(process, + PAGE_SIZE, + PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, + MmVadGeneric, + &zeroBase); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s); + + s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s); + + PHYSICAL_ADDRESS zeroPhys = 0; + uint64_t zeroFlags = 0; + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags); + if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n", + s, (unsigned long long)zeroFlags); + + s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s); + + PHYSICAL_ADDRESS privatePhys = 0; + uint64_t privateFlags = 0; + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags); + if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n", + s, + (unsigned long long)zeroPhys, + (unsigned long long)privatePhys, + (unsigned long long)privateFlags); + + ((uint8_t*)zeroBase)[0] = 0x9C; + if (((uint8_t*)zeroBase)[0] != 0x9C) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n"); + + (void)MmVadDestroy(process, zeroBase); + DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n"); +} + + +// Verifies that a private resident VAD page can move to swap and fault back in. +static void KiRunVadSwapSelfTest(void) { + if (!MmSwapIsEnabled()) { + DbgPrint("VAD swap selftest: SKIP, swap is disabled\n"); + return; + } + + PROCESS* process = KeGetCurrentProcess(); + if (!process) + process = KeGetSystemProcess(); + if (!process) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n"); + + void* base = (void*)0; + KSTATUS s = MmVadCreateCommitted(process, + PAGE_SIZE, + PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, + MmVadGeneric, + &base); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s); + + s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s); + + uint8_t* p = (uint8_t*)base; + p[0] = 0xC1; + p[PAGE_SIZE - 1] = 0x5E; + + void* trimmed = (void*)0; + s = MmVadTrimOnePageForTest(process, &trimmed); + if (!KSUCCESS(s) || trimmed != base) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=%p base=%p\n", s, trimmed, base); + + s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0); + if (s != STATUS_NOT_FOUND) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s); + + s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s); + + if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E) + KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]); + + (void)MmVadDestroy(process, base); + DbgPrint("VAD swap selftest: private pageout/pagein PASS\n"); } // String-view ABI self-test. @@ -614,6 +1046,8 @@ static void KiRunAsyncFileIoSelfTest(void) { &st.Iosb, st.CompletionEvent, &st.Apc, + (MM_LOCKED_USER_BUFFER*)0, + (MM_LOCKED_USER_BUFFER*)0, &readReq); if (s != STATUS_PENDING) KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s); @@ -683,6 +1117,8 @@ static void KiRunAsyncFileIoSelfTest(void) { &wiosb, wev, (KAPC*)0, + (MM_LOCKED_USER_BUFFER*)0, + (MM_LOCKED_USER_BUFFER*)0, &writeReq); if (s != STATUS_PENDING) KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s); @@ -832,7 +1268,11 @@ void KiRunEarlyKernelSelfTests(void) { void KiRunKernelSelfTests(const HAL_API* hal) { DbgPrint("Kernel selftests: begin\n"); KiRunKernelApcSelfTest(); + KiRunPfnDatabaseSelfTest(); KiRunMemoryMappingSelfTest(); + KiRunVadSparseSelfTest(); + KiRunVadCopyOnWriteSelfTest(); + KiRunVadSwapSelfTest(); KiRunStringViewSelfTest(); KiRunIrpKernelApcSelfTest(hal); KiRunAsyncFileIoSelfTest(); diff --git a/code/kernel/ke_semaphore.c b/code/kernel/ke_semaphore.c index 01e7770..ec6c1ea 100644 --- a/code/kernel/ke_semaphore.c +++ b/code/kernel/ke_semaphore.c @@ -68,6 +68,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) { // consuming a count later. LIST_ENTRY* node = ListPopHead(&sem->WaitList); THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeSetThreadWaitList(waiter, (LIST_ENTRY*)0); KeReleaseSpinlock(&sem->Lock, flags); KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS); return; diff --git a/code/kernel/ke_smp.c b/code/kernel/ke_smp.c index a1ed696..a49898f 100644 --- a/code/kernel/ke_smp.c +++ b/code/kernel/ke_smp.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -141,15 +142,17 @@ static KSTATUS KiBuildApTrampoline(void) { DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n", (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN, (uint64_t)AP_TRAMPOLINE_MAX_PHYS); - frame = MmAllocateFrameInRange(AP_TRAMPOLINE_PREFERRED_MIN, - AP_TRAMPOLINE_MAX_PHYS); + frame = MmPfnAllocateInRange(AP_TRAMPOLINE_PREFERRED_MIN, + AP_TRAMPOLINE_MAX_PHYS, + MmPfnUseKernel); } if (!frame) { DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n", (uint64_t)AP_TRAMPOLINE_FALLBACK_MIN, (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN); - frame = MmAllocateFrameInRange(AP_TRAMPOLINE_FALLBACK_MIN, - AP_TRAMPOLINE_PREFERRED_MIN); + frame = MmPfnAllocateInRange(AP_TRAMPOLINE_FALLBACK_MIN, + AP_TRAMPOLINE_PREFERRED_MIN, + MmPfnUseKernel); } if (!frame) { DbgPrint("SMP: no free PMM frame available below 0x%016llX for AP trampoline\n", @@ -589,7 +592,7 @@ void KeBuildProcessorTopology(void) { // Creates ap Stack. static KSTATUS KiAllocateApStack(KCPU* cpu) { // AP kernel stacks do not need physically contiguous memory. Requiring - // MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being + // MmPfnAllocateContiguous(8) made SMP startup depend on a DMA-style run being // available, which is fragile on real machines and unrelated to AP startup. size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE; void* stack = MmAllocatePool(stackBytes); @@ -967,12 +970,11 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) { cpuIndex, st); } - THREAD* thread = (THREAD*)0; st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker, (uintptr_t)arg, PriorityNormal, (1ULL << cpuIndex), - &thread); + (THREAD**)0); if (!KSUCCESS(st)) { KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: AP%u worker create failed status=0x%x\n", diff --git a/code/kernel/ke_syscall.asm b/code/kernel/ke_syscall.asm index 168b0bd..6c01689 100644 --- a/code/kernel/ke_syscall.asm +++ b/code/kernel/ke_syscall.asm @@ -152,7 +152,7 @@ KeSyscallEntry ENDP ; also shows which register held the null target. ; ; KeFaultDispatchFrame signature (MS x64): -; void KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx +; KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx ; --------------------------------------------------------------------------- KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish @@ -379,8 +379,45 @@ pf_kernel_frame: pf_call: and rsp, -16 sub rsp, FAULT_CALL_SHADOW + mov rbx, rax ; Keep the KFAULT_FRAME pointer across C. mov rcx, rax call KeFaultDispatchFrame + test eax, eax + jnz pf_unhandled_return + + mov rdx, rbx ; rdx = KFAULT_FRAME*. + mov rcx, [rdx + KF_Rcx] + mov r8, [rdx + KF_R8] + mov r9, [rdx + KF_R9] + mov rsi, [rdx + KF_Rsi] + mov rdi, [rdx + KF_Rdi] + mov rbp, [rdx + KF_Rbp] + mov r12, [rdx + KF_R12] + mov r13, [rdx + KF_R13] + mov r14, [rdx + KF_R14] + mov r15, [rdx + KF_R15] + mov rbx, [rdx + KF_Rbx] + + lea rsp, [rdx + KFAULT_SIZE + 24] ; Skip saved RAX/R11 and the CPU error code. + mov rax, [rdx + KF_Rax] + mov r11, [rdx + KF_R11] + mov r10, [rdx + KF_Cs] + test r10, 3 + jz pf_resume_no_swapgs + mov r10, [rdx + KF_R10] + mov rdx, [rdx + KF_Rdx] + swapgs + iretq +pf_resume_no_swapgs: + mov r10, [rdx + KF_R10] + mov rdx, [rdx + KF_Rdx] + iretq + +pf_unhandled_return: + cli +pf_unhandled_halt: + hlt + jmp pf_unhandled_halt KeFaultPF ENDP diff --git a/code/kernel/ke_syscall.c b/code/kernel/ke_syscall.c index 81fc6e4..530bcdc 100644 --- a/code/kernel/ke_syscall.c +++ b/code/kernel/ke_syscall.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #ifndef PROCESS_LAUNCH_DEBUG @@ -52,10 +53,15 @@ #define ProcDbg(...) ((void)0) #endif #include +#include +#include +#include +#include #include #include #include #include +#include #include #include #include @@ -95,6 +101,196 @@ typedef struct SYS_USER_ASYNC_APC_CONTEXT { void* UserContext; } SYS_USER_ASYNC_APC_CONTEXT; + +typedef struct FILE_MAPPING_OBJECT { + OBJECT Header; + FILE_OBJECT* File; + uint64_t MaximumSize; + uint64_t PageFlags; + uint32_t Protect; + uint32_t Flags; +} FILE_MAPPING_OBJECT; + +static void KiFileMappingDestroy(OBJECT* obj) { + FILE_MAPPING_OBJECT* mapping = CONTAINING_RECORD(obj, FILE_MAPPING_OBJECT, Header); + if (mapping->File) { + ObDereferenceObject(&mapping->File->Header); + mapping->File = (FILE_OBJECT*)0; + } +} + +typedef struct SURFACE_OBJECT { + OBJECT Header; + PROCESS* Owner; + uint32_t Width; + uint32_t Height; + uint32_t PitchBytes; + uint32_t Format; + uint32_t Flags; + size_t PayloadSize; + size_t PageCount; + uint8_t StatsRegistered; + uint64_t MapCount; + uint64_t PresentCount; + uint64_t PresentRectCount; + uint64_t PresentClippedRectCount; + uint64_t PresentEmptyClipCount; + uint64_t PresentRowsCopied; + uint64_t PresentBytesCopied; + PHYSICAL_ADDRESS* Frames; + void* UserBase; + uint32_t* RowScratch; + MUTEX* Lock; +} SURFACE_OBJECT; + +static SPINLOCK gSurfaceStatsLock = SPINLOCK_INIT; +static SYS_SURFACE_STATS gSurfaceStats; + +// Registers a surface in the global surface telemetry snapshot. +static void KiSurfaceStatsRegister(SURFACE_OBJECT* surface) { + if (!surface || surface->StatsRegistered) + return; + + uintptr_t flags = KeAcquireSpinlock(&gSurfaceStatsLock); + gSurfaceStats.CreatedSurfaces++; + gSurfaceStats.LiveSurfaces++; + gSurfaceStats.LivePages += surface->PageCount; + gSurfaceStats.LiveBytes += surface->PayloadSize; + surface->StatsRegistered = 1; + KeReleaseSpinlock(&gSurfaceStatsLock, flags); +} + +// Removes a surface from the global surface telemetry snapshot. +static void KiSurfaceStatsUnregister(SURFACE_OBJECT* surface) { + if (!surface || !surface->StatsRegistered) + return; + + uintptr_t flags = KeAcquireSpinlock(&gSurfaceStatsLock); + gSurfaceStats.DestroyedSurfaces++; + if (gSurfaceStats.LiveSurfaces) gSurfaceStats.LiveSurfaces--; + if (gSurfaceStats.LivePages >= surface->PageCount) gSurfaceStats.LivePages -= surface->PageCount; + else gSurfaceStats.LivePages = 0; + if (gSurfaceStats.LiveBytes >= surface->PayloadSize) gSurfaceStats.LiveBytes -= surface->PayloadSize; + else gSurfaceStats.LiveBytes = 0; + surface->StatsRegistered = 0; + KeReleaseSpinlock(&gSurfaceStatsLock, flags); +} + +// Records a successful surface map operation. +static void KiSurfaceStatsRecordMap(SURFACE_OBJECT* surface) { + if (!surface) + return; + + uintptr_t flags = KeAcquireSpinlock(&gSurfaceStatsLock); + surface->MapCount++; + gSurfaceStats.MapCalls++; + KeReleaseSpinlock(&gSurfaceStatsLock, flags); +} + +// Records a successful surface present operation. +static void KiSurfaceStatsRecordPresent(SURFACE_OBJECT* surface, uint64_t inputRects, uint64_t clippedRects, + uint64_t emptyClips, uint64_t rowsCopied, uint64_t bytesCopied) { + if (!surface) + return; + + uintptr_t flags = KeAcquireSpinlock(&gSurfaceStatsLock); + surface->PresentCount++; + surface->PresentRectCount += inputRects; + surface->PresentClippedRectCount += clippedRects; + surface->PresentEmptyClipCount += emptyClips; + surface->PresentRowsCopied += rowsCopied; + surface->PresentBytesCopied += bytesCopied; + gSurfaceStats.PresentCalls++; + gSurfaceStats.PresentRectInputs += inputRects; + gSurfaceStats.PresentClippedRects += clippedRects; + gSurfaceStats.PresentEmptyClips += emptyClips; + gSurfaceStats.PresentRowsCopied += rowsCopied; + gSurfaceStats.PresentBytesCopied += bytesCopied; + KeReleaseSpinlock(&gSurfaceStatsLock, flags); +} + +// Copies the global surface telemetry snapshot. +static void KiSurfaceStatsQuery(SYS_SURFACE_STATS* out) { + if (!out) + return; + + uintptr_t flags = KeAcquireSpinlock(&gSurfaceStatsLock); + *out = gSurfaceStats; + KeReleaseSpinlock(&gSurfaceStatsLock, flags); +} + +static void KiSurfaceDestroy(OBJECT* obj) { + SURFACE_OBJECT* surface = CONTAINING_RECORD(obj, SURFACE_OBJECT, Header); + + KiSurfaceStatsUnregister(surface); + + if (surface->Owner && surface->UserBase) { + (void)MmVadDestroy(surface->Owner, surface->UserBase); + surface->UserBase = (void*)0; + } + + if (surface->Lock) { + ObDereferenceObject(&surface->Lock->Header); + surface->Lock = (MUTEX*)0; + } + + if (surface->Frames) { + for (size_t i = 0; i < surface->PageCount; ++i) { + if (surface->Frames[i]) + (void)MmPfnRelease(surface->Frames[i]); + } + MmFreePool(surface->Frames); + surface->Frames = (PHYSICAL_ADDRESS*)0; + } + + if (surface->RowScratch) { + MmFreePool(surface->RowScratch); + surface->RowScratch = (uint32_t*)0; + } +} + +typedef struct SURFACE_ROW_CONTEXT { + SURFACE_OBJECT* Surface; + uint32_t SourceX; + uint32_t SourceY; +} SURFACE_ROW_CONTEXT; + +static KSTATUS KiSurfaceCopyRow(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount) { + SURFACE_ROW_CONTEXT* ctx = (SURFACE_ROW_CONTEXT*)context; + if (!ctx || !ctx->Surface || !scratch || rowIndex < 0) + return STATUS_INVALID_PARAMETER; + + SURFACE_OBJECT* surface = ctx->Surface; + uint64_t row = (uint64_t)ctx->SourceY + (uint64_t)(uint32_t)rowIndex; + uint64_t offset = row * (uint64_t)surface->PitchBytes + (uint64_t)ctx->SourceX * sizeof(uint32_t); + uint64_t bytes = (uint64_t)pixelCount * sizeof(uint32_t); + if (offset > (uint64_t)surface->PayloadSize || bytes > (uint64_t)surface->PayloadSize - offset) + return STATUS_INVALID_PARAMETER; + + uint8_t* dst = (uint8_t*)scratch; + while (bytes) { + size_t frameIndex = (size_t)(offset / PAGE_SIZE); + size_t pageOffset = (size_t)(offset & (PAGE_SIZE - 1)); + if (frameIndex >= surface->PageCount || !surface->Frames[frameIndex]) + return STATUS_INVALID_PARAMETER; + + size_t chunk = PAGE_SIZE - pageOffset; + if (chunk > bytes) + chunk = (size_t)bytes; + + const uint8_t* src = (const uint8_t*)MmPfnToVirtual(surface->Frames[frameIndex]); + if (!src) + return STATUS_INVALID_PARAMETER; + memcpy(dst, src + pageOffset, chunk); + + dst += chunk; + offset += chunk; + bytes -= chunk; + } + + return STATUS_SUCCESS; +} + // Implements the syscall Init operation. void KeSyscallInit(const HAL_API* hal) { gHal = hal; @@ -141,9 +337,10 @@ static KSTATUS KiAllocHandleForCurrent(OBJECT* obj, uint32_t* handleOut) { } -#define SYSCALL_STRING_MAX 1024 +#define SYSCALL_STRING_MAX SYS_PATH_MAX #define SYSCALL_TEXT_MAX_BYTES (64ULL * 1024ULL) #define SYSCALL_IO_MAX_BYTES (16ULL * 1024ULL * 1024ULL) +#define SYSCALL_FILE_IO_CHUNK_BYTES (64ULL * 1024ULL) #define SYSCALL_BLIT_MAX_BYTES (16ULL * 1024ULL * 1024ULL) // Copies user Request. @@ -225,6 +422,13 @@ static int KiMulSize(size_t a, size_t b, size_t* out) { return 1; } +static size_t KiAlignUpSize(size_t value, size_t alignment) { + if (!alignment) + return value; + size_t mask = alignment - 1; + return (value + mask) & ~mask; +} + // Individual syscall handlers. // SYS_EXIT_PROCESS (0) a0 = exitCode. @@ -247,30 +451,263 @@ static uint64_t SyscallExitThread(uint64_t a0, uint64_t a1, uint64_t a2, while (true); } -// SYS_CREATE_THREAD (2) a0=entry a1=arg a2=priority a3=*tidOut. +// SYS_CREATE_THREAD a0=*SYS_CREATE_THREAD_REQUEST. +// Creates a runnable user thread and optionally returns both its waitable +// handle and diagnostic TID. The handle is the synchronization identity. static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a4; (void)a5; + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_CREATE_THREAD_REQUEST req; + KSTATUS s = KiCopyUserRequest(a0, &req, sizeof(req)); + if (!KSUCCESS(s)) return s; + PROCESS* proc = KeGetCurrentProcess(); if (!proc) return STATUS_FAILURE; - if (!a0) return STATUS_INVALID_PARAMETER; - if (a2 >= PriorityCount) return STATUS_INVALID_PARAMETER; + if (!req.Entry) return STATUS_INVALID_PARAMETER; + if (req.Priority >= PriorityCount) return STATUS_INVALID_PARAMETER; + if (req.Flags) return STATUS_INVALID_PARAMETER; + if (!req.ThreadHandleOut && !req.ThreadIdOut) return STATUS_INVALID_PARAMETER; - KSTATUS s = MmProbeUserRead((const void*)(uintptr_t)a0, 1); + s = MmProbeUserRead((const void*)req.Entry, 1); if (!KSUCCESS(s)) return s; - if (a3) { - s = MmProbeUserWrite((void*)(uintptr_t)a3, sizeof(uint32_t)); + if (req.ThreadHandleOut) { + s = MmProbeUserWrite(req.ThreadHandleOut, sizeof(*req.ThreadHandleOut)); + if (!KSUCCESS(s)) return s; + } + if (req.ThreadIdOut) { + s = MmProbeUserWrite(req.ThreadIdOut, sizeof(*req.ThreadIdOut)); if (!KSUCCESS(s)) return s; } THREAD* newt = NULL; - s = KeCreateThread(proc, (uintptr_t)a0, (uintptr_t)a1, - (THREAD_PRIORITY)a2, &newt); + s = KeCreateThread(proc, req.Entry, req.Argument, + (THREAD_PRIORITY)req.Priority, &newt); if (!KSUCCESS(s)) return s; - if (a3) return KiCopyU32ToUser(a3, newt->ThreadId); + + uint32_t threadHandle = 0; + if (req.ThreadHandleOut) { + s = KeAllocateHandleWithRights(proc, &newt->Header, + HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY, + &threadHandle); + if (!KSUCCESS(s)) { + ObDereferenceObject(&newt->Header); + return s; + } + } + + if (req.ThreadHandleOut) { + s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadHandleOut, threadHandle); + if (!KSUCCESS(s)) { + KeCloseHandle(proc, threadHandle); + ObDereferenceObject(&newt->Header); + return s; + } + } + + if (req.ThreadIdOut) { + s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadIdOut, newt->ThreadId); + if (!KSUCCESS(s)) { + if (threadHandle) KeCloseHandle(proc, threadHandle); + ObDereferenceObject(&newt->Header); + return s; + } + } + + ObDereferenceObject(&newt->Header); return STATUS_SUCCESS; } +static void KiFinishAlertableUserWait(uint8_t alertable) { + if (!alertable) return; + + THREAD* thread = KeGetCurrentThread(); + if (thread && thread->UserApcPending) + thread->UserApcDeliverable = 1; +} + +// Maps a waitable handle to its passive event backing object. +static KSTATUS KiHandleToWaitEvent(PROCESS* proc, uint32_t handle, EVENT** eventOut, OBJECT** objectOut) { + if (!proc || !eventOut || !objectOut) return STATUS_INVALID_PARAMETER; + *eventOut = (EVENT*)0; + *objectOut = (OBJECT*)0; + + OBJECT* obj = KeLookupHandleWithRights(proc, handle, HANDLE_RIGHT_WAIT); + if (!obj) return STATUS_INVALID_HANDLE; + + EVENT* event = (EVENT*)0; + switch (obj->Type) { + case ObjectTypeProcess: + event = CONTAINING_RECORD(obj, PROCESS, Header)->ExitEvent; + break; + case ObjectTypeThread: + event = CONTAINING_RECORD(obj, THREAD, Header)->ExitEvent; + break; + case ObjectTypeEvent: + event = CONTAINING_RECORD(obj, EVENT, Header); + break; + case ObjectTypeAsyncIo: + event = IoGetAsyncFileRequestWaitEvent((IO_ASYNC_FILE_REQUEST_OBJECT*)obj); + break; + case ObjectTypeInputQueue: + event = KeInputQueueWaitEvent(CONTAINING_RECORD(obj, INPUT_QUEUE, Header)); + break; + default: + ObDereferenceObject(obj); + return STATUS_NOT_SUPPORTED; + } + + if (!event) { + ObDereferenceObject(obj); + return STATUS_OBJECT_DEAD; + } + + *eventOut = event; + *objectOut = obj; + return STATUS_SUCCESS; +} + +// SYS_WAIT_SINGLE a0=handle a1=SYS_WAIT_FLAG_* a2=timeoutMs. +// Waits on one handle. Passive objects use the wait-many engine; mutexes and +// semaphores keep their native consuming semantics for single-object waits. +static uint64_t SyscallWaitSingle(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a3; (void)a4; (void)a5; + + uint32_t flags = (uint32_t)a1; + uint64_t timeoutMs = a2; + if (flags & ~SYS_WAIT_FLAG_ALERTABLE) return STATUS_INVALID_PARAMETER; + + PROCESS* proc = KeGetCurrentProcess(); + if (!proc) return STATUS_FAILURE; + + OBJECT* obj = KeLookupHandleWithRights(proc, (uint32_t)a0, HANDLE_RIGHT_WAIT); + if (!obj) return STATUS_INVALID_HANDLE; + + uint8_t alertable = (flags & SYS_WAIT_FLAG_ALERTABLE) ? 1 : 0; + KSTATUS s = STATUS_NOT_SUPPORTED; + + if (obj->Type == ObjectTypeMutex) { + if (alertable) s = STATUS_NOT_SUPPORTED; + else if (timeoutMs == 0) { + s = KeTryAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header)); + if (s == STATUS_FAILURE) s = STATUS_TIMEOUT; + } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) { + s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header)); + } else { + s = STATUS_NOT_SUPPORTED; + } + ObDereferenceObject(obj); + return s; + } + + if (obj->Type == ObjectTypeSemaphore) { + if (alertable) s = STATUS_NOT_SUPPORTED; + else if (timeoutMs == 0) { + s = KeTrySemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header)); + if (s == STATUS_FAILURE) s = STATUS_TIMEOUT; + } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) { + s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header)); + } else { + s = STATUS_NOT_SUPPORTED; + } + ObDereferenceObject(obj); + return s; + } + + ObDereferenceObject(obj); + + EVENT* event = (EVENT*)0; + OBJECT* ref = (OBJECT*)0; + s = KiHandleToWaitEvent(proc, (uint32_t)a0, &event, &ref); + if (!KSUCCESS(s)) return s; + + uint32_t selected = 0; + s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected); + ObDereferenceObject(ref); + if (s != STATUS_TIMEOUT && s != STATUS_NOT_SUPPORTED) + KiFinishAlertableUserWait(alertable); + return s; +} + +// SYS_WAIT_MANY a0=*SYS_WAIT_MANY_REQUEST. +// Waits until any passive waitable handle becomes signaled. +static uint64_t SyscallWaitMany(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_WAIT_MANY_REQUEST req; + KSTATUS s = KiCopyUserRequest(a0, &req, sizeof(req)); + if (!KSUCCESS(s)) return s; + if (req.Size != sizeof(req)) return STATUS_INVALID_PARAMETER; + if (req.Flags & ~SYS_WAIT_FLAG_ALERTABLE) return STATUS_INVALID_PARAMETER; + if (!req.Handles || req.Count == 0 || req.Count > SYS_WAIT_MANY_MAX_HANDLES) + return STATUS_INVALID_PARAMETER; + + uint32_t handles[SYS_WAIT_MANY_MAX_HANDLES]; + size_t handleBytes = (size_t)req.Count * sizeof(handles[0]); + s = CopyFromUser(handles, req.Handles, handleBytes); + if (!KSUCCESS(s)) return s; + + PROCESS* proc = KeGetCurrentProcess(); + if (!proc) return STATUS_FAILURE; + + EVENT* events[SYS_WAIT_MANY_MAX_HANDLES]; + OBJECT* refs[SYS_WAIT_MANY_MAX_HANDLES]; + for (uint32_t i = 0; i < SYS_WAIT_MANY_MAX_HANDLES; i++) { + events[i] = (EVENT*)0; + refs[i] = (OBJECT*)0; + } + + for (uint32_t i = 0; i < req.Count; i++) { + s = KiHandleToWaitEvent(proc, handles[i], &events[i], &refs[i]); + if (!KSUCCESS(s)) { + for (uint32_t j = 0; j < i; j++) + ObDereferenceObject(refs[j]); + return s; + } + } + + uint8_t alertable = (req.Flags & SYS_WAIT_FLAG_ALERTABLE) ? 1 : 0; + uint32_t selected = 0; + s = KeWaitForMultipleEvents(events, req.Count, alertable, req.TimeoutMilliseconds, &selected); + + for (uint32_t i = 0; i < req.Count; i++) + ObDereferenceObject(refs[i]); + + if (KSUCCESS(s) && req.SignaledIndexOut) { + KSTATUS cs = KiCopyU32ToUser((uint64_t)(uintptr_t)req.SignaledIndexOut, selected); + if (!KSUCCESS(cs)) return cs; + } + + if (s != STATUS_TIMEOUT && s != STATUS_NOT_SUPPORTED) + KiFinishAlertableUserWait(alertable); + return s; +} + +// SYS_OPEN_INPUT_QUEUE a0=flags a1=*handleOut. +static uint64_t SyscallOpenInputQueue(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a2; (void)a3; (void)a4; (void)a5; + if (a0 != 0) return STATUS_INVALID_PARAMETER; + if (!a1) return STATUS_INVALID_PARAMETER; + + INPUT_QUEUE* queue = (INPUT_QUEUE*)0; + KSTATUS s = KeInputOpenQueue(&queue); + if (!KSUCCESS(s)) return s; + + uint32_t handle = 0; + s = KiAllocHandleForCurrentWithRights(&queue->Header, HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY, &handle); + if (!KSUCCESS(s)) return s; + + s = KiCopyHandleToUser(a1, handle); + if (!KSUCCESS(s)) { + PROCESS* proc = KeGetCurrentProcess(); + if (proc) KeCloseHandle(proc, handle); + } + return s; +} + // SYS_GET_CURRENT_TID (3). static uint64_t SyscallGetCurrentTid(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -299,11 +736,7 @@ static uint64_t SyscallYield(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallSleep(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (!gHal) return STATUS_FAILURE; - uint64_t deadline = gHal->Timer.QueryTicks() + ((uint64_t)a0 * 1000u); - while (gHal->Timer.QueryTicks() < deadline) - KeYield(); - return STATUS_SUCCESS; + return KeDelayThread(a0); } // SYS_SET_USER_APC_DISPATCHER a0 = dispatcher entry in syscall.dll/user image. @@ -391,24 +824,16 @@ static uint64_t SyscallVmAlloc(uint64_t a0, uint64_t a1, uint64_t a2, if (!process) return STATUS_FAILURE; void* base = (void*)0; - s = MmCreateUserRegion(process, pageCount * PAGE_SIZE, PAGE_SIZE, - pageFlags ? pageFlags : (PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX), - USER_REGION_ZERO, - UserRegionGeneric, + s = MmVadCreateCommitted(process, pageCount * PAGE_SIZE, PAGE_SIZE, + pageFlags, + MM_VAD_ZERO, + MmVadGeneric, &base); if (!KSUCCESS(s)) return s; - if (protect == SYS_VM_NOACCESS) { - s = MmProtectUserRegion(process, base, pageCount * PAGE_SIZE, 0); - if (!KSUCCESS(s)) { - MmDestroyUserRegion(process, base); - return s; - } - } - s = CopyToUser(baseOut, &base, sizeof(base)); if (!KSUCCESS(s)) { - MmDestroyUserRegion(process, base); + MmVadDestroy(process, base); return s; } return STATUS_SUCCESS; @@ -419,7 +844,7 @@ static uint64_t SyscallVmFree(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; PROCESS* process = KeGetCurrentProcess(); if (!process || !a0) return STATUS_INVALID_PARAMETER; - return MmDestroyUserRegion(process, (void*)(uintptr_t)a0); + return MmVadDestroy(process, (void*)(uintptr_t)a0); } static uint64_t SyscallVmProtect(uint64_t a0, uint64_t a1, uint64_t a2, @@ -439,16 +864,16 @@ static uint64_t SyscallVmProtect(uint64_t a0, uint64_t a1, uint64_t a2, void* regionBase = (void*)0; size_t regionSize = 0; uint64_t oldFlags = 0; - USER_REGION_TYPE type = UserRegionGeneric; + MM_VAD_TYPE type = MmVadGeneric; uint32_t regionFlags = 0; - s = MmQueryUserRegion(process, (void*)(uintptr_t)a0, ®ionBase, ®ionSize, &oldFlags, &type, ®ionFlags); + s = MmVadQuery(process, (void*)(uintptr_t)a0, ®ionBase, ®ionSize, &oldFlags, &type, ®ionFlags); if (!KSUCCESS(s)) return s; uint32_t oldProtect = KiPageFlagsToVmProtect(oldFlags); s = CopyToUser((void*)(uintptr_t)a3, &oldProtect, sizeof(oldProtect)); if (!KSUCCESS(s)) return s; } - return MmProtectUserRegion(process, (void*)(uintptr_t)a0, (size_t)a1 * PAGE_SIZE, pageFlags); + return MmVadProtect(process, (void*)(uintptr_t)a0, (size_t)a1 * PAGE_SIZE, pageFlags); } static uint64_t SyscallVmQuery(uint64_t a0, uint64_t a1, uint64_t a2, @@ -462,9 +887,9 @@ static uint64_t SyscallVmQuery(uint64_t a0, uint64_t a1, uint64_t a2, void* base = (void*)0; size_t size = 0; uint64_t pageFlags = 0; - USER_REGION_TYPE type = UserRegionGeneric; + MM_VAD_TYPE type = MmVadGeneric; uint32_t regionFlags = 0; - KSTATUS s = MmQueryUserRegion(process, (void*)(uintptr_t)a0, &base, &size, &pageFlags, &type, ®ionFlags); + KSTATUS s = MmVadQuery(process, (void*)(uintptr_t)a0, &base, &size, &pageFlags, &type, ®ionFlags); if (!KSUCCESS(s)) return s; SYS_VM_REGION_INFO info; @@ -496,17 +921,6 @@ static uint64_t SyscallMutexCreate(uint64_t a0, uint64_t a1, uint64_t a2, return s; } -// SYS_ACQUIRE_MUTEX (12) a0 = handle. -static uint64_t SyscallMutexAcquire(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex, HANDLE_RIGHT_WAIT); - if (!mx) return STATUS_NOT_FOUND; - KSTATUS s = KeAcquireMutex(mx); - ObDereferenceObject(&mx->Header); - return s; -} - // SYS_TRY_ACQUIRE_MUTEX (13) a0 = handle. static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -548,17 +962,6 @@ static uint64_t SyscallSemCreate(uint64_t a0, uint64_t a1, uint64_t a2, return s; } -// SYS_WAIT_SEM (16) a0 = handle. -static uint64_t SyscallSemWait(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore, HANDLE_RIGHT_WAIT); - if (!sem) return STATUS_NOT_FOUND; - KSTATUS s = KeWaitSemaphore(sem); - ObDereferenceObject(&sem->Header); - return s; -} - // SYS_TRY_WAIT_SEM (17) a0 = handle. static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -622,23 +1025,6 @@ static uint64_t SyscallEventClear(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_SUCCESS; } -// SYS_WAIT_EVENT a0 = handle, a1 = alertable. -static uint64_t SyscallEventWait(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a2; (void)a3; (void)a4; (void)a5; - EVENT* ev = (EVENT*)KiHandleToObject((uint32_t)a0, ObjectTypeEvent, HANDLE_RIGHT_WAIT); - if (!ev) return STATUS_NOT_FOUND; - uint8_t alertable = a1 ? 1 : 0; - KSTATUS s = KeWaitForEvent(ev, alertable); - if (alertable) { - THREAD* thread = KeGetCurrentThread(); - if (thread && thread->UserApcPending) - thread->UserApcDeliverable = 1; - } - ObDereferenceObject(&ev->Header); - return s; -} - // SYS_CREATE_QUEUE (23) a0 = *handleOut. static uint64_t SyscallQueueCreate(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -698,11 +1084,193 @@ static uint64_t SyscallQueueRecv(uint64_t a0, uint64_t a1, uint64_t a2, // on 'obj'. If that was the last reference, ObDereferenceObject runs the // type-specific Destroy callback (KiFileDestroy, etc.) and frees the pool memory. // No per-type close syscall needed or wanted. + +static uint64_t SyscallCreateFileMapping(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a5; + uint32_t fileHandle = (uint32_t)a0; + uint32_t protect = (uint32_t)a1; + uint32_t flags = (uint32_t)a2; + uint64_t maximumSize = a3; + uint32_t* handleOut = (uint32_t*)(uintptr_t)a4; + + if (!fileHandle || !handleOut) + return STATUS_INVALID_PARAMETER; + + uint64_t pageFlags = 0; + KSTATUS s = KiVmProtectToPageFlags(protect, &pageFlags); + if (!KSUCCESS(s)) return s; + if (!pageFlags) return STATUS_ACCESS_DENIED; + + uint32_t neededRights = HANDLE_RIGHT_READ | HANDLE_RIGHT_QUERY; + if (pageFlags & PAGE_WRITE) + neededRights |= HANDLE_RIGHT_WRITE; + + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject(fileHandle, ObjectTypeFile, neededRights); + if (!file) return STATUS_NOT_FOUND; + + if (maximumSize == 0) + maximumSize = IoGetFileSize(file); + + FILE_MAPPING_OBJECT* mapping = (FILE_MAPPING_OBJECT*)ObAllocateObject(ObjectTypeFileMapping, + sizeof(FILE_MAPPING_OBJECT)); + if (!mapping) { + ObDereferenceObject(&file->Header); + return STATUS_NO_MEMORY; + } + + mapping->Header.Destroy = KiFileMappingDestroy; + mapping->File = file; // KiHandleToObject returned the mapping object's file reference. + mapping->MaximumSize = maximumSize; + mapping->PageFlags = pageFlags; + mapping->Protect = protect; + mapping->Flags = flags; + + uint32_t h = 0; + s = KiAllocHandleForCurrentWithRights(&mapping->Header, + HANDLE_RIGHT_READ | HANDLE_RIGHT_QUERY, + &h); + if (!KSUCCESS(s)) + return s; + + s = KiCopyHandleToUser((uint64_t)(uintptr_t)handleOut, h); + if (!KSUCCESS(s)) { + PROCESS* proc = KeGetCurrentProcess(); + if (proc) KeCloseHandle(proc, h); + } + return s; +} + +static KSTATUS KiMapAccessToPageFlags(const FILE_MAPPING_OBJECT* mapping, + uint32_t access, + uint64_t* pageFlagsOut) { + if (!mapping || !pageFlagsOut) + return STATUS_INVALID_PARAMETER; + + uint64_t mappingFlags = mapping->PageFlags; + if (!mappingFlags) + return STATUS_ACCESS_DENIED; + + if (access == 0) + access = SYS_MAP_ACCESS_READ; + + if ((access & SYS_MAP_ACCESS_WRITE) && (access & SYS_MAP_ACCESS_COPY)) + return STATUS_INVALID_PARAMETER; + + if (((access & SYS_MAP_ACCESS_COPY) || (access & SYS_MAP_ACCESS_WRITE)) && + (mappingFlags & PAGE_WRITE) == 0) + return STATUS_ACCESS_DENIED; + + uint64_t flags = PAGE_PRESENT | PAGE_USER; + if ((access & SYS_MAP_ACCESS_COPY) || (access & SYS_MAP_ACCESS_WRITE)) + flags |= PAGE_WRITE; + + if (access & SYS_MAP_ACCESS_EXECUTE) { + if (mappingFlags & PAGE_NX) + return STATUS_ACCESS_DENIED; + } else { + flags |= PAGE_NX; + } + + *pageFlagsOut = flags; + return STATUS_SUCCESS; +} + +static uint64_t SyscallMapFile(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + uint32_t mappingHandle = (uint32_t)a0; + uint64_t offset = a1; + size_t size = (size_t)a2; + uint32_t access = (uint32_t)a3; + uint32_t flags = (uint32_t)a4; + void** baseOut = (void**)(uintptr_t)a5; + + if (!mappingHandle || !baseOut) + return STATUS_INVALID_PARAMETER; + if (offset & (PAGE_SIZE - 1)) + return STATUS_INVALID_PARAMETER; + + FILE_MAPPING_OBJECT* mapping = (FILE_MAPPING_OBJECT*)KiHandleToObject(mappingHandle, + ObjectTypeFileMapping, + HANDLE_RIGHT_READ); + if (!mapping) + return STATUS_NOT_FOUND; + + uint64_t maximumSize = mapping->MaximumSize; + if (maximumSize == 0) + maximumSize = IoGetFileSize(mapping->File); + + if (offset >= maximumSize) { + ObDereferenceObject(&mapping->Header); + return STATUS_INVALID_PARAMETER; + } + if (size == 0) { + uint64_t tail = maximumSize - offset; + if (tail > (uint64_t)((size_t)-1)) { + ObDereferenceObject(&mapping->Header); + return STATUS_NO_MEMORY; + } + size = (size_t)tail; + } + if (size == 0) { + ObDereferenceObject(&mapping->Header); + return STATUS_INVALID_PARAMETER; + } + if ((uint64_t)size > maximumSize - offset) { + ObDereferenceObject(&mapping->Header); + return STATUS_INVALID_PARAMETER; + } + + uint64_t pageFlags = 0; + KSTATUS s = KiMapAccessToPageFlags(mapping, access, &pageFlags); + if (!KSUCCESS(s)) { + ObDereferenceObject(&mapping->Header); + return s; + } + + PROCESS* process = KeGetCurrentProcess(); + if (!process) { + ObDereferenceObject(&mapping->Header); + return STATUS_FAILURE; + } + + uint32_t vadFlags = flags; + if (access & SYS_MAP_ACCESS_WRITE) + vadFlags |= MM_VAD_SHARED_WRITE; + + void* base = (void*)0; + s = MmVadCreateFileMapping(process, mapping->File, offset, size, + pageFlags, vadFlags, &base); + ObDereferenceObject(&mapping->Header); + if (!KSUCCESS(s)) + return s; + + s = CopyToUser(baseOut, &base, sizeof(base)); + if (!KSUCCESS(s)) { + MmVadDestroy(process, base); + return s; + } + return STATUS_SUCCESS; +} + static uint64_t SyscallCloseHandle(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; PROCESS* proc = KeGetCurrentProcess(); if (!proc) return STATUS_FAILURE; + + OBJECT* obj = KeLookupHandleWithRights(proc, (uint32_t)a0, 0); + if (obj) { + if (obj->Type == ObjectTypeFile) { + KSTATUS fs = IoFlushFile((FILE_OBJECT*)obj); + ObDereferenceObject(obj); + if (!KSUCCESS(fs)) + return fs; + } else { + ObDereferenceObject(obj); + } + } + KeCloseHandle(proc, (uint32_t)a0); return STATUS_SUCCESS; } @@ -739,70 +1307,165 @@ static uint64_t SyscallOpenFile(uint64_t a0, uint64_t a1, uint64_t a2, return s; } -// SYS_READ_FILE (28) a0=handle a1=buf a2=size a3=*bytesReadOut. +static void* KiAllocateSyscallIoBuffer(size_t requested, size_t* actualOut) { + if (actualOut) *actualOut = 0; + if (!requested) return NULL; + + size_t chunk = requested < SYSCALL_FILE_IO_CHUNK_BYTES ? requested : SYSCALL_FILE_IO_CHUNK_BYTES; + for (;;) { + void* buf = MmAllocatePoolTag(chunk, MM_POOL_TAG('s', 'i', 'o', ' ')); + if (buf) { + if (actualOut) *actualOut = chunk; + return buf; + } + if (chunk <= PAGE_SIZE) + break; + chunk >>= 1; + if (chunk < PAGE_SIZE) + chunk = PAGE_SIZE; + } + return NULL; +} + +// Imports and validates a synchronous file I/O request. +static KSTATUS KiImportSyncFileIoRequest(uint64_t arg, SYS_FILE_IO_REQUEST* req) { + KSTATUS s = KiCopyUserRequest(arg, req, sizeof(*req)); + if (!KSUCCESS(s)) + return s; + + if (req->Flags != 0) + return STATUS_INVALID_PARAMETER; + if (req->Size > SYSCALL_IO_MAX_BYTES) + return STATUS_INVALID_PARAMETER; + if (req->Size && !req->Buffer) + return STATUS_INVALID_PARAMETER; + + return STATUS_SUCCESS; +} + +// SYS_READ_FILE a0=*SYS_FILE_IO_REQUEST. static uint64_t SyscallReadFile(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a4; (void)a5; - size_t size = (size_t)a2; - if (size && !a1) return STATUS_INVALID_PARAMETER; - if (size > SYSCALL_IO_MAX_BYTES) return STATUS_INVALID_PARAMETER; + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (size == 0) { - if (a3) return KiCopySizeToUser(a3, 0); + SYS_FILE_IO_REQUEST req; + KSTATUS s = KiImportSyncFileIoRequest(a0, &req); + if (!KSUCCESS(s)) + return s; + + if (req.Size == 0) { + if (req.TransferredOut) + return KiCopySizeToUser((uint64_t)(uintptr_t)req.TransferredOut, 0); return STATUS_SUCCESS; } - KSTATUS s = MmProbeUserWrite((void*)(uintptr_t)a1, size); - if (!KSUCCESS(s)) return s; + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject(req.FileHandle, ObjectTypeFile, HANDLE_RIGHT_READ); + if (!file) return STATUS_NOT_FOUND; - void* kbuf = MmAllocatePool(size); - if (!kbuf) return STATUS_NO_MEMORY; + size_t chunkSize = 0; + void* kbuf = KiAllocateSyscallIoBuffer(req.Size, &chunkSize); + if (!kbuf) { + ObDereferenceObject(&file->Header); + return STATUS_NO_MEMORY; + } - FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile, HANDLE_RIGHT_READ); - if (!file) { MmFreePool(kbuf); return STATUS_NOT_FOUND; } + uintptr_t user = (uintptr_t)req.Buffer; + size_t total = 0; + s = STATUS_SUCCESS; - size_t got = 0; - s = IoReadFile(file, kbuf, size, &got); + while (total < req.Size) { + size_t todo = req.Size - total; + if (todo > chunkSize) todo = chunkSize; + + if (user > (uintptr_t)-1 - total) { + s = STATUS_INVALID_PARAMETER; + break; + } + + void* userChunk = (void*)(user + total); + s = MmProbeUserWrite(userChunk, todo); + if (!KSUCCESS(s)) break; + + size_t got = 0; + s = IoReadFile(file, kbuf, todo, &got); + if (!KSUCCESS(s)) break; + + if (got) { + s = CopyToUser(userChunk, kbuf, got); + if (!KSUCCESS(s)) break; + total += got; + } + + if (got < todo) + break; + } + + MmFreePool(kbuf); ObDereferenceObject(&file->Header); - if (KSUCCESS(s)) - s = CopyToUser((void*)(uintptr_t)a1, kbuf, got); - MmFreePool(kbuf); - - if (KSUCCESS(s) && a3) - s = KiCopySizeToUser(a3, got); + if (req.TransferredOut) { + KSTATUS cs = KiCopySizeToUser((uint64_t)(uintptr_t)req.TransferredOut, total); + if (KSUCCESS(s)) s = cs; + } return s; } -// SYS_WRITE_FILE (29) a0=handle a1=buf a2=size a3=*bytesWrittenOut. +// SYS_WRITE_FILE a0=*SYS_FILE_IO_REQUEST. static uint64_t SyscallWriteFile(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a4; (void)a5; - size_t size = (size_t)a2; - if (size && !a1) return STATUS_INVALID_PARAMETER; - if (size > SYSCALL_IO_MAX_BYTES) return STATUS_INVALID_PARAMETER; + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (size == 0) { - if (a3) return KiCopySizeToUser(a3, 0); + SYS_FILE_IO_REQUEST req; + KSTATUS s = KiImportSyncFileIoRequest(a0, &req); + if (!KSUCCESS(s)) + return s; + + if (req.Size == 0) { + if (req.TransferredOut) + return KiCopySizeToUser((uint64_t)(uintptr_t)req.TransferredOut, 0); return STATUS_SUCCESS; } - void* kbuf = MmAllocatePool(size); - if (!kbuf) return STATUS_NO_MEMORY; + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject(req.FileHandle, ObjectTypeFile, HANDLE_RIGHT_WRITE); + if (!file) return STATUS_NOT_FOUND; - KSTATUS s = CopyFromUser(kbuf, (const void*)(uintptr_t)a1, size); - if (!KSUCCESS(s)) { MmFreePool(kbuf); return s; } + size_t chunkSize = 0; + void* kbuf = KiAllocateSyscallIoBuffer(req.Size, &chunkSize); + if (!kbuf) { + ObDereferenceObject(&file->Header); + return STATUS_NO_MEMORY; + } - FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile, HANDLE_RIGHT_WRITE); - if (!file) { MmFreePool(kbuf); return STATUS_NOT_FOUND; } + uintptr_t user = (uintptr_t)req.Buffer; + size_t total = 0; + s = STATUS_SUCCESS; + + while (total < req.Size) { + size_t todo = req.Size - total; + if (todo > chunkSize) todo = chunkSize; + + if (user > (uintptr_t)-1 - total) { + s = STATUS_INVALID_PARAMETER; + break; + } + + s = CopyFromUser(kbuf, (const void*)(user + total), todo); + if (!KSUCCESS(s)) break; + + size_t written = 0; + s = IoWriteFile(file, kbuf, todo, &written); + if (!KSUCCESS(s)) break; + + total += written; + if (written < todo) + break; + } - size_t written = 0; - s = IoWriteFile(file, kbuf, size, &written); - ObDereferenceObject(&file->Header); MmFreePool(kbuf); + ObDereferenceObject(&file->Header); - if (a3) { - KSTATUS cs = KiCopySizeToUser(a3, written); + if (req.TransferredOut) { + KSTATUS cs = KiCopySizeToUser((uint64_t)(uintptr_t)req.TransferredOut, total); if (KSUCCESS(s)) s = cs; } return s; @@ -879,14 +1542,8 @@ static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) { if (!req.Buffer && req.Size) return STATUS_INVALID_PARAMETER; if (req.Size > SYSCALL_IO_MAX_BYTES) return STATUS_INVALID_PARAMETER; - s = MmProbeUserWrite(req.Iosb, sizeof(*req.Iosb)); - if (!KSUCCESS(s)) return s; - - if (req.Size) { - s = write ? MmProbeUserRead(req.Buffer, req.Size) - : MmProbeUserWrite(req.Buffer, req.Size); - if (!KSUCCESS(s)) return s; - } + PROCESS* proc = KeGetCurrentProcess(); + if (!proc) return STATUS_FAILURE; if (req.RequestHandleOut) { s = MmProbeUserWrite(req.RequestHandleOut, sizeof(*req.RequestHandleOut)); @@ -916,7 +1573,7 @@ static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) { return STATUS_INVALID_PARAMETER; } - apcCtx = (SYS_USER_ASYNC_APC_CONTEXT*)MmAllocatePool(sizeof(SYS_USER_ASYNC_APC_CONTEXT)); + apcCtx = (SYS_USER_ASYNC_APC_CONTEXT*)MmAllocatePoolTag(sizeof(SYS_USER_ASYNC_APC_CONTEXT), MM_POOL_TAG('u', 'a', 'p', 'c')); if (!apcCtx) { if (event) ObDereferenceObject(&event->Header); ObDereferenceObject(&file->Header); @@ -937,32 +1594,53 @@ static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) { completionApc = &apcCtx->KernelApc; } + MM_LOCKED_USER_BUFFER* lockedBuffer = (MM_LOCKED_USER_BUFFER*)0; + MM_LOCKED_USER_BUFFER* lockedIosb = (MM_LOCKED_USER_BUFFER*)0; + + s = MmLockUserBuffer(proc, req.Iosb, sizeof(*req.Iosb), 1, &lockedIosb); + if (!KSUCCESS(s)) { + if (apcCtx) + SyscallUserAsyncApcContextFree(apcCtx); + if (event) ObDereferenceObject(&event->Header); + ObDereferenceObject(&file->Header); + return s; + } + + if (req.Size) { + s = MmLockUserBuffer(proc, req.Buffer, req.Size, write ? 0 : 1, &lockedBuffer); + if (!KSUCCESS(s)) { + MmUnlockUserBuffer(lockedIosb); + if (apcCtx) + SyscallUserAsyncApcContextFree(apcCtx); + if (event) ObDereferenceObject(&event->Header); + ObDereferenceObject(&file->Header); + return s; + } + } + IO_ASYNC_FILE_REQUEST_OBJECT* asyncReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0; if (write) { s = IoWriteFileAsyncKernel(file, req.Buffer, req.Size, req.Offset, (IO_STATUS_BLOCK*)req.Iosb, event, completionApc, + lockedBuffer, + lockedIosb, &asyncReq); } else { s = IoReadFileAsyncKernel(file, req.Buffer, req.Size, req.Offset, (IO_STATUS_BLOCK*)req.Iosb, event, completionApc, + lockedBuffer, + lockedIosb, &asyncReq); } + lockedBuffer = (MM_LOCKED_USER_BUFFER*)0; + lockedIosb = (MM_LOCKED_USER_BUFFER*)0; if (s == STATUS_PENDING && req.RequestHandleOut && asyncReq) { - PROCESS* proc = KeGetCurrentProcess(); - if (!proc) { - (void)IoCancelAsyncFileRequest(asyncReq); - ObDereferenceObject(IoAsyncFileRequestObject(asyncReq)); - if (event) ObDereferenceObject(&event->Header); - ObDereferenceObject(&file->Header); - return STATUS_FAILURE; - } - uint32_t requestHandle = 0; KSTATUS hs = KeAllocateHandleWithRights(proc, IoAsyncFileRequestObject(asyncReq), - HANDLE_RIGHT_QUERY | HANDLE_RIGHT_TERMINATE, + HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY | HANDLE_RIGHT_TERMINATE, &requestHandle); if (!KSUCCESS(hs)) { (void)IoCancelAsyncFileRequest(asyncReq); @@ -986,7 +1664,7 @@ static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) { KSTATUS cs = CopyToUser(req.RequestHandleOut, &zeroHandle, sizeof(zeroHandle)); if (!KSUCCESS(cs)) { if (asyncReq) ObDereferenceObject(IoAsyncFileRequestObject(asyncReq)); - if (!KSUCCESS(s) && s != STATUS_PENDING && apcCtx) + if (s != STATUS_PENDING && apcCtx) SyscallUserAsyncApcContextFree(apcCtx); if (event) ObDereferenceObject(&event->Header); ObDereferenceObject(&file->Header); @@ -997,7 +1675,7 @@ static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) { if (asyncReq) ObDereferenceObject(IoAsyncFileRequestObject(asyncReq)); // drop creator reference - if (!KSUCCESS(s) && s != STATUS_PENDING && apcCtx) + if (s != STATUS_PENDING && apcCtx) SyscallUserAsyncApcContextFree(apcCtx); if (event) ObDereferenceObject(&event->Header); @@ -1093,6 +1771,17 @@ static uint64_t SyscallTruncateFile(uint64_t a0, uint64_t a1, uint64_t a2, return s; } +// SYS_FLUSH_FILE a0=handle. +static uint64_t SyscallFlushFile(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile, HANDLE_RIGHT_WRITE); + if (!file) return STATUS_NOT_FOUND; + KSTATUS s = IoFlushFile(file); + ObDereferenceObject(&file->Header); + return s; +} + // SYS_GET_FILE_POS (33) a0=handle -> current seek offset. static uint64_t SyscallGetFilePos(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -1128,31 +1817,280 @@ static uint64_t SyscallGetFileSize(uint64_t a0, uint64_t a1, uint64_t a2, return sz; } -// SYS_BLIT_SCREEN (66) a0=x a1=y a2=w a3=h a4=*pixels. -static uint64_t SyscallBlitScreen(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { +// SYS_CREATE_SURFACE (99) a0=width a1=height a2=format a3=flags a4=*handleOut. +static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { (void)a5; - if (!a4) return STATUS_FAILURE; + uint32_t width = (uint32_t)a0; + uint32_t height = (uint32_t)a1; + uint32_t format = (uint32_t)a2; + uint32_t flags = (uint32_t)a3; + if (!width || !height || !a4) + return STATUS_INVALID_PARAMETER; + if (format != SYS_SURFACE_FORMAT_XRGB8888 || flags != SYS_SURFACE_FLAG_NONE) + return STATUS_NOT_SUPPORTED; - int w = (int)a2; - int h = (int)a3; - if (w <= 0 || h <= 0) return STATUS_INVALID_PARAMETER; - - size_t pixels = 0; - size_t bytes = 0; - if (!KiMulSize((size_t)w, (size_t)h, &pixels) || - !KiMulSize(pixels, sizeof(uint32_t), &bytes) || - bytes > SYSCALL_BLIT_MAX_BYTES) + DISPLAY_MODE mode; + KSTATUS s = KeDisplayGetMode(&mode); + if (!KSUCCESS(s)) + return s; + if (width > mode.Width || height > mode.Height) return STATUS_INVALID_PARAMETER; - uint32_t* kpixels = (uint32_t*)MmAllocatePool(bytes); - if (!kpixels) return STATUS_NO_MEMORY; + size_t rowBytes = 0; + if (!KiMulSize((size_t)width, sizeof(uint32_t), &rowBytes)) + return STATUS_INVALID_PARAMETER; + size_t pitchBytes = KiAlignUpSize(rowBytes, 64); + if (pitchBytes < rowBytes) + return STATUS_INVALID_PARAMETER; + + size_t payloadSize = 0; + if (!KiMulSize(pitchBytes, (size_t)height, &payloadSize)) + return STATUS_INVALID_PARAMETER; + size_t pageCount = (payloadSize + PAGE_SIZE - 1) / PAGE_SIZE; + if (!pageCount || pageCount > (SIZE_MAX / sizeof(PHYSICAL_ADDRESS))) + return STATUS_INVALID_PARAMETER; + + SURFACE_OBJECT* surface = (SURFACE_OBJECT*)ObAllocateObject(ObjectTypeSurface, sizeof(SURFACE_OBJECT)); + if (!surface) + return STATUS_NO_MEMORY; + surface->Header.Destroy = KiSurfaceDestroy; + surface->Owner = KeGetCurrentProcess(); + surface->Width = width; + surface->Height = height; + surface->PitchBytes = (uint32_t)pitchBytes; + surface->Format = format; + surface->Flags = flags; + surface->PayloadSize = payloadSize; + surface->PageCount = pageCount; + + surface->Frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS), MM_POOL_TAG('s', 'f', 'r', 'm')); + surface->RowScratch = (uint32_t*)MmAllocatePoolTag(rowBytes, MM_POOL_TAG('s', 'r', 'o', 'w')); + s = KeMutexCreate(&surface->Lock); + if (!surface->Frames || !surface->RowScratch || !KSUCCESS(s)) { + ObDereferenceObject(&surface->Header); + return STATUS_NO_MEMORY; + } + memset(surface->Frames, 0, pageCount * sizeof(PHYSICAL_ADDRESS)); + + for (size_t i = 0; i < pageCount; ++i) { + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseSurface); + if (!frame) { + ObDereferenceObject(&surface->Header); + return STATUS_NO_MEMORY; + } + surface->Frames[i] = frame; + void* kva = MmPfnToVirtual(frame); + if (kva) + memset(kva, 0, PAGE_SIZE); + } + + uint32_t handle = 0; + s = KiAllocHandleForCurrentWithRights(&surface->Header, HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE | HANDLE_RIGHT_QUERY, &handle); + if (!KSUCCESS(s)) + return s; + + KiSurfaceStatsRegister(surface); + + s = CopyToUser((void*)(uintptr_t)a4, &handle, sizeof(handle)); + if (!KSUCCESS(s)) { + KeCloseHandle(KeGetCurrentProcess(), handle); + return s; + } + return STATUS_SUCCESS; +} + +// SYS_MAP_SURFACE (100) a0=surfaceHandle a1=*baseOut a2=*pitchBytesOut. +static uint64_t SyscallMapSurface(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a3; (void)a4; (void)a5; + if (!a1 || !a2) + return STATUS_INVALID_PARAMETER; + + SURFACE_OBJECT* surface = (SURFACE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeSurface, HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE); + if (!surface) + return STATUS_INVALID_PARAMETER; + + PROCESS* current = KeGetCurrentProcess(); + if (surface->Owner != current) { + ObDereferenceObject(&surface->Header); + return STATUS_ACCESS_DENIED; + } + + KSTATUS s = KeAcquireMutex(surface->Lock); + if (!KSUCCESS(s)) { + ObDereferenceObject(&surface->Header); + return s; + } + + if (!surface->UserBase) { + s = MmVadCreateSharedViewFromFrames(current, surface->Frames, surface->PageCount, + surface->PayloadSize, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, MmVadShared, &surface->UserBase); + } + + void* userBase = surface->UserBase; + uint32_t pitchBytes = surface->PitchBytes; + KeReleaseMutex(surface->Lock); - KSTATUS s = CopyFromUser(kpixels, (const void*)(uintptr_t)a4, bytes); if (KSUCCESS(s)) - s = KeDisplayBlit((int)a0, (int)a1, w, h, kpixels); + s = CopyToUser((void*)(uintptr_t)a1, &userBase, sizeof(userBase)); + if (KSUCCESS(s)) + s = CopyToUser((void*)(uintptr_t)a2, &pitchBytes, sizeof(pitchBytes)); + if (KSUCCESS(s)) + KiSurfaceStatsRecordMap(surface); - MmFreePool(kpixels); + ObDereferenceObject(&surface->Header); + return s; +} + +// SYS_QUERY_SURFACE (101) a0=surfaceHandle a1=*SYS_SURFACE_DESC. +static uint64_t SyscallQuerySurface(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a2; (void)a3; (void)a4; (void)a5; + if (!a1) + return STATUS_INVALID_PARAMETER; + + SURFACE_OBJECT* surface = (SURFACE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeSurface, HANDLE_RIGHT_QUERY); + if (!surface) + return STATUS_INVALID_PARAMETER; + + if (surface->Owner != KeGetCurrentProcess()) { + ObDereferenceObject(&surface->Header); + return STATUS_ACCESS_DENIED; + } + + SYS_SURFACE_DESC desc; + memset(&desc, 0, sizeof(desc)); + + KSTATUS s = KeAcquireMutex(surface->Lock); + if (KSUCCESS(s)) { + desc.Width = surface->Width; + desc.Height = surface->Height; + desc.PitchBytes = surface->PitchBytes; + desc.Format = surface->Format; + desc.Base = surface->UserBase; + desc.PayloadBytes = surface->PayloadSize; + desc.PageCount = surface->PageCount; + KeReleaseMutex(surface->Lock); + + uintptr_t statsFlags = KeAcquireSpinlock(&gSurfaceStatsLock); + desc.MapCount = surface->MapCount; + desc.PresentCount = surface->PresentCount; + desc.PresentRectCount = surface->PresentRectCount; + desc.PresentClippedRectCount = surface->PresentClippedRectCount; + desc.PresentEmptyClipCount = surface->PresentEmptyClipCount; + desc.PresentRowsCopied = surface->PresentRowsCopied; + desc.PresentBytesCopied = surface->PresentBytesCopied; + KeReleaseSpinlock(&gSurfaceStatsLock, statsFlags); + + s = CopyToUser((void*)(uintptr_t)a1, &desc, sizeof(desc)); + } + + ObDereferenceObject(&surface->Header); + return s; +} + +static int KiClipSurfaceRect(const SURFACE_OBJECT* surface, const DISPLAY_RECT* in, DISPLAY_RECT* out) { + if (!surface || !out) + return 0; + DISPLAY_RECT r; + if (in) { + r = *in; + } else { + r.X = 0; + r.Y = 0; + r.W = (int32_t)surface->Width; + r.H = (int32_t)surface->Height; + } + if (r.W <= 0 || r.H <= 0) + return 0; + + int64_t x0 = r.X; + int64_t y0 = r.Y; + int64_t x1 = x0 + r.W; + int64_t y1 = y0 + r.H; + if (x0 < 0) x0 = 0; + if (y0 < 0) y0 = 0; + if (x1 > surface->Width) x1 = surface->Width; + if (y1 > surface->Height) y1 = surface->Height; + if (x0 >= x1 || y0 >= y1) + return 0; + + out->X = (int32_t)x0; + out->Y = (int32_t)y0; + out->W = (int32_t)(x1 - x0); + out->H = (int32_t)(y1 - y0); + return 1; +} + +// SYS_PRESENT_SURFACE (102) a0=surfaceHandle a1=*DISPLAY_RECT[] a2=rectCount. +static uint64_t SyscallPresentSurface(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a3; (void)a4; (void)a5; + uint32_t rectCount = (uint32_t)a2; + if (rectCount > SYS_SURFACE_MAX_RECTS) + return STATUS_INVALID_PARAMETER; + + if (rectCount != 0 && !a1) + return STATUS_INVALID_PARAMETER; + + SURFACE_OBJECT* surface = (SURFACE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeSurface, HANDLE_RIGHT_READ); + if (!surface) + return STATUS_INVALID_PARAMETER; + if (surface->Owner != KeGetCurrentProcess()) { + ObDereferenceObject(&surface->Header); + return STATUS_ACCESS_DENIED; + } + + DISPLAY_RECT rects[SYS_SURFACE_MAX_RECTS]; + if (rectCount == 0) { + rects[0].X = 0; + rects[0].Y = 0; + rects[0].W = (int32_t)surface->Width; + rects[0].H = (int32_t)surface->Height; + rectCount = 1; + } else { + KSTATUS cs = CopyFromUser(rects, (const void*)(uintptr_t)a1, (size_t)rectCount * sizeof(DISPLAY_RECT)); + if (!KSUCCESS(cs)) { + ObDereferenceObject(&surface->Header); + return cs; + } + } + + uint64_t inputRects = (uint64_t)a2; + uint64_t clippedRects = 0; + uint64_t emptyClips = 0; + uint64_t rowsCopied = 0; + uint64_t bytesCopied = 0; + + KSTATUS s = STATUS_SUCCESS; + for (uint32_t i = 0; i < rectCount; ++i) { + DISPLAY_RECT clipped; + if (!KiClipSurfaceRect(surface, &rects[i], &clipped)) { + emptyClips++; + continue; + } + + SURFACE_ROW_CONTEXT ctx; + ctx.Surface = surface; + ctx.SourceX = (uint32_t)clipped.X; + ctx.SourceY = (uint32_t)clipped.Y; + s = KeDisplayBlitRows(clipped.X, clipped.Y, clipped.W, clipped.H, + KiSurfaceCopyRow, &ctx, surface->RowScratch); + if (!KSUCCESS(s)) + break; + + clippedRects++; + rowsCopied += (uint64_t)(uint32_t)clipped.H; + bytesCopied += (uint64_t)(uint32_t)clipped.W * (uint64_t)(uint32_t)clipped.H * sizeof(uint32_t); + } + + if (KSUCCESS(s)) + KiSurfaceStatsRecordPresent(surface, inputRects, clippedRects, emptyClips, rowsCopied, bytesCopied); + + ObDereferenceObject(&surface->Header); return s; } @@ -1267,20 +2205,36 @@ static uint64_t SyscallKbIsKeyDown(uint64_t a0, uint64_t a1, uint64_t a2, return KeInputIsKeyDown((INPUT_KEY)a0) ? 1u : 0u; } -// SYS_READ_KEYBOARD_CHAR (72) -> char (blocks, yields while waiting). +// SYS_READ_KEYBOARD_CHAR (72) -> char. static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - while (!KeInputKbPoll()) KeYield(); - return (uint64_t)(uint8_t)KeInputReadChar(); + + for (;;) { + char ch = KeInputReadChar(); + if (ch) + return (uint64_t)(uint8_t)ch; + + KSTATUS s = KeInputWaitKey(0); + if (!KSUCCESS(s)) + return 0; + } } -// SYS_READ_KEY (73) -> INPUT_KEY (blocks, yields while waiting). +// SYS_READ_KEY (73) -> INPUT_KEY. static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - while (!KeInputKbPoll()) KeYield(); - return (uint64_t)KeInputReadKey(); + + for (;;) { + INPUT_KEY key = KeInputReadKey(); + if (key != INPUT_KEY_NONE) + return (uint64_t)key; + + KSTATUS s = KeInputWaitKey(0); + if (!KSUCCESS(s)) + return (uint64_t)INPUT_KEY_NONE; + } } // SYS_GET_KEY_MODIFIERS (74) -> uint32_t INPUT_MOD_* flags. @@ -1290,6 +2244,36 @@ static uint64_t SyscallKbGetModifiers(uint64_t a0, uint64_t a1, uint64_t a2, return (uint64_t)KeInputGetModifiers(); } +// SYS_READ_INPUT_EVENT (104) a0=*INPUT_KEY_RECORD -> KSTATUS. +static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + if (!a0) return STATUS_INVALID_PARAMETER; + + INPUT_KEY_RECORD record; + for (;;) { + if (KeInputReadEvent(&record)) + return CopyToUser((void*)(uintptr_t)a0, &record, sizeof(record)); + + KSTATUS s = KeInputWaitKey(0); + if (!KSUCCESS(s)) + return s; + } +} + +// SYS_PEEK_INPUT_EVENT (105) a0=*INPUT_KEY_RECORD -> KSTATUS. +static uint64_t SyscallKbPeekInputEvent(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + if (!a0) return STATUS_INVALID_PARAMETER; + + INPUT_KEY_RECORD record; + if (!KeInputPeekEvent(&record)) + return STATUS_TIMEOUT; + + return CopyToUser((void*)(uintptr_t)a0, &record, sizeof(record)); +} + // SYS_POLL_MOUSE (75) -> 1 or 0. static uint64_t SyscallMousePoll(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -1423,18 +2407,11 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t size = IoGetFileSize(f); ProcDbg("CreateProcess: fileSize=%llu\n", size); - void* mem = MmAllocatePool(size); - if (!mem) { IoCloseFile(f); ProcDbg("CreateProcess: raw image pool alloc failed size=%llu\n", size); return STATUS_NO_MEMORY; } - - size_t got; - s = IoReadFile(f, mem, size, &got); - IoCloseFile(f); - if (!KSUCCESS(s)) { ProcDbg("CreateProcess: IoReadFile failed st=%d got=%llu\n", s, (unsigned long long)got); MmFreePool(mem); return s; } // Create the process object. PROCESS* proc; s = KeCreateProcess(pathBuf, &proc); - if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateProcess failed st=%d\n", s); MmFreePool(mem); return s; } + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateProcess failed st=%d\n", s); IoCloseFile(f); return s; } ProcDbg("CreateProcess: created pid=%u name=%s\n", proc->ProcessId, proc->Name); // Inherit CWD from parent. @@ -1454,19 +2431,19 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, path, cmdlineView, SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength)); - if (!KSUCCESS(s)) { ProcDbg("CreateProcess: env setup failed st=%d\n", s); MmFreePool(mem); ObDereferenceObject(&proc->Header); return s; } + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: env setup failed st=%d\n", s); IoCloseFile(f); ObDereferenceObject(&proc->Header); return s; } // Map the PE image into memory. uintptr_t imageBase = 0; - s = LdrLoadImageForProcess(proc, mem, size, &imageBase); - MmFreePool(mem); - if (!KSUCCESS(s)) { ProcDbg("CreateProcess: LdrLoadImage failed st=%d\n", s); ObDereferenceObject(&proc->Header); return s; } + s = LdrLoadImageFileForProcess(proc, f, &imageBase); + IoCloseFile(f); + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: LdrLoadImageFileForProcess failed st=%d\n", s); ObDereferenceObject(&proc->Header); return s; } ProcDbg("CreateProcess: image mapped pid=%u base=%016llx size=%llx\n", proc->ProcessId, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(proc, imageBase)); s = LdrResolveImportsForProcess(proc, imageBase); if (!KSUCCESS(s)) { ProcDbg("CreateProcess: resolve imports failed st=%d pid=%u image=%016llx\n", s, proc->ProcessId, (unsigned long long)imageBase); - MmDebugDumpUserRegions(proc); + MmVadDebugDumpProcess(proc); ObDereferenceObject(&proc->Header); return s; } @@ -1475,7 +2452,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, s = LdrProtectImageForProcess(proc, imageBase, true); if (!KSUCCESS(s)) { ProcDbg("CreateProcess: protect image failed st=%d pid=%u\n", s, proc->ProcessId); - MmDebugDumpUserRegions(proc); + MmVadDebugDumpProcess(proc); ObDereferenceObject(&proc->Header); return s; } @@ -1503,16 +2480,18 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread); if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint); - MmDebugDumpUserRegions(proc); + MmVadDebugDumpProcess(proc); KeCloseHandle(caller, handle); ObDereferenceObject(&proc->Header); return s; } - ProcDbg("CreateProcess: thread created pid=%u tid=%llu entry=%016llx handle=%u\n", proc->ProcessId, (unsigned long long)thread->ThreadId, (unsigned long long)entryPoint, handle); + uint32_t tid = thread->ThreadId; + ProcDbg("CreateProcess: thread created pid=%u tid=%llu entry=%016llx handle=%u\n", proc->ProcessId, (unsigned long long)tid, (unsigned long long)entryPoint, handle); + ObDereferenceObject(&thread->Header); s = KiCopyHandleToUser((uint64_t)(uintptr_t)req.HandleOut, handle); if (!KSUCCESS(s)) { - KeTerminateProcess(proc, -1); + KeTerminateProcess(proc, (int)STATUS_PROCESS_PAGE_FAULT); KeCloseHandle(caller, handle); ObDereferenceObject(&proc->Header); return s; @@ -1523,20 +2502,6 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, ObDereferenceObject(&proc->Header); return STATUS_SUCCESS; } -// SYS_WAIT_FOR_PROCESS (51) a0=handle -// Blocks until the target process signals its ExitEvent. -static uint64_t SyscallWaitForProcess(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - - PROCESS* proc = (PROCESS*)KiHandleToObject((uint32_t)a0, ObjectTypeProcess, HANDLE_RIGHT_WAIT); - if (!proc) return STATUS_INVALID_HANDLE; - - KSTATUS s = KeWaitForEvent(proc->ExitEvent, 0); - ObDereferenceObject(&proc->Header); - return s; -} - // SYS_GET_PROCESS_EXIT_CODE (52) a0=handle a1=*exitCodeOut. static uint64_t SyscallGetProcessExitCode(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -1677,48 +2642,6 @@ static uint64_t SyscallGetCwd(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_SUCCESS; } -// SYS_WAIT_THREAD (58) a0=tid -// Blocks the caller until the thread with the given TID exits. -// Searches the current process's thread list; cross-process join is not -// supported in v1 (POSIX pthread_join is always intra-process). -// Returns STATUS_NOT_FOUND if the TID is unknown (already reaped or never -// existed) - treat the same as a successful join at the POSIX layer. -static uint64_t SyscallWaitThread(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - uint32_t targetTid = (uint32_t)a0; - - PROCESS* proc = KeGetCurrentProcess(); - if (!proc) return STATUS_FAILURE; - - // Walk the thread list under the spinlock to find the target, then bump - // its ExitEvent refcount so the event survives even if the thread exits - // and is destroyed while we wait. - EVENT* exitEvent = (EVENT*)0; - uintptr_t flags = KeAcquireSpinlock(&proc->HandleLock); - LIST_ENTRY* entry = proc->ThreadList.Fwd; - while (entry != &proc->ThreadList) { - THREAD* t = CONTAINING_RECORD(entry, THREAD, ProcessLink); - if (t->ThreadId == targetTid && t->ExitEvent) { - exitEvent = t->ExitEvent; - ObReferenceObject(&exitEvent->Header); // keep event alive - break; - } - entry = entry->Fwd; - } - KeReleaseSpinlock(&proc->HandleLock, flags); - - if (!exitEvent) { - // TID not found - either already exited and reaped, or invalid. - // Treat as success at the POSIX layer (join on an already-dead thread). - return STATUS_NOT_FOUND; - } - - KSTATUS s = KeWaitForEvent(exitEvent, 0); - ObDereferenceObject(&exitEvent->Header); - return s; -} - // SYS_GET_PROCESS_PID (59) a0=procHandle a1=*pidOut // Returns the numeric PID of a process identified by handle. // Used by posix_spawn to map the kernel handle back to a pid_t for waitpid. @@ -1968,7 +2891,7 @@ static uint64_t SyscallGetProcessList(uint64_t a0, uint64_t a1, uint64_t a2, size_t bytes = 0; if (!KiMulSize(toCopy, sizeof(SYS_PROCESS_INFO), &bytes)) return STATUS_INVALID_PARAMETER; - SYS_PROCESS_INFO* tmp = (SYS_PROCESS_INFO*)MmAllocatePool(bytes); + SYS_PROCESS_INFO* tmp = (SYS_PROCESS_INFO*)MmAllocatePoolTag(bytes, MM_POOL_TAG('s', 'y', 's', ' ')); if (!tmp) return STATUS_NO_MEMORY; size_t copied = KeCopyProcessInfo(tmp, toCopy); @@ -1983,6 +2906,45 @@ static uint64_t SyscallGetProcessList(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_SUCCESS; } +// SYS_GET_THREAD_LIST (103) a0=pid a1=*SYS_THREAD_INFO[] a2=maxCount a3=*countOut. +static uint64_t SyscallGetThreadList(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a4; (void)a5; + uint32_t pid = (uint32_t)a0; + size_t maxCount = (size_t)a2; + + if (maxCount && !a1) return STATUS_INVALID_PARAMETER; + if (a3) { + KSTATUS s = MmProbeUserWrite((void*)(uintptr_t)a3, sizeof(size_t)); + if (!KSUCCESS(s)) return s; + } + + size_t total = 0; + KSTATUS s = KeCopyProcessThreadInfo(pid, NULL, 0, &total); + if (!KSUCCESS(s)) return s; + + size_t toCopy = total < maxCount ? total : maxCount; + if (toCopy) { + size_t bytes = 0; + if (!KiMulSize(toCopy, sizeof(SYS_THREAD_INFO), &bytes)) return STATUS_INVALID_PARAMETER; + + SYS_THREAD_INFO* tmp = (SYS_THREAD_INFO*)MmAllocatePoolTag(bytes, MM_POOL_TAG('s', 't', 'h', 'r')); + if (!tmp) return STATUS_NO_MEMORY; + + size_t copied = 0; + s = KeCopyProcessThreadInfo(pid, tmp, toCopy, &copied); + if (KSUCCESS(s)) { + if (copied < toCopy) toCopy = copied; + s = CopyToUser((void*)(uintptr_t)a1, tmp, toCopy * sizeof(SYS_THREAD_INFO)); + } + MmFreePool(tmp); + if (!KSUCCESS(s)) return s; + } + + if (a3) return KiCopySizeToUser(a3, total); + return STATUS_SUCCESS; +} + // SYS_RENAME_FILE (66) a0=*oldPath a1=*newPath. static uint64_t SyscallRenameFile(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { @@ -2076,6 +3038,12 @@ typedef struct SYS_MEMORY_STATUS_K { uint64_t FreePhysicalBytes; uint64_t KernelPoolBytes; uint64_t UserHeapBytes; + uint64_t ReclaimablePhysicalBytes; + uint64_t AvailablePhysicalBytes; + uint64_t PinnedPhysicalBytes; + uint64_t NonEvictablePhysicalBytes; + uint64_t UserPhysicalBytes; + uint64_t PageTablePhysicalBytes; } SYS_MEMORY_STATUS_K; #define SYS_POLL_MAX_FDS 64u @@ -2224,7 +3192,7 @@ static uint64_t SyscallSend(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_SUCCESS; } - void* kbuf = MmAllocatePool(length); + void* kbuf = MmAllocatePoolTag(length, MM_POOL_TAG('s', 'y', 's', ' ')); if (!kbuf) return STATUS_NO_MEMORY; KSTATUS s = CopyFromUser(kbuf, (const void*)(uintptr_t)a1, length); @@ -2261,7 +3229,7 @@ static uint64_t SyscallRecv(uint64_t a0, uint64_t a1, uint64_t a2, KSTATUS s = MmProbeUserWrite((void*)(uintptr_t)a1, length); if (!KSUCCESS(s)) return s; - void* kbuf = MmAllocatePool(length); + void* kbuf = MmAllocatePoolTag(length, MM_POOL_TAG('s', 'y', 's', ' ')); if (!kbuf) return STATUS_NO_MEMORY; SOCKET_OBJECT* socket = (SOCKET_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeSocket, HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE); @@ -2313,7 +3281,7 @@ static uint64_t SyscallSendTo(uint64_t a0, uint64_t a1, uint64_t a2, void* kbuf = NULL; if (req.Length) { - kbuf = MmAllocatePool(req.Length); + kbuf = MmAllocatePoolTag(req.Length, MM_POOL_TAG('s', 'y', 's', ' ')); if (!kbuf) return STATUS_NO_MEMORY; s = CopyFromUser(kbuf, req.Buffer, req.Length); if (!KSUCCESS(s)) { MmFreePool(kbuf); return s; } @@ -2369,7 +3337,7 @@ static uint64_t SyscallRecvFrom(uint64_t a0, uint64_t a1, uint64_t a2, void* kbuf = NULL; if (req.Length) { - kbuf = MmAllocatePool(req.Length); + kbuf = MmAllocatePoolTag(req.Length, MM_POOL_TAG('s', 'y', 's', ' ')); if (!kbuf) return STATUS_NO_MEMORY; } @@ -2521,7 +3489,7 @@ static uint64_t SyscallPoll(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_INVALID_PARAMETER; size_t bytes = count * sizeof(SYS_POLLFD_K); - SYS_POLLFD_K* fds = (SYS_POLLFD_K*)MmAllocatePool(bytes); + SYS_POLLFD_K* fds = (SYS_POLLFD_K*)MmAllocatePoolTag(bytes, MM_POOL_TAG('s', 'y', 's', ' ')); if (!fds) return STATUS_NO_MEMORY; KSTATUS s = CopyFromUser(fds, (const void*)(uintptr_t)a0, bytes); @@ -2573,10 +3541,20 @@ static uint64_t SyscallGetMemoryStatus(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; if (!a0) return STATUS_INVALID_PARAMETER; + MM_PFN_STATS pfn; + MmPfnQueryStats(&pfn); + SYS_MEMORY_STATUS_K status; - status.FreePhysicalBytes = MmFreeFrameCount() * PAGE_SIZE; + memset(&status, 0, sizeof(status)); + status.FreePhysicalBytes = pfn.FreeFrames * PAGE_SIZE; status.KernelPoolBytes = MmGetKernelPoolUsage(); status.UserHeapBytes = MmGetUserHeapUsage(); + status.ReclaimablePhysicalBytes = pfn.ReclaimableFrames * PAGE_SIZE; + status.AvailablePhysicalBytes = pfn.AvailableFrames * PAGE_SIZE; + status.PinnedPhysicalBytes = pfn.PinnedFrames * PAGE_SIZE; + status.NonEvictablePhysicalBytes = pfn.NonEvictableFrames * PAGE_SIZE; + status.UserPhysicalBytes = pfn.UserFrames * PAGE_SIZE; + status.PageTablePhysicalBytes = pfn.PageTableFrames * PAGE_SIZE; return CopyToUser((void*)(uintptr_t)a0, &status, sizeof(status)); } @@ -2661,6 +3639,267 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); } + + case SYS_KERNEL_STATS_PFN: { + MM_PFN_STATS km; + SYS_PFN_STATS um; + MmPfnQueryStats(&km); + memset(&um, 0, sizeof(um)); + um.TotalFrames = km.TotalFrames; + um.FreeFrames = km.FreeFrames; + um.UsedFrames = km.UsedFrames; + um.ReservedFrames = km.ReservedFrames; + um.ZeroedFrames = km.ZeroedFrames; + um.ActiveFrames = km.ActiveFrames; + um.StandbyFrames = km.StandbyFrames; + um.ModifiedFrames = km.ModifiedFrames; + um.PinnedFrames = km.PinnedFrames; + um.EvictableFrames = km.EvictableFrames; + um.NonEvictableFrames = km.NonEvictableFrames; + um.ReclaimableFrames = km.ReclaimableFrames; + um.AvailableFrames = km.AvailableFrames; + um.MappedFrames = km.MappedFrames; + um.KernelFrames = km.KernelFrames; + um.PageTableFrames = km.PageTableFrames; + um.UserFrames = km.UserFrames; + um.CleanFileCacheFrames = km.CleanFileCacheFrames; + um.ActiveCleanFileCacheFrames = km.ActiveCleanFileCacheFrames; + um.StandbyCleanFileCacheFrames = km.StandbyCleanFileCacheFrames; + um.ImageFrames = km.ImageFrames; + um.ActiveImageFrames = km.ActiveImageFrames; + um.StandbyImageFrames = km.StandbyImageFrames; + um.MappedFileFrames = km.MappedFileFrames; + um.AnonymousFrames = km.AnonymousFrames; + um.CowFrames = km.CowFrames; + um.SurfaceFrames = km.SurfaceFrames; + um.DriverFrames = km.DriverFrames; + um.DmaFrames = km.DmaFrames; + um.BootloaderFrames = km.BootloaderFrames; + um.PfnDatabaseFrames = km.PfnDatabaseFrames; + um.MmioFrames = km.MmioFrames; + um.AllocationCount = km.AllocationCount; + um.FreeCount = km.FreeCount; + um.RetainCount = km.RetainCount; + um.ReleaseCount = km.ReleaseCount; + um.PinCount = km.PinCount; + um.UnpinCount = km.UnpinCount; + um.FailedAllocations = km.FailedAllocations; + um.FailedContiguousAllocations = km.FailedContiguousAllocations; + um.FailedInvalidRange = km.FailedInvalidRange; + um.FailedEmptyRange = km.FailedEmptyRange; + um.FailedReserveDenied = km.FailedReserveDenied; + um.FailedNoFrame = km.FailedNoFrame; + um.FailedContiguousInvalidRequest = km.FailedContiguousInvalidRequest; + um.FailedContiguousReserveDenied = km.FailedContiguousReserveDenied; + um.FailedContiguousNoRun = km.FailedContiguousNoRun; + um.ReclaimAttempts = km.ReclaimAttempts; + um.ReclaimPagesRequested = km.ReclaimPagesRequested; + um.ReclaimPagesSucceeded = km.ReclaimPagesSucceeded; + um.ReclaimPagesFailed = km.ReclaimPagesFailed; + um.ReclaimReentrantSkips = km.ReclaimReentrantSkips; + return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); + } + + case SYS_KERNEL_STATS_PTE: { + MM_PTE_STATS km; + SYS_PTE_STATS um; + MmPteQueryStats(&km); + memset(&um, 0, sizeof(um)); + um.MapCalls = km.MapCalls; + um.MapFailures = km.MapFailures; + um.UnmapCalls = km.UnmapCalls; + um.UnmapFailures = km.UnmapFailures; + um.ProtectCalls = km.ProtectCalls; + um.ProtectFailures = km.ProtectFailures; + um.QueryCalls = km.QueryCalls; + um.QueryFailures = km.QueryFailures; + for (size_t i = 0; i < 6 && i < MmPteQueryPurposeCount; i++) { + um.QueryByPurpose[i] = km.QueryByPurpose[i]; + um.QueryFailuresByPurpose[i] = km.QueryFailuresByPurpose[i]; + } + um.TranslateCalls = km.TranslateCalls; + um.LocalFlushes = km.LocalFlushes; + um.GlobalFlushes = km.GlobalFlushes; + um.ShootdownRequests = km.ShootdownRequests; + um.ShootdownTargetCpus = km.ShootdownTargetCpus; + um.ShootdownSkippedNoTargets = km.ShootdownSkippedNoTargets; + um.ShootdownAcks = km.ShootdownAcks; + um.ShootdownTimeouts = km.ShootdownTimeouts; + um.NewMappingNoFlushes = km.NewMappingNoFlushes; + um.ReplacedMappingFlushes = km.ReplacedMappingFlushes; + um.PageTablePagesAllocated = km.PageTablePagesAllocated; + um.CurrentLeafMappings = km.CurrentLeafMappings; + return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); + } + + case SYS_KERNEL_STATS_VAD: { + MM_VAD_STATS km; + SYS_VAD_STATS um; + MmVadQueryStats(&km); + memset(&um, 0, sizeof(um)); + um.ResolveFaultCalls = km.ResolveFaultCalls; + um.ResolvedFaults = km.ResolvedFaults; + um.FailedFaults = km.FailedFaults; + um.ProtectionFaults = km.ProtectionFaults; + um.NoVadFaults = km.NoVadFaults; + um.UncommittedFaults = km.UncommittedFaults; + um.NoAccessFaults = km.NoAccessFaults; + um.DemandZeroPages = km.DemandZeroPages; + um.ZeroPageMaps = km.ZeroPageMaps; + um.ZeroPageCowSplits = km.ZeroPageCowSplits; + um.CowFaults = km.CowFaults; + um.CowPagesCopied = km.CowPagesCopied; + um.CowViewsCreated = km.CowViewsCreated; + um.CowPagesShared = km.CowPagesShared; + um.EnsureResidentCalls = km.EnsureResidentCalls; + um.EnsureResidentPages = km.EnsureResidentPages; + um.AlreadyResidentPages = km.AlreadyResidentPages; + um.FileBackedFaults = km.FileBackedFaults; + um.FileBackedReads = km.FileBackedReads; + um.FileBackedReadFailures = km.FileBackedReadFailures; + um.FileBackedDiscards = km.FileBackedDiscards; + um.FileBackedSharedWritableFaults = km.FileBackedSharedWritableFaults; + um.FileBackedDirtyFaults = km.FileBackedDirtyFaults; + um.FileBackedDirtySyncs = km.FileBackedDirtySyncs; + um.ImageBackedFaults = km.ImageBackedFaults; + um.ImageBackedZeroFaults = km.ImageBackedZeroFaults; + um.ImageBackedPrivateFaults = km.ImageBackedPrivateFaults; + um.ImageFixupPrivateCalls = km.ImageFixupPrivateCalls; + um.ImageFixupPrivatePages = km.ImageFixupPrivatePages; + um.SwapPageOuts = km.SwapPageOuts; + um.SwapPageIns = km.SwapPageIns; + return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); + } + + + case SYS_KERNEL_STATS_FILE_CACHE: { + MM_FILE_CACHE_STATS km; + SYS_FILE_CACHE_STATS um; + MmFileCacheQueryStats(&km); + memset(&um, 0, sizeof(um)); + um.Lookups = km.Lookups; + um.Hits = km.Hits; + um.Misses = km.Misses; + um.Loads = km.Loads; + um.LoadFailures = km.LoadFailures; + um.RaceDiscards = km.RaceDiscards; + um.InflightWaits = km.InflightWaits; + um.Invalidations = km.Invalidations; + um.DirtyPages = km.DirtyPages; + um.MaxDirtyPages = km.MaxDirtyPages; + um.DirtyMarks = km.DirtyMarks; + um.WritebackRequests = km.WritebackRequests; + um.WritebackPages = km.WritebackPages; + um.WritebackBytes = km.WritebackBytes; + um.WritebackFailures = km.WritebackFailures; + um.FlushFileCalls = km.FlushFileCalls; + um.FlushFileFailures = km.FlushFileFailures; + um.MappedDirtySyncs = km.MappedDirtySyncs; + um.MappedDirtyPages = km.MappedDirtyPages; + um.Drops = km.Drops; + um.DropFailures = km.DropFailures; + um.StandbyTransitions = km.StandbyTransitions; + um.StandbyHits = km.StandbyHits; + um.ReadAheadRequests = km.ReadAheadRequests; + um.ReadAheadPagesRequested = km.ReadAheadPagesRequested; + um.ReadAheadPagesQueued = km.ReadAheadPagesQueued; + um.ReadAheadPagesLoaded = km.ReadAheadPagesLoaded; + um.ReadAheadPagesSkipped = km.ReadAheadPagesSkipped; + um.ReadAheadFailures = km.ReadAheadFailures; + um.ReadAheadPagesThrottled = km.ReadAheadPagesThrottled; + um.MaxReadAheadPagesPerRequest = km.MaxReadAheadPagesPerRequest; + um.BatchReads = km.BatchReads; + um.BatchPagesRequested = km.BatchPagesRequested; + um.BatchPagesLoaded = km.BatchPagesLoaded; + um.MaxBatchPagesRequested = km.MaxBatchPagesRequested; + um.MaxBatchPagesLoaded = km.MaxBatchPagesLoaded; + um.ResidentPages = km.ResidentPages; + um.ActivePages = km.ActivePages; + um.StandbyPages = km.StandbyPages; + um.MaxResidentPages = km.MaxResidentPages; + return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); + } + + case SYS_KERNEL_STATS_SURFACE: { + SYS_SURFACE_STATS stats; + memset(&stats, 0, sizeof(stats)); + KiSurfaceStatsQuery(&stats); + return KiReturnKernelStats(a1, (size_t)a2, a3, &stats, sizeof(stats)); + } + + case SYS_KERNEL_STATS_SWAP: { + MM_SWAP_STATS km; + SYS_SWAP_STATS um; + MmSwapQueryStats(&km); + memset(&um, 0, sizeof(um)); + um.Enabled = km.Enabled; + um.TotalSlots = km.TotalSlots; + um.UsedSlots = km.UsedSlots; + um.FreeSlots = km.FreeSlots; + um.HighWaterSlots = km.HighWaterSlots; + um.PageOuts = km.PageOuts; + um.PageIns = km.PageIns; + um.PageOutFailures = km.PageOutFailures; + um.PageInFailures = km.PageInFailures; + um.SlotAllocations = km.SlotAllocations; + um.SlotFrees = km.SlotFrees; + um.SlotAllocationFailures = km.SlotAllocationFailures; + return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); + } + + case SYS_KERNEL_STATS_FILESYSTEM: { + IO_FILESYSTEM_STATS km; + SYS_FILESYSTEM_STATS um; + IoQueryFilesystemStats(&km); + memset(&um, 0, sizeof(um)); + um.PathLookups = km.PathLookups; + um.PathCacheHits = km.PathCacheHits; + um.PathCacheMisses = km.PathCacheMisses; + um.PathCacheStores = km.PathCacheStores; + um.PathCacheInvalidations = km.PathCacheInvalidations; + um.PathCacheGeneration = km.PathCacheGeneration; + um.DirectorySearches = km.DirectorySearches; + um.DirectoryClustersRead = km.DirectoryClustersRead; + um.DirectoryEntriesScanned = km.DirectoryEntriesScanned; + um.FatChainCacheLookups = km.FatChainCacheLookups; + um.FatChainCacheHits = km.FatChainCacheHits; + um.FatChainCacheMisses = km.FatChainCacheMisses; + um.FatChainCacheStores = km.FatChainCacheStores; + um.FatChainCacheInvalidations = km.FatChainCacheInvalidations; + um.FatChainWalkSteps = km.FatChainWalkSteps; + um.DataReadRuns = km.DataReadRuns; + um.DataReadSectors = km.DataReadSectors; + um.DataReadRunMerges = km.DataReadRunMerges; + um.MaxDataReadSpanSectors = km.MaxDataReadSpanSectors; + return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); + } + + case SYS_KERNEL_STATS_DISK: { + IO_DISK_STATS km; + SYS_DISK_STATS um; + IoQueryDiskStats(&km); + memset(&um, 0, sizeof(um)); + um.RegisteredDisks = km.RegisteredDisks; + um.ReadRequests = km.ReadRequests; + um.ReadSectors = km.ReadSectors; + um.WriteRequests = km.WriteRequests; + um.WriteSectors = km.WriteSectors; + um.MaxReadSectors = km.MaxReadSectors; + um.MaxWriteSectors = km.MaxWriteSectors; + um.ReadaheadHits = km.ReadaheadHits; + um.ReadaheadMisses = km.ReadaheadMisses; + um.ReadaheadStores = km.ReadaheadStores; + um.ReadaheadInvalidations = km.ReadaheadInvalidations; + um.ReadaheadBytesServed = km.ReadaheadBytesServed; + um.ReadaheadSectorsLoaded = km.ReadaheadSectorsLoaded; + um.ReadaheadSequentialDetections = km.ReadaheadSequentialDetections; + um.ReadaheadSuppressed = km.ReadaheadSuppressed; + um.ReadaheadAllocFailures = km.ReadaheadAllocFailures; + um.ReadaheadReadFailures = km.ReadaheadReadFailures; + um.MaxReadaheadSectors = km.MaxReadaheadSectors; + return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); + } + case SYS_KERNEL_STATS_CPU: { SYS_CPU_STATS stats; memset(&stats, 0, sizeof(stats)); @@ -2681,6 +3920,7 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, if (cpu->IsBsp) out->Flags |= SYS_CPU_FLAG_BSP; if (cpu->IsPrimaryThread) out->Flags |= SYS_CPU_FLAG_PRIMARY; out->CurrentTid = cpu->CurrentThread ? cpu->CurrentThread->ThreadId : 0; + if (cpu->CurrentThread && cpu->CurrentThread->IsIdleThread) out->CurrentThreadFlags |= SYS_CPU_THREAD_FLAG_IDLE; out->SchedulerTicks = cpu->SchedulerTicks; out->ContextSwitches = cpu->ContextSwitches; } @@ -2701,13 +3941,13 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { SyscallExitProcess, // 00 SYS_EXIT_PROCESS. SyscallExitThread, // 01 SYS_EXIT_THREAD. SyscallCreateProcess, // 02 SYS_CREATE_PROCESS. - SyscallWaitForProcess, // 03 SYS_WAIT_FOR_PROCESS. + SyscallWaitSingle, // 03 SYS_WAIT_SINGLE. SyscallGetProcessExitCode, // 04 SYS_GET_PROCESS_EXIT_CODE. SyscallKillProcess, // 05 SYS_KILL_PROCESS. SyscallGetProcessList, // 06 SYS_GET_PROCESS_LIST. SyscallGetProcessPid, // 07 SYS_GET_PROCESS_PID. SyscallCreateThread, // 08 SYS_CREATE_THREAD. - SyscallWaitThread, // 09 SYS_WAIT_THREAD. + 0, // 09 retired SYS_WAIT_THREAD. SyscallGetCurrentTid, // 10 SYS_GET_CURRENT_TID. SyscallGetCurrentPid, // 11 SYS_GET_CURRENT_PID. SyscallSetThreadPriority, // 12 SYS_SET_THREAD_PRIORITY. @@ -2721,17 +3961,17 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { SyscallVmQuery, // 20 SYS_QUERY_VM. SyscallCloseHandle, // 21 SYS_CLOSE_HANDLE. SyscallMutexCreate, // 22 SYS_CREATE_MUTEX. - SyscallMutexAcquire, // 23 SYS_ACQUIRE_MUTEX. + 0, // 23 retired SYS_ACQUIRE_MUTEX. SyscallMutexTryAcquire, // 24 SYS_TRY_ACQUIRE_MUTEX. SyscallMutexRelease, // 25 SYS_RELEASE_MUTEX. SyscallSemCreate, // 26 SYS_CREATE_SEM. - SyscallSemWait, // 27 SYS_WAIT_SEM. + 0, // 27 retired SYS_WAIT_SEM. SyscallSemTryWait, // 28 SYS_TRY_WAIT_SEM. SyscallSemSignal, // 29 SYS_SIGNAL_SEM. SyscallEventCreate, // 30 SYS_CREATE_EVENT. SyscallEventSignal, // 31 SYS_SIGNAL_EVENT. SyscallEventClear, // 32 SYS_CLEAR_EVENT. - SyscallEventWait, // 33 SYS_WAIT_EVENT. + 0, // 33 retired SYS_WAIT_EVENT. SyscallQueueCreate, // 34 SYS_CREATE_QUEUE. SyscallQueueSend, // 35 SYS_SEND_QUEUE. SyscallQueueRecv, // 36 SYS_RECV_QUEUE. @@ -2764,38 +4004,49 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { SyscallFillConsoleCharacter, // 63 SYS_FILL_CONSOLE_CHARACTER. SyscallFillConsoleAttribute, // 64 SYS_FILL_CONSOLE_ATTRIBUTE. SyscallPutScreenChar, // 65 SYS_PUT_SCREEN_CHAR. - SyscallBlitScreen, // 66 SYS_BLIT_SCREEN. - SyscallClearScreen, // 67 SYS_CLEAR_SCREEN. - SyscallGetScreenSize, // 68 SYS_GET_SCREEN_SIZE. - SyscallQueryDisplayInfo, // 69 SYS_QUERY_DISPLAY_INFO. - SyscallKbPoll, // 70 SYS_POLL_KEYBOARD. - SyscallKbIsKeyDown, // 71 SYS_IS_KEY_DOWN. - SyscallKbReadChar, // 72 SYS_READ_KEYBOARD_CHAR. - SyscallKbReadKey, // 73 SYS_READ_KEY. - SyscallKbGetModifiers, // 74 SYS_GET_KEY_MODIFIERS. - SyscallMousePoll, // 75 SYS_POLL_MOUSE. - SyscallMouseRead, // 76 SYS_READ_MOUSE. - SyscallMouseGetPos, // 77 SYS_GET_MOUSE_POS. - SyscallGetTick, // 78 SYS_GET_TICK. - SyscallDebugPrint, // 79 SYS_PRINT_DEBUG. - SyscallQueryTrapFrame, // 80 SYS_QUERY_TRAP_FRAME. - SyscallSocket, // 81 SYS_CREATE_SOCKET. - SyscallBind, // 82 SYS_BIND_SOCKET. - SyscallListen, // 83 SYS_LISTEN_SOCKET. - SyscallAccept, // 84 SYS_ACCEPT_SOCKET. - SyscallConnect, // 85 SYS_CONNECT_SOCKET. - SyscallSend, // 86 SYS_SEND_SOCKET. - SyscallRecv, // 87 SYS_RECV_SOCKET. - SyscallShutdown, // 88 SYS_SHUTDOWN_SOCKET. - SyscallSendTo, // 89 SYS_SEND_TO_SOCKET. - SyscallRecvFrom, // 90 SYS_RECV_FROM_SOCKET. - SyscallResolveHost, // 91 SYS_RESOLVE_HOST. - SyscallSetSocketOption, // 92 SYS_SET_SOCKET_OPTION. - SyscallGetSocketOption, // 93 SYS_GET_SOCKET_OPTION. - SyscallPoll, // 94 SYS_POLL. - SyscallGetMemoryStatus, // 95 SYS_GET_MEMORY_STATUS. - SyscallQueryKernelStats, // 96 SYS_QUERY_KERNEL_STATS. - SyscallGetSystemTime // 97 SYS_GET_SYSTEM_TIME. + SyscallClearScreen, // 66 SYS_CLEAR_SCREEN. + SyscallGetScreenSize, // 67 SYS_GET_SCREEN_SIZE. + SyscallQueryDisplayInfo, // 68 SYS_QUERY_DISPLAY_INFO. + SyscallKbPoll, // 69 SYS_POLL_KEYBOARD. + SyscallKbIsKeyDown, // 70 SYS_IS_KEY_DOWN. + SyscallKbReadChar, // 71 SYS_READ_KEYBOARD_CHAR. + SyscallKbReadKey, // 72 SYS_READ_KEY. + SyscallKbGetModifiers, // 73 SYS_GET_KEY_MODIFIERS. + SyscallMousePoll, // 74 SYS_POLL_MOUSE. + SyscallMouseRead, // 75 SYS_READ_MOUSE. + SyscallMouseGetPos, // 76 SYS_GET_MOUSE_POS. + SyscallGetTick, // 77 SYS_GET_TICK. + SyscallDebugPrint, // 78 SYS_PRINT_DEBUG. + SyscallQueryTrapFrame, // 79 SYS_QUERY_TRAP_FRAME. + SyscallSocket, // 80 SYS_CREATE_SOCKET. + SyscallBind, // 81 SYS_BIND_SOCKET. + SyscallListen, // 82 SYS_LISTEN_SOCKET. + SyscallAccept, // 83 SYS_ACCEPT_SOCKET. + SyscallConnect, // 84 SYS_CONNECT_SOCKET. + SyscallSend, // 85 SYS_SEND_SOCKET. + SyscallRecv, // 86 SYS_RECV_SOCKET. + SyscallShutdown, // 87 SYS_SHUTDOWN_SOCKET. + SyscallSendTo, // 88 SYS_SEND_TO_SOCKET. + SyscallRecvFrom, // 89 SYS_RECV_FROM_SOCKET. + SyscallResolveHost, // 90 SYS_RESOLVE_HOST. + SyscallSetSocketOption, // 91 SYS_SET_SOCKET_OPTION. + SyscallGetSocketOption, // 92 SYS_GET_SOCKET_OPTION. + SyscallPoll, // 93 SYS_POLL. + SyscallGetMemoryStatus, // 94 SYS_GET_MEMORY_STATUS. + SyscallQueryKernelStats, // 95 SYS_QUERY_KERNEL_STATS. + SyscallGetSystemTime, // 96 SYS_GET_SYSTEM_TIME. + SyscallCreateFileMapping, // 97 SYS_CREATE_FILE_MAPPING. + SyscallMapFile, // 98 SYS_MAP_FILE. + SyscallCreateSurface, // 99 SYS_CREATE_SURFACE. + SyscallMapSurface, // 100 SYS_MAP_SURFACE. + SyscallQuerySurface, // 101 SYS_QUERY_SURFACE. + SyscallPresentSurface, // 102 SYS_PRESENT_SURFACE. + SyscallGetThreadList, // 103 SYS_GET_THREAD_LIST. + SyscallKbReadInputEvent, // 104 SYS_READ_INPUT_EVENT. + SyscallKbPeekInputEvent, // 105 SYS_PEEK_INPUT_EVENT. + SyscallWaitMany, // 106 SYS_WAIT_MANY. + SyscallOpenInputQueue, // 107 SYS_OPEN_INPUT_QUEUE. + SyscallFlushFile, // 108 SYS_FLUSH_FILE. }; // KeSyscallDispatch is entered from ke_syscall.asm after the assembly stub has reordered registers into the C calling convention. diff --git a/code/kernel/ke_system_root.c b/code/kernel/ke_system_root.c new file mode 100644 index 0000000..37abd85 --- /dev/null +++ b/code/kernel/ke_system_root.c @@ -0,0 +1,97 @@ +// System root volume selection. This file keeps boot-volume identity in one place so loaders do not guess. + +#include +#include +#include +#include +#include +#include +#include + +static uint8_t gSystemRootDrive; +static bool gSystemRootReady; + +// Builds a drive-qualified path without depending on any process CWD. +static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) { + if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES) + return false; + + size_t relLen = strlen(relative); + if (relLen + 4 > outSize) + return false; + + out[0] = (char)('0' + drive); + out[1] = ':'; + out[2] = '/'; + memcpy(out + 3, relative, relLen + 1); + return true; +} + +// Returns true when the candidate volume looks like the volume that booted the kernel. +static bool KiDriveLooksLikeSystemRoot(uint8_t drive) { + static const char* required[] = { + "system/colonel.exe", + "system/hal.dll", + "drivers/drivers.ini", + }; + + char path[1024]; + for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) { + if (!KiBuildDrivePath(drive, required[i], path, sizeof(path))) + return false; + + FILE_OBJECT* file = NULL; + KSTATUS s = IoOpenFile(SysSvFromCstr(path), &file); + if (!KSUCCESS(s)) + return false; + IoCloseFile(file); + } + + return true; +} + +// Selects the mounted volume that contains the kernel/system tree. +KSTATUS KeInitializeSystemRootVolume(void) { + for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) { + if (!IoFilesystemPresent(drive)) + continue; + if (!KiDriveLooksLikeSystemRoot(drive)) + continue; + + gSystemRootDrive = drive; + gSystemRootReady = true; + DbgPrint("System root: drive=%u\n", drive); + return STATUS_SUCCESS; + } + + return STATUS_NOT_FOUND; +} + +// Returns whether the system root volume has been selected. +bool KeSystemRootReady(void) { + return gSystemRootReady; +} + +// Returns the selected system root drive, or zero before discovery. +uint8_t KeGetSystemRootDrive(void) { + return gSystemRootReady ? gSystemRootDrive : 0; +} + +// Builds an absolute path rooted at the selected system volume. +bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize) { + if (!gSystemRootReady) + return false; + return KiBuildDrivePath(gSystemRootDrive, relative, out, outSize); +} + +// Opens a file rooted at the selected system volume. +KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out) { + if (!out) + return STATUS_INVALID_PARAMETER; + *out = NULL; + + char path[1024]; + if (!KeBuildSystemRootPath(relative, path, sizeof(path))) + return STATUS_NOT_FOUND; + return IoOpenFile(SysSvFromCstr(path), out); +} diff --git a/code/kernel/ke_thread.c b/code/kernel/ke_thread.c index f73200d..6e2b92d 100644 --- a/code/kernel/ke_thread.c +++ b/code/kernel/ke_thread.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -36,6 +38,11 @@ extern void KeThreadEntry(void); static void KiThreadDestroy(OBJECT* obj) { THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header); + // A wait that uses auxiliary stack wait blocks must be removed before the + // kernel stack is released. A terminated thread may never return to the + // wait syscall to run normal cleanup. + KeRundownThreadWait(thread, STATUS_ALERTED); + KiRundownApcState(thread); if (thread->ExitEvent) { @@ -43,16 +50,16 @@ static void KiThreadDestroy(OBJECT* obj) { thread->ExitEvent = (EVENT*)0; } - PROCESS* ownerForUserRegions = thread->Owner; - if (ownerForUserRegions) { + PROCESS* ownerForVadList = thread->Owner; + if (ownerForVadList) { if (thread->Teb && thread->TebSize) { - MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb); + MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb); thread->Teb = (void*)0; thread->TebSize = 0; } if (thread->UserStackBase) { - MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase); + MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase); thread->UserStackBase = (void *)0; } } @@ -66,10 +73,13 @@ static void KiThreadDestroy(OBJECT* obj) { remaining = owner->ThreadCount; KeReleaseSpinlock(&owner->HandleLock, flags); - if (remaining == 0 && owner->Terminating && - // Implements the interlocked Exchange operation. - _InterlockedExchange(&owner->Exited, 1) == 0) { - if (owner->ExitEvent) KeSignalEvent(owner->ExitEvent); + if (remaining == 0) { + // The last thread leaving a process is process exit even if user + // code called ExitThread instead of ExitProcess. KeExitProcess / + // KeTerminateProcess set Terminating earlier; this covers the last + // thread voluntarily exiting without an explicit process-exit call. + _InterlockedExchange(&owner->Terminating, 1); + KeCompleteProcessExitIfLastThread(owner); } ObDereferenceObject(&owner->Header); @@ -111,11 +121,13 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui thread->WaitStatus = STATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); thread->TrapFrame = (KTRAP_FRAME*)0; - void* kstack = MmAllocatePool(KSTACK_SIZE); + void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k')); if (!kstack) { ObDereferenceObject(&thread->Header); return STATUS_NO_MEMORY; @@ -127,10 +139,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui KSTATUS s; void* ustack = (void*)0; - s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE, + s = MmVadCreateCommitted(owner, USTACK_SIZE, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, - USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH, - UserRegionStack, + MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH, + MmVadStack, &ustack); if (!KSUCCESS(s) || !ustack) { MmFreePool(kstack); @@ -139,11 +151,19 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui return STATUS_NO_MEMORY; } thread->UserStackBase = ustack; - // Ring-3 starts at the image entry via IRET, not CALL, so there is no - // pushed return address. The x64 ABI expects RSP % 16 == 8 at function - // entry; bias the initial stack as if a call had happened so compiler - // generated aligned SSE spills (movaps) are safe. - uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8; + // Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI + // nevertheless lets callees immediately use the caller-provided 32-byte + // home/shadow area above RSP, and MSVC commonly emits a first instruction + // like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic + // entry stack as if a CALL with a normal home area had happened: + // + // RSP % 16 == 8 at function entry + // [RSP + 0] fake return-address slot + // [RSP + 8..39] home/shadow space + // + // Without the 32-byte home area, the first spill lands in the high stack + // guard page and every freshly-created MSVC thread faults immediately. + uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8 - 32; SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB)); if (!teb) { @@ -211,8 +231,16 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui KeReleaseSpinlock(&owner->HandleLock, flags); KeInitThreadFpuContext(thread); - KeReadyThread(thread); + + // Hand the creator a short-lived reference before making the thread + // runnable. On SMP a new user thread can run and exit before + // KeCreateThread() returns; without this reference, callers that merely + // read ThreadId for logging or syscall output can race the scheduler's + // dead-thread dereference and touch freed pool. + ObReferenceObject(&thread->Header); *out = thread; + + KeReadyThread(thread); return STATUS_SUCCESS; } @@ -283,6 +311,8 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, thread->WaitStatus = STATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); thread->TrapFrame = (KTRAP_FRAME*)0; @@ -290,7 +320,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx thread->TebSize = 0; - void* kstack = MmAllocatePool(KSTACK_SIZE); + void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k')); if (!kstack) { ObDereferenceObject(&thread->Header); return STATUS_NO_MEMORY; @@ -320,7 +350,16 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, owner->ThreadCount++; KeReleaseSpinlock(&owner->HandleLock, flags); - if (out) *out = thread; + // Hand the creator a short-lived reference before making the kernel + // thread runnable. Kernel workers can run, signal their completion event, + // exit, and be reaped on another CPU before KeCreateKernelThread()'s + // caller drops its returned pointer. The live scheduler reference remains + // separate and is released by KeSchedulerPostSwitch() after the dead + // thread has switched away. + if (out) { + ObReferenceObject(&thread->Header); + *out = thread; + } KeInitThreadFpuContext(thread); KeReadyThread(thread); @@ -352,6 +391,8 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) thread->WaitStatus = STATUS_PENDING; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = alertable ? 1 : 0; + thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0; + thread->WaitRundownContext = (void*)0; } @@ -362,12 +403,41 @@ void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) { thread->WaitListHead = waitListHead; } +// Registers cleanup for waits with auxiliary wait blocks. +void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context) { + KeAssertIrqlAtOrBelow(DISPATCH_LEVEL); + if (!thread) return; + thread->WaitRundownRoutine = routine; + thread->WaitRundownContext = context; +} + +// Clears auxiliary wait cleanup after the owning wait has removed its blocks. +void KeClearThreadWaitRundown(THREAD* thread) { + KeAssertIrqlAtOrBelow(DISPATCH_LEVEL); + if (!thread) return; + thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0; + thread->WaitRundownContext = (void*)0; +} + +// Runs auxiliary wait cleanup for abort and thread teardown paths. +void KeRundownThreadWait(THREAD* thread, KSTATUS status) { + if (!thread || !thread->WaitRundownRoutine) return; + + KE_WAIT_RUNDOWN_ROUTINE routine = thread->WaitRundownRoutine; + void* context = thread->WaitRundownContext; + thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0; + thread->WaitRundownContext = (void*)0; + routine(thread, context, status); +} + // Implements the finish Wait operation. static void KiFinishWait(THREAD* thread, KSTATUS status) { thread->WaitObject = (OBJECT*)0; thread->WaitStatus = status; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; } // Implements the commit Block Thread operation. @@ -385,6 +455,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { if (thread->State != ThreadWaiting) { KSTATUS status = thread->WaitStatus; thread->AlertableWait = 0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; return status; } @@ -406,6 +478,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { KiDeliverKernelApcs(); thread->AlertableWait = 0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; return thread->WaitStatus; } @@ -423,6 +497,92 @@ void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) { KiSchedulerWakeThread(thread, status); } +// Returns whether the scheduler link still belongs to an object wait list. +static int KiThreadWaitLinkIsLinked(const THREAD* thread) { + if (!thread) return 0; + return thread->SchedulerLink.Fwd != &thread->SchedulerLink || + thread->SchedulerLink.Bwd != &thread->SchedulerLink; +} + +// Removes a thread from one wait list while that wait object's lock is held. +static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) { + if (!thread || !waitListHead) return 0; + if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead) + return 0; + + // The signaling side pops the wait-list entry before it wakes the thread. + // In that handoff window the scheduler link is already self-linked; abort + // must not remove it again or turn a successful mutex/semaphore handoff + // into an aborted wait for a thread that may already own the object. + if (!KiThreadWaitLinkIsLinked(thread)) + return 0; + + ListRemove(&thread->SchedulerLink); + thread->WaitListHead = (LIST_ENTRY*)0; + thread->SleepUntilTick = 0; + thread->TimeoutStatus = STATUS_SUCCESS; + return 1; +} + +// Removes a thread from an object-owned wait queue using that object's lock. +static int KiAbortObjectWait(THREAD* thread) { + if (!thread || !thread->WaitObject || !thread->WaitListHead) + return 0; + + OBJECT* object = thread->WaitObject; + switch (object->Type) { + case ObjectTypeEvent: { + EVENT* event = CONTAINING_RECORD(object, EVENT, Header); + uintptr_t flags = KeAcquireSpinlock(&event->Lock); + int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &event->WaitList); + KeReleaseSpinlock(&event->Lock, flags); + return removed; + } + case ObjectTypeMutex: { + MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header); + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList); + KeReleaseSpinlock(&mutex->Lock, flags); + return removed; + } + case ObjectTypeSemaphore: { + SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header); + uintptr_t flags = KeAcquireSpinlock(&semaphore->Lock); + int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &semaphore->WaitList); + KeReleaseSpinlock(&semaphore->Lock, flags); + return removed; + } + default: + return 0; + } +} + +// Aborts a thread's current wait regardless of whether the wait was marked +// alertable. Process teardown uses this for crash/kill paths: a background +// worker blocked forever on a semaphore must not keep the process exit event +// unsignaled after the foreground thread has faulted. Object wait-list removal +// must use the object's own lock; otherwise signal/abort races can leave stale +// scheduler links in an event, mutex, or semaphore wait queue. +void KeAbortThreadWait(THREAD* thread, KSTATUS status) { + if (!thread) return; + if (thread->State != ThreadWaiting) return; + + if (thread->WaitRundownRoutine) { + KeRundownThreadWait(thread, status); + return; + } + + if (KiAbortObjectWait(thread)) { + KeUnblockThreadWithStatus(thread, status); + return; + } + + // Sleep/timeouts and already-dequeued handoffs are owned by the scheduler. + // This path removes from the scheduler sleep list when present and otherwise + // completes the wait in place. + KeUnblockThreadFromWaitListWithStatus(thread, status); +} + // Implements the unblock Thread operation. void KeUnblockThread(THREAD* thread) { KeUnblockThreadWithStatus(thread, STATUS_SUCCESS); diff --git a/code/kernel/ke_work.c b/code/kernel/ke_work.c new file mode 100644 index 0000000..4f41b38 --- /dev/null +++ b/code/kernel/ke_work.c @@ -0,0 +1,117 @@ +// Kernel passive-level work queue. This file owns the generic mechanism used +// to move ISR/DPC-originated work to a sleepable system thread context. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +static SPINLOCK gWorkQueueLock = SPINLOCK_INIT; +static LIST_ENTRY gWorkQueueList; +static EVENT* gWorkQueueEvent; +static volatile long gWorkQueueListReady; +static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started. + +static void KiWorkQueueWorker(void* arg) { + (void)arg; + KeAssertIrql(PASSIVE_LEVEL); + + for (;;) { + (void)KeWaitForEvent(gWorkQueueEvent, 0); + + for (;;) { + KWORK_ITEM* item = (KWORK_ITEM*)0; + uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock); + if (!LIST_IS_EMPTY(&gWorkQueueList)) { + LIST_ENTRY* node = ListPopHead(&gWorkQueueList); + item = CONTAINING_RECORD(node, KWORK_ITEM, Link); + } + KeReleaseSpinlock(&gWorkQueueLock, flags); + + if (!item) + break; + + // Allow the item to be queued again while it is running. This keeps + // level-triggered users such as cursor updates naturally coalesced + // but does not lose an update that arrives during the callback. + _InterlockedExchange(&item->Queued, 0); + if (item->Routine) + item->Routine(item->Context); + } + } +} + +void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context) { + if (!item) + return; + LIST_INIT(&item->Link); + item->Routine = routine; + item->Context = context; + item->Queued = 0; +} + +bool KeIsWorkQueueStarted(void) { + return _InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 2; +} + +bool KeQueueWorkItem(KWORK_ITEM* item) { + if (!item || !item->Routine) + return false; + + if (_InterlockedCompareExchange(&item->Queued, 1, 0) != 0) + return false; + + if (!_InterlockedCompareExchange(&gWorkQueueListReady, 1, 1)) { + _InterlockedExchange(&item->Queued, 0); + return false; + } + + uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock); + ListInsertTail(&gWorkQueueList, &item->Link); + EVENT* event = gWorkQueueEvent; + KeReleaseSpinlock(&gWorkQueueLock, flags); + + if (event) + KeSignalEvent(event); + return true; +} + +KSTATUS KeStartWorkQueue(void) { + long state = _InterlockedCompareExchange(&gWorkQueueStartState, 1, 0); + if (state == 2) + return STATUS_SUCCESS; + if (state == 1) { + while (_InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 1) + KeYield(); + return KeIsWorkQueueStarted() ? STATUS_SUCCESS : STATUS_FAILURE; + } + + LIST_INIT(&gWorkQueueList); + _InterlockedExchange(&gWorkQueueListReady, 1); + + KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent); + if (KSUCCESS(s)) { + THREAD* worker = (THREAD*)0; + s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker, + 0, + PriorityNormal, + &worker); + // Persistent system worker; keep the creator reference for kernel life. + (void)worker; + } + + if (KSUCCESS(s)) { + _InterlockedExchange(&gWorkQueueStartState, 2); + KeSignalEvent(gWorkQueueEvent); + } else { + _InterlockedExchange(&gWorkQueueStartState, 0); + } + return s; +} diff --git a/code/kernel/kmain.c b/code/kernel/kmain.c index 94e526f..c47a259 100644 --- a/code/kernel/kmain.c +++ b/code/kernel/kmain.c @@ -7,7 +7,9 @@ #pragma intrinsic(_enable) #include #include +#include #include +#include #include #include #include @@ -28,6 +30,9 @@ #include #include #include +#include +#include +#include #include typedef struct { @@ -35,6 +40,90 @@ typedef struct { const HAL_API* hal; } SYSTEM_INIT_ARGS; +#define KI_SWAP_MIN_MB 4ull +#define KI_SWAP_MAX_MB 8192ull + +// Parses a required unsigned decimal integer from settings.ini. +// Configuration errors are treated as boot-time corruption instead of guessed defaults. +static uint64_t KiParseRequiredUInt64Setting(const char* section, const char* key) { + char value[64]; + KSTATUS s = KeGetSettingString(section, key, value, sizeof(value)); + if (!KSUCCESS(s)) { + KePanic(KE_PANIC_CODE_INIT, + "settings: missing required [%s] %s entry, status=0x%x\n", + section, + key, + s); + } + + uint64_t result = 0; + const char* p = value; + if (!*p) { + KePanic(KE_PANIC_CODE_INIT, + "settings: empty numeric value for [%s] %s\n", + section, + key); + } + + while (*p) { + if (*p < '0' || *p > '9') { + KePanic(KE_PANIC_CODE_INIT, + "settings: invalid numeric value for [%s] %s: %s\n", + section, + key, + value); + } + + uint64_t digit = (uint64_t)(*p - '0'); + if (result > (UINT64_MAX - digit) / 10ull) { + KePanic(KE_PANIC_CODE_INIT, + "settings: numeric value for [%s] %s is too large: %s\n", + section, + key, + value); + } + + result = result * 10ull + digit; + ++p; + } + + return result; +} + +// Initializes swap according to settings.ini. +// A size of zero is the explicit opt-out. Any missing or invalid size is a broken configuration. +static void KiInitializeSwapFromSettings(void) { + uint64_t swapMb = KiParseRequiredUInt64Setting("vm", "swap_size_mb"); + if (swapMb == 0) { + DbgPrint("swap: disabled by settings\n"); + return; + } + + if (swapMb < KI_SWAP_MIN_MB || swapMb > KI_SWAP_MAX_MB) { + KePanic(KE_PANIC_CODE_INIT, + "settings: [vm] swap_size_mb=%llu is outside the allowed range %llu..%llu MB\n", + (unsigned long long)swapMb, + (unsigned long long)KI_SWAP_MIN_MB, + (unsigned long long)KI_SWAP_MAX_MB); + } + + char pagefilePath[1024]; + if (!KeBuildSystemRootPath("pagefile.sys", pagefilePath, sizeof(pagefilePath))) { + KePanic(KE_PANIC_CODE_INIT, + "swap: failed to build system-root pagefile path\n"); + } + + uint64_t swapBytes = swapMb * 1024ull * 1024ull; + KSTATUS s = MmSwapInitialize(SysSvFromCstr(pagefilePath), swapBytes); + if (!KSUCCESS(s)) { + KePanic(KE_PANIC_CODE_INIT, + "swap: failed to initialize %llu MB pagefile at %s, status=0x%x\n", + (unsigned long long)swapMb, + pagefilePath, + s); + } +} + typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *); // HAL pointer cached at boot for subsystems that still need platform services. @@ -68,7 +157,7 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) { for (uintptr_t page = base; page < end; page += PAGE_SIZE) { if (!KiPageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue; - MmFreeFrame((PHYSICAL_ADDRESS)page); + MmPfnFree((PHYSICAL_ADDRESS)page); } } } @@ -86,6 +175,7 @@ static void KiResolveSystemImports(BOOT_INFO* bootInfo) { } } + // Phase Helper: Initialize dynamically loaded .sys drivers. static void KiDriversIniCb(const char* section, const char* key, const char* val, void* ud) { @@ -120,7 +210,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { // Map the PE image into kernel space. uintptr_t mappedBase = 0; - KSTATUS st = LdrLoadImage(rawPeBuffer, mod->ImageSize, &mappedBase, false); + KSTATUS st = LdrLoadKernelImage(rawPeBuffer, mod->ImageSize, &mappedBase); if (!KSUCCESS(st)) { DbgPrint("ERROR: Failed to map driver %s (0x%X)\n", mod->Name, st); continue; @@ -129,7 +219,9 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { // Update the struct so other modules can find it. mod->VirtBase = mappedBase; - // Resolve imports (can now see Kernel, HAL, and previous drivers). + // Resolve imports while the image is still writable. Kernel driver images + // stay writable until the loader grows real section-backed mappings; the + // loader still clears NX for the whole image before returning it. if (!KSUCCESS(LdrResolveImports(mappedBase))) { DbgPrint("ERROR: Failed to resolve imports for %s\n", mod->Name); continue; @@ -152,9 +244,24 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { KSTATUS s; FILE_OBJECT* f; - s = IoOpenFile(SYS_SV_LITERAL("drivers\\drivers.ini"), &f); + s = KeInitializeSystemRootVolume(); if (s != STATUS_SUCCESS) { - KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini (status %d)\n", s); + KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to find system root volume (status %d)\n", s); + } + + char root[4]; + root[0] = (char)('0' + KeGetSystemRootDrive()); + root[1] = ':'; + root[2] = '/'; + root[3] = '\0'; + s = IoSetCwd(SysSvFromCstr(root)); + if (s != STATUS_SUCCESS) { + KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to set system CWD to %s (status %d)\n", root, s); + } + + s = KeOpenSystemRootFile("drivers\\drivers.ini", &f); + if (s != STATUS_SUCCESS) { + KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini on system root (status %d)\n", s); } uint64_t size = IoGetFileSize(f) + 1; @@ -174,26 +281,27 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) { DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]); - // Load the driver. + char path[1024]; + char relative[512]; size_t len = strlen(runtimeKeys[i]); - char* path = MmAllocatePool(len + 10); - if (!path) { - DbgPrint("KernelMain: Failed to allocate memory for driver path. How?\n"); + if (len + 9 > sizeof(relative)) { + DbgPrint("KiInitializeDrivers: runtime driver path too long: %s\n", runtimeKeys[i]); continue; } + memcpy(relative, "drivers\\", 8); + memcpy(relative + 8, runtimeKeys[i], len + 1); - // memcpy maneuvers. - memcpy(path, "drivers\\", 8); - memcpy(path + 8, runtimeKeys[i], len + 1); + if (!KeBuildSystemRootPath(relative, path, sizeof(path))) { + DbgPrint("KiInitializeDrivers: failed to build runtime driver path: %s\n", runtimeKeys[i]); + continue; + } s = IoOpenFile(SysSvFromCstr(path), &f); if (s != STATUS_SUCCESS) { DbgPrint("KiInitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path); - continue; // Skip this driver but keep going with others + continue; } - MmFreePool(path); - uint64_t drvSize = IoGetFileSize(f); void* drvMem = MmAllocatePool(drvSize); if (!KSUCCESS(IoReadFile(f, drvMem, drvSize, &got))) { @@ -204,7 +312,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { IoCloseFile(f); uintptr_t mappedBase = 0; - if (!KSUCCESS(LdrLoadImage(drvMem, drvSize, &mappedBase, false))) { // Load into kernel space + if (!KSUCCESS(LdrLoadKernelImage(drvMem, drvSize, &mappedBase))) { // Load into kernel space DbgPrint("KiInitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]); MmFreePool(drvMem); continue; @@ -215,7 +323,6 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]); continue; } - // Runtime modules may also export DDIs to drivers loaded later. Boot // drivers were already registered above; keep the module model uniform. LdrRegisterModule(runtimeKeys[i], mappedBase); @@ -250,28 +357,15 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { KSTATUS s; FILE_OBJECT* f; - // Read the executable file into a raw buffer. - s = IoOpenFile(SysSvFromCstr(filePath), &f); + // Kernel-launched user images are rooted at the selected system volume. + s = KeOpenSystemRootFile(filePath, &f); if (s != STATUS_SUCCESS) { - DbgPrint("Failed to load file: \"%s\"!\n", filePath); + DbgPrint("Failed to load system file: \"%s\" (status %d)!\n", filePath, s); return 0; } uint64_t size = IoGetFileSize(f); - - void* mem = MmAllocatePool(size); - if (!mem) { - IoCloseFile(f); - return 0; - } - - size_t got = 0; - s = IoReadFile(f, mem, size, &got); - IoCloseFile(f); - if (!KSUCCESS(s)) { - MmFreePool(mem); - return 0; - } + (void)size; // Create the process object. PROCESS* proc; @@ -280,7 +374,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { s = KeCreateProcess(filePath, &proc); if (!KSUCCESS(s)) { DbgPrint("Failed to create process. System halted.\n"); - MmFreePool(mem); + IoCloseFile(f); return 0; } DbgPrint("Created process (PID %u) with command line: %s\n", proc->ProcessId, (args && args[0]) ? args : filePath); @@ -291,17 +385,17 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { SysSvFromCstr(proc->CurrentDirectory)); if (!KSUCCESS(s)) { DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s); - MmFreePool(mem); + IoCloseFile(f); ObDereferenceObject(&proc->Header); return 0; } - // Parse the raw PE file and map it into executable memory. + // Map the executable through an image section backed by the open file. uintptr_t imageBase = 0; - s = LdrLoadImageForProcess(proc, mem, size, &imageBase); - MmFreePool(mem); + s = LdrLoadImageFileForProcess(proc, f, &imageBase); + IoCloseFile(f); if (!KSUCCESS(s)) { - DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s); + DbgPrint("KiLaunchUserProcess: LdrLoadImageFileForProcess failed with status: %d\n", s); ObDereferenceObject(&proc->Header); return 0; } @@ -313,6 +407,13 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { return 0; } + s = LdrProtectImageForProcess(proc, imageBase, true); + if (!KSUCCESS(s)) { + DbgPrint("KiLaunchUserProcess: Failed to protect EXE image for %s (status %d)\n", filePath, s); + ObDereferenceObject(&proc->Header); + return 0; + } + proc->ImageBase = imageBase; proc->ImageSize = LdrGetMappedImageSize(proc, imageBase); @@ -327,7 +428,9 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { ObDereferenceObject(&proc->Header); return 0; } - DbgPrint("KiLaunchUserProcess: Thread created and queued!\n"); + uint32_t tid = thread->ThreadId; + DbgPrint("KiLaunchUserProcess: Thread created and queued! TID=%u\n", tid); + ObDereferenceObject(&thread->Header); // Caller owns the reference returned by KeCreateProcess. return proc; @@ -344,35 +447,39 @@ void SystemInitThread(void* arg) { KiInitializeDrivers(args->bootInfo, args->hal); KeTimeReloadSettings(); + KiInitializeSwapFromSettings(); + KiResolveSystemImports(args->bootInfo); KiReclaimBootloaderMemory(args->bootInfo); if (args->bootInfo->Debug & KE_DEBUG_SELFTEST) KiRunKernelSelfTests(args->hal); - // Launch User-Mode init.exe and restart it until it exits cleanly. - while (true) { - PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine); - if (!shellProc) { - DbgPrint("SystemInitThread: Failed to start system/init.exe; retrying.\n"); - KeYield(); - continue; - } - - KeWaitForEvent(shellProc->ExitEvent, 0); - int exitCode = shellProc->ExitCode; - DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode); - ObDereferenceObject(&shellProc->Header); - - if (exitCode == 0) break; - DbgPrint("SystemInitThread: Restarting init.exe.\n"); + // Launch User-Mode init.exe. + PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine); + if (!shellProc) { + DbgPrint("SystemInitThread: Failed to start system/init.exe.\n"); + KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n"); } - DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n"); + KeWaitForEvent(shellProc->ExitEvent, 0); + int exitCode = shellProc->ExitCode; + if (exitCode != 0) { + KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init.exe exited with code %d.\n", exitCode); + } + DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode); + ObDereferenceObject(&shellProc->Header); + if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) { - KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log")); - if (!KSUCCESS(logStatus)) - DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus); + char logPath[1024]; + KSTATUS logStatus = STATUS_NOT_FOUND; + if (KeBuildSystemRootPath("kernel.log", logPath, sizeof(logPath))) { + logStatus = KeFlushDebugLogToFile(SysSvFromCstr(logPath)); + if (!KSUCCESS(logStatus)) + DbgPrint("debuglog: shutdown flush to %s failed st=0x%x\n", logPath, logStatus); + } else { + DbgPrint("debuglog: shutdown flush skipped; system root path unavailable\n"); + } } gKmainHal->Power.Shutdown(); MmFreePool(args); @@ -459,6 +566,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) { KeInitializeScheduler(); + KSTATUS workStatus = KeStartWorkQueue(); + if (!KSUCCESS(workStatus)) { + KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus); + } + KSTATUS smpSchedStatus = KeAdmitApplicationProcessorsToScheduler(); if (!KSUCCESS(smpSchedStatus)) { KePanic(KE_PANIC_CODE_INIT, "SMP: failed to admit APs to scheduler, status=0x%x\n", smpSchedStatus); @@ -471,11 +583,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) { initArgs->bootInfo = bootInfo; initArgs->hal = hal; - // Spawn the Initialization Thread. - THREAD* initThread = (THREAD*)0; - KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread, - (uintptr_t)initArgs, - PriorityNormal, &initThread); + // Spawn the Initialization Thread. No creator reference is needed; the + // scheduler owns the live thread lifetime. + KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread, + (uintptr_t)initArgs, + PriorityNormal, (THREAD**)0); if (!KSUCCESS(st)) { KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st); diff --git a/code/kernel/ldr_image.c b/code/kernel/ldr_image.c index fb97fc7..5fe64c1 100644 --- a/code/kernel/ldr_image.c +++ b/code/kernel/ldr_image.c @@ -3,11 +3,13 @@ #include #include #include +#include #include #include #include #include #include +#include #ifndef LDR_DEBUG_VERBOSE #define LDR_DEBUG_VERBOSE 0 @@ -155,6 +157,8 @@ typedef struct { #define IMAGE_REL_BASED_DIR64 10 #define IMAGE_SCN_CNT_CODE 0x00000020u +#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040u +#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u #define IMAGE_SCN_MEM_EXECUTE 0x20000000u #define IMAGE_SCN_MEM_READ 0x40000000u #define IMAGE_SCN_MEM_WRITE 0x80000000u @@ -171,6 +175,11 @@ typedef struct { static MODULE_ENTRY gModules[MAX_MODULES]; static uint32_t gModuleCount = 0; +// Ensures a user image page is privately writable before the kernel patches it. +static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t address, size_t size) { + if (!user) return STATUS_SUCCESS; + return MmVadMakeImageRangePrivate(process, (void*)address, size); +} // Implements the canonical Module Name operation. static const char* KiCanonicalModuleName(const char* name) { @@ -310,10 +319,10 @@ size_t LdrGetImageSize(uintptr_t base) { size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) { if (!base) return 0; if (process) { - size_t regionSize = MmGetUserRegionSize(process, (const void*)base); + size_t regionSize = MmVadGetSize(process, (const void*)base); if (regionSize) return regionSize; } - size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base); + size_t globalRegionSize = MmVadGetSizeByBase((const void*)base); if (globalRegionSize) return globalRegionSize; return LdrGetImageSize(base); } @@ -327,13 +336,26 @@ static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, b else MmFreePool((void*)imageBase); } -// Releases loaded Image. -static void KiFreeLoadedImage(uintptr_t imageBase, bool user) { - KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user); +// Returns the final page flags for a PE section. +static uint64_t KiSectionPageProtection(const SECTION_HEADER* section, bool user) { + uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0); + + // Some old toolchains mark BSS-like sections as uninitialized data without + // setting MEM_WRITE. Treating those pages as read-only breaks programs that + // legitimately write their zero-fill storage. + if ((section->Characteristics & IMAGE_SCN_MEM_WRITE) || + (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)) { + prot |= PAGE_WRITE; + } + + if (!((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) || + (section->Characteristics & IMAGE_SCN_CNT_CODE))) { + prot |= PAGE_NX; + } + return prot; } - -// Implements the ldr Protect Image For Process operation. +// Applies final page protections to a mapped PE image. KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) { if (!imageBase) return STATUS_INVALID_PARAMETER; if (user && !process) return STATUS_INVALID_PARAMETER; @@ -353,6 +375,14 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset) return STATUS_FAILURE; + size_t pageCount = (mappedSize + PAGE_SIZE - 1) / PAGE_SIZE; + uint64_t* pageProt = (uint64_t*)MmAllocatePoolTag(pageCount * sizeof(uint64_t), MM_POOL_TAG('i', 'p', 'r', 't')); + if (!pageProt) return STATUS_NO_MEMORY; + + uint64_t headerProt = PAGE_PRESENT | (user ? PAGE_USER : 0) | PAGE_NX; + for (size_t i = 0; i < pageCount; ++i) + pageProt[i] = headerProt; + SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset); for (int i = 0; i < pe->NumberOfSections; i++) { size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData; @@ -368,35 +398,56 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us (unsigned long long)imageBase, (unsigned long long)mappedSize); continue; } - if (secEnd < secBase) return STATUS_NO_MEMORY; + if (secEnd < secBase) { + MmFreePool(pageProt); + return STATUS_NO_MEMORY; + } if (secEnd > mapEnd) { LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n", i, (unsigned long long)secEnd, (unsigned long long)mapEnd); - protectSize = (size_t)(mapEnd - secBase); + secEnd = mapEnd; } - uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0); - if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE) - prot |= PAGE_WRITE; - if (!((sec[i].Characteristics & IMAGE_SCN_MEM_EXECUTE) || - (sec[i].Characteristics & IMAGE_SCN_CNT_CODE))) - prot |= PAGE_NX; - - KSTATUS s; - if (user) { - s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot); - } else { - s = MmSetPageProtect(secBase, protectSize, prot); - } - if (!KSUCCESS(s)) { - LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n", - i, (unsigned long long)secBase, (unsigned long long)protectSize, - (unsigned long long)prot, s, - (unsigned long long)imageBase, (unsigned long long)mappedSize); - return s; + size_t firstPage = (size_t)((secBase - imageBase) / PAGE_SIZE); + size_t lastPage = (size_t)((secEnd - 1 - imageBase) / PAGE_SIZE); + uint64_t prot = KiSectionPageProtection(&sec[i], user); + for (size_t page = firstPage; page <= lastPage && page < pageCount; ++page) { + // Protection is page-granular. If sections ever share a page, the + // hardware page must allow the union of their access requirements. + if (prot & PAGE_WRITE) + pageProt[page] |= PAGE_WRITE; + if ((prot & PAGE_NX) == 0) + pageProt[page] &= ~PAGE_NX; } } - return STATUS_SUCCESS; + + size_t runStart = 0; + KSTATUS s = STATUS_SUCCESS; + while (runStart < pageCount) { + uint64_t prot = pageProt[runStart]; + size_t runEnd = runStart + 1; + while (runEnd < pageCount && pageProt[runEnd] == prot) + runEnd++; + + uintptr_t va = imageBase + runStart * PAGE_SIZE; + size_t bytes = (runEnd - runStart) * PAGE_SIZE; + if (user) + s = MmVadProtect(process, (void*)va, bytes, prot); + else + s = MmPteProtectRange(va, bytes, prot); + + if (!KSUCCESS(s)) { + LdrDbg("LDR: protect failed va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n", + (unsigned long long)va, (unsigned long long)bytes, + (unsigned long long)prot, s, + (unsigned long long)imageBase, (unsigned long long)mappedSize); + break; + } + runStart = runEnd; + } + + MmFreePool(pageProt); + return s; } @@ -408,7 +459,7 @@ uintptr_t LdrGetEntryPoint(uintptr_t base) { } // Base Relocation. -static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) { +static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, uintptr_t delta, bool user) { uint32_t peOffset = *(uint32_t *)(base + 0x3C); PE_HEADER* pe = (PE_HEADER *)(base + peOffset); OPT_HEADER64* opt = &pe->OptionalHeader; @@ -432,6 +483,8 @@ static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) { // For 64-bit PEs, we mostly care about DIR64 (Type 10). if (type == IMAGE_REL_BASED_DIR64) { uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset); + KSTATUS ws = KiEnsureUserImageWrite(process, user, (uintptr_t)ptr, sizeof(*ptr)); + if (!KSUCCESS(ws)) return ws; *ptr += delta; } } @@ -534,6 +587,8 @@ static KSTATUS LdrResolveImportsCommon( return STATUS_NOT_FOUND; } + KSTATUS ws = KiEnsureUserImageWrite(process, process != (PROCESS*)0, (uintptr_t)thunk, sizeof(*thunk)); + if (!KSUCCESS(ws)) return ws; *thunk = (uint64_t)addr; thunk++; @@ -566,6 +621,41 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) { ); } +// Loads a user image from an open file through the image-section object model. +KSTATUS LdrLoadImageFileForProcess(PROCESS* process, FILE_OBJECT* file, uintptr_t* baseOut) { + if (!process || !file || !baseOut) + return STATUS_INVALID_PARAMETER; + *baseOut = 0; + + IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)0; + KSTATUS s = LdrCreateImageSectionFromFile(file, §ion); + if (!KSUCCESS(s)) + return s; + + void* view = (void*)0; + s = LdrMapImageSection(process, section, &view); + if (!KSUCCESS(s)) { + LdrDereferenceImageSection(section); + return s; + } + + const IMAGE_SECTION_METADATA* metadata = LdrImageSectionMetadata(section); + uintptr_t imageBase = (uintptr_t)view; + uintptr_t delta = imageBase - metadata->PreferredBase; + if (delta != 0) { + s = LdrApplyRelocationsForProcess(process, imageBase, delta, true); + if (!KSUCCESS(s)) { + KiFreeLoadedImageForProcess(process, imageBase, true); + LdrDereferenceImageSection(section); + return s; + } + } + + LdrDereferenceImageSection(section); + *baseOut = imageBase; + return STATUS_SUCCESS; +} + // Implements the ldr Register Process Module operation. static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) { if (!process || !name || !base) return STATUS_INVALID_PARAMETER; @@ -602,36 +692,27 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_ FILE_OBJECT* f; KSTATUS s = IoOpenFile(SysSvFromCstr(filename), &f); if (!KSUCCESS(s)) { - char path[260]; - memcpy(path, "0:/system/", 10); - memcpy(path + 10, filename, strnlen(filename, 248) + 1); - path[259] = '\0'; - s = IoOpenFile(SysSvFromCstr(path), &f); + char relative[260]; + size_t nameLen = strnlen(filename, sizeof(relative) - 8); + if (nameLen + 8 >= sizeof(relative)) + return STATUS_BUFFER_TOO_SMALL; + memcpy(relative, "system/", 7); + memcpy(relative + 7, filename, nameLen); + relative[7 + nameLen] = '\0'; + + s = KeOpenSystemRootFile(relative, &f); if (!KSUCCESS(s)) { - DbgPrint("LdrLoadModuleForProcess: Failed to open %s (%d)\n", filename, s); + DbgPrint("LdrLoadModuleForProcess: Failed to open %s on system root (%d)\n", filename, s); return s; } } uint64_t fileSize = IoGetFileSize(f); LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize); - void* mem = MmAllocatePool((size_t)fileSize); - if (!mem) { - IoCloseFile(f); - return STATUS_NO_MEMORY; - } - - size_t got = 0; - s = IoReadFile(f, mem, (size_t)fileSize, &got); - IoCloseFile(f); - if (!KSUCCESS(s)) { - MmFreePool(mem); - return s; - } uintptr_t imageBase = 0; - s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase); - MmFreePool(mem); + s = LdrLoadImageFileForProcess(process, f, &imageBase); + IoCloseFile(f); if (!KSUCCESS(s)) { DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s); return s; @@ -665,10 +746,9 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_ return STATUS_SUCCESS; } -// LdrLoadImage - load a PE from a raw buffer into virtual space. -static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) { +// Loads a boot/kernel PE image from a raw memory buffer into kernel space. +static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, uintptr_t* baseOut) { if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER; - if (user && !process) return STATUS_INVALID_PARAMETER; if (size < 0x40) return STATUS_FAILURE; const uint8_t* bytes = (const uint8_t*)buffer; @@ -687,13 +767,13 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t uint16_t optionalMagic = *(const uint16_t *)(bytes + peOffset + 24); if (machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) { - DbgPrint("LdrLoadImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n", + DbgPrint("LdrLoadKernelImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n", machine, optionalMagic); return STATUS_NOT_SUPPORTED; } if (machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) { - DbgPrint("LdrLoadImage: unsupported PE image (machine=0x%X magic=0x%X)\n", + DbgPrint("LdrLoadKernelImage: unsupported PE image (machine=0x%X magic=0x%X)\n", machine, optionalMagic); return STATUS_NOT_SUPPORTED; } @@ -704,17 +784,16 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize); if (!KSUCCESS(sizeStatus)) return sizeStatus; - LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n", + LdrDbg("LDR: kernel image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u\n", opt->SizeOfImage, (unsigned long long)imageSize, (unsigned long long)opt->ImageBase, - pe->NumberOfSections, - user ? 1 : 0, - process ? process->ProcessId : 0); - void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize); + pe->NumberOfSections); + + void* mem = MmAllocatePoolTag(imageSize, MM_POOL_TAG('i', 'm', 'g', ' ')); if (!mem) { - DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n", - (unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0); + DbgPrint("LDR: kernel image allocation failed required=%llx\n", + (unsigned long long)imageSize); return STATUS_NO_MEMORY; } memset(mem, 0, imageSize); @@ -730,27 +809,37 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t memcpy(dst, src, copySize); } - uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0); - MmSetPageProtect((uintptr_t)mem + sec[i].VirtualAddress, + MmPteProtectRange((uintptr_t)mem + sec[i].VirtualAddress, sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData, - prot); + PAGE_PRESENT | PAGE_WRITE); } uintptr_t delta = (uintptr_t)mem - opt->ImageBase; if (delta != 0) { - LdrApplyRelocations((uintptr_t)mem, delta); + KSTATUS rs = LdrApplyRelocationsForProcess((PROCESS*)0, (uintptr_t)mem, delta, false); + if (!KSUCCESS(rs)) { + MmFreePool(mem); + return rs; + } + } + + // Kernel drivers are still pool-backed images. Section protections cannot + // safely make pages read-only yet because imports and driver-owned writable + // image data are not separated from callback code by a section object. What + // must be true is simpler: every page in the loaded image must be executable + // enough for driver callbacks, including section-alignment slack near the + // end of the image. + KSTATUS xs = MmPteProtectRange((uintptr_t)mem, imageSize, PAGE_PRESENT | PAGE_WRITE); + if (!KSUCCESS(xs)) { + MmFreePool(mem); + return xs; } *baseOut = (uintptr_t)mem; return STATUS_SUCCESS; } -// Implements the ldr Load Image For Process operation. -KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) { - return KiLdrLoadImageCommon(process, buffer, size, baseOut, true); -} - -// Implements the ldr Load Image operation. -KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) { - return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user); +// Loads a kernel image from a bootloader or driver raw buffer. +KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut) { + return KiLdrLoadKernelImageFromBuffer(buffer, size, baseOut); } diff --git a/code/kernel/ldr_section.c b/code/kernel/ldr_section.c new file mode 100644 index 0000000..7324bfb --- /dev/null +++ b/code/kernel/ldr_section.c @@ -0,0 +1,736 @@ +// Image section objects. This file owns parsed PE image identity, sparse prototype pages, and image-section caching. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PE_SIGNATURE 0x00004550UL +#define IMAGE_FILE_MACHINE_AMD64 0x8664 +#define IMAGE_FILE_MACHINE_I386 0x014c +#define IMAGE_NT_OPTIONAL_HDR32_MAGIC 0x10b +#define IMAGE_NT_OPTIONAL_HDR64_MAGIC 0x20b +#define IMAGE_SCN_CNT_CODE 0x00000020u +#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040u +#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u +#define IMAGE_SCN_MEM_EXECUTE 0x20000000u +#define IMAGE_SCN_MEM_READ 0x40000000u +#define IMAGE_SCN_MEM_WRITE 0x80000000u + +#define LDR_IMAGE_SECTION_CACHE_MAX 32u + +#pragma pack(push, 1) +typedef struct LDR_OPT_HEADER64 { + uint16_t Magic; + uint8_t MajorLinkerVersion; + uint8_t MinorLinkerVersion; + uint32_t SizeOfCode; + uint32_t SizeOfInitializedData; + uint32_t SizeOfUninitializedData; + uint32_t AddressOfEntryPoint; + uint32_t BaseOfCode; + uint64_t ImageBase; + uint32_t SectionAlignment; + uint32_t FileAlignment; + uint16_t MajorOSVersion; + uint16_t MinorOSVersion; + uint16_t MajorImageVersion; + uint16_t MinorImageVersion; + uint16_t MajorSubsystemVersion; + uint16_t MinorSubsystemVersion; + uint32_t Win32VersionValue; + uint32_t SizeOfImage; + uint32_t SizeOfHeaders; + uint32_t CheckSum; + uint16_t Subsystem; + uint16_t DllCharacteristics; + uint64_t SizeOfStackReserve; + uint64_t SizeOfStackCommit; + uint64_t SizeOfHeapReserve; + uint64_t SizeOfHeapCommit; + uint32_t LoaderFlags; + uint32_t NumberOfRvaAndSizes; + struct { uint32_t VirtualAddress; uint32_t Size; } DataDirectory[16]; +} LDR_OPT_HEADER64; + +typedef struct LDR_PE_HEADER { + uint32_t Signature; + uint16_t Machine; + uint16_t NumberOfSections; + uint32_t TimeDateStamp; + uint32_t PointerToSymbolTable; + uint32_t NumberOfSymbols; + uint16_t SizeOfOptionalHeader; + uint16_t Characteristics; + LDR_OPT_HEADER64 OptionalHeader; +} LDR_PE_HEADER; + +typedef struct LDR_SECTION_HEADER { + char Name[8]; + uint32_t VirtualSize; + uint32_t VirtualAddress; + uint32_t SizeOfRawData; + uint32_t PointerToRawData; + uint32_t PointerToRelocations; + uint32_t PointerToLinenumbers; + uint16_t NumberOfRelocations; + uint16_t NumberOfLinenumbers; + uint32_t Characteristics; +} LDR_SECTION_HEADER; +#pragma pack(pop) + +struct IMAGE_SECTION_OBJECT { + OBJECT Header; + IMAGE_SECTION_METADATA Metadata; + IMAGE_SECTION_DESCRIPTOR* Sections; + PHYSICAL_ADDRESS* Frames; + FILE_OBJECT* BackingFile; + size_t HeadersFileSize; + SPINLOCK Lock; +}; + +static IMAGE_SECTION_OBJECT* gImageSectionCache[LDR_IMAGE_SECTION_CACHE_MAX]; +static SPINLOCK gImageSectionCacheLock = SPINLOCK_INIT; + +// Aligns a PE image size to the hardware page size. +static size_t KiImageSectionAlignSize(size_t value) { + return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); +} + +// Mixes filesystem identity into a compact cache key. +static uint64_t KiImageSectionIdentityKey(const IO_FILE_IDENTITY* identity) { + if (!identity) + return 0; + + uint64_t h = 1469598103934665603ULL; + h ^= identity->VolumeId; + h *= 1099511628211ULL; + h ^= identity->FileId; + h *= 1099511628211ULL; + h ^= identity->Generation; + h *= 1099511628211ULL; + h ^= identity->Size; + h *= 1099511628211ULL; + return h ? h : 1; +} + +// Converts PE section characteristics to the final user page protection union for that section. +static uint64_t KiImageSectionProtection(uint32_t characteristics) { + uint64_t prot = PAGE_PRESENT | PAGE_USER | PAGE_NX; + if (characteristics & IMAGE_SCN_MEM_WRITE) + prot |= PAGE_WRITE; + if (characteristics & IMAGE_SCN_MEM_EXECUTE) + prot &= ~PAGE_NX; + return prot; +} + +// Computes the required image mapping size after validating raw section extents. +static KSTATUS KiImageSectionComputeSize(const LDR_PE_HEADER* pe, size_t fileSize, size_t* outSize) { + if (!pe || !outSize) return STATUS_INVALID_PARAMETER; + + const LDR_OPT_HEADER64* opt = &pe->OptionalHeader; + size_t required = KiImageSectionAlignSize(opt->SizeOfImage); + size_t headers = KiImageSectionAlignSize(opt->SizeOfHeaders); + if (headers > required) required = headers; + + const LDR_SECTION_HEADER* sec = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader); + for (uint32_t i = 0; i < pe->NumberOfSections; i++) { + size_t va = (size_t)sec[i].VirtualAddress; + size_t virt = (size_t)sec[i].VirtualSize; + size_t raw = (size_t)sec[i].SizeOfRawData; + size_t span = virt ? virt : raw; + if (raw > span) span = raw; + if (!span) continue; + + if (va > (size_t)-1 - span) return STATUS_NO_MEMORY; + size_t end = KiImageSectionAlignSize(va + span); + if (end > required) required = end; + + if (raw) { + size_t rawBase = (size_t)sec[i].PointerToRawData; + if (rawBase > fileSize || raw > fileSize - rawBase) + return STATUS_FAILURE; + } + } + + if (!required) return STATUS_FAILURE; + *outSize = required; + return STATUS_SUCCESS; +} + +// Parses and validates the PE header that backs an image section object. +static KSTATUS KiImageSectionParseHeader(const void* buffer, + size_t bufferSize, + size_t fileSize, + const LDR_PE_HEADER** peOut, + const LDR_SECTION_HEADER** sectionsOut, + size_t* imageSizeOut) { + if (!buffer || !bufferSize || !peOut || !sectionsOut || !imageSizeOut) + return STATUS_INVALID_PARAMETER; + if (bufferSize < 0x40) return STATUS_FAILURE; + + const uint8_t* bytes = (const uint8_t*)buffer; + uint32_t peOffset = *(const uint32_t*)(bytes + 0x3C); + if ((size_t)peOffset > bufferSize || bufferSize - (size_t)peOffset < 24) + return STATUS_FAILURE; + + const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)(bytes + peOffset); + if (pe->Signature != PE_SIGNATURE) + return STATUS_FAILURE; + + uint16_t optionalSize = pe->SizeOfOptionalHeader; + if (optionalSize < sizeof(LDR_OPT_HEADER64) || + (size_t)peOffset + 24 + optionalSize > bufferSize) { + return STATUS_FAILURE; + } + + uint16_t optionalMagic = pe->OptionalHeader.Magic; + if (pe->Machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) { + DbgPrint("LdrImageSection: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n", + pe->Machine, optionalMagic); + return STATUS_NOT_SUPPORTED; + } + + if (pe->Machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) { + DbgPrint("LdrImageSection: unsupported PE image (machine=0x%X magic=0x%X)\n", + pe->Machine, optionalMagic); + return STATUS_NOT_SUPPORTED; + } + + if (!pe->NumberOfSections) + return STATUS_FAILURE; + + const LDR_SECTION_HEADER* sections = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader); + size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader; + size_t sectionTableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER); + if (sectionTableOffset > bufferSize || sectionTableBytes > bufferSize - sectionTableOffset) + return STATUS_FAILURE; + + size_t imageSize = 0; + KSTATUS s = KiImageSectionComputeSize(pe, fileSize, &imageSize); + if (!KSUCCESS(s)) return s; + + *peOut = pe; + *sectionsOut = sections; + *imageSizeOut = imageSize; + return STATUS_SUCCESS; +} + +// Releases all resources owned by an image section object. +static void KiDestroyImageSectionObject(OBJECT* object) { + IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)object; + if (!section) return; + + if (section->Frames) { + for (size_t i = 0; i < section->Metadata.PageCount; ++i) { + if (section->Frames[i]) + (void)MmPfnFree(section->Frames[i]); + } + MmFreePool(section->Frames); + section->Frames = (PHYSICAL_ADDRESS*)0; + } + + if (section->Sections) { + MmFreePool(section->Sections); + section->Sections = (IMAGE_SECTION_DESCRIPTOR*)0; + } + + if (section->BackingFile) { + IoCloseFile(section->BackingFile); + section->BackingFile = (FILE_OBJECT*)0; + } +} + +// Finds a cached image section while the image-section cache lock is held. +static IMAGE_SECTION_OBJECT* KiFindImageSectionLocked(const IO_FILE_IDENTITY* identity, uint64_t key) { + if (!identity || !key) + return (IMAGE_SECTION_OBJECT*)0; + + for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) { + IMAGE_SECTION_OBJECT* section = gImageSectionCache[i]; + if (section && + section->Metadata.ContentKey == key && + section->Metadata.VolumeId == identity->VolumeId && + section->Metadata.FileId == identity->FileId && + section->Metadata.FileGeneration == identity->Generation && + section->Metadata.RawSize == identity->Size) { + return section; + } + } + return (IMAGE_SECTION_OBJECT*)0; +} + + +// Verifies that a cached image section is still backed by the same file generation. +static KSTATUS KiValidateImageSectionBackingFile(IMAGE_SECTION_OBJECT* section) { + if (!section || !section->BackingFile) + return STATUS_INVALID_PARAMETER; + + IO_FILE_IDENTITY identity; + KSTATUS s = IoQueryFileIdentity(section->BackingFile, &identity); + if (!KSUCCESS(s)) + return s; + + if (identity.VolumeId != section->Metadata.VolumeId || + identity.FileId != section->Metadata.FileId || + identity.Generation != section->Metadata.FileGeneration || + identity.Size != section->Metadata.RawSize) { + return STATUS_CHECKSUM_MISMATCH; + } + + return STATUS_SUCCESS; +} + +// Returns true when a virtual image page has initialized bytes in the backing file. +static bool KiImageSectionPageHasFileBytes(const IMAGE_SECTION_OBJECT* section, size_t pageIndex) { + if (!section || pageIndex >= section->Metadata.PageCount) + return false; + + size_t pageStart = pageIndex * PAGE_SIZE; + size_t pageEnd = pageStart + PAGE_SIZE; + if (pageStart < section->HeadersFileSize && pageEnd > 0) + return true; + + for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) { + const IMAGE_SECTION_DESCRIPTOR* desc = §ion->Sections[i]; + size_t raw = desc->RawDataSize; + if (desc->VirtualSize && desc->VirtualSize < raw) + raw = desc->VirtualSize; + if (!raw) + continue; + + size_t start = desc->VirtualAddress; + size_t end = start + raw; + if (end < start) + continue; + if (pageStart < end && pageEnd > start) + return true; + } + + return false; +} + +// Copies a file range into an image prototype page, leaving absent bytes zero-filled. +static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section, + uint64_t fileOffset, + void* dst, + size_t size) { + if (!section || !section->BackingFile || (!dst && size)) + return STATUS_INVALID_PARAMETER; + if (!size) + return STATUS_SUCCESS; + + size_t got = 0; + KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got); + if (!KSUCCESS(s)) + return s; + if (got != size) + return STATUS_FAILURE; + return STATUS_SUCCESS; +} + +// Fills one prototype frame from the image backing file. +static KSTATUS KiFillImageSectionFrameFromFile(IMAGE_SECTION_OBJECT* section, + size_t pageIndex, + PHYSICAL_ADDRESS frame) { + if (!section || !frame || pageIndex >= section->Metadata.PageCount) + return STATUS_INVALID_PARAMETER; + + uint8_t* page = (uint8_t*)MmPfnToVirtual(frame); + if (!page) + return STATUS_FAILURE; + memset(page, 0, PAGE_SIZE); + + size_t pageStart = pageIndex * PAGE_SIZE; + size_t pageEnd = pageStart + PAGE_SIZE; + + if (pageStart < section->HeadersFileSize) { + size_t copyStart = pageStart; + size_t copyEnd = pageEnd; + if (copyEnd > section->HeadersFileSize) + copyEnd = section->HeadersFileSize; + KSTATUS s = KiReadImageSectionFileBytes(section, + (uint64_t)copyStart, + page, + copyEnd - copyStart); + if (!KSUCCESS(s)) + return s; + } + + for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) { + const IMAGE_SECTION_DESCRIPTOR* desc = §ion->Sections[i]; + size_t raw = desc->RawDataSize; + if (desc->VirtualSize && desc->VirtualSize < raw) + raw = desc->VirtualSize; + if (!raw) + continue; + + size_t sectionStart = desc->VirtualAddress; + size_t sectionEnd = sectionStart + raw; + if (sectionEnd < sectionStart) + return STATUS_FAILURE; + if (pageStart >= sectionEnd || pageEnd <= sectionStart) + continue; + + size_t copyStart = pageStart > sectionStart ? pageStart : sectionStart; + size_t copyEnd = pageEnd < sectionEnd ? pageEnd : sectionEnd; + size_t dstOffset = copyStart - pageStart; + uint64_t srcOffset = (uint64_t)desc->RawDataOffset + (uint64_t)(copyStart - sectionStart); + + KSTATUS s = KiReadImageSectionFileBytes(section, + srcOffset, + page + dstOffset, + copyEnd - copyStart); + if (!KSUCCESS(s)) + return s; + } + + return STATUS_SUCCESS; +} + +// Ensures a prototype page exists for initialized image bytes. +static KSTATUS KiEnsureImageSectionFrame(IMAGE_SECTION_OBJECT* section, size_t pageIndex) { + if (!section || pageIndex >= section->Metadata.PageCount) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(§ion->Lock); + PHYSICAL_ADDRESS existing = section->Frames[pageIndex]; + KeReleaseSpinlock(§ion->Lock, flags); + if (existing) + return STATUS_SUCCESS; + + if (section->BackingFile && !KiImageSectionPageHasFileBytes(section, pageIndex)) + return STATUS_SUCCESS; + + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseImage); + if (!frame) + return STATUS_NO_MEMORY; + + void* va = MmPfnToVirtual(frame); + if (!va) { + MmPfnFree(frame); + return STATUS_FAILURE; + } + + KSTATUS s = STATUS_SUCCESS; + if (section->BackingFile) { + s = KiValidateImageSectionBackingFile(section); + if (KSUCCESS(s)) + s = KiFillImageSectionFrameFromFile(section, pageIndex, frame); + } else { + memset(va, 0, PAGE_SIZE); + } + if (!KSUCCESS(s)) { + MmPfnFree(frame); + return s; + } + + flags = KeAcquireSpinlock(§ion->Lock); + if (section->Frames[pageIndex]) { + KeReleaseSpinlock(§ion->Lock, flags); + MmPfnFree(frame); + return STATUS_SUCCESS; + } + section->Frames[pageIndex] = frame; + KeReleaseSpinlock(§ion->Lock, flags); + return STATUS_SUCCESS; +} + +// Builds an uncached image section object backed directly by an open file. +static KSTATUS KiBuildImageSectionObjectFromFile(FILE_OBJECT* file, + const void* headerBuffer, + size_t headerBufferSize, + const IO_FILE_IDENTITY* identity, + uint64_t key, + IMAGE_SECTION_OBJECT** sectionOut) { + if (!file || !headerBuffer || !headerBufferSize || !identity || !sectionOut) + return STATUS_INVALID_PARAMETER; + *sectionOut = (IMAGE_SECTION_OBJECT*)0; + + size_t fileSize = (size_t)identity->Size; + const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)0; + const LDR_SECTION_HEADER* rawSections = (const LDR_SECTION_HEADER*)0; + size_t imageSize = 0; + KSTATUS s = KiImageSectionParseHeader(headerBuffer, + headerBufferSize, + fileSize, + &pe, + &rawSections, + &imageSize); + if (!KSUCCESS(s)) + return s; + + size_t pageCount = (imageSize + PAGE_SIZE - 1) / PAGE_SIZE; + if (!pageCount || pageCount > ((size_t)-1 / sizeof(PHYSICAL_ADDRESS))) + return STATUS_FAILURE; + + IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)ObAllocateObject(ObjectTypeImageSection, + sizeof(IMAGE_SECTION_OBJECT)); + if (!section) + return STATUS_NO_MEMORY; + section->Header.Destroy = KiDestroyImageSectionObject; + + PHYSICAL_ADDRESS* frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS), + MM_POOL_TAG('i', 's', 'f', 'r')); + if (!frames) { + ObDereferenceObject(§ion->Header); + return STATUS_NO_MEMORY; + } + memset(frames, 0, pageCount * sizeof(PHYSICAL_ADDRESS)); + + IMAGE_SECTION_DESCRIPTOR* descriptors = (IMAGE_SECTION_DESCRIPTOR*)MmAllocatePoolTag( + (size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR), + MM_POOL_TAG('i', 's', 'e', 'c')); + if (!descriptors) { + MmFreePool(frames); + ObDereferenceObject(§ion->Header); + return STATUS_NO_MEMORY; + } + memset(descriptors, 0, (size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR)); + + section->Frames = frames; + section->Sections = descriptors; + section->BackingFile = file; + section->HeadersFileSize = pe->OptionalHeader.SizeOfHeaders; + section->Lock = (SPINLOCK)SPINLOCK_INIT; + ObReferenceObject(&file->Header); + + section->Metadata.ContentKey = key; + section->Metadata.VolumeId = identity->VolumeId; + section->Metadata.FileId = identity->FileId; + section->Metadata.FileGeneration = identity->Generation; + section->Metadata.PreferredBase = pe->OptionalHeader.ImageBase; + section->Metadata.EntryRva = pe->OptionalHeader.AddressOfEntryPoint; + section->Metadata.HeaderSize = pe->OptionalHeader.SizeOfHeaders; + section->Metadata.RawSize = fileSize; + section->Metadata.ImageSize = imageSize; + section->Metadata.PageCount = pageCount; + section->Metadata.SectionCount = pe->NumberOfSections; + + for (uint32_t i = 0; i < pe->NumberOfSections; ++i) { + IMAGE_SECTION_DESCRIPTOR* dst = §ion->Sections[i]; + memcpy(dst->Name, rawSections[i].Name, sizeof(rawSections[i].Name)); + dst->Name[8] = '\0'; + dst->VirtualAddress = rawSections[i].VirtualAddress; + dst->VirtualSize = rawSections[i].VirtualSize; + dst->RawDataOffset = rawSections[i].PointerToRawData; + dst->RawDataSize = rawSections[i].SizeOfRawData; + dst->Characteristics = rawSections[i].Characteristics; + dst->PageProtection = KiImageSectionProtection(rawSections[i].Characteristics); + } + + *sectionOut = section; + return STATUS_SUCCESS; +} + +// Creates or references an image section object backed by an open file. +KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file, + IMAGE_SECTION_OBJECT** sectionOut) { + if (!file || !sectionOut) + return STATUS_INVALID_PARAMETER; + *sectionOut = (IMAGE_SECTION_OBJECT*)0; + + IO_FILE_IDENTITY identity; + KSTATUS s = IoQueryFileIdentity(file, &identity); + if (!KSUCCESS(s)) + return s; + + size_t fileSize = (size_t)identity.Size; + if (!fileSize) + return STATUS_FAILURE; + + size_t firstRead = fileSize < 4096u ? fileSize : 4096u; + uint8_t stackHeader[4096]; + size_t got = 0; + s = IoReadFileAt(file, 0, stackHeader, firstRead, &got); + if (!KSUCCESS(s) || got != firstRead) + return KSUCCESS(s) ? STATUS_FAILURE : s; + if (firstRead < 0x40) + return STATUS_FAILURE; + + uint32_t peOffset = *(const uint32_t*)(stackHeader + 0x3C); + if ((size_t)peOffset > fileSize || fileSize - (size_t)peOffset < 24) + return STATUS_FAILURE; + + size_t minimumHeader = (size_t)peOffset + 24u; + if (minimumHeader > firstRead) { + if (minimumHeader > 65536u || minimumHeader > fileSize) + return STATUS_FAILURE; + void* tmp = MmAllocatePoolTag(minimumHeader, MM_POOL_TAG('i', 's', 'h', 'd')); + if (!tmp) + return STATUS_NO_MEMORY; + s = IoReadFileAt(file, 0, tmp, minimumHeader, &got); + if (!KSUCCESS(s) || got != minimumHeader) { + MmFreePool(tmp); + return KSUCCESS(s) ? STATUS_FAILURE : s; + } + uint16_t optSize = *(const uint16_t*)((const uint8_t*)tmp + peOffset + 20u); + size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize; + MmFreePool(tmp); + if (fullHeader > firstRead) { + firstRead = fullHeader; + } + } else { + uint16_t optSize = *(const uint16_t*)(stackHeader + peOffset + 20u); + size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize; + if (fullHeader > firstRead) + firstRead = fullHeader; + } + + if (firstRead > fileSize || firstRead > 65536u) + return STATUS_FAILURE; + + void* header = stackHeader; + bool heapHeader = false; + if (firstRead > sizeof(stackHeader)) { + header = MmAllocatePoolTag(firstRead, MM_POOL_TAG('i', 's', 'h', 'd')); + if (!header) + return STATUS_NO_MEMORY; + heapHeader = true; + s = IoReadFileAt(file, 0, header, firstRead, &got); + if (!KSUCCESS(s) || got != firstRead) { + MmFreePool(header); + return KSUCCESS(s) ? STATUS_FAILURE : s; + } + } + + const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)((const uint8_t*)header + peOffset); + size_t tableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER); + size_t tableEnd = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader + tableBytes; + if (tableEnd > firstRead) { + if (tableEnd > fileSize || tableEnd > 65536u) { + if (heapHeader) MmFreePool(header); + return STATUS_FAILURE; + } + void* bigger = MmAllocatePoolTag(tableEnd, MM_POOL_TAG('i', 's', 'h', 'd')); + if (!bigger) { + if (heapHeader) MmFreePool(header); + return STATUS_NO_MEMORY; + } + s = IoReadFileAt(file, 0, bigger, tableEnd, &got); + if (heapHeader) MmFreePool(header); + if (!KSUCCESS(s) || got != tableEnd) { + MmFreePool(bigger); + return KSUCCESS(s) ? STATUS_FAILURE : s; + } + header = bigger; + firstRead = tableEnd; + heapHeader = true; + } + + uint64_t key = KiImageSectionIdentityKey(&identity); + + uintptr_t flags = KeAcquireSpinlock(&gImageSectionCacheLock); + IMAGE_SECTION_OBJECT* cached = KiFindImageSectionLocked(&identity, key); + if (cached) { + ObReferenceObject(&cached->Header); + KeReleaseSpinlock(&gImageSectionCacheLock, flags); + if (heapHeader) MmFreePool(header); + *sectionOut = cached; + return STATUS_SUCCESS; + } + KeReleaseSpinlock(&gImageSectionCacheLock, flags); + + IMAGE_SECTION_OBJECT* built = (IMAGE_SECTION_OBJECT*)0; + s = KiBuildImageSectionObjectFromFile(file, header, firstRead, &identity, key, &built); + if (heapHeader) MmFreePool(header); + if (!KSUCCESS(s)) + return s; + + flags = KeAcquireSpinlock(&gImageSectionCacheLock); + cached = KiFindImageSectionLocked(&identity, key); + if (cached) { + ObReferenceObject(&cached->Header); + KeReleaseSpinlock(&gImageSectionCacheLock, flags); + ObDereferenceObject(&built->Header); + *sectionOut = cached; + return STATUS_SUCCESS; + } + + for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) { + if (!gImageSectionCache[i]) { + ObReferenceObject(&built->Header); + gImageSectionCache[i] = built; + break; + } + } + KeReleaseSpinlock(&gImageSectionCacheLock, flags); + + *sectionOut = built; + return STATUS_SUCCESS; +} + +// Adds a reference to an existing image section object. +void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section) { + if (!section) return; + ObReferenceObject(§ion->Header); +} + +// Releases a reference returned by LdrCreateImageSectionFromFile. +void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section) { + if (!section) return; + ObDereferenceObject(§ion->Header); +} + +// Returns immutable metadata for an image section object. +const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section) { + if (!section) return (const IMAGE_SECTION_METADATA*)0; + return §ion->Metadata; +} + +// Returns the immutable PE section table owned by the image section object. +const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section, + uint32_t* countOut) { + if (countOut) *countOut = section ? section->Metadata.SectionCount : 0; + if (!section) return (const IMAGE_SECTION_DESCRIPTOR*)0; + return section->Sections; +} + +// Retains one prototype page frame owned by an image section object. +KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section, + size_t pageIndex, + PHYSICAL_ADDRESS* frameOut) { + if (!section || !frameOut) return STATUS_INVALID_PARAMETER; + *frameOut = 0; + if (pageIndex >= section->Metadata.PageCount) + return STATUS_INVALID_PARAMETER; + + KSTATUS es = KiEnsureImageSectionFrame(section, pageIndex); + if (!KSUCCESS(es)) + return es; + + uintptr_t flags = KeAcquireSpinlock(§ion->Lock); + PHYSICAL_ADDRESS frame = section->Frames[pageIndex]; + KeReleaseSpinlock(§ion->Lock, flags); + if (!frame) + return STATUS_SUCCESS; + + KSTATUS s = MmPfnRetain(frame); + if (!KSUCCESS(s)) + return s; + + *frameOut = frame; + return STATUS_SUCCESS; +} + +// Maps an image section into a process as an image-backed VAD. +KSTATUS LdrMapImageSection(PROCESS* process, + IMAGE_SECTION_OBJECT* section, + void** baseOut) { + if (!process || !section || !baseOut) return STATUS_INVALID_PARAMETER; + *baseOut = (void*)0; + + return MmVadCreateImageSectionMapping(process, + section, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH, + baseOut); +} diff --git a/code/kernel/mm_file_cache.c b/code/kernel/mm_file_cache.c new file mode 100644 index 0000000..dabc914 --- /dev/null +++ b/code/kernel/mm_file_cache.c @@ -0,0 +1,1273 @@ +// Coherent file page cache. +// +// The cache owns one resident page per file/offset. Reads, buffered writes, and +// mapped-file faults converge on this page so writeback has a single source of +// truth. Dirty pages are flushed explicitly, by close, or by the background +// writeback worker under pressure. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MM_FILE_CACHE_BUCKETS 64u +#define MM_FILE_CACHE_SOFT_MAX_PAGES 512u +#define MM_FILE_CACHE_READAHEAD_MAX_REQUESTS 64u +#define MM_FILE_CACHE_READAHEAD_MAX_PAGES 32u +#define MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET 128u +#define MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES 16u +#define MM_FILE_CACHE_BATCH_MAX_PAGES 16u +#define MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES 64u +#define MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES 128u +#define MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES 192u +#define MM_FILE_CACHE_WRITEBACK_BATCH_PAGES 16u + +typedef struct MM_FILE_CACHE_ENTRY { + struct MM_FILE_CACHE_ENTRY* Next; + FILE_OBJECT* File; + uint64_t Offset; + PHYSICAL_ADDRESS Frame; + uint32_t ValidBytes; + uint8_t Referenced; + uint8_t Standby; + uint8_t Loading; + uint8_t Invalidated; + uint8_t Dirty; + uint8_t Writeback; + uint64_t DirtySequence; +} MM_FILE_CACHE_ENTRY; + +typedef struct MM_FILE_CACHE_READAHEAD_REQUEST { + LIST_ENTRY Link; + FILE_OBJECT* File; + uint64_t Offset; + uint32_t Pages; +} MM_FILE_CACHE_READAHEAD_REQUEST; + +static SPINLOCK gFileCacheLock = SPINLOCK_INIT; +static MM_FILE_CACHE_ENTRY* gFileCacheBuckets[MM_FILE_CACHE_BUCKETS]; +static uint64_t gFileCacheResidentPages; +static uint64_t gFileCacheActivePages; +static uint64_t gFileCacheStandbyPages; +static uint64_t gFileCacheDirtyPages; +static uint64_t gFileCacheMaxDirtyPages; +static uint64_t gFileCacheMaxResidentPages; +static MM_FILE_CACHE_STATS gFileCacheStats; + +static SPINLOCK gReadAheadLock = SPINLOCK_INIT; +static LIST_ENTRY gReadAheadQueue; +static SEMAPHORE* gReadAheadSemaphore; +static uint32_t gReadAheadQueuedRequests; +static uint32_t gReadAheadQueuedPages; +static int gReadAheadQueueReady; +static int gReadAheadStartState; + +static SPINLOCK gWritebackLock = SPINLOCK_INIT; +static SEMAPHORE* gWritebackSemaphore; +static int gWritebackStartState; + +static uint32_t KiFileCacheHash(FILE_OBJECT* file, uint64_t offset) { + uintptr_t f = (uintptr_t)file >> 4; + uint64_t p = offset >> 12; + return (uint32_t)((f ^ p ^ (p >> 11)) & (MM_FILE_CACHE_BUCKETS - 1u)); +} + +static MM_FILE_CACHE_ENTRY* KiFileCacheFindLocked(FILE_OBJECT* file, uint64_t offset, uint32_t bucket) { + for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[bucket]; e; e = e->Next) { + if (e->File == file && e->Offset == offset) + return e; + } + return (MM_FILE_CACHE_ENTRY*)0; +} + +static void KiFileCacheBumpResidentLocked(int delta, int standby) { + if (delta > 0) { + gFileCacheResidentPages += (uint64_t)delta; + if (standby) gFileCacheStandbyPages += (uint64_t)delta; + else gFileCacheActivePages += (uint64_t)delta; + if (gFileCacheResidentPages > gFileCacheMaxResidentPages) + gFileCacheMaxResidentPages = gFileCacheResidentPages; + } else if (delta < 0) { + uint64_t dec = (uint64_t)(-delta); + gFileCacheResidentPages = gFileCacheResidentPages > dec ? gFileCacheResidentPages - dec : 0; + if (standby) gFileCacheStandbyPages = gFileCacheStandbyPages > dec ? gFileCacheStandbyPages - dec : 0; + else gFileCacheActivePages = gFileCacheActivePages > dec ? gFileCacheActivePages - dec : 0; + } +} + +// Updates dirty-page accounting while the cache lock is held. +static void KiFileCacheSetDirtyLocked(MM_FILE_CACHE_ENTRY* e, int dirty) { + if (!e) + return; + if (dirty) { + if (!e->Dirty) { + e->Dirty = 1; + gFileCacheDirtyPages++; + gFileCacheStats.DirtyPages = gFileCacheDirtyPages; + if (gFileCacheDirtyPages > gFileCacheMaxDirtyPages) { + gFileCacheMaxDirtyPages = gFileCacheDirtyPages; + gFileCacheStats.MaxDirtyPages = gFileCacheMaxDirtyPages; + } + } + e->DirtySequence++; + gFileCacheStats.DirtyMarks++; + } else { + if (e->Dirty) { + e->Dirty = 0; + if (gFileCacheDirtyPages) + gFileCacheDirtyPages--; + gFileCacheStats.DirtyPages = gFileCacheDirtyPages; + } + } +} + +static void KiFileCacheMoveEntryToStandbyLocked(MM_FILE_CACHE_ENTRY* e) { + if (!e || e->Standby) + return; + e->Standby = 1; + if (gFileCacheActivePages) gFileCacheActivePages--; + gFileCacheStandbyPages++; + gFileCacheStats.StandbyTransitions++; +} + +static void KiFileCacheActivateEntryLocked(MM_FILE_CACHE_ENTRY* e) { + if (!e || !e->Standby) + return; + e->Standby = 0; + if (gFileCacheStandbyPages) gFileCacheStandbyPages--; + gFileCacheActivePages++; + gFileCacheStats.StandbyHits++; +} + +static void KiFileCacheFreeEntry(MM_FILE_CACHE_ENTRY* e) { + if (!e) return; + if (e->File) ObDereferenceObject(&e->File->Header); + if (e->Frame) (void)MmPfnRelease(e->Frame); + MmFreePool(e); +} + +static void KiFileCacheRemoveEntryLocked(uint32_t bucket, MM_FILE_CACHE_ENTRY* prev, MM_FILE_CACHE_ENTRY* e) { + if (!e) return; + if (prev) prev->Next = e->Next; + else gFileCacheBuckets[bucket] = e->Next; + if (e->Frame) + KiFileCacheBumpResidentLocked(-1, e->Standby); + if (e->Dirty) + KiFileCacheSetDirtyLocked(e, 0); +} + +static int KiFileCacheIsReclaimCandidateLocked(MM_FILE_CACHE_ENTRY* e, FILE_OBJECT* skipFile, uint64_t skipOffset) { + if (!e) return 0; + if (skipFile && e->File == skipFile && e->Offset == skipOffset) + return 0; + if (e->Loading || e->Invalidated || !e->Frame || e->Dirty || e->Writeback) + return 0; + + MM_PFN* pfn = MmPfnGet(e->Frame); + return pfn && pfn->PinCount == 0 && pfn->MappedCount == 0 && pfn->RefCount == 1; +} + +static KSTATUS KiFileCacheDemoteOnePage(FILE_OBJECT* skipFile, uint64_t skipOffset) { + // A cold clean page first moves to PFN standby so the cache can reactivate it + // without disk I/O if it becomes hot again before final pressure reclaim. + for (int pass = 0; pass < 2; ++pass) { + for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { + MM_FILE_CACHE_ENTRY* demote = (MM_FILE_CACHE_ENTRY*)0; + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; e = e->Next) { + if (e->Standby || !KiFileCacheIsReclaimCandidateLocked(e, skipFile, skipOffset)) + continue; + + if (e->Referenced && pass == 0) { + e->Referenced = 0; + continue; + } + + demote = e; + break; + } + + if (demote) { + KSTATUS s = MmPfnMoveToStandby(demote->Frame); + if (KSUCCESS(s)) { + KiFileCacheMoveEntryToStandbyLocked(demote); + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_SUCCESS; + } + } + KeReleaseSpinlock(&gFileCacheLock, flags); + } + } + return STATUS_NOT_FOUND; +} + +static KSTATUS KiFileCacheDropOneStandbyPage(FILE_OBJECT* skipFile, uint64_t skipOffset) { + for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { + MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0; + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; + for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) { + if (!e->Standby || !KiFileCacheIsReclaimCandidateLocked(e, skipFile, skipOffset)) + continue; + + victim = e; + KiFileCacheRemoveEntryLocked(b, prev, e); + gFileCacheStats.Drops++; + break; + } + KeReleaseSpinlock(&gFileCacheLock, flags); + if (victim) { + KiFileCacheFreeEntry(victim); + return STATUS_SUCCESS; + } + } + return STATUS_NOT_FOUND; +} + +static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOffset) { + KSTATUS s = KiFileCacheDropOneStandbyPage(skipFile, skipOffset); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + + s = KiFileCacheDemoteOnePage(skipFile, skipOffset); + if (KSUCCESS(s)) { + s = KiFileCacheDropOneStandbyPage(skipFile, skipOffset); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + } + + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + gFileCacheStats.DropFailures++; + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_NOT_FOUND; +} + + +static void KiFileCacheAddReadAheadStat(uint64_t* field, uint64_t value) { + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + *field += value; + KeReleaseSpinlock(&gFileCacheLock, flags); +} + +// Wakes the writeback worker after dirty pressure crosses the high watermark. +static void KiFileCacheSignalWriteback(void) { + if (gWritebackSemaphore) + KeSignalSemaphore(gWritebackSemaphore); +} + +// Updates a maximum counter while the file-cache stats lock is held. +static void KiFileCacheMaxStatLocked(uint64_t* field, uint64_t value) { + if (*field < value) + *field = value; +} + +static int KiFileCachePageExistsLocked(FILE_OBJECT* file, uint64_t offset) { + uint32_t bucket = KiFileCacheHash(file, offset); + return KiFileCacheFindLocked(file, offset, bucket) != (MM_FILE_CACHE_ENTRY*)0; +} + +// Finds the lowest dirty page for a file, or globally when file is NULL. +static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) { + MM_FILE_CACHE_ENTRY* best = (MM_FILE_CACHE_ENTRY*)0; + for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { + for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; e = e->Next) { + if (!e->Dirty || e->Writeback || e->Loading || e->Invalidated || !e->Frame) + continue; + if (file && e->File != file) + continue; + if (!best || e->Offset < best->Offset) + best = e; + } + } + return best; +} + +static MM_FILE_CACHE_READAHEAD_REQUEST* KiFileCacheReadAheadDequeue(void) { + uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); + LIST_ENTRY* node = ListPopHead(&gReadAheadQueue); + if (node) { + MM_FILE_CACHE_READAHEAD_REQUEST* req = CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link); + if (gReadAheadQueuedRequests) + gReadAheadQueuedRequests--; + if (gReadAheadQueuedPages >= req->Pages) + gReadAheadQueuedPages -= req->Pages; + else + gReadAheadQueuedPages = 0; + } + KeReleaseSpinlock(&gReadAheadLock, flags); + + if (!node) + return (MM_FILE_CACHE_READAHEAD_REQUEST*)0; + return CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link); +} + +static void KiFileCacheReadAheadWorker(void* arg) { + (void)arg; + KeAssertIrql(PASSIVE_LEVEL); + + for (;;) { + KSTATUS ws = KeWaitSemaphore(gReadAheadSemaphore); + if (!KSUCCESS(ws)) { + KeYield(); + continue; + } + + MM_FILE_CACHE_READAHEAD_REQUEST* req = KiFileCacheReadAheadDequeue(); + if (!req) + continue; + + for (uint32_t i = 0; i < req->Pages; ++i) { + uint64_t offset = req->Offset + ((uint64_t)i * PAGE_SIZE); + + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + if (gFileCacheResidentPages + MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES >= + MM_FILE_CACHE_SOFT_MAX_PAGES) { + gFileCacheStats.ReadAheadPagesThrottled += (uint64_t)(req->Pages - i); + KeReleaseSpinlock(&gFileCacheLock, flags); + break; + } + int exists = KiFileCachePageExistsLocked(req->File, offset); + if (exists) + gFileCacheStats.ReadAheadPagesSkipped++; + KeReleaseSpinlock(&gFileCacheLock, flags); + if (exists) + continue; + + MM_FILE_CACHE_PAGE page; + KSTATUS s = MmFileCacheGetPage(req->File, offset, &page); + flags = KeAcquireSpinlock(&gFileCacheLock); + if (KSUCCESS(s)) + gFileCacheStats.ReadAheadPagesLoaded++; + else + gFileCacheStats.ReadAheadFailures++; + KeReleaseSpinlock(&gFileCacheLock, flags); + + if (!KSUCCESS(s)) + break; + MmFileCacheReleasePage(&page); + } + + ObDereferenceObject(&req->File->Header); + MmFreePool(req); + } +} + +static KSTATUS KiFileCacheEnsureReadAheadStarted(void) { + uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); + if (gReadAheadStartState == 2) { + KeReleaseSpinlock(&gReadAheadLock, flags); + return STATUS_SUCCESS; + } + if (gReadAheadStartState == 1) { + KeReleaseSpinlock(&gReadAheadLock, flags); + while (gReadAheadStartState == 1) + KeYield(); + return gReadAheadStartState == 2 ? STATUS_SUCCESS : STATUS_FAILURE; + } + + gReadAheadStartState = 1; + KeReleaseSpinlock(&gReadAheadLock, flags); + + LIST_INIT(&gReadAheadQueue); + gReadAheadQueuedRequests = 0; + gReadAheadQueuedPages = 0; + gReadAheadQueueReady = 1; + + KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gReadAheadSemaphore); + if (KSUCCESS(s)) { + THREAD* worker = (THREAD*)0; + s = KeCreateKernelThread((uintptr_t)KiFileCacheReadAheadWorker, + 0, + PriorityNormal, + &worker); + (void)worker; + } + + flags = KeAcquireSpinlock(&gReadAheadLock); + gReadAheadStartState = KSUCCESS(s) ? 2 : 0; + if (!KSUCCESS(s)) + gReadAheadQueueReady = 0; + KeReleaseSpinlock(&gReadAheadLock, flags); + return s; +} + +// Flushes the next dirty cache page for file, or globally when file is NULL. +static KSTATUS KiFileCacheFlushOneDirtyPage(FILE_OBJECT* file, int* didWork) { + if (didWork) *didWork = 0; + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + MM_FILE_CACHE_ENTRY* e = KiFileCacheFindLowestDirtyLocked(file); + if (!e) { + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_NOT_FOUND; + } + + KSTATUS rs = MmPfnRetain(e->Frame); + if (!KSUCCESS(rs)) { + gFileCacheStats.WritebackFailures++; + KeReleaseSpinlock(&gFileCacheLock, flags); + return rs; + } + + FILE_OBJECT* target = e->File; + ObReferenceObject(&target->Header); + PHYSICAL_ADDRESS frame = e->Frame; + uint64_t offset = e->Offset; + uint32_t validBytes = e->ValidBytes ? e->ValidBytes : PAGE_SIZE; + uint64_t sequence = e->DirtySequence; + e->Writeback = 1; + gFileCacheStats.WritebackRequests++; + KeReleaseSpinlock(&gFileCacheLock, flags); + + void* page = MmPfnToVirtual(frame); + size_t written = 0; + KSTATUS s = STATUS_FAILURE; + if (page && target->Fs && target->Fs->Write) + s = target->Fs->Write(target->Fs, target->FileCtx, offset, page, validBytes, &written); + if (KSUCCESS(s) && written != validBytes) + s = STATUS_FAILURE; + + flags = KeAcquireSpinlock(&gFileCacheLock); + uint32_t bucket = KiFileCacheHash(target, offset); + MM_FILE_CACHE_ENTRY* cur = KiFileCacheFindLocked(target, offset, bucket); + if (cur && cur->Frame == frame) { + if (KSUCCESS(s) && cur->Dirty && cur->DirtySequence == sequence) { + KiFileCacheSetDirtyLocked(cur, 0); + gFileCacheStats.WritebackPages++; + gFileCacheStats.WritebackBytes += validBytes; + } else if (!KSUCCESS(s)) { + gFileCacheStats.WritebackFailures++; + } + cur->Writeback = 0; + } + KeReleaseSpinlock(&gFileCacheLock, flags); + + (void)MmPfnRelease(frame); + ObDereferenceObject(&target->Header); + if (didWork) *didWork = 1; + return s; +} + +// Background writeback worker drains dirty cache pressure best-effort. +static void KiFileCacheWritebackWorker(void* arg) { + (void)arg; + KeAssertIrql(PASSIVE_LEVEL); + + for (;;) { + KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore); + if (!KSUCCESS(ws)) { + KeYield(); + continue; + } + + for (uint32_t i = 0; i < MM_FILE_CACHE_WRITEBACK_BATCH_PAGES; ++i) { + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + int overHigh = gFileCacheDirtyPages >= MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES; + KeReleaseSpinlock(&gFileCacheLock, flags); + if (!overHigh) + break; + + int didWork = 0; + KSTATUS s = KiFileCacheFlushOneDirtyPage((FILE_OBJECT*)0, &didWork); + if (!didWork || !KSUCCESS(s)) + break; + } + } +} + +// Starts the writeback worker once. +static KSTATUS KiFileCacheEnsureWritebackStarted(void) { + uintptr_t flags = KeAcquireSpinlock(&gWritebackLock); + if (gWritebackStartState == 2) { + KeReleaseSpinlock(&gWritebackLock, flags); + return STATUS_SUCCESS; + } + if (gWritebackStartState == 1) { + KeReleaseSpinlock(&gWritebackLock, flags); + while (gWritebackStartState == 1) + KeYield(); + return gWritebackStartState == 2 ? STATUS_SUCCESS : STATUS_FAILURE; + } + + gWritebackStartState = 1; + KeReleaseSpinlock(&gWritebackLock, flags); + + KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gWritebackSemaphore); + if (KSUCCESS(s)) { + THREAD* worker = (THREAD*)0; + s = KeCreateKernelThread((uintptr_t)KiFileCacheWritebackWorker, + 0, + PriorityNormal, + &worker); + (void)worker; + } + + flags = KeAcquireSpinlock(&gWritebackLock); + gWritebackStartState = KSUCCESS(s) ? 2 : 0; + KeReleaseSpinlock(&gWritebackLock, flags); + return s; +} + +static size_t KiMinSize(size_t a, size_t b) { return a < b ? a : b; } + +static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_PAGE* outPage, int waitForLoading) { + if (outPage) + memset(outPage, 0, sizeof(*outPage)); + + uint32_t bucket = KiFileCacheHash(file, aligned); + int counted = 0; + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + if (!counted) { + gFileCacheStats.Lookups++; + counted = 1; + } + + MM_FILE_CACHE_ENTRY* hit = KiFileCacheFindLocked(file, aligned, bucket); + if (!hit) { + gFileCacheStats.Misses++; + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_NOT_FOUND; + } + + if (hit->Loading) { + if (!waitForLoading) { + gFileCacheStats.Misses++; + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_PENDING; + } + + gFileCacheStats.InflightWaits++; + KeReleaseSpinlock(&gFileCacheLock, flags); + KeYield(); + continue; + } + + if (hit->Invalidated || !hit->Frame) { + gFileCacheStats.Misses++; + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_NOT_FOUND; + } + + if (hit->Standby) { + KSTATUS as = MmPfnActivate(hit->Frame); + if (KSUCCESS(as)) + KiFileCacheActivateEntryLocked(hit); + } + + KSTATUS rs = STATUS_SUCCESS; + if (outPage) + rs = MmPfnRetain(hit->Frame); + if (KSUCCESS(rs)) { + hit->Referenced = 1; + gFileCacheStats.Hits++; + if (outPage) { + outPage->File = file; + outPage->Offset = aligned; + outPage->Frame = hit->Frame; + outPage->ValidBytes = hit->ValidBytes; + } + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_SUCCESS; + } + + // A retained cache entry should never point at an invalid frame, but do not + // return corrupt data if the PFN layer rejects it. + gFileCacheStats.Misses++; + KeReleaseSpinlock(&gFileCacheLock, flags); + return rs; + } +} + +static void KiFileCacheKeepBounded(FILE_OBJECT* skipFile, uint64_t skipOffset) { + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + int over = gFileCacheResidentPages > MM_FILE_CACHE_SOFT_MAX_PAGES; + KeReleaseSpinlock(&gFileCacheLock, flags); + if (!over) break; + if (!KSUCCESS(KiFileCacheReclaimOnePage(skipFile, skipOffset))) break; + } +} + +static MM_FILE_CACHE_ENTRY* KiFileCacheAllocateLoadingEntry(FILE_OBJECT* file, uint64_t aligned) { + MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)MmAllocatePoolTag(sizeof(MM_FILE_CACHE_ENTRY), MM_POOL_TAG('p', 'c', 'a', 'c')); + if (!entry) + return (MM_FILE_CACHE_ENTRY*)0; + + memset(entry, 0, sizeof(*entry)); + entry->File = file; + entry->Offset = aligned; + entry->Referenced = 1; + entry->Loading = 1; + ObReferenceObject(&file->Header); + return entry; +} + +static void KiFileCacheFreeLoadingEntry(MM_FILE_CACHE_ENTRY* entry) { + if (!entry) + return; + entry->Frame = 0; + entry->Loading = 0; + KiFileCacheFreeEntry(entry); +} + +static int KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry) { + if (!entry) + return 0; + + uint32_t bucket = KiFileCacheHash(entry->File, entry->Offset); + MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; + for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[bucket]; e; prev = e, e = e->Next) { + if (e == entry) { + KiFileCacheRemoveEntryLocked(bucket, prev, e); + return 1; + } + } + return 0; +} + +static KSTATUS KiFileCacheReserveLoadingPage(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_ENTRY** outEntry) { + if (outEntry) + *outEntry = (MM_FILE_CACHE_ENTRY*)0; + + MM_FILE_CACHE_ENTRY* entry = KiFileCacheAllocateLoadingEntry(file, aligned); + if (!entry) + return STATUS_NO_MEMORY; + + uint32_t bucket = KiFileCacheHash(file, aligned); + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + if (KiFileCacheFindLocked(file, aligned, bucket)) { + KeReleaseSpinlock(&gFileCacheLock, flags); + KiFileCacheFreeLoadingEntry(entry); + return STATUS_ALREADY_EXISTS; + } + + entry->Next = gFileCacheBuckets[bucket]; + gFileCacheBuckets[bucket] = entry; + KeReleaseSpinlock(&gFileCacheLock, flags); + + if (outEntry) + *outEntry = entry; + return STATUS_SUCCESS; +} + +static void KiFileCacheCancelLoadingPage(MM_FILE_CACHE_ENTRY* entry) { + if (!entry) + return; + + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + int unlinked = KiFileCacheUnlinkEntryLocked(entry); + KeReleaseSpinlock(&gFileCacheLock, flags); + + if (unlinked) + KiFileCacheFreeLoadingEntry(entry); +} + +static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry, + PHYSICAL_ADDRESS frame, + uint32_t validBytes, + MM_FILE_CACHE_PAGE* requestedPage, + uint64_t* publishedCount) { + if (!entry || !frame) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + if (entry->Invalidated) { + (void)KiFileCacheUnlinkEntryLocked(entry); + KeReleaseSpinlock(&gFileCacheLock, flags); + (void)MmPfnRelease(frame); + KiFileCacheFreeLoadingEntry(entry); + return STATUS_NOT_FOUND; + } + + entry->Frame = frame; + entry->ValidBytes = validBytes; + entry->Referenced = 1; + entry->Standby = 0; + entry->Loading = 0; + gFileCacheStats.Loads++; + KiFileCacheBumpResidentLocked(1, 0); + if (publishedCount) + (*publishedCount)++; + + if (requestedPage) { + KSTATUS rs = MmPfnRetain(frame); + if (!KSUCCESS(rs)) { + (void)KiFileCacheUnlinkEntryLocked(entry); + gFileCacheStats.LoadFailures++; + KeReleaseSpinlock(&gFileCacheLock, flags); + KiFileCacheFreeEntry(entry); + return rs; + } + requestedPage->File = entry->File; + requestedPage->Offset = entry->Offset; + requestedPage->Frame = frame; + requestedPage->ValidBytes = validBytes; + } + + KeReleaseSpinlock(&gFileCacheLock, flags); + return STATUS_SUCCESS; +} + + +static uint32_t KiFileCacheChooseForegroundBatchPages(void) { + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + uint64_t resident = gFileCacheResidentPages; + KeReleaseSpinlock(&gFileCacheLock, flags); + + if (resident >= MM_FILE_CACHE_SOFT_MAX_PAGES) + return 1; + + uint64_t headroom = MM_FILE_CACHE_SOFT_MAX_PAGES - resident; + if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) + return 1; + if (headroom <= MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES) + return 2; + if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 2u)) + return 4; + if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 4u)) + return 8; + return MM_FILE_CACHE_BATCH_MAX_PAGES; +} + +static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file, + uint64_t aligned, + uint32_t maxPages, + MM_FILE_CACHE_PAGE* requestedPage, + uint32_t* loadedPagesOut) { + if (loadedPagesOut) + *loadedPagesOut = 0; + if (!file || maxPages == 0) + return STATUS_INVALID_PARAMETER; + if (maxPages > MM_FILE_CACHE_BATCH_MAX_PAGES) + maxPages = MM_FILE_CACHE_BATCH_MAX_PAGES; + + uintptr_t statFlags = KeAcquireSpinlock(&gFileCacheLock); + gFileCacheStats.BatchPagesRequested += maxPages; + KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesRequested, maxPages); + KeReleaseSpinlock(&gFileCacheLock, statFlags); + + uintptr_t fileFlags = KeAcquireSpinlock(&file->Lock); + uint64_t fileSize = file->Size; + KeReleaseSpinlock(&file->Lock, fileFlags); + if (aligned >= fileSize) + return STATUS_SUCCESS; + + MM_FILE_CACHE_ENTRY* entries[MM_FILE_CACHE_BATCH_MAX_PAGES]; + PHYSICAL_ADDRESS frames[MM_FILE_CACHE_BATCH_MAX_PAGES]; + memset(entries, 0, sizeof(entries)); + memset(frames, 0, sizeof(frames)); + + uint32_t pages = 0; + KSTATUS reserveStatus = STATUS_SUCCESS; + while (pages < maxPages) { + uint64_t pageOffset = aligned + ((uint64_t)pages * PAGE_SIZE); + if (pageOffset >= fileSize) + break; + + reserveStatus = KiFileCacheReserveLoadingPage(file, pageOffset, &entries[pages]); + if (!KSUCCESS(reserveStatus)) + break; + pages++; + } + + if (pages == 0) { + if (requestedPage) + return KiFileCacheTryHit(file, aligned, requestedPage, 1); + return reserveStatus == STATUS_ALREADY_EXISTS ? STATUS_ALREADY_EXISTS : reserveStatus; + } + + uint8_t* readBuffer = (uint8_t*)MmAllocatePoolTag((size_t)pages * PAGE_SIZE, MM_POOL_TAG('f', 'c', 'b', 'l')); + if (!readBuffer) { + for (uint32_t i = 0; i < pages; ++i) + KiFileCacheCancelLoadingPage(entries[i]); + KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); + return STATUS_NO_MEMORY; + } + memset(readBuffer, 0, (size_t)pages * PAGE_SIZE); + + for (uint32_t i = 0; i < pages; ++i) { + frames[i] = MmPfnAllocate(MmPfnUseCleanFileCache); + if (!frames[i]) { + for (uint32_t j = 0; j < i; ++j) + (void)MmPfnRelease(frames[j]); + for (uint32_t j = 0; j < pages; ++j) + KiFileCacheCancelLoadingPage(entries[j]); + MmFreePool(readBuffer); + KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); + return STATUS_NO_MEMORY; + } + } + + size_t toRead = KiMinSize((size_t)pages * PAGE_SIZE, (size_t)(fileSize - aligned)); + size_t got = 0; + KSTATUS s = STATUS_NOT_SUPPORTED; + if (file->Fs && file->Fs->Read) + s = file->Fs->Read(file->Fs, file->FileCtx, aligned, readBuffer, toRead, &got); + if (!KSUCCESS(s)) { + for (uint32_t i = 0; i < pages; ++i) { + if (frames[i]) + (void)MmPfnRelease(frames[i]); + KiFileCacheCancelLoadingPage(entries[i]); + } + MmFreePool(readBuffer); + KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); + return s; + } + if (got > toRead) + got = toRead; + + uint64_t published = 0; + KSTATUS requestedStatus = STATUS_NOT_FOUND; + for (uint32_t i = 0; i < pages; ++i) { + uint64_t pageOffset = aligned + ((uint64_t)i * PAGE_SIZE); + size_t pageBase = (size_t)i * PAGE_SIZE; + if (pageBase >= got && pageOffset >= fileSize) { + (void)MmPfnRelease(frames[i]); + frames[i] = 0; + KiFileCacheCancelLoadingPage(entries[i]); + continue; + } + + uint32_t validBytes = 0; + if (pageBase < got) { + size_t avail = got - pageBase; + if (avail > PAGE_SIZE) avail = PAGE_SIZE; + validBytes = (uint32_t)avail; + } + + void* page = MmPfnToVirtual(frames[i]); + if (!page) { + (void)MmPfnRelease(frames[i]); + frames[i] = 0; + KiFileCacheCancelLoadingPage(entries[i]); + continue; + } + memcpy(page, readBuffer + pageBase, validBytes); + if (validBytes < PAGE_SIZE) + memset((uint8_t*)page + validBytes, 0, PAGE_SIZE - validBytes); + + MM_FILE_CACHE_PAGE* out = (pageOffset == aligned) ? requestedPage : (MM_FILE_CACHE_PAGE*)0; + KSTATUS ps = KiFileCacheCompleteLoadingPage(entries[i], frames[i], validBytes, out, &published); + if (KSUCCESS(ps)) { + if (pageOffset == aligned) + requestedStatus = STATUS_SUCCESS; + frames[i] = 0; + } else { + if (pageOffset == aligned) + requestedStatus = ps; + frames[i] = 0; + } + } + + MmFreePool(readBuffer); + + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + if (published) { + gFileCacheStats.BatchReads++; + gFileCacheStats.BatchPagesLoaded += published; + KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesLoaded, published); + } + KeReleaseSpinlock(&gFileCacheLock, flags); + + if (loadedPagesOut) + *loadedPagesOut = (uint32_t)published; + + KiFileCacheKeepBounded(file, aligned); + + if (requestedPage) { + if (KSUCCESS(requestedStatus)) + return STATUS_SUCCESS; + return KiFileCacheTryHit(file, aligned, requestedPage, 1); + } + return published ? STATUS_SUCCESS : STATUS_NOT_FOUND; +} + +// Returns a retained cache page suitable for in-place buffered writes. +static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file, + uint64_t aligned, + uint64_t oldFileSize, + int needExistingBytes, + MM_FILE_CACHE_PAGE* outPage) { + if (!file || !outPage) + return STATUS_INVALID_PARAMETER; + memset(outPage, 0, sizeof(*outPage)); + + KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + + if (needExistingBytes && aligned < oldFileSize) { + s = KiFileCacheLoadSpan(file, aligned, 1, outPage, (uint32_t*)0); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + if (s != STATUS_NOT_FOUND) + return s; + } + + MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)0; + s = KiFileCacheReserveLoadingPage(file, aligned, &entry); + if (s == STATUS_ALREADY_EXISTS) + return KiFileCacheTryHit(file, aligned, outPage, 1); + if (!KSUCCESS(s)) + return s; + + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseCleanFileCache); + if (!frame) { + KiFileCacheCancelLoadingPage(entry); + return STATUS_NO_MEMORY; + } + + void* page = MmPfnToVirtual(frame); + if (!page) { + (void)MmPfnRelease(frame); + KiFileCacheCancelLoadingPage(entry); + return STATUS_FAILURE; + } + memset(page, 0, PAGE_SIZE); + + uint32_t validBytes = PAGE_SIZE; + if (oldFileSize > aligned && oldFileSize - aligned < PAGE_SIZE) + validBytes = (uint32_t)(oldFileSize - aligned); + + s = KiFileCacheCompleteLoadingPage(entry, frame, validBytes, outPage, (uint64_t*)0); + if (!KSUCCESS(s)) + return s; + return STATUS_SUCCESS; +} + +// Marks one cache page dirty and updates its valid byte range. +static void KiFileCacheMarkPageDirty(FILE_OBJECT* file, uint64_t aligned, uint32_t validBytes) { + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + uint32_t bucket = KiFileCacheHash(file, aligned); + MM_FILE_CACHE_ENTRY* e = KiFileCacheFindLocked(file, aligned, bucket); + if (e) { + if (validBytes > e->ValidBytes) + e->ValidBytes = validBytes; + if (e->Standby) + KiFileCacheActivateEntryLocked(e); + KiFileCacheSetDirtyLocked(e, 1); + } + uint64_t dirty = gFileCacheDirtyPages; + KeReleaseSpinlock(&gFileCacheLock, flags); + + if (dirty >= MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES) { + if (KSUCCESS(KiFileCacheEnsureWritebackStarted())) + KiFileCacheSignalWriteback(); + } +} + + +KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset, + uint64_t fileSizeBefore, int needExistingBytes, + MM_FILE_CACHE_PAGE* outPage) { + if (!outPage) + return STATUS_INVALID_PARAMETER; + uint64_t aligned = offset & ~(uint64_t)(PAGE_SIZE - 1); + return KiFileCacheGetWritablePage(file, aligned, fileSizeBefore, needExistingBytes, outPage); +} + +void MmFileCacheMarkDirty(FILE_OBJECT* file, uint64_t offset, uint32_t validBytes) { + uint64_t aligned = offset & ~(uint64_t)(PAGE_SIZE - 1); + if (validBytes > PAGE_SIZE) + validBytes = PAGE_SIZE; + if (validBytes == 0) + validBytes = PAGE_SIZE; + KiFileCacheMarkPageDirty(file, aligned, validBytes); + + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + gFileCacheStats.MappedDirtySyncs++; + gFileCacheStats.MappedDirtyPages++; + KeReleaseSpinlock(&gFileCacheLock, flags); +} + +KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf, + size_t size, uint64_t fileSizeBefore, size_t* writtenOut) { + if (writtenOut) *writtenOut = 0; + if (!file || (!buf && size) || !writtenOut) + return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) + return STATUS_INVALID_IRQL; + if (!file->Fs || !file->Fs->Write) + return STATUS_NOT_SUPPORTED; + if (size == 0) + return STATUS_SUCCESS; + + size_t total = 0; + const uint8_t* src = (const uint8_t*)buf; + while (total < size) { + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + uint64_t dirty = gFileCacheDirtyPages; + KeReleaseSpinlock(&gFileCacheLock, flags); + if (dirty >= MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES) { + KSTATUS fs = MmFileCacheFlushFile(file); + if (!KSUCCESS(fs) && total == 0) + return fs; + } + + uint64_t absolute = offset + total; + if (absolute < offset) + break; + uint64_t aligned = absolute & ~(uint64_t)(PAGE_SIZE - 1); + size_t inPage = (size_t)(absolute - aligned); + size_t chunk = PAGE_SIZE - inPage; + if (chunk > size - total) + chunk = size - total; + + int coversFullPage = (inPage == 0 && chunk == PAGE_SIZE); + int needExisting = !coversFullPage; + MM_FILE_CACHE_PAGE page; + KSTATUS s = KiFileCacheGetWritablePage(file, aligned, fileSizeBefore, needExisting, &page); + if (!KSUCCESS(s)) + return total ? STATUS_SUCCESS : s; + + void* dst = MmPfnToVirtual(page.Frame); + if (!dst) { + MmFileCacheReleasePage(&page); + return total ? STATUS_SUCCESS : STATUS_FAILURE; + } + + memcpy((uint8_t*)dst + inPage, src + total, chunk); + uint32_t valid = (uint32_t)(inPage + chunk); + uint64_t pageEnd = aligned + valid; + uint64_t newEnd = offset + total + chunk; + if (pageEnd > newEnd) + valid = (uint32_t)(newEnd - aligned); + KiFileCacheMarkPageDirty(file, aligned, valid); + MmFileCacheReleasePage(&page); + + total += chunk; + } + + *writtenOut = total; + return STATUS_SUCCESS; +} + +KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file) { + if (!file) + return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) + return STATUS_INVALID_IRQL; + + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + gFileCacheStats.FlushFileCalls++; + KeReleaseSpinlock(&gFileCacheLock, flags); + + KSTATUS firstError = STATUS_SUCCESS; + for (;;) { + int didWork = 0; + KSTATUS s = KiFileCacheFlushOneDirtyPage(file, &didWork); + if (!didWork) + break; + if (!KSUCCESS(s) && KSUCCESS(firstError)) + firstError = s; + } + + if (!KSUCCESS(firstError)) { + flags = KeAcquireSpinlock(&gFileCacheLock); + gFileCacheStats.FlushFileFailures++; + KeReleaseSpinlock(&gFileCacheLock, flags); + } + return firstError; +} + +KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAGE* outPage) { + if (!outPage) return STATUS_INVALID_PARAMETER; + memset(outPage, 0, sizeof(*outPage)); + if (!file) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + + uint64_t aligned = offset & ~(uint64_t)(PAGE_SIZE - 1); + KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + return KiFileCacheLoadSpan(file, aligned, KiFileCacheChooseForegroundBatchPages(), outPage, (uint32_t*)0); +} + +void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page) { + if (!page || !page->Frame) return; + (void)MmPfnRelease(page->Frame); + memset(page, 0, sizeof(*page)); +} + +void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) { + if (!file) return; + if (size == 0) return; + uint64_t start = offset & ~(uint64_t)(PAGE_SIZE - 1); + uint64_t end = offset + size; + if (end < offset || size == (size_t)-1) { + end = UINT64_MAX; + } else { + end = (end + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1); + } + + for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { + for (;;) { + MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0; + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; + for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) { + uint64_t pageEnd = e->Offset + PAGE_SIZE; + if (e->File == file && e->Offset < end && pageEnd > start) { + if (e->Loading) { + if (!e->Invalidated) { + e->Invalidated = 1; + gFileCacheStats.Invalidations++; + } + continue; + } + if (e->Dirty || e->Writeback) { + e->Invalidated = 1; + continue; + } + victim = e; + KiFileCacheRemoveEntryLocked(b, prev, e); + gFileCacheStats.Invalidations++; + break; + } + } + KeReleaseSpinlock(&gFileCacheLock, flags); + if (!victim) break; + KiFileCacheFreeEntry(victim); + } + } +} + +void MmFileCacheInvalidateAll(FILE_OBJECT* file) { + if (!file) return; + for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { + for (;;) { + MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0; + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; + for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) { + if (e->File == file) { + if (e->Loading) { + if (!e->Invalidated) { + e->Invalidated = 1; + gFileCacheStats.Invalidations++; + } + continue; + } + if (e->Dirty || e->Writeback) { + e->Invalidated = 1; + continue; + } + victim = e; + KiFileCacheRemoveEntryLocked(b, prev, e); + gFileCacheStats.Invalidations++; + break; + } + } + KeReleaseSpinlock(&gFileCacheLock, flags); + if (!victim) break; + KiFileCacheFreeEntry(victim); + } + } +} + +KSTATUS MmFileCacheDropOnePageForPressure(void) { + return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0); +} + + +KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages) { + if (!file || pages == 0) + return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) + return STATUS_INVALID_IRQL; + + uint32_t requestedPages = pages; + if (pages > MM_FILE_CACHE_READAHEAD_MAX_PAGES) + pages = MM_FILE_CACHE_READAHEAD_MAX_PAGES; + + uintptr_t cacheFlags = KeAcquireSpinlock(&gFileCacheLock); + gFileCacheStats.ReadAheadPagesRequested += requestedPages; + KiFileCacheMaxStatLocked(&gFileCacheStats.MaxReadAheadPagesPerRequest, requestedPages); + uint64_t resident = gFileCacheResidentPages; + uint64_t headroom = resident < MM_FILE_CACHE_SOFT_MAX_PAGES + ? MM_FILE_CACHE_SOFT_MAX_PAGES - resident + : 0; + if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) { + gFileCacheStats.ReadAheadPagesThrottled += pages; + KeReleaseSpinlock(&gFileCacheLock, cacheFlags); + return STATUS_QUEUE_FULL; + } + headroom -= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES; + if ((uint64_t)pages > headroom) + pages = (uint32_t)headroom; + KeReleaseSpinlock(&gFileCacheLock, cacheFlags); + if (pages == 0) + return STATUS_QUEUE_FULL; + + KSTATUS s = KiFileCacheEnsureReadAheadStarted(); + if (!KSUCCESS(s)) { + KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1); + return s; + } + + MM_FILE_CACHE_READAHEAD_REQUEST* req = + (MM_FILE_CACHE_READAHEAD_REQUEST*)MmAllocatePoolTag(sizeof(*req), MM_POOL_TAG('r', 'd', 'a', 'h')); + if (!req) { + KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1); + return STATUS_NO_MEMORY; + } + + LIST_INIT(&req->Link); + req->File = file; + req->Offset = offset & ~(uint64_t)(PAGE_SIZE - 1); + req->Pages = pages; + ObReferenceObject(&file->Header); + + uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); + if (!gReadAheadQueueReady || + gReadAheadQueuedRequests >= MM_FILE_CACHE_READAHEAD_MAX_REQUESTS || + gReadAheadQueuedPages >= MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET) { + KeReleaseSpinlock(&gReadAheadLock, flags); + ObDereferenceObject(&file->Header); + MmFreePool(req); + KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadPagesThrottled, pages); + return STATUS_QUEUE_FULL; + } + uint32_t queueHeadroom = MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET - gReadAheadQueuedPages; + if (pages > queueHeadroom) + pages = queueHeadroom; + req->Pages = pages; + + ListInsertTail(&gReadAheadQueue, &req->Link); + gReadAheadQueuedRequests++; + gReadAheadQueuedPages += pages; + KeReleaseSpinlock(&gReadAheadLock, flags); + + flags = KeAcquireSpinlock(&gFileCacheLock); + gFileCacheStats.ReadAheadRequests++; + gFileCacheStats.ReadAheadPagesQueued += pages; + KeReleaseSpinlock(&gFileCacheLock, flags); + + KeSignalSemaphore(gReadAheadSemaphore); + return STATUS_SUCCESS; +} + +void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out) { + if (!out) return; + uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); + MM_FILE_CACHE_STATS st = gFileCacheStats; + st.ResidentPages = gFileCacheResidentPages; + st.ActivePages = gFileCacheActivePages; + st.StandbyPages = gFileCacheStandbyPages; + st.DirtyPages = gFileCacheDirtyPages; + st.MaxDirtyPages = gFileCacheMaxDirtyPages; + st.MaxResidentPages = gFileCacheMaxResidentPages; + KeReleaseSpinlock(&gFileCacheLock, flags); + *out = st; +} diff --git a/code/kernel/mm_phys.c b/code/kernel/mm_phys.c index cf60293..5bdf0da 100644 --- a/code/kernel/mm_phys.c +++ b/code/kernel/mm_phys.c @@ -1,143 +1,305 @@ -// Physical-memory manager. This file owns frame accounting and boot memory-map consumption. +// Physical frame database. +// This file consumes the boot memory map and builds one PFN descriptor for each +// managed physical frame. The first implementation still allocates by scanning, +// but callers now get typed, reference-counted frames instead of anonymous +// bitmap bits. -// Bitmap allocator over 4 KB physical frames. One bitmap records which PFNs -// exist in usable RAM, and one bitmap records which usable PFNs are currently -// free. This keeps single-frame allocation simple while making contiguous -// allocation and memory visualization deterministic. -#include +#include #include +#include #include -// Physical-to-virtual translation via the direct map window. -static uint64_t gPhysBase = 0; // set by MmInitializePhysical +extern KSTATUS MmVadTrimOnePageForPressure(void); -// Implements the phys To Virt operation. +static uint64_t gPhysBase = 0; +static MM_PFN* gPfnDatabase = (MM_PFN*)0; +static size_t gMaxPfn = 0; +static size_t gTotalFrames = 0; +static size_t gFreeFrames = 0; +static SPINLOCK gPfnLock = SPINLOCK_INIT; +static uint64_t gPfnAllocations = 0; +static uint64_t gPfnFrees = 0; +static uint64_t gPfnRetains = 0; +static uint64_t gPfnReleases = 0; +static uint64_t gPfnPins = 0; +static uint64_t gPfnUnpins = 0; +static uint64_t gPfnFailedAllocations = 0; +static uint64_t gPfnFailedContiguousAllocations = 0; +static uint64_t gPfnFailedInvalidRange = 0; +static uint64_t gPfnFailedEmptyRange = 0; +static uint64_t gPfnFailedReserveDenied = 0; +static uint64_t gPfnFailedNoFrame = 0; +static uint64_t gPfnFailedContiguousInvalidRequest = 0; +static uint64_t gPfnFailedContiguousReserveDenied = 0; +static uint64_t gPfnFailedContiguousNoRun = 0; +static uint64_t gPfnReclaimAttempts = 0; +static uint64_t gPfnReclaimPagesRequested = 0; +static uint64_t gPfnReclaimPagesSucceeded = 0; +static uint64_t gPfnReclaimPagesFailed = 0; +static uint64_t gPfnReclaimReentrantSkips = 0; +static volatile int gPfnReclaiming = 0; + +static PHYSICAL_ADDRESS KiClaimPfnLocked(size_t pfn, MM_PFN_USAGE usage); + +#define MM_PFN_KERNEL_RESERVE_FRAMES 128u +#define MM_PFN_RECLAIM_LOW_WATER 512u +#define MM_PFN_RECLAIM_HIGH_WATER 768u + +// Converts a physical address into the direct-map alias used during early memory setup. static inline void* KiPhysToVirt(PHYSICAL_ADDRESS phys) { - return (void *)(phys + gPhysBase); + return (void*)(phys + gPhysBase); } -// Frame bitmap. - -typedef struct { - uint8_t* Usable; // bit = 1 when PFN belongs to usable RAM - uint8_t* Free; // bit = 1 when PFN is currently free - size_t BitmapBytes; - size_t MaxPfn; - size_t TotalFrames; // usable frames available to the allocator - size_t FreeFrames; - SPINLOCK Lock; -} FRAME_BITMAP; - -static FRAME_BITMAP gFrameBitmap; - -// Implements the align Up operation. +// Rounds a physical address up to the next whole frame boundary. static inline PHYSICAL_ADDRESS KiAlignUp(PHYSICAL_ADDRESS value) { return (value + PAGE_SIZE - 1) & PAGE_MASK; } -// Implements the align Down operation. +// Rounds a physical address down to the previous whole frame boundary. static inline PHYSICAL_ADDRESS KiAlignDown(PHYSICAL_ADDRESS value) { return value & PAGE_MASK; } -// Implements the bit Test operation. -static inline int KiBitTest(const uint8_t* bits, size_t index) { - return (bits[index >> 3] & (uint8_t)(1u << (index & 7))) != 0; +// Returns whether a boot memory type describes RAM that the PFN database should track. +static int KiMemoryTypeManaged(MEMORY_TYPE type) { + return type == MemoryUsable || type == MemoryBootloader || type == MemoryKernel; } -// Implements the bit Set operation. -static inline void KiBitSet(uint8_t* bits, size_t index) { - bits[index >> 3] |= (uint8_t)(1u << (index & 7)); +// Returns whether an allocation may consume the emergency kernel reserve. +static int KiPfnUsageMayUseReserve(MM_PFN_USAGE usage) { + return usage == MmPfnUseKernel || + usage == MmPfnUsePageTable || + usage == MmPfnUseDma || + usage == MmPfnUseBootloader || + usage == MmPfnUsePfnDatabase || + usage == MmPfnUseMmio; } -// Implements the bit Clear operation. -static inline void KiBitClear(uint8_t* bits, size_t index) { - bits[index >> 3] &= (uint8_t)~(1u << (index & 7)); +// Converts allocation purpose into the initial PFN state. +static MM_PFN_STATE KiAllocatedStateForUsage(MM_PFN_USAGE usage) { + (void)usage; + return MmPfnStateActive; } -// Implements the pfn Usable Locked operation. -static inline int KiPfnUsableLocked(size_t pfn) { - return pfn < gFrameBitmap.MaxPfn && KiBitTest(gFrameBitmap.Usable, pfn); +// Marks a PFN as used by a permanent structure that must never be returned to the free list. +static void KiReservePfnLocked(size_t pfn, MM_PFN_USAGE usage) { + if (pfn >= gMaxPfn) + return; + + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) { + if (gFreeFrames) + gFreeFrames--; + } + + if (entry->State == MmPfnStateInvalid) + gTotalFrames++; + + entry->State = MmPfnStateActive; + entry->Usage = usage; + entry->RefCount = 1; + entry->PinCount = 1; + entry->Flags = 0; + entry->MappedCount = 0; + entry->Owner = (void*)0; + entry->BackingPage = 0; } -// Implements the pfn Free Locked operation. -static inline int KiPfnFreeLocked(size_t pfn) { - return KiPfnUsableLocked(pfn) && KiBitTest(gFrameBitmap.Free, pfn); -} +// Marks a page-aligned RAM range as managed by the PFN database. +static void KiRegisterManagedRange(PHYSICAL_ADDRESS base, + PHYSICAL_ADDRESS end, + MM_PFN_STATE state, + MM_PFN_USAGE usage) { + size_t firstPfn = (size_t)(base / PAGE_SIZE); + size_t count = (size_t)((end - base) / PAGE_SIZE); -// Implements the mark Usable Free Range operation. -static void KiMarkUsableFreeRange(size_t firstPfn, size_t count) { for (size_t i = 0; i < count; i++) { size_t pfn = firstPfn + i; - if (pfn >= gFrameBitmap.MaxPfn) break; - if (!KiBitTest(gFrameBitmap.Usable, pfn)) { - KiBitSet(gFrameBitmap.Usable, pfn); - gFrameBitmap.TotalFrames++; - } - if (!KiBitTest(gFrameBitmap.Free, pfn)) { - KiBitSet(gFrameBitmap.Free, pfn); - gFrameBitmap.FreeFrames++; + if (pfn >= gMaxPfn) + break; + + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateInvalid) + gTotalFrames++; + + if (entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) { + if (gFreeFrames) + gFreeFrames--; } + + entry->State = state; + entry->Usage = usage; + entry->RefCount = (state == MmPfnStateFree || state == MmPfnStateZeroed) ? 0 : 1; + entry->PinCount = 0; + entry->Flags = 0; + entry->MappedCount = 0; + entry->Owner = (void*)0; + entry->BackingPage = 0; + + if (state == MmPfnStateFree || state == MmPfnStateZeroed) + gFreeFrames++; } } -// Releases usable Pfn. -static void KiRemoveUsablePfn(size_t pfn) { - if (pfn >= gFrameBitmap.MaxPfn) return; - if (KiBitTest(gFrameBitmap.Free, pfn)) { - KiBitClear(gFrameBitmap.Free, pfn); - if (gFrameBitmap.FreeFrames) gFrameBitmap.FreeFrames--; - } - if (KiBitTest(gFrameBitmap.Usable, pfn)) { - KiBitClear(gFrameBitmap.Usable, pfn); - if (gFrameBitmap.TotalFrames) gFrameBitmap.TotalFrames--; - } + +// Returns whether the future page-replacement code may consider this frame a candidate. +static int KiPfnEvictableLocked(const MM_PFN* entry) { + if (!entry || entry->PinCount) + return 0; + if (entry->State != MmPfnStateActive && entry->State != MmPfnStateStandby && entry->State != MmPfnStateModified) + return 0; + return entry->Usage == MmPfnUseUser || + entry->Usage == MmPfnUseCleanFileCache || + entry->Usage == MmPfnUseImage || + entry->Usage == MmPfnUseMappedFile || + entry->Usage == MmPfnUseAnonymous || + entry->Usage == MmPfnUseCow; } -// Initialisation. + + +// Returns whether a standby frame can be counted as immediately reclaimable. +static int KiPfnReclaimableLocked(const MM_PFN* entry) { + if (!entry || entry->PinCount || entry->MappedCount) + return 0; + if (entry->State != MmPfnStateStandby || entry->RefCount != 1) + return 0; + return entry->Usage == MmPfnUseCleanFileCache || entry->Usage == MmPfnUseImage; +} + +// Returns true when a PFN can satisfy a new allocation. +static int KiPfnAvailableLocked(size_t pfn) { + if (pfn >= gMaxPfn) + return 0; + MM_PFN* entry = &gPfnDatabase[pfn]; + return entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed; +} + +// Tries to create breathing room by swapping private user pages before an allocation fails. +static void KiPfnReclaimForPressure(void) { + if (!KeCanWait()) + return; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + if (gPfnReclaiming) { + gPfnReclaimReentrantSkips++; + KeReleaseSpinlock(&gPfnLock, flags); + return; + } + if (gFreeFrames >= MM_PFN_RECLAIM_LOW_WATER) { + KeReleaseSpinlock(&gPfnLock, flags); + return; + } + + gPfnReclaiming = 1; + gPfnReclaimAttempts++; + KeReleaseSpinlock(&gPfnLock, flags); + + for (uint32_t i = 0; i < 64; ++i) { + flags = KeAcquireSpinlock(&gPfnLock); + if (gFreeFrames >= MM_PFN_RECLAIM_HIGH_WATER) { + KeReleaseSpinlock(&gPfnLock, flags); + break; + } + gPfnReclaimPagesRequested++; + KeReleaseSpinlock(&gPfnLock, flags); + + KSTATUS s = MmVadTrimOnePageForPressure(); + + flags = KeAcquireSpinlock(&gPfnLock); + if (KSUCCESS(s)) + gPfnReclaimPagesSucceeded++; + else + gPfnReclaimPagesFailed++; + KeReleaseSpinlock(&gPfnLock, flags); + + if (!KSUCCESS(s)) + break; + } + + flags = KeAcquireSpinlock(&gPfnLock); + gPfnReclaiming = 0; + KeReleaseSpinlock(&gPfnLock, flags); +} + +// Claims a free PFN in the requested range while the PFN lock is held. +static PHYSICAL_ADDRESS KiClaimAvailableInRangeLocked(size_t firstPfn, size_t limitPfn, MM_PFN_USAGE usage) { + for (size_t pfn = limitPfn; pfn > firstPfn; pfn--) { + size_t candidate = pfn - 1; + if (KiPfnAvailableLocked(candidate)) + return KiClaimPfnLocked(candidate, usage); + } + return (PHYSICAL_ADDRESS)0; +} + +// Claims one free PFN for the requested usage. +static PHYSICAL_ADDRESS KiClaimPfnLocked(size_t pfn, MM_PFN_USAGE usage) { + MM_PFN* entry = &gPfnDatabase[pfn]; + entry->State = KiAllocatedStateForUsage(usage); + entry->Usage = usage; + entry->RefCount = 1; + entry->PinCount = (usage == MmPfnUsePageTable) ? 1 : 0; + entry->Flags = 0; + entry->MappedCount = 0; + entry->Owner = (void*)0; + entry->BackingPage = 0; + if (gFreeFrames) + gFreeFrames--; + gPfnAllocations++; + return MmPfnAddress(pfn); +} + +// Initializes the PFN database from the bootloader memory map. bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) { - if (!map) return false; + if (!map) + return false; + gPhysBase = physicalWindowBase; PHYSICAL_ADDRESS maxEnd = 0; for (uint32_t i = 0; i < map->RegionCount; i++) { const MEMORY_REGION* r = &map->Regions[i]; - if (r->Type != MemoryUsable) continue; + if (!KiMemoryTypeManaged(r->Type)) + continue; PHYSICAL_ADDRESS base = KiAlignUp(r->Base); PHYSICAL_ADDRESS end = r->Base + r->Length; - if (end < r->Base) end = (PHYSICAL_ADDRESS)-1; + if (end < r->Base) + end = (PHYSICAL_ADDRESS)-1; end = KiAlignDown(end); - if (end <= base) continue; - if (end > maxEnd) maxEnd = end; + if (end <= base) + continue; + if (end > maxEnd) + maxEnd = end; } - size_t maxPfn = (size_t)(maxEnd / PAGE_SIZE); - if (!maxPfn) return false; + gMaxPfn = (size_t)(maxEnd / PAGE_SIZE); + if (!gMaxPfn) + return false; - size_t bitmapBytes = (maxPfn + 7) / 8; - size_t metadataBytes = bitmapBytes * 2; - size_t metadataPages = (metadataBytes + PAGE_SIZE - 1) / PAGE_SIZE; - PHYSICAL_ADDRESS metadataSize = (PHYSICAL_ADDRESS)metadataPages * PAGE_SIZE; + size_t pfnBytes = gMaxPfn * sizeof(MM_PFN); + size_t pfnPages = (pfnBytes + PAGE_SIZE - 1) / PAGE_SIZE; + PHYSICAL_ADDRESS metadataSize = (PHYSICAL_ADDRESS)pfnPages * PAGE_SIZE; - // Keep allocator metadata out of low memory. The bitmap is much smaller - // than the old free-frame stack, so a first-fit search can otherwise choose - // a conventional-memory hole below 1 MiB. That range is still useful for - // AP startup/legacy scratch, and stealing it makes early boot failures - // unnecessarily silent because PMM initializes before normal debug output. const PHYSICAL_ADDRESS metadataMin = 0x100000ULL; PHYSICAL_ADDRESS metadataPhys = 0; int metadataFound = 0; + for (uint32_t i = 0; i < map->RegionCount; i++) { const MEMORY_REGION* r = &map->Regions[i]; - if (r->Type != MemoryUsable) continue; + if (r->Type != MemoryUsable) + continue; PHYSICAL_ADDRESS base = KiAlignUp(r->Base); - if (base < metadataMin) base = metadataMin; + if (base < metadataMin) + base = metadataMin; PHYSICAL_ADDRESS end = r->Base + r->Length; - if (end < r->Base) end = (PHYSICAL_ADDRESS)-1; + if (end < r->Base) + end = (PHYSICAL_ADDRESS)-1; end = KiAlignDown(end); - if (end <= base) continue; + if (end <= base) + continue; + if (end - base >= metadataSize) { metadataPhys = base; metadataFound = 1; @@ -145,155 +307,529 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) { } } - if (!metadataFound) return false; + if (!metadataFound) + return false; - gFrameBitmap.BitmapBytes = bitmapBytes; - gFrameBitmap.MaxPfn = maxPfn; - gFrameBitmap.TotalFrames = 0; - gFrameBitmap.FreeFrames = 0; - gFrameBitmap.Lock = (SPINLOCK)SPINLOCK_INIT; - gFrameBitmap.Usable = (uint8_t*)KiPhysToVirt(metadataPhys); - gFrameBitmap.Free = gFrameBitmap.Usable + bitmapBytes; + gPfnDatabase = (MM_PFN*)KiPhysToVirt(metadataPhys); + memset(gPfnDatabase, 0, metadataSize); + gTotalFrames = 0; + gFreeFrames = 0; + gPfnLock = (SPINLOCK)SPINLOCK_INIT; + gPfnAllocations = 0; + gPfnFrees = 0; + gPfnRetains = 0; + gPfnReleases = 0; + gPfnPins = 0; + gPfnUnpins = 0; + gPfnFailedAllocations = 0; + gPfnFailedContiguousAllocations = 0; + gPfnFailedInvalidRange = 0; + gPfnFailedEmptyRange = 0; + gPfnFailedReserveDenied = 0; + gPfnFailedNoFrame = 0; + gPfnFailedContiguousInvalidRequest = 0; + gPfnFailedContiguousReserveDenied = 0; + gPfnFailedContiguousNoRun = 0; + gPfnReclaimAttempts = 0; + gPfnReclaimPagesRequested = 0; + gPfnReclaimPagesSucceeded = 0; + gPfnReclaimPagesFailed = 0; + gPfnReclaimReentrantSkips = 0; - memset(gFrameBitmap.Usable, 0, metadataBytes); - - // Mark page-aligned usable RAM as free. Unaligned tails are ignored rather - // than handing malformed frame addresses to the rest of the kernel. for (uint32_t i = 0; i < map->RegionCount; i++) { const MEMORY_REGION* r = &map->Regions[i]; - if (r->Type != MemoryUsable) continue; + if (!KiMemoryTypeManaged(r->Type)) + continue; PHYSICAL_ADDRESS base = KiAlignUp(r->Base); PHYSICAL_ADDRESS end = r->Base + r->Length; - if (end < r->Base) end = (PHYSICAL_ADDRESS)-1; + if (end < r->Base) + end = (PHYSICAL_ADDRESS)-1; end = KiAlignDown(end); - if (end <= base) continue; + if (end <= base) + continue; - size_t firstPfn = (size_t)(base / PAGE_SIZE); - size_t count = (size_t)((end - base) / PAGE_SIZE); - KiMarkUsableFreeRange(firstPfn, count); + if (r->Type == MemoryUsable) { + KiRegisterManagedRange(base, end, MmPfnStateFree, MmPfnUseNone); + } else if (r->Type == MemoryBootloader) { + KiRegisterManagedRange(base, end, MmPfnStateActive, MmPfnUseBootloader); + } else { + KiRegisterManagedRange(base, end, MmPfnStateActive, MmPfnUseKernel); + for (size_t pfn = (size_t)(base / PAGE_SIZE); pfn < (size_t)(end / PAGE_SIZE) && pfn < gMaxPfn; pfn++) + gPfnDatabase[pfn].PinCount = 1; + } } - // PFN 0 is never a valid allocation result because 0 is the failure value. - KiRemoveUsablePfn(0); + // PFN 0 remains reserved because a zero physical address is the allocation failure value. + KiReservePfnLocked(0, MmPfnUseKernel); - // Hide the PMM metadata pages from the allocator entirely. They are not - // merely allocated; they are structural storage that must not become free if - // some caller accidentally passes their PFNs to MmFreeFrame(). + // The PFN database owns its storage permanently. It is structural metadata, + // not an allocation that can later be freed through a stale physical address. size_t metadataFirstPfn = (size_t)(metadataPhys / PAGE_SIZE); - for (size_t i = 0; i < metadataPages; i++) - KiRemoveUsablePfn(metadataFirstPfn + i); + for (size_t i = 0; i < pfnPages; i++) + KiReservePfnLocked(metadataFirstPfn + i, MmPfnUsePfnDatabase); return true; } -// Public API. -PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive, - PHYSICAL_ADDRESS maxExclusive) { +// Converts a physical frame address to a PFN index. +size_t MmPfnIndex(PHYSICAL_ADDRESS frame) { + if (frame & (PAGE_SIZE - 1)) + return MM_PFN_INVALID_INDEX; + size_t pfn = (size_t)(frame / PAGE_SIZE); + if (pfn >= gMaxPfn) + return MM_PFN_INVALID_INDEX; + return pfn; +} + +// Converts a PFN index to a physical frame address. +PHYSICAL_ADDRESS MmPfnAddress(size_t pfn) { + return (PHYSICAL_ADDRESS)pfn * PAGE_SIZE; +} + +// Returns the PFN descriptor for a page-aligned physical frame. +__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return (MM_PFN*)0; + return &gPfnDatabase[pfn]; +} + + +// Returns the direct-map virtual address for a managed physical frame. +__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame) { + if (MmPfnIndex(frame) == MM_PFN_INVALID_INDEX) + return (void*)0; + return KiPhysToVirt(frame); +} + +// Allocates one physical frame for the requested kernel subsystem. +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage) { + return MmPfnAllocateInRange(PAGE_SIZE, (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE, usage); +} + +// Allocates one physical frame below a physical address limit. +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage) { + return MmPfnAllocateInRange(PAGE_SIZE, maxExclusive, usage); +} + +// Allocates one physical frame inside a physical address range. +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive, + MM_PFN_USAGE usage) { minInclusive = KiAlignUp(minInclusive); maxExclusive = KiAlignDown(maxExclusive); - if (maxExclusive <= minInclusive) return (PHYSICAL_ADDRESS)0; + if (maxExclusive <= minInclusive) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedAllocations++; + gPfnFailedInvalidRange++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } size_t firstPfn = (size_t)(minInclusive / PAGE_SIZE); size_t limitPfn = (size_t)(maxExclusive / PAGE_SIZE); - if (limitPfn > gFrameBitmap.MaxPfn) limitPfn = gFrameBitmap.MaxPfn; - if (firstPfn >= limitPfn) return (PHYSICAL_ADDRESS)0; - - uintptr_t flags = KeAcquireSpinlock(&gFrameBitmap.Lock); - - for (size_t pfn = limitPfn; pfn > firstPfn; pfn--) { - size_t candidate = pfn - 1; - if (KiPfnFreeLocked(candidate)) { - KiBitClear(gFrameBitmap.Free, candidate); - if (gFrameBitmap.FreeFrames) gFrameBitmap.FreeFrames--; - KeReleaseSpinlock(&gFrameBitmap.Lock, flags); - return (PHYSICAL_ADDRESS)candidate * PAGE_SIZE; - } + if (limitPfn > gMaxPfn) + limitPfn = gMaxPfn; + if (firstPfn >= limitPfn) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedAllocations++; + gPfnFailedEmptyRange++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; } - KeReleaseSpinlock(&gFrameBitmap.Lock, flags); + KiPfnReclaimForPressure(); + + int mayUseReserve = KiPfnUsageMayUseReserve(usage); + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + if (!mayUseReserve && gFreeFrames <= MM_PFN_KERNEL_RESERVE_FRAMES) { + KeReleaseSpinlock(&gPfnLock, flags); + } else { + PHYSICAL_ADDRESS phys = KiClaimAvailableInRangeLocked(firstPfn, limitPfn, usage); + if (phys) { + KeReleaseSpinlock(&gPfnLock, flags); + return phys; + } + KeReleaseSpinlock(&gPfnLock, flags); + } + + KiPfnReclaimForPressure(); + + flags = KeAcquireSpinlock(&gPfnLock); + if (!mayUseReserve && gFreeFrames <= MM_PFN_KERNEL_RESERVE_FRAMES) { + gPfnFailedAllocations++; + gPfnFailedReserveDenied++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + + PHYSICAL_ADDRESS phys = KiClaimAvailableInRangeLocked(firstPfn, limitPfn, usage); + if (phys) { + KeReleaseSpinlock(&gPfnLock, flags); + return phys; + } + gPfnFailedAllocations++; + gPfnFailedNoFrame++; + KeReleaseSpinlock(&gPfnLock, flags); return (PHYSICAL_ADDRESS)0; } -PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) { - return MmAllocateFrameInRange(PAGE_SIZE, maxExclusive); -} - -// Creates frame. -__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void) { - return MmAllocateFrameInRange(PAGE_SIZE, (PHYSICAL_ADDRESS)gFrameBitmap.MaxPfn * PAGE_SIZE); -} - -// Releases frame. -void MmFreeFrame(PHYSICAL_ADDRESS frame) { - if (!frame || (frame & (PAGE_SIZE - 1))) return; - size_t pfn = (size_t)(frame / PAGE_SIZE); - - uintptr_t flags = KeAcquireSpinlock(&gFrameBitmap.Lock); - if (KiPfnUsableLocked(pfn) && !KiBitTest(gFrameBitmap.Free, pfn)) { - KiBitSet(gFrameBitmap.Free, pfn); - gFrameBitmap.FreeFrames++; +// Allocates a physically contiguous run for DMA or hardware structures. +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) { + if (!count || count > gMaxPfn) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; } - KeReleaseSpinlock(&gFrameBitmap.Lock, flags); -} -// Creates contiguous. -__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) { - if (!count || count > gFrameBitmap.MaxPfn) return (PHYSICAL_ADDRESS)0; + if (!KiPfnUsageMayUseReserve(usage)) + KiPfnReclaimForPressure(); - uintptr_t flags = KeAcquireSpinlock(&gFrameBitmap.Lock); + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + if (!KiPfnUsageMayUseReserve(usage) && gFreeFrames < MM_PFN_KERNEL_RESERVE_FRAMES + count) { + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousReserveDenied++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } size_t runStart = 0; size_t runLength = 0; - for (size_t pfn = 1; pfn < gFrameBitmap.MaxPfn; pfn++) { - if (KiPfnFreeLocked(pfn)) { - if (!runLength) runStart = pfn; + + for (size_t pfn = 1; pfn < gMaxPfn; pfn++) { + if (KiPfnAvailableLocked(pfn)) { + if (!runLength) + runStart = pfn; runLength++; if (runLength == count) { for (size_t i = 0; i < count; i++) - KiBitClear(gFrameBitmap.Free, runStart + i); - if (gFrameBitmap.FreeFrames >= count) gFrameBitmap.FreeFrames -= count; - else gFrameBitmap.FreeFrames = 0; - KeReleaseSpinlock(&gFrameBitmap.Lock, flags); - return (PHYSICAL_ADDRESS)runStart * PAGE_SIZE; + KiClaimPfnLocked(runStart + i, usage); + KeReleaseSpinlock(&gPfnLock, flags); + return MmPfnAddress(runStart); } } else { runLength = 0; } } - KeReleaseSpinlock(&gFrameBitmap.Lock, flags); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousNoRun++; + KeReleaseSpinlock(&gPfnLock, flags); return (PHYSICAL_ADDRESS)0; } -// Releases frame Count. +// Increments the reference count on an allocated frame. +__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + if (entry->RefCount == UINT16_MAX) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_FAILURE; + } + entry->RefCount++; + gPfnRetains++; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + +// Releases a reference and returns the frame to the free list when the last owner leaves. +__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + if (entry->PinCount) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_ACCESS_DENIED; + } + gPfnReleases++; + if (entry->RefCount > 1) { + entry->RefCount--; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; + } + if (entry->MappedCount) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_ACCESS_DENIED; + } + + entry->State = MmPfnStateFree; + entry->Usage = MmPfnUseNone; + entry->RefCount = 0; + entry->PinCount = 0; + entry->Flags = 0; + entry->MappedCount = 0; + entry->Owner = (void*)0; + entry->BackingPage = 0; + gFreeFrames++; + gPfnFrees++; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + +// Frees a frame that is expected to have a single owner. +__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame) { + return MmPfnRelease(frame); +} + +// Pins a frame so later page replacement cannot evict it. +__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + if (entry->PinCount == UINT16_MAX) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_FAILURE; + } + entry->PinCount++; + gPfnPins++; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + +// Removes one pin from a frame. +__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (!entry->PinCount) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + entry->PinCount--; + gPfnUnpins++; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + +// Records that a VAD now maps this frame into a user address space. +__declspec(dllexport) KSTATUS MmPfnAddMapping(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + if (entry->MappedCount == UINT32_MAX) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_FAILURE; + } + entry->MappedCount++; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + +// Removes one user VAD mapping reference from a frame. +__declspec(dllexport) KSTATUS MmPfnRemoveMapping(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateInvalid || entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed || + entry->MappedCount == 0) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + entry->MappedCount--; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + + + +// Moves a clean, unshared frame to standby so pressure can reclaim it later. +__declspec(dllexport) KSTATUS MmPfnMoveToStandby(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State != MmPfnStateActive || entry->PinCount || entry->MappedCount || entry->RefCount != 1) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_ACCESS_DENIED; + } + if (entry->Usage != MmPfnUseCleanFileCache && entry->Usage != MmPfnUseImage) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + + entry->State = MmPfnStateStandby; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + +// Reactivates a standby frame after a cache or image lookup found it still useful. +__declspec(dllexport) KSTATUS MmPfnActivate(PHYSICAL_ADDRESS frame) { + size_t pfn = MmPfnIndex(frame); + if (pfn == MM_PFN_INVALID_INDEX) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State != MmPfnStateStandby || !entry->RefCount) { + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_INVALID_PARAMETER; + } + + entry->State = MmPfnStateActive; + KeReleaseSpinlock(&gPfnLock, flags); + return STATUS_SUCCESS; +} + +// Returns the number of frames that are immediately available for allocation. size_t MmFreeFrameCount(void) { - uintptr_t flags = KeAcquireSpinlock(&gFrameBitmap.Lock); - size_t count = gFrameBitmap.FreeFrames; - KeReleaseSpinlock(&gFrameBitmap.Lock, flags); + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + size_t count = gFreeFrames; + KeReleaseSpinlock(&gPfnLock, flags); return count; } -// Implements the total Frame Count operation. +// Returns the number of RAM frames managed by the PFN database. size_t MmTotalFrameCount(void) { - return gFrameBitmap.TotalFrames; + return gTotalFrames; } -// Returns physical Stats. + +// Returns detailed PFN state counters for VMM self-tests and diagnostics. +void MmPfnQueryStats(MM_PFN_STATS* out) { + if (!out) + return; + + MM_PFN_STATS stats; + memset(&stats, 0, sizeof(stats)); + + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + stats.TotalFrames = (uint64_t)gTotalFrames; + stats.FreeFrames = (uint64_t)gFreeFrames; + stats.UsedFrames = stats.TotalFrames - stats.FreeFrames; + + for (size_t pfn = 0; pfn < gMaxPfn; pfn++) { + const MM_PFN* entry = &gPfnDatabase[pfn]; + if (entry->State == MmPfnStateInvalid) + continue; + switch ((MM_PFN_STATE)entry->State) { + case MmPfnStateReserved: stats.ReservedFrames++; break; + case MmPfnStateZeroed: stats.ZeroedFrames++; break; + case MmPfnStateActive: stats.ActiveFrames++; break; + case MmPfnStateStandby: stats.StandbyFrames++; break; + case MmPfnStateModified: stats.ModifiedFrames++; break; + default: break; + } + if (entry->PinCount) + stats.PinnedFrames++; + if (entry->MappedCount) + stats.MappedFrames++; + if (KiPfnEvictableLocked(entry)) + stats.EvictableFrames++; + else if (entry->State != MmPfnStateFree && entry->State != MmPfnStateZeroed) + stats.NonEvictableFrames++; + if (KiPfnReclaimableLocked(entry)) + stats.ReclaimableFrames++; + + switch ((MM_PFN_USAGE)entry->Usage) { + case MmPfnUseKernel: stats.KernelFrames++; break; + case MmPfnUsePageTable: stats.PageTableFrames++; break; + case MmPfnUseUser: stats.UserFrames++; break; + case MmPfnUseCleanFileCache: + stats.CleanFileCacheFrames++; + if (entry->State == MmPfnStateStandby) stats.StandbyCleanFileCacheFrames++; + else if (entry->State == MmPfnStateActive) stats.ActiveCleanFileCacheFrames++; + break; + case MmPfnUseImage: + stats.ImageFrames++; + if (entry->State == MmPfnStateStandby) stats.StandbyImageFrames++; + else if (entry->State == MmPfnStateActive) stats.ActiveImageFrames++; + break; + case MmPfnUseMappedFile: stats.MappedFileFrames++; break; + case MmPfnUseAnonymous: stats.AnonymousFrames++; break; + case MmPfnUseCow: stats.CowFrames++; break; + case MmPfnUseSurface: stats.SurfaceFrames++; break; + case MmPfnUseDriver: stats.DriverFrames++; break; + case MmPfnUseDma: stats.DmaFrames++; break; + case MmPfnUseBootloader: stats.BootloaderFrames++; break; + case MmPfnUsePfnDatabase: stats.PfnDatabaseFrames++; break; + case MmPfnUseMmio: stats.MmioFrames++; break; + default: break; + } + } + + stats.AvailableFrames = stats.FreeFrames + stats.ReclaimableFrames; + + stats.AllocationCount = gPfnAllocations; + stats.FreeCount = gPfnFrees; + stats.RetainCount = gPfnRetains; + stats.ReleaseCount = gPfnReleases; + stats.PinCount = gPfnPins; + stats.UnpinCount = gPfnUnpins; + stats.FailedAllocations = gPfnFailedAllocations; + stats.FailedContiguousAllocations = gPfnFailedContiguousAllocations; + stats.FailedInvalidRange = gPfnFailedInvalidRange; + stats.FailedEmptyRange = gPfnFailedEmptyRange; + stats.FailedReserveDenied = gPfnFailedReserveDenied; + stats.FailedNoFrame = gPfnFailedNoFrame; + stats.FailedContiguousInvalidRequest = gPfnFailedContiguousInvalidRequest; + stats.FailedContiguousReserveDenied = gPfnFailedContiguousReserveDenied; + stats.FailedContiguousNoRun = gPfnFailedContiguousNoRun; + stats.ReclaimAttempts = gPfnReclaimAttempts; + stats.ReclaimPagesRequested = gPfnReclaimPagesRequested; + stats.ReclaimPagesSucceeded = gPfnReclaimPagesSucceeded; + stats.ReclaimPagesFailed = gPfnReclaimPagesFailed; + stats.ReclaimReentrantSkips = gPfnReclaimReentrantSkips; + + KeReleaseSpinlock(&gPfnLock, flags); + *out = stats; +} + +// Returns the PMM counters expected by the existing stats syscall. void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out) { - if (!out) return; + if (!out) + return; MM_PHYSICAL_STATS stats; memset(&stats, 0, sizeof(stats)); - uintptr_t flags = KeAcquireSpinlock(&gFrameBitmap.Lock); - - stats.TotalFrames = (uint64_t)gFrameBitmap.TotalFrames; - stats.FreeFrames = (uint64_t)gFrameBitmap.FreeFrames; + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + stats.TotalFrames = (uint64_t)gTotalFrames; + stats.FreeFrames = (uint64_t)gFreeFrames; stats.UsedFrames = stats.TotalFrames - stats.FreeFrames; uint64_t run = 0; - for (size_t pfn = 1; pfn < gFrameBitmap.MaxPfn; pfn++) { - if (KiPfnFreeLocked(pfn)) { + for (size_t pfn = 1; pfn < gMaxPfn; pfn++) { + if (KiPfnAvailableLocked(pfn)) { run++; continue; } @@ -328,7 +864,6 @@ void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out) { else stats.FreeRunHistogram[7]++; } - KeReleaseSpinlock(&gFrameBitmap.Lock, flags); + KeReleaseSpinlock(&gPfnLock, flags); *out = stats; } - diff --git a/code/kernel/mm_pte.c b/code/kernel/mm_pte.c new file mode 100644 index 0000000..6e79e45 --- /dev/null +++ b/code/kernel/mm_pte.c @@ -0,0 +1,575 @@ +// Kernel page-table manager. This file owns x86-64 PTE manipulation, TLB invalidation, and shared-CR3 activation. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +// Page-table frames are reached through the physical direct map while the kernel still owns one shared CR3. +static uint64_t gPhysBase = PHYS_MAP_BASE; + +// Converts a physical address to the kernel physical-window alias used for page-table editing. +static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) { + return (void *)(p + gPhysBase); +} +// Returns the entry pointer for a page-table frame that is already mapped in the physical window. +static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) { + return (uint64_t *)KiPhysToVirt(tablePhys) + idx; +} + +// The bootloader builds the initial PML4, and this layer owns all later edits to it. +static PHYSICAL_ADDRESS gPml4Phys = 0; +static SPINLOCK gPageTableLock = SPINLOCK_INIT; + +static volatile long gTlbShootdownLock = 0; +static volatile long gTlbShootdownActive = 0; +static volatile long gTlbShootdownAcks = 0; +static uintptr_t gTlbShootdownVirt = 0; +static MM_PTE_STATS gPteStats; + +#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u + +// These macros extract the hardware page-table index for each level. +#define PML4_IDX(va) (((va) >> 39) & 0x1FF) +#define PDPT_IDX(va) (((va) >> 30) & 0x1FF) +#define PDT_IDX(va) (((va) >> 21) & 0x1FF) +#define PT_IDX(va) (((va) >> 12) & 0x1FF) + +#define PAGE_PS (1ULL << 7) +#define PAGE_FLAGS_MASK (0xFFFULL | PAGE_NX) +#define PAGE_FRAME_4K_MASK 0x000FFFFFFFFFF000ULL +#define PAGE_FRAME_2M_MASK 0x000FFFFFFFE00000ULL +#define PAGE_FRAME_1G_MASK 0x000FFFFFC0000000ULL + +// Bumps a telemetry counter. These counters are diagnostic only; exactness is +// less important than avoiding extra locks in hot page-table paths. +static inline void KiPteStatInc(uint64_t* counter) { + (*counter)++; +} + +// Clamps query attribution so corrupt callers cannot index outside telemetry arrays. +static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose) { + if ((uint32_t)purpose >= (uint32_t)MmPteQueryPurposeCount) + return MmPteQueryGeneric; + return purpose; +} + +// Ensures that one parent entry points to a child page table. +// New child tables are zeroed before publication so no stale mappings become visible. +static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys, + unsigned index, uint64_t extraFlags) { + uint64_t* entry = KiPtePtr(parentPhys, index); + if (*entry & PAGE_PRESENT) { + *entry |= extraFlags; + return *entry & PAGE_FRAME_4K_MASK; + } + PHYSICAL_ADDRESS childPhys = MmPfnAllocate(MmPfnUsePageTable); + if (!childPhys) return 0; + KiPteStatInc(&gPteStats.PageTablePagesAllocated); + memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE); + *entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags; + return childPhys; +} + + +// Initializes the page-table manager around the bootloader-created PML4. +void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys) { + gPml4Phys = bootPml4Phys; + gPageTableLock = (SPINLOCK)SPINLOCK_INIT; + gTlbShootdownLock = 0; + gTlbShootdownActive = 0; + gTlbShootdownAcks = 0; + gTlbShootdownVirt = 0; + memset(&gPteStats, 0, sizeof(gPteStats)); +} + +// Installs the shared PML4 into CR3. +void MmPteActivate(void) { + __writecr3(gPml4Phys); +} + +// Flushes one virtual address from the local TLB. +void MmPteFlushLocal(uintptr_t virt) { + KiPteStatInc(&gPteStats.LocalFlushes); + __invlpg((void*)virt); +} + +// Acquires the shootdown state lock without disabling interrupts. +static void KiAcquireTlbShootdownLock(void) { + // Do not use the normal spinlock here: it masks interrupts while spinning. + // A CPU spinning with IF=0 on the shootdown lock could not acknowledge a + // concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is + // meant to avoid. + while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0) + _mm_pause(); + _ReadWriteBarrier(); +} + +// Releases the shootdown state lock after all shared fields are stable. +static void KiReleaseTlbShootdownLock(void) { + _ReadWriteBarrier(); + _InterlockedExchange(&gTlbShootdownLock, 0); +} + +// Flushes one kernel mapping on every online CPU that shares this PML4. +void MmPteFlushKernelAddress(uintptr_t virt) { + KiPteStatInc(&gPteStats.GlobalFlushes); + MmPteFlushLocal(virt); + + if (gCpuCount <= 1) { + KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets); + return; + } + + KCPU* self = KeGetCurrentCpu(); + uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU; + + KiAcquireTlbShootdownLock(); + + long targets = 0; + gTlbShootdownVirt = virt; + _InterlockedExchange(&gTlbShootdownAcks, 0); + _ReadWriteBarrier(); + _InterlockedExchange(&gTlbShootdownActive, 1); + _ReadWriteBarrier(); + + for (uint32_t i = 0; i < gCpuCount; i++) { + KCPU* cpu = &gCpus[i]; + if (i == selfIndex) + continue; + if (!cpu->Present || !cpu->Started || !cpu->Online) + continue; + targets++; + KiPteStatInc(&gPteStats.ShootdownTargetCpus); + HalApicSendTlbShootdownIpi(cpu->LapicId); + } + + if (!targets) { + KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets); + _InterlockedExchange(&gTlbShootdownActive, 0); + _ReadWriteBarrier(); + KiReleaseTlbShootdownLock(); + return; + } + + KiPteStatInc(&gPteStats.ShootdownRequests); + + for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) { + if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets) + break; + _mm_pause(); + } + + long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0); + _InterlockedExchange(&gTlbShootdownActive, 0); + _ReadWriteBarrier(); + KiReleaseTlbShootdownLock(); + + gPteStats.ShootdownAcks += (uint64_t)acks; + if (acks < targets) { + KiPteStatInc(&gPteStats.ShootdownTimeouts); + KePanic(KE_PANIC_CODE_GENERIC, + "TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n", + (uint64_t)virt, targets, acks, selfIndex); + } +} + +// Handles an incoming shootdown request on the current CPU. +static int KiHandleTlbShootdown(void) { + if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0) + return 0; + + uintptr_t va = gTlbShootdownVirt; + _ReadWriteBarrier(); + MmPteFlushLocal(va); + _ReadWriteBarrier(); + _InterlockedIncrement(&gTlbShootdownAcks); + return 1; +} + +// Handles the NMI vector reserved for TLB shootdowns. +int MmPteHandleShootdownNmi(void) { + return KiHandleTlbShootdown(); +} + +// Maps or replaces one 4 KiB leaf PTE in the shared page tables. +__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) { + KiPteStatInc(&gPteStats.MapCalls); + unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0; + + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + + PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag); + if (!pdpt) { + KiPteStatInc(&gPteStats.MapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NO_MEMORY; + } + PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag); + if (!pdt) { + KiPteStatInc(&gPteStats.MapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NO_MEMORY; + } + PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag); + if (!pt) { + KiPteStatInc(&gPteStats.MapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NO_MEMORY; + } + + uint64_t* pte = KiPtePtr(pt, PT_IDX(virt)); + if ((*pte & PAGE_PRESENT) && mode == MmPteMapFailIfPresent) { + KiPteStatInc(&gPteStats.MapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_ALREADY_EXISTS; + } + int wasPresent = ((*pte & PAGE_PRESENT) != 0); + *pte = phys | flags; + if (!wasPresent) { + KiPteStatInc(&gPteStats.CurrentLeafMappings); + KiPteStatInc(&gPteStats.NewMappingNoFlushes); + } + + KeReleaseSpinlock(&gPageTableLock, lockFlags); + + if (wasPresent) { + KiPteStatInc(&gPteStats.ReplacedMappingFlushes); + MmPteFlushKernelAddress(virt); + } + return STATUS_SUCCESS; +} + + +// Removes one 4 KiB leaf PTE from the shared page tables. +__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) { + KiPteStatInc(&gPteStats.UnmapCalls); + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + + uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt)); + if (!(*e4 & PAGE_PRESENT)) { + KiPteStatInc(&gPteStats.UnmapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NOT_FOUND; + } + uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt)); + if (!(*e3 & PAGE_PRESENT)) { + KiPteStatInc(&gPteStats.UnmapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NOT_FOUND; + } + uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt)); + if (!(*e2 & PAGE_PRESENT)) { + KiPteStatInc(&gPteStats.UnmapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NOT_FOUND; + } + uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt)); + if (!(*pte & PAGE_PRESENT)) { + KiPteStatInc(&gPteStats.UnmapFailures); + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NOT_FOUND; + } + *pte = 0; + if (gPteStats.CurrentLeafMappings) + gPteStats.CurrentLeafMappings--; + + KeReleaseSpinlock(&gPageTableLock, lockFlags); + MmPteFlushKernelAddress(virt); + return STATUS_SUCCESS; +} + + +// Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them. +__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) { + KiPteStatInc(&gPteStats.ProtectCalls); + if (!size) return STATUS_SUCCESS; + + // The protection operation is page-granular even when callers pass an interior address. + uintptr_t base = virt & PAGE_MASK; + uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK; + + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + KSTATUS status = STATUS_SUCCESS; + uintptr_t flushEnd = base; + + for (uintptr_t va = base; va < end; va += PAGE_SIZE) { + uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va)); + if (!(*e4 & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } + uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va)); + if (!(*e3 & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } + if (*e3 & PAGE_PS) { + status = STATUS_NOT_SUPPORTED; + break; + } + uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va)); + if (!(*e2 & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } + if (*e2 & PAGE_PS) { + status = STATUS_NOT_SUPPORTED; + break; + } + uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va)); + if (!(*pte & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } + + // The physical frame stays attached while access and cache attributes change. + PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK; + *pte = phys | flags; + flushEnd = va + PAGE_SIZE; + } + + if (!KSUCCESS(status)) + KiPteStatInc(&gPteStats.ProtectFailures); + + KeReleaseSpinlock(&gPageTableLock, lockFlags); + + for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE) + MmPteFlushKernelAddress(va); + + return status; +} + + +// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf. This is +// used by the pressure trimmer as a second-chance test for clean file-backed +// pages. If the page was accessed recently, reclaim skips it once and flushes +// the TLB entry so future accesses can set PAGE_ACCESSED again. +__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt) { + if (!gPml4Phys) + return 0; + + uintptr_t va = virt & PAGE_MASK; + int wasAccessed = 0; + + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + + uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va)); + if (!(*e4 & PAGE_PRESENT)) + goto out; + uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va)); + if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS)) + goto out; + uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va)); + if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS)) + goto out; + uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va)); + if (!(*pte & PAGE_PRESENT)) + goto out; + + if (*pte & PAGE_ACCESSED) { + *pte &= ~PAGE_ACCESSED; + wasAccessed = 1; + } + +out: + KeReleaseSpinlock(&gPageTableLock, lockFlags); + + if (wasAccessed) + MmPteFlushKernelAddress(va); + return wasAccessed; +} + +// Tests and clears the hardware dirty bit on a mapped 4 KiB leaf. +__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt) { + if (!gPml4Phys) + return 0; + + uintptr_t va = virt & PAGE_MASK; + int wasDirty = 0; + + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + + uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va)); + if (!(*e4 & PAGE_PRESENT)) + goto out; + uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va)); + if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS)) + goto out; + uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va)); + if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS)) + goto out; + uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va)); + if (!(*pte & PAGE_PRESENT)) + goto out; + + if (*pte & PAGE_DIRTY) { + *pte &= ~PAGE_DIRTY; + wasDirty = 1; + } + +out: + KeReleaseSpinlock(&gPageTableLock, lockFlags); + + if (wasDirty) + MmPteFlushKernelAddress(va); + return wasDirty; +} + +// Translates a mapped virtual address to a physical address. +__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt) { + KiPteStatInc(&gPteStats.TranslateCalls); + if (!gPml4Phys) return 0; + + PHYSICAL_ADDRESS result = 0; + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + + uint64_t pml4_idx = (virt >> 39) & 0x1FF; + uint64_t pdpt_idx = (virt >> 30) & 0x1FF; + uint64_t pd_idx = (virt >> 21) & 0x1FF; + uint64_t pt_idx = (virt >> 12) & 0x1FF; + + uint64_t* pml4 = KiPhysToVirt(gPml4Phys); + if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out; + + uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK); + if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out; + + // A present PDPT leaf maps a 1 GiB huge page. + if (pdpt[pdpt_idx] & PAGE_PS) { + result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF); + goto out; + } + + uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK); + if (!(pd[pd_idx] & PAGE_PRESENT)) goto out; + + // A present PDT leaf maps a 2 MiB huge page. + if (pd[pd_idx] & PAGE_PS) { + result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF); + goto out; + } + + uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK); + if (!(pt[pt_idx] & PAGE_PRESENT)) goto out; + + // A normal PT leaf maps a 4 KiB page. + result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF); + +out: + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return result; +} + +// Queries one virtual address through the shared page-table hierarchy. +KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose, + uintptr_t virt, + PHYSICAL_ADDRESS* physOut, + uint64_t* flagsOut) { + purpose = KiNormalizeQueryPurpose(purpose); + KiPteStatInc(&gPteStats.QueryCalls); + KiPteStatInc(&gPteStats.QueryByPurpose[purpose]); + if (!gPml4Phys) { + KiPteStatInc(&gPteStats.QueryFailures); + KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]); + return STATUS_NOT_FOUND; + } + + KSTATUS status = STATUS_NOT_FOUND; + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + + uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys); + uint64_t e4 = pml4[PML4_IDX(virt)]; + if (!(e4 & PAGE_PRESENT)) goto out; + + uint64_t effectiveWrite = e4 & PAGE_WRITE; + uint64_t effectiveUser = e4 & PAGE_USER; + uint64_t effectiveNx = e4 & PAGE_NX; + + uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK); + uint64_t e3 = pdpt[PDPT_IDX(virt)]; + if (!(e3 & PAGE_PRESENT)) goto out; + effectiveWrite &= e3 & PAGE_WRITE; + effectiveUser &= e3 & PAGE_USER; + effectiveNx |= e3 & PAGE_NX; + + if (e3 & PAGE_PS) { + if (physOut) + *physOut = (e3 & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFFULL); + if (flagsOut) { + uint64_t flags = (e3 & PAGE_FLAGS_MASK) | effectiveNx; + if (!effectiveWrite) flags &= ~PAGE_WRITE; + if (!effectiveUser) flags &= ~PAGE_USER; + *flagsOut = flags; + } + status = STATUS_SUCCESS; + goto out; + } + + uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK); + uint64_t e2 = pd[PDT_IDX(virt)]; + if (!(e2 & PAGE_PRESENT)) goto out; + effectiveWrite &= e2 & PAGE_WRITE; + effectiveUser &= e2 & PAGE_USER; + effectiveNx |= e2 & PAGE_NX; + + if (e2 & PAGE_PS) { + if (physOut) + *physOut = (e2 & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFFULL); + if (flagsOut) { + uint64_t flags = (e2 & PAGE_FLAGS_MASK) | effectiveNx; + if (!effectiveWrite) flags &= ~PAGE_WRITE; + if (!effectiveUser) flags &= ~PAGE_USER; + *flagsOut = flags; + } + status = STATUS_SUCCESS; + goto out; + } + + uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK); + uint64_t e1 = pt[PT_IDX(virt)]; + if (!(e1 & PAGE_PRESENT)) goto out; + effectiveWrite &= e1 & PAGE_WRITE; + effectiveUser &= e1 & PAGE_USER; + effectiveNx |= e1 & PAGE_NX; + + if (physOut) + *physOut = (e1 & PAGE_FRAME_4K_MASK) + (virt & 0xFFFULL); + if (flagsOut) { + uint64_t flags = (e1 & PAGE_FLAGS_MASK) | effectiveNx; + if (!effectiveWrite) flags &= ~PAGE_WRITE; + if (!effectiveUser) flags &= ~PAGE_USER; + *flagsOut = flags; + } + status = STATUS_SUCCESS; + +out: + if (!KSUCCESS(status)) { + KiPteStatInc(&gPteStats.QueryFailures); + KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]); + } + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return status; +} + +// Queries one virtual address without caller-specific attribution. +__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, + PHYSICAL_ADDRESS* physOut, + uint64_t* flagsOut) { + return MmPteQueryPageFor(MmPteQueryGeneric, virt, physOut, flagsOut); +} + +// Returns page-table telemetry for diagnostics. +void MmPteQueryStats(MM_PTE_STATS* out) { + if (!out) + return; + *out = gPteStats; +} + + diff --git a/code/kernel/mm_swap.c b/code/kernel/mm_swap.c new file mode 100644 index 0000000..be1fb39 --- /dev/null +++ b/code/kernel/mm_swap.c @@ -0,0 +1,243 @@ +// Kernel swap manager. +// The first backend is a fixed-size pagefile accessed through VFS offset I/O. +// Kernel memory and filesystem code remain non-pageable, so this layer is safe +// to use before the VMM has direct block-extents for pageout. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct MM_SWAP_STATE { + SPINLOCK Lock; + FILE_OBJECT* File; + uint8_t* Bitmap; + uint32_t SlotCount; + uint32_t UsedSlots; + uint32_t HighWaterSlots; + int Enabled; + MM_SWAP_STATS Stats; +} MM_SWAP_STATE; + +static MM_SWAP_STATE gSwap = { .Lock = SPINLOCK_INIT }; + +// Returns whether a swap slot is marked in use. +static int KiSwapTestSlot(uint32_t slot) { + return (gSwap.Bitmap[slot >> 3] & (uint8_t)(1u << (slot & 7))) != 0; +} + +// Marks a swap slot as allocated or free. +static void KiSwapSetSlot(uint32_t slot, int used) { + uint8_t mask = (uint8_t)(1u << (slot & 7)); + if (used) + gSwap.Bitmap[slot >> 3] |= mask; + else + gSwap.Bitmap[slot >> 3] &= (uint8_t)~mask; +} + +// Allocates one pagefile slot. +static KSTATUS KiSwapAllocateSlot(uint32_t* slotOut) { + if (!slotOut) return STATUS_INVALID_PARAMETER; + *slotOut = 0; + + uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock); + if (!gSwap.Enabled || !gSwap.Bitmap || !gSwap.SlotCount) { + KeReleaseSpinlock(&gSwap.Lock, flags); + return STATUS_NOT_SUPPORTED; + } + + for (uint32_t slot = 0; slot < gSwap.SlotCount; ++slot) { + if (!KiSwapTestSlot(slot)) { + KiSwapSetSlot(slot, 1); + gSwap.UsedSlots++; + if (gSwap.UsedSlots > gSwap.HighWaterSlots) + gSwap.HighWaterSlots = gSwap.UsedSlots; + gSwap.Stats.SlotAllocations++; + *slotOut = slot; + KeReleaseSpinlock(&gSwap.Lock, flags); + return STATUS_SUCCESS; + } + } + + gSwap.Stats.SlotAllocationFailures++; + KeReleaseSpinlock(&gSwap.Lock, flags); + return STATUS_NO_MEMORY; +} + +// Initializes the fixed pagefile backend. +KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) { + if (!path.Data || !path.Length || sizeBytes < PAGE_SIZE) + return STATUS_INVALID_PARAMETER; + if (sizeBytes & (PAGE_SIZE - 1)) + sizeBytes = (sizeBytes + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1); + + uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock); + if (gSwap.Enabled) { + KeReleaseSpinlock(&gSwap.Lock, flags); + return STATUS_SUCCESS; + } + KeReleaseSpinlock(&gSwap.Lock, flags); + + FILE_OBJECT* file = (FILE_OBJECT*)0; + KSTATUS s = IoOpenFile(path, &file); + if (!KSUCCESS(s)) { + s = IoCreateFile(path, &file); + if (!KSUCCESS(s)) + return s; + } + + // Swap slots are never read before MmSwapPageOut overwrites the whole + // page, so the filesystem may allocate backing clusters without zeroing + // them. Normal files still get zero-on-extend semantics. + file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND; + + s = IoTruncateFile(file, sizeBytes); + if (!KSUCCESS(s)) { + IoCloseFile(file); + return s; + } + + uint64_t slots64 = sizeBytes / PAGE_SIZE; + if (!slots64 || slots64 > 0xFFFFFFFFull) { + IoCloseFile(file); + return STATUS_INVALID_PARAMETER; + } + + uint32_t slotCount = (uint32_t)slots64; + size_t bitmapBytes = (slotCount + 7u) / 8u; + uint8_t* bitmap = (uint8_t*)MmAllocatePoolTag(bitmapBytes, MM_POOL_TAG('s', 'w', 'a', 'p')); + if (!bitmap) { + IoCloseFile(file); + return STATUS_NO_MEMORY; + } + memset(bitmap, 0, bitmapBytes); + + flags = KeAcquireSpinlock(&gSwap.Lock); + if (gSwap.Enabled) { + KeReleaseSpinlock(&gSwap.Lock, flags); + MmFreePool(bitmap); + IoCloseFile(file); + return STATUS_SUCCESS; + } + + gSwap.File = file; + gSwap.Bitmap = bitmap; + gSwap.SlotCount = slotCount; + gSwap.UsedSlots = 0; + gSwap.HighWaterSlots = 0; + gSwap.Enabled = 1; + memset(&gSwap.Stats, 0, sizeof(gSwap.Stats)); + gSwap.Stats.Enabled = 1; + gSwap.Stats.TotalSlots = slotCount; + gSwap.Stats.FreeSlots = slotCount; + KeReleaseSpinlock(&gSwap.Lock, flags); + + DbgPrint("swap: initialized %u slots (%llu KiB) at %v\n", + slotCount, + (unsigned long long)(sizeBytes / 1024), + path); + return STATUS_SUCCESS; +} + +// Returns whether swap is available for pageout. +int MmSwapIsEnabled(void) { + return gSwap.Enabled; +} + +// Writes one physical page to a newly allocated swap slot. +KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut) { + if (!slotOut || !frame) + return STATUS_INVALID_PARAMETER; + *slotOut = 0; + if (!gSwap.Enabled || !gSwap.File) + return STATUS_NOT_SUPPORTED; + + uint32_t slot = 0; + KSTATUS s = KiSwapAllocateSlot(&slot); + if (!KSUCCESS(s)) { + gSwap.Stats.PageOutFailures++; + return s; + } + + void* page = MmPfnToVirtual(frame); + if (!page) { + MmSwapFreeSlot(slot); + gSwap.Stats.PageOutFailures++; + return STATUS_FAILURE; + } + + size_t written = 0; + s = IoWriteFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &written); + if (!KSUCCESS(s) || written != PAGE_SIZE) { + MmSwapFreeSlot(slot); + gSwap.Stats.PageOutFailures++; + return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s; + } + + gSwap.Stats.PageOuts++; + *slotOut = slot; + return STATUS_SUCCESS; +} + +// Reads one swap slot into an already allocated physical page. +KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame) { + if (!frame) + return STATUS_INVALID_PARAMETER; + if (!gSwap.Enabled || !gSwap.File) + return STATUS_NOT_SUPPORTED; + + uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock); + int valid = slot < gSwap.SlotCount && KiSwapTestSlot(slot); + KeReleaseSpinlock(&gSwap.Lock, flags); + if (!valid) { + gSwap.Stats.PageInFailures++; + return STATUS_NOT_FOUND; + } + + void* page = MmPfnToVirtual(frame); + if (!page) { + gSwap.Stats.PageInFailures++; + return STATUS_FAILURE; + } + + size_t read = 0; + KSTATUS s = IoReadFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &read); + if (!KSUCCESS(s) || read != PAGE_SIZE) { + gSwap.Stats.PageInFailures++; + return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s; + } + + gSwap.Stats.PageIns++; + return STATUS_SUCCESS; +} + +// Releases a swap slot after the owning VAD page no longer references it. +void MmSwapFreeSlot(uint32_t slot) { + uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock); + if (gSwap.Enabled && slot < gSwap.SlotCount && KiSwapTestSlot(slot)) { + KiSwapSetSlot(slot, 0); + if (gSwap.UsedSlots) + gSwap.UsedSlots--; + gSwap.Stats.SlotFrees++; + } + KeReleaseSpinlock(&gSwap.Lock, flags); +} + +// Copies swap telemetry for user diagnostics. +void MmSwapQueryStats(MM_SWAP_STATS* out) { + if (!out) return; + uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock); + MM_SWAP_STATS stats = gSwap.Stats; + stats.Enabled = gSwap.Enabled ? 1 : 0; + stats.TotalSlots = gSwap.SlotCount; + stats.UsedSlots = gSwap.UsedSlots; + stats.FreeSlots = gSwap.SlotCount >= gSwap.UsedSlots ? gSwap.SlotCount - gSwap.UsedSlots : 0; + stats.HighWaterSlots = gSwap.HighWaterSlots; + KeReleaseSpinlock(&gSwap.Lock, flags); + *out = stats; +} diff --git a/code/kernel/mm_virt.c b/code/kernel/mm_virt.c index 716a47b..4fbd846 100644 --- a/code/kernel/mm_virt.c +++ b/code/kernel/mm_virt.c @@ -1,414 +1,36 @@ -// Kernel virtual-memory manager. This file owns kernel page-table construction, mapping attributes, and direct-map maintenance. - -// Page tables: -// One PML4 shared by all processes. -// Upper half (bit 47 set) has U/S = 0 - ring-3 access always faults. -// Lower half pages are PAGE_USER. +// Kernel virtual-memory manager. This file owns kernel pool allocation, vad bookkeeping, and user-copy validation. +// Page-table construction lives in mm_pte.c so the higher VMM can grow toward VADs, faults, COW, and swap without mixing those policies with raw PTE edits. // Kernel pool: -// A boundary-tag (header + footer) free-list allocator starting at POOL_BASE. -// Pages are committed on demand from the PMM. +// A boundary-tag allocator starts at POOL_BASE and commits pages on demand. -// User heap: -// A single best-fit free list in the lower half. -// Page-granular lower-half regions owned by PROCESS metadata. Sub-page -// allocation is a user-mode runtime concern. +// User mappings: +// VADs separate reservation from commitment. Committed pages become resident +// when a user fault or kernel prefault path materializes them. #include #include #include #include +#include +#include +#include +#include +#include +#include +#include #include -#include #include #include #include -#include +#include -// Helpers - physical window access. -static uint64_t gPhysBase = PHYS_MAP_BASE; - -// Implements the phys To Virt operation. -static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) { - return (void *)(p + gPhysBase); -} -// Implements the pte Ptr operation. -static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) { - return (uint64_t *)KiPhysToVirt(tablePhys) + idx; -} - -// PML4 / page-table helpers. -static PHYSICAL_ADDRESS gPml4Phys = 0; -static SPINLOCK gPageTableLock = SPINLOCK_INIT; - -static volatile long gTlbShootdownLock = 0; -static volatile long gTlbShootdownActive = 0; -static volatile long gTlbShootdownAcks = 0; -static uintptr_t gTlbShootdownVirt = 0; - -#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u - -// Extract the page-table index for each level from a virtual address. -#define PML4_IDX(va) (((va) >> 39) & 0x1FF) -#define PDPT_IDX(va) (((va) >> 30) & 0x1FF) -#define PDT_IDX(va) (((va) >> 21) & 0x1FF) -#define PT_IDX(va) (((va) >> 12) & 0x1FF) - -#define PAGE_PS (1ULL << 7) -#define PAGE_FLAGS_MASK (0xFFFULL | PAGE_NX) -#define PAGE_FRAME_4K_MASK 0x000FFFFFFFFFF000ULL -#define PAGE_FRAME_2M_MASK 0x000FFFFFFFE00000ULL -#define PAGE_FRAME_1G_MASK 0x000FFFFFC0000000ULL - -// Ensure a page-table entry at 'parentPhys[index]' points to a child table. -// Allocates and zeros a new frame if the entry is not present. -// Returns the physical address of the child table, or 0 on OOM. -static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys, - unsigned index, uint64_t extraFlags) { - uint64_t* entry = KiPtePtr(parentPhys, index); - if (*entry & PAGE_PRESENT) { - *entry |= extraFlags; - return *entry & PAGE_FRAME_4K_MASK; - } - PHYSICAL_ADDRESS childPhys = MmAllocateFrame(); - if (!childPhys) return 0; - memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE); - *entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags; - return childPhys; -} - -// TLB/cache coherency hooks. -void MmFlushTlbLocal(uintptr_t virt) { - __invlpg((void*)virt); -} - -// Implements the acquire Tlb Shootdown Lock operation. -static void KiAcquireTlbShootdownLock(void) { - // Do not use the normal spinlock here: it masks interrupts while spinning. - // A CPU spinning with IF=0 on the shootdown lock could not acknowledge a - // concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is - // meant to avoid. - while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0) - _mm_pause(); - _ReadWriteBarrier(); -} - -// Releases tlb Shootdown Lock. -static void KiReleaseTlbShootdownLock(void) { - _ReadWriteBarrier(); - _InterlockedExchange(&gTlbShootdownLock, 0); -} - -// Flushes tlb For Kernel Mapping. -void MmFlushTlbForKernelMapping(uintptr_t virt) { - MmFlushTlbLocal(virt); - - if (gCpuCount <= 1) - return; - - KCPU* self = KeGetCurrentCpu(); - uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU; - - KiAcquireTlbShootdownLock(); - - long targets = 0; - gTlbShootdownVirt = virt; - _InterlockedExchange(&gTlbShootdownAcks, 0); - _ReadWriteBarrier(); - _InterlockedExchange(&gTlbShootdownActive, 1); - _ReadWriteBarrier(); - - for (uint32_t i = 0; i < gCpuCount; i++) { - KCPU* cpu = &gCpus[i]; - if (i == selfIndex) - continue; - if (!cpu->Present || !cpu->Started || !cpu->Online) - continue; - targets++; - HalApicSendTlbShootdownIpi(cpu->LapicId); - } - - for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) { - if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets) - break; - _mm_pause(); - } - - long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0); - _InterlockedExchange(&gTlbShootdownActive, 0); - _ReadWriteBarrier(); - KiReleaseTlbShootdownLock(); - - if (acks < targets) { - KePanic(KE_PANIC_CODE_GENERIC, - "TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n", - (uint64_t)virt, targets, acks, selfIndex); - } -} - -// Handles tlb Shootdown. -static int KiHandleTlbShootdown(void) { - if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0) - return 0; - - uintptr_t va = gTlbShootdownVirt; - _ReadWriteBarrier(); - MmFlushTlbLocal(va); - _ReadWriteBarrier(); - _InterlockedIncrement(&gTlbShootdownAcks); - return 1; -} - -// Handles tlb Shootdown Nmi. -int MmHandleTlbShootdownNmi(void) { - return KiHandleTlbShootdown(); -} - -// Public mapping. -__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode) { - unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0; - - uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); - - PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag); - if (!pdpt) { - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return STATUS_NO_MEMORY; - } - PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag); - if (!pdt) { - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return STATUS_NO_MEMORY; - } - PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag); - if (!pt) { - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return STATUS_NO_MEMORY; - } - - uint64_t* pte = KiPtePtr(pt, PT_IDX(virt)); - if ((*pte & PAGE_PRESENT) && mode == MmMapFailIfPresent) { - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return STATUS_ALREADY_EXISTS; - } - *pte = phys | flags; - - KeReleaseSpinlock(&gPageTableLock, lockFlags); - MmFlushTlbForKernelMapping(virt); - return STATUS_SUCCESS; -} - - -// Releases page. -__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) { - uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); - - uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt)); - if (!(*e4 & PAGE_PRESENT)) { - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return STATUS_NOT_FOUND; - } - uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt)); - if (!(*e3 & PAGE_PRESENT)) { - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return STATUS_NOT_FOUND; - } - uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt)); - if (!(*e2 & PAGE_PRESENT)) { - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return STATUS_NOT_FOUND; - } - uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt)); - *pte = 0; - - KeReleaseSpinlock(&gPageTableLock, lockFlags); - MmFlushTlbForKernelMapping(virt); - return STATUS_SUCCESS; -} - - -// Changes page attributes for 4 KiB mappings and refuses huge-page leaves that cannot be edited in place. -__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags) { - if (!size) return STATUS_SUCCESS; - - // Round down to page boundary, round size up. - uintptr_t base = virt & PAGE_MASK; - uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK; - - uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); - KSTATUS status = STATUS_SUCCESS; - uintptr_t flushEnd = base; - - for (uintptr_t va = base; va < end; va += PAGE_SIZE) { - uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va)); - if (!(*e4 & PAGE_PRESENT)) { - status = STATUS_NOT_FOUND; - break; - } - uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va)); - if (!(*e3 & PAGE_PRESENT)) { - status = STATUS_NOT_FOUND; - break; - } - if (*e3 & PAGE_PS) { - status = STATUS_NOT_SUPPORTED; - break; - } - uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va)); - if (!(*e2 & PAGE_PRESENT)) { - status = STATUS_NOT_FOUND; - break; - } - if (*e2 & PAGE_PS) { - status = STATUS_NOT_SUPPORTED; - break; - } - uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va)); - if (!(*pte & PAGE_PRESENT)) { - status = STATUS_NOT_FOUND; - break; - } - - // Preserve the physical address, replace everything else. - PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK; - *pte = phys | flags; - flushEnd = va + PAGE_SIZE; - } - - KeReleaseSpinlock(&gPageTableLock, lockFlags); - - for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE) - MmFlushTlbForKernelMapping(va); - - return status; -} - - -// Returns physical Address. -__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) { - if (!gPml4Phys) return 0; // Not initialized - - PHYSICAL_ADDRESS result = 0; - uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); - - uint64_t pml4_idx = (virt >> 39) & 0x1FF; - uint64_t pdpt_idx = (virt >> 30) & 0x1FF; - uint64_t pd_idx = (virt >> 21) & 0x1FF; - uint64_t pt_idx = (virt >> 12) & 0x1FF; - - uint64_t* pml4 = KiPhysToVirt(gPml4Phys); - if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out; - - uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK); - if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out; - - // Check for 1GB huge pages (bit 7 - PS). - if (pdpt[pdpt_idx] & PAGE_PS) { - result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF); - goto out; - } - - uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK); - if (!(pd[pd_idx] & PAGE_PRESENT)) goto out; - - // Check for 2MB huge pages (bit 7 - PS). - if (pd[pd_idx] & PAGE_PS) { - result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF); - goto out; - } - - uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK); - if (!(pt[pt_idx] & PAGE_PRESENT)) goto out; - - // Extract the physical frame and add the offset within the page. - result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF); - -out: - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return result; -} - - - -// Page query and user/kernel copy boundary helpers. -__declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, - PHYSICAL_ADDRESS* physOut, - uint64_t* flagsOut) { - if (!gPml4Phys) return STATUS_NOT_FOUND; - - KSTATUS status = STATUS_NOT_FOUND; - uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); - - uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys); - uint64_t e4 = pml4[PML4_IDX(virt)]; - if (!(e4 & PAGE_PRESENT)) goto out; - - uint64_t effectiveWrite = e4 & PAGE_WRITE; - uint64_t effectiveUser = e4 & PAGE_USER; - uint64_t effectiveNx = e4 & PAGE_NX; - - uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK); - uint64_t e3 = pdpt[PDPT_IDX(virt)]; - if (!(e3 & PAGE_PRESENT)) goto out; - effectiveWrite &= e3 & PAGE_WRITE; - effectiveUser &= e3 & PAGE_USER; - effectiveNx |= e3 & PAGE_NX; - - if (e3 & PAGE_PS) { - if (physOut) - *physOut = (e3 & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFFULL); - if (flagsOut) { - uint64_t flags = (e3 & PAGE_FLAGS_MASK) | effectiveNx; - if (!effectiveWrite) flags &= ~PAGE_WRITE; - if (!effectiveUser) flags &= ~PAGE_USER; - *flagsOut = flags; - } - status = STATUS_SUCCESS; - goto out; - } - - uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK); - uint64_t e2 = pd[PDT_IDX(virt)]; - if (!(e2 & PAGE_PRESENT)) goto out; - effectiveWrite &= e2 & PAGE_WRITE; - effectiveUser &= e2 & PAGE_USER; - effectiveNx |= e2 & PAGE_NX; - - if (e2 & PAGE_PS) { - if (physOut) - *physOut = (e2 & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFFULL); - if (flagsOut) { - uint64_t flags = (e2 & PAGE_FLAGS_MASK) | effectiveNx; - if (!effectiveWrite) flags &= ~PAGE_WRITE; - if (!effectiveUser) flags &= ~PAGE_USER; - *flagsOut = flags; - } - status = STATUS_SUCCESS; - goto out; - } - - uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK); - uint64_t e1 = pt[PT_IDX(virt)]; - if (!(e1 & PAGE_PRESENT)) goto out; - effectiveWrite &= e1 & PAGE_WRITE; - effectiveUser &= e1 & PAGE_USER; - effectiveNx |= e1 & PAGE_NX; - - if (physOut) - *physOut = (e1 & PAGE_FRAME_4K_MASK) + (virt & 0xFFFULL); - if (flagsOut) { - uint64_t flags = (e1 & PAGE_FLAGS_MASK) | effectiveNx; - if (!effectiveWrite) flags &= ~PAGE_WRITE; - if (!effectiveUser) flags &= ~PAGE_USER; - *flagsOut = flags; - } - status = STATUS_SUCCESS; - -out: - KeReleaseSpinlock(&gPageTableLock, lockFlags); - return status; -} +static KSTATUS KiVadEnsureResidentRange(PROCESS* process, uintptr_t start, size_t size, uint64_t accessFlags); +static KSTATUS KiCopyFromUserPaged(void* kernelDst, const void* userSrc, size_t size); +static KSTATUS KiCopyToUserPaged(void* userDst, const void* kernelSrc, size_t size); +static PHYSICAL_ADDRESS gVadZeroPageFrame; +// User/kernel copy boundary helpers. // Implements the user Range Valid operation. static int KiUserRangeValid(uintptr_t addr, size_t size) { if (size == 0) return 1; @@ -418,7 +40,7 @@ static int KiUserRangeValid(uintptr_t addr, size_t size) { return 1; } -// Probes user Range. +// Probes user Range and materializes demand-zero pages before the kernel copies through it. static KSTATUS KiProbeUserRange(uintptr_t addr, size_t size, uint64_t requiredFlags) { if (!KiUserRangeValid(addr, size)) return STATUS_ACCESS_DENIED; @@ -426,22 +48,11 @@ static KSTATUS KiProbeUserRange(uintptr_t addr, size_t size, uint64_t requiredFl if (size == 0) return STATUS_SUCCESS; - uintptr_t va = addr & PAGE_MASK; - uintptr_t last = (addr + size - 1) & PAGE_MASK; + PROCESS* process = KeGetCurrentProcess(); + if (!process) + return STATUS_ACCESS_DENIED; - for (;;) { - uint64_t flags = 0; - KSTATUS s = MmQueryPage(va, (PHYSICAL_ADDRESS*)0, &flags); - if (!KSUCCESS(s)) - return STATUS_ACCESS_DENIED; - if ((flags & requiredFlags) != requiredFlags) - return STATUS_ACCESS_DENIED; - if (va == last) - break; - va += PAGE_SIZE; - } - - return STATUS_SUCCESS; + return KiVadEnsureResidentRange(process, addr, size, requiredFlags); } // Probes user Read. @@ -454,26 +65,14 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size) { return KiProbeUserRange((uintptr_t)ptr, size, PAGE_PRESENT | PAGE_USER | PAGE_WRITE); } -// Copies from User. +// Copies from a user buffer through page-sized, pinned segments. __declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size) { - if (!kernelDst && size) return STATUS_INVALID_PARAMETER; - - KSTATUS s = MmProbeUserRead(userSrc, size); - if (!KSUCCESS(s)) return s; - - if (size) memcpy(kernelDst, userSrc, size); - return STATUS_SUCCESS; + return KiCopyFromUserPaged(kernelDst, userSrc, size); } -// Copies to User. +// Copies to a user buffer through page-sized, pinned segments. __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size) { - if (!kernelSrc && size) return STATUS_INVALID_PARAMETER; - - KSTATUS s = MmProbeUserWrite(userDst, size); - if (!KSUCCESS(s)) return s; - - if (size) memcpy(userDst, kernelSrc, size); - return STATUS_SUCCESS; + return KiCopyToUserPaged(userDst, kernelSrc, size); } // Boundary-tag allocator (used for both pool and user heap) @@ -483,12 +82,33 @@ __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, s #define BLOCK_MAGIC 0xB10CB10CUL +#define BLOCK_STATE_USED 0u +#define BLOCK_STATE_FREE 1u +#define BLOCK_STATE_QUARANTINE 2u + +#define BLOCK_FLAG_CANARY 0x00000001u +#define BLOCK_FLAG_POISONED 0x00000002u + +#define POOL_TAIL_CANARY_BASE 0xC0FFEE1234ABCDEFULL +#define POOL_POISON_FREE 0xDD +#define POOL_QUARANTINE_SLOTS 256u +#define POOL_TAG_DEFAULT 0x6C6F6F70u /* 'pool' */ + typedef struct BLOCK_HDR { uint32_t Magic; - uint32_t Free; // 1 = free, 0 = used - size_t Size; // usable bytes (excludes header + footer) + uint32_t Free; // BLOCK_STATE_* + size_t Size; // usable allocator bytes (excludes header + footer) struct BLOCK_HDR* PrevFree; struct BLOCK_HDR* NextFree; + + // Debug metadata. Present in all builds so boot.ini bits can enable + // diagnostics without changing allocator layout or ABI. + size_t RequestedSize; + uint32_t Tag; + uint32_t Flags; + uint64_t Sequence; + uintptr_t AllocCaller; + uintptr_t FreeCaller; } BLOCK_HDR; typedef struct BLOCK_FTR { @@ -519,43 +139,143 @@ static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) { return (BLOCK_FTR *)((char *)h - FTR_SIZE); } -typedef struct HEAP { +typedef struct HEAP HEAP; + +struct HEAP { BLOCK_HDR* FreeList; // doubly-linked list of free blocks uintptr_t Start; // virtual base of the arena uintptr_t Brk; // current committed end (page-aligned) uintptr_t Limit; // hard upper bound uint64_t MapFlags; // PAGE_* flags for new committed pages SPINLOCK Lock; -} HEAP; + int DecommitTailOnFree; -// Implements the heap Ensure Mapped operation. -static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) { - if (!h || !size) return 1; - if (start < h->Start) return 0; - if (start > (uintptr_t)-1 - (size - 1)) return 0; + uint64_t NextSequence; + BLOCK_HDR* Quarantine[POOL_QUARANTINE_SLOTS]; + size_t QuarantineCursor; + size_t QuarantineCount; +}; - uintptr_t begin = start & PAGE_MASK; - uintptr_t end = (start + size + PAGE_SIZE - 1) & PAGE_MASK; - if (end < begin || end > h->Limit) return 0; +static inline uint32_t KiPoolDebugFlags(void) { + return KeGetDebugFlags() & (KE_DEBUG_POOL_CANARY | + KE_DEBUG_POOL_POISON | + KE_DEBUG_POOL_QUARANTINE | + KE_DEBUG_POOL_CALLERS); +} - for (uintptr_t va = begin; va < end; va += PAGE_SIZE) { - PHYSICAL_ADDRESS phys = 0; - uint64_t flags = 0; - KSTATUS qs = MmQueryPage(va, &phys, &flags); - if (KSUCCESS(qs) && (flags & PAGE_PRESENT)) - continue; +static inline uint64_t KiPoolTailCanaryValue(const BLOCK_HDR* blk) { + return POOL_TAIL_CANARY_BASE ^ + ((uint64_t)(uintptr_t)blk << 13) ^ + ((uint64_t)blk->RequestedSize << 1) ^ + blk->Sequence; +} - PHYSICAL_ADDRESS frame = MmAllocateFrame(); - if (!frame) return 0; - KSTATUS ms = MmMapPage(va, frame, h->MapFlags, MmMapFailIfPresent); - if (!KSUCCESS(ms)) { - MmFreeFrame(frame); - return 0; - } - memset((void*)va, 0, PAGE_SIZE); - DbgPrint("MM: repaired missing heap page va=%016llx\n", (unsigned long long)va); +static inline uint8_t* KiBlockUserBase(BLOCK_HDR* blk) { + return (uint8_t*)blk + HDR_SIZE; +} + +static inline uint64_t* KiBlockTailCanary(BLOCK_HDR* blk) { + return (uint64_t*)(void*)(KiBlockUserBase(blk) + blk->RequestedSize); +} + +static const char* KiPoolStateName(uint32_t state) { + switch (state) { + case BLOCK_STATE_USED: return "used"; + case BLOCK_STATE_FREE: return "free"; + case BLOCK_STATE_QUARANTINE: return "quarantine"; + default: return "bad-state"; + } +} + +static _Noreturn void KiPoolPanic(const char* reason, HEAP* h, BLOCK_HDR* blk, const void* ptr) { + KePanic(KE_PANIC_CODE_GENERIC, + "MM: pool corruption: %s ptr=%p hdr=%p heap=[%016llx,%016llx) magic=%08x state=%u(%s) size=%llu req=%llu tag=%08x flags=%08x seq=%llu alloc=%p free=%p\n", + reason ? reason : "unknown", + ptr, + blk, + (unsigned long long)(h ? h->Start : 0), + (unsigned long long)(h ? h->Brk : 0), + blk ? blk->Magic : 0, + blk ? blk->Free : 0, + blk ? KiPoolStateName(blk->Free) : "", + (unsigned long long)(blk ? blk->Size : 0), + (unsigned long long)(blk ? blk->RequestedSize : 0), + blk ? blk->Tag : 0, + blk ? blk->Flags : 0, + (unsigned long long)(blk ? blk->Sequence : 0), + blk ? (void*)blk->AllocCaller : (void*)0, + blk ? (void*)blk->FreeCaller : (void*)0); +} + +static void KiPoolValidateFooterLocked(HEAP* h, BLOCK_HDR* blk, const char* where) { + if (!h || !blk) + KiPoolPanic(where, h, blk, blk ? KiBlockUserBase(blk) : (void*)0); + if (blk->Magic != BLOCK_MAGIC) + KiPoolPanic(where, h, blk, KiBlockUserBase(blk)); + if ((uintptr_t)blk < h->Start || (uintptr_t)blk >= h->Brk) + KiPoolPanic("block outside heap", h, blk, KiBlockUserBase(blk)); + if (blk->Size > (size_t)(h->Brk - (uintptr_t)blk - OVERHEAD)) + KiPoolPanic("block size outside heap", h, blk, KiBlockUserBase(blk)); + BLOCK_FTR* ftr = KiHdrToFtr(blk); + if ((uintptr_t)ftr + FTR_SIZE > h->Brk) + KiPoolPanic("footer outside heap", h, blk, KiBlockUserBase(blk)); + if (ftr->Magic != BLOCK_MAGIC || ftr->Size != blk->Size || ftr->Free != blk->Free) + KiPoolPanic(where, h, blk, KiBlockUserBase(blk)); +} + +static void KiPoolWriteTailCanary(BLOCK_HDR* blk) { + if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY)) + return; + *KiBlockTailCanary(blk) = KiPoolTailCanaryValue(blk); +} + +static void KiPoolValidateTailCanaryLocked(HEAP* h, BLOCK_HDR* blk, const char* where) { + if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY)) + return; + if (blk->RequestedSize + sizeof(uint64_t) > blk->Size) + KiPoolPanic("invalid canary layout", h, blk, KiBlockUserBase(blk)); + uint64_t actual = *KiBlockTailCanary(blk); + uint64_t expect = KiPoolTailCanaryValue(blk); + if (actual != expect) + KiPoolPanic(where, h, blk, KiBlockUserBase(blk)); +} + +static void KiPoolPoisonFreedPayload(BLOCK_HDR* blk) { + if (!blk) + return; + size_t bytes = blk->RequestedSize; + if (bytes > blk->Size) + bytes = blk->Size; + memset(KiBlockUserBase(blk), POOL_POISON_FREE, bytes); + blk->Flags |= BLOCK_FLAG_POISONED; + KiPoolWriteTailCanary(blk); +} + +static void KiPoolValidateFreedPoisonLocked(HEAP* h, BLOCK_HDR* blk, const char* where) { + if (!blk || !(blk->Flags & BLOCK_FLAG_POISONED)) + return; + size_t bytes = blk->RequestedSize; + if (bytes > blk->Size) + bytes = blk->Size; + const uint8_t* p = KiBlockUserBase(blk); + for (size_t i = 0; i < bytes; i++) { + if (p[i] != POOL_POISON_FREE) + KiPoolPanic(where, h, blk, KiBlockUserBase(blk)); + } +} + +// Rolls back a failed heap grow attempt. KiHeapGrow maps pages before the +// new free block is linked into the allocator; if a later frame allocation or +// page-table allocation fails, those earlier mappings must be returned or the +// pool silently leaks physical frames and leaves stale mappings below Brk. +static void KiHeapRollbackGrow(uintptr_t start, uintptr_t end) { + for (uintptr_t va = start; va < end; va += PAGE_SIZE) { + PHYSICAL_ADDRESS frame = MmPteTranslate(va); + if (!frame) + continue; + if (KSUCCESS(MmPteUnmapPage(va))) + (void)MmPfnFree(frame); } - return 1; } // Extend heap by at least 'need' bytes (rounded up to page boundary). @@ -563,33 +283,71 @@ static int KiHeapGrow(HEAP* h, size_t need) { size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); if (h->Brk + grow > h->Limit) return 0; - for (uintptr_t va = h->Brk; va < h->Brk + grow; va += PAGE_SIZE) { - PHYSICAL_ADDRESS frame = MmAllocateFrame(); - if (!frame) return 0; - if (!KSUCCESS(MmMapPage(va, frame, h->MapFlags, MmMapFailIfPresent))) { - MmFreeFrame(frame); + uintptr_t start = h->Brk; + uintptr_t mappedEnd = start; + for (uintptr_t va = start; va < start + grow; va += PAGE_SIZE) { + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel); + if (!frame) { + KiHeapRollbackGrow(start, mappedEnd); return 0; } + if (!KSUCCESS(MmPteMapPage(va, frame, h->MapFlags, MmPteMapFailIfPresent))) { + MmPfnFree(frame); + KiHeapRollbackGrow(start, mappedEnd); + return 0; + } + mappedEnd = va + PAGE_SIZE; memset((void *)va, 0, PAGE_SIZE); } - if (!KiHeapEnsureMapped(h, h->Brk, grow)) - return 0; + // The grow path owns the only transition from reserved heap VA to mapped heap + // pages, so a successful loop above is the heap's committed-frontier proof. + // Re-querying every heap page on later allocations only burns page-table locks. + // Missing pool pages are corruption now, not something the allocator repairs. + + uintptr_t oldBrk = h->Brk; + uintptr_t newBrk = oldBrk + grow; + + // If the current tail block is already free, extend it in-place instead of + // leaving adjacent free blocks. This matters once tail decommit is enabled: + // a later grow should rebuild the same tail block, not fragment the arena + // into many page-sized free runs. + if (oldBrk > h->Start) { + BLOCK_FTR* prevFtr = (BLOCK_FTR*)(oldBrk - FTR_SIZE); + if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free == BLOCK_STATE_FREE) { + BLOCK_HDR* prev = KiFtrToHdr(prevFtr); + if ((uintptr_t)prev >= h->Start && prev->Magic == BLOCK_MAGIC && prev->Free == BLOCK_STATE_FREE) { + prev->Size += grow; + h->Brk = newBrk; + KiHdrToFtr(prev)->Magic = BLOCK_MAGIC; + KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE; + KiHdrToFtr(prev)->Size = prev->Size; + return 1; + } + } + } // Make the new pages a single free block. - BLOCK_HDR* blk = (BLOCK_HDR *)h->Brk; + BLOCK_HDR* blk = (BLOCK_HDR *)oldBrk; blk->Magic = BLOCK_MAGIC; - blk->Free = 1; + blk->Free = BLOCK_STATE_FREE; blk->Size = grow - OVERHEAD; blk->PrevFree = (BLOCK_HDR *)0; blk->NextFree = h->FreeList; + blk->RequestedSize = 0; + blk->Tag = 0; + blk->Flags = 0; + blk->Sequence = 0; + blk->AllocCaller = 0; + blk->FreeCaller = 0; if (h->FreeList) h->FreeList->PrevFree = blk; h->FreeList = blk; + + h->Brk = newBrk; KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; - KiHdrToFtr(blk)->Free = 1; + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; KiHdrToFtr(blk)->Size = blk->Size; - h->Brk += grow; return 1; } @@ -603,41 +361,201 @@ static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) { // Implements the heap Try Decommit Tail operation. static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) { - (void)h; - if (!blk || blk->Free == 0) return; + if (!h || !blk || blk->Free == BLOCK_STATE_USED) return; - // Do not decommit kernel-pool tail pages for now. The pool allocator keeps - // free-list nodes and boundary tags inside the managed arena; shrinking the - // mapped tail while large launch-failure cleanup is walking/freeing many - // objects can leave stale metadata pointers into unmapped pages. Physical - // pages are still reclaimed for user regions by MmDestroyUserRegion(); pool - // pages remain committed until we have a dedicated, audited heap shrinker. + // Do not eagerly decommit the kernel pool on every free. Large transient + // syscall bounce buffers, especially SysBlitScreen's frame-sized buffer, + // are often allocated/freed once per frame. If the tail block is unmapped + // immediately, the next frame has to map the same pages again, producing + // massive page-table churn and global TLB shootdowns. Keep the committed + // pool high-water resident during normal operation; add an explicit idle or + // pressure pool trim later if needed. + if (!h->DecommitTailOnFree) { + KiHdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; + return; + } + + uintptr_t blockBase = (uintptr_t)blk; + uintptr_t oldBrk = h->Brk; + + // Only the last free block can return committed heap pages. Interior free + // blocks must remain mapped because the allocator's boundary tags and free + // list links are stored inside those blocks. + if ((uintptr_t)KiNextBlock(blk) != oldBrk) { + KiHdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; + return; + } + + // Keep enough mapped space for the tail block's header, footer, and a small + // usable payload, then decommit whole pages above that. The footer moves to + // the new Brk so future coalescing can still walk the boundary tag chain. + uintptr_t minEnd = blockBase + MIN_BLOCK; + uintptr_t newBrk = (minEnd + PAGE_SIZE - 1) & ~(uintptr_t)(PAGE_SIZE - 1); + + if (newBrk >= oldBrk) { + KiHdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; + return; + } + + for (uintptr_t va = newBrk; va < oldBrk; va += PAGE_SIZE) { + PHYSICAL_ADDRESS frame = MmPteTranslate(va); + if (!frame) + continue; + if (KSUCCESS(MmPteUnmapPage(va))) + (void)MmPfnFree(frame); + } + + h->Brk = newBrk; + blk->Size = newBrk - blockBase - OVERHEAD; KiHdrToFtr(blk)->Size = blk->Size; KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; - KiHdrToFtr(blk)->Free = 1; + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; +} + +// Releases a previously quarantined block into the real free list. Caller holds h->Lock. +static void KiHeapReleaseFreeBlockLocked(HEAP* h, BLOCK_HDR* blk) { + if (!h || !blk) return; + + KiPoolValidateFooterLocked(h, blk, "release free footer corrupt"); + KiPoolValidateTailCanaryLocked(h, blk, "release free tail canary corrupt"); + KiPoolValidateFreedPoisonLocked(h, blk, "use-after-free payload write detected"); + + blk->Free = BLOCK_STATE_FREE; + blk->RequestedSize = 0; + blk->Tag = 0; + blk->Flags = 0; + blk->AllocCaller = 0; + // Keep FreeCaller/Sequence for diagnostics until the block is reused. + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Size = blk->Size; + + // Coalesce with next block if it is really free. Quarantined neighbors are + // intentionally not coalesced; their payload must remain intact for UAF checks. + BLOCK_HDR* next = KiNextBlock(blk); + if ((uintptr_t)next < h->Brk) { + KiPoolValidateFooterLocked(h, next, "next footer corrupt while freeing"); + if (next->Free == BLOCK_STATE_FREE) { + KiFreeListRemove(h, next); + blk->Size += OVERHEAD + next->Size; + KiHdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; + } + } + + // Coalesce with previous block if it is really free. + if ((uintptr_t)blk > h->Start) { + BLOCK_FTR* prevFtr = KiPrevFtr(blk); + if (prevFtr->Magic == BLOCK_MAGIC) { + BLOCK_HDR* prev = KiFtrToHdr(prevFtr); + KiPoolValidateFooterLocked(h, prev, "previous footer corrupt while freeing"); + if (prev->Free == BLOCK_STATE_FREE) { + KiFreeListRemove(h, prev); + prev->Size += OVERHEAD + blk->Size; + KiHdrToFtr(prev)->Size = prev->Size; + KiHdrToFtr(prev)->Magic = BLOCK_MAGIC; + KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE; + blk = prev; + } + } + } + + // Insert at head of free list. + blk->PrevFree = (BLOCK_HDR *)0; + blk->NextFree = h->FreeList; + if (h->FreeList) h->FreeList->PrevFree = blk; + h->FreeList = blk; + + KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Size = blk->Size; + + KiHeapTryDecommitTail(h, blk); +} + +static void KiHeapReleaseQuarantineSlotLocked(HEAP* h, size_t index) { + if (!h || index >= POOL_QUARANTINE_SLOTS) + return; + BLOCK_HDR* old = h->Quarantine[index]; + if (!old) + return; + h->Quarantine[index] = (BLOCK_HDR*)0; + if (h->QuarantineCount) + h->QuarantineCount--; + if (old->Free != BLOCK_STATE_QUARANTINE) + KiPoolPanic("quarantine slot state corrupt", h, old, KiBlockUserBase(old)); + KiHeapReleaseFreeBlockLocked(h, old); +} + +static void KiHeapDrainQuarantineLocked(HEAP* h) { + if (!h) + return; + for (size_t i = 0; i < POOL_QUARANTINE_SLOTS; i++) + KiHeapReleaseQuarantineSlotLocked(h, i); + h->QuarantineCursor = 0; +} + +static void KiHeapQuarantineBlockLocked(HEAP* h, BLOCK_HDR* blk) { + if (!h || !blk) + return; + size_t idx = h->QuarantineCursor % POOL_QUARANTINE_SLOTS; + KiHeapReleaseQuarantineSlotLocked(h, idx); + + blk->Free = BLOCK_STATE_QUARANTINE; + KiHdrToFtr(blk)->Free = BLOCK_STATE_QUARANTINE; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Size = blk->Size; + + h->Quarantine[idx] = blk; + h->QuarantineCursor = (idx + 1u) % POOL_QUARANTINE_SLOTS; + h->QuarantineCount++; } // Implements the heap Alloc operation. -static void* KiHeapAlloc(HEAP* h, size_t size) { - if (!size) return (void *)0; +static void* KiHeapAllocEx(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t caller) { + if (!requestedSize) return (void *)0; + + uint32_t dbg = KiPoolDebugFlags(); + size_t size = requestedSize; + if (dbg & KE_DEBUG_POOL_CANARY) { + if (size > (size_t)-1 - sizeof(uint64_t)) + return (void*)0; + size += sizeof(uint64_t); + } size = (size + 7) & ~7ULL; // 8-byte align uintptr_t flags = KeAcquireSpinlock(&h->Lock); - if (h->Brk > h->Start && !KiHeapEnsureMapped(h, h->Start, h->Brk - h->Start)) { - KeReleaseSpinlock(&h->Lock, flags); - return (void*)0; - } - // Best-fit search. BLOCK_HDR* best = (BLOCK_HDR *)0; for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) { + KiPoolValidateFooterLocked(h, cur, "free list block corrupt"); + if (cur->Free != BLOCK_STATE_FREE) + KiPoolPanic("free list contains non-free block", h, cur, KiBlockUserBase(cur)); if (cur->Size >= size) { if (!best || cur->Size < best->Size) best = cur; if (best->Size == size) break; } } + if (!best && h->QuarantineCount) { + KiHeapDrainQuarantineLocked(h); + for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) { + KiPoolValidateFooterLocked(h, cur, "free list block corrupt after quarantine drain"); + if (cur->Size >= size) { + if (!best || cur->Size < best->Size) best = cur; + if (best->Size == size) break; + } + } + } + if (!best) { if (!KiHeapGrow(h, size + OVERHEAD)) { KeReleaseSpinlock(&h->Lock, flags); @@ -650,6 +568,7 @@ static void* KiHeapAlloc(HEAP* h, size_t size) { KeReleaseSpinlock(&h->Lock, flags); return (void *)0; } + KiPoolValidateFooterLocked(h, best, "grown free block corrupt"); } KiFreeListRemove(h, best); @@ -658,130 +577,176 @@ static void* KiHeapAlloc(HEAP* h, size_t size) { if (best->Size >= size + OVERHEAD + sizeof(void*)) { BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE); rem->Magic = BLOCK_MAGIC; - rem->Free = 1; + rem->Free = BLOCK_STATE_FREE; rem->Size = best->Size - size - OVERHEAD; rem->PrevFree = (BLOCK_HDR *)0; rem->NextFree = h->FreeList; + rem->RequestedSize = 0; + rem->Tag = 0; + rem->Flags = 0; + rem->Sequence = 0; + rem->AllocCaller = 0; + rem->FreeCaller = 0; if (h->FreeList) h->FreeList->PrevFree = rem; h->FreeList = rem; KiHdrToFtr(rem)->Magic = BLOCK_MAGIC; - KiHdrToFtr(rem)->Free = 1; + KiHdrToFtr(rem)->Free = BLOCK_STATE_FREE; KiHdrToFtr(rem)->Size = rem->Size; best->Size = size; } - best->Free = 0; - KiHdrToFtr(best)->Free = 0; + best->Free = BLOCK_STATE_USED; + best->RequestedSize = requestedSize; + best->Tag = tag ? tag : POOL_TAG_DEFAULT; + best->Flags = 0; + best->Sequence = ++h->NextSequence; + best->AllocCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0; + best->FreeCaller = 0; + if (dbg & KE_DEBUG_POOL_CANARY) + best->Flags |= BLOCK_FLAG_CANARY; + + KiHdrToFtr(best)->Free = BLOCK_STATE_USED; KiHdrToFtr(best)->Magic = BLOCK_MAGIC; KiHdrToFtr(best)->Size = best->Size; - memset((char *)best + HDR_SIZE, 0, size); + memset(KiBlockUserBase(best), 0, requestedSize); + if (best->Size > requestedSize) + memset(KiBlockUserBase(best) + requestedSize, 0, best->Size - requestedSize); + KiPoolWriteTailCanary(best); + KeReleaseSpinlock(&h->Lock, flags); - return (char *)best + HDR_SIZE; + return KiBlockUserBase(best); +} + +static void* KiHeapAlloc(HEAP* h, size_t size) { + return KiHeapAllocEx(h, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress()); } // Implements the heap Free operation. -static void KiHeapFree(HEAP* h, void* ptr) { +static void KiHeapFreeEx(HEAP* h, void* ptr, uintptr_t caller) { if (!ptr) return; - BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE); - if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt + + uintptr_t u = (uintptr_t)ptr; + if (!h || u < h->Start + HDR_SIZE || u >= h->Brk) + KiPoolPanic("invalid free pointer outside heap", h, (BLOCK_HDR*)0, ptr); + + BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE); uintptr_t flags = KeAcquireSpinlock(&h->Lock); - blk->Free = 1; + if (blk->Magic != BLOCK_MAGIC) + KiPoolPanic("invalid free pointer/bad header magic", h, blk, ptr); + KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on free"); + if (blk->Free == BLOCK_STATE_FREE) + KiPoolPanic("double free", h, blk, ptr); + if (blk->Free == BLOCK_STATE_QUARANTINE) + KiPoolPanic("double free/quarantined UAF", h, blk, ptr); + if (blk->Free != BLOCK_STATE_USED) + KiPoolPanic("invalid block state on free", h, blk, ptr); - // Coalesce with next block if it is free. - BLOCK_HDR* next = KiNextBlock(blk); - if ((uintptr_t)next < h->Brk && next->Magic == BLOCK_MAGIC && next->Free) { - KiFreeListRemove(h, next); - blk->Size += OVERHEAD + next->Size; - KiHdrToFtr(blk)->Size = blk->Size; - KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; - KiHdrToFtr(blk)->Free = 1; + KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on free"); + + uint32_t dbg = KiPoolDebugFlags(); + blk->FreeCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0; + if (dbg & KE_DEBUG_POOL_POISON) + KiPoolPoisonFreedPayload(blk); + + if (dbg & KE_DEBUG_POOL_QUARANTINE) { + KiHeapQuarantineBlockLocked(h, blk); + } else { + KiHeapReleaseFreeBlockLocked(h, blk); } - // Coalesce with previous block if it is free. - if ((uintptr_t)blk > h->Start) { - BLOCK_FTR* prevFtr = KiPrevFtr(blk); - if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free) { - BLOCK_HDR* prev = KiFtrToHdr(prevFtr); - KiFreeListRemove(h, prev); - prev->Size += OVERHEAD + blk->Size; - KiHdrToFtr(prev)->Size = prev->Size; - KiHdrToFtr(prev)->Magic = BLOCK_MAGIC; - KiHdrToFtr(prev)->Free = 1; - blk = prev; - } - } - - // Insert at head of free list. - blk->PrevFree = (BLOCK_HDR *)0; - blk->NextFree = h->FreeList; - if (h->FreeList) h->FreeList->PrevFree = blk; - h->FreeList = blk; - - KiHdrToFtr(blk)->Free = 1; - KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; - - KiHeapTryDecommitTail(h, blk); - KeReleaseSpinlock(&h->Lock, flags); } +static void KiHeapFree(HEAP* h, void* ptr) { + KiHeapFreeEx(h, ptr, (uintptr_t)_ReturnAddress()); +} + // Implements the heap Realloc operation. -static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) { +static void* KiHeapReallocEx(HEAP* h, void* ptr, size_t newSize, uintptr_t caller) { // If ptr is NULL, realloc acts like malloc. if (!ptr) { - return KiHeapAlloc(h, new_size); + return KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller); } // If size is 0, realloc acts like free. - if (new_size == 0) { - KiHeapFree(h, ptr); + if (newSize == 0) { + KiHeapFreeEx(h, ptr, caller); return (void*)0; } - // Retrieve the header of the existing block. - BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE); + uintptr_t u = (uintptr_t)ptr; + if (!h || u < h->Start + HDR_SIZE || u >= h->Brk) + KiPoolPanic("invalid realloc pointer outside heap", h, (BLOCK_HDR*)0, ptr); - // Sanity checks to prevent operating on corrupted or already-freed pointers. - if (blk->Magic != BLOCK_MAGIC || blk->Free) { - return (void*)0; - } + BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE); - // Align the requested size (matches KiHeapAlloc 8-byte alignment). - size_t aligned_size = (new_size + 7) & ~7ULL; + uintptr_t flags = KeAcquireSpinlock(&h->Lock); + if (blk->Magic != BLOCK_MAGIC) + KiPoolPanic("invalid realloc pointer/bad header magic", h, blk, ptr); + KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on realloc"); + if (blk->Free != BLOCK_STATE_USED) + KiPoolPanic("realloc of non-allocated block", h, blk, ptr); + KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on realloc"); - // If the current block is already big enough, we can reuse it in-place. - // (Note: To save space, you could optionally split the block here if - // the new size is significantly smaller, similar to KiHeapAlloc's logic). - if (blk->Size >= aligned_size) { + size_t oldRequested = blk->RequestedSize; + size_t oldCopy = oldRequested < newSize ? oldRequested : newSize; + if (newSize <= oldRequested) { + blk->RequestedSize = newSize; + if (blk->Flags & BLOCK_FLAG_CANARY) + KiPoolWriteTailCanary(blk); + KeReleaseSpinlock(&h->Lock, flags); return ptr; } + KeReleaseSpinlock(&h->Lock, flags); - // The block is too small. Allocate a new, larger block. - void* new_ptr = KiHeapAlloc(h, aligned_size); - if (!new_ptr) { - return (void*)0; // Out of memory, original block is left untouched - } - memset(new_ptr, 0, aligned_size); + void* newPtr = KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller); + if (!newPtr) + return (void*)0; + memcpy(newPtr, ptr, oldCopy); + KiHeapFreeEx(h, ptr, caller); + return newPtr; +} - // Copy the old data to the new block. - // We only copy up to the old block's usable size. - memcpy(new_ptr, ptr, blk->Size); - - // Free the old block. - KiHeapFree(h, ptr); - - return new_ptr; +static void* KiHeapRealloc(HEAP* h, void* ptr, size_t newSize) { + return KiHeapReallocEx(h, ptr, newSize, (uintptr_t)_ReturnAddress()); } // Kernel pool. static HEAP gPoolHeap; -// User virtual region arena. -typedef struct USER_VA_REGION { +// VAD arena. +typedef enum MM_VAD_PAGE_STATE { + MmVadPageFree = 0, + MmVadPageCommitted = 1, +} MM_VAD_PAGE_STATE; + +#define MM_VAD_PAGE_COW 0x00000001u +#define MM_VAD_PAGE_ZERO_PAGE 0x00000002u +#define MM_VAD_PAGE_SWAPPED 0x00000004u +#define MM_VAD_PAGE_SHARED_FRAME 0x00000008u +#define MM_VAD_PAGE_PAGING 0x00000010u +#define MM_VAD_PAGE_FILE_BACKED 0x00000020u +#define MM_VAD_PAGE_IMAGE_BACKED 0x00000040u +#define MM_VAD_PAGE_IMAGE_PRIVATE 0x00000080u +#define MM_VAD_PAGE_IMAGE_FIXUP 0x00000100u +#define MM_VAD_PAGE_FILE_SHARED_WRITE 0x00000200u +#define MM_VAD_PAGE_FILE_DIRTY_TRACK 0x00000400u + +typedef struct MM_VAD_PAGE { + uint64_t PageFlags; + PHYSICAL_ADDRESS Frame; + uint8_t State; + uint8_t Resident; + uint16_t Reserved; + uint32_t Flags; + uint32_t SwapSlot; +} MM_VAD_PAGE; + +typedef struct MM_VAD { LIST_ENTRY ProcessLink; LIST_ENTRY GlobalLink; PROCESS* Owner; @@ -790,97 +755,132 @@ typedef struct USER_VA_REGION { size_t ReservationSize; uintptr_t Base; size_t Size; - size_t CommittedSize; + size_t PayloadSize; + size_t PageCount; + size_t CommittedPages; uint64_t PageFlags; - USER_REGION_TYPE Type; + MM_VAD_TYPE Type; uint32_t Flags; -} USER_VA_REGION; + uint32_t LockCount; + MM_VAD_PAGE* Pages; -static LIST_ENTRY gUserRegionList; -static SPINLOCK gUserRegionLock = SPINLOCK_INIT; -static int gUserRegionListReady = 0; + FILE_OBJECT* FileObject; + uint64_t FileOffset; + size_t FileLength; + uint32_t FileMappingFlags; -#define USER_REGION_ARENA_BASE 0x0000100000000000ULL + IMAGE_SECTION_OBJECT* ImageSection; +} MM_VAD; -// Module init. +static LIST_ENTRY gVadList; +static SPINLOCK gVadLock = SPINLOCK_INIT; +static int gVadListReady = 0; +static MM_VAD_STATS gVadStats; + +#define MM_VAD_ARENA_BASE 0x0000100000000000ULL + + +// Initializes the VAD list after the shared page table and kernel pool are ready. +static void KiVadInitialize(void) { + LIST_INIT(&gVadList); + gVadLock = (SPINLOCK)SPINLOCK_INIT; + memset(&gVadStats, 0, sizeof(gVadStats)); + gVadListReady = 1; +} + +// Initializes the shared page tables, kernel pool, and user VAD arena. void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) { - gPml4Phys = bootPml4Phys; - gPageTableLock = (SPINLOCK)SPINLOCK_INIT; + MmPteInitialize(bootPml4Phys); - // Kernel pool heap. gPoolHeap.FreeList = (BLOCK_HDR *)0; gPoolHeap.Start = POOL_BASE; gPoolHeap.Brk = POOL_BASE; gPoolHeap.Limit = POOL_LIMIT; gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX; gPoolHeap.Lock = (SPINLOCK)SPINLOCK_INIT; + gPoolHeap.DecommitTailOnFree = 0; + gPoolHeap.NextSequence = 0; + gPoolHeap.QuarantineCursor = 0; + gPoolHeap.QuarantineCount = 0; + memset(gPoolHeap.Quarantine, 0, sizeof(gPoolHeap.Quarantine)); - // Pre-commit the requested pool size. if (poolSize) { poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); KiHeapGrow(&gPoolHeap, poolSize); } - // User VA arena. One global lower-half virtual-address bucket, with - // per-process ownership tracked by USER_VA_REGION descriptors. - LIST_INIT(&gUserRegionList); - gUserRegionLock = (SPINLOCK)SPINLOCK_INIT; - gUserRegionListReady = 1; + KiVadInitialize(); + + gVadZeroPageFrame = MmPfnAllocate(MmPfnUseAnonymous); + if (!gVadZeroPageFrame) { + KePanic(KE_PANIC_CODE_INIT, "MM: failed to allocate global zero page.\n"); + } + memset(MmPfnToVirtual(gVadZeroPageFrame), 0, PAGE_SIZE); + (void)MmPfnPin(gVadZeroPageFrame); } -// Implements the activate operation. -void MmActivate(void) { - __writecr3(gPml4Phys); + +// Allocates zeroed kernel-pool memory from the non-user heap. +__declspec(dllexport) void* MmAllocatePool(size_t size) { + return KiHeapAllocEx(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress()); } -// Public pool API. -__declspec(dllexport) void* MmAllocatePool(size_t size) { return KiHeapAlloc(&gPoolHeap, size); } -// Releases pool. -__declspec(dllexport) void MmFreePool(void* ptr) { KiHeapFree(&gPoolHeap, ptr); } -// Implements the reallocate Pool operation. -__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); } +// Allocates zeroed kernel-pool memory with a diagnostic tag. +__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag) { + return KiHeapAllocEx(&gPoolHeap, size, tag, (uintptr_t)_ReturnAddress()); +} -// User virtual region API. +// Releases kernel-pool memory back to the non-user heap. +__declspec(dllexport) void MmFreePool(void* ptr) { + KiHeapFreeEx(&gPoolHeap, ptr, (uintptr_t)_ReturnAddress()); +} + +// Reallocates kernel-pool memory while preserving existing contents. +__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { + return KiHeapReallocEx(&gPoolHeap, ptr, newSize, (uintptr_t)_ReturnAddress()); +} + +// VAD utility helpers. static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) { if (align < PAGE_SIZE) align = PAGE_SIZE; if (align & (align - 1)) return 0; return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1); } -// Implements the align Up Size operation. +// Aligns a size to a power-of-two boundary and rejects invalid alignments. static inline size_t KiAlignUpSize(size_t v, size_t align) { if (align < PAGE_SIZE) align = PAGE_SIZE; if (align & (align - 1)) return 0; return (v + align - 1) & ~(align - 1); } -// Implements the ranges Overlap operation. +// Returns whether two half-open address ranges intersect. static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) { uintptr_t aEnd = aBase + aSize; uintptr_t bEnd = bBase + bSize; return aBase < bEnd && bBase < aEnd; } -// Returns user Va Range. -static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) { +// Finds a free range in the global user VA arena. +static KSTATUS KiFindVadRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) { if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER; - if (!gUserRegionListReady) return STATUS_FAILURE; + if (!gVadListReady) return STATUS_FAILURE; - uintptr_t candidate = KiAlignUpPtr(USER_REGION_ARENA_BASE, alignment); + uintptr_t candidate = KiAlignUpPtr(MM_VAD_ARENA_BASE, alignment); if (!candidate) return STATUS_INVALID_PARAMETER; for (;;) { - if (candidate < USER_REGION_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate)) + if (candidate < MM_VAD_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate)) return STATUS_NO_MEMORY; uintptr_t next = candidate; int collided = 0; - for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); - if (KiRangesOverlap(candidate, reservationSize, r->ReservationBase, r->ReservationSize)) { - uintptr_t end = r->ReservationBase + r->ReservationSize; + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, GlobalLink); + if (KiRangesOverlap(candidate, reservationSize, vad->ReservationBase, vad->ReservationSize)) { + uintptr_t end = vad->ReservationBase + vad->ReservationSize; next = KiAlignUpPtr(end, alignment); if (!next || next <= candidate) return STATUS_NO_MEMORY; collided = 1; @@ -896,131 +896,1092 @@ static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintp } } -// Returns user Region Locked. -static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) { - if (!process || !base) return (USER_VA_REGION*)0; - uintptr_t b = (uintptr_t)base; - for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); - if (r->Base == b) return r; - } - return (USER_VA_REGION*)0; +// Returns the page index for an address inside the VAD payload. +static size_t KiVadPageIndex(const MM_VAD* vad, uintptr_t va) { + return (size_t)((va - vad->Base) / PAGE_SIZE); } -// Returns user Region Containing Locked. -static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) { - if (!process || !base || !size) return (USER_VA_REGION*)0; - if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0; +// Returns the page count covered by a page-aligned half-open range. +static size_t KiVadRangePageCount(uintptr_t alignedStart, uintptr_t alignedEnd) { + return (size_t)((alignedEnd - alignedStart) / PAGE_SIZE); +} + +// Finds a VAD by its public base address while the owning process VAD lock is held. +static MM_VAD* KiFindVadByBaseLocked(PROCESS* process, const void* base) { + if (!process || !base) return (MM_VAD*)0; + uintptr_t b = (uintptr_t)base; + for (LIST_ENTRY* node = process->VadList.Fwd; node != &process->VadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, ProcessLink); + if (vad->Base == b) return vad; + } + return (MM_VAD*)0; +} + +// Finds a VAD whose reserved payload fully contains the supplied range. +static MM_VAD* KiFindReservedVadContainingLocked(PROCESS* process, uintptr_t base, size_t size) { + if (!process || !base || !size) return (MM_VAD*)0; + if (base > (uintptr_t)-1 - (size - 1)) return (MM_VAD*)0; uintptr_t end = base + size; - for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); - uintptr_t regionEnd = r->Base + r->CommittedSize; - if (base >= r->Base && end <= regionEnd) return r; + for (LIST_ENTRY* node = process->VadList.Fwd; node != &process->VadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, ProcessLink); + uintptr_t vadEnd = vad->Base + vad->PayloadSize; + if (base >= vad->Base && end <= vadEnd) return vad; } - return (USER_VA_REGION*)0; + return (MM_VAD*)0; } -// Releases user Region Payload. -static void KiUnmapUserRegionPayload(USER_VA_REGION* r) { - if (!r) return; - uintptr_t end = r->Base + r->CommittedSize; - for (uintptr_t va = r->Base; va < end; va += PAGE_SIZE) { - PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); - if (!phys) continue; - if (KSUCCESS(MmUnmapPage(va))) { - MmFreeFrame(phys); +// Returns whether every page in the aligned range is committed in the VAD metadata. +static int KiVadRangeCommitted(const MM_VAD* vad, uintptr_t alignedStart, uintptr_t alignedEnd) { + if (!vad || alignedStart < vad->Base || alignedEnd > vad->Base + vad->PayloadSize) + return 0; + size_t first = KiVadPageIndex(vad, alignedStart); + size_t count = KiVadRangePageCount(alignedStart, alignedEnd); + if (first > vad->PageCount || count > vad->PageCount - first) + return 0; + + for (size_t i = 0; i < count; ++i) { + if (vad->Pages[first + i].State != MmVadPageCommitted) + return 0; + } + return 1; +} + +static uint64_t KiVadHardwarePageFlags(const MM_VAD_PAGE* page); +static KSTATUS KiVadMapPageTracked(uintptr_t va, PHYSICAL_ADDRESS frame, uint64_t flags, MM_PTE_MAP_MODE mode); +static KSTATUS KiVadUnmapPageTracked(uintptr_t va, PHYSICAL_ADDRESS frame); + +static KSTATUS KiVadMarkSharedFilePageDirty(MM_VAD* vad, size_t index, int writeProtect) { + if (!vad || index >= vad->PageCount || !vad->FileObject) + return STATUS_INVALID_PARAMETER; + + MM_VAD_PAGE* page = &vad->Pages[index]; + if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) == 0 || + (page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) == 0) + return STATUS_NOT_FOUND; + + uint64_t pageDelta = (uint64_t)index * (uint64_t)PAGE_SIZE; + uint64_t fileOffset = vad->FileOffset + pageDelta; + if (fileOffset < vad->FileOffset) + return STATUS_INVALID_PARAMETER; + + uint32_t validBytes = PAGE_SIZE; + if (pageDelta >= (uint64_t)vad->FileLength) { + validBytes = PAGE_SIZE; + } else if ((uint64_t)vad->FileLength - pageDelta < PAGE_SIZE) { + validBytes = (uint32_t)((uint64_t)vad->FileLength - pageDelta); + if (validBytes == 0) + validBytes = PAGE_SIZE; + } + + MmFileCacheMarkDirty(vad->FileObject, fileOffset, validBytes); + + uint64_t end = fileOffset + validBytes; + uintptr_t flags = KeAcquireSpinlock(&vad->FileObject->Lock); + if (end > vad->FileObject->Size) + vad->FileObject->Size = end; + KeReleaseSpinlock(&vad->FileObject->Lock, flags); + + if (writeProtect && page->Resident && page->Frame && page->PageFlags) { + uintptr_t va = vad->Base + index * PAGE_SIZE; + page->Flags |= MM_VAD_PAGE_FILE_DIRTY_TRACK; + (void)MmPteProtectRange(va, PAGE_SIZE, KiVadHardwarePageFlags(page)); + } + + gVadStats.FileBackedDirtySyncs++; + return STATUS_SUCCESS; +} + +static void KiVadSyncSharedFilePageBeforeRelease(MM_VAD* vad, size_t index) { + if (!vad || index >= vad->PageCount) + return; + MM_VAD_PAGE* page = &vad->Pages[index]; + if ((page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) == 0) + return; + uintptr_t va = vad->Base + index * PAGE_SIZE; + if (page->Resident && page->Frame && MmPteTestAndClearDirty(va)) + (void)KiVadMarkSharedFilePageDirty(vad, index, 0); +} + +// Marks a page uncommitted and releases any physical frame it still owns. +static void KiVadReleasePage(MM_VAD* vad, size_t index) { + if (!vad || index >= vad->PageCount) return; + MM_VAD_PAGE* page = &vad->Pages[index]; + uintptr_t va = vad->Base + index * PAGE_SIZE; + + KiVadSyncSharedFilePageBeforeRelease(vad, index); + + if (page->Resident && page->PageFlags) { + (void)KiVadUnmapPageTracked(va, page->Frame); + } + if (page->Resident && page->Frame && !(page->Flags & MM_VAD_PAGE_ZERO_PAGE)) { + (void)MmPfnRelease(page->Frame); + } + if ((page->Flags & MM_VAD_PAGE_SWAPPED) && page->SwapSlot != 0xFFFFFFFFu) { + MmSwapFreeSlot(page->SwapSlot); + } + + if (page->State == MmVadPageCommitted && vad->CommittedPages) + vad->CommittedPages--; + + page->PageFlags = 0; + page->Frame = 0; + page->State = MmVadPageFree; + page->Resident = 0; + page->Flags = 0; + page->SwapSlot = 0xFFFFFFFFu; +} + +// Unmaps all resident pages in a VAD and releases their PFNs. +static void KiUnmapVadPayload(MM_VAD* vad) { + if (!vad || !vad->Pages) return; + for (size_t i = 0; i < vad->PageCount; ++i) { + if (vad->Pages[i].State == MmVadPageCommitted) + KiVadReleasePage(vad, i); + } +} + +// Returns the hardware PTE flags used for a resident VAD page. +static uint64_t KiVadHardwarePageFlags(const MM_VAD_PAGE* page) { + if (!page || !page->PageFlags) + return 0; + uint64_t flags = page->PageFlags; + if (page->Flags & MM_VAD_PAGE_COW) + flags &= ~PAGE_WRITE; + if (page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) + flags &= ~PAGE_WRITE; + return flags; +} + +// Maps a user VAD PTE and records the mapping in the PFN database. +static KSTATUS KiVadMapPageTracked(uintptr_t va, PHYSICAL_ADDRESS frame, uint64_t flags, MM_PTE_MAP_MODE mode) { + KSTATUS s = MmPteMapPage(va, frame, flags, mode); + if (!KSUCCESS(s)) + return s; + + s = MmPfnAddMapping(frame); + if (!KSUCCESS(s)) { + (void)MmPteUnmapPage(va); + return s; + } + return STATUS_SUCCESS; +} + +// Unmaps a user VAD PTE and removes the matching PFN mapping reference. +static KSTATUS KiVadUnmapPageTracked(uintptr_t va, PHYSICAL_ADDRESS frame) { + KSTATUS s = MmPteUnmapPage(va); + if (KSUCCESS(s)) + (void)MmPfnRemoveMapping(frame); + return s; +} + +// Returns whether the committed page allows the requested fault or probe access. +static KSTATUS KiVadCheckPageAccess(const MM_VAD_PAGE* page, int wantWrite, int wantExecute) { + if (!page || page->State != MmVadPageCommitted) + return STATUS_NOT_FOUND; + if (!page->PageFlags) + return STATUS_ACCESS_DENIED; + if ((page->PageFlags & PAGE_USER) == 0) + return STATUS_ACCESS_DENIED; + if (wantWrite && (page->PageFlags & PAGE_WRITE) == 0) + return STATUS_ACCESS_DENIED; + if (wantExecute && (page->PageFlags & PAGE_NX)) + return STATUS_ACCESS_DENIED; + return STATUS_SUCCESS; +} + +// Waits for another CPU/thread to finish a page-in/page-out transition on a VAD page. +// The pager performs filesystem I/O outside the VAD lock; this transient bit keeps +// concurrent faults from treating half-transitioned metadata as either resident or +// fully swapped. +static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) { + while (page && (_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0) & + MM_VAD_PAGE_PAGING)) + KeYield(); +} + +// Atomically claims the transient paging bit. This is deliberately not a spinlock: +// a page-in/page-out may allocate frames or do filesystem I/O, so waiters must yield +// instead of holding IRQL-raised locks. Without this CAS, two CPUs faulting the same +// swapped/demand-zero page can both allocate/read/map it; one map fails and rolls +// metadata back to SWAPPED while the other has already published a valid PTE. That +// leaves stale or freed swap slots behind and corrupts user data under pressure. +static int KiVadTryBeginPaging(MM_VAD_PAGE* page) { + if (!page) return 0; + + for (;;) { + uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0); + if (oldFlags & MM_VAD_PAGE_PAGING) + return 0; + uint32_t newFlags = oldFlags | MM_VAD_PAGE_PAGING; + if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, + (long)newFlags, + (long)oldFlags) == oldFlags) + return 1; + _mm_pause(); + } +} + +static void KiVadEndPaging(MM_VAD_PAGE* page) { + if (!page) return; + + for (;;) { + uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0); + uint32_t newFlags = oldFlags & ~MM_VAD_PAGE_PAGING; + if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, + (long)newFlags, + (long)oldFlags) == oldFlags) + return; + _mm_pause(); + } +} + +// Reads one page of a file-backed VAD into a freshly allocated user frame. +// Bytes beyond EOF or beyond the requested mapping length are zero-filled, +// matching the useful mmap/CreateFileMapping contract without materializing the +// whole file up front. +static KSTATUS KiVadFillFileBackedPage(MM_VAD* vad, size_t pageIndex, PHYSICAL_ADDRESS frame) { + if (!vad || !vad->FileObject || !frame || pageIndex >= vad->PageCount) + return STATUS_INVALID_PARAMETER; + + void* pageVa = MmPfnToVirtual(frame); + if (!pageVa) + return STATUS_FAILURE; + memset(pageVa, 0, PAGE_SIZE); + + uint64_t pageDelta = (uint64_t)pageIndex * (uint64_t)PAGE_SIZE; + if (pageDelta >= (uint64_t)vad->FileLength) + return STATUS_SUCCESS; + + size_t remaining = vad->FileLength - (size_t)pageDelta; + size_t toRead = remaining < PAGE_SIZE ? remaining : PAGE_SIZE; + + uint64_t fileOffset = vad->FileOffset + pageDelta; + if (fileOffset < vad->FileOffset) + return STATUS_INVALID_PARAMETER; + + size_t got = 0; + KSTATUS s = IoReadFileAt(vad->FileObject, fileOffset, pageVa, toRead, &got); + if (!KSUCCESS(s)) { + gVadStats.FileBackedReadFailures++; + return s; + } + gVadStats.FileBackedReads++; + if (got < toRead) + memset((uint8_t*)pageVa + got, 0, toRead - got); + return STATUS_SUCCESS; +} + + +// Materializes one image-backed prototype page or zero-fill page for an image VAD. +static KSTATUS KiVadFillImageBackedPage(MM_VAD* vad, size_t pageIndex, int wantWrite) { + if (!vad || !vad->ImageSection || pageIndex >= vad->PageCount) + return STATUS_INVALID_PARAMETER; + + MM_VAD_PAGE* page = &vad->Pages[pageIndex]; + uintptr_t va = vad->Base + pageIndex * PAGE_SIZE; + + PHYSICAL_ADDRESS frame = 0; + KSTATUS s = LdrRetainImageSectionPage(vad->ImageSection, pageIndex, &frame); + if (!KSUCCESS(s)) + return s; + + if (frame) { + page->Frame = frame; + page->Resident = 1; + page->Flags &= ~(MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SWAPPED); + page->Flags |= MM_VAD_PAGE_IMAGE_BACKED | MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_COW; + page->SwapSlot = 0xFFFFFFFFu; + + s = KiVadMapPageTracked(va, frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + page->Frame = 0; + page->Resident = 0; + page->Flags &= ~(MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_COW); + (void)MmPfnRelease(frame); + return s; + } + + gVadStats.ImageBackedFaults++; + return STATUS_SUCCESS; + } + + if (!wantWrite && gVadZeroPageFrame) { + page->Frame = gVadZeroPageFrame; + page->Resident = 1; + page->Flags |= MM_VAD_PAGE_IMAGE_BACKED | MM_VAD_PAGE_ZERO_PAGE; + if (page->PageFlags & PAGE_WRITE) + page->Flags |= MM_VAD_PAGE_COW; + page->SwapSlot = 0xFFFFFFFFu; + + s = KiVadMapPageTracked(va, page->Frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + page->Frame = 0; + page->Resident = 0; + page->Flags &= ~(MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_COW); + return s; + } + + gVadStats.ImageBackedZeroFaults++; + return STATUS_SUCCESS; + } + + frame = MmPfnAllocate(MmPfnUseImage); + if (!frame) + return STATUS_NO_MEMORY; + + void* pageVa = MmPfnToVirtual(frame); + if (!pageVa) { + MmPfnFree(frame); + return STATUS_FAILURE; + } + memset(pageVa, 0, PAGE_SIZE); + + page->Frame = frame; + page->Resident = 1; + page->Flags &= ~(MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_SWAPPED); + page->Flags |= MM_VAD_PAGE_IMAGE_BACKED; + page->SwapSlot = 0xFFFFFFFFu; + + s = KiVadMapPageTracked(va, frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + page->Frame = 0; + page->Resident = 0; + MmPfnFree(frame); + return s; + } + + gVadStats.ImageBackedPrivateFaults++; + return STATUS_SUCCESS; +} + +// Materializes one committed VAD page as either the shared zero page or a private frame. +static KSTATUS KiVadEnsurePageResident(MM_VAD* vad, size_t pageIndex, int wantWrite, int wantExecute) { + if (!vad || pageIndex >= vad->PageCount) + return STATUS_NOT_FOUND; + + MM_VAD_PAGE* page = &vad->Pages[pageIndex]; +retry_after_paging: + KiVadWaitForPagingComplete(page); + + KSTATUS s = KiVadCheckPageAccess(page, wantWrite, wantExecute); + if (!KSUCCESS(s)) + return s; + + uintptr_t va = vad->Base + pageIndex * PAGE_SIZE; + if (page->Resident) { + gVadStats.AlreadyResidentPages++; + return STATUS_SUCCESS; + } + + if (!KiVadTryBeginPaging(page)) + goto retry_after_paging; + + // Re-check after claiming the transition bit. Another CPU may have resolved + // the page between our initial wait/access check and the CAS above. + s = KiVadCheckPageAccess(page, wantWrite, wantExecute); + if (!KSUCCESS(s)) { + KiVadEndPaging(page); + return s; + } + if (page->Resident) { + KiVadEndPaging(page); + gVadStats.AlreadyResidentPages++; + return STATUS_SUCCESS; + } + + if ((page->Flags & MM_VAD_PAGE_SWAPPED) && page->SwapSlot != 0xFFFFFFFFu) { + uint32_t oldSlot = page->SwapSlot; + + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseAnonymous); + if (!frame) { + KiVadEndPaging(page); + return STATUS_NO_MEMORY; + } + + KSTATUS swapStatus = MmSwapPageIn(oldSlot, frame); + if (!KSUCCESS(swapStatus)) { + MmPfnFree(frame); + KiVadEndPaging(page); + return swapStatus; + } + + page->Frame = frame; + page->Resident = 1; + page->Flags &= ~MM_VAD_PAGE_SWAPPED; + page->SwapSlot = 0xFFFFFFFFu; + + KSTATUS mapStatus = KiVadMapPageTracked(va, frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(mapStatus)) { + page->Frame = 0; + page->Resident = 0; + page->Flags |= MM_VAD_PAGE_SWAPPED; + page->SwapSlot = oldSlot; + KiVadEndPaging(page); + MmPfnFree(frame); + return mapStatus; + } + + KiVadEndPaging(page); + MmSwapFreeSlot(oldSlot); + gVadStats.SwapPageIns++; + return STATUS_SUCCESS; + } + + if (page->Flags & MM_VAD_PAGE_IMAGE_BACKED) { + s = KiVadFillImageBackedPage(vad, pageIndex, wantWrite); + KiVadEndPaging(page); + return s; + } + + if (page->Flags & MM_VAD_PAGE_FILE_BACKED) { + PHYSICAL_ADDRESS frame = 0; + MM_FILE_CACHE_PAGE cachePage; + memset(&cachePage, 0, sizeof(cachePage)); + + uint64_t fileOffset = vad->FileOffset + (uint64_t)pageIndex * (uint64_t)PAGE_SIZE; + if (fileOffset < vad->FileOffset) { + KiVadEndPaging(page); + return STATUS_INVALID_PARAMETER; + } + + if (page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) { + uintptr_t f = KeAcquireSpinlock(&vad->FileObject->Lock); + uint64_t oldSize = vad->FileObject->Size; + KeReleaseSpinlock(&vad->FileObject->Lock, f); + int needExisting = fileOffset < oldSize; + s = MmFileCacheGetWritablePage(vad->FileObject, fileOffset, oldSize, needExisting, &cachePage); + if (!KSUCCESS(s)) { + gVadStats.FileBackedReadFailures++; + KiVadEndPaging(page); + return s; + } + frame = cachePage.Frame; + page->Flags |= MM_VAD_PAGE_FILE_DIRTY_TRACK; + gVadStats.FileBackedSharedWritableFaults++; + } else if ((page->PageFlags & PAGE_WRITE) == 0) { + s = MmFileCacheGetPage(vad->FileObject, fileOffset, &cachePage); + if (!KSUCCESS(s)) { + gVadStats.FileBackedReadFailures++; + KiVadEndPaging(page); + return s; + } + frame = cachePage.Frame; + } else { + frame = MmPfnAllocate(MmPfnUseMappedFile); + if (!frame) { + KiVadEndPaging(page); + return STATUS_NO_MEMORY; + } + + s = KiVadFillFileBackedPage(vad, pageIndex, frame); + if (!KSUCCESS(s)) { + KiVadEndPaging(page); + MmPfnFree(frame); + return s; + } + } + + page->Frame = frame; + page->Resident = 1; + page->Flags &= ~(MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_SWAPPED); + page->SwapSlot = 0xFFFFFFFFu; + + s = KiVadMapPageTracked(va, frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + page->Frame = 0; + page->Resident = 0; + KiVadEndPaging(page); + if (cachePage.Frame) MmFileCacheReleasePage(&cachePage); + else MmPfnFree(frame); + return s; + } + + KiVadEndPaging(page); + gVadStats.FileBackedFaults++; + return STATUS_SUCCESS; + } + + if (!wantWrite && gVadZeroPageFrame) { + uint32_t savedFlags = page->Flags; + page->Frame = gVadZeroPageFrame; + page->Resident = 1; + page->Flags |= MM_VAD_PAGE_ZERO_PAGE; + if (page->PageFlags & PAGE_WRITE) + page->Flags |= MM_VAD_PAGE_COW; + + s = KiVadMapPageTracked(va, page->Frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + page->Frame = 0; + page->Resident = 0; + page->Flags = savedFlags; + KiVadEndPaging(page); + return s; + } + + KiVadEndPaging(page); + gVadStats.ZeroPageMaps++; + return STATUS_SUCCESS; + } + + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseAnonymous); + if (!frame) { + KiVadEndPaging(page); + return STATUS_NO_MEMORY; + } + + void* pageVa = MmPfnToVirtual(frame); + if (!pageVa) { + KiVadEndPaging(page); + MmPfnFree(frame); + return STATUS_FAILURE; + } + memset(pageVa, 0, PAGE_SIZE); + + page->Frame = frame; + page->Resident = 1; + page->Flags &= ~(MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_SWAPPED); + page->SwapSlot = 0xFFFFFFFFu; + + s = KiVadMapPageTracked(va, frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + page->Frame = 0; + page->Resident = 0; + KiVadEndPaging(page); + MmPfnFree(frame); + return s; + } + + KiVadEndPaging(page); + gVadStats.DemandZeroPages++; + return STATUS_SUCCESS; +} + +// Splits a copy-on-write page so this VAD receives a private writable frame. +static KSTATUS KiVadBreakCopyOnWrite(MM_VAD* vad, size_t pageIndex) { + if (!vad || pageIndex >= vad->PageCount) + return STATUS_NOT_FOUND; + + MM_VAD_PAGE* page = &vad->Pages[pageIndex]; + if (page->State != MmVadPageCommitted || !(page->Flags & MM_VAD_PAGE_COW)) + return STATUS_ACCESS_DENIED; + if (!page->Resident || !page->Frame) + return STATUS_NOT_FOUND; + if ((page->PageFlags & PAGE_WRITE) == 0) + return STATUS_ACCESS_DENIED; + + PHYSICAL_ADDRESS newFrame = MmPfnAllocate(MmPfnUseCow); + if (!newFrame) + return STATUS_NO_MEMORY; + + void* newPage = MmPfnToVirtual(newFrame); + if (!newPage) { + MmPfnFree(newFrame); + return STATUS_FAILURE; + } + + if (page->Flags & MM_VAD_PAGE_ZERO_PAGE) { + memset(newPage, 0, PAGE_SIZE); + gVadStats.ZeroPageCowSplits++; + } else { + void* oldPage = MmPfnToVirtual(page->Frame); + if (!oldPage) { + MmPfnFree(newFrame); + return STATUS_FAILURE; + } + memcpy(newPage, oldPage, PAGE_SIZE); + } + + uintptr_t va = vad->Base + pageIndex * PAGE_SIZE; + KSTATUS s = KiVadUnmapPageTracked(va, page->Frame); + if (!KSUCCESS(s)) { + MmPfnFree(newFrame); + return s; + } + + s = KiVadMapPageTracked(va, newFrame, page->PageFlags, MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + (void)KiVadMapPageTracked(va, page->Frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + MmPfnFree(newFrame); + return s; + } + + if (!(page->Flags & MM_VAD_PAGE_ZERO_PAGE)) + (void)MmPfnRelease(page->Frame); + page->Frame = newFrame; + page->Flags &= ~(MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_IMAGE_BACKED); + page->Resident = 1; + gVadStats.CowPagesCopied++; + return STATUS_SUCCESS; +} + +// Ensures that every committed page in a range is resident and accessible. +static KSTATUS KiVadEnsureResidentRange(PROCESS* process, uintptr_t start, size_t size, uint64_t accessFlags) { + if (!process || !size) + return STATUS_INVALID_PARAMETER; + if (start > (uintptr_t)-1 - (size - 1)) + return STATUS_NO_MEMORY; + + uintptr_t alignedStart = start & PAGE_MASK; + uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; + if (alignedEnd <= alignedStart) + return STATUS_NO_MEMORY; + + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindReservedVadContainingLocked(process, alignedStart, alignedEnd - alignedStart); + KeReleaseSpinlock(&process->VadLock, procFlags); + if (!vad) + return STATUS_NOT_FOUND; + + int wantWrite = (accessFlags & PAGE_WRITE) ? 1 : 0; + int wantExecute = (accessFlags & PAGE_NX) ? 1 : 0; + size_t first = KiVadPageIndex(vad, alignedStart); + size_t count = KiVadRangePageCount(alignedStart, alignedEnd); + if (first > vad->PageCount || count > vad->PageCount - first) + return STATUS_ACCESS_DENIED; + + gVadStats.EnsureResidentCalls++; + for (size_t i = 0; i < count; ++i) { + KSTATUS s = KiVadEnsurePageResident(vad, first + i, wantWrite, wantExecute); + if (!KSUCCESS(s)) + return s; + if (wantWrite && (vad->Pages[first + i].Flags & MM_VAD_PAGE_COW)) { + s = KiVadBreakCopyOnWrite(vad, first + i); + if (!KSUCCESS(s)) + return s; + } + gVadStats.EnsureResidentPages++; + } + + return STATUS_SUCCESS; +} + +// Makes an image VAD range private before the loader patches relocations or imports. +KSTATUS MmVadMakeImageRangePrivate(PROCESS* process, void* base, size_t size) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process || !base || !size) + return STATUS_INVALID_PARAMETER; + + uintptr_t start = (uintptr_t)base; + if (start > (uintptr_t)-1 - (size - 1)) + return STATUS_NO_MEMORY; + + uintptr_t alignedStart = start & PAGE_MASK; + uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; + if (alignedEnd <= alignedStart) + return STATUS_NO_MEMORY; + + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindReservedVadContainingLocked(process, alignedStart, alignedEnd - alignedStart); + KeReleaseSpinlock(&process->VadLock, procFlags); + if (!vad) + return STATUS_NOT_FOUND; + if (vad->Type != MmVadImage) + return STATUS_INVALID_PARAMETER; + + size_t first = KiVadPageIndex(vad, alignedStart); + size_t count = KiVadRangePageCount(alignedStart, alignedEnd); + if (first > vad->PageCount || count > vad->PageCount - first) + return STATUS_ACCESS_DENIED; + + gVadStats.ImageFixupPrivateCalls++; + for (size_t i = 0; i < count; ++i) { + MM_VAD_PAGE* page = &vad->Pages[first + i]; + + KSTATUS s = KiVadEnsurePageResident(vad, first + i, 1, 0); + if (!KSUCCESS(s)) + return s; + + if (page->Flags & MM_VAD_PAGE_COW) { + s = KiVadBreakCopyOnWrite(vad, first + i); + if (!KSUCCESS(s)) + return s; + } + + if ((page->Flags & MM_VAD_PAGE_IMAGE_PRIVATE) == 0) { + page->Flags |= MM_VAD_PAGE_IMAGE_PRIVATE | MM_VAD_PAGE_IMAGE_FIXUP; + page->Flags &= ~(MM_VAD_PAGE_IMAGE_BACKED | MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_ZERO_PAGE); + gVadStats.ImageFixupPrivatePages++; + } else { + page->Flags |= MM_VAD_PAGE_IMAGE_FIXUP; } } + + return STATUS_SUCCESS; } -KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment, - uint64_t pageFlags, uint32_t regionFlags, - USER_REGION_TYPE type, void** outBase) { +// Pins the user page that contains the supplied address after materializing it. +// The returned KernelAddress is a direct-map alias for the physical frame, so +// callers do not continue touching the original user VA after validation. This +// closes the unmap-after-pin race and is the primitive future zero-copy-ish I/O +// paths should build on. +static KSTATUS KiVadPinUserPage(PROCESS* process, uintptr_t address, + uint64_t accessFlags, MM_PINNED_USER_PAGE* out) { + if (!process || !out) + return STATUS_INVALID_PARAMETER; + memset(out, 0, sizeof(*out)); + + if (!KiUserRangeValid(address, 1)) + return STATUS_ACCESS_DENIED; + + uintptr_t pageBase = address & PAGE_MASK; + KSTATUS s = KiVadEnsureResidentRange(process, pageBase, PAGE_SIZE, accessFlags | PAGE_USER); + if (!KSUCCESS(s)) + return s; + + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindReservedVadContainingLocked(process, pageBase, PAGE_SIZE); + PHYSICAL_ADDRESS frame = 0; + uint64_t pageFlags = 0; + if (vad) { + size_t index = KiVadPageIndex(vad, pageBase); + if (index < vad->PageCount) { + MM_VAD_PAGE* page = &vad->Pages[index]; + if (page->State == MmVadPageCommitted && page->Resident && page->Frame && page->PageFlags) { + frame = page->Frame; + pageFlags = page->PageFlags; + } + } + } + KeReleaseSpinlock(&process->VadLock, procFlags); + + if (!frame) + return STATUS_NOT_FOUND; + + s = MmPfnPin(frame); + if (!KSUCCESS(s)) + return s; + + procFlags = KeAcquireSpinlock(&process->VadLock); + vad = KiFindReservedVadContainingLocked(process, pageBase, PAGE_SIZE); + int stillValid = 0; + if (vad) { + size_t index = KiVadPageIndex(vad, pageBase); + if (index < vad->PageCount) { + MM_VAD_PAGE* page = &vad->Pages[index]; + stillValid = page->State == MmVadPageCommitted && + page->Resident && + page->Frame == frame && + page->PageFlags != 0 && + ((accessFlags & PAGE_WRITE) == 0 || (page->PageFlags & PAGE_WRITE)); + if (stillValid) + pageFlags = page->PageFlags; + } + } + KeReleaseSpinlock(&process->VadLock, procFlags); + + if (!stillValid) { + (void)MmPfnUnpin(frame); + return STATUS_NOT_FOUND; + } + + out->UserPage = pageBase; + out->PageOffset = (size_t)(address & (PAGE_SIZE - 1)); + out->Frame = frame; + out->KernelAddress = MmPfnToVirtual(frame); + out->PageFlags = pageFlags; + if (!out->KernelAddress) { + MmUnpinUserPage(out); + return STATUS_FAILURE; + } + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS MmPinUserPageForRead(PROCESS* process, const void* userAddress, MM_PINNED_USER_PAGE* out) { + return KiVadPinUserPage(process, (uintptr_t)userAddress, PAGE_PRESENT | PAGE_USER, out); +} + +__declspec(dllexport) KSTATUS MmPinUserPageForWrite(PROCESS* process, void* userAddress, MM_PINNED_USER_PAGE* out) { + return KiVadPinUserPage(process, (uintptr_t)userAddress, PAGE_PRESENT | PAGE_USER | PAGE_WRITE, out); +} + +__declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page) { + if (!page || !page->Frame) + return; + (void)MmPfnUnpin(page->Frame); + memset(page, 0, sizeof(*page)); +} + +typedef struct MM_LOCKED_USER_BUFFER { + PROCESS* Process; + void* UserAddress; + size_t Size; + size_t PageCount; + int WriteAccess; + MM_PINNED_USER_PAGE Pages[1]; +} MM_LOCKED_USER_BUFFER; + +// Returns the per-page VAD lock array that follows the pinned-page array. +static MM_VAD** KiLockedUserBufferVads(MM_LOCKED_USER_BUFFER* buffer) { + return buffer ? (MM_VAD**)(buffer->Pages + buffer->PageCount) : (MM_VAD**)0; +} + +// Adds one outstanding user-buffer lock to the VAD that owns a page. +static KSTATUS KiVadAddUserBufferPageLock(PROCESS* process, uintptr_t pageBase, MM_VAD** vadOut) { + if (vadOut) *vadOut = (MM_VAD*)0; + if (!process || !vadOut) + return STATUS_INVALID_PARAMETER; + + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindReservedVadContainingLocked(process, pageBase, PAGE_SIZE); + if (!vad) { + KeReleaseSpinlock(&process->VadLock, procFlags); + return STATUS_NOT_FOUND; + } + if (vad->LockCount == UINT32_MAX) { + KeReleaseSpinlock(&process->VadLock, procFlags); + return STATUS_FAILURE; + } + vad->LockCount++; + *vadOut = vad; + KeReleaseSpinlock(&process->VadLock, procFlags); + return STATUS_SUCCESS; +} + +// Removes one outstanding user-buffer lock from a VAD. +static void KiVadRemoveUserBufferPageLock(PROCESS* process, MM_VAD* vad) { + if (!process || !vad) + return; + + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + if (vad->LockCount) + vad->LockCount--; + KeReleaseSpinlock(&process->VadLock, procFlags); +} + +// Locks a user buffer for asynchronous kernel access. +__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, + void* userAddress, + size_t size, + int writeAccess, + MM_LOCKED_USER_BUFFER** out) { + if (!out) + return STATUS_INVALID_PARAMETER; + *out = (MM_LOCKED_USER_BUFFER*)0; + + if (size == 0) + return STATUS_SUCCESS; + if (!process || !userAddress) + return STATUS_INVALID_PARAMETER; + + uintptr_t start = (uintptr_t)userAddress; + if (!KiUserRangeValid(start, size)) + return STATUS_ACCESS_DENIED; + + uintptr_t alignedStart = start & PAGE_MASK; + uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; + if (alignedEnd <= alignedStart) + return STATUS_NO_MEMORY; + + size_t pageCount = KiVadRangePageCount(alignedStart, alignedEnd); + if (!pageCount) + return STATUS_INVALID_PARAMETER; + if (pageCount > (((size_t)-1) - sizeof(MM_LOCKED_USER_BUFFER)) / + (sizeof(MM_PINNED_USER_PAGE) + sizeof(MM_VAD*))) + return STATUS_NO_MEMORY; + + size_t allocSize = sizeof(MM_LOCKED_USER_BUFFER) + + (pageCount - 1) * sizeof(MM_PINNED_USER_PAGE) + + pageCount * sizeof(MM_VAD*); + MM_LOCKED_USER_BUFFER* buffer = (MM_LOCKED_USER_BUFFER*)MmAllocatePoolTag(allocSize, MM_POOL_TAG('u', 'l', 'o', 'k')); + if (!buffer) + return STATUS_NO_MEMORY; + + memset(buffer, 0, allocSize); + buffer->Process = process; + buffer->UserAddress = userAddress; + buffer->Size = size; + buffer->PageCount = pageCount; + buffer->WriteAccess = writeAccess ? 1 : 0; + ObReferenceObject(&process->Header); + + MM_VAD** vads = KiLockedUserBufferVads(buffer); + KSTATUS s = STATUS_SUCCESS; + size_t lockedPages = 0; + for (size_t i = 0; i < pageCount; ++i) { + uintptr_t pageBase = alignedStart + i * PAGE_SIZE; + s = KiVadAddUserBufferPageLock(process, pageBase, &vads[i]); + if (!KSUCCESS(s)) + break; + + s = writeAccess + ? MmPinUserPageForWrite(process, (void*)pageBase, &buffer->Pages[i]) + : MmPinUserPageForRead(process, (const void*)pageBase, &buffer->Pages[i]); + if (!KSUCCESS(s)) { + KiVadRemoveUserBufferPageLock(process, vads[i]); + vads[i] = (MM_VAD*)0; + break; + } + lockedPages++; + } + + if (!KSUCCESS(s)) { + for (size_t i = 0; i < lockedPages; ++i) + MmUnpinUserPage(&buffer->Pages[i]); + for (size_t i = 0; i < pageCount; ++i) + KiVadRemoveUserBufferPageLock(process, vads[i]); + ObDereferenceObject(&process->Header); + MmFreePool(buffer); + return s; + } + + *out = buffer; + return STATUS_SUCCESS; +} + +// Unlocks a user buffer locked for asynchronous kernel access. +__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer) { + if (!buffer) + return; + + MM_VAD** vads = KiLockedUserBufferVads(buffer); + for (size_t i = 0; i < buffer->PageCount; ++i) + MmUnpinUserPage(&buffer->Pages[i]); + for (size_t i = 0; i < buffer->PageCount; ++i) + KiVadRemoveUserBufferPageLock(buffer->Process, vads[i]); + + if (buffer->Process) { + ObDereferenceObject(&buffer->Process->Header); + buffer->Process = (PROCESS*)0; + } + MmFreePool(buffer); +} + +// Copies user memory one page at a time while the current page is pinned. +// The pin is short-lived and exists only to keep pressure reclaim from removing +// the PTE while the kernel is executing the memcpy. +static KSTATUS KiCopyUserPaged(void* dst, const void* src, size_t size, int toUser) { + if (size == 0) + return STATUS_SUCCESS; + if (!dst || !src) + return STATUS_INVALID_PARAMETER; + + uintptr_t user = toUser ? (uintptr_t)dst : (uintptr_t)src; + if (!KiUserRangeValid(user, size)) + return STATUS_ACCESS_DENIED; + + PROCESS* process = KeGetCurrentProcess(); + if (!process) + return STATUS_ACCESS_DENIED; + + uint8_t* d = (uint8_t*)dst; + const uint8_t* s = (const uint8_t*)src; + size_t done = 0; + uint64_t access = PAGE_PRESENT | PAGE_USER | (toUser ? PAGE_WRITE : 0); + + while (done < size) { + uintptr_t userAddress = user + done; + size_t pageOffset = (size_t)(userAddress & (PAGE_SIZE - 1)); + size_t chunk = PAGE_SIZE - pageOffset; + if (chunk > size - done) + chunk = size - done; + + MM_PINNED_USER_PAGE pinned; + KSTATUS st = toUser + ? MmPinUserPageForWrite(process, (void*)userAddress, &pinned) + : MmPinUserPageForRead(process, (const void*)userAddress, &pinned); + if (!KSUCCESS(st)) + return st; + + uint8_t* kernelPage = (uint8_t*)pinned.KernelAddress; + if (toUser) + memcpy(kernelPage + pinned.PageOffset, s + done, chunk); + else + memcpy(d + done, kernelPage + pinned.PageOffset, chunk); + + MmUnpinUserPage(&pinned); + done += chunk; + } + + return STATUS_SUCCESS; +} + +// Copies from user memory after pinning each source page for the duration of the copy. +static KSTATUS KiCopyFromUserPaged(void* kernelDst, const void* userSrc, size_t size) { + return KiCopyUserPaged(kernelDst, userSrc, size, 0); +} + +// Copies to user memory after pinning each destination page for the duration of the copy. +static KSTATUS KiCopyToUserPaged(void* userDst, const void* kernelSrc, size_t size) { + return KiCopyUserPaged(userDst, kernelSrc, size, 1); +} + + +// Reserves a VAD in the global user VA arena without committing physical pages. +KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment, + uint64_t pageFlags, uint32_t vadFlags, + MM_VAD_TYPE type, void** outBase) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !outBase || !size) return STATUS_INVALID_PARAMETER; *outBase = (void*)0; - if (!(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER; - if (!(pageFlags & PAGE_PRESENT)) pageFlags |= PAGE_PRESENT; - size_t payloadSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); if (payloadSize < size) return STATUS_NO_MEMORY; - size_t guardLow = (regionFlags & USER_REGION_GUARD_LOW) ? PAGE_SIZE : 0; - size_t guardHigh = (regionFlags & USER_REGION_GUARD_HIGH) ? PAGE_SIZE : 0; + size_t guardLow = (vadFlags & MM_VAD_GUARD_LOW) ? PAGE_SIZE : 0; + size_t guardHigh = (vadFlags & MM_VAD_GUARD_HIGH) ? PAGE_SIZE : 0; size_t reservationSize = guardLow + payloadSize + guardHigh; if (reservationSize < payloadSize) return STATUS_NO_MEMORY; if (alignment < PAGE_SIZE) alignment = PAGE_SIZE; if (alignment & (alignment - 1)) return STATUS_INVALID_PARAMETER; - USER_VA_REGION* r = (USER_VA_REGION*)MmAllocatePool(sizeof(USER_VA_REGION)); - if (!r) return STATUS_NO_MEMORY; - memset(r, 0, sizeof(*r)); - LIST_INIT(&r->ProcessLink); - LIST_INIT(&r->GlobalLink); + size_t pageCount = payloadSize / PAGE_SIZE; + if (!pageCount) return STATUS_INVALID_PARAMETER; + + if (pageFlags && !(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER; + + MM_VAD* vad = (MM_VAD*)MmAllocatePoolTag(sizeof(MM_VAD), MM_POOL_TAG('v', 'a', 'd', ' ')); + if (!vad) return STATUS_NO_MEMORY; + memset(vad, 0, sizeof(*vad)); + + MM_VAD_PAGE* pages = (MM_VAD_PAGE*)MmAllocatePoolTag(pageCount * sizeof(MM_VAD_PAGE), MM_POOL_TAG('v', 'p', 'g', 's')); + if (!pages) { + MmFreePool(vad); + return STATUS_NO_MEMORY; + } + memset(pages, 0, pageCount * sizeof(MM_VAD_PAGE)); + + LIST_INIT(&vad->ProcessLink); + LIST_INIT(&vad->GlobalLink); - uintptr_t lockFlags = KeAcquireSpinlock(&gUserRegionLock); uintptr_t reservationBase = 0; - KSTATUS s = KiFindUserVaRange(reservationSize, alignment, &reservationBase); + uintptr_t globalFlags = KeAcquireSpinlock(&gVadLock); + KSTATUS s = KiFindVadRange(reservationSize, alignment, &reservationBase); if (KSUCCESS(s)) { - r->Owner = process; - r->ReservationBase = reservationBase; - r->ReservationSize = reservationSize; - r->Base = reservationBase + guardLow; - r->Size = size; - r->CommittedSize = payloadSize; - r->PageFlags = pageFlags; - r->Type = type; - r->Flags = regionFlags; - ListInsertTail(&gUserRegionList, &r->GlobalLink); + vad->Owner = process; + vad->ReservationBase = reservationBase; + vad->ReservationSize = reservationSize; + vad->Base = reservationBase + guardLow; + vad->Size = size; + vad->PayloadSize = payloadSize; + vad->PageCount = pageCount; + vad->CommittedPages = 0; + vad->PageFlags = pageFlags; + vad->Type = type; + vad->Flags = vadFlags; + vad->Pages = pages; + ListInsertTail(&gVadList, &vad->GlobalLink); } - KeReleaseSpinlock(&gUserRegionLock, lockFlags); + KeReleaseSpinlock(&gVadLock, globalFlags); if (!KSUCCESS(s)) { - MmFreePool(r); + MmFreePool(pages); + MmFreePool(vad); return s; } - uintptr_t va = r->Base; - for (; va < r->Base + payloadSize; va += PAGE_SIZE) { - PHYSICAL_ADDRESS frame = MmAllocateFrame(); - if (!frame) { - s = STATUS_NO_MEMORY; - break; - } - s = MmMapPage(va, frame, pageFlags, MmMapFailIfPresent); - if (!KSUCCESS(s)) { - MmFreeFrame(frame); - break; - } - } + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + ListInsertTail(&process->VadList, &vad->ProcessLink); + KeReleaseSpinlock(&process->VadLock, procFlags); - if (!KSUCCESS(s)) { - r->CommittedSize = (va > r->Base) ? (size_t)(va - r->Base) : 0; - KiUnmapUserRegionPayload(r); - lockFlags = KeAcquireSpinlock(&gUserRegionLock); - ListRemove(&r->GlobalLink); - KeReleaseSpinlock(&gUserRegionLock, lockFlags); - MmFreePool(r); - return s; - } - - if (regionFlags & USER_REGION_ZERO) { - memset((void*)r->Base, 0, payloadSize); - } - - uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); - ListInsertTail(&process->UserRegions, &r->ProcessLink); - KeReleaseSpinlock(&process->UserRegionLock, procFlags); - - *outBase = (void*)r->Base; + *outBase = (void*)vad->Base; return STATUS_SUCCESS; } -// Implements the ensure User Region Mapped operation. -KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) { +// Commits pages in an existing VAD without allocating PFNs until access. +KSTATUS MmVadCommit(PROCESS* process, void* base, size_t size, + uint64_t pageFlags, uint32_t commitFlags) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !base || !size) return STATUS_INVALID_PARAMETER; @@ -1031,51 +1992,845 @@ KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY; - uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); - USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart); - uintptr_t regionBase = r ? r->Base : 0; - uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0; - uint64_t defaultFlags = r ? r->PageFlags : 0; - KeReleaseSpinlock(&process->UserRegionLock, procFlags); + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindReservedVadContainingLocked(process, alignedStart, alignedEnd - alignedStart); + KeReleaseSpinlock(&process->VadLock, procFlags); - if (!r) { - DbgPrint("MM: ensure user mapping failed: pid=%u range=%016llx-%016llx not inside owned committed region\n", + if (!vad) { + DbgPrint("MM: commit failed: pid=%u range=%016llx-%016llx not inside reserved VAD\n", process->ProcessId, (unsigned long long)alignedStart, (unsigned long long)alignedEnd); return STATUS_NOT_FOUND; } - uint64_t flags = pageFlags ? pageFlags : defaultFlags; - flags |= PAGE_PRESENT | PAGE_USER; + uint64_t flags = pageFlags ? pageFlags : vad->PageFlags; + if (flags) flags |= PAGE_PRESENT | PAGE_USER; - uint32_t committed = 0; - for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { - if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED; + size_t first = KiVadPageIndex(vad, alignedStart); + size_t count = KiVadRangePageCount(alignedStart, alignedEnd); + if (first > vad->PageCount || count > vad->PageCount - first) + return STATUS_ACCESS_DENIED; - PHYSICAL_ADDRESS phys = 0; - uint64_t existingFlags = 0; - KSTATUS qs = MmQueryPage(va, &phys, &existingFlags); - if (KSUCCESS(qs) && (existingFlags & PAGE_PRESENT)) + uint8_t* newlyCommitted = (uint8_t*)MmAllocatePool(count ? count : 1); + if (!newlyCommitted) return STATUS_NO_MEMORY; + memset(newlyCommitted, 0, count ? count : 1); + + KSTATUS s = STATUS_SUCCESS; + for (size_t i = 0; i < count; ++i) { + size_t pageIndex = first + i; + MM_VAD_PAGE* page = &vad->Pages[pageIndex]; + + if (page->State == MmVadPageCommitted) { + continue; + } + + page->State = MmVadPageCommitted; + page->PageFlags = flags; + page->Frame = 0; + page->Resident = 0; + page->Flags = 0; + page->SwapSlot = 0xFFFFFFFFu; + vad->CommittedPages++; + newlyCommitted[i] = 1; + } + + if ((commitFlags & MM_VAD_PREFETCH) && flags) { + s = KiVadEnsureResidentRange(process, alignedStart, alignedEnd - alignedStart, flags); + } + + if (!KSUCCESS(s)) { + for (size_t i = 0; i < count; ++i) { + if (newlyCommitted[i]) + KiVadReleasePage(vad, first + i); + } + } + + MmFreePool(newlyCommitted); + return s; +} + +// Reserves and commits a VAD in one step. +KSTATUS MmVadCreateCommitted(PROCESS* process, size_t size, size_t alignment, + uint64_t pageFlags, uint32_t vadFlags, + MM_VAD_TYPE type, void** outBase) { + if (!outBase) return STATUS_INVALID_PARAMETER; + *outBase = (void*)0; + + void* base = (void*)0; + KSTATUS s = MmVadReserve(process, size, alignment, pageFlags, vadFlags, type, &base); + if (!KSUCCESS(s)) return s; + + s = MmVadCommit(process, base, size, pageFlags, vadFlags); + if (!KSUCCESS(s)) { + MmVadDestroy(process, base); + return s; + } + + *outBase = base; + return STATUS_SUCCESS; +} + +// Creates a demand-paged file-backed user mapping. The VAD owns a reference to +// the FILE_OBJECT so the caller may close the original file handle immediately +// after a view is established. Pages are read from the file on first fault and +// then behave like ordinary private user pages: if memory pressure trims them +// after they are resident, the swap path preserves later private modifications. +KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file, + uint64_t fileOffset, size_t size, + uint64_t pageFlags, uint32_t vadFlags, + void** outBase) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process || !file || !size || !outBase) + return STATUS_INVALID_PARAMETER; + *outBase = (void*)0; + + if (fileOffset & (PAGE_SIZE - 1)) + return STATUS_INVALID_PARAMETER; + + size_t alignedSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (!alignedSize || alignedSize < size) + return STATUS_NO_MEMORY; + + if (pageFlags && !(pageFlags & PAGE_USER)) + pageFlags |= PAGE_USER; + pageFlags |= PAGE_PRESENT | PAGE_USER; + + void* base = (void*)0; + KSTATUS s = MmVadReserve(process, alignedSize, PAGE_SIZE, pageFlags, + vadFlags, MmVadMappedFile, &base); + if (!KSUCCESS(s)) + return s; + + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindVadByBaseLocked(process, base); + KeReleaseSpinlock(&process->VadLock, procFlags); + if (!vad) { + (void)MmVadDestroy(process, base); + return STATUS_NOT_FOUND; + } + + ObReferenceObject(&file->Header); + vad->FileObject = file; + vad->FileOffset = fileOffset; + vad->FileLength = size; + vad->FileMappingFlags = vadFlags; + + for (size_t i = 0; i < vad->PageCount; ++i) { + MM_VAD_PAGE* page = &vad->Pages[i]; + page->State = MmVadPageCommitted; + page->PageFlags = pageFlags; + page->Frame = 0; + page->Resident = 0; + page->Flags = MM_VAD_PAGE_FILE_BACKED; + if (vadFlags & MM_VAD_SHARED_WRITE) + page->Flags |= MM_VAD_PAGE_FILE_SHARED_WRITE | MM_VAD_PAGE_FILE_DIRTY_TRACK; + page->SwapSlot = 0xFFFFFFFFu; + vad->CommittedPages++; + } + + *outBase = base; + return STATUS_SUCCESS; +} + + +// Creates an image-backed VAD whose pages fault from a shared image section object. +KSTATUS MmVadCreateImageSectionMapping(PROCESS* process, IMAGE_SECTION_OBJECT* section, + uint64_t pageFlags, uint32_t vadFlags, + void** outBase) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process || !section || !outBase) + return STATUS_INVALID_PARAMETER; + *outBase = (void*)0; + + const IMAGE_SECTION_METADATA* metadata = LdrImageSectionMetadata(section); + if (!metadata || !metadata->ImageSize || !metadata->PageCount) + return STATUS_INVALID_PARAMETER; + + pageFlags |= PAGE_PRESENT | PAGE_USER; + + void* base = (void*)0; + KSTATUS s = MmVadReserve(process, metadata->ImageSize, PAGE_SIZE, pageFlags, + vadFlags, MmVadImage, &base); + if (!KSUCCESS(s)) + return s; + + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindVadByBaseLocked(process, base); + KeReleaseSpinlock(&process->VadLock, procFlags); + if (!vad) { + (void)MmVadDestroy(process, base); + return STATUS_NOT_FOUND; + } + + LdrReferenceImageSection(section); + vad->ImageSection = section; + + for (size_t i = 0; i < vad->PageCount; ++i) { + MM_VAD_PAGE* page = &vad->Pages[i]; + page->State = MmVadPageCommitted; + page->PageFlags = pageFlags; + page->Frame = 0; + page->Resident = 0; + page->Flags = MM_VAD_PAGE_IMAGE_BACKED; + page->SwapSlot = 0xFFFFFFFFu; + vad->CommittedPages++; + } + + *outBase = base; + return STATUS_SUCCESS; +} + +// Materializes committed VAD pages before the kernel or a recovered user fault touches them. +KSTATUS MmVadEnsureResident(PROCESS* process, void* base, size_t size, uint64_t accessFlags) { + if (!process || !base || !size) + return STATUS_INVALID_PARAMETER; + return KiVadEnsureResidentRange(process, (uintptr_t)base, size, accessFlags | PAGE_USER); +} + +// Creates a private copy-on-write view of a resident committed VAD range. +KSTATUS MmVadCreateCowView(PROCESS* sourceProcess, void* sourceBase, size_t size, + PROCESS* targetProcess, void** outBase) { + KeAssertIrql(PASSIVE_LEVEL); + if (!sourceProcess || !sourceBase || !targetProcess || !outBase || !size) + return STATUS_INVALID_PARAMETER; + *outBase = (void*)0; + + uintptr_t start = (uintptr_t)sourceBase; + if (start & (PAGE_SIZE - 1)) + return STATUS_INVALID_PARAMETER; + size_t alignedSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (!alignedSize || alignedSize < size) + return STATUS_NO_MEMORY; + + KSTATUS s = KiVadEnsureResidentRange(sourceProcess, start, alignedSize, PAGE_USER); + if (!KSUCCESS(s)) + return s; + + uintptr_t srcFlags = KeAcquireSpinlock(&sourceProcess->VadLock); + MM_VAD* sourceVad = KiFindReservedVadContainingLocked(sourceProcess, start, alignedSize); + KeReleaseSpinlock(&sourceProcess->VadLock, srcFlags); + if (!sourceVad || !KiVadRangeCommitted(sourceVad, start, start + alignedSize)) + return STATUS_NOT_FOUND; + + void* targetBase = (void*)0; + s = MmVadReserve(targetProcess, alignedSize, PAGE_SIZE, sourceVad->PageFlags, sourceVad->Flags, sourceVad->Type, &targetBase); + if (!KSUCCESS(s)) + return s; + + uintptr_t dstFlags = KeAcquireSpinlock(&targetProcess->VadLock); + MM_VAD* targetVad = KiFindVadByBaseLocked(targetProcess, targetBase); + KeReleaseSpinlock(&targetProcess->VadLock, dstFlags); + if (!targetVad) { + (void)MmVadDestroy(targetProcess, targetBase); + return STATUS_NOT_FOUND; + } + + uintptr_t dst = (uintptr_t)targetBase; + size_t first = KiVadPageIndex(sourceVad, start); + size_t count = alignedSize / PAGE_SIZE; + + for (size_t i = 0; i < count; ++i) { + MM_VAD_PAGE* srcPage = &sourceVad->Pages[first + i]; + if (srcPage->State != MmVadPageCommitted || !srcPage->Resident || !srcPage->Frame) { + s = STATUS_NOT_FOUND; + break; + } + + s = MmPfnRetain(srcPage->Frame); + if (!KSUCCESS(s)) + break; + + MM_VAD_PAGE* dstPage = &targetVad->Pages[i]; + dstPage->State = MmVadPageCommitted; + dstPage->PageFlags = srcPage->PageFlags; + dstPage->Frame = srcPage->Frame; + dstPage->Resident = 1; + dstPage->Flags = MM_VAD_PAGE_SHARED_FRAME | ((srcPage->PageFlags & PAGE_WRITE) ? MM_VAD_PAGE_COW : 0); + dstPage->SwapSlot = 0xFFFFFFFFu; + targetVad->CommittedPages++; + + if (dstPage->Flags) { + srcPage->Flags |= MM_VAD_PAGE_COW | MM_VAD_PAGE_SHARED_FRAME; + if (srcPage->Resident && srcPage->PageFlags) + (void)MmPteProtectRange(sourceVad->Base + (first + i) * PAGE_SIZE, PAGE_SIZE, KiVadHardwarePageFlags(srcPage)); + } + + s = KiVadMapPageTracked(dst + i * PAGE_SIZE, dstPage->Frame, KiVadHardwarePageFlags(dstPage), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) + break; + + gVadStats.CowPagesShared++; + } + + if (!KSUCCESS(s)) { + (void)MmVadDestroy(targetProcess, targetBase); + return s; + } + + gVadStats.CowViewsCreated++; + *outBase = targetBase; + return STATUS_SUCCESS; +} + +// Creates a COW view from loader-owned template frames. +KSTATUS MmVadCreateCowViewFromFrames(PROCESS* targetProcess, + const PHYSICAL_ADDRESS* frames, + size_t pageCount, + size_t payloadSize, + uint64_t pageFlags, + uint32_t vadFlags, + MM_VAD_TYPE type, + void** outBase) { + KeAssertIrql(PASSIVE_LEVEL); + if (!targetProcess || !frames || !pageCount || !payloadSize || !outBase) + return STATUS_INVALID_PARAMETER; + *outBase = (void*)0; + + size_t alignedSize = (payloadSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (!alignedSize || alignedSize / PAGE_SIZE > pageCount) + return STATUS_INVALID_PARAMETER; + + void* targetBase = (void*)0; + KSTATUS s = MmVadReserve(targetProcess, alignedSize, PAGE_SIZE, pageFlags, vadFlags, type, &targetBase); + if (!KSUCCESS(s)) + return s; + + uintptr_t dstFlags = KeAcquireSpinlock(&targetProcess->VadLock); + MM_VAD* targetVad = KiFindVadByBaseLocked(targetProcess, targetBase); + KeReleaseSpinlock(&targetProcess->VadLock, dstFlags); + if (!targetVad) { + (void)MmVadDestroy(targetProcess, targetBase); + return STATUS_NOT_FOUND; + } + + uintptr_t dst = (uintptr_t)targetBase; + size_t count = alignedSize / PAGE_SIZE; + for (size_t i = 0; i < count; ++i) { + MM_VAD_PAGE* page = &targetVad->Pages[i]; + page->State = MmVadPageCommitted; + page->PageFlags = pageFlags | PAGE_PRESENT | PAGE_USER; + page->Frame = frames[i]; + page->Resident = frames[i] ? 1 : 0; + page->Flags = (frames[i] ? MM_VAD_PAGE_SHARED_FRAME : 0) | ((page->PageFlags & PAGE_WRITE) ? MM_VAD_PAGE_COW : 0); + page->SwapSlot = 0xFFFFFFFFu; + targetVad->CommittedPages++; + + if (!frames[i]) continue; - PHYSICAL_ADDRESS frame = MmAllocateFrame(); - if (!frame) return STATUS_NO_MEMORY; + s = MmPfnRetain(frames[i]); + if (!KSUCCESS(s)) + break; - KSTATUS ms = MmMapPage(va, frame, flags, MmMapFailIfPresent); - if (!KSUCCESS(ms)) { - MmFreeFrame(frame); - return ms; + s = KiVadMapPageTracked(dst + i * PAGE_SIZE, page->Frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + (void)MmPfnRelease(frames[i]); + break; } - memset((void*)va, 0, PAGE_SIZE); - committed++; + + gVadStats.CowPagesShared++; } + if (!KSUCCESS(s)) { + (void)MmVadDestroy(targetProcess, targetBase); + return s; + } + + gVadStats.CowViewsCreated++; + *outBase = targetBase; return STATUS_SUCCESS; } -// Implements the protect User Region operation. -KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) { +KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess, + const PHYSICAL_ADDRESS* frames, + size_t pageCount, + size_t payloadSize, + uint64_t pageFlags, + uint32_t vadFlags, + MM_VAD_TYPE type, + void** outBase) { + KeAssertIrql(PASSIVE_LEVEL); + if (!targetProcess || !frames || !pageCount || !payloadSize || !outBase) + return STATUS_INVALID_PARAMETER; + *outBase = (void*)0; + + size_t alignedSize = (payloadSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (!alignedSize || alignedSize / PAGE_SIZE > pageCount) + return STATUS_INVALID_PARAMETER; + + pageFlags |= PAGE_PRESENT | PAGE_USER; + + void* targetBase = (void*)0; + KSTATUS s = MmVadReserve(targetProcess, alignedSize, PAGE_SIZE, pageFlags, + vadFlags, type, &targetBase); + if (!KSUCCESS(s)) + return s; + + uintptr_t dstFlags = KeAcquireSpinlock(&targetProcess->VadLock); + MM_VAD* targetVad = KiFindVadByBaseLocked(targetProcess, targetBase); + KeReleaseSpinlock(&targetProcess->VadLock, dstFlags); + if (!targetVad) { + (void)MmVadDestroy(targetProcess, targetBase); + return STATUS_NOT_FOUND; + } + + uintptr_t dst = (uintptr_t)targetBase; + size_t count = alignedSize / PAGE_SIZE; + for (size_t i = 0; i < count; ++i) { + if (!frames[i]) { + s = STATUS_INVALID_PARAMETER; + break; + } + + s = MmPfnRetain(frames[i]); + if (!KSUCCESS(s)) + break; + + MM_VAD_PAGE* page = &targetVad->Pages[i]; + page->State = MmVadPageCommitted; + page->PageFlags = pageFlags; + page->Frame = frames[i]; + page->Resident = 1; + page->Flags = MM_VAD_PAGE_SHARED_FRAME; + page->SwapSlot = 0xFFFFFFFFu; + targetVad->CommittedPages++; + + s = KiVadMapPageTracked(dst + i * PAGE_SIZE, page->Frame, + KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + if (!KSUCCESS(s)) { + page->PageFlags = 0; + page->Frame = 0; + page->Resident = 0; + page->Flags = 0; + page->State = MmVadPageFree; + page->SwapSlot = 0xFFFFFFFFu; + if (targetVad->CommittedPages) + targetVad->CommittedPages--; + (void)MmPfnRelease(frames[i]); + break; + } + + gVadStats.CowPagesShared++; + } + + if (!KSUCCESS(s)) { + (void)MmVadDestroy(targetProcess, targetBase); + return s; + } + + gVadStats.CowViewsCreated++; + *outBase = targetBase; + return STATUS_SUCCESS; +} + +// Returns whether a resident file-backed page can be discarded instead of +// swapped. This is safe only while the page is still clean by construction: +// read-only mappings cannot have been modified, and the backing file can +// repopulate the page on the next fault. Writable/private file mappings still +// fall back to the swap path because this VMM does not yet track dirty bits. +static int KiVadPageCanDiscardCleanFile(const MM_VAD* vad, const MM_VAD_PAGE* page) { + if (!vad || !page || vad->Type != MmVadMappedFile) + return 0; + if (!vad->FileObject || page->State != MmVadPageCommitted) + return 0; + if (!page->Resident || !page->Frame || !page->PageFlags) + return 0; + if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) == 0) + return 0; + if (page->PageFlags & PAGE_WRITE) + return 0; + if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | + MM_VAD_PAGE_SWAPPED | MM_VAD_PAGE_PAGING)) + return 0; + return 1; +} + +static int KiVadPageCanDiscardCleanFileAfterPagingClaim(const MM_VAD* vad, const MM_VAD_PAGE* page) { + if (!vad || !page || vad->Type != MmVadMappedFile) + return 0; + if (!vad->FileObject || page->State != MmVadPageCommitted) + return 0; + if (!page->Resident || !page->Frame || !page->PageFlags) + return 0; + if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) == 0) + return 0; + if (page->PageFlags & PAGE_WRITE) + return 0; + if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | + MM_VAD_PAGE_SWAPPED)) + return 0; + return (page->Flags & MM_VAD_PAGE_PAGING) != 0; +} + +// Returns whether a resident VAD page is safe to move to swap. Clean read-only +// file-backed pages are intentionally excluded; the pressure trimmer discards +// them and reloads them from the file instead of wasting swap slots. +static int KiVadPageCanSwap(const MM_VAD_PAGE* page) { + if (!page || page->State != MmVadPageCommitted) + return 0; + if (!page->Resident || !page->Frame || !page->PageFlags) + return 0; + if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) && ((page->PageFlags & PAGE_WRITE) == 0)) + return 0; + if (page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) + return 0; + if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | + MM_VAD_PAGE_SWAPPED | MM_VAD_PAGE_PAGING)) + return 0; + return 1; +} + +static int KiVadPageCanSwapAfterPagingClaim(const MM_VAD_PAGE* page) { + if (!page || page->State != MmVadPageCommitted) + return 0; + if (!page->Resident || !page->Frame || !page->PageFlags) + return 0; + if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) && ((page->PageFlags & PAGE_WRITE) == 0)) + return 0; + if (page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) + return 0; + if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | + MM_VAD_PAGE_SWAPPED)) + return 0; + if ((page->Flags & MM_VAD_PAGE_PAGING) == 0) + return 0; + return 1; +} + +// Discards one resident clean file-backed page. No swap slot is allocated; the +// VAD keeps its FILE_BACKED state and refaults the page from the file later. +static KSTATUS KiVadDiscardCleanFilePage(MM_VAD* vad, size_t pageIndex, void** baseOut) { + if (baseOut) *baseOut = (void*)0; + if (!vad || pageIndex >= vad->PageCount) + return STATUS_INVALID_PARAMETER; + + MM_VAD_PAGE* page = &vad->Pages[pageIndex]; + if (!KiVadPageCanDiscardCleanFile(vad, page)) + return STATUS_NOT_FOUND; + + PHYSICAL_ADDRESS frame = page->Frame; + MM_PFN* pfn = MmPfnGet(frame); + if (!pfn || pfn->PinCount) + return STATUS_NOT_FOUND; + + uintptr_t va = vad->Base + pageIndex * PAGE_SIZE; + + // Second chance for hot mapped-file pages. Dropping the active video/WAD + // working set is much worse than swapping cold anonymous pages: it causes + // repeated file reads, page-table churn, and global TLB shootdowns. Clear + // the hardware accessed bit and skip the page once; if it stays cold until + // a later pressure scan, it may be discarded then. + if (MmPteTestAndClearAccessed(va)) + return STATUS_NOT_FOUND; + + if (!KiVadTryBeginPaging(page)) + return STATUS_NOT_FOUND; + + if (!KiVadPageCanDiscardCleanFileAfterPagingClaim(vad, page) || page->Frame != frame) { + KiVadEndPaging(page); + return STATUS_NOT_FOUND; + } + + KSTATUS s = MmPfnPin(frame); + if (!KSUCCESS(s)) { + KiVadEndPaging(page); + return s; + } + + s = KiVadUnmapPageTracked(va, frame); + if (!KSUCCESS(s)) { + KiVadEndPaging(page); + (void)MmPfnUnpin(frame); + return s; + } + + page->Frame = 0; + page->Resident = 0; + page->Flags &= ~(MM_VAD_PAGE_SWAPPED | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_COW); + page->Flags |= MM_VAD_PAGE_FILE_BACKED; + page->SwapSlot = 0xFFFFFFFFu; + KiVadEndPaging(page); + + (void)MmPfnUnpin(frame); + (void)MmPfnRelease(frame); + if (vad->FileObject) { + uint64_t fileOffset = vad->FileOffset + (uint64_t)pageIndex * (uint64_t)PAGE_SIZE; + MmFileCacheInvalidate(vad->FileObject, fileOffset, PAGE_SIZE); + } + + gVadStats.FileBackedDiscards++; + if (baseOut) *baseOut = (void*)va; + return STATUS_SUCCESS; +} + +// Moves one private resident VAD page to the swap backend. +static KSTATUS KiVadTrimPageToSwap(MM_VAD* vad, size_t pageIndex, void** baseOut) { + if (baseOut) *baseOut = (void*)0; + if (!vad || pageIndex >= vad->PageCount) + return STATUS_INVALID_PARAMETER; + + MM_VAD_PAGE* page = &vad->Pages[pageIndex]; + if (!KiVadPageCanSwap(page)) + return STATUS_NOT_FOUND; + + PHYSICAL_ADDRESS frame = page->Frame; + MM_PFN* pfn = MmPfnGet(frame); + if (!pfn || pfn->RefCount != 1 || pfn->PinCount) + return STATUS_NOT_FOUND; + + // Claim the transient paging bit atomically before pinning/removing the + // user PTE. This serializes page-out against another page-out and against + // a concurrent fault on the same VAD page without holding a spinlock across + // pagefile I/O. + if (!KiVadTryBeginPaging(page)) + return STATUS_NOT_FOUND; + + if (!KiVadPageCanSwapAfterPagingClaim(page) || page->Frame != frame) { + KiVadEndPaging(page); + return STATUS_NOT_FOUND; + } + + KSTATUS s = MmPfnPin(frame); + if (!KSUCCESS(s)) { + KiVadEndPaging(page); + return s; + } + + uintptr_t va = vad->Base + pageIndex * PAGE_SIZE; + s = KiVadUnmapPageTracked(va, frame); + if (!KSUCCESS(s)) { + KiVadEndPaging(page); + (void)MmPfnUnpin(frame); + return s; + } + + page->Resident = 0; + + uint32_t slot = 0; + s = MmSwapPageOut(frame, &slot); + if (!KSUCCESS(s)) { + page->Resident = 1; + KSTATUS mapStatus = KiVadMapPageTracked(va, frame, KiVadHardwarePageFlags(page), MmPteMapFailIfPresent); + KiVadEndPaging(page); + (void)MmPfnUnpin(frame); + return KSUCCESS(mapStatus) ? s : mapStatus; + } + + page->Frame = 0; + page->Flags |= MM_VAD_PAGE_SWAPPED; + page->SwapSlot = slot; + KiVadEndPaging(page); + + (void)MmPfnUnpin(frame); + (void)MmPfnRelease(frame); + + gVadStats.SwapPageOuts++; + if (baseOut) *baseOut = (void*)va; + return STATUS_SUCCESS; +} + +// Finds one private resident page in a process and swaps it out. +static KSTATUS KiVadTrimOnePageInProcess(PROCESS* process, void** baseOut) { + if (baseOut) *baseOut = (void*)0; + if (!process) return STATUS_INVALID_PARAMETER; + + for (LIST_ENTRY* node = process->VadList.Fwd; node != &process->VadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, ProcessLink); + for (size_t i = 0; i < vad->PageCount; ++i) { + KSTATUS s = KiVadTrimPageToSwap(vad, i, baseOut); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + } + } + return STATUS_NOT_FOUND; +} + +// Moves one private resident user page to the swap backend for self-tests. +KSTATUS MmVadTrimOnePageForTest(PROCESS* process, void** baseOut) { + KeAssertIrql(PASSIVE_LEVEL); + if (!MmSwapIsEnabled()) return STATUS_NOT_SUPPORTED; + return KiVadTrimOnePageInProcess(process, baseOut); +} + +// Reclaims one resident user page from any process during allocation pressure. +// Prefer clean file-backed pages because they can be dropped and re-read without +// consuming swap. Fall back to anonymous/private swap only when no such page is +// available. +KSTATUS MmVadTrimOnePageForPressure(void) { + if (!gVadListReady) return STATUS_NOT_SUPPORTED; + + KSTATUS cacheDrop = MmFileCacheDropOnePageForPressure(); + if (KSUCCESS(cacheDrop)) + return STATUS_SUCCESS; + + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, GlobalLink); + for (size_t i = 0; i < vad->PageCount; ++i) { + KSTATUS s = KiVadDiscardCleanFilePage(vad, i, (void**)0); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + } + } + + if (!MmSwapIsEnabled()) return STATUS_NOT_SUPPORTED; + + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, GlobalLink); + for (size_t i = 0; i < vad->PageCount; ++i) { + KSTATUS s = KiVadTrimPageToSwap(vad, i, (void**)0); + if (KSUCCESS(s)) + return STATUS_SUCCESS; + } + } + return STATUS_NOT_FOUND; +} + +KSTATUS MmVadSyncFileMappings(FILE_OBJECT* file) { + if (!file) + return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gVadLock); + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, GlobalLink); + if (vad->Type != MmVadMappedFile || vad->FileObject != file || !vad->Pages) + continue; + + for (size_t i = 0; i < vad->PageCount; ++i) { + MM_VAD_PAGE* page = &vad->Pages[i]; + if ((page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) == 0) + continue; + if (!page->Resident || !page->Frame || !page->PageFlags) + continue; + + uintptr_t va = vad->Base + i * PAGE_SIZE; + int wasDirty = MmPteTestAndClearDirty(va); + if (wasDirty) + (void)KiVadMarkSharedFilePageDirty(vad, i, 0); + + if ((page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0) { + page->Flags |= MM_VAD_PAGE_FILE_DIRTY_TRACK; + (void)MmPteProtectRange(va, PAGE_SIZE, KiVadHardwarePageFlags(page)); + } + } + } + KeReleaseSpinlock(&gVadLock, flags); + return STATUS_SUCCESS; +} + +// Resolves a user-mode page fault against committed VAD metadata. +// Resolves a user-mode page fault against committed VAD metadata. +KSTATUS MmVadResolveUserPageFault(PROCESS* process, uintptr_t faultAddress, uint64_t errorCode) { + gVadStats.ResolveFaultCalls++; + + if (!process) + return STATUS_ACCESS_DENIED; + if (!KiUserRangeValid(faultAddress, 1)) { + gVadStats.NoVadFaults++; + gVadStats.FailedFaults++; + return STATUS_ACCESS_DENIED; + } + + uintptr_t va = faultAddress & PAGE_MASK; + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindReservedVadContainingLocked(process, va, PAGE_SIZE); + KeReleaseSpinlock(&process->VadLock, procFlags); + if (!vad) { + gVadStats.NoVadFaults++; + gVadStats.FailedFaults++; + return STATUS_NOT_FOUND; + } + + size_t index = KiVadPageIndex(vad, va); + if (index >= vad->PageCount || vad->Pages[index].State != MmVadPageCommitted) { + gVadStats.UncommittedFaults++; + gVadStats.FailedFaults++; + return STATUS_NOT_FOUND; + } + if (!vad->Pages[index].PageFlags) { + gVadStats.NoAccessFaults++; + gVadStats.FailedFaults++; + return STATUS_ACCESS_DENIED; + } + + int presentFault = (errorCode & 0x1) ? 1 : 0; + int wantWrite = (errorCode & 0x2) ? 1 : 0; + int wantExecute = (errorCode & 0x10) ? 1 : 0; + if (presentFault) { + if (wantWrite && + (vad->Pages[index].Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) && + (vad->Pages[index].Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK)) { + KSTATUS dirty = KiVadMarkSharedFilePageDirty(vad, index, 0); + if (KSUCCESS(dirty)) { + vad->Pages[index].Flags &= ~MM_VAD_PAGE_FILE_DIRTY_TRACK; + KSTATUS ps = MmPteProtectRange(va, PAGE_SIZE, KiVadHardwarePageFlags(&vad->Pages[index])); + if (KSUCCESS(ps)) { + gVadStats.FileBackedDirtyFaults++; + gVadStats.ResolvedFaults++; + return STATUS_SUCCESS; + } + dirty = ps; + } + gVadStats.ProtectionFaults++; + gVadStats.FailedFaults++; + return dirty; + } + if (wantWrite && (vad->Pages[index].Flags & MM_VAD_PAGE_COW)) { + KSTATUS cow = KiVadBreakCopyOnWrite(vad, index); + if (KSUCCESS(cow)) { + gVadStats.CowFaults++; + gVadStats.ResolvedFaults++; + return STATUS_SUCCESS; + } + } + gVadStats.ProtectionFaults++; + gVadStats.FailedFaults++; + return STATUS_ACCESS_DENIED; + } + + KSTATUS s = KiVadEnsurePageResident(vad, index, wantWrite, wantExecute); + if (!KSUCCESS(s)) { + gVadStats.FailedFaults++; + return s; + } + + gVadStats.ResolvedFaults++; + return STATUS_SUCCESS; +} + + +// Resolves a page fault by looking up the owning VAD in the global user arena. +// The single address space intentionally allows the kernel and other processes +// to touch any lower-half mapping, so fault ownership is derived from the VAD +// that owns the virtual address rather than from the currently running process. +KSTATUS MmVadResolvePageFaultByAddress(uintptr_t faultAddress, uint64_t errorCode) { + if (!KiUserRangeValid(faultAddress, 1)) + return STATUS_ACCESS_DENIED; + if (!gVadListReady) + return STATUS_NOT_SUPPORTED; + + uintptr_t va = faultAddress & PAGE_MASK; + PROCESS* owner = (PROCESS*)0; + + uintptr_t flags = KeAcquireSpinlock(&gVadLock); + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, GlobalLink); + uintptr_t end = vad->Base + vad->PayloadSize; + if (va >= vad->Base && va < end) { + owner = vad->Owner; + break; + } + } + KeReleaseSpinlock(&gVadLock, flags); + + if (!owner) + return STATUS_NOT_FOUND; + + return MmVadResolveUserPageFault(owner, faultAddress, errorCode); +} + +// Changes page protections on a committed VAD range. +KSTATUS MmVadProtect(PROCESS* process, void* base, size_t size, uint64_t pageFlags) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !base || !size) return STATUS_INVALID_PARAMETER; @@ -1086,240 +2841,279 @@ KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY; - uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); - USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart); - uintptr_t regionBase = r ? r->Base : 0; - uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0; - KeReleaseSpinlock(&process->UserRegionLock, procFlags); + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindReservedVadContainingLocked(process, alignedStart, alignedEnd - alignedStart); + KeReleaseSpinlock(&process->VadLock, procFlags); - if (!r) { - DbgPrint("MM: protect user region failed: pid=%u range=%016llx-%016llx not owned/committed\n", + if (!vad || !KiVadRangeCommitted(vad, alignedStart, alignedEnd)) { + DbgPrint("MM: protect VAD failed: pid=%u range=%016llx-%016llx not committed\n", process->ProcessId, (unsigned long long)alignedStart, (unsigned long long)alignedEnd); return STATUS_NOT_FOUND; } + if (vad->LockCount) + return STATUS_ACCESS_DENIED; - if (pageFlags == 0) { - for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { - if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED; - PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); - if (phys && KSUCCESS(MmUnmapPage(va))) - MmFreeFrame(phys); - } - return STATUS_SUCCESS; - } + uint64_t finalFlags = pageFlags ? (pageFlags | PAGE_PRESENT | PAGE_USER) : 0; + size_t first = KiVadPageIndex(vad, alignedStart); + size_t count = KiVadRangePageCount(alignedStart, alignedEnd); - uint64_t finalFlags = pageFlags | PAGE_PRESENT | PAGE_USER; + for (size_t i = 0; i < count; ++i) { + size_t pageIndex = first + i; + MM_VAD_PAGE* page = &vad->Pages[pageIndex]; + uintptr_t va = vad->Base + pageIndex * PAGE_SIZE; - for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { - if (va < regionBase || va >= regionEnd) { - DbgPrint("MM: protect user region hit guard/outside page pid=%u va=%016llx region=%016llx-%016llx\n", - process->ProcessId, - (unsigned long long)va, - (unsigned long long)regionBase, - (unsigned long long)regionEnd); - return STATUS_ACCESS_DENIED; + if (!finalFlags) { + if (page->Resident && page->PageFlags) + (void)KiVadUnmapPageTracked(va, page->Frame); + page->PageFlags = 0; + continue; } - PHYSICAL_ADDRESS phys = 0; - uint64_t existingFlags = 0; - KSTATUS qs = MmQueryPage(va, &phys, &existingFlags); - if (!KSUCCESS(qs) || !(existingFlags & PAGE_PRESENT)) { - PHYSICAL_ADDRESS frame = MmAllocateFrame(); - if (!frame) return STATUS_NO_MEMORY; - KSTATUS ms = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MmMapFailIfPresent); - if (!KSUCCESS(ms)) { - MmFreeFrame(frame); - return ms; - } - memset((void*)va, 0, PAGE_SIZE); - DbgPrint("MM: committed missing user page during protect pid=%u va=%016llx\n", - process->ProcessId, (unsigned long long)va); + uint64_t oldFlags = page->PageFlags; + page->PageFlags = finalFlags; + if ((finalFlags & PAGE_WRITE) == 0 && + (page->Flags & (MM_VAD_PAGE_SHARED_FRAME | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_IMAGE_BACKED)) == 0) { + page->Flags &= ~MM_VAD_PAGE_COW; } - KSTATUS ps = MmSetPageProtect(va, PAGE_SIZE, finalFlags); - if (!KSUCCESS(ps)) { - PHYSICAL_ADDRESS dbgPhys = 0; - uint64_t dbgFlags = 0; - KSTATUS dbg = MmQueryPage(va, &dbgPhys, &dbgFlags); - DbgPrint("MM: protect user page failed pid=%u va=%016llx st=%d query=%d phys=%016llx flags=%016llx final=%016llx\n", - process->ProcessId, - (unsigned long long)va, - ps, dbg, - (unsigned long long)dbgPhys, - (unsigned long long)dbgFlags, - (unsigned long long)finalFlags); - return ps; + if (page->Resident) { + uint64_t hardwareFlags = KiVadHardwarePageFlags(page); + KSTATUS s = oldFlags + ? MmPteProtectRange(va, PAGE_SIZE, hardwareFlags) + : KiVadMapPageTracked(va, page->Frame, hardwareFlags, MmPteMapFailIfPresent); + if (!KSUCCESS(s)) return s; } } return STATUS_SUCCESS; } -// Releases user Region. -KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) { +// Destroys a VAD and releases all resident payload pages. +KSTATUS MmVadDestroy(PROCESS* process, void* base) { KeAssertIrql(PASSIVE_LEVEL); if (!process || !base) return STATUS_INVALID_PARAMETER; - uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); - USER_VA_REGION* r = KiFindUserRegionLocked(process, base); - if (!r) { - KeReleaseSpinlock(&process->UserRegionLock, procFlags); + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindVadByBaseLocked(process, base); + if (!vad) { + KeReleaseSpinlock(&process->VadLock, procFlags); return STATUS_NOT_FOUND; } - ListRemove(&r->ProcessLink); - KeReleaseSpinlock(&process->UserRegionLock, procFlags); + if (vad->LockCount) { + KeReleaseSpinlock(&process->VadLock, procFlags); + return STATUS_ACCESS_DENIED; + } + ListRemove(&vad->ProcessLink); + KeReleaseSpinlock(&process->VadLock, procFlags); - uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock); - ListRemove(&r->GlobalLink); - KeReleaseSpinlock(&gUserRegionLock, globalFlags); + uintptr_t globalFlags = KeAcquireSpinlock(&gVadLock); + ListRemove(&vad->GlobalLink); + KeReleaseSpinlock(&gVadLock, globalFlags); - KiUnmapUserRegionPayload(r); - MmFreePool(r); + KiUnmapVadPayload(vad); + if (vad->FileObject) { + ObDereferenceObject(&vad->FileObject->Header); + vad->FileObject = (FILE_OBJECT*)0; + } + if (vad->ImageSection) { + LdrDereferenceImageSection(vad->ImageSection); + vad->ImageSection = (IMAGE_SECTION_OBJECT*)0; + } + MmFreePool(vad->Pages); + MmFreePool(vad); return STATUS_SUCCESS; } -// Releases all User Regions. -void MmDestroyAllUserRegions(PROCESS* process) { +// Destroys every VAD owned by a process during teardown. +void MmVadDestroyAllForProcess(PROCESS* process) { KeAssertIrql(PASSIVE_LEVEL); if (!process) return; for (;;) { - uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); - LIST_ENTRY* node = ListPopHead(&process->UserRegions); - KeReleaseSpinlock(&process->UserRegionLock, procFlags); - if (!node) break; + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + LIST_ENTRY* node = (LIST_ENTRY*)0; + int sawLocked = 0; + for (LIST_ENTRY* it = process->VadList.Fwd; it != &process->VadList; it = it->Fwd) { + MM_VAD* candidate = CONTAINING_RECORD(it, MM_VAD, ProcessLink); + if (candidate->LockCount) { + sawLocked = 1; + continue; + } + node = it; + ListRemove(node); + break; + } + KeReleaseSpinlock(&process->VadLock, procFlags); + if (!node) { + if (sawLocked) { + KeYield(); + continue; + } + break; + } - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); - uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock); - ListRemove(&r->GlobalLink); - KeReleaseSpinlock(&gUserRegionLock, globalFlags); - KiUnmapUserRegionPayload(r); - MmFreePool(r); + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, ProcessLink); + uintptr_t globalFlags = KeAcquireSpinlock(&gVadLock); + ListRemove(&vad->GlobalLink); + KeReleaseSpinlock(&gVadLock, globalFlags); + KiUnmapVadPayload(vad); + if (vad->FileObject) { + ObDereferenceObject(&vad->FileObject->Header); + vad->FileObject = (FILE_OBJECT*)0; + } + if (vad->ImageSection) { + LdrDereferenceImageSection(vad->ImageSection); + vad->ImageSection = (IMAGE_SECTION_OBJECT*)0; + } + MmFreePool(vad->Pages); + MmFreePool(vad); } } -KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, - uint64_t* pageFlagsOut, USER_REGION_TYPE* typeOut, uint32_t* regionFlagsOut) { +// Queries the committed VAD range containing an address. +KSTATUS MmVadQuery(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, + uint64_t* pageFlagsOut, MM_VAD_TYPE* typeOut, uint32_t* vadFlagsOut) { if (!process || !address) return STATUS_INVALID_PARAMETER; uintptr_t addr = (uintptr_t)address; - uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); - for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); - uintptr_t end = r->Base + r->CommittedSize; - if (addr >= r->Base && addr < end) { - if (baseOut) *baseOut = (void*)r->Base; - if (sizeOut) *sizeOut = r->CommittedSize; - if (pageFlagsOut) { - PHYSICAL_ADDRESS phys = 0; - uint64_t flags = 0; - KSTATUS qs = MmQueryPage(addr & PAGE_MASK, &phys, &flags); - *pageFlagsOut = KSUCCESS(qs) ? flags : 0; - } - if (typeOut) *typeOut = r->Type; - if (regionFlagsOut) *regionFlagsOut = r->Flags; - KeReleaseSpinlock(&process->UserRegionLock, procFlags); - return STATUS_SUCCESS; + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + for (LIST_ENTRY* node = process->VadList.Fwd; node != &process->VadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, ProcessLink); + if (addr < vad->Base || addr >= vad->Base + vad->PayloadSize) + continue; + + size_t index = KiVadPageIndex(vad, addr & PAGE_MASK); + if (index >= vad->PageCount || vad->Pages[index].State != MmVadPageCommitted) { + KeReleaseSpinlock(&process->VadLock, procFlags); + return STATUS_NOT_FOUND; } + + uint64_t flags = vad->Pages[index].PageFlags; + size_t runStart = index; + while (runStart > 0 && + vad->Pages[runStart - 1].State == MmVadPageCommitted && + vad->Pages[runStart - 1].PageFlags == flags) { + runStart--; + } + + size_t runEnd = index + 1; + while (runEnd < vad->PageCount && + vad->Pages[runEnd].State == MmVadPageCommitted && + vad->Pages[runEnd].PageFlags == flags) { + runEnd++; + } + + if (baseOut) *baseOut = (void*)(vad->Base + runStart * PAGE_SIZE); + if (sizeOut) *sizeOut = (runEnd - runStart) * PAGE_SIZE; + if (pageFlagsOut) *pageFlagsOut = flags; + if (typeOut) *typeOut = vad->Type; + if (vadFlagsOut) *vadFlagsOut = vad->Flags; + KeReleaseSpinlock(&process->VadLock, procFlags); + return STATUS_SUCCESS; } - KeReleaseSpinlock(&process->UserRegionLock, procFlags); + KeReleaseSpinlock(&process->VadLock, procFlags); return STATUS_NOT_FOUND; } - -// Returns process User Memory. -void MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t* reservedBytesOut, size_t* committedBytesOut) { - size_t regions = 0; +// Returns per-process VAD accounting for ps and memory telemetry. +void MmVadQueryProcessMemory(PROCESS* process, size_t* vadCountOut, size_t* reservedBytesOut, size_t* committedBytesOut) { + size_t vads = 0; size_t reserved = 0; size_t committed = 0; if (process) { - uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock); - for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); - regions++; - reserved += r->ReservationSize; - committed += r->CommittedSize; + uintptr_t flags = KeAcquireSpinlock(&process->VadLock); + for (LIST_ENTRY* node = process->VadList.Fwd; node != &process->VadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, ProcessLink); + vads++; + reserved += vad->ReservationSize; + committed += vad->CommittedPages * PAGE_SIZE; } - KeReleaseSpinlock(&process->UserRegionLock, flags); + KeReleaseSpinlock(&process->VadLock, flags); } - if (regionCountOut) *regionCountOut = regions; + if (vadCountOut) *vadCountOut = vads; if (reservedBytesOut) *reservedBytesOut = reserved; if (committedBytesOut) *committedBytesOut = committed; } -// Returns user Region Size. -size_t MmGetUserRegionSize(PROCESS* process, const void* base) { +// Returns VAD and demand-fault telemetry. +void MmVadQueryStats(MM_VAD_STATS* out) { + if (!out) return; + *out = gVadStats; +} + +// Returns the requested payload size for a VAD base. +size_t MmVadGetSize(PROCESS* process, const void* base) { if (!process || !base) return 0; - uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); - USER_VA_REGION* r = KiFindUserRegionLocked(process, base); - size_t size = r ? r->Size : 0; - KeReleaseSpinlock(&process->UserRegionLock, procFlags); + uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); + MM_VAD* vad = KiFindVadByBaseLocked(process, base); + size_t size = vad ? vad->Size : 0; + KeReleaseSpinlock(&process->VadLock, procFlags); return size; } -// Returns user Region Size By Base. -size_t MmGetUserRegionSizeByBase(const void* base) { +// Returns the requested payload size for a VAD base without knowing the owner. +size_t MmVadGetSizeByBase(const void* base) { if (!base) return 0; uintptr_t b = (uintptr_t)base; - uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock); - for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); - if (r->Base == b) { - size_t size = r->Size; - KeReleaseSpinlock(&gUserRegionLock, flags); + uintptr_t flags = KeAcquireSpinlock(&gVadLock); + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* vad = CONTAINING_RECORD(node, MM_VAD, GlobalLink); + if (vad->Base == b) { + size_t size = vad->Size; + KeReleaseSpinlock(&gVadLock, flags); return size; } } - KeReleaseSpinlock(&gUserRegionLock, flags); + KeReleaseSpinlock(&gVadLock, flags); return 0; } - -// Implements the user Region Type Name operation. -static const char* KiUserRegionTypeName(USER_REGION_TYPE type) { +// Implements the VAD Type Name operation. +static const char* KiVadTypeName(MM_VAD_TYPE type) { switch (type) { - case UserRegionGeneric: return "generic"; - case UserRegionHeap: return "heap"; - case UserRegionImage: return "image"; - case UserRegionStack: return "stack"; - case UserRegionTeb: return "teb"; - case UserRegionPeb: return "peb"; - case UserRegionEnvironment: return "env"; - case UserRegionMappedFile: return "mapped"; - case UserRegionShared: return "shared"; - default: return "?"; + case MmVadGeneric: return "generic"; + case MmVadHeap: return "heap"; + case MmVadImage: return "image"; + case MmVadStack: return "stack"; + case MmVadTeb: return "teb"; + case MmVadPeb: return "peb"; + case MmVadEnvironment: return "env"; + case MmVadMappedFile: return "mapped"; + case MmVadShared: return "shared"; + default: return "?"; } } -// Implements the debug Dump User Regions operation. -void MmDebugDumpUserRegions(PROCESS* process) { +// Implements the debug Dump VADs operation. +void MmVadDebugDumpProcess(PROCESS* process) { if (!process) { - DbgPrint("MM: user regions: \n"); + DbgPrint("MM: VADs: \n"); return; } - DbgPrint("MM: user regions for pid=%u name=%s:\n", process->ProcessId, process->Name); - uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock); + DbgPrint("MM: VADs for pid=%u name=%s:\n", process->ProcessId, process->Name); + uintptr_t flags = KeAcquireSpinlock(&process->VadLock); uint32_t count = 0; - for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); - DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx flags=%x pte=%llx\n", + for (LIST_ENTRY* node = process->VadList.Fwd; node != &process->VadList; node = node->Fwd) { + MM_VAD* r = CONTAINING_RECORD(node, MM_VAD, ProcessLink); + DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx/%llx flags=%x pte=%llx\n", count, - KiUserRegionTypeName(r->Type), + KiVadTypeName(r->Type), (unsigned long long)r->ReservationBase, (unsigned long long)(r->ReservationBase + r->ReservationSize), (unsigned long long)r->Base, (unsigned long long)r->Size, - (unsigned long long)r->CommittedSize, + (unsigned long long)(r->CommittedPages * PAGE_SIZE), + (unsigned long long)r->PayloadSize, r->Flags, (unsigned long long)r->PageFlags); count++; } - KeReleaseSpinlock(&process->UserRegionLock, flags); + KeReleaseSpinlock(&process->VadLock, flags); if (!count) DbgPrint(" \n"); } @@ -1351,14 +3145,34 @@ static void KiDebugDescribePoolAddress(uintptr_t addr) { uintptr_t data = hdr + HDR_SIZE; uintptr_t end = data + cur->Size; uintptr_t ftr = end; + BLOCK_FTR* footer = KiHdrToFtr(cur); + if ((uintptr_t)footer + FTR_SIZE > gPoolHeap.Brk || + footer->Magic != BLOCK_MAGIC || footer->Size != cur->Size || footer->Free != cur->Free) { + DbgPrint("MM: pool walk corrupt footer hdr=%p state=%u size=%llx footer=%p fmagic=%08x fstate=%u fsize=%llx\n", + cur, + cur->Free, + (unsigned long long)cur->Size, + footer, + ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Magic : 0, + ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Free : 0, + ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? (unsigned long long)footer->Size : 0ULL); + break; + } if (addr >= hdr && addr < ftr + FTR_SIZE) { - DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx free=%u\n", + DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx req=%llx state=%u(%s) tag=%08x flags=%08x seq=%llu alloc=%p free=%p\n", (unsigned long long)addr, (unsigned long long)hdr, (unsigned long long)data, (unsigned long long)end, (unsigned long long)cur->Size, - cur->Free); + (unsigned long long)cur->RequestedSize, + cur->Free, + KiPoolStateName(cur->Free), + cur->Tag, + cur->Flags, + (unsigned long long)cur->Sequence, + (void*)cur->AllocCaller, + (void*)cur->FreeCaller); break; } if (KiNextBlock(cur) <= cur) { @@ -1373,7 +3187,7 @@ static void KiDebugDescribePoolAddress(uintptr_t addr) { void MmDebugDescribeAddress(uintptr_t addr) { PHYSICAL_ADDRESS phys = 0; uint64_t flags = 0; - KSTATUS qs = MmQueryPage(addr, &phys, &flags); + KSTATUS qs = MmPteQueryPageFor(MmPteQueryDebug, addr, &phys, &flags); DbgPrint("MM: describe va=%016llx query=%d phys=%016llx flags=%016llx\n", (unsigned long long)addr, qs, @@ -1385,26 +3199,39 @@ void MmDebugDescribeAddress(uintptr_t addr) { return; } - if (addr >= USER_REGION_ARENA_BASE && addr < USER_LIMIT) { - uintptr_t gl = KeAcquireSpinlock(&gUserRegionLock); - for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); + if (addr >= MM_VAD_ARENA_BASE && addr < USER_LIMIT) { + uintptr_t gl = KeAcquireSpinlock(&gVadLock); + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* r = CONTAINING_RECORD(node, MM_VAD, GlobalLink); if (addr >= r->ReservationBase && addr < r->ReservationBase + r->ReservationSize) { - DbgPrint("MM: address belongs to pid=%u name=%s region=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n", + const char* pageState = "guard"; + if (addr >= r->Base && addr < r->Base + r->PayloadSize) { + size_t pageIndex = KiVadPageIndex(r, addr & PAGE_MASK); + if (pageIndex < r->PageCount) { + MM_VAD_PAGE* page = &r->Pages[pageIndex]; + if (page->State == MmVadPageCommitted) + pageState = page->PageFlags ? "mapped" : "noaccess"; + else + pageState = "reserved"; + } + } + + DbgPrint("MM: address belongs to pid=%u name=%s vad=%s page=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n", r->Owner ? r->Owner->ProcessId : 0, r->Owner ? r->Owner->Name : "?", - KiUserRegionTypeName(r->Type), + KiVadTypeName(r->Type), + pageState, (unsigned long long)r->ReservationBase, (unsigned long long)(r->ReservationBase + r->ReservationSize), (unsigned long long)r->Base, (unsigned long long)r->Size, r->Flags); - KeReleaseSpinlock(&gUserRegionLock, gl); + KeReleaseSpinlock(&gVadLock, gl); return; } } - KeReleaseSpinlock(&gUserRegionLock, gl); - DbgPrint("MM: address is in user arena but not reserved by any USER_VA_REGION\n"); + KeReleaseSpinlock(&gVadLock, gl); + DbgPrint("MM: address is in user arena but not reserved by any MM_VAD\n"); } } @@ -1412,18 +3239,24 @@ void MmDebugDescribeAddress(uintptr_t addr) { __declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) { void* base = (void*)0; if (!process || !size) return (void*)0; - KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE, + KSTATUS s = MmVadCreateCommitted(process, size, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, - USER_REGION_ZERO, - UserRegionHeap, + MM_VAD_ZERO, + MmVadHeap, &base); - return KSUCCESS(s) ? base : (void*)0; + if (!KSUCCESS(s)) return (void*)0; + s = MmVadEnsureResident(process, base, size, PAGE_WRITE | PAGE_USER); + if (!KSUCCESS(s)) { + (void)MmVadDestroy(process, base); + return (void*)0; + } + return base; } // Releases user Memory For Process. __declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) { if (!process || !ptr) return; - (void)MmDestroyUserRegion(process, ptr); + (void)MmVadDestroy(process, ptr); } // Implements the reallocate User Memory For Process operation. @@ -1435,7 +3268,7 @@ __declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, v return (void*)0; } - size_t oldSize = MmGetUserRegionSize(process, ptr); + size_t oldSize = MmVadGetSize(process, ptr); if (!oldSize) return (void*)0; if (newSize <= oldSize) return ptr; @@ -1468,19 +3301,20 @@ __declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { __declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) { void* base = (void*)0; if (!process || !size) return (void*)0; - KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE, + KSTATUS s = MmVadCreateCommitted(process, size, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, - USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH, - UserRegionImage, + MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH, + MmVadImage, &base); - return KSUCCESS(s) ? base : (void*)0; + if (!KSUCCESS(s)) return (void*)0; + return base; } // Releases user Image For Process. __declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) { (void)size; if (!process || !ptr) return; - (void)MmDestroyUserRegion(process, ptr); + (void)MmVadDestroy(process, ptr); } // Creates user Image. @@ -1504,12 +3338,12 @@ size_t MmGetKernelPoolUsage(void) { // Returns user Heap Usage. size_t MmGetUserHeapUsage(void) { size_t total = 0; - if (!gUserRegionListReady) return 0; - uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock); - for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { - USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); - total += r->CommittedSize; + if (!gVadListReady) return 0; + uintptr_t flags = KeAcquireSpinlock(&gVadLock); + for (LIST_ENTRY* node = gVadList.Fwd; node != &gVadList; node = node->Fwd) { + MM_VAD* r = CONTAINING_RECORD(node, MM_VAD, GlobalLink); + total += r->CommittedPages * PAGE_SIZE; } - KeReleaseSpinlock(&gUserRegionLock, flags); + KeReleaseSpinlock(&gVadLock, flags); return total; } diff --git a/code/kernel/ob_object.c b/code/kernel/ob_object.c index 5276b51..eacfef0 100644 --- a/code/kernel/ob_object.c +++ b/code/kernel/ob_object.c @@ -5,11 +5,36 @@ #include #include -// Creates object. +// Returns a diagnostic pool tag for the object type. +static uint32_t KiObjectPoolTag(OBJECT_TYPE type) { + switch (type) { + case ObjectTypeProcess: return MM_POOL_TAG('p', 'r', 'o', 'c'); + case ObjectTypeThread: return MM_POOL_TAG('t', 'h', 'r', 'd'); + case ObjectTypeMutex: return MM_POOL_TAG('m', 't', 'x', ' '); + case ObjectTypeSemaphore: return MM_POOL_TAG('s', 'e', 'm', ' '); + case ObjectTypeEvent: return MM_POOL_TAG('e', 'v', 'n', 't'); + case ObjectTypeQueue: return MM_POOL_TAG('q', 'u', 'e', 'u'); + case ObjectTypeFile: return MM_POOL_TAG('f', 'i', 'l', 'e'); + case ObjectTypeDir: return MM_POOL_TAG('d', 'i', 'r', ' '); + case ObjectTypeAsyncIo: return MM_POOL_TAG('a', 'i', 'o', ' '); + case ObjectTypeSocket: return MM_POOL_TAG('s', 'o', 'c', 'k'); + case ObjectTypeDevice: return MM_POOL_TAG('d', 'e', 'v', ' '); + case ObjectTypeDriver: return MM_POOL_TAG('d', 'r', 'v', 'r'); + case ObjectTypeUsbDevice: return MM_POOL_TAG('u', 's', 'b', 'd'); + case ObjectTypeUsbPipe: return MM_POOL_TAG('u', 's', 'b', 'p'); + case ObjectTypeFileMapping: return MM_POOL_TAG('m', 'a', 'p', 'f'); + case ObjectTypeSurface: return MM_POOL_TAG('s', 'u', 'r', 'f'); + case ObjectTypeInputQueue: return MM_POOL_TAG('i', 'n', 'p', 'q'); + case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e'); + default: return MM_POOL_TAG('o', 'b', 'j', ' '); + } +} + +// Creates a zeroed, tagged object with one initial reference. __declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) { if (size < sizeof(OBJECT)) size = sizeof(OBJECT); - OBJECT* obj = (OBJECT *)MmAllocatePool(size); + OBJECT* obj = (OBJECT *)MmAllocatePoolTag(size, KiObjectPoolTag(type)); if (!obj) return (OBJECT *)0; memset(obj, 0, size); diff --git a/code/public/ntlite/ddi/hal_api.h b/code/public/ntlite/ddi/hal_api.h index bad5824..89a527a 100644 --- a/code/public/ntlite/ddi/hal_api.h +++ b/code/public/ntlite/ddi/hal_api.h @@ -120,7 +120,7 @@ typedef struct HAL_INIT_PARAMS { void* (*AllocatePool)(size_t size); } HAL_INIT_PARAMS; -#define HAL_API_VERSION 5 +#define HAL_API_VERSION // do not change during development __declspec(dllexport) const HAL_API* HalGetApi(void); __declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)); diff --git a/code/public/ntlite/ddi/io_driver.h b/code/public/ntlite/ddi/io_driver.h index 8f17fc5..e04a951 100644 --- a/code/public/ntlite/ddi/io_driver.h +++ b/code/public/ntlite/ddi/io_driver.h @@ -179,6 +179,27 @@ typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex, DEVICE_OBJECT* device, void* context); +typedef struct IO_DISK_STATS { + uint64_t RegisteredDisks; + uint64_t ReadRequests; + uint64_t ReadSectors; + uint64_t WriteRequests; + uint64_t WriteSectors; + uint64_t MaxReadSectors; + uint64_t MaxWriteSectors; + uint64_t ReadaheadHits; + uint64_t ReadaheadMisses; + uint64_t ReadaheadStores; + uint64_t ReadaheadInvalidations; + uint64_t ReadaheadBytesServed; + uint64_t ReadaheadSectorsLoaded; + uint64_t ReadaheadSequentialDetections; + uint64_t ReadaheadSuppressed; + uint64_t ReadaheadAllocFailures; + uint64_t ReadaheadReadFailures; + uint64_t MaxReadaheadSectors; +} IO_DISK_STATS; + __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, DEVICE_OBJECT* device); __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex); @@ -203,3 +224,4 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, uint64_t lba, uint32_t count, const void* buffer); +__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out); diff --git a/code/public/ntlite/ddi/pci.h b/code/public/ntlite/ddi/pci.h index 0836a6a..0a828d3 100644 --- a/code/public/ntlite/ddi/pci.h +++ b/code/public/ntlite/ddi/pci.h @@ -118,3 +118,6 @@ PCI_API KSTATUS PciGetInterrupt(DEVICE_OBJECT* pdo, PCI_API KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot); + +PCI_API KSTATUS PciEnableMsiAuto(DEVICE_OBJECT* pdo, + uint8_t* outIrqSlot); diff --git a/code/public/ntlite/display.h b/code/public/ntlite/display.h index c2dda13..5854e74 100644 --- a/code/public/ntlite/display.h +++ b/code/public/ntlite/display.h @@ -32,6 +32,43 @@ typedef struct DISPLAY_RECT { int32_t H; } DISPLAY_RECT; + +#define SYS_SURFACE_FORMAT_XRGB8888 1u +#define SYS_SURFACE_FLAG_NONE 0u +#define SYS_SURFACE_MAX_RECTS 64u + +typedef struct SYS_SURFACE_DESC { + uint32_t Width; + uint32_t Height; + uint32_t PitchBytes; + uint32_t Format; + void* Base; + uint64_t PayloadBytes; + uint64_t PageCount; + uint64_t MapCount; + uint64_t PresentCount; + uint64_t PresentRectCount; + uint64_t PresentClippedRectCount; + uint64_t PresentEmptyClipCount; + uint64_t PresentRowsCopied; + uint64_t PresentBytesCopied; +} SYS_SURFACE_DESC; + +typedef struct SYS_SURFACE_STATS { + uint64_t CreatedSurfaces; + uint64_t DestroyedSurfaces; + uint64_t LiveSurfaces; + uint64_t LivePages; + uint64_t LiveBytes; + uint64_t MapCalls; + uint64_t PresentCalls; + uint64_t PresentRectInputs; + uint64_t PresentClippedRects; + uint64_t PresentEmptyClips; + uint64_t PresentRowsCopied; + uint64_t PresentBytesCopied; +} SYS_SURFACE_STATS; + typedef struct DISPLAY_STATS { uint64_t BlitCount; uint64_t FillRectCount; diff --git a/code/public/ntlite/input.h b/code/public/ntlite/input.h index 198d6a3..ef3f2cd 100644 --- a/code/public/ntlite/input.h +++ b/code/public/ntlite/input.h @@ -136,6 +136,13 @@ typedef struct INPUT_KEY_EVENT { bool Pressed; } INPUT_KEY_EVENT; +typedef struct INPUT_KEY_RECORD { + INPUT_KEY Key; + char Char; + uint8_t Reserved; + uint32_t Modifiers; +} INPUT_KEY_RECORD; + typedef struct INPUT_MOUSE_EVENT { int dx; int dy; diff --git a/code/public/ntlite/status.h b/code/public/ntlite/status.h index 026d965..db3386d 100644 --- a/code/public/ntlite/status.h +++ b/code/public/ntlite/status.h @@ -45,4 +45,26 @@ typedef uint32_t KSTATUS; // --- Execution level (0x0060) ---. #define STATUS_INVALID_IRQL ((KSTATUS)0x0060) +// --- Process termination reasons ---. +// Low values are reserved for ordinary KSTATUS returns. Abnormal process +// termination uses a high-bit private range so exit statuses cannot collide +// with normal kernel/API result codes. +// +// Layout: +// 31 : set => abnormal/private status +// 30..24 : subsystem group +// 23..0 : group-local reason +// +// Group 0x61 is process termination/fault status. +#define STATUS_PRIVATE_BASE ((KSTATUS)0x80000000u) +#define STATUS_PROCESS_BASE ((KSTATUS)0xE1000000u) +#define STATUS_PROCESS_KILLED ((KSTATUS)(STATUS_PROCESS_BASE | 0x0000u)) +#define STATUS_PROCESS_PAGE_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0010u)) +#define STATUS_PROCESS_GENERAL_PROTECTION ((KSTATUS)(STATUS_PROCESS_BASE | 0x0011u)) +#define STATUS_PROCESS_ILLEGAL_OPCODE ((KSTATUS)(STATUS_PROCESS_BASE | 0x0012u)) +#define STATUS_PROCESS_DIVIDE_BY_ZERO ((KSTATUS)(STATUS_PROCESS_BASE | 0x0013u)) +#define STATUS_PROCESS_BREAKPOINT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0014u)) +#define STATUS_PROCESS_ALIGNMENT_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0015u)) +#define STATUS_PROCESS_UNKNOWN_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x001Fu)) + #define KSUCCESS(s) ((s) == STATUS_SUCCESS) diff --git a/code/public/ntlite/syscall.h b/code/public/ntlite/syscall.h index 29db87b..314c01d 100644 --- a/code/public/ntlite/syscall.h +++ b/code/public/ntlite/syscall.h @@ -18,7 +18,11 @@ #define SYS_STATUS_PENDING 4u #define SYS_STATUS_ALERTED 5u #define SYS_STATUS_INVALID_PARAMETER 0x10u +#define SYS_STATUS_INVALID_HANDLE 0x11u +#define SYS_STATUS_BUFFER_TOO_SMALL 0x12u +#define SYS_STATUS_NOT_SUPPORTED 0x02u #define SYS_STATUS_NO_MEMORY 0x20u +#define SYS_STATUS_ACCESS_DENIED 0x21u #define SYS_STATUS_KERNEL_APC 6u #define SYS_STATUS_USER_APC 7u #define SYS_STATUS_CANCELLED 0x33u @@ -30,6 +34,7 @@ #define SYS_TEB_MAGIC 0x30424554u // "TEB0" #define SYS_ENV_VERSION 1u #define SYS_CWD_MAX 520u +#define SYS_PATH_MAX 1024u #ifndef SYS_USER_ENV_TYPES_DEFINED #define SYS_USER_ENV_TYPES_DEFINED 1 @@ -52,9 +57,7 @@ typedef struct SYS_TEB { uint32_t ThreadId; uint32_t Magic; - // Native per-thread extension pointer. The kernel leaves this NULL; - // user-mode personalities/libraries may hang their own thread-local - // state here without baking subsystem semantics into SYS_TEB. + // =The kernel leaves this NULL, userspace can modify and use. void* SubsystemData; } SYS_TEB; @@ -80,6 +83,27 @@ typedef void (*SYS_USER_APC_ROUTINE)(void* normalContext, SYS_IO_STATUS_BLOCK* iosb, uint64_t information); +typedef struct SYS_FILE_IO_REQUEST { + uint32_t FileHandle; + uint32_t Flags; + void* Buffer; + size_t Size; + size_t* TransferredOut; +} SYS_FILE_IO_REQUEST; + +static inline SYS_FILE_IO_REQUEST SysMakeFileIoRequest(uint32_t fileHandle, + const void* buffer, + size_t size, + size_t* transferredOut) { + SYS_FILE_IO_REQUEST request; + request.FileHandle = fileHandle; + request.Flags = 0; + request.Buffer = (void*)(uintptr_t)buffer; + request.Size = size; + request.TransferredOut = transferredOut; + return request; +} + typedef struct SYS_ASYNC_FILE_REQUEST { uint32_t FileHandle; uint32_t EventHandle; // optional; 0 means no event signal @@ -181,11 +205,40 @@ typedef struct SYS_NAMED_QUEUE_OWNER_REQUEST { uint32_t* PidOut; } SYS_NAMED_QUEUE_OWNER_REQUEST; +#define SYS_WAIT_FLAG_NONE 0x00000000u +#define SYS_WAIT_FLAG_ALERTABLE 0x00000001u +#define SYS_WAIT_TIMEOUT_INFINITE UINT64_MAX +#define SYS_WAIT_MANY_MAX_HANDLES 32u + +typedef struct SYS_WAIT_MANY_REQUEST { + uint32_t Size; + uint32_t Flags; + const uint32_t* Handles; + uint32_t Count; + uint32_t* SignaledIndexOut; + uint64_t TimeoutMilliseconds; +} SYS_WAIT_MANY_REQUEST; + +typedef struct SYS_CREATE_THREAD_REQUEST { + uintptr_t Entry; + uintptr_t Argument; + uint32_t Priority; + uint32_t Flags; + uint32_t* ThreadHandleOut; + uint32_t* ThreadIdOut; +} SYS_CREATE_THREAD_REQUEST; + // Process & thread lifecycle (0-14). void SysExitProcess(int exitCode); void SysExitThread(int exitCode); uint32_t SysCreateProcess(const SYS_CREATE_PROCESS_REQUEST* request); -uint32_t SysWaitForProcess(uint32_t handle); +uint32_t SysWaitSingle(uint32_t handle, uint32_t flags, uint64_t timeoutMilliseconds); +uint32_t SysWaitMany(const SYS_WAIT_MANY_REQUEST* request); +uint32_t SysOpenInputQueue(uint32_t flags, uint32_t* handleOut); + +static inline uint32_t SysWait(uint32_t handle, uint32_t flags) { + return SysWaitSingle(handle, flags, SYS_WAIT_TIMEOUT_INFINITE); +} uint32_t SysGetProcessExitCode(uint32_t handle, int* exitCodeOut); uint32_t SysKillProcess(uint32_t pid, int exitCode); @@ -200,6 +253,61 @@ uint32_t SysKillProcess(uint32_t pid, int exitCode); #define SYS_PROCESS_FLAG_CURRENT 0x00000002u #define SYS_PROCESS_FLAG_TERMINATING 0x00000004u #define SYS_PROCESS_FLAG_EXITED 0x00000008u +#define SYS_PROCESS_FLAG_IDLE 0x00000010u + +#define SYS_THREAD_STATE_READY 0u +#define SYS_THREAD_STATE_RUNNING 1u +#define SYS_THREAD_STATE_WAITING 2u +#define SYS_THREAD_STATE_DEAD 3u + +#define SYS_THREAD_FLAG_CURRENT 0x00000001u +#define SYS_THREAD_FLAG_IDLE 0x00000002u +#define SYS_THREAD_FLAG_KERNEL 0x00000004u +#define SYS_THREAD_FLAG_QUEUED 0x00000008u +#define SYS_THREAD_FLAG_ALERTABLE_WAIT 0x00000010u +#define SYS_THREAD_FLAG_KERNEL_APC_PENDING 0x00000020u +#define SYS_THREAD_FLAG_USER_APC_PENDING 0x00000040u + +#define SYS_THREAD_WAIT_NONE 0u +#define SYS_THREAD_WAIT_PROCESS 1u +#define SYS_THREAD_WAIT_THREAD 2u +#define SYS_THREAD_WAIT_MUTEX 3u +#define SYS_THREAD_WAIT_SEMAPHORE 4u +#define SYS_THREAD_WAIT_EVENT 5u +#define SYS_THREAD_WAIT_QUEUE 6u +#define SYS_THREAD_WAIT_FILE 7u +#define SYS_THREAD_WAIT_DIR 8u +#define SYS_THREAD_WAIT_ASYNC_IO 9u +#define SYS_THREAD_WAIT_SOCKET 10u +#define SYS_THREAD_WAIT_DEVICE 11u +#define SYS_THREAD_WAIT_DRIVER 12u +#define SYS_THREAD_WAIT_USB_DEVICE 13u +#define SYS_THREAD_WAIT_USB_PIPE 14u +#define SYS_THREAD_WAIT_FILE_MAPPING 15u +#define SYS_THREAD_WAIT_SURFACE 16u +#define SYS_THREAD_WAIT_UNKNOWN 0xffffffffu + +typedef struct SYS_THREAD_INFO { + uint32_t Size; + uint32_t ProcessId; + uint32_t ThreadId; + uint32_t State; + + uint32_t Priority; + uint32_t Flags; + uint32_t RunningCpu; + uint32_t LastCpu; + + uint32_t QuantumRemaining; + uint32_t QuantumTicks; + uint32_t CpuBurstTicks; + uint32_t LastRunTicks; + + uint32_t WaitObjectType; + uint32_t WaitStatus; + int32_t KernelApcDisable; + uint32_t Reserved0; +} SYS_THREAD_INFO; typedef struct SYS_PROCESS_INFO { uint32_t Size; @@ -219,7 +327,7 @@ typedef struct SYS_PROCESS_INFO { size_t ImageSize; uintptr_t PebBase; - size_t UserRegionCount; + size_t VadCount; size_t UserReservedBytes; size_t UserCommittedBytes; @@ -229,9 +337,9 @@ typedef struct SYS_PROCESS_INFO { uint32_t SysGetProcessList(SYS_PROCESS_INFO* processesOut, size_t maxCount, size_t* countOut); uint32_t SysGetProcessPid(uint32_t handle, uint32_t* pidOut); +uint32_t SysGetThreadList(uint32_t pid, SYS_THREAD_INFO* threadsOut, size_t maxCount, size_t* countOut); -uint32_t SysCreateThread(uintptr_t entry, uintptr_t arg, uint32_t priority, uint32_t* tidOut); -uint32_t SysWaitThread(uint32_t tid); +uint32_t SysCreateThread(const SYS_CREATE_THREAD_REQUEST* request); uint32_t SysGetCurrentTid(void); uint32_t SysGetCurrentPid(void); uint32_t SysSetThreadPriority(uint32_t priority); @@ -266,23 +374,32 @@ uint32_t SysAllocVm(void* preferredBase, size_t pageCount, uint32_t protect, uin uint32_t SysFreeVm(void* base); uint32_t SysProtectVm(void* base, size_t pageCount, uint32_t protect, uint32_t* oldProtectOut); uint32_t SysQueryVm(void* address, SYS_VM_REGION_INFO* out); + +#define SYS_MAP_ACCESS_READ 0x00000001u +#define SYS_MAP_ACCESS_WRITE 0x00000002u +#define SYS_MAP_ACCESS_COPY 0x00000004u +#define SYS_MAP_ACCESS_EXECUTE 0x00000008u +#define SYS_MAP_FLAG_NONE 0x00000000u + +uint32_t SysCreateFileMapping(uint32_t fileHandle, uint32_t protect, uint32_t flags, + uint64_t maximumSize, uint32_t* mappingHandleOut); +uint32_t SysMapFile(uint32_t mappingHandle, uint64_t offset, size_t size, + uint32_t access, uint32_t flags, void** baseOut); uint32_t SysCloseHandle(uint32_t handle); // Synchronization primitives (19-31). uint32_t SysCreateMutex(uint32_t* handleOut); -uint32_t SysAcquireMutex(uint32_t handle); uint32_t SysTryAcquireMutex(uint32_t handle); uint32_t SysReleaseMutex(uint32_t handle); uint32_t SysCreateSem(int32_t initial, int32_t max, uint32_t* handleOut); -uint32_t SysWaitSem(uint32_t handle); uint32_t SysTryWaitSem(uint32_t handle); uint32_t SysSignalSem(uint32_t handle); uint32_t SysCreateEvent(uint32_t type, uint32_t initialState, uint32_t* handleOut); uint32_t SysSignalEvent(uint32_t handle); uint32_t SysClearEvent(uint32_t handle); -uint32_t SysWaitEvent(uint32_t handle, uint32_t alertable); + // Queues and named queues (32-38). uint32_t SysCreateQueue(uint32_t* handleOut); @@ -296,8 +413,8 @@ uint32_t SysGetNamedQueueOwnerPid(const SYS_NAMED_QUEUE_OWNER_REQUEST* request); // Files, directories, and current directory (39-56). uint32_t SysOpenFile(const SYS_PATH_HANDLE_REQUEST* request); uint32_t SysCreateFile(const SYS_PATH_HANDLE_REQUEST* request); -uint32_t SysReadFile(uint32_t handle, void* buf, size_t size, size_t* bytesRead); -uint32_t SysWriteFile(uint32_t handle, const void* buf, size_t size, size_t* written); +uint32_t SysReadFile(const SYS_FILE_IO_REQUEST* request); +uint32_t SysWriteFile(const SYS_FILE_IO_REQUEST* request); uint32_t SysReadFileAsync(const SYS_ASYNC_FILE_REQUEST* request); uint32_t SysWriteFileAsync(const SYS_ASYNC_FILE_REQUEST* request); uint32_t SysCancelIo(uint32_t asyncRequestHandle); @@ -305,6 +422,7 @@ uint32_t SysSeekFile(uint32_t handle, uint64_t offset); uint64_t SysGetFilePos(uint32_t handle); uint64_t SysGetFileSize(uint32_t handle); uint32_t SysTruncateFile(uint32_t handle, uint64_t newSize); +uint32_t SysFlushFile(uint32_t handle); uint32_t SysDeleteFile(const SYS_PATH_REQUEST* request); uint32_t SysRenameFile(const SYS_RENAME_FILE_REQUEST* request); @@ -330,7 +448,10 @@ uint32_t SysSetConsoleCursor(int x, int y); uint32_t SysFillConsoleCharacter(char c, int x, int y, size_t length, size_t* writtenOut); uint32_t SysFillConsoleAttribute(uint16_t attr, int x, int y, size_t length, size_t* writtenOut); uint32_t SysPutScreenChar(char c, int x, int y, uint32_t fg, uint32_t bg); -uint32_t SysBlitScreen(int x, int y, int w, int h, const uint32_t* pixels); +uint32_t SysCreateSurface(uint32_t width, uint32_t height, uint32_t format, uint32_t flags, uint32_t* handleOut); +uint32_t SysMapSurface(uint32_t surfaceHandle, void** baseOut, uint32_t* pitchBytesOut); +uint32_t SysQuerySurface(uint32_t surfaceHandle, SYS_SURFACE_DESC* descOut); +uint32_t SysPresentSurface(uint32_t surfaceHandle, const DISPLAY_RECT* rects, uint32_t rectCount); uint32_t SysClearScreen(uint32_t color); uint32_t SysGetScreenSize(int* widthOut, int* heightOut); uint32_t SysQueryDisplayInfo(SYS_DISPLAY_INFO* infoOut); @@ -341,6 +462,8 @@ uint64_t SysIsKeyDown(INPUT_KEY key); char SysReadKeyboardChar(void); INPUT_KEY SysReadKey(void); uint32_t SysGetKeyModifiers(void); +uint32_t SysReadInputEvent(INPUT_KEY_RECORD* recordOut); +uint32_t SysPeekInputEvent(INPUT_KEY_RECORD* recordOut); uint64_t SysPollMouse(void); uint64_t SysReadMouse(int* dx, int* dy, uint8_t* buttons); @@ -437,11 +560,25 @@ typedef struct SYS_MEMORY_STATUS { uint64_t FreePhysicalBytes; uint64_t KernelPoolBytes; uint64_t UserHeapBytes; + uint64_t ReclaimablePhysicalBytes; + uint64_t AvailablePhysicalBytes; + uint64_t PinnedPhysicalBytes; + uint64_t NonEvictablePhysicalBytes; + uint64_t UserPhysicalBytes; + uint64_t PageTablePhysicalBytes; } SYS_MEMORY_STATUS; #define SYS_KERNEL_STATS_SCHEDULER 1u #define SYS_KERNEL_STATS_PMM 2u #define SYS_KERNEL_STATS_CPU 3u +#define SYS_KERNEL_STATS_PFN 4u +#define SYS_KERNEL_STATS_PTE 5u +#define SYS_KERNEL_STATS_VAD 6u +#define SYS_KERNEL_STATS_SWAP 7u +#define SYS_KERNEL_STATS_FILE_CACHE 8u +#define SYS_KERNEL_STATS_SURFACE 9u +#define SYS_KERNEL_STATS_FILESYSTEM 10u +#define SYS_KERNEL_STATS_DISK 11u #define SYS_KERNEL_STATS_MAX_CPUS 32u @@ -469,11 +606,15 @@ typedef struct SYS_PMM_STATS { uint64_t FreeRunHistogram[8]; } SYS_PMM_STATS; +#define SYS_CPU_THREAD_FLAG_IDLE 0x00000001u + typedef struct SYS_CPU_STAT { uint32_t CpuIndex; uint32_t LapicId; uint32_t Flags; uint32_t CurrentTid; + uint32_t CurrentThreadFlags; + uint32_t Reserved0; uint64_t SchedulerTicks; uint64_t ContextSwitches; } SYS_CPU_STAT; @@ -484,6 +625,228 @@ typedef struct SYS_CPU_STATS { SYS_CPU_STAT Cpu[SYS_KERNEL_STATS_MAX_CPUS]; } SYS_CPU_STATS; +typedef struct SYS_PFN_STATS { + uint64_t TotalFrames; + uint64_t FreeFrames; + uint64_t UsedFrames; + + uint64_t ReservedFrames; + uint64_t ZeroedFrames; + uint64_t ActiveFrames; + uint64_t StandbyFrames; + uint64_t ModifiedFrames; + uint64_t PinnedFrames; + uint64_t EvictableFrames; + uint64_t NonEvictableFrames; + uint64_t ReclaimableFrames; + uint64_t AvailableFrames; + uint64_t MappedFrames; + + uint64_t KernelFrames; + uint64_t PageTableFrames; + uint64_t UserFrames; + uint64_t CleanFileCacheFrames; + uint64_t ActiveCleanFileCacheFrames; + uint64_t StandbyCleanFileCacheFrames; + uint64_t ImageFrames; + uint64_t ActiveImageFrames; + uint64_t StandbyImageFrames; + uint64_t MappedFileFrames; + uint64_t AnonymousFrames; + uint64_t CowFrames; + uint64_t SurfaceFrames; + uint64_t DriverFrames; + uint64_t DmaFrames; + uint64_t BootloaderFrames; + uint64_t PfnDatabaseFrames; + uint64_t MmioFrames; + + uint64_t AllocationCount; + uint64_t FreeCount; + uint64_t RetainCount; + uint64_t ReleaseCount; + uint64_t PinCount; + uint64_t UnpinCount; + uint64_t FailedAllocations; + uint64_t FailedContiguousAllocations; + + uint64_t FailedInvalidRange; + uint64_t FailedEmptyRange; + uint64_t FailedReserveDenied; + uint64_t FailedNoFrame; + + uint64_t FailedContiguousInvalidRequest; + uint64_t FailedContiguousReserveDenied; + uint64_t FailedContiguousNoRun; + + uint64_t ReclaimAttempts; + uint64_t ReclaimPagesRequested; + uint64_t ReclaimPagesSucceeded; + uint64_t ReclaimPagesFailed; + uint64_t ReclaimReentrantSkips; +} SYS_PFN_STATS; + +typedef struct SYS_VAD_STATS { + uint64_t ResolveFaultCalls; + uint64_t ResolvedFaults; + uint64_t FailedFaults; + uint64_t ProtectionFaults; + uint64_t NoVadFaults; + uint64_t UncommittedFaults; + uint64_t NoAccessFaults; + uint64_t DemandZeroPages; + uint64_t ZeroPageMaps; + uint64_t ZeroPageCowSplits; + uint64_t CowFaults; + uint64_t CowPagesCopied; + uint64_t CowViewsCreated; + uint64_t CowPagesShared; + uint64_t EnsureResidentCalls; + uint64_t EnsureResidentPages; + uint64_t AlreadyResidentPages; + uint64_t FileBackedFaults; + uint64_t FileBackedReads; + uint64_t FileBackedReadFailures; + uint64_t FileBackedDiscards; + uint64_t FileBackedSharedWritableFaults; + uint64_t FileBackedDirtyFaults; + uint64_t FileBackedDirtySyncs; + uint64_t ImageBackedFaults; + uint64_t ImageBackedZeroFaults; + uint64_t ImageBackedPrivateFaults; + uint64_t ImageFixupPrivateCalls; + uint64_t ImageFixupPrivatePages; + uint64_t SwapPageOuts; + uint64_t SwapPageIns; +} SYS_VAD_STATS; + + +typedef struct SYS_FILE_CACHE_STATS { + uint64_t Lookups; + uint64_t Hits; + uint64_t Misses; + uint64_t Loads; + uint64_t LoadFailures; + uint64_t RaceDiscards; + uint64_t InflightWaits; + uint64_t Invalidations; + uint64_t DirtyPages; + uint64_t MaxDirtyPages; + uint64_t DirtyMarks; + uint64_t WritebackRequests; + uint64_t WritebackPages; + uint64_t WritebackBytes; + uint64_t WritebackFailures; + uint64_t FlushFileCalls; + uint64_t FlushFileFailures; + uint64_t MappedDirtySyncs; + uint64_t MappedDirtyPages; + uint64_t Drops; + uint64_t DropFailures; + uint64_t StandbyTransitions; + uint64_t StandbyHits; + uint64_t ReadAheadRequests; + uint64_t ReadAheadPagesRequested; + uint64_t ReadAheadPagesQueued; + uint64_t ReadAheadPagesLoaded; + uint64_t ReadAheadPagesSkipped; + uint64_t ReadAheadFailures; + uint64_t ReadAheadPagesThrottled; + uint64_t MaxReadAheadPagesPerRequest; + uint64_t BatchReads; + uint64_t BatchPagesRequested; + uint64_t BatchPagesLoaded; + uint64_t MaxBatchPagesRequested; + uint64_t MaxBatchPagesLoaded; + uint64_t ResidentPages; + uint64_t ActivePages; + uint64_t StandbyPages; + uint64_t MaxResidentPages; +} SYS_FILE_CACHE_STATS; + +typedef struct SYS_FILESYSTEM_STATS { + uint64_t PathLookups; + uint64_t PathCacheHits; + uint64_t PathCacheMisses; + uint64_t PathCacheStores; + uint64_t PathCacheInvalidations; + uint64_t PathCacheGeneration; + uint64_t DirectorySearches; + uint64_t DirectoryClustersRead; + uint64_t DirectoryEntriesScanned; + uint64_t FatChainCacheLookups; + uint64_t FatChainCacheHits; + uint64_t FatChainCacheMisses; + uint64_t FatChainCacheStores; + uint64_t FatChainCacheInvalidations; + uint64_t FatChainWalkSteps; + uint64_t DataReadRuns; + uint64_t DataReadSectors; + uint64_t DataReadRunMerges; + uint64_t MaxDataReadSpanSectors; +} SYS_FILESYSTEM_STATS; + +typedef struct SYS_DISK_STATS { + uint64_t RegisteredDisks; + uint64_t ReadRequests; + uint64_t ReadSectors; + uint64_t WriteRequests; + uint64_t WriteSectors; + uint64_t MaxReadSectors; + uint64_t MaxWriteSectors; + uint64_t ReadaheadHits; + uint64_t ReadaheadMisses; + uint64_t ReadaheadStores; + uint64_t ReadaheadInvalidations; + uint64_t ReadaheadBytesServed; + uint64_t ReadaheadSectorsLoaded; + uint64_t ReadaheadSequentialDetections; + uint64_t ReadaheadSuppressed; + uint64_t ReadaheadAllocFailures; + uint64_t ReadaheadReadFailures; + uint64_t MaxReadaheadSectors; +} SYS_DISK_STATS; + +typedef struct SYS_SWAP_STATS { + uint64_t Enabled; + uint64_t TotalSlots; + uint64_t UsedSlots; + uint64_t FreeSlots; + uint64_t HighWaterSlots; + uint64_t PageOuts; + uint64_t PageIns; + uint64_t PageOutFailures; + uint64_t PageInFailures; + uint64_t SlotAllocations; + uint64_t SlotFrees; + uint64_t SlotAllocationFailures; +} SYS_SWAP_STATS; + +typedef struct SYS_PTE_STATS { + uint64_t MapCalls; + uint64_t MapFailures; + uint64_t UnmapCalls; + uint64_t UnmapFailures; + uint64_t ProtectCalls; + uint64_t ProtectFailures; + uint64_t QueryCalls; + uint64_t QueryFailures; + uint64_t QueryByPurpose[6]; + uint64_t QueryFailuresByPurpose[6]; + uint64_t TranslateCalls; + uint64_t LocalFlushes; + uint64_t GlobalFlushes; + uint64_t ShootdownRequests; + uint64_t ShootdownTargetCpus; + uint64_t ShootdownSkippedNoTargets; + uint64_t ShootdownAcks; + uint64_t ShootdownTimeouts; + uint64_t NewMappingNoFlushes; + uint64_t ReplacedMappingFlushes; + uint64_t PageTablePagesAllocated; + uint64_t CurrentLeafMappings; +} SYS_PTE_STATS; + #define SYS_CPU_FLAG_PRESENT 0x00000001u #define SYS_CPU_FLAG_STARTED 0x00000002u #define SYS_CPU_FLAG_PARKED 0x00000004u diff --git a/code/settings.ini b/code/settings.ini index 8d64eff..d72ae7b 100644 --- a/code/settings.ini +++ b/code/settings.ini @@ -24,3 +24,7 @@ daylight_saving=auto hardware_clock=utc ; Used when timezone is a custom/fixed policy; minutes east of UTC. standard_offset_minutes=60 + +[vm] +; 0 disables swap. Nonzero swap must be between 4 MB and 8192 MB. +swap_size_mb=32 diff --git a/code/user/apps/doom/doom.c b/code/user/apps/doom/doom.c index cc41b7c..a6f7188 100644 --- a/code/user/apps/doom/doom.c +++ b/code/user/apps/doom/doom.c @@ -88,7 +88,8 @@ static int doom_read_wrapper(void* handle, void* buf, int count) if (!handle || count <= 0) return 0; DOOM_FILE_HANDLE* f = (DOOM_FILE_HANDLE*)handle; size_t read = 0; - SysReadFile(f->Handle, buf, (size_t)count, &read); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(f->Handle, buf, (size_t)count, &read); + SysReadFile(&req); f->Pos += read; if (f->Pos > f->Size) f->Size = f->Pos; return (int)read; @@ -99,7 +100,8 @@ static int doom_write_wrapper(void* handle, const void* buf, int count) if (!handle || count <= 0) return 0; DOOM_FILE_HANDLE* f = (DOOM_FILE_HANDLE*)handle; size_t written = 0; - SysWriteFile(f->Handle, buf, (size_t)count, &written); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(f->Handle, buf, (size_t)count, &written); + SysWriteFile(&req); f->Pos += written; if (f->Pos > f->Size) f->Size = f->Pos; return (int)written; @@ -262,7 +264,7 @@ static void poll_input(void) } // --------------------------------------------------------------------------- -// Render — Performs lightweight software upscale and blits to the screen +// Render — writes directly into a mapped kernel surface and presents it. // --------------------------------------------------------------------------- #define DOOM_W 320 #define DOOM_H 200 @@ -270,32 +272,65 @@ static void poll_input(void) #define DISP_W (DOOM_W * SCALE) #define DISP_H (DOOM_H * SCALE) -static uint32_t g_blit_buf[DISP_W * DISP_H]; +static uint32_t g_surface_handle; +static uint32_t* g_surface_pixels; +static uint32_t g_surface_pitch_pixels; + +static void init_surface(void) +{ + uint32_t s = SysCreateSurface(DISP_W, DISP_H, SYS_SURFACE_FORMAT_XRGB8888, + SYS_SURFACE_FLAG_NONE, &g_surface_handle); + if (s != SYS_STATUS_SUCCESS || !g_surface_handle) { + doom_print_wrapper("doom: SysCreateSurface failed\n"); + SysExitProcess(1); + } + + void* base = 0; + uint32_t pitchBytes = 0; + s = SysMapSurface(g_surface_handle, &base, &pitchBytes); + if (s != SYS_STATUS_SUCCESS || !base || pitchBytes < DISP_W * sizeof(uint32_t)) { + doom_print_wrapper("doom: SysMapSurface failed\n"); + SysExitProcess(1); + } + + g_surface_pixels = (uint32_t*)base; + g_surface_pitch_pixels = pitchBytes / sizeof(uint32_t); +} static void render_frame(const uint8_t* rgba) { const uint32_t* src = (const uint32_t*)rgba; - - // Software upscale loop mapping 320x200 buffer directly to scaled window buffer + uint32_t* dstBase = g_surface_pixels; + uint32_t dstPitch = g_surface_pitch_pixels; + + if (!dstBase || dstPitch < DISP_W) { + doom_print_wrapper("doom: surface not initialized\n"); + SysExitProcess(1); + } + for (int y = 0; y < DOOM_H; y++) { for (int x = 0; x < DOOM_W; x++) { uint32_t p = src[y * DOOM_W + x]; - - // RGBA -> BGRA conversion (standard for linear x86/64 framebuffers) + + // RGBA -> BGRA/XRGB8888 conversion for the current display path. uint32_t bgra = (p & 0xFF00FF00u) | ((p & 0x00FF0000u) >> 16) | ((p & 0x000000FFu) << 16); - // Populate SCALE * SCALE pixel block + uint32_t* block = dstBase + (size_t)(y * SCALE) * dstPitch + (size_t)(x * SCALE); for (int sy = 0; sy < SCALE; sy++) { for (int sx = 0; sx < SCALE; sx++) { - g_blit_buf[(y * SCALE + sy) * DISP_W + (x * SCALE + sx)] = bgra; + block[(size_t)sy * dstPitch + sx] = bgra; } } } } - SysBlitScreen(0, 0, DISP_W, DISP_H, g_blit_buf); + DISPLAY_RECT rect = { 0, 0, DISP_W, DISP_H }; + uint32_t code = SysPresentSurface(g_surface_handle, &rect, 1); + if (code != SYS_STATUS_SUCCESS) { + SysExitProcess(code); + } } static int build_argv_from_command_line(const char* cmdline, char* storage, size_t storageSize, @@ -373,6 +408,7 @@ int main(void) char argvStorage[512]; const char* argv[32]; int argc = build_argv_from_command_line(cmdline, argvStorage, sizeof(argvStorage), argv, 32); + init_surface(); doom_init(argc, argv, 0); do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: entering game loop\n") }; SysWriteConsole(&req); } while (0); diff --git a/code/user/apps/edit/edit.c b/code/user/apps/edit/edit.c index dec31ef..6579ace 100644 --- a/code/user/apps/edit/edit.c +++ b/code/user/apps/edit/edit.c @@ -380,12 +380,14 @@ static void save_file(void) { for (int i = 0; i < gLineCount; i++) { size_t written = 0; if (gLines[i].len) { - st = SysWriteFile(h, gLines[i].text, (size_t)gLines[i].len, &written); + SYS_FILE_IO_REQUEST writeReq = SysMakeFileIoRequest(h, gLines[i].text, (size_t)gLines[i].len, &written); + st = SysWriteFile(&writeReq); if (st != 0) { SysCloseHandle(h); msg_set("save failed: write error"); return; } } if (i + 1 < gLineCount) { const char nl = '\n'; - st = SysWriteFile(h, &nl, 1, &written); + SYS_FILE_IO_REQUEST newlineReq = SysMakeFileIoRequest(h, &nl, 1, &written); + st = SysWriteFile(&newlineReq); if (st != 0) { SysCloseHandle(h); msg_set("save failed: write error"); return; } } } @@ -418,7 +420,9 @@ static void load_file(void) { char buf[512]; size_t got = 0; bool truncated = false; - while (SysReadFile(h, buf, sizeof(buf), &got) == 0 && got > 0) { + for (;;) { + SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(h, buf, sizeof(buf), &got); + if (SysReadFile(&readReq) != 0 || got == 0) break; for (size_t i = 0; i < got; i++) { char c = buf[i]; if (c == '\r') continue; diff --git a/code/user/apps/memtop/memtop.c b/code/user/apps/memtop/memtop.c new file mode 100644 index 0000000..72ea855 --- /dev/null +++ b/code/user/apps/memtop/memtop.c @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include + +static int u64_to_string(uint64_t n, char* buf) { + int i = 0; + if (n == 0) { + buf[i++] = '0'; + buf[i] = '\0'; + return i; + } + char rev[24]; + int r_idx = 0; + while (n > 0) { + rev[r_idx++] = (char)('0' + (n % 10)); + n /= 10; + } + while (r_idx > 0) { + buf[i++] = rev[--r_idx]; + } + buf[i] = '\0'; + return i; +} + +int main(void) { + SYS_CONSOLE_INFO console; + SYS_MEMORY_STATUS mem; + char metricsBuf[128]; + + while (true) { + if (SysGetConsoleInfo(&console) == SYS_STATUS_SUCCESS && + SysGetMemoryStatus(&mem) == SYS_STATUS_SUCCESS) { + + // If the system isn't reporting columns correctly, default to a safe standard width + int columns = (console.Columns > 0) ? (int)console.Columns : 80; + + char freeStr[24], poolStr[24], heapStr[24]; + u64_to_string(mem.FreePhysicalBytes / 1024u, freeStr); + u64_to_string(mem.KernelPoolBytes / 1024u, poolStr); + u64_to_string(mem.UserHeapBytes / 1024u, heapStr); + + metricsBuf[0] = '\0'; + strcat(metricsBuf, "[ Free: "); + strcat(metricsBuf, freeStr); + strcat(metricsBuf, "K | Pool: "); + strcat(metricsBuf, poolStr); + strcat(metricsBuf, "K | Heap: "); + strcat(metricsBuf, heapStr); + strcat(metricsBuf, "K ]"); + + size_t textLen = strlen(metricsBuf); + int targetX = columns - (int)textLen; + int targetY = 0; + + if (targetX >= 0) { + size_t written = 0; + + // Loop through the string and use the console's dedicated text filling API + // This utilizes the console's native font configuration instead of drawing raw screen bits + for (size_t i = 0; i < textLen; i++) { + SysFillConsoleCharacter(metricsBuf[i], targetX + (int)i, targetY, 1, &written); + } + } + } + + SysSleep(500); + } + return 0; +} \ No newline at end of file diff --git a/code/user/apps/shell_native/shell_native.c b/code/user/apps/shell_native/shell_native.c index 5d638cb..c36b796 100644 --- a/code/user/apps/shell_native/shell_native.c +++ b/code/user/apps/shell_native/shell_native.c @@ -5,9 +5,13 @@ #include #include #include +#include #define MAX_CMD 256 +#define SHELL_FOREGROUND_POLL_MS 30u #define MAX_PROCS 64 // Maximum number of processes we will fetch at once +#define MAX_THREADS 128 +#define SHELL_PROMPT "> " // Output helpers. static void print(const char* s) { SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; SysWriteConsole(&req); } @@ -62,6 +66,26 @@ static void printInt(int32_t n) { else { printUInt((uint32_t)n); } } +// Parses an unsigned decimal integer. +static bool parseUInt(const char* text, uint32_t* out) { + if (!text || !out) return false; + while (*text == ' ' || *text == '\t') text++; + if (*text < '0' || *text > '9') return false; + + uint32_t value = 0; + while (*text >= '0' && *text <= '9') { + uint32_t digit = (uint32_t)(*text - '0'); + if (value > (0xffffffffu - digit) / 10u) return false; + value = value * 10u + digit; + text++; + } + + while (*text == ' ' || *text == '\t') text++; + if (*text != '\0') return false; + *out = value; + return true; +} + // Prints u Int64. static void printUInt64(uint64_t n) { if (n == 0) { print("0"); return; } @@ -103,6 +127,21 @@ static void printUInt64Width(uint64_t n, size_t width) { printUInt64(n); } +// Names nt-lite process-abort exit statuses. +static const char* processExitStatusName(uint32_t code) { + switch (code) { + case STATUS_PROCESS_KILLED: return "killed"; + case STATUS_PROCESS_PAGE_FAULT: return "page-fault"; + case STATUS_PROCESS_GENERAL_PROTECTION: return "general-protection"; + case STATUS_PROCESS_ILLEGAL_OPCODE: return "illegal-opcode"; + case STATUS_PROCESS_DIVIDE_BY_ZERO: return "divide-by-zero"; + case STATUS_PROCESS_BREAKPOINT: return "breakpoint"; + case STATUS_PROCESS_ALIGNMENT_FAULT: return "alignment-fault"; + case STATUS_PROCESS_UNKNOWN_FAULT: return "unknown-fault"; + default: return NULL; + } +} + // Prints hex32. static void printHex32(uint32_t v) { static const char hex[] = "0123456789abcdef"; @@ -140,18 +179,399 @@ static void printStringWidth(const char* s, size_t width) { if (width > len) printSpaces(width - len); } -// Line editor. -static void readLine(char* buf, int len) { - int pos = 0; - while (true) { - char c = SysReadKeyboardChar(); - if (!c) { SysYield(); continue; } - if (c == '\r' || c == '\n') { print("\n"); break; } - if ((c == '\b' || c == 127) && pos > 0) { pos--; print("\b \b"); continue; } - if (c < 0x20 || c == 127) continue; - if (pos < len - 1) { buf[pos++] = c; char echo[2] = {c, '\0'}; print(echo); } +#define SHELL_HISTORY_DEPTH 16 + +static char gHistory[SHELL_HISTORY_DEPTH][MAX_CMD]; +static size_t gHistoryCount; + +static const char* const gShellBuiltinCommands[] = { + "cd", + "clear", + "crash", + "date", + "del", + "dir", + "display", + "exit", + "gpu", + "help", + "kill", + "mem", + "ps", + "stats", + "telemetry", + "threads", + "time", + "type", + "write", +}; + +// Converts a key event into the shell's printable character model. +static char shellKeyToChar(INPUT_KEY key, uint32_t modifiers) { + bool shifted = (modifiers & INPUT_MOD_SHIFT) != 0; + bool caps = (modifiers & INPUT_MOD_CAPS_LOCK) != 0; + + if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) { + char c = (char)('a' + (key - INPUT_KEY_A)); + if (shifted ^ caps) c = (char)(c - 'a' + 'A'); + return c; } - buf[pos] = '\0'; + + if (key >= INPUT_KEY_1 && key <= INPUT_KEY_9) { + static const char shiftedDigits[] = "!@#$%^&*("; + char normal = (char)('1' + (key - INPUT_KEY_1)); + return shifted ? shiftedDigits[key - INPUT_KEY_1] : normal; + } + + switch (key) { + case INPUT_KEY_0: return shifted ? ')' : '0'; + case INPUT_KEY_MINUS: return shifted ? '_' : '-'; + case INPUT_KEY_EQUAL: return shifted ? '+' : '='; + case INPUT_KEY_LEFT_BRACKET: return shifted ? '{' : '['; + case INPUT_KEY_RIGHT_BRACKET: return shifted ? '}' : ']'; + case INPUT_KEY_BACKSLASH: return shifted ? '|' : '\\'; + case INPUT_KEY_SEMICOLON: return shifted ? ':' : ';'; + case INPUT_KEY_APOSTROPHE: return shifted ? '"' : '\''; + case INPUT_KEY_GRAVE: return shifted ? '~' : '`'; + case INPUT_KEY_COMMA: return shifted ? '<' : ','; + case INPUT_KEY_PERIOD: return shifted ? '>' : '.'; + case INPUT_KEY_SLASH: return shifted ? '?' : '/'; + case INPUT_KEY_SPACE: return ' '; + case INPUT_KEY_TAB: return '\t'; + case INPUT_KEY_KP_DIVIDE: return '/'; + case INPUT_KEY_KP_MULTIPLY: return '*'; + case INPUT_KEY_KP_MINUS: return '-'; + case INPUT_KEY_KP_PLUS: return '+'; + case INPUT_KEY_KP_0: return '0'; + case INPUT_KEY_KP_1: return '1'; + case INPUT_KEY_KP_2: return '2'; + case INPUT_KEY_KP_3: return '3'; + case INPUT_KEY_KP_4: return '4'; + case INPUT_KEY_KP_5: return '5'; + case INPUT_KEY_KP_6: return '6'; + case INPUT_KEY_KP_7: return '7'; + case INPUT_KEY_KP_8: return '8'; + case INPUT_KEY_KP_9: return '9'; + case INPUT_KEY_KP_PERIOD: return '.'; + default: return 0; + } +} + +// Adds a non-empty command to the in-memory history ring. +static void shellHistoryAdd(const char* cmd) { + if (!cmd || !*cmd) return; + if (gHistoryCount > 0 && strcmp(gHistory[gHistoryCount - 1], cmd) == 0) return; + + if (gHistoryCount == SHELL_HISTORY_DEPTH) { + memmove(gHistory[0], gHistory[1], (SHELL_HISTORY_DEPTH - 1) * MAX_CMD); + gHistoryCount--; + } + + strncpy(gHistory[gHistoryCount], cmd, MAX_CMD - 1); + gHistory[gHistoryCount][MAX_CMD - 1] = '\0'; + gHistoryCount++; +} + +// Replaces the visible line while keeping the cursor at the end. +static void shellEditorReplaceLine(char* buf, int len, int* pos, const char* text) { + int oldPos = *pos; + for (int i = 0; i < oldPos; i++) print("\b"); + for (int i = 0; i < oldPos; i++) print(" "); + for (int i = 0; i < oldPos; i++) print("\b"); + + int newPos = 0; + if (text) { + while (text[newPos] && newPos < len - 1) { + buf[newPos] = text[newPos]; + newPos++; + } + } + buf[newPos] = '\0'; + *pos = newPos; + print(buf); +} + +// Returns true when a character separates shell tokens. +static bool shellIsTokenSeparator(char c) { + return c == ' ' || c == '\t'; +} + +// Returns true when a character separates path components. +static bool shellIsPathSeparator(char c) { + return c == '/' || c == '\\'; +} + +// Returns true when the token already names an explicit path. +static bool shellTokenHasPathSeparator(const char* token, size_t length) { + for (size_t i = 0; i < length; i++) { + if (shellIsPathSeparator(token[i])) return true; + } + return false; +} + +// Returns true when name starts with prefix. +static bool shellNameStartsWith(const char* name, const char* prefix, size_t prefixLength) { + if (!name || !prefix) return false; + for (size_t i = 0; i < prefixLength; i++) { + if (name[i] == '\0' || name[i] != prefix[i]) return false; + } + return true; +} + +// Extends an in-progress common prefix with one completion candidate. +static void shellUpdateCommonPrefix(char* common, size_t commonSize, const char* name, uint32_t matchCount) { + if (!common || commonSize == 0 || !name) return; + if (matchCount == 0) { + strncpy(common, name, commonSize - 1); + common[commonSize - 1] = '\0'; + return; + } + + size_t i = 0; + while (common[i] && name[i] && common[i] == name[i]) i++; + common[i] = '\0'; +} + +// Appends text to the edited line and echoes it. +static bool shellEditorAppend(char* buf, int len, int* pos, const char* text) { + if (!buf || len <= 0 || !pos || !text) return false; + + bool changed = false; + while (*text && *pos < len - 1) { + buf[*pos] = *text; + (*pos)++; + buf[*pos] = '\0'; + char echo[2] = { *text, '\0' }; + print(echo); + text++; + changed = true; + } + return changed; +} + +// Prints one completion candidate. +static void shellPrintCompletionCandidate(const char* name, bool directory) { + print(" "); + print(name); + if (directory) print("/"); + print("\n"); +} + +// Tracks completion candidates without storing the full candidate list. +typedef struct SHELL_COMPLETION_STATE { + uint32_t MatchCount; + bool LastIsDirectory; + bool LastAddsSpace; + char Common[SYS_DIR_NAME_MAX]; +} SHELL_COMPLETION_STATE; + +// Records one completion candidate. +static void shellRecordCompletion(SHELL_COMPLETION_STATE* state, const char* name, bool directory, bool addsSpace) { + if (!state || !name || !*name) return; + shellUpdateCommonPrefix(state->Common, sizeof(state->Common), name, state->MatchCount); + state->MatchCount++; + state->LastIsDirectory = directory; + state->LastAddsSpace = addsSpace; +} + +// Collects builtin command matches for a command-position token. +static void shellCollectBuiltinCompletions(const char* prefix, size_t prefixLength, + SHELL_COMPLETION_STATE* state, + bool printMatches) { + size_t count = sizeof(gShellBuiltinCommands) / sizeof(gShellBuiltinCommands[0]); + for (size_t i = 0; i < count; i++) { + const char* name = gShellBuiltinCommands[i]; + if (!shellNameStartsWith(name, prefix, prefixLength)) continue; + if (printMatches) shellPrintCompletionCandidate(name, false); + else shellRecordCompletion(state, name, false, true); + } +} + +// Splits a token into a directory to scan and a filename prefix to match. +static bool shellSplitCompletionPath(const char* token, size_t tokenLength, + char* dir, size_t dirSize, + const char** prefixOut, size_t* prefixLengthOut) { + if (!token || !dir || dirSize == 0 || !prefixOut || !prefixLengthOut) return false; + + size_t lastSeparator = (size_t)-1; + for (size_t i = 0; i < tokenLength; i++) { + if (shellIsPathSeparator(token[i])) lastSeparator = i; + } + + if (lastSeparator == (size_t)-1) { + if (dirSize < 2) return false; + dir[0] = '.'; + dir[1] = '\0'; + *prefixOut = token; + *prefixLengthOut = tokenLength; + return true; + } + + size_t dirLength = lastSeparator; + if (lastSeparator == 0) dirLength = 1; + if (lastSeparator >= 2 && token[1] == ':' && shellIsPathSeparator(token[2])) dirLength = lastSeparator + 1; + if (dirLength + 1 > dirSize) return false; + + memcpy(dir, token, dirLength); + dir[dirLength] = '\0'; + *prefixOut = token + lastSeparator + 1; + *prefixLengthOut = tokenLength - lastSeparator - 1; + return true; +} + +// Collects filesystem path matches using the existing directory enumeration ABI. +static void shellCollectPathCompletions(const char* token, size_t tokenLength, + SHELL_COMPLETION_STATE* state, + bool printMatches) { + char dir[MAX_CMD]; + const char* prefix = token; + size_t prefixLength = tokenLength; + + if (!shellSplitCompletionPath(token, tokenLength, dir, sizeof(dir), &prefix, &prefixLength)) return; + + uint32_t hDir = 0; + if (open_dir(dir, &hDir) != 0) return; + + SYS_DIR_ENTRY entry; + while (SysReadDir(hDir, &entry) == 0) { + if (!entry.Name[0]) continue; + if (!shellNameStartsWith(entry.Name, prefix, prefixLength)) continue; + + bool directory = (entry.Attr & 0x10u) != 0; + if (printMatches) shellPrintCompletionCandidate(entry.Name, directory); + else shellRecordCompletion(state, entry.Name, directory, true); + } + + SysCloseHandle(hDir); +} + +// Redraws the prompt and current edit buffer after a completion list. +static void shellRedrawPromptAndLine(const char* buf) { + print(SHELL_PROMPT); + print(buf ? buf : ""); +} + +// Completes the token at the end of the current line. +static void shellCompleteLine(char* buf, int len, int* pos) { + if (!buf || len <= 0 || !pos || *pos < 0) return; + + size_t cursor = (size_t)*pos; + size_t tokenStart = cursor; + while (tokenStart > 0 && !shellIsTokenSeparator(buf[tokenStart - 1])) tokenStart--; + + const char* token = buf + tokenStart; + size_t tokenLength = cursor - tokenStart; + bool commandPosition = tokenStart == 0; + bool explicitPath = shellTokenHasPathSeparator(token, tokenLength); + + char completionDir[MAX_CMD]; + const char* namePrefix = token; + size_t namePrefixLength = tokenLength; + (void)shellSplitCompletionPath(token, tokenLength, completionDir, sizeof(completionDir), &namePrefix, &namePrefixLength); + + SHELL_COMPLETION_STATE state; + memset(&state, 0, sizeof(state)); + + if (commandPosition && !explicitPath) { + shellCollectBuiltinCompletions(token, tokenLength, &state, false); + } + shellCollectPathCompletions(token, tokenLength, &state, false); + + if (state.MatchCount == 0) return; + + size_t commonLength = strlen(state.Common); + bool expanded = commonLength > namePrefixLength; + if (expanded) { + shellEditorAppend(buf, len, pos, state.Common + namePrefixLength); + } + + if (state.MatchCount == 1) { + if (state.LastIsDirectory) { + if (*pos < len - 1 && *pos > 0 && !shellIsPathSeparator(buf[*pos - 1])) shellEditorAppend(buf, len, pos, "/"); + } else if (state.LastAddsSpace) { + if (*pos < len - 1 && (*pos == 0 || buf[*pos - 1] != ' ')) shellEditorAppend(buf, len, pos, " "); + } + return; + } + + if (expanded) return; + + print("\n"); + if (commandPosition && !explicitPath) { + shellCollectBuiltinCompletions(token, tokenLength, NULL, true); + } + shellCollectPathCompletions(token, tokenLength, NULL, true); + shellRedrawPromptAndLine(buf); +} + +// Reads one editable command line from the keyboard. +static bool readLine(char* buf, int len) { + int pos = 0; + size_t historyCursor = gHistoryCount; + if (len <= 0) return false; + buf[0] = '\0'; + + while (true) { + INPUT_KEY_RECORD input; + if (SysReadInputEvent(&input) != 0) { SysYield(); continue; } + + INPUT_KEY key = input.Key; + if (key == INPUT_KEY_NONE) { SysYield(); continue; } + + uint32_t modifiers = input.Modifiers; + if ((modifiers & INPUT_MOD_CTRL) && key == INPUT_KEY_C) { + buf[0] = '\0'; + print("^C\n"); + return true; + } + + if (key == INPUT_KEY_ENTER || key == INPUT_KEY_KP_ENTER) { + print("\n"); + break; + } + + if (key == INPUT_KEY_BACKSPACE) { + if (pos > 0) { + pos--; + buf[pos] = '\0'; + print("\b \b"); + } + continue; + } + + if (key == INPUT_KEY_UP) { + if (historyCursor > 0) { + historyCursor--; + shellEditorReplaceLine(buf, len, &pos, gHistory[historyCursor]); + } + continue; + } + + if (key == INPUT_KEY_DOWN) { + if (historyCursor < gHistoryCount) { + historyCursor++; + shellEditorReplaceLine(buf, len, &pos, + historyCursor < gHistoryCount ? gHistory[historyCursor] : ""); + } + continue; + } + + if (key == INPUT_KEY_TAB) { + shellCompleteLine(buf, len, &pos); + continue; + } + + char c = input.Char ? input.Char : shellKeyToChar(key, modifiers); + if (c < 0x20 || c == 127) continue; + if (pos < len - 1) { + buf[pos++] = c; + buf[pos] = '\0'; + char echo[2] = { c, '\0' }; + print(echo); + } + } + + return false; } // Implements the trim operation. @@ -170,8 +590,10 @@ static void cmd_help() { " mem - Show memory usage\n" " time - Show system wall-clock time\n" " display - Show display provider and counters\n" - " stats - Show scheduler/CPU/PMM telemetry\n" + " stats - Show scheduler/CPU/MM telemetry\n" " ps - List processes\n" + " threads - Show one process's thread statuses\n" + " kill - Terminate a process\n" " dir [path] - List directory contents\n" " cd - Change directory\n" " type - Display file contents\n" @@ -209,6 +631,14 @@ static void cmd_stats(void) { SYS_SCHEDULER_STATS sched; SYS_PMM_STATS pmm; SYS_CPU_STATS cpu; + SYS_PFN_STATS pfn; + SYS_VAD_STATS vad; + SYS_SWAP_STATS swap; + SYS_FILE_CACHE_STATS fileCache; + SYS_FILESYSTEM_STATS filesystem; + SYS_DISK_STATS disk; + SYS_PTE_STATS pte; + SYS_SURFACE_STATS surface; uint32_t status; status = SysQueryKernelStats(SYS_KERNEL_STATS_SCHEDULER, &sched, sizeof(sched), NULL); @@ -235,6 +665,70 @@ static void cmd_stats(void) { return; } + status = SysQueryKernelStats(SYS_KERNEL_STATS_PFN, &pfn, sizeof(pfn), NULL); + if (status != 0) { + print("stats: pfn query failed: "); + printUInt(status); + print("\n"); + return; + } + + status = SysQueryKernelStats(SYS_KERNEL_STATS_VAD, &vad, sizeof(vad), NULL); + if (status != 0) { + print("stats: vad query failed: "); + printUInt(status); + print("\n"); + return; + } + + status = SysQueryKernelStats(SYS_KERNEL_STATS_SWAP, &swap, sizeof(swap), NULL); + if (status != 0) { + print("stats: swap query failed: "); + printUInt(status); + print("\n"); + return; + } + + status = SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, &fileCache, sizeof(fileCache), NULL); + if (status != 0) { + print("stats: file cache query failed: "); + printUInt(status); + print("\n"); + return; + } + + status = SysQueryKernelStats(SYS_KERNEL_STATS_FILESYSTEM, &filesystem, sizeof(filesystem), NULL); + if (status != 0) { + print("stats: filesystem query failed: "); + printUInt(status); + print("\n"); + return; + } + + status = SysQueryKernelStats(SYS_KERNEL_STATS_DISK, &disk, sizeof(disk), NULL); + if (status != 0) { + print("stats: disk query failed: "); + printUInt(status); + print("\n"); + return; + } + + status = SysQueryKernelStats(SYS_KERNEL_STATS_PTE, &pte, sizeof(pte), NULL); + if (status != 0) { + print("stats: pte query failed: "); + printUInt(status); + print("\n"); + return; + } + + status = SysQueryKernelStats(SYS_KERNEL_STATS_SURFACE, &surface, sizeof(surface), NULL); + if (status != 0) { + print("stats: surface query failed: "); + printUInt(status); + print("\n"); + return; + } + print("Scheduler:\n"); print(" ticks: "); printUInt64(sched.Ticks); print("\n"); print(" context switches: "); printUInt64(sched.ContextSwitches); print("\n"); @@ -256,7 +750,12 @@ static void cmd_stats(void) { SYS_CPU_STAT* c = &cpu.Cpu[i]; print(" cpu"); printUInt(c->CpuIndex); print(" lapic="); printUInt(c->LapicId); - print(" tid="); printUInt(c->CurrentTid); + print(" current="); + if (c->CurrentThreadFlags & SYS_CPU_THREAD_FLAG_IDLE) { + print("idle/"); printUInt(c->CpuIndex); + } else { + print("tid/"); printUInt(c->CurrentTid); + } print(" ticks="); printUInt64(c->SchedulerTicks); print(" switches="); printUInt64(c->ContextSwitches); print(" flags="); printCpuFlags(c->Flags); @@ -269,6 +768,8 @@ static void cmd_stats(void) { print(" free: "); printPagesKiB(pmm.FreeFrames); print(" (frames="); printUInt64(pmm.FreeFrames); print(")\n"); print(" largest free run: "); printPagesKiB(pmm.LargestFreeRun); print(" (frames="); printUInt64(pmm.LargestFreeRun); print(")\n"); print(" free runs: "); printUInt64(pmm.FreeRunCount); print("\n"); + print(" available+reclaim: "); printSizeKiB(pfn.AvailableFrames * 4096ull); + print(" (reclaimable="); printSizeKiB(pfn.ReclaimableFrames * 4096ull); print(")\n"); print(" run histogram:\n"); print(" 1: "); printUInt64(pmm.FreeRunHistogram[0]); print("\n"); print(" 2-3: "); printUInt64(pmm.FreeRunHistogram[1]); print("\n"); @@ -278,6 +779,236 @@ static void cmd_stats(void) { print(" 64-255: "); printUInt64(pmm.FreeRunHistogram[5]); print("\n"); print(" 256-1023: "); printUInt64(pmm.FreeRunHistogram[6]); print("\n"); print(" 1024+: "); printUInt64(pmm.FreeRunHistogram[7]); print("\n"); + + print("PFN database:\n"); + print(" states: active="); printUInt64(pfn.ActiveFrames); + print(" free="); printUInt64(pfn.FreeFrames); + print(" zeroed="); printUInt64(pfn.ZeroedFrames); + print(" standby="); printUInt64(pfn.StandbyFrames); + print(" modified="); printUInt64(pfn.ModifiedFrames); + print(" reserved="); printUInt64(pfn.ReservedFrames); print("\n"); + print(" eviction: evictable="); printUInt64(pfn.EvictableFrames); + print(" reclaimable="); printUInt64(pfn.ReclaimableFrames); + print(" available="); printUInt64(pfn.AvailableFrames); + print(" mapped="); printUInt64(pfn.MappedFrames); + print(" non-evictable="); printUInt64(pfn.NonEvictableFrames); + print(" pinned="); printUInt64(pfn.PinnedFrames); print("\n"); + print(" usage frames: kernel="); printUInt64(pfn.KernelFrames); + print(" pt="); printUInt64(pfn.PageTableFrames); + print(" user="); printUInt64(pfn.UserFrames); + print(" driver="); printUInt64(pfn.DriverFrames); + print(" dma="); printUInt64(pfn.DmaFrames); + print(" boot="); printUInt64(pfn.BootloaderFrames); + print(" pfn-db="); printUInt64(pfn.PfnDatabaseFrames); + print(" mmio="); printUInt64(pfn.MmioFrames); print("\n"); + print(" buckets: clean-file="); printUInt64(pfn.CleanFileCacheFrames); + print(" (active="); printUInt64(pfn.ActiveCleanFileCacheFrames); + print(" standby="); printUInt64(pfn.StandbyCleanFileCacheFrames); print(")"); + print(" image="); printUInt64(pfn.ImageFrames); + print(" (active="); printUInt64(pfn.ActiveImageFrames); + print(" standby="); printUInt64(pfn.StandbyImageFrames); print(")"); + print(" mapped-file="); printUInt64(pfn.MappedFileFrames); + print(" anon="); printUInt64(pfn.AnonymousFrames); + print(" cow="); printUInt64(pfn.CowFrames); + print(" surface="); printUInt64(pfn.SurfaceFrames); print("\n"); + print(" operations: alloc="); printUInt64(pfn.AllocationCount); + print(" free="); printUInt64(pfn.FreeCount); + print(" retain="); printUInt64(pfn.RetainCount); + print(" release="); printUInt64(pfn.ReleaseCount); + print(" pin="); printUInt64(pfn.PinCount); + print(" unpin="); printUInt64(pfn.UnpinCount); print("\n"); + print(" failures: alloc="); printUInt64(pfn.FailedAllocations); + print(" contiguous="); printUInt64(pfn.FailedContiguousAllocations); print("\n"); + print(" alloc reasons: invalid-range="); printUInt64(pfn.FailedInvalidRange); + print(" empty-range="); printUInt64(pfn.FailedEmptyRange); + print(" reserve-denied="); printUInt64(pfn.FailedReserveDenied); + print(" no-frame="); printUInt64(pfn.FailedNoFrame); print("\n"); + print(" contig reasons: invalid="); printUInt64(pfn.FailedContiguousInvalidRequest); + print(" reserve-denied="); printUInt64(pfn.FailedContiguousReserveDenied); + print(" no-run="); printUInt64(pfn.FailedContiguousNoRun); print("\n"); + print(" reclaim: attempts="); printUInt64(pfn.ReclaimAttempts); + print(" requested="); printUInt64(pfn.ReclaimPagesRequested); + print(" succeeded="); printUInt64(pfn.ReclaimPagesSucceeded); + print(" failed="); printUInt64(pfn.ReclaimPagesFailed); + print(" reentrant-skips="); printUInt64(pfn.ReclaimReentrantSkips); print("\n"); + + print("VAD faults:\n"); + print(" resolve: calls="); printUInt64(vad.ResolveFaultCalls); + print(" resolved="); printUInt64(vad.ResolvedFaults); + print(" failed="); printUInt64(vad.FailedFaults); print("\n"); + print(" failure reasons: protection="); printUInt64(vad.ProtectionFaults); + print(" no-vad="); printUInt64(vad.NoVadFaults); + print(" uncommitted="); printUInt64(vad.UncommittedFaults); + print(" noaccess="); printUInt64(vad.NoAccessFaults); print("\n"); + print(" materialization: demand-zero="); printUInt64(vad.DemandZeroPages); + print(" zero-maps="); printUInt64(vad.ZeroPageMaps); + print(" zero-splits="); printUInt64(vad.ZeroPageCowSplits); + print(" ensure-calls="); printUInt64(vad.EnsureResidentCalls); + print(" ensure-pages="); printUInt64(vad.EnsureResidentPages); + print(" already-resident="); printUInt64(vad.AlreadyResidentPages); print("\n"); + print(" cow: faults="); printUInt64(vad.CowFaults); + print(" copied="); printUInt64(vad.CowPagesCopied); + print(" views="); printUInt64(vad.CowViewsCreated); + print(" shared-pages="); printUInt64(vad.CowPagesShared); print("\n"); + print(" file-backed: faults="); printUInt64(vad.FileBackedFaults); + print(" reads="); printUInt64(vad.FileBackedReads); + print(" read-fail="); printUInt64(vad.FileBackedReadFailures); + print(" discards="); printUInt64(vad.FileBackedDiscards); print("\n"); + print(" mapped-file: shared-faults="); printUInt64(vad.FileBackedSharedWritableFaults); + print(" dirty-faults="); printUInt64(vad.FileBackedDirtyFaults); + print(" dirty-syncs="); printUInt64(vad.FileBackedDirtySyncs); print("\n"); + print(" image-backed: faults="); printUInt64(vad.ImageBackedFaults); + print(" zero-faults="); printUInt64(vad.ImageBackedZeroFaults); + print(" private-faults="); printUInt64(vad.ImageBackedPrivateFaults); print("\n"); + print(" image fixups: calls="); printUInt64(vad.ImageFixupPrivateCalls); + print(" private-pages="); printUInt64(vad.ImageFixupPrivatePages); print("\n"); + print(" swap-backed: page-outs="); printUInt64(vad.SwapPageOuts); + print(" page-ins="); printUInt64(vad.SwapPageIns); print("\n"); + + print("Swap:\n"); + print(" enabled: "); printUInt64(swap.Enabled); print("\n"); + print(" slots: total="); printUInt64(swap.TotalSlots); + print(" used="); printUInt64(swap.UsedSlots); + print(" free="); printUInt64(swap.FreeSlots); + print(" high-water="); printUInt64(swap.HighWaterSlots); print("\n"); + print(" io: out="); printUInt64(swap.PageOuts); + print(" in="); printUInt64(swap.PageIns); + print(" out-fail="); printUInt64(swap.PageOutFailures); + print(" in-fail="); printUInt64(swap.PageInFailures); print("\n"); + print(" slots ops: alloc="); printUInt64(swap.SlotAllocations); + print(" free="); printUInt64(swap.SlotFrees); + print(" alloc-fail="); printUInt64(swap.SlotAllocationFailures); print("\n"); + + print("File cache:\n"); + print(" lookups: total="); printUInt64(fileCache.Lookups); + print(" hits="); printUInt64(fileCache.Hits); + print(" misses="); printUInt64(fileCache.Misses); print("\n"); + print(" loads: ok="); printUInt64(fileCache.Loads); + print(" fail="); printUInt64(fileCache.LoadFailures); + print(" races="); printUInt64(fileCache.RaceDiscards); + print(" waits="); printUInt64(fileCache.InflightWaits); print("\n"); + print(" lifetime: resident="); printUInt64(fileCache.ResidentPages); + print(" active="); printUInt64(fileCache.ActivePages); + print(" standby="); printUInt64(fileCache.StandbyPages); + print(" max="); printUInt64(fileCache.MaxResidentPages); + print(" invalidations="); printUInt64(fileCache.Invalidations); print("\n"); + print(" dirty/writeback: dirty="); printUInt64(fileCache.DirtyPages); + print(" max-dirty="); printUInt64(fileCache.MaxDirtyPages); + print(" marks="); printUInt64(fileCache.DirtyMarks); + print(" wb-req="); printUInt64(fileCache.WritebackRequests); + print(" wb-pages="); printUInt64(fileCache.WritebackPages); + print(" wb-bytes="); printUInt64(fileCache.WritebackBytes); + print(" wb-fail="); printUInt64(fileCache.WritebackFailures); print("\n"); + print(" mapped dirty: syncs="); printUInt64(fileCache.MappedDirtySyncs); + print(" pages="); printUInt64(fileCache.MappedDirtyPages); print("\n"); + print(" flushes: calls="); printUInt64(fileCache.FlushFileCalls); + print(" fail="); printUInt64(fileCache.FlushFileFailures); print("\n"); + print(" reclaim: drops="); printUInt64(fileCache.Drops); + print(" standby="); printUInt64(fileCache.StandbyTransitions); + print(" standby-hits="); printUInt64(fileCache.StandbyHits); + print(" drop-fail="); printUInt64(fileCache.DropFailures); print("\n"); + print(" read-ahead: requests="); printUInt64(fileCache.ReadAheadRequests); + print(" requested="); printUInt64(fileCache.ReadAheadPagesRequested); + print(" queued="); printUInt64(fileCache.ReadAheadPagesQueued); + print(" loaded="); printUInt64(fileCache.ReadAheadPagesLoaded); + print(" skipped="); printUInt64(fileCache.ReadAheadPagesSkipped); + print(" fail="); printUInt64(fileCache.ReadAheadFailures); + print(" throttled="); printUInt64(fileCache.ReadAheadPagesThrottled); print("\n"); + print(" read-ahead max: pages/request="); printUInt64(fileCache.MaxReadAheadPagesPerRequest); print("\n"); + print(" batch fills: reads="); printUInt64(fileCache.BatchReads); + print(" requested="); printUInt64(fileCache.BatchPagesRequested); + print(" loaded="); printUInt64(fileCache.BatchPagesLoaded); + print(" max-request="); printUInt64(fileCache.MaxBatchPagesRequested); + print(" max-loaded="); printUInt64(fileCache.MaxBatchPagesLoaded); print("\n"); + + print("Filesystem:\n"); + print(" path cache: lookups="); printUInt64(filesystem.PathLookups); + print(" hits="); printUInt64(filesystem.PathCacheHits); + print(" misses="); printUInt64(filesystem.PathCacheMisses); + print(" stores="); printUInt64(filesystem.PathCacheStores); print("\n"); + print(" invalidation: count="); printUInt64(filesystem.PathCacheInvalidations); + print(" generation="); printUInt64(filesystem.PathCacheGeneration); print("\n"); + print(" directory search: searches="); printUInt64(filesystem.DirectorySearches); + print(" clusters="); printUInt64(filesystem.DirectoryClustersRead); + print(" entries="); printUInt64(filesystem.DirectoryEntriesScanned); print("\n"); + print(" FAT chain cache: lookups="); printUInt64(filesystem.FatChainCacheLookups); + print(" hits="); printUInt64(filesystem.FatChainCacheHits); + print(" misses="); printUInt64(filesystem.FatChainCacheMisses); + print(" stores="); printUInt64(filesystem.FatChainCacheStores); print("\n"); + print(" FAT chain work: invalidations="); printUInt64(filesystem.FatChainCacheInvalidations); + print(" fat-steps="); printUInt64(filesystem.FatChainWalkSteps); print("\n"); + print(" data read spans: runs="); printUInt64(filesystem.DataReadRuns); + print(" sectors="); printUInt64(filesystem.DataReadSectors); + print(" merges="); printUInt64(filesystem.DataReadRunMerges); + print(" max-sectors="); printUInt64(filesystem.MaxDataReadSpanSectors); print("\n"); + + print("Disk I/O:\n"); + print(" devices: registered="); printUInt64(disk.RegisteredDisks); print("\n"); + print(" reads: requests="); printUInt64(disk.ReadRequests); + print(" sectors="); printUInt64(disk.ReadSectors); + print(" max-sectors="); printUInt64(disk.MaxReadSectors); print("\n"); + print(" writes: requests="); printUInt64(disk.WriteRequests); + print(" sectors="); printUInt64(disk.WriteSectors); + print(" max-sectors="); printUInt64(disk.MaxWriteSectors); print("\n"); + print(" read-ahead: hits="); printUInt64(disk.ReadaheadHits); + print(" misses="); printUInt64(disk.ReadaheadMisses); + print(" stores="); printUInt64(disk.ReadaheadStores); + print(" sectors="); printUInt64(disk.ReadaheadSectorsLoaded); + print(" max-sectors="); printUInt64(disk.MaxReadaheadSectors); print("\n"); + print(" read-ahead misc: served-bytes="); printUInt64(disk.ReadaheadBytesServed); + print(" seq="); printUInt64(disk.ReadaheadSequentialDetections); + print(" invalidations="); printUInt64(disk.ReadaheadInvalidations); + print(" suppressed="); printUInt64(disk.ReadaheadSuppressed); + print(" alloc-fail="); printUInt64(disk.ReadaheadAllocFailures); + print(" read-fail="); printUInt64(disk.ReadaheadReadFailures); print("\n"); + + print("Surfaces:\n"); + print(" lifetime: created="); printUInt64(surface.CreatedSurfaces); + print(" destroyed="); printUInt64(surface.DestroyedSurfaces); + print(" live="); printUInt64(surface.LiveSurfaces); print("\n"); + print(" live memory: pages="); printUInt64(surface.LivePages); + print(" bytes="); printUInt64(surface.LiveBytes); print("\n"); + print(" maps/presents: maps="); printUInt64(surface.MapCalls); + print(" presents="); printUInt64(surface.PresentCalls); + print(" input-rects="); printUInt64(surface.PresentRectInputs); print("\n"); + print(" present work: clipped="); printUInt64(surface.PresentClippedRects); + print(" empty="); printUInt64(surface.PresentEmptyClips); + print(" rows="); printUInt64(surface.PresentRowsCopied); + print(" bytes="); printUInt64(surface.PresentBytesCopied); print("\n"); + + print("Page tables:\n"); + print(" leaf mappings: "); printUInt64(pte.CurrentLeafMappings); print("\n"); + print(" page-table pages: "); printUInt64(pte.PageTablePagesAllocated); print("\n"); + print(" map/unmap: map="); printUInt64(pte.MapCalls); + print(" failed="); printUInt64(pte.MapFailures); + print(" unmap="); printUInt64(pte.UnmapCalls); + print(" failed="); printUInt64(pte.UnmapFailures); print("\n"); + print(" protect: calls="); printUInt64(pte.ProtectCalls); + print(" failed="); printUInt64(pte.ProtectFailures); print("\n"); + print(" query: total="); printUInt64(pte.QueryCalls); + print(" failed="); printUInt64(pte.QueryFailures); + print(" translate="); printUInt64(pte.TranslateCalls); print("\n"); + print(" generic: "); printUInt64(pte.QueryByPurpose[0]); + print(" failed="); printUInt64(pte.QueryFailuresByPurpose[0]); print("\n"); + print(" user-probe: "); printUInt64(pte.QueryByPurpose[1]); + print(" failed="); printUInt64(pte.QueryFailuresByPurpose[1]); print("\n"); + print(" heap: "); printUInt64(pte.QueryByPurpose[2]); + print(" failed="); printUInt64(pte.QueryFailuresByPurpose[2]); print("\n"); + print(" vad: "); printUInt64(pte.QueryByPurpose[3]); + print(" failed="); printUInt64(pte.QueryFailuresByPurpose[3]); print("\n"); + print(" debug: "); printUInt64(pte.QueryByPurpose[4]); + print(" failed="); printUInt64(pte.QueryFailuresByPurpose[4]); print("\n"); + print(" self-test: "); printUInt64(pte.QueryByPurpose[5]); + print(" failed="); printUInt64(pte.QueryFailuresByPurpose[5]); print("\n"); + print(" tlb: local="); printUInt64(pte.LocalFlushes); + print(" global="); printUInt64(pte.GlobalFlushes); + print(" new-map-skip="); printUInt64(pte.NewMappingNoFlushes); + print(" replace-flush="); printUInt64(pte.ReplacedMappingFlushes); print("\n"); + print(" shootdowns: requests="); printUInt64(pte.ShootdownRequests); + print(" targets="); printUInt64(pte.ShootdownTargetCpus); + print(" skipped="); printUInt64(pte.ShootdownSkippedNoTargets); + print(" acks="); printUInt64(pte.ShootdownAcks); + print(" timeouts="); printUInt64(pte.ShootdownTimeouts); print("\n"); } @@ -369,11 +1100,23 @@ static void cmd_mem(void) { } print("Memory Status:\n"); - print(" Free physical: "); + print(" Free physical: "); printSizeKiB(st.FreePhysicalBytes); - print("\n Kernel pool: "); + print("\n Reclaimable: "); + printSizeKiB(st.ReclaimablePhysicalBytes); + print("\n Available physical: "); + printSizeKiB(st.AvailablePhysicalBytes); + print("\n Pinned physical: "); + printSizeKiB(st.PinnedPhysicalBytes); + print("\n Non-evictable: "); + printSizeKiB(st.NonEvictablePhysicalBytes); + print("\n User frames: "); + printSizeKiB(st.UserPhysicalBytes); + print("\n Page tables: "); + printSizeKiB(st.PageTablePhysicalBytes); + print("\n Kernel pool: "); printSizeKiB(st.KernelPoolBytes); - print("\n User heap: "); + print("\n User heap: "); printSizeKiB(st.UserHeapBytes); print("\n"); } @@ -388,6 +1131,57 @@ static const char* processStateName(uint32_t state) { } } +// Names public thread states. +static const char* threadStateName(uint32_t state) { + switch (state) { + case SYS_THREAD_STATE_READY: return "ready"; + case SYS_THREAD_STATE_RUNNING: return "run"; + case SYS_THREAD_STATE_WAITING: return "wait"; + case SYS_THREAD_STATE_DEAD: return "dead"; + default: return "?"; + } +} + +// Names public wait object types. +static const char* threadWaitName(uint32_t waitType) { + switch (waitType) { + case SYS_THREAD_WAIT_NONE: return "-"; + case SYS_THREAD_WAIT_PROCESS: return "process"; + case SYS_THREAD_WAIT_THREAD: return "thread"; + case SYS_THREAD_WAIT_MUTEX: return "mutex"; + case SYS_THREAD_WAIT_SEMAPHORE: return "sem"; + case SYS_THREAD_WAIT_EVENT: return "event"; + case SYS_THREAD_WAIT_QUEUE: return "queue"; + case SYS_THREAD_WAIT_FILE: return "file"; + case SYS_THREAD_WAIT_DIR: return "dir"; + case SYS_THREAD_WAIT_ASYNC_IO: return "async"; + case SYS_THREAD_WAIT_SOCKET: return "socket"; + case SYS_THREAD_WAIT_DEVICE: return "device"; + case SYS_THREAD_WAIT_DRIVER: return "driver"; + case SYS_THREAD_WAIT_USB_DEVICE: return "usbdev"; + case SYS_THREAD_WAIT_USB_PIPE: return "usbpipe"; + case SYS_THREAD_WAIT_FILE_MAPPING: return "mapping"; + case SYS_THREAD_WAIT_SURFACE: return "surface"; + default: return "unknown"; + } +} + +// Prints thread status flags. +static void printThreadFlags(uint32_t flags) { + char buf[8]; + size_t n = 0; + if (flags & SYS_THREAD_FLAG_CURRENT) buf[n++] = '*'; + if (flags & SYS_THREAD_FLAG_IDLE) buf[n++] = 'I'; + if (flags & SYS_THREAD_FLAG_KERNEL) buf[n++] = 'K'; + if (flags & SYS_THREAD_FLAG_QUEUED) buf[n++] = 'Q'; + if (flags & SYS_THREAD_FLAG_ALERTABLE_WAIT) buf[n++] = 'A'; + if (flags & SYS_THREAD_FLAG_KERNEL_APC_PENDING) buf[n++] = 'k'; + if (flags & SYS_THREAD_FLAG_USER_APC_PENDING) buf[n++] = 'u'; + if (n == 0) buf[n++] = '-'; + buf[n] = '\0'; + printStringWidth(buf, 7); +} + // Implements the cmd ps operation. static void cmd_ps(void) { SYS_PROCESS_INFO procs[MAX_PROCS]; @@ -403,14 +1197,16 @@ static void cmd_ps(void) { printUInt((uint32_t)count); print("):\n"); print(" PID State Thr Hnd Mod UserKiB ImageKiB Flags Name\n"); + print(" Flags: S=system I=idle *=current T=terminating X=exited.\n"); size_t displayCount = count < MAX_PROCS ? count : MAX_PROCS; for (size_t i = 0; i < displayCount; i++) { SYS_PROCESS_INFO* p = &procs[i]; - char flags[5]; + char flags[6]; size_t flagLen = 0; if (p->Flags & SYS_PROCESS_FLAG_SYSTEM) flags[flagLen++] = 'S'; + if (p->Flags & SYS_PROCESS_FLAG_IDLE) flags[flagLen++] = 'I'; if (p->Flags & SYS_PROCESS_FLAG_CURRENT) flags[flagLen++] = '*'; if (p->Flags & SYS_PROCESS_FLAG_TERMINATING) flags[flagLen++] = 'T'; if (p->Flags & SYS_PROCESS_FLAG_EXITED) flags[flagLen++] = 'X'; @@ -449,6 +1245,86 @@ static void cmd_ps(void) { } } +// Implements the cmd threads operation. +static void cmd_threads(const char* args) { + uint32_t pid = 0; + if (!parseUInt(args, &pid)) { + print("threads: usage: threads \n"); + return; + } + + SYS_THREAD_INFO threads[MAX_THREADS]; + size_t count = 0; + uint32_t status = SysGetThreadList(pid, threads, MAX_THREADS, &count); + if (status != 0) { + print("threads: query failed status="); + printUInt(status); + print("\n"); + return; + } + + print("Threads for PID "); + printUInt(pid); + print(" (Count: "); + printUInt((uint32_t)count); + print("):\n"); + print(" TID State Pri CPU LastCPU QRem QTick Burst Wait WStat Flags\n"); + print(" Flags: *=current I=idle K=kernel Q=queued A=alertable k=kernel-apc u=user-apc.\n"); + + size_t displayCount = count < MAX_THREADS ? count : MAX_THREADS; + for (size_t i = 0; i < displayCount; i++) { + SYS_THREAD_INFO* t = &threads[i]; + print(" "); + printUIntWidth(t->ThreadId, 3); + print(" "); + printStringWidth(threadStateName(t->State), 5); + print(" "); + printUIntWidth(t->Priority, 3); + print(" "); + if (t->RunningCpu == 0xffffffffu) printStringWidth("-", 3); + else printUIntWidth(t->RunningCpu, 3); + print(" "); + if (t->LastCpu == 0xffffffffu) printStringWidth("-", 7); + else printUIntWidth(t->LastCpu, 7); + print(" "); + printUIntWidth(t->QuantumRemaining, 4); + print(" "); + printUIntWidth(t->QuantumTicks, 5); + print(" "); + printUIntWidth(t->CpuBurstTicks, 5); + print(" "); + printStringWidth(threadWaitName(t->WaitObjectType), 8); + print(" "); + printUIntWidth(t->WaitStatus, 5); + print(" "); + printThreadFlags(t->Flags); + print("\n"); + } + + if (count > MAX_THREADS) { + print(" ... and "); + printUInt((uint32_t)(count - MAX_THREADS)); + print(" more threads.\n"); + } +} + +static void cmd_kill(const char* args) { + uint32_t pid = 0; + if (!parseUInt(args, &pid) || pid == 0) { print("kill: usage: kill \n"); return; } + uint32_t status = SysKillProcess(pid, (int)STATUS_PROCESS_KILLED); + if (status != 0) { + print("shell: failed to kill process "); + printUInt(pid); + print(", error code: "); + printUInt(status); + print("\n"); + } else { + print("Process "); + printUInt(pid); + print(" terminated successfully.\n"); + } +} + // Implements the cmd write operation. static void cmd_write(const char* args) { char filename[MAX_CMD]; int i = 0; @@ -460,7 +1336,8 @@ static void cmd_write(const char* args) { uint32_t hFile; if (create_file(filename, &hFile) != 0) { print("write: could not create file\n"); return; } size_t written; - SysWriteFile(hFile, args + i + 1, strlen(args + i + 1), &written); + SYS_FILE_IO_REQUEST writeReq = SysMakeFileIoRequest(hFile, args + i + 1, strlen(args + i + 1), &written); + SysWriteFile(&writeReq); SysCloseHandle(hFile); print("File written successfully.\n"); } @@ -479,22 +1356,16 @@ static void cmd_dir(const char* path) { while (SysReadDir(hDir, &ke) == 0) { printDirTime(&ke.LastWriteTime); - // Corrected Attribute Check: 0x10 is the Directory bit in FAT. if (ke.Attr & 0x10) { print(" "); + print(ke.Name); } else { print(" "); - } - - print(ke.Name); - - // Optional: print file size for non-directories. - if (!(ke.Attr & 0x10)) { + print(ke.Name); print(" ("); - printUInt(ke.FileSize); + printUInt64(ke.FileSize); print(" bytes)"); } - print("\n"); } SysCloseHandle(hDir); @@ -505,7 +1376,9 @@ static void cmd_type(const char* filename) { uint32_t hFile; if (open_file(filename, &hFile) != 0) { print("type: cannot open file\n"); return; } char buffer[512]; size_t bytesRead; - while (SysReadFile(hFile, buffer, sizeof(buffer) - 1, &bytesRead) == 0 && bytesRead > 0) { + for (;;) { + SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(hFile, buffer, sizeof(buffer) - 1, &bytesRead); + if (SysReadFile(&readReq) != 0 || bytesRead == 0) break; buffer[bytesRead] = '\0'; print(buffer); } SysCloseHandle(hFile); print("\n"); @@ -533,6 +1406,72 @@ static bool extractProgramPath(const char* cmdline, char* out, size_t outSize) { return pos != 0; } +// Consumes queued Ctrl-C input without taking ordinary foreground stdin. +static bool shellConsumeQueuedCtrlC(void) { + for (int i = 0; i < 4; i++) { + INPUT_KEY_RECORD input; + if (SysPeekInputEvent(&input) != 0) return false; + + if ((input.Modifiers & INPUT_MOD_CTRL) && input.Key == INPUT_KEY_C) { + (void)SysReadInputEvent(&input); + return true; + } + + if (input.Key == INPUT_KEY_LEFT_CTRL || input.Key == INPUT_KEY_RIGHT_CTRL) { + (void)SysReadInputEvent(&input); + continue; + } + + return false; + } + + return false; +} + +// Waits for a foreground process while watching Ctrl-C without stealing ordinary stdin. +static uint32_t waitForegroundProcess(uint32_t hProc) { + uint32_t pid = 0; + bool havePid = SysGetProcessPid(hProc, &pid) == 0 && pid != 0; + bool sentBreak = false; + + uint32_t inputQueue = 0; + if (SysOpenInputQueue(0, &inputQueue) != 0) + return SysWait(hProc, SYS_WAIT_FLAG_NONE); + + uint32_t handles[2] = { hProc, inputQueue }; + uint32_t index = 0; + SYS_WAIT_MANY_REQUEST waitReq; + waitReq.Size = sizeof(waitReq); + waitReq.Flags = SYS_WAIT_FLAG_NONE; + waitReq.Handles = handles; + waitReq.Count = 2; + waitReq.SignaledIndexOut = &index; + waitReq.TimeoutMilliseconds = SYS_WAIT_TIMEOUT_INFINITE; + + for (;;) { + index = 0; + uint32_t status = SysWaitMany(&waitReq); + if (status != SYS_STATUS_SUCCESS) { + SysCloseHandle(inputQueue); + return status; + } + + if (index == 0) { + SysCloseHandle(inputQueue); + return status; + } + + uint32_t modifiers = SysGetKeyModifiers(); + if (((modifiers & INPUT_MOD_CTRL) && SysIsKeyDown(INPUT_KEY_C)) || shellConsumeQueuedCtrlC()) { + if (!sentBreak) { + print("^C\n"); + if (havePid) (void)SysKillProcess(pid, (int)STATUS_PROCESS_KILLED); + sentBreak = true; + } + } + } +} + // Run one command. static int runCommand(const char* cmd) { if (!cmd || *cmd == '\0') return 0; @@ -582,6 +1521,8 @@ static int runCommand(const char* cmd) { if (strcmp(cmd, "display") == 0 || strcmp(cmd, "gpu") == 0) { cmd_display(); return 0; } if (strcmp(cmd, "stats") == 0 || strcmp(cmd, "telemetry") == 0) { cmd_stats(); return 0; } if (strcmp(cmd, "ps") == 0) { cmd_ps(); return 0; } + if (strncmp(cmd, "threads ", 8) == 0) { cmd_threads(cmd + 8); return 0; } + if (strncmp(cmd, "kill ", 5) == 0) { cmd_kill(cmd + 5); return 0; } if (strcmp(cmd, "dir") == 0) { cmd_dir("."); return 0; } if (strncmp(cmd, "dir ", 4) == 0) { cmd_dir(cmd + 4); return 0; } if (strncmp(cmd, "cd ", 3) == 0) { if (set_cwd(cmd + 3) != 0) print("cd: failed\n"); return 0; } @@ -589,15 +1530,64 @@ static int runCommand(const char* cmd) { if (strncmp(cmd, "type ", 5) == 0) { cmd_type(cmd + 5); return 0; } if (strncmp(cmd, "write ", 6) == 0) { cmd_write(cmd + 6); return 0; } + // Make a local copy of the command line so we can mutate it safely if needed + char modifiedCmd[MAX_CMD]; + strncpy(modifiedCmd, cmd, sizeof(modifiedCmd) - 1); + modifiedCmd[sizeof(modifiedCmd) - 1] = '\0'; + + // Parse background indicator execution flag (&) + bool background = false; + size_t cmdLen = strlen(modifiedCmd); + if (cmdLen > 0 && modifiedCmd[cmdLen - 1] == '&') { + background = true; + modifiedCmd[cmdLen - 1] = '\0'; // Strip the ampersand + + // Clean up any trailing whitespace left before the ampersand + cmdLen = strlen(modifiedCmd); + while (cmdLen > 0 && (modifiedCmd[cmdLen - 1] == ' ' || modifiedCmd[cmdLen - 1] == '\t')) { + modifiedCmd[cmdLen - 1] = '\0'; + cmdLen--; + } + } + char program[MAX_CMD]; - if (!extractProgramPath(cmd, program, sizeof(program))) return 0; + if (!extractProgramPath(modifiedCmd, program, sizeof(program))) return 0; uint32_t hProc; - if (create_process(program, &hProc, cmd) == 0) { - SysWaitForProcess(hProc); - SysCloseHandle(hProc); - while (SysPollKeyboard()) (void)SysReadKeyboardChar(); - } else { print("shell: launch failed\n"); } + if (create_process(program, &hProc, modifiedCmd) == 0) { + if (background) { + // Background mode: Clean up the handle immediately and skip waiting. + print("[Launched in background, handle: "); + printUInt(hProc); + print("]\n"); + SysCloseHandle(hProc); + } else { + // Foreground mode: Wait for completion and flush standard keyboard stream. + uint32_t waitStatus = waitForegroundProcess(hProc); + if (waitStatus != SYS_STATUS_SUCCESS) { + print("shell: wait failed: "); + printUInt(waitStatus); + print("\n"); + } + print("Process exited with code: "); + int exitCode; + SysGetProcessExitCode(hProc, &exitCode); + printInt(exitCode); + const char* exitName = processExitStatusName((uint32_t)exitCode); + if (exitName) { + print(" (0x"); + printHex32((uint32_t)exitCode); + print(" "); + print(exitName); + print(")"); + } + print("\n"); + SysCloseHandle(hProc); + while (SysPollKeyboard()) (void)SysReadKey(); + } + } else { + print("shell: launch failed\n"); + } return 0; } @@ -611,9 +1601,11 @@ int main(void) { print("\n"); print("shell: ready\n"); while (true) { - print("> "); - readLine(lineBuf, MAX_CMD); - if (runCommand(trim(lineBuf))) break; + print(SHELL_PROMPT); + if (readLine(lineBuf, MAX_CMD)) continue; + char* cmd = trim(lineBuf); + shellHistoryAdd(cmd); + if (runCommand(cmd)) break; } print("shell: goodbye\n"); SysExitProcess(0); diff --git a/code/user/apps/shell_win32/shell_win32.c b/code/user/apps/shell_win32/shell_win32.c index 3f3c017..cdff8fa 100644 --- a/code/user/apps/shell_win32/shell_win32.c +++ b/code/user/apps/shell_win32/shell_win32.c @@ -147,6 +147,8 @@ static int runCommand(const char* cmd) { WaitForSingleObject(pi.hProcess, INFINITE); if (pi.hThread && pi.hThread != INVALID_HANDLE_VALUE) CloseHandle(pi.hThread); + if (pi.hProcess && pi.hProcess != INVALID_HANDLE_VALUE) + CloseHandle(pi.hProcess); FlushConsoleInputBuffer(hIn); } else { print("shell: launch failed\n"); diff --git a/code/user/apps/video/video.c b/code/user/apps/video/video.c index eb49abf..e91cba6 100644 --- a/code/user/apps/video/video.c +++ b/code/user/apps/video/video.c @@ -1,110 +1,231 @@ -// User application entry points for video. +// Native NTLite Bink video player. +// +// This application intentionally uses the native syscall ABI directly. Do not +// include here; Win32 compatibility belongs in the Win32 personality +// layer, not in native apps. -#include #include #include #include #include +#include #include "bink.h" -// Implements the command Line Second Token operation. -static const char* commandLineSecondToken(const char* cmdline, char* out, size_t outSize) { - if (!out || outSize == 0) return "video.bk2"; - strcpy(out, "video.bk2"); - if (!cmdline) return out; +#define VIDEO_DEFAULT_PATH "video.bk2" - while (*cmdline == ' ' || *cmdline == '\t') cmdline++; - if (*cmdline == '\0') return out; +static void print_span(const char* s, size_t len) { + if (!s || !len) return; + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(s, len) }; + SysWriteConsole(&req); +} - if (*cmdline == '"') { - cmdline++; - while (*cmdline && *cmdline != '"') cmdline++; - if (*cmdline == '"') cmdline++; +static void print(const char* s) { + if (!s) return; + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { + print("0"); + return; + } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i) { + char c = buf[--i]; + print_span(&c, 1); + } +} + +static void fail_status(const char* what, uint32_t status) { + print("video: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + for (;;) SysExitProcess(1); +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static SYS_STRING_VIEW command_line_view(void) { + SYS_PEB* peb = SysGetPeb(); + if (!peb || peb->Magic != SYS_PEB_MAGIC) + return SYS_SV_NULL; + return peb->CommandLine; +} + +static int sv_is_space(char c) { + return c == ' ' || c == '\t'; +} + +static SYS_STRING_VIEW skip_one_token(SYS_STRING_VIEW s) { + size_t i = 0; + while (i < s.Length && sv_is_space(s.Data[i])) + i++; + + if (i < s.Length && s.Data[i] == '"') { + i++; + while (i < s.Length && s.Data[i] != '"') + i++; + if (i < s.Length && s.Data[i] == '"') + i++; } else { - while (*cmdline && *cmdline != ' ' && *cmdline != '\t') cmdline++; + while (i < s.Length && !sv_is_space(s.Data[i])) + i++; } - while (*cmdline == ' ' || *cmdline == '\t') cmdline++; - if (*cmdline == '\0') return out; + while (i < s.Length && sv_is_space(s.Data[i])) + i++; - size_t pos = 0; - if (*cmdline == '"') { - cmdline++; - while (*cmdline && *cmdline != '"' && pos + 1 < outSize) - out[pos++] = *cmdline++; + return SysSvFromParts(s.Data + i, s.Length - i); +} + +static SYS_STRING_VIEW command_line_second_token(SYS_STRING_VIEW cmdline) { + SYS_STRING_VIEW rest = skip_one_token(cmdline); + if (!rest.Data || rest.Length == 0) + return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH); + + size_t begin = 0; + size_t end = 0; + if (rest.Data[0] == '"') { + begin = 1; + end = begin; + while (end < rest.Length && rest.Data[end] != '"') + end++; } else { - while (*cmdline && *cmdline != ' ' && *cmdline != '\t' && pos + 1 < outSize) - out[pos++] = *cmdline++; + begin = 0; + end = 0; + while (end < rest.Length && !sv_is_space(rest.Data[end])) + end++; } - out[pos] = '\0'; + + if (end <= begin) + return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH); + + return SysSvFromParts(rest.Data + begin, end - begin); +} + +static const char* path_to_cstr(SYS_STRING_VIEW path, char* out, size_t outSize) { + if (!out || outSize == 0) + return VIDEO_DEFAULT_PATH; + + if (!path.Data || path.Length == 0) + path = SYS_SV_LITERAL(VIDEO_DEFAULT_PATH); + + size_t n = path.Length; + if (n + 1 > outSize) + n = outSize - 1; + + memcpy(out, path.Data, n); + out[n] = '\0'; return out; } -// Runs the requested operation. +static uint32_t open_file_view(SYS_STRING_VIEW path, uint32_t* handleOut) { + SYS_PATH_HANDLE_REQUEST req = { path, handleOut }; + return SysOpenFile(&req); +} + int main(void) { - const char* cmdline = GetCommandLineA(); - char videoPath[260]; - const char* path = commandLineSecondToken(cmdline, videoPath, sizeof(videoPath)); + SYS_STRING_VIEW cmdline = command_line_view(); + SYS_STRING_VIEW pathView = command_line_second_token(cmdline); - void* h = GetStdHandle(STD_OUTPUT_HANDLE); - WriteConsoleA(h, "video: starting\n", 16, 0, 0); - - WriteConsoleA(h, cmdline, (unsigned int)strlen(cmdline), 0, 0); - WriteConsoleA(h, "\n", 1, 0, 0); - - void* f = CreateFileA(path, 0, 0, 0, 3, 0, 0); - if (!f || f == (void*)(unsigned long long)0xFFFFFFFF) { // fuckass - WriteConsoleA(h, "video: failed to open input file\n", 33, 0, 0); - ExitProcess(1); + print("video: starting\n"); + if (cmdline.Data && cmdline.Length) { + print_span(cmdline.Data, cmdline.Length); + print("\n"); } - unsigned long s = GetFileSize(f, 0); - if (!s || s == 0xFFFFFFFF) { - WriteConsoleA(h, "video: GetFileSize failed!\n", 27, 0, 0); - ExitProcess(1); - } - void* buf = HeapAlloc(0, 0, s); - if (!buf) { - WriteConsoleA(h, "video: HeapAlloc failed\n", 24, 0, 0); - ExitProcess(1); + uint32_t file = 0; + uint32_t status = open_file_view(pathView, &file); + if (status != SYS_STATUS_SUCCESS) { + char pathBuf[260]; + print("video: failed to open input file: "); + print(path_to_cstr(pathView, pathBuf, sizeof(pathBuf))); + print("\n"); + fail_status("SysOpenFile", status); } - int r = ReadFile(f, buf, s, 0, 0); - if (!r) { - WriteConsoleA(h, "video: ReadFile failed\n", 23, 0, 0); - ExitProcess(1); + uint64_t fileSize = SysGetFileSize(file); + if (fileSize == 0) { + SysCloseHandle(file); + print("video: input file is empty\n"); + exit_now(1); } - WriteConsoleA(h, "video: input file loaded\n", 25, 0, 0); - - CloseHandle(h); - HBINK b = BinkOpen(buf, BINKFROMMEMORY); + uint32_t mapping = 0; + status = SysCreateFileMapping(file, SYS_VM_READ, SYS_MAP_FLAG_NONE, fileSize, &mapping); + if (status != SYS_STATUS_SUCCESS || !mapping) { + SysCloseHandle(file); + fail_status("SysCreateFileMapping", status); + } + + void* buf = 0; + status = SysMapFile(mapping, 0, (size_t)fileSize, + SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &buf); + if (status != SYS_STATUS_SUCCESS || !buf) { + SysCloseHandle(mapping); + SysCloseHandle(file); + fail_status("SysMapFile", status); + } + + // The mapped VAD owns the file reference now; keep only the mapped pointer + // alive while Bink is open. This avoids a second whole-file heap allocation + // and lets the VM fault file pages in on demand. + SysCloseHandle(mapping); + SysCloseHandle(file); + + print("video: input file mapped\n"); + + HBINK b = BinkOpen((const char*)buf, BINKFROMMEMORY); if (!b) { - WriteConsoleA(h, "video: BinkOpen failed: ", 24, 0, 0); + print("video: BinkOpen failed: "); const char* err = BinkGetError(); - WriteConsoleA(h, err, (unsigned int)strlen(err), 0, 0); - WriteConsoleA(h, "\n", 1, 0, 0); - ExitProcess(1); + print(err ? err : "unknown"); + print("\n"); + SysFreeVm(buf); + exit_now(1); } - uint32_t width = b->Width; + uint32_t width = b->Width; uint32_t height = b->Height; - - uint32_t* fb = HeapAlloc(0, 0, width * height * sizeof(uint32_t)); - if (!fb) { - WriteConsoleA(h, "video: framebuffer alloc failed\n", 32, 0, 0); - ExitProcess(1); + if (!width || !height) { + print("video: invalid video dimensions\n"); + BinkClose(b); + SysFreeVm(buf); + exit_now(1); } - // Prepare the Win32 BITMAPINFO structure. - BITMAPINFO bmi; - memset(&bmi, 0, sizeof(bmi)); - bmi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER); - bmi.bmiHeader.biWidth = width; - bmi.bmiHeader.biHeight = -(LONG)height; // Negative means top-down drawing! - bmi.bmiHeader.biPlanes = 1; - bmi.bmiHeader.biBitCount = 32; - bmi.bmiHeader.biCompression = BI_RGB; + uint32_t surface = 0; + status = SysCreateSurface(width, height, SYS_SURFACE_FORMAT_XRGB8888, + SYS_SURFACE_FLAG_NONE, &surface); + if (status != SYS_STATUS_SUCCESS || !surface) { + BinkClose(b); + SysFreeVm(buf); + fail_status("SysCreateSurface", status); + } + + void* surfaceBase = 0; + uint32_t pitchBytes = 0; + status = SysMapSurface(surface, &surfaceBase, &pitchBytes); + if (status != SYS_STATUS_SUCCESS || !surfaceBase || pitchBytes < width * sizeof(uint32_t)) { + SysCloseHandle(surface); + BinkClose(b); + SysFreeVm(buf); + fail_status("SysMapSurface", status); + } + + print("video: native surface ready\n"); + + DISPLAY_RECT fullFrame = { 0, 0, (int32_t)width, (int32_t)height }; while (1) { if (BinkWait(b)) { @@ -120,33 +241,26 @@ int main(void) { continue; } - BinkCopyToBuffer(b, fb, width * 4, height, 0, 0, - BINKSURFACE32); + BinkCopyToBuffer(b, surfaceBase, pitchBytes, height, 0, 0, BINKSURFACE32); - // Welcome to the GDI pipeline! - PAINTSTRUCT ps; - // Using NULL/0 as the HWND since we don't have a real window handle yet. - HDC hdc = BeginPaint((HWND)0, &ps); - - StretchDIBits( - hdc, - 0, 0, width, height, // Destination rect - 0, 0, width, height, // Source rect - fb, - &bmi, - DIB_RGB_COLORS, - SRCCOPY - ); - - EndPaint((HWND)0, &ps); + status = SysPresentSurface(surface, &fullFrame, 1); + if (status != SYS_STATUS_SUCCESS) { + SysCloseHandle(surface); + BinkClose(b); + SysFreeVm(buf); + fail_status("SysPresentSurface", status); + } if (b->FrameNum == b->Frames) break; BinkNextFrame(b); } - - WriteConsoleA(h, "video: goodbye\n", 15, 0, 0); - ExitProcess(0); + SysCloseHandle(surface); + BinkClose(b); + SysFreeVm(buf); + + print("video: goodbye\n"); + exit_now(0); } diff --git a/code/user/personalities/posix/posix.h b/code/user/personalities/posix/posix.h index c3cfdab..f1d772c 100644 --- a/code/user/personalities/posix/posix.h +++ b/code/user/personalities/posix/posix.h @@ -72,12 +72,27 @@ extern int errno; // defined in posix_fd.c; single global in v1 #define SEEK_CUR 1 #define SEEK_END 2 +// mmap() protection and mapping flags. +#define PROT_NONE 0x0 +#define PROT_READ 0x1 +#define PROT_WRITE 0x2 +#define PROT_EXEC 0x4 +#define MAP_SHARED 0x01 +#define MAP_PRIVATE 0x02 +#define MAP_FAILED ((void*)(intptr_t)-1) +#define MS_SYNC 0x04 +#define MS_ASYNC 0x01 +#define MS_INVALIDATE 0x02 + // Scalar types. typedef int32_t pid_t; typedef int32_t off_t; typedef uint32_t mode_t; typedef int64_t ssize_t; +__declspec(dllexport) void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset); +__declspec(dllexport) int munmap(void* addr, size_t length); +__declspec(dllexport) int msync(void* addr, size_t length, int flags); // Sockets. typedef uint16_t sa_family_t; @@ -188,7 +203,7 @@ __declspec(dllexport) int getaddrinfo(const char* node, const char* service, con __declspec(dllexport) void freeaddrinfo(struct addrinfo* res); // pthread types. -typedef struct { uint32_t Tid; } pthread_t; +typedef struct { uint32_t Handle; uint32_t Tid; } pthread_t; typedef struct { uint32_t Handle; } pthread_mutex_t; typedef void* pthread_attr_t; // unused in v1 typedef void* pthread_mutexattr_t; diff --git a/code/user/personalities/posix/posix_io.c b/code/user/personalities/posix/posix_io.c index 6579da5..38b1323 100644 --- a/code/user/personalities/posix/posix_io.c +++ b/code/user/personalities/posix/posix_io.c @@ -81,7 +81,8 @@ __declspec(dllexport) ssize_t read(int fd, void* buf, size_t count) { return (ssize_t)count; case FD_FILE: { size_t got = 0; - uint32_t s = SysReadFile(e->KHandle, buf, count, &got); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(e->KHandle, buf, count, &got); + uint32_t s = SysReadFile(&req); if (s != 0 && got == 0) { errno = PosixStatusToErrno(s); return -1; } return (ssize_t)got; } @@ -115,7 +116,8 @@ __declspec(dllexport) ssize_t write(int fd, const void* buf, size_t count) { return (ssize_t)count; case FD_FILE: { size_t written = 0; - uint32_t s = SysWriteFile(e->KHandle, buf, count, &written); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(e->KHandle, buf, count, &written); + uint32_t s = SysWriteFile(&req); if (s != 0 && written == 0) { errno = PosixStatusToErrno(s); return -1; } return (ssize_t)written; } diff --git a/code/user/personalities/posix/posix_mman.c b/code/user/personalities/posix/posix_mman.c new file mode 100644 index 0000000..562371a --- /dev/null +++ b/code/user/personalities/posix/posix_mman.c @@ -0,0 +1,85 @@ +// POSIX compatibility support for mmap-style file mappings. + +#include +#include +#include +#include "posix.h" +#include "posix_fd.h" + +static uint32_t PosixProtToSysMapAccess(int prot, int flags) { + uint32_t access = 0; + if (prot & PROT_EXEC) access |= SYS_MAP_ACCESS_EXECUTE; + if (prot & PROT_WRITE) { + access |= (flags & MAP_SHARED) ? SYS_MAP_ACCESS_WRITE : SYS_MAP_ACCESS_COPY; + } else if (flags & MAP_PRIVATE) { + access |= SYS_MAP_ACCESS_COPY; + } + if (prot & PROT_READ) access |= SYS_MAP_ACCESS_READ; + if (!access) access = SYS_MAP_ACCESS_READ; + return access; +} + +__declspec(dllexport) void* mmap(void* addr, size_t length, int prot, + int flags, int fd, off_t offset) { + if (addr || length == 0 || offset < 0 || ((uint64_t)offset & 0xFFFu)) { + errno = EINVAL; + return MAP_FAILED; + } + + if (!(flags & MAP_PRIVATE) && !(flags & MAP_SHARED)) { + errno = EINVAL; + return MAP_FAILED; + } + + uint32_t pid = SysGetCurrentPid(); + FD_ENTRY* e = FdGet(pid, fd); + if (!e) return MAP_FAILED; + if (e->Type != FD_FILE) { + errno = EBADF; + return MAP_FAILED; + } + + uint64_t maxSize = (uint64_t)offset + (uint64_t)length; + if (maxSize < (uint64_t)offset) { + errno = EINVAL; + return MAP_FAILED; + } + + uint32_t protect = (prot & PROT_WRITE) ? SYS_VM_READWRITE : SYS_VM_READ; + if (prot & PROT_EXEC) + protect = (prot & PROT_WRITE) ? SYS_VM_EXECUTE_READWRITE : SYS_VM_EXECUTE_READ; + + uint32_t mapping = 0; + uint32_t s = SysCreateFileMapping(e->KHandle, protect, SYS_MAP_FLAG_NONE, + maxSize, &mapping); + if (s != SYS_STATUS_SUCCESS) { + errno = PosixStatusToErrno(s); + return MAP_FAILED; + } + + void* base = (void*)0; + s = SysMapFile(mapping, (uint64_t)offset, length, + PosixProtToSysMapAccess(prot, flags), + SYS_MAP_FLAG_NONE, &base); + SysCloseHandle(mapping); + if (s != SYS_STATUS_SUCCESS) { + errno = PosixStatusToErrno(s); + return MAP_FAILED; + } + return base; +} + +__declspec(dllexport) int munmap(void* addr, size_t length) { + (void)length; + if (!addr) { errno = EINVAL; return -1; } + uint32_t s = SysFreeVm(addr); + if (s != SYS_STATUS_SUCCESS) { errno = PosixStatusToErrno(s); return -1; } + return 0; +} + +__declspec(dllexport) int msync(void* addr, size_t length, int flags) { + (void)addr; + (void)length; + (void)flags; + return 0; +} diff --git a/code/user/personalities/posix/posix_proc.c b/code/user/personalities/posix/posix_proc.c index 5c377fb..dec224a 100644 --- a/code/user/personalities/posix/posix_proc.c +++ b/code/user/personalities/posix/posix_proc.c @@ -126,7 +126,7 @@ __declspec(dllexport) pid_t waitpid(pid_t pid, int* status, int options) { uint32_t handle = ProcTableFind(pid); if (!handle) { errno = ESRCH; return -1; } - SysWaitForProcess(handle); + SysWait(handle, SYS_WAIT_FLAG_NONE); int exitCode = 0; SysGetProcessExitCode(handle, &exitCode); diff --git a/code/user/personalities/posix/posix_thread.c b/code/user/personalities/posix/posix_thread.c index 998e3a2..6334509 100644 --- a/code/user/personalities/posix/posix_thread.c +++ b/code/user/personalities/posix/posix_thread.c @@ -84,28 +84,41 @@ __declspec(dllexport) int pthread_create(pthread_t* t, const pthread_attr_t* att tr->Pid = pid; uint32_t tid = 0; - uint32_t s = SysCreateThread((uint64_t)PthreadTrampoline, - (uint64_t)tr, 1, &tid); + uint32_t handle = 0; + SYS_CREATE_THREAD_REQUEST req = { + (uintptr_t)PthreadTrampoline, + (uintptr_t)tr, + 1, + 0, + &handle, + &tid + }; + uint32_t s = SysCreateThread(&req); if (s != 0) { free(tr); return EAGAIN; } // The trampoline allocates the retval slot itself after it knows its real // TID, avoiding the creator/new-thread race. - if (t) t->Tid = tid; + if (t) { + t->Handle = handle; + t->Tid = tid; + } else { + SysCloseHandle(handle); + } return 0; } // Implements the pthread join operation. __declspec(dllexport) int pthread_join(pthread_t t, void** retval) { uint32_t pid = SysGetCurrentPid(); - uint32_t s = SysWaitThread(t.Tid); - // STATUS_NOT_FOUND (0x22) means already dead - treat as success. - if (s != 0 && s != 0x0022) return ESRCH; + uint32_t s = SysWait(t.Handle, SYS_WAIT_FLAG_NONE); + if (s != 0) return ESRCH; PTHREAD_SLOT* slot = PthreadSlotFind(pid, t.Tid); if (retval) *retval = slot ? slot->RetVal : (void*)0; if (slot) slot->Used = false; + SysCloseHandle(t.Handle); return 0; } @@ -138,7 +151,7 @@ __declspec(dllexport) int pthread_mutex_destroy(pthread_mutex_t* m) { // Implements the pthread mutex lock operation. __declspec(dllexport) int pthread_mutex_lock(pthread_mutex_t* m) { - uint32_t s = SysAcquireMutex(m->Handle); + uint32_t s = SysWait(m->Handle, SYS_WAIT_FLAG_NONE); return s == 0 ? 0 : EINVAL; } diff --git a/code/user/personalities/win32/gdi32.c b/code/user/personalities/win32/gdi32.c index 504fe96..fc73f7e 100644 --- a/code/user/personalities/win32/gdi32.c +++ b/code/user/personalities/win32/gdi32.c @@ -63,101 +63,16 @@ static int* srcX_map = NULL; static int currentDestCapacity = 0; static int currentXMapCapacity = 0; -__declspec(dllexport) int StretchDIBits( - HDC hdc, - int xDest, int yDest, int DestWidth, int DestHeight, +__declspec(dllexport) int StretchDIBits(HDC hdc, int xDest, int yDest, int DestWidth, int DestHeight, int xSrc, int ySrc, int SrcWidth, int SrcHeight, const void *lpBits, const BITMAPINFO *lpbmi, UINT iUsage, DWORD rop) { - (void)hdc; (void)lpbmi; (void)iUsage; (void)rop; + (void)hdc; (void)xDest; (void)yDest; (void)DestWidth; (void)DestHeight; + (void)xSrc; (void)ySrc; (void)SrcWidth; (void)SrcHeight; + (void)lpBits; (void)lpbmi; (void)iUsage; (void)rop; - // Safety check to prevent division by zero. - if (DestWidth <= 0 || DestHeight <= 0 || SrcWidth <= 0 || SrcHeight <= 0) { - return 0; - } - - // Handle trivial case (1:1 copy) to bypass allocation and scaling overhead. - if (DestWidth == SrcWidth && DestHeight == SrcHeight && xSrc == 0 && ySrc == 0) { - SysBlitScreen(xDest, yDest, DestWidth, DestHeight, (const uint32_t*)lpBits); - return DestHeight; - } - - const uint32_t* srcPixels = (const uint32_t*)lpBits; - - // Only reallocate if the new size is LARGER than what we already have. - int requiredPixels = DestWidth * DestHeight; - if (requiredPixels > currentDestCapacity) { - free(destBuffer); - destBuffer = (uint32_t*)malloc(requiredPixels * sizeof(uint32_t)); - currentDestCapacity = requiredPixels; - } - - if (DestWidth > currentXMapCapacity) { - free(srcX_map); - srcX_map = (int*)malloc(DestWidth * sizeof(int)); - currentXMapCapacity = DestWidth; - } - - for (int dx = 0; dx < DestWidth; ++dx) { - // Nearest neighbor coordinate mapping. - srcX_map[dx] = xSrc + (dx * SrcWidth) / DestWidth; - - // Clamp to safe bounds. - if (srcX_map[dx] < 0) srcX_map[dx] = 0; - if (srcX_map[dx] >= SrcWidth) srcX_map[dx] = SrcWidth - 1; - } - - // Process the image row by row. - for (int dy = 0; dy < DestHeight; ++dy) { - // Map destination Y to source Y. - int sy = ySrc + (dy * SrcHeight) / DestHeight; - if (sy < 0) sy = 0; - if (sy >= SrcHeight) sy = SrcHeight - 1; - - // Assumes stride == SrcWidth (tightly packed 32bpp). - const uint32_t* srcRow = srcPixels + (sy * SrcWidth); - uint32_t* dstRow = destBuffer + (dy * DestWidth); - - int dx = 0; - -#if defined(__SSE2__) || defined(_M_X64) - // Process 4 pixels (16 bytes) per loop iteration using SSE2. - for (; dx <= DestWidth - 4; dx += 4) { - // Gather 4 source pixels using our precomputed X map. - // Note: _mm_set_epi32 takes arguments in reverse order (3, 2, 1, 0). - __m128i pixels = _mm_set_epi32( - srcRow[srcX_map[dx + 3]], - srcRow[srcX_map[dx + 2]], - srcRow[srcX_map[dx + 1]], - srcRow[srcX_map[dx + 0]] - ); - // Issue a single 128-bit unaligned store. - _mm_storeu_si128((__m128i*)&dstRow[dx], pixels); - } -#elif defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM64) - // Process 4 pixels per loop iteration using ARM NEON. - for (; dx <= DestWidth - 4; dx += 4) { - // Initialize a vector and populate lanes (avoids stack-memory hop). - uint32x4_t pixels = vdupq_n_u32(0); - pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 0]], pixels, 0); - pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 1]], pixels, 1); - pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 2]], pixels, 2); - pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 3]], pixels, 3); - - // Issue a single 128-bit store. - vst1q_u32(&dstRow[dx], pixels); - } -#endif - - // Epilogue: Handle remaining pixels if DestWidth isn't a multiple of 4. - for (; dx < DestWidth; ++dx) { - dstRow[dx] = srcRow[srcX_map[dx]]; - } - } - - // Direct mapping to kernel screen blit using the stretched buffer. - SysBlitScreen(xDest, yDest, DestWidth, DestHeight, destBuffer); - - return DestHeight; + // There is no implicit display blit ABI. Native graphics clients own a + // surface and present it explicitly. + return 0; } diff --git a/code/user/personalities/win32/kernel32.c b/code/user/personalities/win32/kernel32.c index 649472b..af89d31 100644 --- a/code/user/personalities/win32/kernel32.c +++ b/code/user/personalities/win32/kernel32.c @@ -27,6 +27,8 @@ #include #include +extern size_t _malloc_usable_size(const void* ptr); + // Types. typedef void* HANDLE; typedef unsigned long DWORD; @@ -58,6 +60,7 @@ typedef struct _CONSOLE_SCREEN_BUFFER_INFO { #define MEM_RELEASE 0x00008000UL #define MEM_FREE 0x00010000UL #define MEM_PRIVATE 0x00020000UL +#define MEM_MAPPED 0x00040000UL #define PAGE_NOACCESS 0x01UL #define PAGE_READONLY 0x02UL #define PAGE_READWRITE 0x04UL @@ -65,6 +68,13 @@ typedef struct _CONSOLE_SCREEN_BUFFER_INFO { #define PAGE_EXECUTE 0x10UL #define PAGE_EXECUTE_READ 0x20UL #define PAGE_EXECUTE_READWRITE 0x40UL +#define SEC_COMMIT 0x08000000UL + +#define FILE_MAP_COPY 0x00000001UL +#define FILE_MAP_WRITE 0x00000002UL +#define FILE_MAP_READ 0x00000004UL +#define FILE_MAP_EXECUTE 0x00000020UL +#define FILE_MAP_ALL_ACCESS 0x000F001FUL typedef struct _MEMORY_BASIC_INFORMATION { void* BaseAddress; @@ -90,6 +100,26 @@ static uint32_t WinProtectToSysVm(DWORD protect) { } } +static uint32_t WinFileMappingProtectToSysVm(DWORD protect) { + // Strip section attributes; the low byte carries PAGE_* protection. + DWORD pageProtect = protect & 0xFFu; + if (pageProtect == PAGE_WRITECOPY) + return SYS_VM_READ; // MapViewOfFile(FILE_MAP_COPY) asks for the private writable view. + return WinProtectToSysVm(pageProtect ? pageProtect : PAGE_READONLY); +} + +static uint32_t WinFileMapAccessToSys(DWORD desiredAccess) { + uint32_t access = 0; + if (desiredAccess & FILE_MAP_EXECUTE) access |= SYS_MAP_ACCESS_EXECUTE; + if (desiredAccess & FILE_MAP_COPY) access |= SYS_MAP_ACCESS_COPY; + if (desiredAccess & FILE_MAP_WRITE) access |= SYS_MAP_ACCESS_WRITE; + if (desiredAccess & FILE_MAP_READ) access |= SYS_MAP_ACCESS_READ; + if ((desiredAccess & FILE_MAP_ALL_ACCESS) == FILE_MAP_ALL_ACCESS) + access |= SYS_MAP_ACCESS_READ | SYS_MAP_ACCESS_COPY; + if (!access) access = SYS_MAP_ACCESS_READ; + return access; +} + // Implements the protect Vm To Win operation. static DWORD SysProtectVmToWin(uint32_t protect) { switch (protect) { @@ -153,24 +183,9 @@ typedef struct _WIN32_FIND_DATAA { static inline HANDLE KH_TO_HANDLE(uint32_t kh) { return (HANDLE)(uintptr_t)kh; } static inline uint32_t HANDLE_TO_KH(HANDLE h) { return (uint32_t)(uintptr_t)h; } -// Thread IDs are not kernel handles. Keep Win32 thread pseudo-handles out of -// the low numeric handle namespace so CloseHandle/WaitForSingleObject cannot -// accidentally operate on an unrelated file/process handle with the same value. -#define WIN32_THREAD_HANDLE_TAG ((uintptr_t)1ULL << 63) -#define WIN32_MUTEX_SENTINEL ((uintptr_t)0x4D555458UL) // 'MUTX' -#define WIN32_SEMA_SENTINEL ((uintptr_t)0x53454D41UL) // 'SEMA' -static inline HANDLE TID_TO_THREAD_HANDLE(uint32_t tid) { - return (HANDLE)(WIN32_THREAD_HANDLE_TAG | (uintptr_t)tid); -} -static inline BOOL IS_THREAD_HANDLE(HANDLE h) { - return h && h != INVALID_HANDLE_VALUE && (((uintptr_t)h & WIN32_THREAD_HANDLE_TAG) != 0); -} -static inline uint32_t THREAD_HANDLE_TO_TID(HANDLE h) { - return (uint32_t)((uintptr_t)h & 0xFFFFFFFFu); -} -static inline BOOL IS_NOOP_SYNC_HANDLE(HANDLE h) { +static inline BOOL IS_VALID_KERNEL_HANDLE(HANDLE h) { uintptr_t v = (uintptr_t)h; - return v == WIN32_MUTEX_SENTINEL || v == WIN32_SEMA_SENTINEL; + return v != 0 && v != (uintptr_t)INVALID_HANDLE_VALUE; } // Path translator. @@ -268,11 +283,14 @@ static void K32DebugPrintA(const char* s) { } #define WIN32_TLS_SLOTS 64 +#define FLS_MAX_SLOTS 64 +#define FLS_OUT_OF_INDEXES 0xFFFFFFFFUL typedef struct WIN32_THREAD_ENV { DWORD LastError; DWORD Reserved0; void* TlsSlots[WIN32_TLS_SLOTS]; + void* FlsSlots[FLS_MAX_SLOTS]; } WIN32_THREAD_ENV; static WIN32_THREAD_ENV gFallbackThreadEnv; @@ -418,11 +436,27 @@ __declspec(dllexport) uint64_t GetTickCount64(void) { } // Memory (heap argument is ignored - single user-mode heap). +#define HEAP_ZERO_MEMORY 0x00000008UL +#define HEAP_REALLOC_IN_PLACE_ONLY 0x00000010UL + __declspec(dllexport) void* HeapAlloc(void* heap, DWORD flags, size_t size) { - (void)heap; (void)flags; + (void)heap; if (!size) return NULL; + void* ptr = malloc(size); - if (!ptr) gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY + if (!ptr) { + gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY + return NULL; + } + + // Bink and the Microsoft CRT routinely allocate internal state with + // HEAP_ZERO_MEMORY. Ignoring this flag only works accidentally while the + // heap keeps receiving fresh demand-zero pages; under low memory / swap the + // allocator reuses dirty freed blocks and stale pitch/pointer fields can + // corrupt the decoder. + if (flags & HEAP_ZERO_MEMORY) + memset(ptr, 0, size); + return ptr; } @@ -436,9 +470,28 @@ __declspec(dllexport) BOOL HeapFree(void* heap, DWORD flags, void* ptr) { __declspec(dllexport) void* HeapReAlloc(void* heap, DWORD flags, void* ptr, size_t size) { - (void)heap; (void)flags; + (void)heap; + + if (flags & HEAP_REALLOC_IN_PLACE_ONLY) { + // The user heap does not expose an in-place grow primitive. Preserve + // Windows semantics by failing instead of silently moving the block. + size_t oldSize = _malloc_usable_size(ptr); + if (oldSize != (size_t)-1 && oldSize >= size) + return ptr; + gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY + return NULL; + } + + size_t oldSize = ptr ? _malloc_usable_size(ptr) : 0; void* newptr = realloc(ptr, size); - if (!newptr && size) gLastError = 8; + if (!newptr && size) { + gLastError = 8; + return NULL; + } + + if (newptr && (flags & HEAP_ZERO_MEMORY) && oldSize != (size_t)-1 && size > oldSize) + memset((unsigned char*)newptr + oldSize, 0, size - oldSize); + return newptr; } @@ -508,10 +561,88 @@ __declspec(dllexport) size_t VirtualQuery(const void* lpAddress, lpBuffer->RegionSize = info.PageCount * 4096u; lpBuffer->State = MEM_COMMIT; lpBuffer->Protect = protect; - lpBuffer->Type = MEM_PRIVATE; + lpBuffer->Type = (info.Type == 7u) ? MEM_MAPPED : MEM_PRIVATE; // 7 == MmVadMappedFile return sizeof(MEMORY_BASIC_INFORMATION); } +__declspec(dllexport) HANDLE CreateFileMappingA(HANDLE hFile, + void* lpFileMappingAttributes, + DWORD flProtect, + DWORD dwMaximumSizeHigh, + DWORD dwMaximumSizeLow, + const char* lpName) { + (void)lpFileMappingAttributes; + (void)lpName; + if (!hFile || hFile == INVALID_HANDLE_VALUE) { + gLastError = 6; // ERROR_INVALID_HANDLE + return NULL; + } + + uint64_t maximumSize = ((uint64_t)dwMaximumSizeHigh << 32) | (uint64_t)dwMaximumSizeLow; + uint32_t kh = 0; + uint32_t s = SysCreateFileMapping(HANDLE_TO_KH(hFile), + WinFileMappingProtectToSysVm(flProtect), + SYS_MAP_FLAG_NONE, + maximumSize, + &kh); + if (s != SYS_STATUS_SUCCESS) { + gLastError = (s == SYS_STATUS_NOT_SUPPORTED) ? 120 : ((s == SYS_STATUS_NO_MEMORY) ? 8 : 87); + return NULL; + } + return KH_TO_HANDLE(kh); +} + +__declspec(dllexport) HANDLE CreateFileMappingW(HANDLE hFile, + void* lpFileMappingAttributes, + DWORD flProtect, + DWORD dwMaximumSizeHigh, + DWORD dwMaximumSizeLow, + const unsigned short* lpName) { + (void)lpName; + return CreateFileMappingA(hFile, lpFileMappingAttributes, flProtect, + dwMaximumSizeHigh, dwMaximumSizeLow, NULL); +} + +__declspec(dllexport) void* MapViewOfFile(HANDLE hFileMappingObject, + DWORD dwDesiredAccess, + DWORD dwFileOffsetHigh, + DWORD dwFileOffsetLow, + size_t dwNumberOfBytesToMap) { + if (!hFileMappingObject || hFileMappingObject == INVALID_HANDLE_VALUE) { + gLastError = 6; + return NULL; + } + + uint64_t offset = ((uint64_t)dwFileOffsetHigh << 32) | (uint64_t)dwFileOffsetLow; + void* base = NULL; + uint32_t s = SysMapFile(HANDLE_TO_KH(hFileMappingObject), + offset, + dwNumberOfBytesToMap, + WinFileMapAccessToSys(dwDesiredAccess), + SYS_MAP_FLAG_NONE, + &base); + if (s != SYS_STATUS_SUCCESS) { + gLastError = (s == SYS_STATUS_NOT_SUPPORTED) ? 120 : ((s == SYS_STATUS_NO_MEMORY) ? 8 : 87); + return NULL; + } + return base; +} + +__declspec(dllexport) BOOL UnmapViewOfFile(const void* lpBaseAddress) { + if (!lpBaseAddress) { gLastError = 87; return FALSE; } + uint32_t s = SysFreeVm((void*)lpBaseAddress); + if (s != SYS_STATUS_SUCCESS) { gLastError = 87; return FALSE; } + return TRUE; +} + +__declspec(dllexport) BOOL FlushViewOfFile(const void* lpBaseAddress, size_t dwNumberOfBytesToFlush) { + (void)dwNumberOfBytesToFlush; + if (!lpBaseAddress) { gLastError = 87; return FALSE; } + // Native flush is file-handle based for now; unmap/close also harvests dirty + // PTEs, so FlushViewOfFile remains a successful cache-coherency hint. + return TRUE; +} + // --- Win32 API ---. // GetAsyncKeyState implementation over the canonical input key namespace. @@ -720,7 +851,7 @@ __declspec(dllexport) HANDLE CreateFileA(const char* name, // GetFileSize - fills optional high-DWORD pointer, returns low DWORD. // Returns 0xFFFFFFFF on error, matching Win32 behaviour. __declspec(dllexport) DWORD GetFileSize(HANDLE hFile, DWORD* fileSizeHigh) { - if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) { gLastError = 6; return 0xFFFFFFFFUL; } + if (!hFile || hFile == INVALID_HANDLE_VALUE) { gLastError = 6; return 0xFFFFFFFFUL; } uint64_t sz = SysGetFileSize(HANDLE_TO_KH(hFile)); if (fileSizeHigh) *fileSizeHigh = (DWORD)(sz >> 32); return (DWORD)(sz & 0xFFFFFFFFULL); @@ -731,25 +862,27 @@ __declspec(dllexport) BOOL ReadFile(HANDLE hFile, void* buf, DWORD toRead, DWORD* bytesRead, void* overlapped) { (void)overlapped; if (bytesRead) *bytesRead = 0; - if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile) || !buf) { + if (!hFile || hFile == INVALID_HANDLE_VALUE || !buf) { gLastError = 6; return FALSE; } size_t got = 0; - uint32_t s = SysReadFile(HANDLE_TO_KH(hFile), buf, (size_t)toRead, &got); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(HANDLE_TO_KH(hFile), buf, (size_t)toRead, &got); + uint32_t s = SysReadFile(&req); if (bytesRead) *bytesRead = (DWORD)got; - // Harden callers that accidentally look past *bytesRead. EOF/short reads - // are legal for files, but consuming stale stack/heap bytes after them makes - // the failure mode look like random game-data corruption. - if (got < (size_t)toRead) - memset((unsigned char*)buf + got, 0, (size_t)toRead - got); - if (s != 0) { gLastError = 1; return FALSE; } + + // Harden callers that accidentally look past *bytesRead. EOF/short reads + // are legal for files, but do not touch the caller's remaining buffer after + // a failed read because that can fault in megabytes under low memory. + if (got < (size_t)toRead) + memset((unsigned char*)buf + got, 0, (size_t)toRead - got); + return TRUE; } @@ -758,9 +891,10 @@ __declspec(dllexport) BOOL ReadFile(HANDLE hFile, void* buf, DWORD toRead, __declspec(dllexport) BOOL WriteFile(HANDLE hFile, const void* buf, DWORD toWrite, DWORD* bytesWritten, void* overlapped) { (void)overlapped; - if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile) || !buf) { gLastError = 6; return FALSE; } + if (!hFile || hFile == INVALID_HANDLE_VALUE || !buf) { gLastError = 6; return FALSE; } size_t written = 0; - uint32_t s = SysWriteFile(HANDLE_TO_KH(hFile), buf, (size_t)toWrite, &written); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(HANDLE_TO_KH(hFile), buf, (size_t)toWrite, &written); + uint32_t s = SysWriteFile(&req); if (bytesWritten) *bytesWritten = (DWORD)written; if (s != 0) { gLastError = 29; return FALSE; } // ERROR_WRITE_FAULT return TRUE; @@ -770,7 +904,7 @@ __declspec(dllexport) BOOL WriteFile(HANDLE hFile, const void* buf, DWORD toWrit // Returns new position low DWORD, or 0xFFFFFFFF on error. __declspec(dllexport) DWORD SetFilePointer(HANDLE hFile, long dist, long* distHigh, DWORD method) { - if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) { + if (!hFile || hFile == INVALID_HANDLE_VALUE) { gLastError = 6; return 0xFFFFFFFFUL; } @@ -813,7 +947,7 @@ __declspec(dllexport) DWORD SetFilePointer(HANDLE hFile, long dist, // SetEndOfFile - truncate or extend the file to the current seek position. __declspec(dllexport) BOOL SetEndOfFile(HANDLE hFile) { - if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) { gLastError = 6; return FALSE; } + if (!hFile || hFile == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; } uint32_t kh = HANDLE_TO_KH(hFile); uint64_t pos = SysGetFilePos(kh); if (pos == (uint64_t)-1) { gLastError = 6; return FALSE; } @@ -834,14 +968,10 @@ __declspec(dllexport) BOOL DeleteFileA(const char* lpFileName) { return TRUE; } -// CloseHandle - unified close for files, mutexes, semaphores, events, queues. +// CloseHandle - unified close for files, threads, mutexes, semaphores, events, queues, and processes. __declspec(dllexport) BOOL CloseHandle(HANDLE hObject) { if (!hObject || hObject == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; } - // Our thread/sentinel sync handles are not entries in the kernel handle table. - // There is nothing to close; the thread object is reaped by the scheduler. - if (IS_THREAD_HANDLE(hObject) || IS_NOOP_SYNC_HANDLE(hObject)) return TRUE; - SysCloseHandle(HANDLE_TO_KH(hObject)); return TRUE; } @@ -870,8 +1000,11 @@ __declspec(dllexport) BOOL HeapDestroy(HANDLE hHeap) { // Implements the heap Size operation. __declspec(dllexport) size_t HeapSize(HANDLE heap, DWORD flags, const void* ptr) { - (void)heap; (void)flags; (void)ptr; - return (size_t)-1; // Win32 returns (SIZE_T)-1 on error / unsupported + (void)heap; (void)flags; + size_t size = _malloc_usable_size(ptr); + if (size == (size_t)-1) + gLastError = 87; // ERROR_INVALID_PARAMETER + return size; } __declspec(dllexport) BOOL HeapSetInformation(HANDLE HeapHandle, @@ -1050,9 +1183,29 @@ __declspec(dllexport) DWORD GetModuleFileNameA(HANDLE hModule, // Implements the terminate Process operation. __declspec(dllexport) BOOL TerminateProcess(HANDLE hProcess, unsigned int uExitCode) { - (void)hProcess; - SysExitProcess((int)uExitCode); - while (true) ; + if (hProcess == GetCurrentProcess()) { + SysExitProcess((int)uExitCode); + while (true) ; + } + + if (!hProcess) { + gLastError = 6; // ERROR_INVALID_HANDLE + return FALSE; + } + + uint32_t pid = 0; + uint32_t s = SysGetProcessPid(HANDLE_TO_KH(hProcess), &pid); + if (s != 0 || pid == 0) { + gLastError = 6; // ERROR_INVALID_HANDLE + return FALSE; + } + + s = SysKillProcess(pid, (int)uExitCode); + if (s != 0) { + gLastError = 5; // ERROR_ACCESS_DENIED / generic failure for now + return FALSE; + } + return TRUE; } // STARTUPINFOA - only cb matters for our stub. @@ -1099,35 +1252,42 @@ __declspec(dllexport) HANDLE CreateThread(void* lpThreadAttributes, DWORD* lpThreadId) { (void)lpThreadAttributes; (void)dwStackSize; (void)dwCreationFlags; uint32_t tid = 0; - uint32_t s = SysCreateThread((uint64_t)(uintptr_t)lpStartAddress, - (uint64_t)(uintptr_t)lpParameter, - 0, &tid); + uint32_t kh = 0; + SYS_CREATE_THREAD_REQUEST req = { + (uintptr_t)lpStartAddress, + (uintptr_t)lpParameter, + 0, + 0, + &kh, + &tid + }; + uint32_t s = SysCreateThread(&req); if (s != 0) { gLastError = 11; return INVALID_HANDLE_VALUE; } if (lpThreadId) *lpThreadId = (DWORD)tid; - return TID_TO_THREAD_HANDLE(tid); + return KH_TO_HANDLE(kh); } -// WaitForSingleObject - blocks on process handles via SysWaitForProcess; -// thread and other handle types are not yet waitable so return immediately. +// WaitForSingleObject - blocks on native waitable handles. A zero timeout is +// only a probe for mutexes/semaphores until native timeout waits exist. __declspec(dllexport) DWORD WaitForSingleObject(HANDLE hHandle, DWORD dwMilliseconds) { - (void)dwMilliseconds; - if (!hHandle || hHandle == INVALID_HANDLE_VALUE) { gLastError = 6; return 0xFFFFFFFFUL; } - - if (IS_THREAD_HANDLE(hHandle)) { - uint32_t tid = THREAD_HANDLE_TO_TID(hHandle); - uint32_t s = SysWaitThread(tid); - if (s != 0 && s != 0x0022) { gLastError = 6; return 0xFFFFFFFFUL; } - return WAIT_OBJECT_0; - } - - if (IS_NOOP_SYNC_HANDLE(hHandle)) return WAIT_OBJECT_0; + if (!IS_VALID_KERNEL_HANDLE(hHandle)) { gLastError = 6; return 0xFFFFFFFFUL; } uint32_t kh = HANDLE_TO_KH(hHandle); - if (kh) { - uint32_t s = SysWaitForProcess(kh); - if (s != 0) { gLastError = 6; return 0xFFFFFFFFUL; } + if (dwMilliseconds == 0) { + uint32_t s = SysTryAcquireMutex(kh); + if (s == 0) return WAIT_OBJECT_0; + + s = SysTryWaitSem(kh); + if (s == 0) return WAIT_OBJECT_0; + + return WAIT_TIMEOUT; } - return WAIT_OBJECT_0; + + uint32_t s = SysWait(kh, SYS_WAIT_FLAG_NONE); + if (s == 0) return WAIT_OBJECT_0; + + gLastError = 6; + return 0xFFFFFFFFUL; } @@ -1250,7 +1410,7 @@ __declspec(dllexport) BOOL CreateProcessA( __declspec(dllexport) BOOL GetExitCodeProcess(HANDLE hProcess, DWORD* lpExitCode) { if (!lpExitCode) { gLastError = 87; return FALSE; } uint32_t kh = HANDLE_TO_KH(hProcess); - if (!kh || IS_THREAD_HANDLE(hProcess)) { gLastError = 6; return FALSE; } // ERROR_INVALID_HANDLE + if (!kh) { gLastError = 6; return FALSE; } // ERROR_INVALID_HANDLE int code = 0; uint32_t s = SysGetProcessExitCode(kh, &code); if (s != 0) { gLastError = 6; return FALSE; } @@ -1791,57 +1951,123 @@ __declspec(dllexport) DWORD GetCurrentDirectoryA(DWORD nBufferLength, char* lpBu __declspec(dllexport) HANDLE CreateMutexA(void* lpMutexAttributes, BOOL bInitialOwner, const char* lpName) { - (void)lpMutexAttributes; (void)bInitialOwner; (void)lpName; - return (HANDLE)WIN32_MUTEX_SENTINEL; + (void)lpMutexAttributes; (void)lpName; + uint32_t kh = 0; + uint32_t s = SysCreateMutex(&kh); + if (s != 0 || !kh) { gLastError = 8; return NULL; } + if (bInitialOwner) { + s = SysWait(kh, SYS_WAIT_FLAG_NONE); + if (s != 0) { + SysCloseHandle(kh); + gLastError = 1; + return NULL; + } + } + return KH_TO_HANDLE(kh); } // Releases mutex. __declspec(dllexport) BOOL ReleaseMutex(HANDLE hMutex) { - (void)hMutex; + if (!IS_VALID_KERNEL_HANDLE(hMutex)) { gLastError = 6; return FALSE; } + uint32_t s = SysReleaseMutex(HANDLE_TO_KH(hMutex)); + if (s != 0) { gLastError = 6; return FALSE; } return TRUE; } __declspec(dllexport) HANDLE CreateSemaphoreA(void* lpSemaphoreAttributes, long lInitialCount, long lMaximumCount, const char* lpName) { - (void)lpSemaphoreAttributes; (void)lInitialCount; - (void)lMaximumCount; (void)lpName; - return (HANDLE)WIN32_SEMA_SENTINEL; + (void)lpSemaphoreAttributes; (void)lpName; + uint32_t kh = 0; + uint32_t s = SysCreateSem((int32_t)lInitialCount, (int32_t)lMaximumCount, &kh); + if (s != 0 || !kh) { gLastError = 8; return NULL; } + return KH_TO_HANDLE(kh); } __declspec(dllexport) BOOL ReleaseSemaphore(HANDLE hSemaphore, long lReleaseCount, long* lpPreviousCount) { - (void)hSemaphore; (void)lReleaseCount; + if (!IS_VALID_KERNEL_HANDLE(hSemaphore) || lReleaseCount < 0) { gLastError = 6; return FALSE; } if (lpPreviousCount) *lpPreviousCount = 0; + for (long i = 0; i < lReleaseCount; ++i) { + uint32_t s = SysSignalSem(HANDLE_TO_KH(hSemaphore)); + if (s != 0) { gLastError = 6; return FALSE; } + } return TRUE; } -// Critical Sections & Condition Variables -// Single-threaded: all locks are no-ops. - -// Minimal opaque types; callers allocate these from their Windows headers - -// we just need the pointer to be accepted by our stubs. +// Critical Sections & Condition Variables. +// Match the size of Win64 RTL_CRITICAL_SECTION (5 pointer-sized fields) while +// using the storage as a small recursive spin lock. Bink uses these from its +// worker/copy paths, so the old no-op implementation allowed internal state +// races as soon as CreateThread started working. typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION; typedef struct { void* _ptr; } CONDITION_VARIABLE; +#define K32_CS_LOCK 0 +#define K32_CS_OWNER 1 +#define K32_CS_RECURSION 2 + +static volatile long* K32CsLockWord(CRITICAL_SECTION* cs) { + return (volatile long*)&cs->_opaque[K32_CS_LOCK]; +} + +static DWORD K32CurrentTidOrOne(void) { + DWORD tid = (DWORD)SysGetCurrentTid(); + return tid ? tid : 1; +} + // Initializes critical Section. __declspec(dllexport) void InitializeCriticalSection(CRITICAL_SECTION* lpCS) { - (void)lpCS; + if (!lpCS) return; + memset(lpCS, 0, sizeof(*lpCS)); } __declspec(dllexport) BOOL InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION* lpCS, DWORD dwSpinCount) { - (void)lpCS; (void)dwSpinCount; + (void)dwSpinCount; + InitializeCriticalSection(lpCS); return TRUE; } // Releases critical Section. __declspec(dllexport) void DeleteCriticalSection(CRITICAL_SECTION* lpCS) { (void)lpCS; } // Implements the enter Critical Section operation. -__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS) { (void)lpCS; } +__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS) { + if (!lpCS) return; + DWORD tid = K32CurrentTidOrOne(); + if ((DWORD)lpCS->_opaque[K32_CS_OWNER] == tid) { + lpCS->_opaque[K32_CS_RECURSION]++; + return; + } + while (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0) + _mm_pause(); + lpCS->_opaque[K32_CS_OWNER] = tid; + lpCS->_opaque[K32_CS_RECURSION] = 1; +} // Implements the leave Critical Section operation. -__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION* lpCS) { (void)lpCS; } +__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION* lpCS) { + if (!lpCS) return; + DWORD tid = K32CurrentTidOrOne(); + if ((DWORD)lpCS->_opaque[K32_CS_OWNER] != tid || lpCS->_opaque[K32_CS_RECURSION] == 0) + return; + lpCS->_opaque[K32_CS_RECURSION]--; + if (lpCS->_opaque[K32_CS_RECURSION] == 0) { + lpCS->_opaque[K32_CS_OWNER] = 0; + _ReadWriteBarrier(); + _InterlockedExchange(K32CsLockWord(lpCS), 0); + } +} // Implements the try Enter Critical Section operation. __declspec(dllexport) BOOL TryEnterCriticalSection(CRITICAL_SECTION* lpCS) { - (void)lpCS; return TRUE; + if (!lpCS) return FALSE; + DWORD tid = K32CurrentTidOrOne(); + if ((DWORD)lpCS->_opaque[K32_CS_OWNER] == tid) { + lpCS->_opaque[K32_CS_RECURSION]++; + return TRUE; + } + if (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0) + return FALSE; + lpCS->_opaque[K32_CS_OWNER] = tid; + lpCS->_opaque[K32_CS_RECURSION] = 1; + return TRUE; } // Initializes condition Variable. @@ -2357,14 +2583,12 @@ __declspec(dllexport) int GetLocaleInfoA(unsigned int Locale, DWORD LCType, return 0; } -// Fiber Local Storage (FLS) -// We implement a minimal 64-slot static table so callers that use FLS for -// thread-local-ish state don't immediately fail. +// Fiber Local Storage (FLS). +// Allocation state is process-global, but the values themselves are per-thread, +// matching the contract that Bink and the CRT expect when worker threads are +// enabled. The old single global value array let unrelated threads overwrite +// each other's decoder-local state. -#define FLS_MAX_SLOTS 64 -#define FLS_OUT_OF_INDEXES 0xFFFFFFFFUL - -static void* gFlsSlots[FLS_MAX_SLOTS]; static int gFlsUsed[FLS_MAX_SLOTS]; // 0 = free, 1 = allocated // Implements the fls Alloc operation. @@ -2372,8 +2596,8 @@ __declspec(dllexport) DWORD FlsAlloc(void* lpCallback) { (void)lpCallback; for (int i = 0; i < FLS_MAX_SLOTS; i++) { if (!gFlsUsed[i]) { - gFlsUsed[i] = 1; - gFlsSlots[i] = NULL; + gFlsUsed[i] = 1; + K32ThreadEnv()->FlsSlots[i] = NULL; return (DWORD)i; } } @@ -2387,7 +2611,7 @@ __declspec(dllexport) void* FlsGetValue(DWORD dwFlsIndex) { gLastError = 87; // ERROR_INVALID_PARAMETER return NULL; } - return gFlsSlots[dwFlsIndex]; + return K32ThreadEnv()->FlsSlots[dwFlsIndex]; } // Implements the fls Set Value operation. @@ -2396,7 +2620,7 @@ __declspec(dllexport) BOOL FlsSetValue(DWORD dwFlsIndex, void* lpFlsData) { gLastError = 87; return FALSE; } - gFlsSlots[dwFlsIndex] = lpFlsData; + K32ThreadEnv()->FlsSlots[dwFlsIndex] = lpFlsData; return TRUE; } @@ -2406,8 +2630,8 @@ __declspec(dllexport) BOOL FlsFree(DWORD dwFlsIndex) { gLastError = 87; return FALSE; } - gFlsUsed[dwFlsIndex] = 0; - gFlsSlots[dwFlsIndex] = NULL; + gFlsUsed[dwFlsIndex] = 0; + K32ThreadEnv()->FlsSlots[dwFlsIndex] = NULL; return TRUE; } diff --git a/code/user/runtime/heap.c b/code/user/runtime/heap.c index 629a180..7b1d323 100644 --- a/code/user/runtime/heap.c +++ b/code/user/runtime/heap.c @@ -210,6 +210,29 @@ void* _calloc(size_t count, size_t size) { return ptr; } + +// Returns the usable payload size of an allocation made by this heap. +size_t _malloc_usable_size(const void* ptr) { + if (!ptr) return (size_t)-1; + + HeapLock(); + HEAP_SEGMENT* seg = SegmentForPointer((void*)ptr); + if (!seg) { + HeapUnlock(); + return (size_t)-1; + } + + HEAP_BLOCK* block = PayloadBlock((void*)ptr); + if (block->Magic != USER_HEAP_MAGIC || block->Free) { + HeapUnlock(); + return (size_t)-1; + } + + size_t size = block->Size; + HeapUnlock(); + return size; +} + // Implements the realloc operation. void* _realloc(void* ptr, size_t newSize) { if (!ptr) return _malloc(newSize); diff --git a/code/user/runtime/heap_exports.c b/code/user/runtime/heap_exports.c index a4d128c..339c8d8 100644 --- a/code/user/runtime/heap_exports.c +++ b/code/user/runtime/heap_exports.c @@ -12,6 +12,7 @@ void* _malloc(size_t size); void _free(void* ptr); void* _calloc(size_t count, size_t size); void* _realloc(void* ptr, size_t size); +size_t _malloc_usable_size(const void* ptr); HEAP_EXPORT void* malloc(size_t size) { return _malloc(size); @@ -28,3 +29,7 @@ HEAP_EXPORT void* calloc(size_t count, size_t size) { HEAP_EXPORT void* realloc(void* ptr, size_t size) { return _realloc(ptr, size); } + +HEAP_EXPORT size_t malloc_usable_size(const void* ptr) { + return _malloc_usable_size(ptr); +} diff --git a/code/user/syscall/syscall_nr.inc b/code/user/syscall/syscall_nr.inc index a88f1fc..f839a86 100644 --- a/code/user/syscall/syscall_nr.inc +++ b/code/user/syscall/syscall_nr.inc @@ -3,13 +3,13 @@ SYS_EXIT_PROCESS EQU 0 SYS_EXIT_THREAD EQU 1 SYS_CREATE_PROCESS EQU 2 -SYS_WAIT_FOR_PROCESS EQU 3 +SYS_WAIT_SINGLE EQU 3 SYS_GET_PROCESS_EXIT_CODE EQU 4 SYS_KILL_PROCESS EQU 5 SYS_GET_PROCESS_LIST EQU 6 SYS_GET_PROCESS_PID EQU 7 SYS_CREATE_THREAD EQU 8 -SYS_WAIT_THREAD EQU 9 +SYS_RETIRED_WAIT_THREAD EQU 9 SYS_GET_CURRENT_TID EQU 10 SYS_GET_CURRENT_PID EQU 11 SYS_SET_THREAD_PRIORITY EQU 12 @@ -23,17 +23,17 @@ SYS_PROTECT_VM EQU 19 SYS_QUERY_VM EQU 20 SYS_CLOSE_HANDLE EQU 21 SYS_CREATE_MUTEX EQU 22 -SYS_ACQUIRE_MUTEX EQU 23 +SYS_RETIRED_ACQUIRE_MUTEX EQU 23 SYS_TRY_ACQUIRE_MUTEX EQU 24 SYS_RELEASE_MUTEX EQU 25 SYS_CREATE_SEM EQU 26 -SYS_WAIT_SEM EQU 27 +SYS_RETIRED_WAIT_SEM EQU 27 SYS_TRY_WAIT_SEM EQU 28 SYS_SIGNAL_SEM EQU 29 SYS_CREATE_EVENT EQU 30 SYS_SIGNAL_EVENT EQU 31 SYS_CLEAR_EVENT EQU 32 -SYS_WAIT_EVENT EQU 33 +SYS_RETIRED_WAIT_EVENT EQU 33 SYS_CREATE_QUEUE EQU 34 SYS_SEND_QUEUE EQU 35 SYS_RECV_QUEUE EQU 36 @@ -66,36 +66,47 @@ SYS_SET_CONSOLE_CURSOR EQU 62 SYS_FILL_CONSOLE_CHARACTER EQU 63 SYS_FILL_CONSOLE_ATTRIBUTE EQU 64 SYS_PUT_SCREEN_CHAR EQU 65 -SYS_BLIT_SCREEN EQU 66 -SYS_CLEAR_SCREEN EQU 67 -SYS_GET_SCREEN_SIZE EQU 68 -SYS_QUERY_DISPLAY_INFO EQU 69 -SYS_POLL_KEYBOARD EQU 70 -SYS_IS_KEY_DOWN EQU 71 -SYS_READ_KEYBOARD_CHAR EQU 72 -SYS_READ_KEY EQU 73 -SYS_GET_KEY_MODIFIERS EQU 74 -SYS_POLL_MOUSE EQU 75 -SYS_READ_MOUSE EQU 76 -SYS_GET_MOUSE_POS EQU 77 -SYS_GET_TICK EQU 78 -SYS_PRINT_DEBUG EQU 79 -SYS_QUERY_TRAP_FRAME EQU 80 -SYS_CREATE_SOCKET EQU 81 -SYS_BIND_SOCKET EQU 82 -SYS_LISTEN_SOCKET EQU 83 -SYS_ACCEPT_SOCKET EQU 84 -SYS_CONNECT_SOCKET EQU 85 -SYS_SEND_SOCKET EQU 86 -SYS_RECV_SOCKET EQU 87 -SYS_SHUTDOWN_SOCKET EQU 88 -SYS_SEND_TO_SOCKET EQU 89 -SYS_RECV_FROM_SOCKET EQU 90 -SYS_RESOLVE_HOST EQU 91 -SYS_SET_SOCKET_OPTION EQU 92 -SYS_GET_SOCKET_OPTION EQU 93 -SYS_POLL EQU 94 -SYS_GET_MEMORY_STATUS EQU 95 -SYS_QUERY_KERNEL_STATS EQU 96 -SYS_GET_SYSTEM_TIME EQU 97 -SYS_MAX EQU 98 +SYS_CLEAR_SCREEN EQU 66 +SYS_GET_SCREEN_SIZE EQU 67 +SYS_QUERY_DISPLAY_INFO EQU 68 +SYS_POLL_KEYBOARD EQU 69 +SYS_IS_KEY_DOWN EQU 70 +SYS_READ_KEYBOARD_CHAR EQU 71 +SYS_READ_KEY EQU 72 +SYS_GET_KEY_MODIFIERS EQU 73 +SYS_POLL_MOUSE EQU 74 +SYS_READ_MOUSE EQU 75 +SYS_GET_MOUSE_POS EQU 76 +SYS_GET_TICK EQU 77 +SYS_PRINT_DEBUG EQU 78 +SYS_QUERY_TRAP_FRAME EQU 79 +SYS_CREATE_SOCKET EQU 80 +SYS_BIND_SOCKET EQU 81 +SYS_LISTEN_SOCKET EQU 82 +SYS_ACCEPT_SOCKET EQU 83 +SYS_CONNECT_SOCKET EQU 84 +SYS_SEND_SOCKET EQU 85 +SYS_RECV_SOCKET EQU 86 +SYS_SHUTDOWN_SOCKET EQU 87 +SYS_SEND_TO_SOCKET EQU 88 +SYS_RECV_FROM_SOCKET EQU 89 +SYS_RESOLVE_HOST EQU 90 +SYS_SET_SOCKET_OPTION EQU 91 +SYS_GET_SOCKET_OPTION EQU 92 +SYS_POLL EQU 93 +SYS_GET_MEMORY_STATUS EQU 94 +SYS_QUERY_KERNEL_STATS EQU 95 +SYS_GET_SYSTEM_TIME EQU 96 +SYS_CREATE_FILE_MAPPING EQU 97 +SYS_MAP_FILE EQU 98 +SYS_CREATE_SURFACE EQU 99 +SYS_MAP_SURFACE EQU 100 +SYS_QUERY_SURFACE EQU 101 +SYS_PRESENT_SURFACE EQU 102 +SYS_GET_THREAD_LIST EQU 103 +SYS_READ_INPUT_EVENT EQU 104 +SYS_PEEK_INPUT_EVENT EQU 105 +SYS_WAIT_MANY EQU 106 +SYS_OPEN_INPUT_QUEUE EQU 107 +SYS_FLUSH_FILE EQU 108 +SYS_MAX EQU 109 diff --git a/code/user/syscall/syscall_stubs.asm b/code/user/syscall/syscall_stubs.asm index 5eb9c13..76b2489 100644 --- a/code/user/syscall/syscall_stubs.asm +++ b/code/user/syscall/syscall_stubs.asm @@ -12,7 +12,8 @@ ; User-mode callers follow standard MS x64: args 1-4 in RCX/RDX/R8/R9, ; args 5+ on the stack above the home area ([rsp+28h], [rsp+30h], ...). ; -; SYSCALL_STUB2 - 4 or fewer args: pure register passing, no stack read. +; SYSCALL_STUB2/3 - 4 or fewer args: pure register passing, no stack read. +; SYSCALL_STUB5 - exactly 5 args: loads [rsp+28h] -> R10 only. ; SYSCALL_STUB6 - 5 or 6 args: loads [rsp+28h] -> R10 and [rsp+30h] -> R11. ; ; Handle lifecycle: @@ -44,6 +45,27 @@ Sys&name PROC EXPORT Sys&name ENDP ENDM +; --------------------------------------------------------------------------- +SYSCALL_STUB3 MACRO name, nr +PUBLIC Sys&name +Sys&name PROC EXPORT + mov rax, nr + int 40h + ret +Sys&name ENDP +ENDM + +; --------------------------------------------------------------------------- +SYSCALL_STUB5 MACRO name, nr +PUBLIC Sys&name +Sys&name PROC EXPORT + mov r10, [rsp+28h] ; 5th argument + mov rax, nr + int 40h + ret +Sys&name ENDP +ENDM + ; --------------------------------------------------------------------------- SYSCALL_STUB6 MACRO name, nr PUBLIC Sys&name @@ -62,14 +84,16 @@ ENDM SYSCALL_STUB2 ExitProcess, SYS_EXIT_PROCESS SYSCALL_STUB2 ExitThread, SYS_EXIT_THREAD SYSCALL_STUB2 CreateProcess, SYS_CREATE_PROCESS -SYSCALL_STUB2 WaitForProcess, SYS_WAIT_FOR_PROCESS +SYSCALL_STUB3 WaitSingle, SYS_WAIT_SINGLE +SYSCALL_STUB2 WaitMany, SYS_WAIT_MANY +SYSCALL_STUB2 OpenInputQueue, SYS_OPEN_INPUT_QUEUE SYSCALL_STUB2 GetProcessExitCode, SYS_GET_PROCESS_EXIT_CODE SYSCALL_STUB2 KillProcess, SYS_KILL_PROCESS SYSCALL_STUB2 GetProcessList, SYS_GET_PROCESS_LIST SYSCALL_STUB2 GetProcessPid, SYS_GET_PROCESS_PID +SYSCALL_STUB2 GetThreadList, SYS_GET_THREAD_LIST SYSCALL_STUB2 CreateThread, SYS_CREATE_THREAD -SYSCALL_STUB2 WaitThread, SYS_WAIT_THREAD SYSCALL_STUB2 GetCurrentTid, SYS_GET_CURRENT_TID SYSCALL_STUB2 GetCurrentPid, SYS_GET_CURRENT_PID SYSCALL_STUB2 SetThreadPriority, SYS_SET_THREAD_PRIORITY @@ -120,19 +144,16 @@ SYSCALL_STUB2 CloseHandle, SYS_CLOSE_HANDLE ; Synchronization objects ; --------------------------------------------------------------------------- SYSCALL_STUB2 CreateMutex, SYS_CREATE_MUTEX -SYSCALL_STUB2 AcquireMutex, SYS_ACQUIRE_MUTEX SYSCALL_STUB2 TryAcquireMutex, SYS_TRY_ACQUIRE_MUTEX SYSCALL_STUB2 ReleaseMutex, SYS_RELEASE_MUTEX SYSCALL_STUB2 CreateSem, SYS_CREATE_SEM -SYSCALL_STUB2 WaitSem, SYS_WAIT_SEM SYSCALL_STUB2 TryWaitSem, SYS_TRY_WAIT_SEM SYSCALL_STUB2 SignalSem, SYS_SIGNAL_SEM SYSCALL_STUB2 CreateEvent, SYS_CREATE_EVENT SYSCALL_STUB2 SignalEvent, SYS_SIGNAL_EVENT SYSCALL_STUB2 ClearEvent, SYS_CLEAR_EVENT -SYSCALL_STUB2 WaitEvent, SYS_WAIT_EVENT ; --------------------------------------------------------------------------- ; Queues and named queues @@ -159,6 +180,7 @@ SYSCALL_STUB2 SeekFile, SYS_SEEK_FILE SYSCALL_STUB2 GetFilePos, SYS_GET_FILE_POS SYSCALL_STUB2 GetFileSize, SYS_GET_FILE_SIZE SYSCALL_STUB2 TruncateFile, SYS_TRUNCATE_FILE +SYSCALL_STUB2 FlushFile, SYS_FLUSH_FILE SYSCALL_STUB2 DeleteFile, SYS_DELETE_FILE SYSCALL_STUB2 RenameFile, SYS_RENAME_FILE @@ -178,7 +200,10 @@ SYSCALL_STUB2 SetConsoleCursor, SYS_SET_CONSOLE_CURSOR SYSCALL_STUB6 FillConsoleCharacter, SYS_FILL_CONSOLE_CHARACTER SYSCALL_STUB6 FillConsoleAttribute, SYS_FILL_CONSOLE_ATTRIBUTE SYSCALL_STUB6 PutScreenChar, SYS_PUT_SCREEN_CHAR -SYSCALL_STUB6 BlitScreen, SYS_BLIT_SCREEN +SYSCALL_STUB5 CreateSurface, SYS_CREATE_SURFACE +SYSCALL_STUB2 MapSurface, SYS_MAP_SURFACE +SYSCALL_STUB2 QuerySurface, SYS_QUERY_SURFACE +SYSCALL_STUB2 PresentSurface, SYS_PRESENT_SURFACE SYSCALL_STUB2 ClearScreen, SYS_CLEAR_SCREEN SYSCALL_STUB2 GetScreenSize, SYS_GET_SCREEN_SIZE SYSCALL_STUB2 QueryDisplayInfo, SYS_QUERY_DISPLAY_INFO @@ -191,6 +216,8 @@ SYSCALL_STUB2 IsKeyDown, SYS_IS_KEY_DOWN SYSCALL_STUB2 ReadKeyboardChar, SYS_READ_KEYBOARD_CHAR SYSCALL_STUB2 ReadKey, SYS_READ_KEY SYSCALL_STUB2 GetKeyModifiers, SYS_GET_KEY_MODIFIERS +SYSCALL_STUB2 ReadInputEvent, SYS_READ_INPUT_EVENT +SYSCALL_STUB2 PeekInputEvent, SYS_PEEK_INPUT_EVENT SYSCALL_STUB2 PollMouse, SYS_POLL_MOUSE SYSCALL_STUB2 ReadMouse, SYS_READ_MOUSE @@ -224,5 +251,7 @@ SYSCALL_STUB2 Poll, SYS_POLL SYSCALL_STUB2 GetMemoryStatus, SYS_GET_MEMORY_STATUS SYSCALL_STUB2 QueryKernelStats, SYS_QUERY_KERNEL_STATS SYSCALL_STUB2 GetSystemTime, SYS_GET_SYSTEM_TIME +SYSCALL_STUB5 CreateFileMapping, SYS_CREATE_FILE_MAPPING +SYSCALL_STUB6 MapFile, SYS_MAP_FILE END diff --git a/code/user/system/init/init.c b/code/user/system/init/init.c index 25545cd..d77b468 100644 --- a/code/user/system/init/init.c +++ b/code/user/system/init/init.c @@ -76,9 +76,9 @@ void _start(void) } debug_print("[init] waiting\n"); - s = SysWaitForProcess(h); + s = SysWait(h, SYS_WAIT_FLAG_NONE); if (s != STATUS_SUCCESS) - debug_print("[init] WARNING: SysWaitForProcess returned error\n"); + debug_print("[init] WARNING: SysWait returned error\n"); int exitCode = 0; if (SysGetProcessExitCode(h, &exitCode) != STATUS_SUCCESS) { diff --git a/code/user/tests/async_apc/async_apc.c b/code/user/tests/async_apc/async_apc.c index f45639f..8304931 100644 --- a/code/user/tests/async_apc/async_apc.c +++ b/code/user/tests/async_apc/async_apc.c @@ -15,10 +15,35 @@ static void print(const char* s) { SysWriteConsole(&req); } -// Opens file. -static uint32_t open_file(const char* path, uint32_t* handleOut) { - SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), handleOut }; - return SysOpenFile(&req); +// Opens the first available executable payload for async I/O. +// Tests are usually launched from /apps, so a bare "system/init.exe" +// resolves as /apps/system/init.exe and fails. Prefer the test binary itself, +// then fall back to absolute/drive-qualified paths. +static uint32_t open_test_file(uint32_t* handleOut, const char** pathOut) { + static const char* paths[] = { + "async_apc.exe", + "/apps/async_apc.exe", + "1:/apps/async_apc.exe", + "0:/apps/async_apc.exe", + "/system/init.exe", + "0:/system/init.exe", + "1:/system/init.exe", + "system/init.exe", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysOpenFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *handleOut = h; + if (pathOut) + *pathOut = paths[i]; + return SYS_STATUS_SUCCESS; + } + } + + return SYS_STATUS_NOT_FOUND; } // Prints hex8. @@ -74,7 +99,8 @@ int main(void) { } uint32_t file = 0; - s = open_file("system/init.exe", &file); + const char* openedPath = 0; + s = open_test_file(&file, &openedPath); if (s != SYS_STATUS_SUCCESS) { print("async_apc: open failed status="); print_u64(s); @@ -82,6 +108,10 @@ int main(void) { SysExitProcess(1); } + print("async_apc: using "); + print(openedPath ? openedPath : ""); + print("\n"); + uint32_t event = 0; s = SysCreateEvent(EVENT_AUTO_RESET, 0, &event); if (s != SYS_STATUS_SUCCESS) { @@ -125,7 +155,7 @@ int main(void) { } while (!state.Delivered) { - s = SysWaitEvent(event, 1); + s = SysWait(event, SYS_WAIT_FLAG_ALERTABLE); print("async_apc: alertable wait returned "); print_u64(s); print("\n"); diff --git a/code/user/tests/async_cancel/async_cancel.c b/code/user/tests/async_cancel/async_cancel.c index 0df69c7..6ea12ed 100644 --- a/code/user/tests/async_cancel/async_cancel.c +++ b/code/user/tests/async_cancel/async_cancel.c @@ -14,10 +14,35 @@ static void print(const char* s) { SysWriteConsole(&req); } -// Opens file. -static uint32_t open_file(const char* path, uint32_t* handleOut) { - SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), handleOut }; - return SysOpenFile(&req); +// Opens the first available executable payload for async I/O. +// Tests are usually launched from /apps, so a bare "system/init.exe" +// resolves as /apps/system/init.exe and fails. Prefer the test binary itself, +// then fall back to absolute/drive-qualified paths. +static uint32_t open_test_file(uint32_t* handleOut, const char** pathOut) { + static const char* paths[] = { + "async_cancel.exe", + "/apps/async_cancel.exe", + "1:/apps/async_cancel.exe", + "0:/apps/async_cancel.exe", + "/system/init.exe", + "0:/system/init.exe", + "1:/system/init.exe", + "system/init.exe", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysOpenFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *handleOut = h; + if (pathOut) + *pathOut = paths[i]; + return SYS_STATUS_SUCCESS; + } + } + + return SYS_STATUS_NOT_FOUND; } // Prints u64. @@ -46,12 +71,17 @@ int main(void) { uint32_t event = 0; uint32_t request = 0; - uint32_t s = open_file("system/init.exe", &file); + const char* openedPath = 0; + uint32_t s = open_test_file(&file, &openedPath); if (s != SYS_STATUS_SUCCESS) { print("async_cancel: open failed status="); print_u64(s); print("\n"); SysExitProcess(1); } + print("async_cancel: using "); + print(openedPath ? openedPath : ""); + print("\n"); + s = SysCreateEvent(EVENT_AUTO_RESET, 0, &event); if (s != SYS_STATUS_SUCCESS) { print("async_cancel: event create failed status="); print_u64(s); print("\n"); @@ -90,7 +120,7 @@ int main(void) { print("async_cancel: cancel returned "); print_u64(cs); print("\n"); while (iosb.Status == SYS_STATUS_PENDING) { - s = SysWaitEvent(event, 1); + s = SysWait(event, SYS_WAIT_FLAG_ALERTABLE); print("async_cancel: wait returned "); print_u64(s); print("\n"); if (s != SYS_STATUS_SUCCESS && s != SYS_STATUS_KERNEL_APC && s != SYS_STATUS_USER_APC && s != SYS_STATUS_ALERTED) break; diff --git a/code/user/tests/dirty_writeback/dirty_writeback.c b/code/user/tests/dirty_writeback/dirty_writeback.c new file mode 100644 index 0000000..91e4ef4 --- /dev/null +++ b/code/user/tests/dirty_writeback/dirty_writeback.c @@ -0,0 +1,227 @@ +// Dirty file-cache writeback regression test. +// Verifies buffered writes dirty cache pages, explicit flush writes them back, and close flushes synchronously. + +#include +#include +#include +#include + +#define PAGE_SIZE_BYTES 4096u +#define TEST_SIZE (PAGE_SIZE_BYTES * 4u) +#define REWRITE_SIZE (PAGE_SIZE_BYTES * 2u) +#define CLOSE_SIZE (PAGE_SIZE_BYTES * 3u) + +static uint8_t gPatternA[TEST_SIZE]; +static uint8_t gPatternB[TEST_SIZE]; +static uint8_t gReadback[TEST_SIZE]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { + print("0"); + return; + } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c[2] = { buf[i], 0 }; + print(c); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("dirty_writeback: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("dirty_writeback: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { + memset(out, 0, sizeof(*out)); + return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); +} + +static void delete_path(const char* path) { + SYS_PATH_REQUEST req = { SysSvFromCstr(path) }; + (void)SysDeleteFile(&req); +} + +static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { + static const char* paths[] = { + "dirty_writeback.tmp", + "/dirty_writeback.tmp", + "1:/dirty_writeback.tmp", + "0:/dirty_writeback.tmp", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + delete_path(paths[i]); + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysCreateFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *fileOut = h; + if (pathOut) *pathOut = paths[i]; + return SYS_STATUS_SUCCESS; + } + } + + return SYS_STATUS_NOT_FOUND; +} + +static uint32_t open_path(const char* path, uint32_t* fileOut) { + uint32_t h = 0; + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), &h }; + uint32_t s = SysOpenFile(&req); + if (s == SYS_STATUS_SUCCESS) + *fileOut = h; + return s; +} + +static void fill_patterns(void) { + for (size_t i = 0; i < sizeof(gPatternA); ++i) { + gPatternA[i] = (uint8_t)(0x11u + (uint8_t)(i * 13u + (i >> 6))); + gPatternB[i] = (uint8_t)(0x80u + (uint8_t)(i * 31u + (i >> 4))); + } +} + +static void checked_write(uint32_t file, const void* buffer, size_t size, const char* what) { + size_t done = 0; + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &done); + uint32_t s = SysWriteFile(&req); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); + if (done != size) fail("short write"); +} + +static void checked_read(uint32_t file, void* buffer, size_t size, size_t* gotOut, const char* what) { + size_t got = 0; + memset(buffer, 0xCC, size); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &got); + uint32_t s = SysReadFile(&req); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); + if (gotOut) *gotOut = got; +} + +static void checked_seek(uint32_t file, uint64_t offset, const char* what) { + uint32_t s = SysSeekFile(file, offset); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); +} + +static void expect_writeback_progress(const SYS_FILE_CACHE_STATS* before, + const SYS_FILE_CACHE_STATS* after, + const char* what) { + if (after->FlushFileFailures != before->FlushFileFailures) + fail("flush failure counter increased"); + if (after->WritebackFailures != before->WritebackFailures) + fail("writeback failure counter increased"); + if (after->WritebackBytes <= before->WritebackBytes) { + print("dirty_writeback: "); + print(what); + print(" did not write back bytes\n"); + exit_now(1); + } +} + +int main(void) { + print("dirty_writeback: start\n"); + fill_patterns(); + + uint32_t file = 0; + const char* path = 0; + uint32_t s = create_temp(&file, &path); + if (s != SYS_STATUS_SUCCESS) fail_status("create temp", s); + + print("dirty_writeback: using "); + print(path ? path : ""); + print("\n"); + + SYS_FILE_CACHE_STATS before; + SYS_FILE_CACHE_STATS afterWrite; + SYS_FILE_CACHE_STATS afterFlush; + SYS_FILE_CACHE_STATS beforeClose; + SYS_FILE_CACHE_STATS afterClose; + + if (query_cache(&before) != SYS_STATUS_SUCCESS) fail("initial stats query failed"); + + checked_write(file, gPatternA, sizeof(gPatternA), "write pattern A"); + + if (query_cache(&afterWrite) != SYS_STATUS_SUCCESS) fail("post-write stats query failed"); + if (afterWrite.DirtyMarks <= before.DirtyMarks) fail("write did not mark dirty cache pages"); + if (afterWrite.DirtyPages <= before.DirtyPages) fail("write did not leave dirty pages resident"); + + checked_seek(file, 0, "seek readback before flush"); + size_t got = 0; + checked_read(file, gReadback, sizeof(gReadback), &got, "read before flush"); + if (got != sizeof(gPatternA)) fail("read before flush size mismatch"); + if (memcmp(gReadback, gPatternA, sizeof(gPatternA)) != 0) fail("cached read before flush mismatch"); + + s = SysFlushFile(file); + if (s != SYS_STATUS_SUCCESS) fail_status("explicit flush", s); + + if (query_cache(&afterFlush) != SYS_STATUS_SUCCESS) fail("post-flush stats query failed"); + if (afterFlush.FlushFileCalls <= afterWrite.FlushFileCalls) fail("flush call counter did not increase"); + expect_writeback_progress(&afterWrite, &afterFlush, "explicit flush"); + + checked_seek(file, 0, "seek rewrite"); + checked_write(file, gPatternB, REWRITE_SIZE, "rewrite pattern B"); + checked_seek(file, REWRITE_SIZE, "seek close-tail"); + checked_write(file, gPatternA + REWRITE_SIZE, CLOSE_SIZE - REWRITE_SIZE, "write close tail"); + + if (query_cache(&beforeClose) != SYS_STATUS_SUCCESS) fail("pre-close stats query failed"); + if (beforeClose.DirtyMarks <= afterFlush.DirtyMarks) fail("rewrite did not mark dirty pages"); + + s = SysCloseHandle(file); + file = 0; + if (s != SYS_STATUS_SUCCESS) fail_status("close", s); + + if (query_cache(&afterClose) != SYS_STATUS_SUCCESS) fail("post-close stats query failed"); + if (afterClose.FlushFileCalls <= beforeClose.FlushFileCalls) fail("close did not flush file"); + expect_writeback_progress(&beforeClose, &afterClose, "close flush"); + + s = open_path(path, &file); + if (s != SYS_STATUS_SUCCESS) fail_status("reopen", s); + + checked_read(file, gReadback, CLOSE_SIZE, &got, "read after reopen"); + if (got != CLOSE_SIZE) fail("read after reopen size mismatch"); + if (memcmp(gReadback, gPatternB, REWRITE_SIZE) != 0) fail("rewrite data not durable after close"); + if (memcmp(gReadback + REWRITE_SIZE, gPatternA + REWRITE_SIZE, + CLOSE_SIZE - REWRITE_SIZE) != 0) fail("tail data not durable after close"); + + s = SysCloseHandle(file); + file = 0; + if (s != SYS_STATUS_SUCCESS) fail_status("close reopened", s); + delete_path(path); + + print("dirty_writeback: dirty-marks+="); + print_u64(afterClose.DirtyMarks - before.DirtyMarks); + print(" writeback-bytes+="); + print_u64(afterClose.WritebackBytes - before.WritebackBytes); + print(" flushes+="); + print_u64(afterClose.FlushFileCalls - before.FlushFileCalls); + print("\n"); + + print("dirty_writeback: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/mmap_smoke/mmap_smoke.c b/code/user/tests/mmap_smoke/mmap_smoke.c new file mode 100644 index 0000000..62670bb --- /dev/null +++ b/code/user/tests/mmap_smoke/mmap_smoke.c @@ -0,0 +1,196 @@ +// User-mode regression test for mmap-style native file mappings. + +#include +#include +#include +#include + +#define TEST_MAP_SIZE 8192u + +static uint8_t gExpected[TEST_MAP_SIZE]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { + print("0"); + return; + } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c = buf[i]; + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(&c, 1) }; + SysWriteConsole(&req); + } +} + +static void exit_now(int code) { + // SysExitProcess should not return, but keep tests from falling through to + // the synthetic entry frame if a broken kernel build returns anyway. + for (;;) + SysExitProcess(code); +} + +static void fail_status(const char* what, uint32_t status) { + print("mmap_smoke: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static void fail_msg(const char* msg) { + print("mmap_smoke: "); + print(msg); + print("\n"); + exit_now(1); +} + +static uint32_t open_first_existing(uint32_t* fileOut, const char** pathOut) { + static const char* candidates[] = { + "mmap_smoke.exe", + "/apps/mmap_smoke.exe", + "0:/apps/mmap_smoke.exe", + "1:/apps/mmap_smoke.exe", + + "/system/init.exe", + "0:/system/init.exe", + "1:/system/init.exe", + "system/init.exe", + }; + + for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { + uint32_t file = 0; + SYS_PATH_HANDLE_REQUEST openReq = { SysSvFromCstr(candidates[i]), &file }; + uint32_t s = SysOpenFile(&openReq); + if (s == SYS_STATUS_SUCCESS) { + *fileOut = file; + if (pathOut) + *pathOut = candidates[i]; + return SYS_STATUS_SUCCESS; + } + } + + return SYS_STATUS_NOT_FOUND; +} + +int main(void) { + print("mmap_smoke: start\n"); + + uint32_t file = 0; + const char* openedPath = 0; + uint32_t s = open_first_existing(&file, &openedPath); + if (s != SYS_STATUS_SUCCESS) + fail_status("open test file", s); + + print("mmap_smoke: using "); + print(openedPath ? openedPath : ""); + print("\n"); + + size_t got = 0; + memset(gExpected, 0xA5, sizeof(gExpected)); + + SYS_FILE_IO_REQUEST ioReq = SysMakeFileIoRequest(file, gExpected, sizeof(gExpected), &got); + s = SysReadFile(&ioReq); + if (s != SYS_STATUS_SUCCESS) + fail_status("read baseline", s); + if (got < 64) + fail_msg("baseline read was unexpectedly short"); + + print("mmap_smoke: baseline read OK\n"); + + uint32_t mapping = 0; + s = SysCreateFileMapping(file, + SYS_VM_READ, + SYS_MAP_FLAG_NONE, + sizeof(gExpected), + &mapping); + if (s != SYS_STATUS_SUCCESS) + fail_status("create file mapping", s); + + print("mmap_smoke: mapping created\n"); + + // The mapping object must hold its own file reference. Closing the source + // handle before SysMapFile catches lifetime bugs in the object layer. + SysCloseHandle(file); + file = 0; + + print("mmap_smoke: source handle closed\n"); + + void* base = 0; + s = SysMapFile(mapping, + 0, + sizeof(gExpected), + SYS_MAP_ACCESS_READ, + SYS_MAP_FLAG_NONE, + &base); + + SysCloseHandle(mapping); + mapping = 0; + + if (s != SYS_STATUS_SUCCESS) + fail_status("map file view", s); + if (!base) + fail_msg("map returned null base"); + + print("mmap_smoke: view mapped\n"); + + volatile uint8_t* view = (volatile uint8_t*)base; + + // Touch both pages before comparing. This ensures the test exercises + // demand-faulted file-backed VAD pages rather than only VAD creation. + (void)view[0]; + print("mmap_smoke: first page fault OK\n"); + + (void)view[4096]; + print("mmap_smoke: second page fault OK\n"); + + for (size_t i = 0; i < got; ++i) { + if (view[i] != gExpected[i]) { + print("mmap_smoke: byte mismatch at "); + print_u64(i); + print(" expected="); + print_u64(gExpected[i]); + print(" got="); + print_u64(view[i]); + print("\n"); + SysFreeVm(base); + exit_now(1); + } + } + + print("mmap_smoke: mapped bytes match\n"); + + // If the file is shorter than TEST_MAP_SIZE, the mapped tail must be + // zero-filled rather than exposing stale physical memory. + for (size_t i = got; i < sizeof(gExpected) && i < got + 64; ++i) { + if (view[i] != 0) { + print("mmap_smoke: nonzero EOF fill at "); + print_u64(i); + print(" got="); + print_u64(view[i]); + print("\n"); + SysFreeVm(base); + exit_now(1); + } + } + + print("mmap_smoke: EOF fill OK\n"); + + s = SysFreeVm(base); + if (s != SYS_STATUS_SUCCESS) + fail_status("unmap", s); + + print("mmap_smoke: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/pcache_invalidate/pcache_invalidate.c b/code/user/tests/pcache_invalidate/pcache_invalidate.c new file mode 100644 index 0000000..b5e1467 --- /dev/null +++ b/code/user/tests/pcache_invalidate/pcache_invalidate.c @@ -0,0 +1,223 @@ +// Page-cache write/truncate invalidation regression test. +// Verifies cached clean pages are not reused after file write or truncate. + +#include +#include +#include +#include + +#define TEST_SIZE 8192u +#define TRUNC_SIZE 1024u + +static uint8_t gOriginal[TEST_SIZE]; +static uint8_t gReplacement[4096]; +static uint8_t gReadback[TEST_SIZE]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c[2] = { buf[i], 0 }; + print(c); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("pcache_invalidate: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("pcache_invalidate: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { + memset(out, 0, sizeof(*out)); + return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); +} + +static void delete_path(const char* path) { + SYS_PATH_REQUEST req = { SysSvFromCstr(path) }; + (void)SysDeleteFile(&req); +} + +static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { + static const char* paths[] = { + "pcache_invalidate.tmp", + "/pcache_invalidate.tmp", + "1:/pcache_invalidate.tmp", + "0:/pcache_invalidate.tmp", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + delete_path(paths[i]); + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysCreateFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *fileOut = h; + if (pathOut) *pathOut = paths[i]; + return SYS_STATUS_SUCCESS; + } + } + + return SYS_STATUS_NOT_FOUND; +} + +static void fill_patterns(void) { + for (size_t i = 0; i < TEST_SIZE; ++i) + gOriginal[i] = (uint8_t)(0x30u + (uint8_t)(i * 17u + (i >> 7))); + for (size_t i = 0; i < sizeof(gReplacement); ++i) + gReplacement[i] = (uint8_t)(0xA0u + (uint8_t)(i * 29u + (i >> 5))); +} + +static void checked_write(uint32_t file, const void* buffer, size_t size, const char* what) { + size_t done = 0; + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &done); + uint32_t s = SysWriteFile(&req); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); + if (done != size) fail("short write"); +} + +static void checked_read(uint32_t file, void* buffer, size_t size, size_t* gotOut, const char* what) { + size_t got = 0; + memset(buffer, 0xCC, size); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &got); + uint32_t s = SysReadFile(&req); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); + if (gotOut) *gotOut = got; +} + +int main(void) { + print("pcache_invalidate: start\n"); + fill_patterns(); + + uint32_t file = 0; + const char* path = 0; + uint32_t s = create_temp(&file, &path); + if (s != SYS_STATUS_SUCCESS) fail_status("create temp", s); + + print("pcache_invalidate: using "); + print(path ? path : ""); + print("\n"); + + checked_write(file, gOriginal, sizeof(gOriginal), "write original"); + + s = SysSeekFile(file, 0); + if (s != SYS_STATUS_SUCCESS) fail_status("seek fill", s); + + SYS_FILE_CACHE_STATS before; + SYS_FILE_CACHE_STATS afterFill; + SYS_FILE_CACHE_STATS afterSecond; + SYS_FILE_CACHE_STATS afterWrite; + SYS_FILE_CACHE_STATS afterRewriteRead; + SYS_FILE_CACHE_STATS afterTruncate; + if (query_cache(&before) != SYS_STATUS_SUCCESS) fail("initial stats query failed"); + + size_t got = 0; + checked_read(file, gReadback, sizeof(gReadback), &got, "first read"); + if (got != sizeof(gOriginal)) fail("first read size mismatch"); + if (memcmp(gReadback, gOriginal, sizeof(gOriginal)) != 0) fail("first read data mismatch"); + + if (query_cache(&afterFill) != SYS_STATUS_SUCCESS) fail("fill stats query failed"); + if (afterFill.Loads <= before.Loads) fail("initial read did not populate cache"); + + s = SysSeekFile(file, 0); + if (s != SYS_STATUS_SUCCESS) fail_status("seek second", s); + checked_read(file, gReadback, sizeof(gReadback), &got, "second read"); + if (got != sizeof(gOriginal)) fail("second read size mismatch"); + if (memcmp(gReadback, gOriginal, sizeof(gOriginal)) != 0) fail("second read data mismatch"); + + if (query_cache(&afterSecond) != SYS_STATUS_SUCCESS) fail("second stats query failed"); + // Do not require this second read to hit: the cache is intentionally + // bounded and pcache_pressure.exe may have left it at the soft limit, so + // the newly-populated temp pages are eligible for immediate pressure + // eviction. This test is about write/truncate freshness; mmap/read reuse + // is covered by pcache_mmap_read_reuse.exe. + + s = SysSeekFile(file, 0); + if (s != SYS_STATUS_SUCCESS) fail_status("seek rewrite", s); + checked_write(file, gReplacement, sizeof(gReplacement), "rewrite first page"); + + if (query_cache(&afterWrite) != SYS_STATUS_SUCCESS) fail("post-write stats query failed"); + if (afterWrite.Invalidations <= afterSecond.Invalidations) fail("write did not invalidate cached page"); + + uint32_t mapping = 0; + s = SysCreateFileMapping(file, SYS_VM_READ, SYS_MAP_FLAG_NONE, sizeof(gReplacement), &mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("create mapping after rewrite", s); + + void* mappedBase = 0; + s = SysMapFile(mapping, 0, sizeof(gReplacement), SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &mappedBase); + SysCloseHandle(mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("map after rewrite", s); + if (!mappedBase) fail("map after rewrite returned null"); + + volatile uint8_t* mappedView = (volatile uint8_t*)mappedBase; + (void)mappedView[0]; + (void)mappedView[sizeof(gReplacement) - 1]; + if (memcmp((const void*)mappedBase, gReplacement, sizeof(gReplacement)) != 0) { + SysFreeVm(mappedBase); + fail("stale mmap data after write invalidation"); + } + s = SysFreeVm(mappedBase); + if (s != SYS_STATUS_SUCCESS) fail_status("unmap after rewrite", s); + + s = SysSeekFile(file, 0); + if (s != SYS_STATUS_SUCCESS) fail_status("seek verify rewrite", s); + checked_read(file, gReadback, sizeof(gReadback), &got, "read after rewrite"); + if (got != sizeof(gOriginal)) fail("rewrite read size mismatch"); + if (memcmp(gReadback, gReplacement, sizeof(gReplacement)) != 0) fail("stale data after write invalidation"); + if (memcmp(gReadback + sizeof(gReplacement), gOriginal + sizeof(gReplacement), + sizeof(gOriginal) - sizeof(gReplacement)) != 0) fail("unwritten page changed after rewrite"); + + if (query_cache(&afterRewriteRead) != SYS_STATUS_SUCCESS) fail("rewrite-read stats query failed"); + + s = SysTruncateFile(file, TRUNC_SIZE); + if (s != SYS_STATUS_SUCCESS) fail_status("truncate", s); + if (SysGetFileSize(file) != TRUNC_SIZE) fail("truncate size mismatch"); + + if (query_cache(&afterTruncate) != SYS_STATUS_SUCCESS) fail("truncate stats query failed"); + if (afterTruncate.Invalidations <= afterRewriteRead.Invalidations) fail("truncate did not invalidate cached tail"); + + s = SysSeekFile(file, 0); + if (s != SYS_STATUS_SUCCESS) fail_status("seek truncated", s); + checked_read(file, gReadback, sizeof(gReadback), &got, "read truncated"); + if (got != TRUNC_SIZE) fail("truncated read length mismatch"); + if (memcmp(gReadback, gReplacement, TRUNC_SIZE) != 0) fail("truncated data mismatch"); + + SysCloseHandle(file); + delete_path(path); + + print("pcache_invalidate: invalidations+="); + print_u64(afterTruncate.Invalidations - before.Invalidations); + print(" loads+="); + print_u64(afterTruncate.Loads - before.Loads); + print(" hits+="); + print_u64(afterTruncate.Hits - before.Hits); + print("\n"); + + print("pcache_invalidate: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c b/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c new file mode 100644 index 0000000..872933e --- /dev/null +++ b/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c @@ -0,0 +1,176 @@ +// Page-cache mmap->read reuse regression test. +// Faults file pages through mmap first, then verifies SysReadFile reuses those pages. + +#include +#include +#include +#include + +#define TEST_SIZE 8192u + +static uint8_t gReadback[TEST_SIZE]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c[2] = { buf[i], 0 }; + print(c); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("pcache_mmap_read_reuse: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("pcache_mmap_read_reuse: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { + memset(out, 0, sizeof(*out)); + return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); +} + +static uint32_t open_test_file(uint32_t* out, const char** pathOut) { + static const char* paths[] = { + "pcache_mmap_read_reuse.exe", + "/apps/pcache_mmap_read_reuse.exe", + "1:/apps/pcache_mmap_read_reuse.exe", + "0:/apps/pcache_mmap_read_reuse.exe", + "pcache_smoke.exe", + "/apps/pcache_smoke.exe", + "1:/apps/pcache_smoke.exe", + "0:/apps/pcache_smoke.exe", + "/system/init.exe", + "0:/system/init.exe", + "1:/system/init.exe", + "system/init.exe", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysOpenFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *out = h; + if (pathOut) *pathOut = paths[i]; + return SYS_STATUS_SUCCESS; + } + } + return SYS_STATUS_NOT_FOUND; +} + +int main(void) { + print("pcache_mmap_read_reuse: start\n"); + + SYS_FILE_CACHE_STATS before; + SYS_FILE_CACHE_STATS afterFault; + SYS_FILE_CACHE_STATS afterRead; + uint32_t s = query_cache(&before); + if (s != SYS_STATUS_SUCCESS) fail_status("query initial stats", s); + + uint32_t file = 0; + const char* path = 0; + s = open_test_file(&file, &path); + if (s != SYS_STATUS_SUCCESS) fail_status("open test file", s); + + uint64_t fileSize = SysGetFileSize(file); + if (fileSize < 4096) fail("test file too small"); + size_t mapSize = (fileSize < TEST_SIZE) ? (size_t)fileSize : TEST_SIZE; + + print("pcache_mmap_read_reuse: using "); + print(path ? path : ""); + print(" size="); + print_u64(fileSize); + print("\n"); + + uint32_t mapping = 0; + s = SysCreateFileMapping(file, SYS_VM_READ, SYS_MAP_FLAG_NONE, mapSize, &mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("create mapping", s); + + void* base = 0; + s = SysMapFile(mapping, 0, mapSize, SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &base); + SysCloseHandle(mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("map file", s); + if (!base) fail("map returned null"); + + volatile uint8_t* view = (volatile uint8_t*)base; + for (size_t i = 0; i < mapSize; i += 4096) + (void)view[i]; + (void)view[mapSize - 1]; + + s = query_cache(&afterFault); + if (s != SYS_STATUS_SUCCESS) fail_status("query after mmap faults", s); + if (afterFault.Loads <= before.Loads) { + SysFreeVm(base); + fail("mmap faults did not populate file cache"); + } + + s = SysSeekFile(file, 0); + if (s != SYS_STATUS_SUCCESS) { + SysFreeVm(base); + fail_status("seek", s); + } + + size_t got = 0; + memset(gReadback, 0xCC, sizeof(gReadback)); + SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(file, gReadback, mapSize, &got); + s = SysReadFile(&readReq); + if (s != SYS_STATUS_SUCCESS) { + SysFreeVm(base); + fail_status("read", s); + } + if (got != mapSize) { + SysFreeVm(base); + fail("read length mismatch"); + } + if (memcmp(gReadback, (const void*)base, mapSize) != 0) { + SysFreeVm(base); + fail("read data differs from mapped data"); + } + + s = query_cache(&afterRead); + if (s != SYS_STATUS_SUCCESS) { + SysFreeVm(base); + fail_status("query after read", s); + } + if (afterRead.Hits <= afterFault.Hits) { + SysFreeVm(base); + fail("read did not reuse mmap-populated cache pages"); + } + + SysFreeVm(base); + SysCloseHandle(file); + + print("pcache_mmap_read_reuse: loads+="); + print_u64(afterRead.Loads - before.Loads); + print(" hits+="); + print_u64(afterRead.Hits - before.Hits); + print("\n"); + print("pcache_mmap_read_reuse: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/pcache_multithread/pcache_multithread.c b/code/user/tests/pcache_multithread/pcache_multithread.c new file mode 100644 index 0000000..53ba344 --- /dev/null +++ b/code/user/tests/pcache_multithread/pcache_multithread.c @@ -0,0 +1,227 @@ +// Multi-threaded file-cache/VAD fault regression test. +// Several threads map the same file mapping and fault/read pages concurrently. + +#include +#include +#include +#include + +#define THREAD_COUNT 6u +#define ITERATIONS 16u +#define TEST_SIZE 32768u + +static uint8_t gPattern[TEST_SIZE]; + +typedef struct THREAD_CTX { + uint32_t Index; + uint32_t Mapping; + volatile uint32_t Status; + volatile uint32_t IterationsDone; +} THREAD_CTX; + +static THREAD_CTX gThreads[THREAD_COUNT]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c[2] = { buf[i], 0 }; + print(c); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("pcache_multithread: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("pcache_multithread: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { + memset(out, 0, sizeof(*out)); + return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); +} + +static void delete_path(const char* path) { + SYS_PATH_REQUEST req = { SysSvFromCstr(path) }; + (void)SysDeleteFile(&req); +} + +static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { + static const char* paths[] = { + "pcache_multithread.tmp", + "/pcache_multithread.tmp", + "1:/pcache_multithread.tmp", + "0:/pcache_multithread.tmp", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + delete_path(paths[i]); + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysCreateFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *fileOut = h; + if (pathOut) *pathOut = paths[i]; + return SYS_STATUS_SUCCESS; + } + } + + return SYS_STATUS_NOT_FOUND; +} + +static void fill_pattern(void) { + for (size_t i = 0; i < TEST_SIZE; ++i) + gPattern[i] = (uint8_t)(0x5Au ^ (uint8_t)(i * 13u) ^ (uint8_t)(i >> 3)); +} + +static uint32_t validate_mapping(const uint8_t* base) { + for (size_t i = 0; i < TEST_SIZE; ++i) { + if (base[i] != gPattern[i]) + return 0x10000000u | (uint32_t)(i & 0x0FFFFFFFu); + } + return SYS_STATUS_SUCCESS; +} + +static void thread_main(uintptr_t arg) { + THREAD_CTX* ctx = (THREAD_CTX*)arg; + if (!ctx) SysExitThread(1); + + for (uint32_t iter = 0; iter < ITERATIONS; ++iter) { + void* base = 0; + uint32_t s = SysMapFile(ctx->Mapping, 0, TEST_SIZE, + SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &base); + if (s != SYS_STATUS_SUCCESS || !base) { + ctx->Status = s ? s : SYS_STATUS_INVALID_PARAMETER; + SysExitThread(1); + } + + volatile uint8_t* touch = (volatile uint8_t*)base; + for (size_t off = 0; off < TEST_SIZE; off += 4096) + (void)touch[off]; + (void)touch[TEST_SIZE - 1]; + + s = validate_mapping((const uint8_t*)base); + uint32_t fs = SysFreeVm(base); + if (s != SYS_STATUS_SUCCESS) { + ctx->Status = s; + SysExitThread(1); + } + if (fs != SYS_STATUS_SUCCESS) { + ctx->Status = fs; + SysExitThread(1); + } + ctx->IterationsDone++; + } + + ctx->Status = SYS_STATUS_SUCCESS; + SysExitThread(0); +} + +int main(void) { + print("pcache_multithread: start\n"); + fill_pattern(); + + uint32_t file = 0; + const char* path = 0; + uint32_t s = create_temp(&file, &path); + if (s != SYS_STATUS_SUCCESS) fail_status("create temp", s); + + size_t written = 0; + SYS_FILE_IO_REQUEST writeReq = SysMakeFileIoRequest(file, gPattern, sizeof(gPattern), &written); + s = SysWriteFile(&writeReq); + if (s != SYS_STATUS_SUCCESS) fail_status("write pattern", s); + if (written != sizeof(gPattern)) fail("short write"); + + uint32_t mapping = 0; + s = SysCreateFileMapping(file, SYS_VM_READ, SYS_MAP_FLAG_NONE, TEST_SIZE, &mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("create mapping", s); + + SYS_FILE_CACHE_STATS before; + SYS_FILE_CACHE_STATS after; + if (query_cache(&before) != SYS_STATUS_SUCCESS) fail("initial stats query failed"); + + uint32_t handles[THREAD_COUNT]; + uint32_t tids[THREAD_COUNT]; + memset(handles, 0, sizeof(handles)); + memset(tids, 0, sizeof(tids)); + memset(gThreads, 0, sizeof(gThreads)); + + for (uint32_t i = 0; i < THREAD_COUNT; ++i) { + gThreads[i].Index = i; + gThreads[i].Mapping = mapping; + gThreads[i].Status = SYS_STATUS_PENDING; + SYS_CREATE_THREAD_REQUEST req = { + (uintptr_t)thread_main, + (uintptr_t)&gThreads[i], + 1, + 0, + &handles[i], + &tids[i] + }; + s = SysCreateThread(&req); + if (s != SYS_STATUS_SUCCESS) fail_status("create thread", s); + } + + for (uint32_t i = 0; i < THREAD_COUNT; ++i) { + s = SysWait(handles[i], SYS_WAIT_FLAG_NONE); + if (s != SYS_STATUS_SUCCESS) fail_status("wait thread", s); + SysCloseHandle(handles[i]); + } + + for (uint32_t i = 0; i < THREAD_COUNT; ++i) { + if (gThreads[i].Status != SYS_STATUS_SUCCESS) { + print("pcache_multithread: worker "); + print_u64(i); + print(" status="); + print_u64(gThreads[i].Status); + print(" iterations="); + print_u64(gThreads[i].IterationsDone); + print("\n"); + exit_now(1); + } + if (gThreads[i].IterationsDone != ITERATIONS) fail("worker iteration count mismatch"); + } + + if (query_cache(&after) != SYS_STATUS_SUCCESS) fail("final stats query failed"); + if (after.Loads <= before.Loads) fail("concurrent mmap did not load cache pages"); + if (after.Hits <= before.Hits) fail("concurrent mmap did not reuse cache pages"); + + SysCloseHandle(mapping); + SysCloseHandle(file); + delete_path(path); + + print("pcache_multithread: loads+="); + print_u64(after.Loads - before.Loads); + print(" hits+="); + print_u64(after.Hits - before.Hits); + print(" races+="); + print_u64(after.RaceDiscards - before.RaceDiscards); + print("\n"); + print("pcache_multithread: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/pcache_pressure/pcache_pressure.c b/code/user/tests/pcache_pressure/pcache_pressure.c new file mode 100644 index 0000000..5f3f7ca --- /dev/null +++ b/code/user/tests/pcache_pressure/pcache_pressure.c @@ -0,0 +1,169 @@ +// Sequential file-cache pressure test. +// Reads a large read-only asset through SysReadFile and checks that the +// read-only page cache stays bounded by dropping clean pages. + +#include +#include +#include +#include + +#define READ_CHUNK 4096u +#define MIN_PRESSURE_BYTES (3u * 1024u * 1024u) +#define RESIDENT_LIMIT_PAGES 640u + +static uint8_t gBuf[READ_CHUNK]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c[2] = { buf[i], 0 }; + print(c); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("pcache_pressure: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("pcache_pressure: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { + return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); +} + +static uint32_t open_first_large(uint32_t* fileOut, const char** pathOut, uint64_t* sizeOut) { + static const char* candidates[] = { + "1:/apps/video.bk2", + "0:/apps/video.bk2", + "/apps/video.bk2", + "video.bk2", + + "1:/apps/doom1.wad", + "0:/apps/doom1.wad", + "/apps/doom1.wad", + "doom1.wad", + }; + + for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { + uint32_t h = 0; + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(candidates[i]), &h }; + uint32_t s = SysOpenFile(&req); + if (s != SYS_STATUS_SUCCESS) + continue; + + uint64_t size = SysGetFileSize(h); + if (size >= MIN_PRESSURE_BYTES) { + *fileOut = h; + *pathOut = candidates[i]; + *sizeOut = size; + return SYS_STATUS_SUCCESS; + } + + SysCloseHandle(h); + } + + return SYS_STATUS_NOT_FOUND; +} + +int main(void) { + print("pcache_pressure: start\n"); + + SYS_FILE_CACHE_STATS before; + SYS_FILE_CACHE_STATS after; + if (query_cache(&before) != SYS_STATUS_SUCCESS) + fail("initial cache stats query failed"); + + uint32_t file = 0; + const char* path = 0; + uint64_t size = 0; + uint32_t s = open_first_large(&file, &path, &size); + if (s != SYS_STATUS_SUCCESS) + fail_status("open large asset", s); + + print("pcache_pressure: using "); + print(path ? path : ""); + print(" size="); + print_u64(size); + print("\n"); + + uint64_t total = 0; + uint64_t target = size < (8u * 1024u * 1024u) ? size : (8u * 1024u * 1024u); + while (total < target) { + size_t got = 0; + SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(file, gBuf, sizeof(gBuf), &got); + s = SysReadFile(&readReq); + if (s != SYS_STATUS_SUCCESS) + fail_status("read", s); + if (got == 0) + break; + total += got; + } + + SysCloseHandle(file); + + if (total < MIN_PRESSURE_BYTES) + fail("asset read was too short to pressure the cache"); + + if (query_cache(&after) != SYS_STATUS_SUCCESS) + fail("final cache stats query failed"); + + uint64_t loads = after.Loads - before.Loads; + uint64_t hits = after.Hits - before.Hits; + uint64_t drops = after.Drops - before.Drops; + uint64_t raReq = after.ReadAheadRequests - before.ReadAheadRequests; + uint64_t raLoad = after.ReadAheadPagesLoaded - before.ReadAheadPagesLoaded; + uint64_t raThrottle = after.ReadAheadPagesThrottled - before.ReadAheadPagesThrottled; + uint64_t batchReads = after.BatchReads - before.BatchReads; + uint64_t batchPages = after.BatchPagesLoaded - before.BatchPagesLoaded; + + print("pcache_pressure: read bytes="); print_u64(total); + print(" loads+="); print_u64(loads); + print(" hits+="); print_u64(hits); + print(" drops+="); print_u64(drops); + print(" resident="); print_u64(after.ResidentPages); + print(" active="); print_u64(after.ActivePages); + print(" standby="); print_u64(after.StandbyPages); + print(" max="); print_u64(after.MaxResidentPages); + print(" readahead-req+="); print_u64(raReq); + print(" readahead-load+="); print_u64(raLoad); + print(" readahead-throttle+="); print_u64(raThrottle); + print(" batch-reads+="); print_u64(batchReads); + print(" batch-pages+="); print_u64(batchPages); + print("\n"); + + if (loads < 512) + fail("expected at least 512 cache page loads"); + if (after.ResidentPages > RESIDENT_LIMIT_PAGES) + fail("resident page cache exceeded pressure bound"); + if (loads > RESIDENT_LIMIT_PAGES && drops == 0) + fail("cache did not drop clean pages under pressure"); + + print("pcache_pressure: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/pcache_smoke/pcache_smoke.c b/code/user/tests/pcache_smoke/pcache_smoke.c new file mode 100644 index 0000000..53de5ba --- /dev/null +++ b/code/user/tests/pcache_smoke/pcache_smoke.c @@ -0,0 +1,156 @@ +// Read-only page-cache smoke test. +// Verifies that ReadFile and file-backed mappings share the same cached pages. + +#include +#include +#include +#include + +#define TEST_SIZE 8192u + +static uint8_t gA[TEST_SIZE]; +static uint8_t gB[TEST_SIZE]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c = buf[i]; + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(&c, 1) }; + SysWriteConsole(&req); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("pcache_smoke: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t s) { + print("pcache_smoke: "); print(what); print(" failed status="); print_u64(s); print("\n"); + exit_now(1); +} + +static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { + memset(out, 0, sizeof(*out)); + return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); +} + +static uint32_t open_self(uint32_t* out) { + static const char* paths[] = { + "pcache_smoke.exe", + "/apps/pcache_smoke.exe", + "Y:/apps/pcache_smoke.exe", + "0:/apps/pcache_smoke.exe", + "1:/apps/pcache_smoke.exe", + "/apps/shell_native.exe", + "0:/apps/shell_native.exe", + "1:/apps/shell_native.exe", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysOpenFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *out = h; + print("pcache_smoke: using "); print(paths[i]); print("\n"); + return SYS_STATUS_SUCCESS; + } + } + return SYS_STATUS_NOT_FOUND; +} + +int main(void) { + print("pcache_smoke: start\n"); + + SYS_FILE_CACHE_STATS before, afterFirst, afterSecond, afterMap; + uint32_t s = query_cache(&before); + if (s != SYS_STATUS_SUCCESS) fail_status("query initial stats", s); + + uint32_t file = 0; + s = open_self(&file); + if (s != SYS_STATUS_SUCCESS) fail_status("open test file", s); + + size_t gotA = 0; + memset(gA, 0xA5, sizeof(gA)); + SYS_FILE_IO_REQUEST readReqA = SysMakeFileIoRequest(file, gA, sizeof(gA), &gotA); + s = SysReadFile(&readReqA); + if (s != SYS_STATUS_SUCCESS) fail_status("first read", s); + if (gotA < 4096) fail("first read too short"); + print("pcache_smoke: first read OK\n"); + + s = query_cache(&afterFirst); + if (s != SYS_STATUS_SUCCESS) fail_status("query after first read", s); + if (afterFirst.Loads <= before.Loads) + fail("first read did not load file-cache pages"); + + s = SysSeekFile(file, 0); + if (s != SYS_STATUS_SUCCESS) fail_status("seek", s); + + size_t gotB = 0; + memset(gB, 0x5A, sizeof(gB)); + SYS_FILE_IO_REQUEST readReqB = SysMakeFileIoRequest(file, gB, sizeof(gB), &gotB); + s = SysReadFile(&readReqB); + if (s != SYS_STATUS_SUCCESS) fail_status("second read", s); + if (gotB != gotA) fail("second read length mismatch"); + if (memcmp(gA, gB, gotA) != 0) fail("second read data mismatch"); + print("pcache_smoke: second read OK\n"); + + s = query_cache(&afterSecond); + if (s != SYS_STATUS_SUCCESS) fail_status("query after second read", s); + if (afterSecond.Hits <= afterFirst.Hits) + fail("second read did not hit file cache"); + + uint32_t mapping = 0; + s = SysCreateFileMapping(file, SYS_VM_READ, SYS_MAP_FLAG_NONE, sizeof(gA), &mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("create mapping", s); + SysCloseHandle(file); + file = 0; + + void* base = 0; + s = SysMapFile(mapping, 0, sizeof(gA), SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &base); + SysCloseHandle(mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("map file", s); + if (!base) fail("map returned null"); + + volatile uint8_t* view = (volatile uint8_t*)base; + (void)view[0]; + (void)view[4096]; + print("pcache_smoke: mmap faults OK\n"); + + if (memcmp((const void*)base, gA, gotA) != 0) { + SysFreeVm(base); + fail("mapped data mismatch"); + } + + s = query_cache(&afterMap); + if (s != SYS_STATUS_SUCCESS) fail_status("query after map", s); + if (afterMap.Hits <= afterSecond.Hits) + fail("mmap faults did not hit already cached pages"); + + SysFreeVm(base); + + print("pcache_smoke: stats loads="); print_u64(afterMap.Loads - before.Loads); + print(" hits="); print_u64(afterMap.Hits - before.Hits); + print(" resident="); print_u64(afterMap.ResidentPages); print("\n"); + print("pcache_smoke: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/shared_mmap/shared_mmap.c b/code/user/tests/shared_mmap/shared_mmap.c new file mode 100644 index 0000000..8ed5799 --- /dev/null +++ b/code/user/tests/shared_mmap/shared_mmap.c @@ -0,0 +1,216 @@ +// Shared writable mmap regression test. +// Verifies that writable mapped-file views dirty the coherent file cache and flush through the normal file path. + +#include +#include +#include +#include + +#define PAGE_SIZE_BYTES 4096u +#define TEST_SIZE (PAGE_SIZE_BYTES * 3u) + +static uint8_t gInitial[TEST_SIZE]; +static uint8_t gExpected[TEST_SIZE]; +static uint8_t gReadback[TEST_SIZE]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c[2] = { buf[i], 0 }; + print(c); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("shared_mmap: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("shared_mmap: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { + memset(out, 0, sizeof(*out)); + return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); +} + +static void delete_path(const char* path) { + SYS_PATH_REQUEST req = { SysSvFromCstr(path) }; + (void)SysDeleteFile(&req); +} + +static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { + static const char* paths[] = { + "shared_mmap.tmp", + "/shared_mmap.tmp", + "1:/shared_mmap.tmp", + "0:/shared_mmap.tmp", + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { + uint32_t h = 0; + delete_path(paths[i]); + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h }; + uint32_t s = SysCreateFile(&req); + if (s == SYS_STATUS_SUCCESS) { + *fileOut = h; + if (pathOut) *pathOut = paths[i]; + return SYS_STATUS_SUCCESS; + } + } + return SYS_STATUS_NOT_FOUND; +} + +static uint32_t open_path(const char* path, uint32_t* fileOut) { + uint32_t h = 0; + SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), &h }; + uint32_t s = SysOpenFile(&req); + if (s == SYS_STATUS_SUCCESS) + *fileOut = h; + return s; +} + +static void checked_seek(uint32_t file, uint64_t offset, const char* what) { + uint32_t s = SysSeekFile(file, offset); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); +} + +static void checked_write(uint32_t file, const void* buffer, size_t size, const char* what) { + size_t done = 0; + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &done); + uint32_t s = SysWriteFile(&req); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); + if (done != size) fail("short write"); +} + +static void checked_read(uint32_t file, void* buffer, size_t size, size_t* gotOut, const char* what) { + size_t got = 0; + memset(buffer, 0xCC, size); + SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &got); + uint32_t s = SysReadFile(&req); + if (s != SYS_STATUS_SUCCESS) fail_status(what, s); + if (gotOut) *gotOut = got; +} + +static void fill_initial(void) { + for (size_t i = 0; i < TEST_SIZE; ++i) { + gInitial[i] = (uint8_t)(0x20u + (uint8_t)(i * 7u + (i >> 5))); + gExpected[i] = gInitial[i]; + } +} + +int main(void) { + print("shared_mmap: start\n"); + fill_initial(); + + uint32_t file = 0; + const char* path = 0; + uint32_t s = create_temp(&file, &path); + if (s != SYS_STATUS_SUCCESS) fail_status("create temp", s); + + print("shared_mmap: creating temporary file "); + print(path ? path : ""); + print("\n"); + + checked_write(file, gInitial, TEST_SIZE, "initial write"); + s = SysFlushFile(file); + if (s != SYS_STATUS_SUCCESS) fail_status("initial flush", s); + + uint32_t mapping = 0; + s = SysCreateFileMapping(file, SYS_VM_READWRITE, SYS_MAP_FLAG_NONE, TEST_SIZE, &mapping); + if (s != SYS_STATUS_SUCCESS) fail_status("create mapping", s); + + uint8_t* view = 0; + s = SysMapFile(mapping, 0, TEST_SIZE, SYS_MAP_ACCESS_READ | SYS_MAP_ACCESS_WRITE, + SYS_MAP_FLAG_NONE, (void**)&view); + if (s != SYS_STATUS_SUCCESS) fail_status("map shared write", s); + + for (size_t i = 128; i < 128 + 3000; ++i) { + view[i] = (uint8_t)(0xA0u + (uint8_t)i); + gExpected[i] = view[i]; + } + for (size_t i = PAGE_SIZE_BYTES + 17; i < PAGE_SIZE_BYTES + 17 + 2048; ++i) { + view[i] = (uint8_t)(0x50u + (uint8_t)(i >> 1)); + gExpected[i] = view[i]; + } + + SYS_FILE_CACHE_STATS beforeFlush; + SYS_FILE_CACHE_STATS afterFlush; + if (query_cache(&beforeFlush) != SYS_STATUS_SUCCESS) fail("pre-flush stats failed"); + + s = SysFlushFile(file); + if (s != SYS_STATUS_SUCCESS) fail_status("flush mapped writes", s); + + if (query_cache(&afterFlush) != SYS_STATUS_SUCCESS) fail("post-flush stats failed"); + if (afterFlush.MappedDirtyPages <= beforeFlush.MappedDirtyPages) + fail("mapped dirty pages were not harvested"); + if (afterFlush.WritebackBytes <= beforeFlush.WritebackBytes) + fail("mapped writes did not write back"); + + checked_seek(file, 0, "seek after mapped flush"); + size_t got = 0; + checked_read(file, gReadback, TEST_SIZE, &got, "read after mapped flush"); + if (got != TEST_SIZE) fail("read after mapped flush size mismatch"); + if (memcmp(gReadback, gExpected, TEST_SIZE) != 0) fail("mapped flush data mismatch"); + + for (size_t i = PAGE_SIZE_BYTES * 2 + 3; i < PAGE_SIZE_BYTES * 2 + 900; ++i) { + view[i] = (uint8_t)(0xC0u ^ (uint8_t)i); + gExpected[i] = view[i]; + } + + s = SysFreeVm(view); + view = 0; + if (s != SYS_STATUS_SUCCESS) fail_status("unmap", s); + + s = SysCloseHandle(mapping); + mapping = 0; + if (s != SYS_STATUS_SUCCESS) fail_status("close mapping", s); + + s = SysCloseHandle(file); + file = 0; + if (s != SYS_STATUS_SUCCESS) fail_status("close file", s); + + s = open_path(path, &file); + if (s != SYS_STATUS_SUCCESS) fail_status("reopen", s); + + checked_read(file, gReadback, TEST_SIZE, &got, "read after close"); + if (got != TEST_SIZE) fail("read after close size mismatch"); + if (memcmp(gReadback, gExpected, TEST_SIZE) != 0) fail("unmap/close data mismatch"); + + s = SysCloseHandle(file); + file = 0; + if (s != SYS_STATUS_SUCCESS) fail_status("close reopened", s); + delete_path(path); + + print("shared_mmap: mapped-dirty-pages+="); + print_u64(afterFlush.MappedDirtyPages - beforeFlush.MappedDirtyPages); + print(" writeback-bytes+="); + print_u64(afterFlush.WritebackBytes - beforeFlush.WritebackBytes); + print("\n"); + print("shared_mmap: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/surface_invalid_rect/surface_invalid_rect.c b/code/user/tests/surface_invalid_rect/surface_invalid_rect.c new file mode 100644 index 0000000..5c64283 --- /dev/null +++ b/code/user/tests/surface_invalid_rect/surface_invalid_rect.c @@ -0,0 +1,105 @@ +// Native surface validation test. +// Exercises invalid parameters and clipped dirty rectangles without depending on windowing. + +#include +#include +#include +#include + +#define SURFACE_W 32u +#define SURFACE_H 24u + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c = buf[i]; + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(&c, 1) }; + SysWriteConsole(&req); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("surface_invalid_rect: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("surface_invalid_rect: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static void expect_status(const char* what, uint32_t got, uint32_t expected) { + if (got == expected) return; + print("surface_invalid_rect: "); + print(what); + print(" returned "); + print_u64(got); + print(" expected "); + print_u64(expected); + print("\n"); + exit_now(1); +} + +int main(void) { + print("surface_invalid_rect: start\n"); + + uint32_t bad = 0; + uint32_t s = SysCreateSurface(0, SURFACE_H, SYS_SURFACE_FORMAT_XRGB8888, SYS_SURFACE_FLAG_NONE, &bad); + expect_status("zero width", s, SYS_STATUS_INVALID_PARAMETER); + + s = SysCreateSurface(SURFACE_W, SURFACE_H, 0x9999u, SYS_SURFACE_FLAG_NONE, &bad); + expect_status("bad format", s, SYS_STATUS_NOT_SUPPORTED); + + uint32_t surface = 0; + s = SysCreateSurface(SURFACE_W, SURFACE_H, SYS_SURFACE_FORMAT_XRGB8888, + SYS_SURFACE_FLAG_NONE, &surface); + if (s != SYS_STATUS_SUCCESS) fail_status("create surface", s); + + void* base = 0; + uint32_t pitch = 0; + s = SysMapSurface(surface, &base, &pitch); + if (s != SYS_STATUS_SUCCESS) fail_status("map surface", s); + if (!base || pitch < SURFACE_W * sizeof(uint32_t)) fail("invalid mapping"); + + s = SysPresentSurface(surface, 0, 1); + expect_status("null rect array with nonzero count", s, SYS_STATUS_INVALID_PARAMETER); + + DISPLAY_RECT rects[SYS_SURFACE_MAX_RECTS + 1]; + memset(rects, 0, sizeof(rects)); + s = SysPresentSurface(surface, rects, SYS_SURFACE_MAX_RECTS + 1); + expect_status("too many rects", s, SYS_STATUS_INVALID_PARAMETER); + + DISPLAY_RECT clipped = { -8, -8, 24, 24 }; + s = SysPresentSurface(surface, &clipped, 1); + if (s != SYS_STATUS_SUCCESS) fail_status("present clipped rect", s); + + DISPLAY_RECT empty = { 10000, 10000, 16, 16 }; + s = SysPresentSurface(surface, &empty, 1); + if (s != SYS_STATUS_SUCCESS) fail_status("present empty clipped rect", s); + + SysCloseHandle(surface); + print("surface_invalid_rect: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/surface_multithread_present/surface_multithread_present.c b/code/user/tests/surface_multithread_present/surface_multithread_present.c new file mode 100644 index 0000000..a382e64 --- /dev/null +++ b/code/user/tests/surface_multithread_present/surface_multithread_present.c @@ -0,0 +1,157 @@ +// Native surface multithreaded present test. +// Multiple user threads present one process-private surface to catch mapping and scratch-buffer races. + +#include +#include +#include +#include + +#define SURFACE_W 96u +#define SURFACE_H 64u +#define THREAD_COUNT 4u +#define ITERATIONS 32u + +typedef struct WORKER_STATE { + uint32_t Index; + uint32_t Surface; + uint8_t* Base; + uint32_t Pitch; + uint32_t Status; + uint32_t IterationsDone; +} WORKER_STATE; + +static WORKER_STATE gWorkers[THREAD_COUNT]; + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c = buf[i]; + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(&c, 1) }; + SysWriteConsole(&req); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("surface_multithread_present: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("surface_multithread_present: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +static void worker_main(uintptr_t arg) { + WORKER_STATE* st = (WORKER_STATE*)arg; + if (!st) SysExitThread(1); + uint32_t bandHeight = SURFACE_H / THREAD_COUNT; + uint32_t y0 = st->Index * bandHeight; + uint32_t h = bandHeight; + if (st->Index == THREAD_COUNT - 1) + h = SURFACE_H - y0; + + for (uint32_t iter = 0; iter < ITERATIONS; ++iter) { + for (uint32_t y = 0; y < h; ++y) { + uint32_t* row = (uint32_t*)(st->Base + (size_t)(y0 + y) * st->Pitch); + for (uint32_t x = 0; x < SURFACE_W; ++x) + row[x] = 0x00010101u * ((st->Index + 1u) * 32u + (iter & 31u)); + } + + DISPLAY_RECT rect = { 0, (int32_t)y0, (int32_t)SURFACE_W, (int32_t)h }; + uint32_t s = SysPresentSurface(st->Surface, &rect, 1); + if (s != SYS_STATUS_SUCCESS) { + st->Status = s; + SysExitThread(1); + } + st->IterationsDone++; + SysYield(); + } + + st->Status = SYS_STATUS_SUCCESS; + SysExitThread(0); +} + +int main(void) { + print("surface_multithread_present: start\n"); + + uint32_t surface = 0; + uint32_t s = SysCreateSurface(SURFACE_W, SURFACE_H, SYS_SURFACE_FORMAT_XRGB8888, + SYS_SURFACE_FLAG_NONE, &surface); + if (s != SYS_STATUS_SUCCESS) fail_status("create surface", s); + + void* base = 0; + uint32_t pitch = 0; + s = SysMapSurface(surface, &base, &pitch); + if (s != SYS_STATUS_SUCCESS) fail_status("map surface", s); + if (!base || pitch < SURFACE_W * sizeof(uint32_t)) fail("invalid mapping"); + + uint32_t handles[THREAD_COUNT]; + uint32_t tids[THREAD_COUNT]; + memset(handles, 0, sizeof(handles)); + memset(tids, 0, sizeof(tids)); + memset(gWorkers, 0, sizeof(gWorkers)); + + for (uint32_t i = 0; i < THREAD_COUNT; ++i) { + gWorkers[i].Index = i; + gWorkers[i].Surface = surface; + gWorkers[i].Base = (uint8_t*)base; + gWorkers[i].Pitch = pitch; + gWorkers[i].Status = SYS_STATUS_PENDING; + SYS_CREATE_THREAD_REQUEST req = { + (uintptr_t)worker_main, + (uintptr_t)&gWorkers[i], + 1, + 0, + &handles[i], + &tids[i] + }; + s = SysCreateThread(&req); + if (s != SYS_STATUS_SUCCESS) fail_status("create thread", s); + } + + for (uint32_t i = 0; i < THREAD_COUNT; ++i) { + s = SysWait(handles[i], SYS_WAIT_FLAG_NONE); + if (s != SYS_STATUS_SUCCESS) fail_status("wait thread", s); + SysCloseHandle(handles[i]); + } + + for (uint32_t i = 0; i < THREAD_COUNT; ++i) { + if (gWorkers[i].Status != SYS_STATUS_SUCCESS) { + print("surface_multithread_present: worker "); + print_u64(i); + print(" status="); + print_u64(gWorkers[i].Status); + print(" iterations="); + print_u64(gWorkers[i].IterationsDone); + print("\n"); + exit_now(1); + } + if (gWorkers[i].IterationsDone != ITERATIONS) fail("worker iteration count mismatch"); + } + + SysCloseHandle(surface); + print("surface_multithread_present: PASS\n"); + exit_now(0); + return 0; +} diff --git a/code/user/tests/surface_smoke/surface_smoke.c b/code/user/tests/surface_smoke/surface_smoke.c new file mode 100644 index 0000000..c8304b0 --- /dev/null +++ b/code/user/tests/surface_smoke/surface_smoke.c @@ -0,0 +1,93 @@ +// Native surface smoke test. +// Verifies that process-private surface handles can be created, mapped, queried, presented, and closed. + +#include +#include +#include +#include + +#define SURFACE_W 64u +#define SURFACE_H 48u + +static void print(const char* s) { + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; + SysWriteConsole(&req); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { print("0"); return; } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c = buf[i]; + SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(&c, 1) }; + SysWriteConsole(&req); + } +} + +static void exit_now(int code) { + for (;;) SysExitProcess(code); +} + +static void fail(const char* msg) { + print("surface_smoke: "); + print(msg); + print("\n"); + exit_now(1); +} + +static void fail_status(const char* what, uint32_t status) { + print("surface_smoke: "); + print(what); + print(" failed status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +int main(void) { + print("surface_smoke: start\n"); + + uint32_t surface = 0; + uint32_t s = SysCreateSurface(SURFACE_W, SURFACE_H, SYS_SURFACE_FORMAT_XRGB8888, + SYS_SURFACE_FLAG_NONE, &surface); + if (s != SYS_STATUS_SUCCESS) fail_status("create surface", s); + if (!surface) fail("surface handle is zero"); + + void* base = 0; + uint32_t pitch = 0; + s = SysMapSurface(surface, &base, &pitch); + if (s != SYS_STATUS_SUCCESS) fail_status("map surface", s); + if (!base || pitch < SURFACE_W * sizeof(uint32_t)) fail("invalid mapping"); + + SYS_SURFACE_DESC desc; + memset(&desc, 0, sizeof(desc)); + s = SysQuerySurface(surface, &desc); + if (s != SYS_STATUS_SUCCESS) fail_status("query surface", s); + if (desc.Width != SURFACE_W || desc.Height != SURFACE_H) fail("query dimensions mismatch"); + if (desc.PitchBytes != pitch || desc.Format != SYS_SURFACE_FORMAT_XRGB8888) fail("query metadata mismatch"); + if (desc.Base != base) fail("query base mismatch"); + + uint8_t* rows = (uint8_t*)base; + for (uint32_t y = 0; y < SURFACE_H; ++y) { + uint32_t* row = (uint32_t*)(rows + (size_t)y * pitch); + for (uint32_t x = 0; x < SURFACE_W; ++x) + row[x] = 0x00202020u + ((x & 0xffu) << 16) + ((y & 0xffu) << 8); + } + + DISPLAY_RECT rect = { 0, 0, (int32_t)SURFACE_W, (int32_t)SURFACE_H }; + s = SysPresentSurface(surface, &rect, 1); + if (s != SYS_STATUS_SUCCESS) fail_status("present surface", s); + + s = SysPresentSurface(surface, 0, 0); + if (s != SYS_STATUS_SUCCESS) fail_status("present full surface", s); + + SysCloseHandle(surface); + print("surface_smoke: PASS\n"); + exit_now(0); + return 0; +} diff --git a/docs.md b/docs.md new file mode 100644 index 0000000..80605ce --- /dev/null +++ b/docs.md @@ -0,0 +1,93 @@ +# nt-lite development notes + +nt-lite is a clean-design hobby kernel. It is not a stable compatibility platform. Prefer the right design over preserving old APIs, shims, or compatibility aliases. + +## Principles + +- Break old code when the new model is better. +- Do not keep compatibility paths unless they have a clear purpose. +- Plan the model before writing the patch. +- Make one architectural move at a time. +- Delete stale paths once the real path works. +- Prefer compile failures over wrappers that hide stale users. +- Keep the tree internally consistent. +- Keep one canonical API for each operation. +- A small patch can still represent the final model. +- Do not land temporary architecture just to make a feature appear to work. +- Put policy in the layer that owns it. +- Shared state should have one clear owner. + +## Workflow + +- Keep patches reviewable. +- Boot-test meaningful changes. +- Add diagnostics before optimizing. +- Use logs, counters, and assertions instead of guessing. +- Remove stale comments and dead code as part of the same cleanup. +- Treat a fallback as either a supported mode or delete it before it becomes permanent. +- Ask design questions before changing API shape, object lifetime, ownership, cancellation, persistence, or compatibility behavior. +- Do not add a side path to avoid updating callers. +- When changing an interface, update every caller in the same patch. +- When adding a build artifact, update build, packaging, copy, and runtime discovery paths in the same patch. +- Do not count apply checks or syntax-only checks as a real build. +- Be precise about what was actually validated. +- If behavior can be tested cheaply, include a focused runtime test or a visible diagnostic. +- Prefer counters that prove both success and non-use. + +## Breaking changes + +Breaking changes are allowed. When making one: + +1. Define the new model. +2. Move every user to it. +3. Delete the old path. +4. Bump ABI or version constants when a contract changes. +5. Let stale users fail to compile. + +## Patch expectations for AI agents + +- Generate patches from the repository root. +- Paths must be rooted as `code/...`. +- `git apply ` should work from the repository root. +- Do not require `--directory=code`. +- Keep generated artifacts out of the source tree unless requested. +- When asked to provide the full tree, also provide the patch. +- The patch is for review; the full tree is for reproducing the exact baseline. +- Avoid optimistic claims. +- Say exactly whether a change was apply-checked, syntax-checked, built, booted, or runtime-tested. +- Do not introduce compatibility aliases unless they are intentionally supported API. + +## Comment style + +- Comments explain why, not what the next line already says. +- Use `//` comments only. +- Use sentence case. +- End every sentence with a period. +- Every owned source file should start with a short file comment. +- Every function should have a comment, even if it is only a one-line summary. +- Keep the tone practical and technical. +- Humor is allowed rarely, but never instead of the real explanation. +- Debugging history belongs in comments only when it explains a lasting constraint. +- Document workarounds by saying what looks strange, why it is necessary, and what breaks if it is simplified. +- Avoid trailing comments except for flags, bitfields, register tables, and packed layouts. +- Keep real limitations as normal prose. + +## Development lessons + +- Ask before changing the shape of an API or the ownership model behind it. +- The right model matters more than minimizing the diff. +- A patch should be small enough to review, but not so small that it creates a temporary model. +- If a compatibility path is not meant to stay, do not add it. +- If the old path is replaced, remove it. +- Ergonomics matter. A clean API should not make common calls annoying. +- Debug by evidence, not by the subsystem that changed most recently. +- Poison values, fault addresses, counters, and reproduction steps are more useful than guesses. +- Performance fixes need tail-latency checks, not only average throughput. +- Diagnostics must reach every active output path that developers rely on. +- Runtime tests should clean up after themselves and say clearly when they use temporary data. +- A passing test should leave the system in a sane state, not only print `PASS`. +- Build integration is part of the feature. +- Packaging mistakes are real bugs, not clerical issues. +- When a change crosses layers, verify every layer that knows about the new object, file, syscall, or artifact. +- Prefer finishing the feature the user needs over endlessly polishing the audit around it. +- Do not hide uncertainty. Say what was checked and what still needs a real build or boot test. diff --git a/image.png b/image.png index b6a6d87..3a000b3 100644 Binary files a/image.png and b/image.png differ diff --git a/readme.md b/readme.md index 850abe9..b5ac9f6 100644 --- a/readme.md +++ b/readme.md @@ -192,10 +192,10 @@ NT-Lite now has a TCP/IP stack via lwIP running as a runtime driver. The kernel - `SysSocket` / `SysBind` / `SysListen` / `SysAccept` / `SysConnect` - `SysSend` / `SysRecv` / `SysSendTo` / `SysRecvFrom` - `SysShutdown` / `SysPoll` -- `SysResolveHost` for DNS +- `SysResolveHost` for DNS (wow!) - `SysSetSocketOption` / `SysGetSocketOption` -Host byte order. Cry some more. +Host byte order. It's convenient. Cry some more. Test apps ship with the build: `httpget.exe`, `tcpserve.exe`, `udp.exe`, `resolve.exe`, and `ping.exe`. diff --git a/runa.bat b/runa.bat index 7969a4d..914b74f 100644 --- a/runa.bat +++ b/runa.bat @@ -7,7 +7,7 @@ -device ide-hd,bus=ahci.0,drive=disk0 ^ -serial stdio ^ -M q35 ^ --m 256M ^ +-m 128M ^ -smp cpus=4,cores=4,threads=1,sockets=1 ^ -netdev user,id=net0,hostfwd=tcp::8080-:8080 ^ -device rtl8139,netdev=net0 ^