maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

Browse Source
This commit is contained in:
maksw committed 2026-06-20 13:17:39 +02:00
1 parent 61ec2442eb
commit 2c4a68a75f
114 files changed
+18069 -2620
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+4 -1
View File
@@ -2,6 +2,9 @@ build.obj
bin/
obj/
qemu.log
*.vhd
disk.vhd
usb.vhd
disk.vmdk
code.zip
code/build/build.exe
code/build/build.exe.old
OVMF/OVMF_VARS_4M.fd
BIN
View File
Binary file not shown.
OVMF/OVMF_VARS_4M_d.fd
BIN
View File
Binary file not shown.
code/boot.ini
+1 -1
View File
@@ -1,5 +1,5 @@
[kernel]
; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file
debug = 1
debug = 0xffff
[init]
commandline = apps/shell_native.exe
code/boot/uefi/main.c
+16 -15
View File
@@ -291,7 +291,7 @@ static int ParseBootIniInteger(const char* val) {
static void BootIniCb(const char* section, const char* key,
const char* val, void* ud) {
PARSE_STATE* s = (PARSE_STATE *)ud;
// [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log.
// [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log, 8+=kernel diagnostics.
if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug"))
s->debug = ParseBootIniInteger(val);
// [init] commandline = hello.exe.
@@ -446,7 +446,7 @@ EFI_STATUS LoadPeImage(UINT8* rawFile, UINT64 virtDest, EFI_PHYSICAL_ADDRESS* ou
IMAGE_DATA_DIRECTORY* dataDirs = (IMAGE_DATA_DIRECTORY*)(ntHeaders + 24 + 112);
IMAGE_DATA_DIRECTORY* relocDir = &dataDirs[5];
//IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1];.
//IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1];
// Allocate Physical Memory for the mapped image.
UINTN pages = (sizeOfImage + PAGE_SIZE - 1) / PAGE_SIZE;
@@ -619,7 +619,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
return Status;
}
//Print(L"1\n");.
Print(L"1\n");
// boot.ini controls debugger and runtime policy before the kernel has a filesystem service.
UINTN sz;
@@ -632,7 +632,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
FreePool(bootIniRaw);
}
//Print(L"2\n");.
Print(L"2\n");
// drivers.ini decides which boot and runtime drivers are loaded into the initial module list.
UINT8* drvIniRaw = LoadFile(L"drivers\\drivers.ini", &sz);
@@ -643,7 +643,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
FreePool(drvIniRaw);
}
//Print(L"3\n");.
Print(L"3\n");
BOOT_INFO* bi = AllocateZeroPool(sizeof(BOOT_INFO));
bi->Magic = BOOT_INFO_MAGIC;
@@ -675,7 +675,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
// Current mapping cursor for upper half.
UINT64 virtCursor = KERNEL_VIRT_BASE;
//Print(L"4\n");.
Print(L"4\n");
// Core images are loaded first because driver import resolution depends on them.
typedef struct {
@@ -742,7 +742,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
while(true);
}
//Print(L"5\n");.
Print(L"5\n");
// Load driver .sys files into raw buffers.
for (int i = 0; i < ps.driverCount; i++) {
@@ -751,9 +751,10 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
UINTN imgSz;
UINT8* buf = LoadFile(path, &imgSz);
if (!buf) {
Print(L"Driver not found: %s\n", ps.drivers[i]);
continue;
if (!buf) {
Print(L"Fatal: boot driver not found: %s\n", ps.drivers[i]);
while (true);
return EFI_NOT_FOUND;
}
// Store file info for the kernel to process later.
@@ -767,7 +768,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
mod->VirtBase = 0; // Kernel will map this later
}
//Print(L"6\n");.
Print(L"6\n");
// Query GOP FIRST, before the GetMemoryMap loop.
EFI_GRAPHICS_OUTPUT_PROTOCOL* gop;
@@ -795,7 +796,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
}
}
//Print(L"7\n");.
Print(L"7\n");
bi->ApTrampolinePhys = (uintptr_t)AllocateApTrampolinePageBelow1M();
@@ -895,7 +896,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // Physical Window Map, P|W|PCD
}
//Print(L"8\n");.
Print(L"8\n");
// The kernel consumes a compact memory map so it does not need EFI descriptor semantics after boot.
ConvertMemoryMap(&bi->MemoryMap);
@@ -940,12 +941,12 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
}
// Print it out to verify the bootloader actually found it before launching the kernel
//Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);.
Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);
bi->PhysicalMapBase = PHYS_MAP_BASE;
bi->Debug = (UINT32)ps.debug;
//Print(L"9\n");.
Print(L"9\n");
// ExitBootServices is the point of no return for firmware allocation, file I/O, and protocol access.
gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer);
code/build/build.c
+67 -4
View File
@@ -30,6 +30,7 @@ static const Driver_Target PCI_BUS_DRIVER =
static const Driver_Target STANDALONE_DRIVERS[] = {
DRIVER_TARGET("ps2", "obj\\drivers\\ps2", "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"),
DRIVER_TARGET("serial", "obj\\drivers\\serial", "code\\drivers\\input\\serial\\serial.c"),
DRIVER_TARGET_LIBS("ata", "obj\\drivers\\ata", "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"),
DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
@@ -46,7 +47,10 @@ void create_layout() {
"obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init",
"obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video"
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop",
"obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present"
};
for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) nob_mkdir_if_not_exists(dirs[i]);
nob_mkdir_if_not_exists(PCI_BUS_DRIVER.obj_dir);
@@ -71,6 +75,41 @@ Nob_Proc run_async(const char *cmd_str) {
return proc;
}
// Deletes stale object files from a target directory before compiling into it.
// Link commands use directory wildcards, so removed translation units can otherwise
// survive as old .obj files and keep unresolved references alive.
bool clean_object_directory(const char *obj_dir) {
const char *cmd = nob_temp_sprintf("if exist %s\\*.obj del /q %s\\*.obj", obj_dir, obj_dir);
return run_sync(cmd);
}
// Cleans object directories that are linked through wildcards.
// The build is intentionally simple and does not maintain per-file dependency state,
// so stale objects are more dangerous than recompiling a few extra files.
bool clean_build_object_directories(void) {
const char *dirs[] = {
"obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
"obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix",
"obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping",
"obj\\doom", "obj\\video", "obj\\memtop", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present"
};
for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) {
if (!clean_object_directory(dirs[i])) return false;
}
if (!clean_object_directory(PCI_BUS_DRIVER.obj_dir)) return false;
for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) {
if (!clean_object_directory(STANDALONE_DRIVERS[i].obj_dir)) return false;
}
return true;
}
// Creates an empty USB image the first time the build sees a missing usb.vhd.
// The image is intentionally left unmounted and unpopulated after formatting so
// callers can use it as removable FAT32 storage without the build overwriting it.
@@ -111,6 +150,7 @@ int main(int argc, char **argv) {
}
if (!create_usb_vhd_if_missing()) return 1;
create_layout();
if (!clean_build_object_directories()) return 1;
// Core images build sequentially because later images link against earlier import libraries.
nob_log(NOB_INFO, "--- Building Core Subsystems ---");
@@ -196,6 +236,17 @@ int main(int argc, char **argv) {
{"apps\\shell_native", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "shell_native", "/ENTRY:main"},
{"tests\\async_apc", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_apc", "/ENTRY:main"},
{"tests\\async_cancel", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_cancel", "/ENTRY:main"},
{"tests\\mmap_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "mmap_smoke", "/ENTRY:main"},
{"tests\\pcache_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_smoke", "/ENTRY:main"},
{"tests\\pcache_pressure", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_pressure", "/ENTRY:main"},
{"tests\\dirty_writeback", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "dirty_writeback", "/ENTRY:main"},
{"tests\\shared_mmap", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "shared_mmap", "/ENTRY:main"},
{"tests\\pcache_invalidate", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_invalidate", "/ENTRY:main"},
{"tests\\pcache_mmap_read_reuse", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_mmap_read_reuse", "/ENTRY:main"},
{"tests\\pcache_multithread", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_multithread", "/ENTRY:main"},
{"tests\\surface_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_smoke", "/ENTRY:main"},
{"tests\\surface_invalid_rect", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_invalid_rect", "/ENTRY:main"},
{"tests\\surface_multithread_present", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "surface_multithread_present", "/ENTRY:main"},
{"apps\\edit", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "edit", "/ENTRY:main"},
{"apps\\httpget", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "httpget", "/ENTRY:main"},
{"apps\\tcpserve", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "tcpserve", "/ENTRY:main"},
@@ -203,7 +254,8 @@ int main(int argc, char **argv) {
{"apps\\resolve", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "resolve", "/ENTRY:main"},
{"apps\\ping", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "ping", "/ENTRY:main"},
{"apps\\doom", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "doom", "/ENTRY:main"},
{"apps\\video", "bin\\win32.lib obj\\libc.obj code\\user\\apps\\video\\bink2w64.lib", "video", "/ENTRY:main"}
{"apps\\video", "bin\\syscall.lib obj\\libc.obj code\\user\\apps\\video\\bink2w64.lib", "video", "/ENTRY:main"},
{"apps\\memtop", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "memtop", "/ENTRY:main"}
};
for (size_t i = 0; i < NOB_ARRAY_LEN(apps); i++) {
@@ -248,6 +300,7 @@ int main(int argc, char **argv) {
{"code\\settings.ini", "Z:\\system\\settings.ini"}, {"bin\\init.exe", "Z:\\system\\init.exe"},
{"bin\\pci.sys", "Z:\\drivers\\pci.sys"}, {"bin\\ps2.sys", "Z:\\drivers\\ps2.sys"},
{"bin\\serial.sys", "Z:\\drivers\\serial.sys"},
{"bin\\ata.sys", "Z:\\drivers\\ata.sys"}, {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"},
{"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"}, {"bin\\vmwaresvga.sys", "Z:\\drivers\\vmwaresvga.sys"},
{"bin\\lwip.sys", "Z:\\drivers\\lwip.sys"}, {"bin\\usb.sys", "Z:\\drivers\\usb.sys"},
@@ -260,9 +313,19 @@ int main(int argc, char **argv) {
{"bin\\httpget.exe", "Z:\\apps\\httpget.exe"}, {"bin\\tcpserve.exe", "Z:\\apps\\tcpserve.exe"},
{"bin\\udp.exe", "Z:\\apps\\udp.exe"}, {"bin\\resolve.exe", "Z:\\apps\\resolve.exe"},
{"bin\\ping.exe", "Z:\\apps\\ping.exe"}, {"bin\\doom.exe", "Z:\\apps\\doom.exe"},
{"bin\\video.exe", "Z:\\apps\\video.exe"},
{"bin\\video.exe", "Z:\\apps\\video.exe"}, {"bin\\memtop.exe", "Z:\\apps\\memtop.exe"},
{"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"}, {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"}
{"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"}, {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"},
{"bin\\mmap_smoke.exe", "Y:\\apps\\mmap_smoke.exe"}, {"bin\\pcache_smoke.exe", "Y:\\apps\\pcache_smoke.exe"},
{"bin\\pcache_pressure.exe", "Y:\\apps\\pcache_pressure.exe"},
{"bin\\dirty_writeback.exe", "Y:\\apps\\dirty_writeback.exe"},
{"bin\\shared_mmap.exe", "Y:\\apps\\shared_mmap.exe"},
{"bin\\pcache_invalidate.exe", "Y:\\apps\\pcache_invalidate.exe"},
{"bin\\pcache_mmap_read_reuse.exe", "Y:\\apps\\pcache_mmap_read_reuse.exe"},
{"bin\\pcache_multithread.exe", "Y:\\apps\\pcache_multithread.exe"},
{"bin\\surface_smoke.exe", "Y:\\apps\\surface_smoke.exe"},
{"bin\\surface_invalid_rect.exe", "Y:\\apps\\surface_invalid_rect.exe"},
{"bin\\surface_multithread_present.exe", "Y:\\apps\\surface_multithread_present.exe"}
};
for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) nob_copy_file(copies[i][0], copies[i][1]);
code/drivers/bus/pci/pci.c
+24 -1
View File
@@ -13,6 +13,7 @@
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_irq.h>
#include <kernel/hal/hal_apic.h>
int _fltused; // MSVC floating-point helper anchor required by printf.obj
@@ -540,7 +541,7 @@ __declspec(dllexport) KSTATUS PciMapBar(DEVICE_OBJECT* pdo, uint8_t index, uint6
return st;
for (uint64_t off = 0; off < mapLength; off += PAGE_SIZE) {
st = MmMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmMapFailIfPresent);
st = MmPteMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmPteMapFailIfPresent);
if (!KSUCCESS(st))
return st;
}
@@ -659,6 +660,28 @@ __declspec(dllexport) KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot)
return STATUS_SUCCESS;
}
// Enables a single MSI vector, allocating the nt-lite IRQ slot internally.
__declspec(dllexport) KSTATUS PciEnableMsiAuto(DEVICE_OBJECT* pdo, uint8_t* outIrqSlot) {
PCI_PDO_EXTENSION* ext;
if (!outIrqSlot || !PciIsPdo(pdo, &ext))
return STATUS_INVALID_PARAMETER;
if (!PciFindCapability(ext, PCI_CAP_ID_MSI))
return STATUS_NOT_SUPPORTED;
uint8_t irq = 0xFFu;
KSTATUS st = KeIrqAllocateMsiVector(&irq);
if (!KSUCCESS(st))
return st;
st = PciEnableMsi(pdo, irq);
if (!KSUCCESS(st))
return st;
*outIrqSlot = irq;
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal) return STATUS_INVALID_PARAMETER;
code/drivers/bus/usb/usb.h
+1
View File
@@ -21,6 +21,7 @@
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#ifndef USB_DEBUG_VERBOSE
code/drivers/bus/usb/usb_xhci.c
+36 -20
View File
@@ -95,9 +95,9 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
// hid this because it had simple low memory; laptop firmware does not.
PHYSICAL_ADDRESS phys = 0;
if (pages == 1) {
phys = MmAllocateFrame();
phys = MmPfnAllocate(MmPfnUseDma);
} else {
phys = MmAllocateContiguous(pages);
phys = MmPfnAllocateContiguous(pages, MmPfnUseDma);
}
if (!phys) {
@@ -1930,21 +1930,31 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return STATUS_DEVICE_ERROR;
}
st = PciGetInterrupt(xhci->Pdo, &xhci->Irq);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI PciGetInterrupt failed st=0x%x\n", st);
return st;
}
if (xhci->Irq == 0xFFu) {
DbgPrint("usb: xHCI PCI %02x:%02x.%x has no INTx IRQ; MSI is not in pci.sys yet\n",
xhci->Bus, xhci->Dev, xhci->Fn);
return STATUS_NOT_SUPPORTED;
xhci->Irq = 0xFFu;
xhci->UsingMsi = false;
st = PciEnableMsiAuto(xhci->Pdo, &xhci->Irq);
if (KSUCCESS(st)) {
xhci->UsingMsi = true;
DbgPrint("usb: xHCI using MSI IRQ slot %u\n", xhci->Irq);
} else {
KSTATUS intxStatus = PciGetInterrupt(xhci->Pdo, &xhci->Irq);
if (KSUCCESS(intxStatus) && xhci->Irq != 0xFFu) {
DbgPrint("usb: xHCI using legacy INTx IRQ %u\n", xhci->Irq);
} else {
xhci->Irq = 0xFFu;
DbgPrint("usb: xHCI no MSI/INTx interrupt available; using polled event-ring mode (msi=0x%x intx=0x%x)\n",
st, intxStatus);
}
}
st = PciEnableDevice(xhci->Pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX);
uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER;
if (xhci->UsingMsi || xhci->Irq == 0xFFu)
enableFlags |= PCI_DISABLE_INTX;
else
enableFlags |= PCI_ENABLE_INTX;
st = PciEnableDevice(xhci->Pdo, enableFlags);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI PciEnableDevice failed st=0x%x\n", st);
return st;
@@ -2056,12 +2066,18 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return st;
}
KeIrqRegister(xhci->Irq, XhciIsr);
XhciEnableInterrupts(xhci);
xhci->InterruptsEnabled = true;
if (xhci->Irq != 0xFFu) {
KeIrqRegister(xhci->Irq, XhciIsr);
XhciEnableInterrupts(xhci);
xhci->InterruptsEnabled = true;
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
DbgPrint("usb: xHCI ISR registered on IRQ %u (%s); event ring handled by DPC\n",
xhci->Irq, xhci->UsingMsi ? "MSI" : "INTx");
} else {
xhci->InterruptsEnabled = false;
DbgPrint("usb: xHCI started in polling mode; synchronous waits drain event ring\n");
}
xhci->Fdo->PnpState = DevicePnpStarted;
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
DbgPrint("usb: xHCI ISR registered on IRQ %u; event ring handled by DPC\n", xhci->Irq);
st = UsbHubScheduleRootScan(&xhci->Hcd);
if (!KSUCCESS(st))
code/drivers/display/vmwaresvga/vmwaresvga.c
+1 -7
View File
@@ -890,19 +890,13 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT*
result = syncSt;
}
if (v->HardwareCursor && v->CursorVisible)
VmwSvgaCursorCommit(v);
return result;
}
// Implements the display Sync operation.
static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
KSTATUS st = VmwSvgaSyncDevice(v);
if (v && v->HardwareCursor && v->CursorVisible)
VmwSvgaCursorCommit(v);
return st;
return VmwSvgaSyncDevice(v);
}
// Implements the display Set Cursor Image operation.
code/drivers/drivers.ini
+1
View File
@@ -1,6 +1,7 @@
[boot]
pci.sys
ps2.sys
serial.sys
ahci.sys
ata.sys
usb.sys
code/drivers/fs/fat32/fat32.c
+563 -79
View File
@@ -128,12 +128,79 @@ typedef struct {
static FAT_CACHE_ENTRY gFatCache[FAT_CACHE_SLOTS];
#define FAT32_PATH_CACHE_SLOTS 64u
#define FAT32_PATH_CACHE_MAX 512u
#define FAT32_CHAIN_CACHE_SLOTS 16u
#define FAT32_WRITE_RUN_MAX_SECTORS 128u
#define FAT32_READ_RUN_MAX_SECTORS 512u
typedef struct FAT32_PATH_CACHE_ENTRY {
bool Valid;
uint64_t Generation;
uint32_t Hash;
size_t PathLength;
char Path[FAT32_PATH_CACHE_MAX];
uint32_t Cluster;
uint32_t Size;
uint8_t Attr;
uint64_t DirentLba;
uint32_t DirentOffset;
} FAT32_PATH_CACHE_ENTRY;
typedef struct FAT32_CHAIN_CACHE_ENTRY {
bool Valid;
uint32_t LogicalStart;
uint32_t PhysicalStart;
uint32_t Length;
uint32_t NextCluster;
} FAT32_CHAIN_CACHE_ENTRY;
// The FAT sector cache is global and direct-mapped. mmap-backed video can fault
// file pages from multiple Bink worker threads at the same time; without
// serialization one thread can evict/reload a cache slot while another thread is
// still reading a FAT entry from that slot, corrupting cluster-chain traversal.
// Use a sleepable mutex rather than a spinlock because cache misses perform disk
// I/O. The static mutex avoids allocating an object from the paging path.
static MUTEX gFatCacheMutex;
static SPINLOCK gFatCacheMutexInitLock = SPINLOCK_INIT;
static int gFatCacheMutexReady;
static void FatCacheEnsureMutex(void) {
if (gFatCacheMutexReady)
return;
uintptr_t flags = KeAcquireSpinlock(&gFatCacheMutexInitLock);
if (!gFatCacheMutexReady) {
memset(&gFatCacheMutex, 0, sizeof(gFatCacheMutex));
gFatCacheMutex.Header.Type = ObjectTypeMutex;
gFatCacheMutex.Header.RefCount = 1;
gFatCacheMutex.Header.Destroy = (void*)0;
gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
gFatCacheMutex.Owner = (struct THREAD*)0;
LIST_INIT(&gFatCacheMutex.WaitList);
gFatCacheMutexReady = 1;
}
KeReleaseSpinlock(&gFatCacheMutexInitLock, flags);
}
static int FatCacheAcquire(void) {
if (!KeCanWait())
return 0;
FatCacheEnsureMutex();
return KSUCCESS(KeAcquireMutex(&gFatCacheMutex));
}
static void FatCacheRelease(int held) {
if (held)
KeReleaseMutex(&gFatCacheMutex);
}
// Raw (non-cached) sector I/O — forward declarations.
static KSTATUS ReadSectorsRaw (FAT32_VOL *v, uint64_t lba, uint32_t count, void *buf);
static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, const void *buf);
// Read one FAT sector through the cache.
// Returns a pointer into the cache buffer (valid until the next eviction).
// Caller must hold the FAT cache mutex while using the returned pointer.
static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) {
uint32_t slot = (uint32_t)(lba % FAT_CACHE_SLOTS);
FAT_CACHE_ENTRY *e = &gFatCache[slot];
@@ -170,11 +237,13 @@ static void FatCacheMarkDirty(FAT32_VOL *v, uint64_t lba) {
// Implements the fat Cache Invalidate operation.
static void FatCacheInvalidate(void) {
int held = FatCacheAcquire();
for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
gFatCache[i].vol = (FAT32_VOL*)0;
gFatCache[i].valid = false;
gFatCache[i].dirty = false;
}
FatCacheRelease(held);
}
// FlushFatCache — write every dirty slot to all FAT copies, clear dirty flag.
@@ -196,6 +265,10 @@ struct FAT32_VOL_S {
uint64_t DataStart; // absolute LBA of cluster 2
uint32_t TotalClusters; // usable data clusters
uint32_t NextFreeHint; // fix #3: start cluster for next free-scan
SPINLOCK PathCacheLock;
uint64_t PathCacheGeneration;
IO_FILESYSTEM_STATS Stats;
FAT32_PATH_CACHE_ENTRY PathCache[FAT32_PATH_CACHE_SLOTS];
};
typedef struct {
uint32_t StartCluster;
@@ -203,6 +276,9 @@ typedef struct {
FAT32_VOL *Vol;
uint64_t DirentLba;
uint32_t DirentOffset;
SPINLOCK ChainCacheLock;
uint32_t ChainCacheClock;
FAT32_CHAIN_CACHE_ENTRY ChainCache[FAT32_CHAIN_CACHE_SLOTS];
} FAT32_FILE;
// Raw sector I/O (bypasses FAT cache; used for file data and cache eviction).
@@ -231,6 +307,7 @@ static uint64_t ClusterToLba(FAT32_VOL *v, uint32_t cluster) {
// FlushFatCache — needs full FAT32_VOL definition.
static void FlushFatCache(FAT32_VOL *v) {
uint64_t fat0Start = (uint64_t)v->PartitionLba + v->ReservedSectors;
int held = FatCacheAcquire();
for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
FAT_CACHE_ENTRY *e = &gFatCache[i];
@@ -246,6 +323,8 @@ static void FlushFatCache(FAT32_VOL *v) {
}
e->dirty = false;
}
FatCacheRelease(held);
}
// FAT read / write (through cache).
@@ -256,9 +335,14 @@ static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) {
+ fatOffset / v->BytesPerSector;
uint32_t offInSec = fatOffset % v->BytesPerSector;
int held = FatCacheAcquire();
uint8_t *sec;
if (!KSUCCESS(FatCacheRead(v, sector, &sec))) return 0;
return *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
KSTATUS s = FatCacheRead(v, sector, &sec);
uint32_t value = 0;
if (KSUCCESS(s))
value = *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
FatCacheRelease(held);
return value;
}
// Updates fat Entry.
@@ -268,19 +352,24 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) {
+ fatOffset / v->BytesPerSector;
uint32_t offInSec = fatOffset % v->BytesPerSector;
int held = FatCacheAcquire();
uint8_t *sec;
KSTATUS s = FatCacheRead(v, sector, &sec);
if (!KSUCCESS(s)) return s;
if (!KSUCCESS(s)) {
FatCacheRelease(held);
return s;
}
uint32_t existing = *(uint32_t *)(sec + offInSec);
*(uint32_t *)(sec + offInSec) = (existing & 0xF0000000u)
| (value & 0x0FFFFFFFu);
FatCacheMarkDirty(v, sector);
FatCacheRelease(held);
return STATUS_SUCCESS;
}
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan).
static uint32_t AllocCluster(FAT32_VOL *v) {
static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) {
if (v->TotalClusters == 0) return 0;
uint32_t total = v->TotalClusters;
@@ -295,12 +384,18 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
if (!KSUCCESS(WriteFatEntry(v, c, FAT32_EOC_MARK))) return 0;
v->NextFreeHint = c + 1;
// Zero the cluster (required for directory correctness).
uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (zero) {
memset(zero, 0, v->BytesPerCluster);
WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero);
MmFreePool(zero);
// Normal file growth zeroes newly allocated clusters so old
// disk contents are never exposed. Swap/pagefile creation is
// the one exception: every allocated slot is overwritten before
// it can ever be read back, so skipping these writes avoids a
// slow boot-time full-pagefile zero pass.
if (zeroCluster) {
uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (zero) {
memset(zero, 0, v->BytesPerCluster);
WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero);
MmFreePool(zero);
}
}
return c;
}
@@ -309,14 +404,23 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
return 0;
}
// Appends cluster.
static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
uint32_t newCluster = AllocCluster(v);
static uint32_t AllocCluster(FAT32_VOL *v) {
return AllocClusterEx(v, 1);
}
static uint32_t AppendClusterEx(FAT32_VOL *v, uint32_t lastCluster, int zeroCluster) {
uint32_t newCluster = AllocClusterEx(v, zeroCluster);
if (!newCluster) return 0;
if (!KSUCCESS(WriteFatEntry(v, lastCluster, newCluster))) return 0;
return newCluster;
}
// Appends cluster.
static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
return AppendClusterEx(v, lastCluster, 1);
}
// Returns chain End.
static uint32_t GetChainEnd(FAT32_VOL *v, uint32_t start, uint32_t *outCount) {
uint32_t cluster = start, prev = start, count = 0;
@@ -973,6 +1077,302 @@ static int MatchName83V(const char *dirName, const char *dirExt,
return 1;
}
// Returns the path cache canonical byte used for hashes and comparisons.
static char Fat32PathCacheChar(char c) {
if (c == '/') return '\\';
if (c >= 'A' && c <= 'Z') return (char)(c + 32);
return c;
}
// Hashes a drive-relative path using the same case/separator rules as lookup.
static uint32_t Fat32PathCacheHash(SYS_STRING_VIEW path) {
uint32_t h = 2166136261u;
for (size_t i = 0; i < path.Length; i++) {
h ^= (uint8_t)Fat32PathCacheChar(path.Data[i]);
h *= 16777619u;
}
return h ? h : 1u;
}
// Tests whether the cached path names the same FAT path as the caller's view.
static bool Fat32PathCachePathEquals(const FAT32_PATH_CACHE_ENTRY* e, SYS_STRING_VIEW path) {
if (!e || e->PathLength != path.Length) return false;
for (size_t i = 0; i < path.Length; i++) {
if (Fat32PathCacheChar(e->Path[i]) != Fat32PathCacheChar(path.Data[i]))
return false;
}
return true;
}
// Adds directory search counters without holding the lock during disk I/O.
static void Fat32PathCacheAddDirectoryStats(FAT32_VOL* v, uint64_t clusters, uint64_t entries) {
if (!v) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.DirectorySearches++;
v->Stats.DirectoryClustersRead += clusters;
v->Stats.DirectoryEntriesScanned += entries;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Looks up a positive path-cache entry for a full drive-relative path.
static bool Fat32PathCacheLookup(FAT32_VOL* v, SYS_STRING_VIEW path,
uint32_t* outCluster, uint32_t* outSize,
uint8_t* outAttr, uint64_t* outDirentLba,
uint32_t* outDirentOffset) {
if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
return false;
uint32_t hash = Fat32PathCacheHash(path);
uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
bool hit = false;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.PathLookups++;
FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
if (e->Valid && e->Generation == v->PathCacheGeneration &&
e->Hash == hash && Fat32PathCachePathEquals(e, path)) {
if (outCluster) *outCluster = e->Cluster;
if (outSize) *outSize = e->Size;
if (outAttr) *outAttr = e->Attr;
if (outDirentLba) *outDirentLba = e->DirentLba;
if (outDirentOffset) *outDirentOffset = e->DirentOffset;
v->Stats.PathCacheHits++;
hit = true;
} else {
v->Stats.PathCacheMisses++;
}
KeReleaseSpinlock(&v->PathCacheLock, flags);
return hit;
}
// Stores a positive path lookup in the volume-local path cache.
static void Fat32PathCacheStore(FAT32_VOL* v, SYS_STRING_VIEW path,
uint32_t cluster, uint32_t size, uint8_t attr,
uint64_t direntLba, uint32_t direntOffset) {
if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
return;
uint32_t hash = Fat32PathCacheHash(path);
uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
memset(e, 0, sizeof(*e));
e->Valid = true;
e->Generation = v->PathCacheGeneration;
e->Hash = hash;
e->PathLength = path.Length;
memcpy(e->Path, path.Data, path.Length);
e->Path[path.Length] = '\0';
e->Cluster = cluster;
e->Size = size;
e->Attr = attr;
e->DirentLba = direntLba;
e->DirentOffset = direntOffset;
v->Stats.PathCacheStores++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Invalidates every positive path-cache entry after a directory or size mutation.
static void Fat32PathCacheInvalidate(FAT32_VOL* v) {
if (!v) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->PathCacheGeneration++;
if (v->PathCacheGeneration == 0)
v->PathCacheGeneration = 1;
v->Stats.PathCacheInvalidations++;
v->Stats.PathCacheGeneration = v->PathCacheGeneration;
for (uint32_t i = 0; i < FAT32_PATH_CACHE_SLOTS; i++)
v->PathCache[i].Valid = false;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Returns the current filesystem cache and traversal counters.
void Fat32QueryStats(IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out) {
if (!fs || !out) return;
FAT32_VOL* v = (FAT32_VOL*)fs->DriverCtx;
if (!v) { memset(out, 0, sizeof(*out)); return; }
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
*out = v->Stats;
out->PathCacheGeneration = v->PathCacheGeneration;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Adds FAT-chain traversal counters shared by the read path helpers.
static void Fat32StatsAddChainWalk(FAT32_VOL* v, uint64_t steps) {
if (!v || !steps) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainWalkSteps += steps;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Records one data read span and its physically contiguous cluster merges.
static void Fat32StatsAddReadRun(FAT32_VOL* v, uint32_t sectors, uint32_t merges) {
if (!v || !sectors) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.DataReadRuns++;
v->Stats.DataReadSectors += sectors;
v->Stats.DataReadRunMerges += merges;
if (v->Stats.MaxDataReadSpanSectors < sectors)
v->Stats.MaxDataReadSpanSectors = sectors;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Initializes the per-open cluster-chain cache for a FAT file object.
static void Fat32FileInitChainCache(FAT32_FILE* f) {
if (!f) return;
f->ChainCacheLock = (SPINLOCK)SPINLOCK_INIT;
f->ChainCacheClock = 0;
memset(f->ChainCache, 0, sizeof(f->ChainCache));
}
// Invalidates cached FAT extents after the file's cluster chain changes.
static void Fat32FileInvalidateChainCache(FAT32_FILE* f) {
if (!f) return;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
memset(f->ChainCache, 0, sizeof(f->ChainCache));
f->ChainCacheClock = 0;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheInvalidations++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Locates the best cached starting point for a logical cluster index.
static bool Fat32ChainCacheLookupStart(FAT32_FILE* f, uint32_t logicalIndex,
uint32_t* outLogical, uint32_t* outCluster) {
if (!f || !outLogical || !outCluster)
return false;
bool hit = false;
uint32_t bestLogical = 0;
uint32_t bestCluster = 0;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
for (uint32_t i = 0; i < FAT32_CHAIN_CACHE_SLOTS; i++) {
FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[i];
if (!e->Valid || e->Length == 0)
continue;
uint32_t end = e->LogicalStart + e->Length;
if (logicalIndex >= e->LogicalStart && logicalIndex < end) {
*outLogical = logicalIndex;
*outCluster = e->PhysicalStart + (logicalIndex - e->LogicalStart);
hit = true;
break;
}
if (end <= logicalIndex && e->NextCluster >= 2 && e->NextCluster < FAT32_EOC) {
if (!hit || end > bestLogical) {
bestLogical = end;
bestCluster = e->NextCluster;
hit = true;
}
}
}
if (hit && bestCluster) {
*outLogical = bestLogical;
*outCluster = bestCluster;
}
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (v) {
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheLookups++;
if (hit) v->Stats.FatChainCacheHits++;
else v->Stats.FatChainCacheMisses++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
return hit;
}
// Stores a physically contiguous FAT extent in the per-open chain cache.
static void Fat32ChainCacheStore(FAT32_FILE* f, uint32_t logicalStart,
uint32_t physicalStart, uint32_t length,
uint32_t nextCluster) {
if (!f || physicalStart < 2 || physicalStart >= FAT32_EOC || length == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
uint32_t slot = f->ChainCacheClock++ % FAT32_CHAIN_CACHE_SLOTS;
FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[slot];
e->Valid = true;
e->LogicalStart = logicalStart;
e->PhysicalStart = physicalStart;
e->Length = length;
e->NextCluster = nextCluster;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheStores++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Resolves a logical file cluster through the cached FAT-chain extent map.
static uint32_t Fat32ResolveFileCluster(FAT32_FILE* f, uint32_t logicalIndex) {
if (!f || f->StartCluster < 2 || f->StartCluster >= FAT32_EOC)
return 0;
if (logicalIndex == 0)
return f->StartCluster;
uint32_t currentLogical = 0;
uint32_t cluster = f->StartCluster;
if (!Fat32ChainCacheLookupStart(f, logicalIndex, &currentLogical, &cluster)) {
currentLogical = 0;
cluster = f->StartCluster;
}
uint64_t steps = 0;
while (currentLogical < logicalIndex && cluster >= 2 && cluster < FAT32_EOC) {
cluster = FatEntry(f->Vol, cluster);
currentLogical++;
steps++;
}
Fat32StatsAddChainWalk(f->Vol, steps);
return cluster;
}
// Builds and caches one physically contiguous read run from a starting cluster.
static bool Fat32BuildReadRun(FAT32_FILE* f, uint32_t logicalIndex,
uint32_t startCluster, uint32_t maxSectors,
uint32_t* outRunSectors, uint32_t* outNextCluster) {
if (!f || !outRunSectors || !outNextCluster)
return false;
FAT32_VOL* v = f->Vol;
if (!v || startCluster < 2 || startCluster >= FAT32_EOC)
return false;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t runClusters = 1;
uint32_t last = startCluster;
uint32_t next = FatEntry(v, last);
uint64_t steps = 1;
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= maxSectors) {
last = next;
runClusters++;
runSectors += v->SectorsPerCluster;
next = FatEntry(v, last);
steps++;
}
Fat32StatsAddChainWalk(v, steps);
Fat32ChainCacheStore(f, logicalIndex, startCluster, runClusters, next);
*outRunSectors = runSectors;
*outNextCluster = next;
return true;
}
static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
uint32_t *outCluster, uint32_t *outSize, uint8_t *outAttr,
uint64_t *outDirentLba, uint32_t *outDirentOffset) {
@@ -986,15 +1386,20 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
int allowShortMatch = FatNameIsStructurallyShort83V(name);
Make83NameV(name, wantName83, wantExt83);
uint32_t cluster = dirCluster;
uint64_t clustersRead = 0;
uint64_t entriesScanned = 0;
int found = 0;
while (cluster >= 2 && cluster < FAT32_EOC) {
uint64_t clusterLba = ClusterToLba(v, cluster);
if (!KSUCCESS(ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf))) break;
clustersRead++;
uint32_t entries = v->BytesPerCluster / sizeof(DIRENT);
DIRENT *dir = (DIRENT *)clusterBuf;
for (uint32_t i = 0; i < entries; i++) {
entriesScanned++;
if (dir[i].Name[0] == 0x00) goto done;
if ((uint8_t)dir[i].Name[0] == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; }
if (dir[i].Attr == ATTR_LFN) { LfnAppend(lfn, (LFN_ENTRY *)&dir[i]); hasLfn = 1; continue; }
@@ -1016,15 +1421,16 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
if (outDirentLba) *outDirentLba = clusterLba + eb / v->BytesPerSector;
if (outDirentOffset) *outDirentOffset = (uint32_t)(eb % v->BytesPerSector);
}
MmFreePool(clusterBuf);
return 1;
found = 1;
goto done;
}
}
cluster = FatEntry(v, cluster);
}
done:
Fat32PathCacheAddDirectoryStats(v, clustersRead, entriesScanned);
MmFreePool(clusterBuf);
return 0;
return found;
}
static int Fat32PathNextComponent(SYS_STRING_VIEW path, size_t* cursor,
@@ -1058,6 +1464,10 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
if (outDirentLba) *outDirentLba = 0;
if (outDirentOffset) *outDirentOffset = 0;
if (Fat32PathCacheLookup(v, path, outCluster, outSize, &attr,
outDirentLba, outDirentOffset))
return STATUS_SUCCESS;
size_t cursor = 0;
SYS_STRING_VIEW component;
while (Fat32PathNextComponent(path, &cursor, &component)) {
@@ -1078,6 +1488,9 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
*outCluster = cluster;
*outSize = size;
Fat32PathCacheStore(v, path, cluster, size, attr,
outDirentLba ? *outDirentLba : 0,
outDirentOffset ? *outDirentOffset : 0);
return STATUS_SUCCESS;
}
@@ -1220,6 +1633,7 @@ KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = cluster; f->FileSize = size; f->Vol = v;
f->DirentLba = direntLba; f->DirentOffset = direntOffset;
Fat32FileInitChainCache(f);
*fileCtx = f;
return STATUS_SUCCESS;
}
@@ -1233,15 +1647,9 @@ void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
}
// Read (fix #4: contiguous-run coalescing)
// We peek ahead in the FAT chain: if cluster N+1 == cluster N physically,
// extend the current run instead of issuing a new ReadSectors call.
// Run length is capped at MAX_RUN_SECTORS to stay within the AHCI bounce
// buffer (128 sectors = 64 KB).
// FatEntry() calls in the peek loop are nearly free because they hit the
// FAT sector cache (fix #1).
#define MAX_RUN_SECTORS 128u
// Cluster-chain lookup is cached per open file, and data spans are allowed to
// grow beyond the AHCI bounce size because the device layer now splits large
// sector requests for the concrete controller.
KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
uint64_t offset, void *buf,
@@ -1257,14 +1665,12 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
if (size > (size_t)(f->FileSize - offset)) size = (size_t)(f->FileSize - offset);
size_t wanted = size;
// Allocate only as much bounce space as this read can actually use.
// The old code allocated 64 KiB and could read a 64 KiB contiguous run even
// for tiny random reads; Doom does lots of seek/read pairs for small lumps.
uint32_t initialOffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
uint64_t initialBytesCovered = (uint64_t)initialOffsetInCluster + (uint64_t)size;
uint32_t maxUsefulSectors = (uint32_t)((initialBytesCovered + 511u) / 512u);
if (maxUsefulSectors < v->SectorsPerCluster) maxUsefulSectors = v->SectorsPerCluster;
if (maxUsefulSectors > MAX_RUN_SECTORS) maxUsefulSectors = MAX_RUN_SECTORS;
if (maxUsefulSectors > FAT32_READ_RUN_MAX_SECTORS)
maxUsefulSectors = FAT32_READ_RUN_MAX_SECTORS;
uint8_t *runBuf = (uint8_t *)MmAllocatePool((size_t)maxUsefulSectors * 512u);
if (!runBuf) return STATUS_NO_MEMORY;
@@ -1272,35 +1678,27 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
size_t totalRead = 0;
uint8_t *out = (uint8_t *)buf;
KSTATUS readStatus = STATUS_SUCCESS;
uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster);
uint32_t logicalCluster = (uint32_t)(offset / v->BytesPerCluster);
uint32_t offsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
uint32_t cluster = f->StartCluster;
for (uint32_t i = 0; i < skipClusters && cluster < FAT32_EOC; i++)
cluster = FatEntry(v, cluster);
uint32_t cluster = Fat32ResolveFileCluster(f, logicalCluster);
while (size > 0 && cluster >= 2 && cluster < FAT32_EOC) {
// Build the longest physically-contiguous run that can contribute to
// this caller's remaining read. Do not prefetch an unrelated 64 KiB run
// for a tiny random read.
uint64_t bytesCovered = (uint64_t)offsetInCluster + (uint64_t)size;
uint32_t usefulSectors = (uint32_t)((bytesCovered + 511u) / 512u);
if (usefulSectors < v->SectorsPerCluster) usefulSectors = v->SectorsPerCluster;
if (usefulSectors > MAX_RUN_SECTORS) usefulSectors = MAX_RUN_SECTORS;
if (usefulSectors > FAT32_READ_RUN_MAX_SECTORS)
usefulSectors = FAT32_READ_RUN_MAX_SECTORS;
if (usefulSectors > maxUsefulSectors) usefulSectors = maxUsefulSectors;
uint32_t runStart = cluster;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t next = FatEntry(v, cluster);
while (next == cluster + 1 && next < FAT32_EOC
&& runSectors + v->SectorsPerCluster <= usefulSectors) {
cluster = next;
next = FatEntry(v, cluster);
runSectors += v->SectorsPerCluster;
uint32_t runSectors = 0;
uint32_t next = 0;
if (!Fat32BuildReadRun(f, logicalCluster, cluster, usefulSectors,
&runSectors, &next)) {
readStatus = STATUS_DEVICE_ERROR;
break;
}
KSTATUS rs = ReadSectors(v, ClusterToLba(v, runStart), runSectors, runBuf);
KSTATUS rs = ReadSectors(v, ClusterToLba(v, cluster), runSectors, runBuf);
if (!KSUCCESS(rs)) {
readStatus = rs;
break;
@@ -1309,12 +1707,16 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
uint32_t runBytes = runSectors * 512u;
size_t avail = runBytes - offsetInCluster;
size_t copy = avail < size ? avail : size;
uint32_t runClusters = runSectors / v->SectorsPerCluster;
uint32_t merges = runClusters ? runClusters - 1u : 0;
Fat32StatsAddReadRun(v, runSectors, merges);
memcpy(out, runBuf + offsetInCluster, copy);
out += copy;
totalRead += copy;
size -= copy;
offsetInCluster = 0;
logicalCluster += runClusters;
cluster = next;
}
@@ -1345,6 +1747,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
const uint8_t *src = (const uint8_t *)buf;
Fat32FileInvalidateChainCache(f);
// Ensure at least one cluster exists.
if (f->StartCluster < 2) {
@@ -1368,7 +1771,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
cluster = next;
}
uint8_t *runBuf = (uint8_t *)MmAllocatePool(MAX_RUN_SECTORS * 512u);
uint8_t *runBuf = (uint8_t *)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
if (!runBuf) { *bytesWritten = 0; return STATUS_NO_MEMORY; }
size_t written = 0, remaining = size;
@@ -1382,7 +1785,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
if (offsetInCluster == 0) {
while (next == cluster + 1 && next < FAT32_EOC
&& runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
&& runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
cluster = next;
next = FatEntry(v, cluster);
runSectors += v->SectorsPerCluster;
@@ -1439,7 +1842,9 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
u.Flags |= FAT32_DIRENT_UPDATE_SIZE;
u.Size = f->FileSize;
}
DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
KSTATUS ms = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
if (KSUCCESS(ms))
Fat32PathCacheInvalidate(v);
}
*bytesWritten = written;
@@ -1447,8 +1852,6 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
}
// Create.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
// Creates the requested object.
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
@@ -1483,7 +1886,7 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
// Create means overwrite by default for regular files. Reuse the
// existing directory entry, drop the old cluster chain, and return
// an open zero-length file instead of allocating a duplicate name.
KSTATUS ts = Fat32Truncate(fs, f, 0);
KSTATUS ts = Fat32Truncate(fs, f, 0, 0);
if (!KSUCCESS(ts)) { MmFreePool(f); return ts; }
*fileCtx = f;
return STATUS_SUCCESS;
@@ -1558,21 +1961,27 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = 0; f->FileSize = 0; f->Vol = v;
f->DirentLba = direntLba; f->DirentOffset = direntOffset;
Fat32FileInitChainCache(f);
Fat32PathCacheInvalidate(v);
*fileCtx = f;
return STATUS_SUCCESS;
}
// Truncate.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags) {
(void)fs;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
Fat32FileInvalidateChainCache(f);
int zeroNewClusters = (fileFlags & IO_FILE_FLAG_SKIP_ZERO_EXTEND) ? 0 : 1;
if ((uint32_t)newSize == f->FileSize) {
FAT32_DIRENT_UPDATE u;
memset(&u, 0, sizeof(u));
FatQueryEncodedTime(&u.Timestamp);
u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME;
return DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
KSTATUS s = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s;
}
if (newSize == 0) {
@@ -1584,6 +1993,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
f->FileSize = 0;
KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, 0);
FlushFatCache(v);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s;
}
@@ -1604,7 +2014,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
? (f->FileSize + v->BytesPerCluster - 1) / v->BytesPerCluster : 0;
uint32_t lastCluster = 0;
if (f->StartCluster < 2) {
uint32_t first = AllocCluster(v);
uint32_t first = AllocClusterEx(v, zeroNewClusters);
if (!first) return STATUS_NO_MEMORY;
f->StartCluster = first;
DirentUpdateFirstCluster(v, f->DirentLba, f->DirentOffset, first);
@@ -1613,7 +2023,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
lastCluster = GetChainEnd(v, f->StartCluster, NULL);
}
for (uint32_t i = curClusters; i < newClusters; i++) {
uint32_t next = AppendCluster(v, lastCluster);
uint32_t next = AppendClusterEx(v, lastCluster, zeroNewClusters);
if (!next) return STATUS_NO_MEMORY;
lastCluster = next;
}
@@ -1623,6 +2033,7 @@ update_size:
f->FileSize = (uint32_t)newSize;
{ KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, (uint32_t)newSize);
FlushFatCache(v);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s; }
}
@@ -1646,6 +2057,7 @@ KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (cluster >= 2) FreeClusterChain(v, cluster);
FlushFatCache(v);
Fat32PathCacheInvalidate(v);
return STATUS_SUCCESS;
}
@@ -1654,6 +2066,75 @@ uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs; return ((FAT32_FILE *)fileCtx)->FileSize;
}
// Mixes FAT directory metadata into a per-file content generation.
static uint64_t Fat32DirentGeneration(const DIRENT *d) {
if (!d)
return 0;
uint32_t firstCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
uint64_t generation = ((uint64_t)d->WrtDate << 48) |
((uint64_t)d->WrtTime << 32) |
((uint64_t)d->LastAccDate << 16) |
(uint64_t)d->Attr;
generation ^= ((uint64_t)d->CrtDate << 40);
generation ^= ((uint64_t)d->CrtTime << 24);
generation ^= ((uint64_t)d->CrtTimeTenth << 16);
generation ^= ((uint64_t)firstCluster << 1);
generation ^= ((uint64_t)d->FileSize << 17);
return generation ? generation : 1;
}
// Returns a stable FAT file identity for cross-open image-section caches.
KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out) {
(void)fs;
if (!fileCtx || !out)
return STATUS_INVALID_PARAMETER;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
if (!v)
return STATUS_INVALID_PARAMETER;
uint32_t startCluster = f->StartCluster;
uint32_t fileSize = f->FileSize;
uint64_t generation = ((uint64_t)startCluster << 1) ^
((uint64_t)fileSize << 17);
if (f->DirentLba != 0) {
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
if (!sec)
return STATUS_NO_MEMORY;
KSTATUS s = ReadSectors(v, f->DirentLba, 1, sec);
if (!KSUCCESS(s)) {
MmFreePool(sec);
return s;
}
DIRENT *d = (DIRENT *)(sec + f->DirentOffset);
if ((uint8_t)d->Name[0] == 0x00 || (uint8_t)d->Name[0] == 0xE5) {
MmFreePool(sec);
return STATUS_NOT_FOUND;
}
startCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
fileSize = d->FileSize;
generation = Fat32DirentGeneration(d);
MmFreePool(sec);
}
out->VolumeId = ((uint64_t)v->DriveIndex << 56) ^
((uint64_t)v->PartitionLba << 8) ^
((uint64_t)v->RootCluster << 1) ^
0x4641543332494445ULL;
out->FileId = f->DirentLba ^
((uint64_t)f->DirentOffset << 32) ^
((uint64_t)startCluster << 1);
out->Generation = generation ? generation : 1;
out->Size = fileSize;
return STATUS_SUCCESS;
}
// Directory enumeration (fix #2: cluster buffer cached in context)
// Previously: allocate + ReadSectors + return 1 entry + free, every call.
// Now: buffer lives in FAT32_DIR_CTX; ReadSectors fires only on cluster
@@ -1757,8 +2238,7 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
}
ctx->CurCluster = FatEntry(v, ctx->CurCluster);
ctx->EntryIndex = 0; ctx->HasLfn = 0;
memset(ctx->LfnBuf, 0, sizeof(ctx->LfnBuf));
ctx->EntryIndex = 0;
}
return STATUS_NO_MORE_FILES;
@@ -1962,6 +2442,7 @@ typedef struct FAT32_ASYNC_RW {
uint64_t Offset;
uint32_t OffsetInCluster;
uint32_t Cluster;
uint32_t LogicalCluster;
size_t Remaining;
size_t Done;
size_t Requested;
@@ -1972,6 +2453,7 @@ typedef struct FAT32_ASYNC_RW {
uint32_t CurrentNext;
uint32_t CurrentLast;
uint32_t CurrentClusters;
uint32_t CurrentOffset;
size_t CurrentCopy;
@@ -2088,7 +2570,13 @@ static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
ctx->Done += ctx->CurrentCopy;
ctx->Remaining -= ctx->CurrentCopy;
ctx->OffsetInCluster = 0;
ctx->LogicalCluster += ctx->CurrentClusters;
ctx->Cluster = ctx->CurrentNext;
if (ctx->CurrentClusters) {
uint32_t sectors = ctx->CurrentClusters * ctx->Vol->SectorsPerCluster;
uint32_t merges = ctx->CurrentClusters - 1u;
Fat32StatsAddReadRun(ctx->Vol, sectors, merges);
}
Fat32AsyncQueueContinuation(ctx);
}
@@ -2109,15 +2597,12 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
FAT32_VOL* v = ctx->Vol;
uint32_t runStart = ctx->Cluster;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t last = ctx->Cluster;
uint32_t next = FatEntry(v, last);
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
last = next;
next = FatEntry(v, last);
runSectors += v->SectorsPerCluster;
uint32_t runSectors = 0;
uint32_t next = 0;
if (!Fat32BuildReadRun(ctx->File, ctx->LogicalCluster, ctx->Cluster,
FAT32_READ_RUN_MAX_SECTORS, &runSectors, &next)) {
Fat32AsyncFinish(ctx, STATUS_DEVICE_ERROR, ctx->Done);
return;
}
uint32_t runBytes = runSectors * 512u;
@@ -2125,7 +2610,8 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
size_t copy = avail < ctx->Remaining ? avail : ctx->Remaining;
ctx->CurrentNext = next;
ctx->CurrentLast = last;
ctx->CurrentLast = runStart + (runSectors / v->SectorsPerCluster) - 1u;
ctx->CurrentClusters = runSectors / v->SectorsPerCluster;
ctx->CurrentOffset = ctx->OffsetInCluster;
ctx->CurrentCopy = copy;
@@ -2184,7 +2670,7 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
memset(ctx, 0, sizeof(*ctx));
LIST_INIT(&ctx->Link);
ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_READ_RUN_MAX_SECTORS * 512u);
if (!ctx->RunBuf) {
MmFreePool(ctx);
complete(completeContext, STATUS_NO_MEMORY, 0);
@@ -2196,17 +2682,14 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
ctx->IsWrite = 0;
ctx->Offset = offset;
ctx->OffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
ctx->Cluster = f->StartCluster;
ctx->LogicalCluster = (uint32_t)(offset / v->BytesPerCluster);
ctx->Cluster = Fat32ResolveFileCluster(f, ctx->LogicalCluster);
ctx->Remaining = size;
ctx->Requested = size;
ctx->Dst = (uint8_t*)buf;
ctx->Complete = complete;
ctx->CompleteContext = completeContext;
uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster);
for (uint32_t i = 0; i < skipClusters && ctx->Cluster < FAT32_EOC; i++)
ctx->Cluster = FatEntry(v, ctx->Cluster);
Fat32AsyncQueueContinuation(ctx);
return STATUS_PENDING;
}
@@ -2276,7 +2759,7 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
if (ctx->OffsetInCluster == 0) {
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
last = next;
next = FatEntry(v, last);
runSectors += v->SectorsPerCluster;
@@ -2347,6 +2830,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
FAT32_FILE* f = (FAT32_FILE*)fileCtx;
FAT32_VOL* v = f->Vol;
Fat32FileInvalidateChainCache(f);
if (f->StartCluster < 2) {
uint32_t first = AllocCluster(v);
@@ -2380,7 +2864,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
memset(ctx, 0, sizeof(*ctx));
LIST_INIT(&ctx->Link);
ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
if (!ctx->RunBuf) {
MmFreePool(ctx);
complete(completeContext, STATUS_NO_MEMORY, 0);
code/drivers/fs/fat32/fat32.h
+5 -1
View File
@@ -16,6 +16,8 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_time.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_irql.h>
typedef struct FAT32_VOL_S FAT32_VOL;
@@ -27,10 +29,12 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags);
uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out);
KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx);
KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out);
void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx);
void Fat32QueryStats(IO_FILESYSTEM *fs, IO_FILESYSTEM_STATS *out);
void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol);
code/drivers/fs/fat32/fat32_mount.c
+2
View File
@@ -14,9 +14,11 @@ void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
fs->Delete = Fat32Delete;
fs->Truncate = Fat32Truncate;
fs->GetSize = Fat32GetSize;
fs->QueryIdentity = Fat32QueryIdentity;
fs->Close = Fat32Close;
fs->DriverCtx = vol;
fs->OpenDir = Fat32OpenDir;
fs->ReadDir = Fat32ReadDir;
fs->CloseDir = Fat32CloseDir;
fs->QueryStats = Fat32QueryStats;
}
code/drivers/input/serial/serial.c
+301
View File
@@ -0,0 +1,301 @@
// 16550-compatible serial input driver. This file routes COM1 receive bytes into the canonical input core.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <intrin.h>
#include <ntlite/status.h>
#include <ntlite/ddi/hal_api.h>
#include <ntlite/ddi/input_device.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_irq.h>
#define SERIAL_COM1_BASE 0x3F8u
#define SERIAL_COM1_IRQ 4u
#define SERIAL_REG_DATA 0u
#define SERIAL_REG_IER 1u
#define SERIAL_REG_IIR 2u
#define SERIAL_REG_FCR 2u
#define SERIAL_REG_LCR 3u
#define SERIAL_REG_MCR 4u
#define SERIAL_REG_LSR 5u
#define SERIAL_LSR_DATA_READY 0x01u
#define SERIAL_LSR_TX_EMPTY 0x20u
#define SERIAL_LCR_DLAB 0x80u
#define SERIAL_MCR_DTR 0x01u
#define SERIAL_MCR_RTS 0x02u
#define SERIAL_MCR_OUT2 0x08u
#define SERIAL_IER_RX_READY 0x01u
#define SERIAL_FCR_ENABLE 0x01u
#define SERIAL_FCR_CLEAR_RX 0x02u
#define SERIAL_FCR_CLEAR_TX 0x04u
#define SERIAL_FCR_TRIGGER_14 0xC0u
typedef enum SERIAL_ESCAPE_STATE {
SerialEscapeNone = 0,
SerialEscapeEsc,
SerialEscapeCsi,
SerialEscapeCsiNumber,
} SERIAL_ESCAPE_STATE;
static volatile SERIAL_ESCAPE_STATE gSerialEscapeState;
static volatile uint8_t gSerialCsiNumber;
static volatile bool gSerialDropNextLf;
// Reads one byte from a COM register.
static uint8_t SerialRead(uint16_t reg) {
return __inbyte((unsigned short)(SERIAL_COM1_BASE + reg));
}
// Writes one byte to a COM register.
static void SerialWrite(uint16_t reg, uint8_t value) {
__outbyte((unsigned short)(SERIAL_COM1_BASE + reg), value);
}
// Submits one input key transition.
static void SerialSubmitTransition(INPUT_KEY key, bool pressed) {
INPUT_KEY_EVENT event;
event.Key = key;
event.Pressed = pressed;
KeInputSubmitKey(&event);
}
// Submits one logical key press and release.
static void SerialSubmitKey(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return;
SerialSubmitTransition(key, true);
SerialSubmitTransition(key, false);
}
// Submits one shifted key press and release.
static void SerialSubmitShiftedKey(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return;
SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, true);
SerialSubmitKey(key);
SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, false);
}
// Submits one control-modified key press and release.
static void SerialSubmitCtrlKey(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return;
SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, true);
SerialSubmitKey(key);
SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, false);
}
// Converts an ASCII printable byte into input events.
static void SerialSubmitPrintable(uint8_t ch) {
if (ch >= 'a' && ch <= 'z') {
SerialSubmitKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'a')));
return;
}
if (ch >= 'A' && ch <= 'Z') {
SerialSubmitShiftedKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'A')));
return;
}
if (ch >= '1' && ch <= '9') {
SerialSubmitKey((INPUT_KEY)(INPUT_KEY_1 + (ch - '1')));
return;
}
if (ch == '0') {
SerialSubmitKey(INPUT_KEY_0);
return;
}
switch (ch) {
case ' ': SerialSubmitKey(INPUT_KEY_SPACE); return;
case '-': SerialSubmitKey(INPUT_KEY_MINUS); return;
case '=': SerialSubmitKey(INPUT_KEY_EQUAL); return;
case '[': SerialSubmitKey(INPUT_KEY_LEFT_BRACKET); return;
case ']': SerialSubmitKey(INPUT_KEY_RIGHT_BRACKET); return;
case '\\': SerialSubmitKey(INPUT_KEY_BACKSLASH); return;
case ';': SerialSubmitKey(INPUT_KEY_SEMICOLON); return;
case '\'': SerialSubmitKey(INPUT_KEY_APOSTROPHE); return;
case '`': SerialSubmitKey(INPUT_KEY_GRAVE); return;
case ',': SerialSubmitKey(INPUT_KEY_COMMA); return;
case '.': SerialSubmitKey(INPUT_KEY_PERIOD); return;
case '/': SerialSubmitKey(INPUT_KEY_SLASH); return;
case '!': SerialSubmitShiftedKey(INPUT_KEY_1); return;
case '@': SerialSubmitShiftedKey(INPUT_KEY_2); return;
case '#': SerialSubmitShiftedKey(INPUT_KEY_3); return;
case '$': SerialSubmitShiftedKey(INPUT_KEY_4); return;
case '%': SerialSubmitShiftedKey(INPUT_KEY_5); return;
case '^': SerialSubmitShiftedKey(INPUT_KEY_6); return;
case '&': SerialSubmitShiftedKey(INPUT_KEY_7); return;
case '*': SerialSubmitShiftedKey(INPUT_KEY_8); return;
case '(': SerialSubmitShiftedKey(INPUT_KEY_9); return;
case ')': SerialSubmitShiftedKey(INPUT_KEY_0); return;
case '_': SerialSubmitShiftedKey(INPUT_KEY_MINUS); return;
case '+': SerialSubmitShiftedKey(INPUT_KEY_EQUAL); return;
case '{': SerialSubmitShiftedKey(INPUT_KEY_LEFT_BRACKET); return;
case '}': SerialSubmitShiftedKey(INPUT_KEY_RIGHT_BRACKET); return;
case '|': SerialSubmitShiftedKey(INPUT_KEY_BACKSLASH); return;
case ':': SerialSubmitShiftedKey(INPUT_KEY_SEMICOLON); return;
case '"': SerialSubmitShiftedKey(INPUT_KEY_APOSTROPHE); return;
case '~': SerialSubmitShiftedKey(INPUT_KEY_GRAVE); return;
case '<': SerialSubmitShiftedKey(INPUT_KEY_COMMA); return;
case '>': SerialSubmitShiftedKey(INPUT_KEY_PERIOD); return;
case '?': SerialSubmitShiftedKey(INPUT_KEY_SLASH); return;
default: return;
}
}
// Submits a completed CSI escape sequence.
static void SerialSubmitCsi(uint8_t finalByte) {
uint8_t number = gSerialCsiNumber;
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
switch (finalByte) {
case 'A': SerialSubmitKey(INPUT_KEY_UP); return;
case 'B': SerialSubmitKey(INPUT_KEY_DOWN); return;
case 'C': SerialSubmitKey(INPUT_KEY_RIGHT); return;
case 'D': SerialSubmitKey(INPUT_KEY_LEFT); return;
case 'H': SerialSubmitKey(INPUT_KEY_HOME); return;
case 'F': SerialSubmitKey(INPUT_KEY_END); return;
case '~':
switch (number) {
case 1: SerialSubmitKey(INPUT_KEY_HOME); return;
case 3: SerialSubmitKey(INPUT_KEY_DELETE); return;
case 4: SerialSubmitKey(INPUT_KEY_END); return;
case 5: SerialSubmitKey(INPUT_KEY_PAGE_UP); return;
case 6: SerialSubmitKey(INPUT_KEY_PAGE_DOWN); return;
default: return;
}
default: return;
}
}
// Handles one byte after an ESC prefix.
static bool SerialHandleEscapeByte(uint8_t ch) {
if (gSerialEscapeState == SerialEscapeNone) return false;
if (gSerialEscapeState == SerialEscapeEsc) {
if (ch == '[') {
gSerialEscapeState = SerialEscapeCsi;
gSerialCsiNumber = 0;
return true;
}
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
SerialSubmitKey(INPUT_KEY_ESCAPE);
return false;
}
if (gSerialEscapeState == SerialEscapeCsi || gSerialEscapeState == SerialEscapeCsiNumber) {
if (ch >= '0' && ch <= '9') {
gSerialEscapeState = SerialEscapeCsiNumber;
if (gSerialCsiNumber < 25)
gSerialCsiNumber = (uint8_t)(gSerialCsiNumber * 10u + (ch - '0'));
return true;
}
if (ch == ';')
return true;
SerialSubmitCsi(ch);
return true;
}
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
return false;
}
// Routes one received serial byte into the input core.
static void SerialReceiveByte(uint8_t ch) {
if (SerialHandleEscapeByte(ch)) return;
if (ch == 0x1Bu) {
gSerialEscapeState = SerialEscapeEsc;
gSerialCsiNumber = 0;
return;
}
if (ch == '\r') {
SerialSubmitKey(INPUT_KEY_ENTER);
gSerialDropNextLf = true;
return;
}
if (ch == '\n') {
if (gSerialDropNextLf) {
gSerialDropNextLf = false;
return;
}
SerialSubmitKey(INPUT_KEY_ENTER);
return;
}
gSerialDropNextLf = false;
if (ch == '\t') {
SerialSubmitKey(INPUT_KEY_TAB);
return;
}
if (ch == '\b' || ch == 0x7Fu) {
SerialSubmitKey(INPUT_KEY_BACKSPACE);
return;
}
if (ch == 0x03u) {
SerialSubmitCtrlKey(INPUT_KEY_C);
return;
}
if (ch >= 0x20u && ch <= 0x7Eu)
SerialSubmitPrintable(ch);
}
// Handles receive interrupts for COM1.
static void SerialIsr(void) {
for (uint32_t i = 0; i < 256u; i++) {
if ((SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY) == 0)
break;
SerialReceiveByte(SerialRead(SERIAL_REG_DATA));
}
}
// Initializes COM1 as an 8N1 115200 baud input device.
static void SerialInitializeCom1(void) {
SerialWrite(SERIAL_REG_IER, 0x00u);
SerialWrite(SERIAL_REG_LCR, SERIAL_LCR_DLAB);
SerialWrite(SERIAL_REG_DATA, 0x01u);
SerialWrite(SERIAL_REG_IER, 0x00u);
SerialWrite(SERIAL_REG_LCR, 0x03u);
SerialWrite(SERIAL_REG_FCR, SERIAL_FCR_ENABLE | SERIAL_FCR_CLEAR_RX |
SERIAL_FCR_CLEAR_TX | SERIAL_FCR_TRIGGER_14);
SerialWrite(SERIAL_REG_MCR, SERIAL_MCR_DTR | SERIAL_MCR_RTS | SERIAL_MCR_OUT2);
while (SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY)
(void)SerialRead(SERIAL_REG_DATA);
SerialWrite(SERIAL_REG_IER, SERIAL_IER_RX_READY);
}
// DriverEntry registers COM1 as an input source.
KSTATUS DriverEntry(const HAL_API* hal) {
(void)hal;
uint32_t deviceId = 0;
KSTATUS st = KeInputRegisterDevice("serial-com1", 0, &deviceId);
if (!KSUCCESS(st))
return st;
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
gSerialDropNextLf = false;
SerialInitializeCom1();
KeIrqRegister(SERIAL_COM1_IRQ, SerialIsr);
DbgPrint("serial: initialized COM1 input device=%u irq=%u\n", deviceId, SERIAL_COM1_IRQ);
return STATUS_SUCCESS;
}
code/drivers/net/rtl8139/rtl8139.c
+2 -2
View File
@@ -88,8 +88,8 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) {
// Creates dma.
static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
r->RxPhys = MmAllocateContiguous(RTL8139_RX_PAGES);
r->TxPhys = MmAllocateContiguous(RTL8139_TX_PAGES);
r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma);
r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma);
if (!r->RxPhys || !r->TxPhys)
return STATUS_NO_MEMORY;
code/drivers/net/rtl8139/rtl8139.h
+1
View File
@@ -17,6 +17,7 @@
#include <kernel/ke_dpc.h>
#include <kernel/ke_spinlock.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <ntlite/ddi/net_device.h>
code/drivers/net/rtl816x/rtl816x.c
+4 -4
View File
@@ -85,10 +85,10 @@ static void RtlReadMac(RTL816X_DEVICE* r) {
// Implements the rtl Allocate Dma operation.
static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
r->RxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
r->TxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
r->RxBuffersPhys = MmAllocateContiguous(RTL_RXBUF_PAGES);
r->TxBuffersPhys = MmAllocateContiguous(RTL_TXBUF_PAGES);
r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma);
r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma);
if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys)
return STATUS_NO_MEMORY;
code/drivers/net/rtl816x/rtl816x.h
+1
View File
@@ -17,6 +17,7 @@
#include <kernel/ke_dpc.h>
#include <kernel/ke_spinlock.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <ntlite/ddi/net_device.h>
code/drivers/storage/ahci/ahci.c
+40 -16
View File
@@ -58,20 +58,20 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT);
drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma);
if (!drv->structPhys) return STATUS_NO_MEMORY;
drv->structVirt = g_hal->PhysToVirt(drv->structPhys);
// Force NOCACHE on DMA descriptor memory.
MmSetPageProtect((uintptr_t)drv->structVirt,
MmPteProtectRange((uintptr_t)drv->structVirt,
PORT_PAGES_STRUCT * 4096, flags);
memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096);
drv->bouncePhys = MmAllocateContiguous(BOUNCE_PAGES);
drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma);
if (!drv->bouncePhys) return STATUS_NO_MEMORY;
drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys);
// Bounce buffer: NOCACHE to ensure CPU writes reach physical memory
// before the HBA's DMA engine reads them.
MmSetPageProtect((uintptr_t)drv->bounceVirt,
MmPteProtectRange((uintptr_t)drv->bounceVirt,
BOUNCE_PAGES * 4096, flags);
memset(drv->bounceVirt, 0, BOUNCE_BYTES);
@@ -203,6 +203,9 @@ static void AhciIsr(void) {
// Blocks the current thread until the ISR signals completionEvent.
static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
if (drv->controller && drv->controller->pollingMode)
return port_issue_cmd_poll(drv);
HBA_PORT *port = drv->port;
// Clear stale IS and snapshot before issuing so we don't mis-read a
@@ -647,12 +650,25 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
ctl->pciInfo.VendorId,
ctl->pciInfo.DeviceId);
st = PciGetInterrupt(ctl->pdo, &ctl->irq);
if (!KSUCCESS(st) || ctl->irq == 0xFF) {
DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n");
return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
ctl->irq = 0xFFu;
ctl->usingMsi = false;
ctl->pollingMode = false;
st = PciEnableMsiAuto(ctl->pdo, &ctl->irq);
if (KSUCCESS(st)) {
ctl->usingMsi = true;
DbgPrint("AHCI: using MSI IRQ slot %u\n", (unsigned)ctl->irq);
} else {
KSTATUS intxStatus = PciGetInterrupt(ctl->pdo, &ctl->irq);
if (KSUCCESS(intxStatus) && ctl->irq != 0xFFu) {
DbgPrint("AHCI: using legacy INTx IRQ %u\n", (unsigned)ctl->irq);
} else {
ctl->irq = 0xFFu;
ctl->pollingMode = true;
DbgPrint("AHCI: no MSI/INTx interrupt available; using polled command completion (msi=0x%x intx=0x%x)\n",
st, intxStatus);
}
}
DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)ctl->irq);
st = PciGetBar(ctl->pdo, AHCI_PCI_BAR_ABAR, &ctl->abar);
if (!KSUCCESS(st)) {
@@ -674,10 +690,13 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
return st;
}
st = PciEnableDevice(ctl->pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX);
uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER;
if (ctl->usingMsi || ctl->pollingMode)
enableFlags |= PCI_DISABLE_INTX;
else
enableFlags |= PCI_ENABLE_INTX;
st = PciEnableDevice(ctl->pdo, enableFlags);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: PciEnableDevice failed: 0x%x\n", st);
return st;
@@ -831,11 +850,16 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
ctl->pciInfo.Address.Device,
ctl->pciInfo.Address.Function);
KeIrqRegister(ctl->irq, AhciIsr);
if (!ctl->pollingMode) {
KeIrqRegister(ctl->irq, AhciIsr);
DbgPrint("AHCI: ISR registered on IRQ %u (%s); controller drives=%d\n",
(unsigned)ctl->irq, ctl->usingMsi ? "MSI" : "INTx", ctl->driveCount);
} else {
DbgPrint("AHCI: controller started in polling mode; controller drives=%d\n",
ctl->driveCount);
}
ctl->started = true;
ctl->fdo->PnpState = DevicePnpStarted;
DbgPrint("AHCI: ISR registered on IRQ %u; controller drives=%d\n",
(unsigned)ctl->irq, ctl->driveCount);
return STATUS_SUCCESS;
}
code/drivers/storage/ahci/ahci.h
+3
View File
@@ -8,6 +8,7 @@
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddi/pci.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_event.h>
@@ -367,6 +368,8 @@ struct AHCI_CONTROLLER {
HBA_MEM* hba;
uint64_t abarLength;
uint8_t irq;
bool usingMsi;
bool pollingMode;
AHCI_DRIVE drives[MAX_AHCI_DRIVES];
int driveCount;
code/drivers/storage/ahci/ahci_port.c
+4 -4
View File
@@ -12,15 +12,15 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) {
uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys);
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
KSTATUS st = MmMapPage(virt, phys, flags, MmMapFailIfPresent);
KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent);
// Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags
// is possible for the first page if the bootloader was overly generous.
// Use MmSetPageProtect to force NOCACHE in that case.
// Use MmPteProtectRange to force NOCACHE in that case.
if (st == STATUS_ALREADY_EXISTS) {
st = MmSetPageProtect(virt, 4096, flags);
st = MmPteProtectRange(virt, 4096, flags);
}
if (!KSUCCESS(st)) {
DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
pageIndex, (unsigned long long)phys, st);
return st;
}
code/kernel/include/kernel/io_file.h
+60 -1
View File
@@ -25,6 +25,39 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
KSTATUS status,
size_t information);
// Filesystem-level cache and traversal statistics.
typedef struct IO_FILESYSTEM_STATS {
uint64_t PathLookups;
uint64_t PathCacheHits;
uint64_t PathCacheMisses;
uint64_t PathCacheStores;
uint64_t PathCacheInvalidations;
uint64_t PathCacheGeneration;
uint64_t DirectorySearches;
uint64_t DirectoryClustersRead;
uint64_t DirectoryEntriesScanned;
uint64_t FatChainCacheLookups;
uint64_t FatChainCacheHits;
uint64_t FatChainCacheMisses;
uint64_t FatChainCacheStores;
uint64_t FatChainCacheInvalidations;
uint64_t FatChainWalkSteps;
uint64_t DataReadRuns;
uint64_t DataReadSectors;
uint64_t DataReadRunMerges;
uint64_t MaxDataReadSpanSectors;
} IO_FILESYSTEM_STATS;
// Stable identity for a file on a mounted filesystem.
// Filesystems that can provide this let higher layers cache file-backed
// objects by file identity instead of by one FILE_OBJECT instance.
typedef struct IO_FILE_IDENTITY {
uint64_t VolumeId;
uint64_t FileId;
uint64_t Generation;
uint64_t Size;
} IO_FILE_IDENTITY;
// Filesystem driver vtable - filled in by a filesystem driver.
typedef struct IO_FILESYSTEM {
// Open a file by absolute path (e.g. "apps\\shell.exe").
@@ -69,14 +102,21 @@ typedef struct IO_FILESYSTEM {
// Truncate or extend the file to exactly 'newSize' bytes.
KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t newSize);
uint64_t newSize, uint32_t fileFlags);
// Return the file size in bytes.
uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Return a stable file identity for caches that outlive one open handle.
KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx,
IO_FILE_IDENTITY* out);
// Release any resources the driver allocated for fileCtx.
void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Query cache and traversal statistics exposed through kernel stats.
void (*QueryStats)(struct IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out);
// Optional driver-private data.
void* DriverCtx;
@@ -106,9 +146,13 @@ typedef struct IO_STATUS_BLOCK {
struct EVENT;
struct KAPC;
typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
// FILE_OBJECT - the kernel object returned to callers.
#define IO_FILE_FLAG_NO_CACHE 0x00000001u
#define IO_FILE_FLAG_SKIP_ZERO_EXTEND 0x00000002u
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
@@ -116,6 +160,11 @@ typedef struct FILE_OBJECT {
SPINLOCK Lock; // protects cached Size/Offset updates
uint64_t Size;
uint64_t Offset; // current position for sequential reads
uint32_t Flags; // IO_FILE_FLAG_*
uint32_t ReadaheadScore;
uint64_t ReadaheadNextOffset;
uint64_t ReadaheadIssuedUntil;
uint64_t ReadaheadSuppressedUntil;
} FILE_OBJECT;
// DIR_OBJECT - kernel object for an open directory iterator.
@@ -137,6 +186,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESY
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
__declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex);
__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out);
// Kernel-side API.
__declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out);
@@ -149,15 +199,23 @@ __declspec(dllexport) KSTATUS IoWriteFile (FILE_OBJECT* file, const void* buf,
__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
const void* buf, size_t size,
size_t* bytesWritten);
__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file);
__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf,
size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
struct EVENT* completionEvent, struct KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
struct EVENT* completionEvent, struct KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
uint32_t flags);
__declspec(dllexport) struct EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
IO_STATUS_BLOCK* iosbOut);
@@ -169,6 +227,7 @@ __declspec(dllexport) KSTATUS IoRemoveDir (SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize);
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// Directory enumeration API.
code/kernel/include/kernel/ke_debug.h
+11 -1
View File
@@ -12,6 +12,16 @@
#define KE_DEBUG_SELFTEST 0x00000002u
#define KE_DEBUG_FILE_LOG 0x00000004u
#define KE_DEBUG_POOL_CANARY 0x00000008u
#define KE_DEBUG_POOL_POISON 0x00000010u
#define KE_DEBUG_POOL_QUARANTINE 0x00000020u
#define KE_DEBUG_POOL_CALLERS 0x00000040u
#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u
#define KE_DEBUG_DISPLAY_TRACE 0x00000100u
#define KE_DEBUG_INPUT_TRACE 0x00000200u
#define KE_DEBUG_VM_ASSERTS 0x00000400u
#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t));
uint32_t KeGetDebugFlags(void);
@@ -68,7 +78,7 @@ typedef struct KFAULT_FRAME {
uint64_t R15;
} KFAULT_FRAME;
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame);
__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame);
// Compatibility wrapper for any older assembly/object still calling this.
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
code/kernel/include/kernel/ke_display.h
+2
View File
@@ -42,7 +42,9 @@ DISPLAY_API void KeDisplaySuspendConsole(bool suspend);
DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode);
DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight);
typedef KSTATUS (*KE_DISPLAY_ROW_SOURCE)(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount);
DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels);
DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch);
DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color);
DISPLAY_API KSTATUS KeDisplayClear(uint32_t color);
DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h);
code/kernel/include/kernel/ke_event.h
+3
View File
@@ -16,9 +16,12 @@ typedef struct EVENT {
EVENT_TYPE Type;
uint32_t Signaled;
LIST_ENTRY WaitList;
LIST_ENTRY MultiWaitList;
} EVENT;
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out);
__declspec(dllexport) void KeSignalEvent(EVENT* event);
__declspec(dllexport) void KeClearEvent(EVENT* event);
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable);
__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event);
__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count, uint8_t alertable, uint64_t timeoutMs, uint32_t* signaledIndexOut);
code/kernel/include/kernel/ke_input.h
+12
View File
@@ -6,6 +6,13 @@
#include <ntlite/input.h>
#include <ntlite/ddi/hal_api.h>
#include <ntlite/status.h>
#include <kernel/ob_object.h>
#include <kernel/ke_event.h>
typedef struct INPUT_QUEUE {
OBJECT Header;
EVENT* WaitEvent;
} INPUT_QUEUE;
void KeInputInitialize(const HAL_API* hal);
@@ -16,7 +23,12 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons);
INPUT_KEY KeInputKbPoll(void);
KSTATUS KeInputWaitKey(uint8_t alertable);
bool KeInputIsKeyDown(INPUT_KEY key);
bool KeInputPeekEvent(INPUT_KEY_RECORD* record);
bool KeInputReadEvent(INPUT_KEY_RECORD* record);
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out);
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue);
INPUT_KEY KeInputReadKey(void);
char KeInputReadChar(void);
uint32_t KeInputGetModifiers(void);
code/kernel/include/kernel/ke_mutex.h
+4 -4
View File
@@ -14,7 +14,7 @@ typedef struct MUTEX {
LIST_ENTRY WaitList;
} MUTEX;
KSTATUS KeMutexCreate(MUTEX **out);
KSTATUS KeAcquireMutex(MUTEX* mutex);
KSTATUS KeTryAcquireMutex(MUTEX* mutex);
void KeReleaseMutex(MUTEX* mutex);
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out);
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex);
__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex);
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex);
code/kernel/include/kernel/ke_process.h
+8 -2
View File
@@ -62,6 +62,7 @@ typedef struct PROCESS {
int ExitCode;
volatile long Terminating;
volatile long Exited;
volatile long ResourcesReleased;
// Main executable image. Used for cleanup.
uintptr_t ImageBase;
@@ -80,8 +81,8 @@ typedef struct PROCESS {
// Per-process user virtual regions. The actual VA arena is global because
// NT-Lite currently uses one shared address space; this list tracks
// ownership so leaked user mappings can be reclaimed on process exit.
LIST_ENTRY UserRegions;
SPINLOCK UserRegionLock;
LIST_ENTRY VadList;
SPINLOCK VadLock;
// Per-process loaded modules (user processes only).
PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
@@ -102,6 +103,7 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* process,
SYS_STRING_VIEW currentDirectory);
void KeExitProcess(PROCESS* process, int exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode);
void KeCompleteProcessExitIfLastThread(PROCESS* process);
int KeIsProcessTerminating(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut);
@@ -118,8 +120,12 @@ LIST_ENTRY* KeGetProcessList(void);
PROCESS* KeGetCurrentProcess(void);
PROCESS* KeGetSystemProcess(void);
struct SYS_PROCESS_INFO;
struct SYS_THREAD_INFO;
// Copy a snapshot of rich process information into caller-provided storage.
size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount);
// Copy a snapshot of one process's threads into caller-provided storage.
KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut);
// Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject.
PROCESS* KeGetProcessById(uint32_t pid);
code/kernel/include/kernel/ke_sched.h
+4
View File
@@ -58,6 +58,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread);
void KiSchedulerMarkThreadWaiting(THREAD* thread);
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status);
// Blocks the current thread until at least the requested delay has elapsed.
__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds);
KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus);
// Voluntarily yield the CPU. Picks the next runnable thread immediately.
__declspec(dllexport) void KeYield(void);
code/kernel/include/kernel/ke_system_root.h
+25
View File
@@ -0,0 +1,25 @@
// System root volume declarations. This file exposes the one selected kernel root volume to loader and process code.
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <ntlite/status.h>
#include <kernel/io_file.h>
#define KE_SYSTEM_ROOT_MAX_DRIVES 8
// Selects the mounted volume that contains the kernel/system tree.
KSTATUS KeInitializeSystemRootVolume(void);
// Returns whether the system root volume has been selected.
bool KeSystemRootReady(void);
// Returns the selected system root drive, or zero before discovery.
uint8_t KeGetSystemRootDrive(void);
// Builds an absolute path rooted at the selected system volume.
bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize);
// Opens a file rooted at the selected system volume.
KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out);
code/kernel/include/kernel/ke_thread.h
+28 -4
View File
@@ -14,6 +14,8 @@
#define KSTACK_SIZE (64 * 1024) // 64 KB kernel stack
#define USTACK_SIZE (512 * 1024) // 512 KB user stack
typedef struct THREAD THREAD;
typedef enum THREAD_PRIORITY {
PriorityLow = 0,
PriorityNormal = 1,
@@ -28,6 +30,10 @@ typedef enum THREAD_STATE {
ThreadDead = 3,
} THREAD_STATE;
typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread,
void* context,
KSTATUS status);
// Saved register context - matches the layout pushed by KeSwitchContext.
// The saved RSP points to this structure on the kernel stack.
typedef struct KCONTEXT {
@@ -36,7 +42,7 @@ typedef struct KCONTEXT {
uint64_t Rip; // return address back into KeSwitchContext
} KCONTEXT;
typedef struct THREAD {
struct THREAD {
OBJECT Header; // must be first
uint32_t ThreadId;
@@ -82,7 +88,7 @@ typedef struct THREAD {
// Signaled (manual-reset) when the thread exits.
// Allocated in KeCreateThread, signaled in KeExitThread before teardown.
// Waiters block on this via SYS_WAIT_THREAD -> KeWaitForEvent.
// Waiters block on this through waitable thread handles.
EVENT* ExitEvent;
void* FpuContext;
@@ -113,6 +119,12 @@ typedef struct THREAD {
KSTATUS WaitStatus;
LIST_ENTRY* WaitListHead;
uint8_t AlertableWait;
uint64_t SleepUntilTick;
KSTATUS TimeoutStatus;
// Optional cleanup hook for waits that use auxiliary wait blocks.
KE_WAIT_RUNDOWN_ROUTINE WaitRundownRoutine;
void* WaitRundownContext;
// Kernel APC state. APCs are queued to a target thread and delivered
// when that thread reaches PASSIVE_LEVEL with APC delivery enabled.
@@ -135,15 +147,22 @@ typedef struct THREAD {
uintptr_t UserApcDispatcher;
uint8_t UserApcDeliverable;
uint8_t UserApcDeliveryActive;
} THREAD;
};
// Creates and queues a user thread. On success, *out receives a referenced
// THREAD pointer owned by the caller; release it with ObDereferenceObject()
// after copying any needed IDs/state. The live scheduler/thread lifetime is
// separate and remains valid until the thread exits and is reaped.
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
// Create a kernel-mode-only thread. fn is called directly in ring 0 with
// arg; when fn returns the thread exits automatically. No user stack or TEB
// is allocated. The thread is owned by the system process and runs at the
// given priority.
// given priority. On success, non-NULL out receives a referenced THREAD
// pointer owned by the caller; release it with ObDereferenceObject() after
// copying any needed IDs/state or after joining/waiting on caller-owned
// completion state. The scheduler's live thread reference is separate.
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
@@ -167,9 +186,14 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code);
// waiting. Alertable waits are explicit at every call site.
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead);
void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context);
void KeClearThreadWaitRundown(THREAD* thread);
void KeRundownThreadWait(THREAD* thread, KSTATUS status);
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status);
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status);
void KeAbortThreadWait(THREAD* thread, KSTATUS status);
void KeUnblockThread(THREAD* thread);
KSTATUS KeAlertThread(THREAD* thread);
void KiWakeThreadForKernelApc(THREAD* thread);
code/kernel/include/kernel/ke_work.h
+24
View File
@@ -0,0 +1,24 @@
// Kernel passive-level work queue declarations.
#pragma once
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <ntlite/status.h>
// Work items are caller-owned embedded objects. They may be queued from
// interrupt/DPC/spinlock context and run later on a persistent system worker
// thread at PASSIVE_LEVEL. Queueing the same item while it is already queued
// is coalesced.
typedef void (*KWORK_ROUTINE)(void* context);
typedef struct KWORK_ITEM {
LIST_ENTRY Link;
KWORK_ROUTINE Routine;
void* Context;
volatile long Queued;
} KWORK_ITEM;
void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context);
KSTATUS KeStartWorkQueue(void);
bool KeIsWorkQueueStarted(void);
bool KeQueueWorkItem(KWORK_ITEM* item);
code/kernel/include/kernel/ldr_image.h
+5 -4
View File
@@ -9,11 +9,12 @@
#include <ntlite/status.h>
struct PROCESS;
struct FILE_OBJECT;
// Map a PE32+ image from 'buffer' into the kernel virtual address space.
// On success, *baseOut receives the virtual base address.
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user);
KSTATUS LdrLoadImageForProcess(struct PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut);
// Map a PE32+ image from a bootloader/driver raw buffer into kernel space.
// User images must be loaded from FILE_OBJECTs so image sections have stable identity.
KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut);
KSTATUS LdrLoadImageFileForProcess(struct PROCESS* process, struct FILE_OBJECT* file, uintptr_t* baseOut);
// Return the SizeOfImage recorded in the PE header for an already-mapped image.
size_t LdrGetImageSize(uintptr_t base);
code/kernel/include/kernel/ldr_section.h
+67
View File
@@ -0,0 +1,67 @@
// Kernel image section declarations shared by loader and memory-management code.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <kernel/mm_phys.h>
struct PROCESS;
struct FILE_OBJECT;
typedef struct PROCESS PROCESS;
typedef struct FILE_OBJECT FILE_OBJECT;
typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT;
// Describes one PE section in image-address-space coordinates.
typedef struct IMAGE_SECTION_DESCRIPTOR {
char Name[9];
uint32_t VirtualAddress;
uint32_t VirtualSize;
uint32_t RawDataOffset;
uint32_t RawDataSize;
uint32_t Characteristics;
uint64_t PageProtection;
} IMAGE_SECTION_DESCRIPTOR;
// Keeps the parsed identity and layout for a PE image section object.
typedef struct IMAGE_SECTION_METADATA {
uint64_t ContentKey;
uint64_t VolumeId;
uint64_t FileId;
uint64_t FileGeneration;
uint64_t PreferredBase;
uint32_t EntryRva;
uint32_t HeaderSize;
size_t RawSize;
size_t ImageSize;
size_t PageCount;
uint32_t SectionCount;
} IMAGE_SECTION_METADATA;
// Creates or references an image section object backed by an open file.
KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file,
IMAGE_SECTION_OBJECT** sectionOut);
// Adds a reference to an existing image section object.
void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section);
// Releases a reference returned by LdrCreateImageSectionFromFile or LdrReferenceImageSection.
void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section);
// Returns immutable metadata for an image section object.
const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section);
// Returns the immutable PE section table owned by the image section object.
const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section,
uint32_t* countOut);
// Retains one prototype frame owned by an image section object.
KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section,
size_t pageIndex,
PHYSICAL_ADDRESS* frameOut);
// Maps an image section into a process as an image-backed VAD.
KSTATUS LdrMapImageSection(PROCESS* process,
IMAGE_SECTION_OBJECT* section,
void** baseOut);
code/kernel/include/kernel/mm_file_cache.h
+99
View File
@@ -0,0 +1,99 @@
// Kernel file-page cache declarations.
// Caches PAGE_SIZE file pages shared by reads, mmap faults, and writeback.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <ntlite/status.h>
struct FILE_OBJECT;
typedef struct FILE_OBJECT FILE_OBJECT;
typedef struct MM_FILE_CACHE_PAGE {
FILE_OBJECT* File;
uint64_t Offset;
PHYSICAL_ADDRESS Frame;
uint32_t ValidBytes;
uint8_t Dirty;
} MM_FILE_CACHE_PAGE;
typedef struct MM_FILE_CACHE_STATS {
uint64_t Lookups;
uint64_t Hits;
uint64_t Misses;
uint64_t Loads;
uint64_t LoadFailures;
uint64_t RaceDiscards;
uint64_t InflightWaits;
uint64_t Invalidations;
uint64_t DirtyPages;
uint64_t MaxDirtyPages;
uint64_t DirtyMarks;
uint64_t WritebackRequests;
uint64_t WritebackPages;
uint64_t WritebackBytes;
uint64_t WritebackFailures;
uint64_t FlushFileCalls;
uint64_t FlushFileFailures;
uint64_t MappedDirtySyncs;
uint64_t MappedDirtyPages;
uint64_t Drops;
uint64_t DropFailures;
uint64_t StandbyTransitions;
uint64_t StandbyHits;
uint64_t ReadAheadRequests;
uint64_t ReadAheadPagesRequested;
uint64_t ReadAheadPagesQueued;
uint64_t ReadAheadPagesLoaded;
uint64_t ReadAheadPagesSkipped;
uint64_t ReadAheadFailures;
uint64_t ReadAheadPagesThrottled;
uint64_t MaxReadAheadPagesPerRequest;
uint64_t BatchReads;
uint64_t BatchPagesRequested;
uint64_t BatchPagesLoaded;
uint64_t MaxBatchPagesRequested;
uint64_t MaxBatchPagesLoaded;
uint64_t ResidentPages;
uint64_t ActivePages;
uint64_t StandbyPages;
uint64_t MaxResidentPages;
} MM_FILE_CACHE_STATS;
// Returns a retained physical frame containing the file page at page-aligned
// offset. The caller owns one PFN reference and must call
// MmFileCacheReleasePage(). Bytes beyond ValidBytes are zero-filled.
KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAGE* outPage);
void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page);
// Returns a retained coherent cache page suitable for a shared writable mapping.
KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
uint64_t fileSizeBefore, int needExistingBytes,
MM_FILE_CACHE_PAGE* outPage);
// Marks one coherent cache page dirty after a shared writable mapping is written.
void MmFileCacheMarkDirty(FILE_OBJECT* file, uint64_t offset, uint32_t validBytes);
// Writes into coherent cached file pages and marks them dirty.
KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf,
size_t size, uint64_t fileSizeBefore, size_t* writtenOut);
// Flushes dirty cached pages for one file.
KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file);
// Drops cached clean pages overlapping the byte range. Dirty pages are flushed
// by callers before destructive metadata operations.
void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size);
void MmFileCacheInvalidateAll(FILE_OBJECT* file);
// Reclaims one cache-owned page if possible. Safe to call under memory pressure;
// it never writes back because v1 stores only clean file data.
KSTATUS MmFileCacheDropOnePageForPressure(void);
// Queues best-effort read-ahead for clean file pages. The queue owns a file
// reference until the worker has consumed the request.
KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages);
void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out);
code/kernel/include/kernel/mm_pfn.h
+134
View File
@@ -0,0 +1,134 @@
// Physical frame database declarations.
// The PFN layer owns frame state, reference counts, pin counts, and allocation
// policy. Higher VMM layers should ask for pages by purpose instead of treating
// the physical allocator as an untyped bitmap.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <kernel/mm_phys.h>
#define MM_PFN_INVALID_INDEX ((size_t)-1)
typedef enum MM_PFN_STATE {
MmPfnStateInvalid = 0,
MmPfnStateReserved,
MmPfnStateFree,
MmPfnStateZeroed,
MmPfnStateActive,
MmPfnStateStandby,
MmPfnStateModified,
} MM_PFN_STATE;
typedef enum MM_PFN_USAGE {
MmPfnUseNone = 0,
MmPfnUseKernel,
MmPfnUsePageTable,
MmPfnUseUser,
MmPfnUseCleanFileCache,
MmPfnUseImage,
MmPfnUseMappedFile,
MmPfnUseAnonymous,
MmPfnUseCow,
MmPfnUseSurface,
MmPfnUseDriver,
MmPfnUseDma,
MmPfnUseBootloader,
MmPfnUsePfnDatabase,
MmPfnUseMmio,
} MM_PFN_USAGE;
typedef struct MM_PFN {
uint16_t State;
uint16_t Usage;
uint16_t RefCount;
uint16_t PinCount;
uint32_t Flags;
uint32_t MappedCount;
void* Owner;
uint64_t BackingPage;
} MM_PFN;
typedef struct MM_PFN_STATS {
uint64_t TotalFrames;
uint64_t FreeFrames;
uint64_t UsedFrames;
uint64_t ReservedFrames;
uint64_t ZeroedFrames;
uint64_t ActiveFrames;
uint64_t StandbyFrames;
uint64_t ModifiedFrames;
uint64_t PinnedFrames;
uint64_t EvictableFrames;
uint64_t NonEvictableFrames;
uint64_t ReclaimableFrames;
uint64_t AvailableFrames;
uint64_t MappedFrames;
uint64_t KernelFrames;
uint64_t PageTableFrames;
uint64_t UserFrames;
uint64_t CleanFileCacheFrames;
uint64_t ActiveCleanFileCacheFrames;
uint64_t StandbyCleanFileCacheFrames;
uint64_t ImageFrames;
uint64_t ActiveImageFrames;
uint64_t StandbyImageFrames;
uint64_t MappedFileFrames;
uint64_t AnonymousFrames;
uint64_t CowFrames;
uint64_t SurfaceFrames;
uint64_t DriverFrames;
uint64_t DmaFrames;
uint64_t BootloaderFrames;
uint64_t PfnDatabaseFrames;
uint64_t MmioFrames;
uint64_t AllocationCount;
uint64_t FreeCount;
uint64_t RetainCount;
uint64_t ReleaseCount;
uint64_t PinCount;
uint64_t UnpinCount;
uint64_t FailedAllocations;
uint64_t FailedContiguousAllocations;
uint64_t FailedInvalidRange;
uint64_t FailedEmptyRange;
uint64_t FailedReserveDenied;
uint64_t FailedNoFrame;
uint64_t FailedContiguousInvalidRequest;
uint64_t FailedContiguousReserveDenied;
uint64_t FailedContiguousNoRun;
uint64_t ReclaimAttempts;
uint64_t ReclaimPagesRequested;
uint64_t ReclaimPagesSucceeded;
uint64_t ReclaimPagesFailed;
uint64_t ReclaimReentrantSkips;
} MM_PFN_STATS;
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage);
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage);
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive,
MM_PFN_USAGE usage);
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage);
__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnAddMapping(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRemoveMapping(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnMoveToStandby(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnActivate(PHYSICAL_ADDRESS frame);
__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame);
__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame);
size_t MmPfnIndex(PHYSICAL_ADDRESS frame);
PHYSICAL_ADDRESS MmPfnAddress(size_t pfn);
void MmPfnQueryStats(MM_PFN_STATS* out);
code/kernel/include/kernel/mm_phys.h
+8 -22
View File
@@ -1,8 +1,9 @@
// Kernel mm phys declarations shared by implementation files and kernel clients.
// Physical-memory declarations shared by the kernel and drivers.
// The physical layer publishes page-size constants, boot memory-map ingestion,
// and aggregate memory statistics. Frame ownership lives in the PFN database.
#pragma once
// mm_phys.h - physical memory manager (PMM)
// Bitmap allocator over 4 KB frames, seeded from MEMORY_MAP.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
@@ -22,22 +23,7 @@ typedef struct MM_PHYSICAL_STATS {
uint64_t FreeRunHistogram[8];
} MM_PHYSICAL_STATS;
// Called once by KernelMain with the MEMORY_MAP from BOOT_INFO.
bool MmInitializePhysical(const MEMORY_MAP* map,
uint64_t physicalWindowBase);
// Allocate / free a single 4 KB frame.
// MmAllocateFrame returns 0 on failure.
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void);
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive);
PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive);
void MmFreeFrame(PHYSICAL_ADDRESS frame);
// Allocate 'count' physically contiguous 4 KB frames.
// Returns 0 on failure. Use sparingly (DMA, page tables).
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateContiguous(size_t count);
size_t MmFreeFrameCount(void);
size_t MmTotalFrameCount(void);
void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out);
bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase);
size_t MmFreeFrameCount(void);
size_t MmTotalFrameCount(void);
void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out);
code/kernel/include/kernel/mm_pte.h
+105
View File
@@ -0,0 +1,105 @@
// Kernel page-table manager declarations shared by the VMM and low-level kernel clients.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <ntlite/status.h>
// x86-64 PTE bits exposed to the rest of the kernel.
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
#define PAGE_WRITETHROUGH (1ULL << 3)
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_ACCESSED (1ULL << 5)
#define PAGE_DIRTY (1ULL << 6)
#define PAGE_NX (1ULL << 63)
// The physical direct map is a kernel-only alias used to reach page-table frames and early physical memory.
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
// Page-table telemetry separates cumulative operations from the current number
// of 4 KiB leaf mappings owned by this layer. Huge boot mappings may exist but
// are not counted as leaves here.
typedef enum MM_PTE_QUERY_PURPOSE {
MmPteQueryGeneric = 0,
MmPteQueryUserProbe,
MmPteQueryHeap,
MmPteQueryVad,
MmPteQueryDebug,
MmPteQuerySelfTest,
MmPteQueryPurposeCount,
} MM_PTE_QUERY_PURPOSE;
typedef struct MM_PTE_STATS {
uint64_t MapCalls;
uint64_t MapFailures;
uint64_t UnmapCalls;
uint64_t UnmapFailures;
uint64_t ProtectCalls;
uint64_t ProtectFailures;
uint64_t QueryCalls;
uint64_t QueryFailures;
uint64_t QueryByPurpose[MmPteQueryPurposeCount];
uint64_t QueryFailuresByPurpose[MmPteQueryPurposeCount];
uint64_t TranslateCalls;
uint64_t LocalFlushes;
uint64_t GlobalFlushes;
uint64_t ShootdownRequests;
uint64_t ShootdownTargetCpus;
uint64_t ShootdownSkippedNoTargets;
uint64_t ShootdownAcks;
uint64_t ShootdownTimeouts;
uint64_t NewMappingNoFlushes;
uint64_t ReplacedMappingFlushes;
uint64_t PageTablePagesAllocated;
uint64_t CurrentLeafMappings;
} MM_PTE_STATS;
typedef enum MM_PTE_MAP_MODE {
MmPteMapFailIfPresent = 0,
MmPteMapReplaceExisting = 1,
} MM_PTE_MAP_MODE;
// Initializes the page-table manager around the bootloader-created PML4.
void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys);
// Installs the shared PML4 into CR3.
void MmPteActivate(void);
// Maps a 4 KiB page into the shared address space.
__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode);
// Removes a 4 KiB mapping from the shared address space.
__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt);
// Replaces the flags on every mapped 4 KiB page in the range while preserving the physical frame.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags);
// Tests and clears the hardware accessed bit on one mapped 4 KiB leaf.
// Returns nonzero when the page had been accessed and was given a second chance.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt);
// Tests and clears the hardware dirty bit on one mapped 4 KiB leaf.
__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt);
// Translates a virtual address through the current shared page tables.
__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt);
// Returns the physical address and effective flags for a mapped virtual address.
__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose, uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
// Flushes one address from the local TLB.
void MmPteFlushLocal(uintptr_t virt);
// Flushes a kernel mapping from every online CPU that shares the kernel page tables.
void MmPteFlushKernelAddress(uintptr_t virt);
// Handles the fixed-vector NMI used for kernel TLB shootdowns.
int MmPteHandleShootdownNmi(void);
// Returns page-table operation counters used by memory telemetry.
void MmPteQueryStats(MM_PTE_STATS* out);
code/kernel/include/kernel/mm_swap.h
+31
View File
@@ -0,0 +1,31 @@
// Kernel swap manager declarations.
// Swap owns pagefile slot allocation and page-sized I/O for anonymous VAD pages.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <kernel/mm_phys.h>
typedef struct MM_SWAP_STATS {
uint64_t Enabled;
uint64_t TotalSlots;
uint64_t UsedSlots;
uint64_t FreeSlots;
uint64_t HighWaterSlots;
uint64_t PageOuts;
uint64_t PageIns;
uint64_t PageOutFailures;
uint64_t PageInFailures;
uint64_t SlotAllocations;
uint64_t SlotFrees;
uint64_t SlotAllocationFailures;
} MM_SWAP_STATS;
KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes);
int MmSwapIsEnabled(void);
KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut);
KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame);
void MmSwapFreeSlot(uint32_t slot);
void MmSwapQueryStats(MM_SWAP_STATS* out);
code/kernel/include/kernel/mm_virt.h
+140 -76
View File
@@ -2,50 +2,43 @@
#pragma once
// mm_virt.h - virtual memory manager (VMM)
// Manages the single shared PML4, kernel pool, and page-granular user VM regions.
// Manages the single shared PML4, kernel pool, and VAD-backed user VM ownership.
#include <stdint.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pte.h>
#include <ntlite/status.h>
struct PROCESS;
typedef struct PROCESS PROCESS;
struct FILE_OBJECT;
typedef struct FILE_OBJECT FILE_OBJECT;
struct IMAGE_SECTION_OBJECT;
typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT;
typedef enum MM_MAP_MODE {
MmMapFailIfPresent = 0,
MmMapReplaceExisting = 1,
} MM_MAP_MODE;
typedef enum MM_VAD_TYPE {
MmVadGeneric = 0,
MmVadHeap,
MmVadImage,
MmVadStack,
MmVadTeb,
MmVadPeb,
MmVadEnvironment,
MmVadMappedFile,
MmVadShared,
} MM_VAD_TYPE;
typedef enum USER_REGION_TYPE {
UserRegionGeneric = 0,
UserRegionHeap,
UserRegionImage,
UserRegionStack,
UserRegionTeb,
UserRegionPeb,
UserRegionEnvironment,
UserRegionMappedFile,
UserRegionShared,
} USER_REGION_TYPE;
#define USER_REGION_GUARD_LOW 0x00000001u
#define USER_REGION_GUARD_HIGH 0x00000002u
#define USER_REGION_ZERO 0x00000004u
// Page flags (x86-64 PTE bits).
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
#define PAGE_WRITETHROUGH (1ULL << 3)
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_NX (1ULL << 63)
#define MM_VAD_GUARD_LOW 0x00000001u
#define MM_VAD_GUARD_HIGH 0x00000002u
#define MM_VAD_ZERO 0x00000004u
#define MM_VAD_PREFETCH 0x00000008u
#define MM_VAD_SHARED_WRITE 0x00000010u
// Virtual address space constants.
#define USER_BASE 0x0000000000001000ULL
#define USER_LIMIT 0x00007FFFFFFFF000ULL
#define KERNEL_BASE 0xFFFF800000000000ULL
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
@@ -55,37 +48,6 @@ typedef enum USER_REGION_TYPE {
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
void MmActivate(void);
// Page mapping.
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode);
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt);
// Modify protection flags on an already-mapped virtual range.
// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags
// entirely (except the physical address, which is preserved).
// Returns STATUS_NOT_FOUND if any page in the range is not mapped.
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags);
// Walk the page tables to translate a virtual address to physical.
// Returns 0 if the address is not mapped.
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt);
// TLB/cache coherency hooks for the shared kernel address space. Local flushes
// use INVLPG directly; kernel mapping changes broadcast an NMI shootdown to
// other online CPUs and wait for acknowledgements.
void MmFlushTlbLocal(uintptr_t virt);
void MmFlushTlbForKernelMapping(uintptr_t virt);
int MmHandleTlbShootdownNmi(void);
// User/kernel copy boundary helpers
// MmQueryPage returns the physical address and effective PAGE_* bits for a
// mapped virtual address. The reported PAGE_USER/PAGE_WRITE bits account for
// every page-table level, not just the leaf entry.
__declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
// Probe a user virtual range without copying it. These functions only accept
// lower-half user mappings with PAGE_USER set; MmProbeUserWrite additionally
// requires PAGE_WRITE.
@@ -97,31 +59,96 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size);
__declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size);
__declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size);
typedef struct MM_PINNED_USER_PAGE {
uintptr_t UserPage;
size_t PageOffset;
PHYSICAL_ADDRESS Frame;
void* KernelAddress;
uint64_t PageFlags;
} MM_PINNED_USER_PAGE;
typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
// Pins one user page and returns a kernel direct-map alias for the requested
// address. Callers may copy through KernelAddress + PageOffset while the page is
// pinned. The pin is short-lived: never sleep, do file I/O, or hold it across
// arbitrary callbacks.
__declspec(dllexport) KSTATUS MmPinUserPageForRead(PROCESS* process, const void* userAddress, MM_PINNED_USER_PAGE* out);
__declspec(dllexport) KSTATUS MmPinUserPageForWrite(PROCESS* process, void* userAddress, MM_PINNED_USER_PAGE* out);
__declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
// Locks a user buffer for asynchronous kernel access. The containing VADs stay
// alive and protected, and resident PFNs stay pinned until the buffer is
// unlocked. WriteAccess describes kernel writes to the user buffer.
__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
// Kernel pool - ring 0 only, never PAGE_USER.
// Best-fit free-list allocator, thread-safe via spinlock.
#define MM_POOL_TAG(a, b, c, d) \
((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24))
__declspec(dllexport) void* MmAllocatePool(size_t size);
__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag);
__declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// User virtual regions.
// The kernel has one shared user VA arena, but every user mapping is owned by
// a PROCESS so process teardown can reclaim leaked memory. Guard pages are
// reserved VA with no present PTE.
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t regionFlags,
USER_REGION_TYPE type, void** outBase);
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base);
void MmDestroyAllUserRegions(PROCESS* process);
KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, USER_REGION_TYPE* typeOut, uint32_t* regionFlagsOut);
void MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t* reservedBytesOut, size_t* committedBytesOut);
size_t MmGetUserRegionSize(PROCESS* process, const void* base);
size_t MmGetUserRegionSizeByBase(const void* base);
// Virtual address descriptors.
// The single shared address space still has per-process VAD ownership so process
// teardown can reclaim leaked mappings. VADs track sparse per-page commitment,
// and committed pages are materialized by user faults or explicit prefaulting.
KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t vadFlags,
MM_VAD_TYPE type, void** outBase);
KSTATUS MmVadCommit(PROCESS* process, void* base, size_t size,
uint64_t pageFlags, uint32_t commitFlags);
KSTATUS MmVadCreateCommitted(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t vadFlags,
MM_VAD_TYPE type, void** outBase);
KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file,
uint64_t fileOffset, size_t size,
uint64_t pageFlags, uint32_t vadFlags,
void** outBase);
KSTATUS MmVadCreateImageSectionMapping(PROCESS* process, IMAGE_SECTION_OBJECT* section,
uint64_t pageFlags, uint32_t vadFlags,
void** outBase);
KSTATUS MmVadEnsureResident(PROCESS* process, void* base, size_t size, uint64_t accessFlags);
KSTATUS MmVadMakeImageRangePrivate(PROCESS* process, void* base, size_t size);
KSTATUS MmVadSyncFileMappings(FILE_OBJECT* file);
KSTATUS MmVadCreateCowView(PROCESS* sourceProcess, void* sourceBase, size_t size,
PROCESS* targetProcess, void** outBase);
KSTATUS MmVadCreateCowViewFromFrames(PROCESS* targetProcess,
const PHYSICAL_ADDRESS* frames,
size_t pageCount,
size_t payloadSize,
uint64_t pageFlags,
uint32_t vadFlags,
MM_VAD_TYPE type,
void** outBase);
KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess,
const PHYSICAL_ADDRESS* frames,
size_t pageCount,
size_t payloadSize,
uint64_t pageFlags,
uint32_t vadFlags,
MM_VAD_TYPE type,
void** outBase);
KSTATUS MmVadResolveUserPageFault(PROCESS* process, uintptr_t faultAddress, uint64_t errorCode);
KSTATUS MmVadResolvePageFaultByAddress(uintptr_t faultAddress, uint64_t errorCode);
KSTATUS MmVadTrimOnePageForTest(PROCESS* process, void** baseOut);
KSTATUS MmVadTrimOnePageForPressure(void);
KSTATUS MmVadProtect(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmVadDestroy(PROCESS* process, void* base);
void MmVadDestroyAllForProcess(PROCESS* process);
KSTATUS MmVadQuery(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, MM_VAD_TYPE* typeOut, uint32_t* vadFlagsOut);
void MmVadQueryProcessMemory(PROCESS* process, size_t* vadCountOut, size_t* reservedBytesOut, size_t* committedBytesOut);
size_t MmVadGetSize(PROCESS* process, const void* base);
size_t MmVadGetSizeByBase(const void* base);
// Debug-only ownership/address diagnostics used by fault and launch paths.
void MmDebugDescribeAddress(uintptr_t addr);
void MmDebugDumpUserRegions(PROCESS* process);
void MmVadDebugDumpProcess(PROCESS* process);
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size);
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr);
@@ -140,3 +167,40 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
size_t MmGetKernelPoolUsage(void);
size_t MmGetUserHeapUsage(void);
// Demand-paging telemetry used by the shell stats command.
typedef struct MM_VAD_STATS {
uint64_t ResolveFaultCalls;
uint64_t ResolvedFaults;
uint64_t FailedFaults;
uint64_t ProtectionFaults;
uint64_t NoVadFaults;
uint64_t UncommittedFaults;
uint64_t NoAccessFaults;
uint64_t DemandZeroPages;
uint64_t ZeroPageMaps;
uint64_t ZeroPageCowSplits;
uint64_t CowFaults;
uint64_t CowPagesCopied;
uint64_t CowViewsCreated;
uint64_t CowPagesShared;
uint64_t EnsureResidentCalls;
uint64_t EnsureResidentPages;
uint64_t AlreadyResidentPages;
uint64_t FileBackedFaults;
uint64_t FileBackedReads;
uint64_t FileBackedReadFailures;
uint64_t FileBackedDiscards;
uint64_t FileBackedSharedWritableFaults;
uint64_t FileBackedDirtyFaults;
uint64_t FileBackedDirtySyncs;
uint64_t ImageBackedFaults;
uint64_t ImageBackedZeroFaults;
uint64_t ImageBackedPrivateFaults;
uint64_t ImageFixupPrivateCalls;
uint64_t ImageFixupPrivatePages;
uint64_t SwapPageOuts;
uint64_t SwapPageIns;
} MM_VAD_STATS;
void MmVadQueryStats(MM_VAD_STATS* out);
code/kernel/include/kernel/ob_object.h
+4
View File
@@ -23,6 +23,10 @@ typedef enum OBJECT_TYPE {
ObjectTypeDriver = 12,
ObjectTypeUsbDevice = 13,
ObjectTypeUsbPipe = 14,
ObjectTypeFileMapping = 15,
ObjectTypeSurface = 16,
ObjectTypeInputQueue = 17,
ObjectTypeImageSection = 18,
} OBJECT_TYPE;
typedef struct OBJECT {
code/kernel/include/kernel/syscall_nr.h
+49 -38
View File
@@ -11,13 +11,13 @@
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_WAIT_FOR_PROCESS 3
#define SYS_WAIT_SINGLE 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_WAIT_THREAD 9
#define SYS_RETIRED_WAIT_THREAD 9
#define SYS_GET_CURRENT_TID 10
#define SYS_GET_CURRENT_PID 11
#define SYS_SET_THREAD_PRIORITY 12
@@ -31,17 +31,17 @@
#define SYS_QUERY_VM 20
#define SYS_CLOSE_HANDLE 21
#define SYS_CREATE_MUTEX 22
#define SYS_ACQUIRE_MUTEX 23
#define SYS_RETIRED_ACQUIRE_MUTEX 23
#define SYS_TRY_ACQUIRE_MUTEX 24
#define SYS_RELEASE_MUTEX 25
#define SYS_CREATE_SEM 26
#define SYS_WAIT_SEM 27
#define SYS_RETIRED_WAIT_SEM 27
#define SYS_TRY_WAIT_SEM 28
#define SYS_SIGNAL_SEM 29
#define SYS_CREATE_EVENT 30
#define SYS_SIGNAL_EVENT 31
#define SYS_CLEAR_EVENT 32
#define SYS_WAIT_EVENT 33
#define SYS_RETIRED_WAIT_EVENT 33
#define SYS_CREATE_QUEUE 34
#define SYS_SEND_QUEUE 35
#define SYS_RECV_QUEUE 36
@@ -74,36 +74,47 @@
#define SYS_FILL_CONSOLE_CHARACTER 63
#define SYS_FILL_CONSOLE_ATTRIBUTE 64
#define SYS_PUT_SCREEN_CHAR 65
#define SYS_BLIT_SCREEN 66
#define SYS_CLEAR_SCREEN 67
#define SYS_GET_SCREEN_SIZE 68
#define SYS_QUERY_DISPLAY_INFO 69
#define SYS_POLL_KEYBOARD 70
#define SYS_IS_KEY_DOWN 71
#define SYS_READ_KEYBOARD_CHAR 72
#define SYS_READ_KEY 73
#define SYS_GET_KEY_MODIFIERS 74
#define SYS_POLL_MOUSE 75
#define SYS_READ_MOUSE 76
#define SYS_GET_MOUSE_POS 77
#define SYS_GET_TICK 78
#define SYS_PRINT_DEBUG 79
#define SYS_QUERY_TRAP_FRAME 80
#define SYS_CREATE_SOCKET 81
#define SYS_BIND_SOCKET 82
#define SYS_LISTEN_SOCKET 83
#define SYS_ACCEPT_SOCKET 84
#define SYS_CONNECT_SOCKET 85
#define SYS_SEND_SOCKET 86
#define SYS_RECV_SOCKET 87
#define SYS_SHUTDOWN_SOCKET 88
#define SYS_SEND_TO_SOCKET 89
#define SYS_RECV_FROM_SOCKET 90
#define SYS_RESOLVE_HOST 91
#define SYS_SET_SOCKET_OPTION 92
#define SYS_GET_SOCKET_OPTION 93
#define SYS_POLL 94
#define SYS_GET_MEMORY_STATUS 95
#define SYS_QUERY_KERNEL_STATS 96
#define SYS_GET_SYSTEM_TIME 97
#define SYS_MAX 98
#define SYS_CLEAR_SCREEN 66
#define SYS_GET_SCREEN_SIZE 67
#define SYS_QUERY_DISPLAY_INFO 68
#define SYS_POLL_KEYBOARD 69
#define SYS_IS_KEY_DOWN 70
#define SYS_READ_KEYBOARD_CHAR 71
#define SYS_READ_KEY 72
#define SYS_GET_KEY_MODIFIERS 73
#define SYS_POLL_MOUSE 74
#define SYS_READ_MOUSE 75
#define SYS_GET_MOUSE_POS 76
#define SYS_GET_TICK 77
#define SYS_PRINT_DEBUG 78
#define SYS_QUERY_TRAP_FRAME 79
#define SYS_CREATE_SOCKET 80
#define SYS_BIND_SOCKET 81
#define SYS_LISTEN_SOCKET 82
#define SYS_ACCEPT_SOCKET 83
#define SYS_CONNECT_SOCKET 84
#define SYS_SEND_SOCKET 85
#define SYS_RECV_SOCKET 86
#define SYS_SHUTDOWN_SOCKET 87
#define SYS_SEND_TO_SOCKET 88
#define SYS_RECV_FROM_SOCKET 89
#define SYS_RESOLVE_HOST 90
#define SYS_SET_SOCKET_OPTION 91
#define SYS_GET_SOCKET_OPTION 92
#define SYS_POLL 93
#define SYS_GET_MEMORY_STATUS 94
#define SYS_QUERY_KERNEL_STATS 95
#define SYS_GET_SYSTEM_TIME 96
#define SYS_CREATE_FILE_MAPPING 97
#define SYS_MAP_FILE 98
#define SYS_CREATE_SURFACE 99
#define SYS_MAP_SURFACE 100
#define SYS_QUERY_SURFACE 101
#define SYS_PRESENT_SURFACE 102
#define SYS_GET_THREAD_LIST 103
#define SYS_READ_INPUT_EVENT 104
#define SYS_PEEK_INPUT_EVENT 105
#define SYS_WAIT_MANY 106
#define SYS_OPEN_INPUT_QUEUE 107
#define SYS_FLUSH_FILE 108
#define SYS_MAX 109
code/kernel/io_device.c
+336 -6
View File
@@ -7,6 +7,7 @@
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
#include <kernel/ke_sched.h>
#define MAX_DRIVER_OBJECTS 64
#define MAX_PDO_OBJECTS 128
@@ -443,6 +444,170 @@ static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
#define IO_DISK_READAHEAD_MAX_SECTORS 128u
#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u
#define IO_DISK_READAHEAD_MIN_SECTORS 8u
#define IO_DISK_SECTOR_BYTES 512u
typedef struct IO_DISK_READAHEAD_STATE {
SPINLOCK Lock;
uint8_t* Buffer;
uint8_t Valid;
uint8_t Loading;
uint64_t Generation;
uint64_t BaseLba;
uint32_t Count;
uint64_t NextLba;
uint32_t SequentialScore;
IO_DISK_STATS Stats;
} IO_DISK_READAHEAD_STATE;
static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES];
// Updates a maximum statistic while holding the disk read-ahead lock.
static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) {
if (field && *field < value) *field = value;
}
// Prepares the per-disk read-ahead buffer.
static void IoDiskReadAheadPrepare(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
state->Lock = (SPINLOCK)SPINLOCK_INIT;
state->Valid = 0;
state->Loading = 0;
state->Generation = 0;
state->BaseLba = 0;
state->Count = 0;
state->NextLba = 0;
state->SequentialScore = 0;
memset(&state->Stats, 0, sizeof(state->Stats));
if (!state->Buffer) {
state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES,
MM_POOL_TAG('d', 'r', 'a', 'h'));
if (!state->Buffer)
state->Stats.ReadaheadAllocFailures++;
}
}
// Releases the per-disk read-ahead buffer.
static void IoDiskReadAheadRelease(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uint8_t* buffer = (uint8_t*)0;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (!state->Loading) {
buffer = state->Buffer;
state->Buffer = (uint8_t*)0;
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->SequentialScore = 0;
KeReleaseSpinlock(&state->Lock, flags);
break;
}
KeReleaseSpinlock(&state->Lock, flags);
KeYield();
}
if (buffer)
MmFreePool(buffer);
}
// Invalidates cached disk read-ahead data for one disk.
static void IoDiskReadAheadInvalidate(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Returns true when one range fully contains another range.
static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount,
uint64_t innerBase, uint32_t innerCount) {
uint64_t outerEnd = outerBase + outerCount;
uint64_t innerEnd = innerBase + innerCount;
if (outerEnd < outerBase || innerEnd < innerBase)
return false;
return innerBase >= outerBase && innerEnd <= outerEnd;
}
// Attempts to satisfy a disk read from the per-disk read-ahead buffer.
static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) {
if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0)
return false;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool hit = state->Buffer && state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, lba, count);
if (hit) {
size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES;
size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES;
memcpy(buffer, state->Buffer + offset, bytes);
state->Stats.ReadaheadHits++;
state->Stats.ReadaheadBytesServed += bytes;
} else {
state->Stats.ReadaheadMisses++;
}
KeReleaseSpinlock(&state->Lock, flags);
return hit;
}
// Records request size and sequential access state for one disk read.
static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES || count == 0)
return 0;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadRequests++;
state->Stats.ReadSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count);
if (lba == state->NextLba) {
if (state->SequentialScore < 32)
state->SequentialScore++;
} else {
state->SequentialScore = 1;
}
state->NextLba = lba + count;
uint32_t score = state->SequentialScore;
if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE)
state->Stats.ReadaheadSequentialDetections++;
KeReleaseSpinlock(&state->Lock, flags);
return score;
}
// Records one disk write and invalidates stale read-ahead data.
static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.WriteRequests++;
state->Stats.WriteSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Implements the notify Disk Change operation.
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
@@ -477,6 +642,8 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
if (pnpStatus == STATUS_NOT_SUPPORTED)
device->PnpState = DevicePnpStarted;
IoDiskReadAheadPrepare(driveIndex);
IoReferenceDeviceObject(device);
gDiskDevices[driveIndex] = device;
IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
@@ -490,6 +657,8 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return STATUS_NOT_FOUND;
IoDiskReadAheadInvalidate(driveIndex);
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
@@ -518,6 +687,7 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoDiskReadAheadRelease(driveIndex);
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
@@ -632,6 +802,112 @@ __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
return st;
}
// Validates that a sector range fits the disk and IRP byte length.
static KSTATUS IoValidateDiskSectorRange(DEVICE_OBJECT* dev, uint64_t lba, uint32_t count) {
if (!dev || count == 0) return STATUS_INVALID_PARAMETER;
if (count > (0xffffffffu / IO_DISK_SECTOR_BYTES)) return STATUS_INVALID_PARAMETER;
if (lba > dev->u.Disk.SectorCount) return STATUS_INVALID_PARAMETER;
if ((uint64_t)count > dev->u.Disk.SectorCount - lba) return STATUS_INVALID_PARAMETER;
return STATUS_SUCCESS;
}
// Submits one raw disk read IRP.
static KSTATUS IoReadDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
uint32_t count, void* buffer) {
if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
return IoSubmitIrpSync(dev, &irp);
}
// Submits one raw disk write IRP.
static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
uint32_t count, void* buffer) {
if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, buffer);
return IoSubmitIrpSync(dev, &irp);
}
// Chooses a conservative read-ahead span from the current sequential score.
static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) {
uint32_t sectors = currentCount;
if (sectors < IO_DISK_READAHEAD_MIN_SECTORS)
sectors = IO_DISK_READAHEAD_MIN_SECTORS;
if (score >= 8 && sectors < 128u)
sectors = 128u;
else if (score >= 4 && sectors < 64u)
sectors = 64u;
else if (score >= 2 && sectors < 32u)
sectors = 32u;
if (sectors > IO_DISK_READAHEAD_MAX_SECTORS)
sectors = IO_DISK_READAHEAD_MAX_SECTORS;
return sectors;
}
// Loads the next sequential disk range into the read-ahead buffer.
static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev,
uint64_t nextLba, uint32_t currentCount, uint32_t score) {
if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
if (!state->Buffer) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score);
uint64_t diskSectors = dev->u.Disk.SectorCount;
if (nextLba >= diskSectors)
return;
if ((uint64_t)sectors > diskSectors - nextLba)
sectors = (uint32_t)(diskSectors - nextLba);
if (sectors == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool covered = state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors);
if (covered) {
KeReleaseSpinlock(&state->Lock, flags);
return;
}
if (state->Loading) {
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint64_t generation = state->Generation;
state->Loading = 1;
state->Valid = 0;
state->Count = 0;
KeReleaseSpinlock(&state->Lock, flags);
KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer);
flags = KeAcquireSpinlock(&state->Lock);
state->Loading = 0;
if (KSUCCESS(st) && generation == state->Generation) {
state->BaseLba = nextLba;
state->Count = sectors;
state->Valid = 1;
state->Stats.ReadaheadStores++;
state->Stats.ReadaheadSectorsLoaded += sectors;
IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors);
} else {
state->Valid = 0;
state->Count = 0;
if (!KSUCCESS(st))
state->Stats.ReadaheadReadFailures++;
}
KeReleaseSpinlock(&state->Lock, flags);
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
@@ -641,9 +917,23 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
if (!KSUCCESS(rangeStatus)) {
IoDereferenceDeviceObject(dev);
return rangeStatus;
}
if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) {
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
}
KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
if (KSUCCESS(st))
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return st;
@@ -658,10 +948,50 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
if (!KSUCCESS(rangeStatus)) {
IoDereferenceDeviceObject(dev);
return rangeStatus;
}
KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
if (KSUCCESS(st))
IoDiskReadAheadNoteWrite(driveIndex, count);
IoDereferenceDeviceObject(dev);
return st;
}
// Queries aggregate disk I/O statistics.
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (gDiskDevices[i])
out->RegisteredDisks++;
out->ReadRequests += state->Stats.ReadRequests;
out->ReadSectors += state->Stats.ReadSectors;
out->WriteRequests += state->Stats.WriteRequests;
out->WriteSectors += state->Stats.WriteSectors;
if (out->MaxReadSectors < state->Stats.MaxReadSectors)
out->MaxReadSectors = state->Stats.MaxReadSectors;
if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
out->MaxWriteSectors = state->Stats.MaxWriteSectors;
out->ReadaheadHits += state->Stats.ReadaheadHits;
out->ReadaheadMisses += state->Stats.ReadaheadMisses;
out->ReadaheadStores += state->Stats.ReadaheadStores;
out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations;
out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed;
out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded;
out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections;
out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed;
out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures;
out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures;
if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors)
out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors;
KeReleaseSpinlock(&state->Lock, flags);
}
}
code/kernel/io_file.c
+283 -18
View File
@@ -4,6 +4,8 @@
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/mm_file_cache.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_userenv.h>
@@ -13,6 +15,7 @@
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <ntlite/syscall.h>
#define MAX_DRIVES 8
@@ -45,11 +48,15 @@ struct IO_ASYNC_FILE_REQUEST_OBJECT {
uint8_t CancelRequested;
FILE_OBJECT* File; // referenced until object destruction
EVENT* WaitEvent; // internal completion event
EVENT* CompletionEvent; // optional, referenced until object destruction
KAPC* CompletionApc; // caller-owned APC storage
IO_STATUS_BLOCK* Iosb; // caller/user-owned completion block
IO_STATUS_BLOCK IosbSnapshot;
MM_LOCKED_USER_BUFFER* LockedBuffer;
MM_LOCKED_USER_BUFFER* LockedIosb;
void* Buffer; // caller/user-owned; must remain valid until completion/cancel
void* Buffer; // caller-owned; user buffers must be locked until completion/cancel
size_t Size;
uint64_t Offset;
};
@@ -65,6 +72,10 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header);
if (req->WaitEvent) {
ObDereferenceObject(&req->WaitEvent->Header);
req->WaitEvent = (EVENT*)0;
}
if (req->CompletionEvent) {
ObDereferenceObject(&req->CompletionEvent->Header);
req->CompletionEvent = (EVENT*)0;
@@ -73,6 +84,14 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
ObDereferenceObject(&req->File->Header);
req->File = (FILE_OBJECT*)0;
}
if (req->LockedBuffer) {
MmUnlockUserBuffer(req->LockedBuffer);
req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
}
if (req->LockedIosb) {
MmUnlockUserBuffer(req->LockedIosb);
req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
}
}
static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
@@ -80,6 +99,9 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
size_t information) {
if (!req) return;
req->IosbSnapshot.Status = status;
req->IosbSnapshot.Information = information;
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
req->State = (status == STATUS_CANCELLED) ? IoAsyncStateCancelled : IoAsyncStateCompleted;
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
@@ -99,6 +121,18 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
if (req->CompletionEvent)
KeSignalEvent(req->CompletionEvent);
if (req->WaitEvent)
KeSignalEvent(req->WaitEvent);
if (req->LockedBuffer) {
MmUnlockUserBuffer(req->LockedBuffer);
req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
}
if (req->LockedIosb) {
MmUnlockUserBuffer(req->LockedIosb);
req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
}
}
// Implements the io Async File Dequeue operation.
@@ -226,10 +260,24 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
if (!file || !iosb) return STATUS_INVALID_PARAMETER;
if (!buffer && size) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (!file || !iosb) {
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_INVALID_PARAMETER;
}
if (!buffer && size) {
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_INVALID_PARAMETER;
}
if (!KeCanWait()) {
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_INVALID_IRQL;
}
iosb->Status = STATUS_PENDING;
iosb->Information = 0;
@@ -239,6 +287,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
sizeof(IO_ASYNC_FILE_REQUEST_OBJECT));
if (!req) {
iosb->Status = STATUS_NO_MEMORY;
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_NO_MEMORY;
}
@@ -247,10 +297,23 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
req->Op = op;
req->State = IoAsyncStateQueued;
req->CancelRequested = 0;
KSTATUS waitStatus = KeEventCreate(EventManualReset, 0, &req->WaitEvent);
if (!KSUCCESS(waitStatus)) {
iosb->Status = waitStatus;
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
ObDereferenceObject(&req->Header);
return waitStatus;
}
req->File = file;
req->CompletionEvent = completionEvent;
req->CompletionApc = completionApc;
req->Iosb = iosb;
req->IosbSnapshot.Status = STATUS_PENDING;
req->IosbSnapshot.Information = 0;
req->LockedBuffer = lockedBuffer;
req->LockedIosb = lockedIosb;
req->Buffer = buffer;
req->Size = size;
req->Offset = offset;
@@ -288,6 +351,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
ObDereferenceObject(&req->Header); // provider rejected synchronously
iosb->Status = ns;
req->IosbSnapshot.Status = ns;
req->IosbSnapshot.Information = 0;
ObDereferenceObject(&req->Header); // creator reference
return ns;
}
@@ -295,6 +360,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
KSTATUS s = KiIoEnsureAsyncFileIoStarted();
if (!KSUCCESS(s)) {
iosb->Status = s;
req->IosbSnapshot.Status = s;
req->IosbSnapshot.Information = 0;
ObDereferenceObject(&req->Header);
return s;
}
@@ -323,10 +390,12 @@ __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset,
iosb, completionEvent, completionApc,
requestOut);
lockedBuffer, lockedIosb, requestOut);
}
__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
@@ -334,10 +403,12 @@ __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const vo
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset,
iosb, completionEvent, completionApc,
requestOut);
lockedBuffer, lockedIosb, requestOut);
}
// Implements the async File Request Object operation.
@@ -345,6 +416,19 @@ __declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJ
return req ? &req->Header : (OBJECT*)0;
}
// Returns the wait event owned by an async file request.
__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? req->WaitEvent : (EVENT*)0;
}
// Waits for an async file request to complete according to wait flags.
__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, uint32_t flags) {
if (!req || !req->WaitEvent) return STATUS_INVALID_PARAMETER;
if (flags & ~SYS_WAIT_FLAG_ALERTABLE) return STATUS_INVALID_PARAMETER;
return KeWaitForEvent(req->WaitEvent, (flags & SYS_WAIT_FLAG_ALERTABLE) ? 1 : 0);
}
// Cancels async File Request.
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
if (!req) return STATUS_INVALID_PARAMETER;
@@ -377,13 +461,8 @@ __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJ
__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
IO_STATUS_BLOCK* iosbOut) {
if (!req || !iosbOut) return STATUS_INVALID_PARAMETER;
if (req->Iosb) {
iosbOut->Status = req->Iosb->Status;
iosbOut->Information = req->Iosb->Information;
} else {
iosbOut->Status = STATUS_FAILURE;
iosbOut->Information = 0;
}
iosbOut->Status = req->IosbSnapshot.Status;
iosbOut->Information = req->IosbSnapshot.Information;
return STATUS_SUCCESS;
}
@@ -622,10 +701,51 @@ __declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
}
// Aggregates filesystem traversal statistics across mounted volumes.
__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
for (uint8_t i = 0; i < MAX_DRIVES; i++) {
IO_FILESYSTEM* fs = gVolumes[i].Fs;
if (!fs || !fs->QueryStats) continue;
IO_FILESYSTEM_STATS one;
memset(&one, 0, sizeof(one));
fs->QueryStats(fs, &one);
out->PathLookups += one.PathLookups;
out->PathCacheHits += one.PathCacheHits;
out->PathCacheMisses += one.PathCacheMisses;
out->PathCacheStores += one.PathCacheStores;
out->PathCacheInvalidations += one.PathCacheInvalidations;
if (one.PathCacheGeneration > out->PathCacheGeneration)
out->PathCacheGeneration = one.PathCacheGeneration;
out->DirectorySearches += one.DirectorySearches;
out->DirectoryClustersRead += one.DirectoryClustersRead;
out->DirectoryEntriesScanned += one.DirectoryEntriesScanned;
out->FatChainCacheLookups += one.FatChainCacheLookups;
out->FatChainCacheHits += one.FatChainCacheHits;
out->FatChainCacheMisses += one.FatChainCacheMisses;
out->FatChainCacheStores += one.FatChainCacheStores;
out->FatChainCacheInvalidations += one.FatChainCacheInvalidations;
out->FatChainWalkSteps += one.FatChainWalkSteps;
out->DataReadRuns += one.DataReadRuns;
out->DataReadSectors += one.DataReadSectors;
out->DataReadRunMerges += one.DataReadRunMerges;
if (one.MaxDataReadSpanSectors > out->MaxDataReadSpanSectors)
out->MaxDataReadSpanSectors = one.MaxDataReadSpanSectors;
}
}
// Destructor.
static void KiFileDestroy(OBJECT* obj) {
FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx);
if (f->Fs && f->FileCtx) {
(void)IoFlushFile(f);
MmFileCacheInvalidateAll(f);
f->Fs->Close(f->Fs, f->FileCtx);
}
}
// IoOpenFile.
@@ -655,11 +775,73 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
f->Lock = (SPINLOCK)SPINLOCK_INIT;
f->Size = fs->GetSize(fs, fileCtx);
f->Offset = 0;
f->Flags = 0;
f->ReadaheadScore = 0;
f->ReadaheadNextOffset = 0;
f->ReadaheadIssuedUntil = 0;
f->ReadaheadSuppressedUntil = 0;
*out = f;
return STATUS_SUCCESS;
}
static uint64_t KiAlignUpPage64(uint64_t value) {
return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
}
static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) {
if (!file || bytesRead < 512)
return;
if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0)
return;
uint64_t end = offset + bytesRead;
uint64_t start = 0;
uint32_t pages = 0;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
if (offset == file->ReadaheadNextOffset) {
if (file->ReadaheadScore < 32)
file->ReadaheadScore++;
} else {
file->ReadaheadScore = 1;
file->ReadaheadIssuedUntil = end;
}
file->ReadaheadNextOffset = end;
if (end < file->ReadaheadSuppressedUntil) {
pages = 0;
} else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) {
pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u);
start = KiAlignUpPage64(end);
if (start < file->ReadaheadIssuedUntil)
start = KiAlignUpPage64(file->ReadaheadIssuedUntil);
if (start >= fileSize) {
pages = 0;
} else {
uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE);
if (maxEnd > fileSize)
pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE);
file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE);
}
}
KeReleaseSpinlock(&file->Lock, flags);
if (pages) {
KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages);
if (ras == STATUS_QUEUE_FULL) {
flags = KeAcquireSpinlock(&file->Lock);
uint64_t suppressUntil = end + (1024u * 1024u);
if (suppressUntil < end || suppressUntil > fileSize)
suppressUntil = fileSize;
if (file->ReadaheadSuppressedUntil < suppressUntil)
file->ReadaheadSuppressedUntil = suppressUntil;
KeReleaseSpinlock(&file->Lock, flags);
}
}
}
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -679,7 +861,50 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
size_t remaining = (size_t)(fileSize - offset);
if (size > remaining) size = remaining;
return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead);
if ((file->Flags & IO_FILE_FLAG_NO_CACHE) || size < 512)
return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead);
size_t total = 0;
uint8_t* dst = (uint8_t*)buf;
while (total < size) {
uint64_t absolute = offset + total;
uint64_t pageOffset = absolute & ~(uint64_t)(PAGE_SIZE - 1);
size_t inPage = (size_t)(absolute - pageOffset);
size_t chunk = PAGE_SIZE - inPage;
if (chunk > size - total) chunk = size - total;
MM_FILE_CACHE_PAGE page;
KSTATUS s = MmFileCacheGetPage(file, pageOffset, &page);
if (!KSUCCESS(s)) {
if (total) {
*bytesRead = total;
return STATUS_SUCCESS;
}
return s;
}
if (inPage >= page.ValidBytes) {
MmFileCacheReleasePage(&page);
break;
}
size_t available = page.ValidBytes - inPage;
if (chunk > available) chunk = available;
void* srcPage = MmPfnToVirtual(page.Frame);
if (!srcPage) {
MmFileCacheReleasePage(&page);
return total ? STATUS_SUCCESS : STATUS_FAILURE;
}
memcpy(dst + total, (const uint8_t*)srcPage + inPage, chunk);
MmFileCacheReleasePage(&page);
total += chunk;
if (chunk == 0) break;
}
*bytesRead = total;
KiIoMaybeScheduleReadAhead(file, offset, total, fileSize);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -691,10 +916,21 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
if (!file->Fs->Write) return STATUS_NOT_SUPPORTED;
if (size == 0) return STATUS_SUCCESS;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t oldSize = file->Size;
KeReleaseSpinlock(&file->Lock, flags);
size_t written = 0;
KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
KSTATUS s;
if (file->Flags & IO_FILE_FLAG_NO_CACHE) {
s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
if (KSUCCESS(s) && written)
MmFileCacheInvalidate(file, offset, written);
} else {
s = MmFileCacheWrite(file, offset, buf, size, oldSize, &written);
}
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
flags = KeAcquireSpinlock(&file->Lock);
uint64_t end = offset + written;
if (end > file->Size) file->Size = end;
KeReleaseSpinlock(&file->Lock, flags);
@@ -704,6 +940,16 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
return s;
}
// Flushes dirty mapped and cached pages for one file object.
__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) {
if (!file) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
KSTATUS sync = MmVadSyncFileMappings(file);
KSTATUS flush = MmFileCacheFlushFile(file);
return KSUCCESS(sync) ? flush : sync;
}
// IoReadFile - sequential read at current offset.
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead) {
@@ -768,6 +1014,11 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o
f->Lock = (SPINLOCK)SPINLOCK_INIT;
f->Size = 0;
f->Offset = 0;
f->Flags = 0;
f->ReadaheadScore = 0;
f->ReadaheadNextOffset = 0;
f->ReadaheadIssuedUntil = 0;
f->ReadaheadSuppressedUntil = 0;
*out = f;
return STATUS_SUCCESS;
@@ -848,8 +1099,12 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (!file->Fs->Truncate) return STATUS_NOT_SUPPORTED;
KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize);
KSTATUS flush = IoFlushFile(file);
if (!KSUCCESS(flush)) return flush;
KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize, file->Flags);
if (KSUCCESS(s)) {
MmFileCacheInvalidate(file, newSize, (size_t)-1);
file->Size = newSize;
// Pull the file pointer back if it now sits past EOF.
if (file->Offset > file->Size)
@@ -875,6 +1130,16 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
return size;
}
// Queries the filesystem identity for a file object.
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) {
if (!file || !out)
return STATUS_INVALID_PARAMETER;
memset(out, 0, sizeof(*out));
if (!file->Fs || !file->Fs->QueryIdentity)
return STATUS_NOT_SUPPORTED;
return file->Fs->QueryIdentity(file->Fs, file->FileCtx, out);
}
// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
if (file) ObDereferenceObject(&file->Header);
code/kernel/ke_apc.c
+1 -1
View File
@@ -204,7 +204,7 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
if (thread->State == ThreadDead)
return STATUS_OBJECT_DEAD;
KUSER_APC* apc = (KUSER_APC*)MmAllocatePool(sizeof(KUSER_APC));
KUSER_APC* apc = (KUSER_APC*)MmAllocatePoolTag(sizeof(KUSER_APC), MM_POOL_TAG('a', 'p', 'c', ' '));
if (!apc)
return STATUS_NO_MEMORY;
code/kernel/ke_debug.c
+254 -22
View File
@@ -1,20 +1,24 @@
// Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <kernel/ke_debug.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_work.h>
#include <kernel/mm_virt.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor
@@ -22,15 +26,25 @@ int gDebugEnabled = 0;
static uint32_t gDebugFlags = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_DEBUG_SCREEN_DRAIN_SIZE 512u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static char* gDebugScreenBuffer;
static size_t gDebugScreenSize;
static size_t gDebugScreenRead;
static size_t gDebugScreenWrite;
static size_t gDebugScreenUsed;
static volatile long gDebugLogFlushing;
static volatile long gDebugConsoleOwnsSerial;
static volatile long gDebugConsoleWorkReady;
static KWORK_ITEM gDebugConsoleWorkItem;
// Implements the debug Log Append Locked operation.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
@@ -55,6 +69,95 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
gDebugLogUsed += len;
}
// Implements the debug Screen Append Locked operation.
static void KiDebugScreenAppendLocked(const char* text, size_t len) {
if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0)
return;
if (len >= gDebugScreenSize) {
text += len - (gDebugScreenSize - 1u);
len = gDebugScreenSize - 1u;
}
while (gDebugScreenUsed + len > gDebugScreenSize) {
gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize;
gDebugScreenUsed--;
}
size_t first = gDebugScreenSize - gDebugScreenWrite;
if (first > len) first = len;
memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first);
if (len > first)
memcpy(gDebugScreenBuffer, text + first, len - first);
gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize;
gDebugScreenUsed += len;
}
// Implements the debug Screen Drain Locked operation.
static size_t KiDebugScreenDrainLocked(char* out, size_t max) {
if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0)
return 0;
size_t len = gDebugScreenUsed;
if (len > max) len = max;
size_t first = gDebugScreenSize - gDebugScreenRead;
if (first > len) first = len;
memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first);
if (len > first)
memcpy(out + first, gDebugScreenBuffer, len - first);
gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize;
gDebugScreenUsed -= len;
return len;
}
// Ensures the asynchronous screen ring exists before console output starts using it.
static bool KiDebugEnsureScreenBuffer(void) {
if (gDebugScreenBuffer && gDebugScreenSize != 0)
return true;
char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r'));
if (!buffer)
return false;
memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE);
bool installed = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (!gDebugScreenBuffer) {
gDebugScreenBuffer = buffer;
gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE;
gDebugScreenRead = 0;
gDebugScreenWrite = 0;
gDebugScreenUsed = 0;
installed = true;
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (!installed)
MmFreePool(buffer);
return true;
}
// Implements the debug Serial Write Char operation.
static void KiDebugSerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n') {
__outbyte(0x3F8, '\r');
while ((__inbyte(0x3FD) & 0x20) == 0) { }
}
__outbyte(0x3F8, (uint8_t)c);
}
// Implements the debug Serial Write Locked operation.
static void KiDebugSerialWriteLocked(const char* text, size_t len) {
if (!text)
return;
for (size_t i = 0; i < len; i++)
KiDebugSerialWriteChar(text[i]);
}
// Implements the debug Console Write operation.
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
@@ -63,6 +166,28 @@ static void KiDebugConsoleWrite(const char* text, size_t len) {
PrintChar(text[i], 0xCCCCCC, 0x012456);
}
// Implements the debug Console Work operation.
static void KiDebugConsoleWork(void* context) {
(void)context;
char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE];
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk));
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (len == 0)
break;
KiDebugConsoleWrite(chunk, len);
}
}
// Queues the debug Console Work operation if the passive work queue is ready.
static void KiDebugQueueConsoleWork(void) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted())
(void)KeQueueWorkItem(&gDebugConsoleWorkItem);
}
static volatile long gPanicInProgress = 0;
static volatile uintptr_t gPanicOwnerToken = 0;
static volatile long gPanicStoppedCpuCount = 0;
@@ -116,7 +241,7 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
if (KiPanicIsActive())
KiPanicSecondaryHalt();
if (MmHandleTlbShootdownNmi())
if (MmPteHandleShootdownNmi())
return 1;
return 0;
@@ -124,6 +249,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
// Implements the kernel Put Char operation.
void KiKernelPutChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xCCCCCC, 0x012456);
}
@@ -135,8 +262,11 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0))
KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0);
if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE);
gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
if (gDebugLogBuffer) {
gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
gDebugLogWrite = 0;
@@ -144,6 +274,9 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
memset(gDebugLogBuffer, 0, gDebugLogSize);
}
}
if (flags & KE_DEBUG_CONSOLE)
(void)KiDebugEnsureScreenBuffer();
}
// Returns debug Flags.
@@ -157,6 +290,13 @@ void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
return;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
// After the display manager owns debug drawing, DbgPrint and user console
// writes mirror to COM1 themselves. The display writer is screen-only so
// asynchronous screen flushing does not duplicate serial bytes.
(void)KiDebugEnsureScreenBuffer();
_InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1);
KiDebugQueueConsoleWork();
}
// Implements the dbg Print operation.
@@ -179,17 +319,37 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (KiPanicIsActive()) {
if (!KiPanicCurrentCpuOwnsReport())
KiPanicSecondaryHalt();
if (gDebugFlags & KE_DEBUG_CONSOLE)
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
KiDebugConsoleWrite(line, len);
}
return;
}
bool queueConsole = false;
bool writeConsoleSync = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE)
KiDebugConsoleWrite(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (KeIsWorkQueueStarted() && gDebugScreenBuffer) {
KiDebugScreenAppendLocked(line, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(line, len);
}
// Flushes debug Log To File.
@@ -242,6 +402,23 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// Attempts to persist the debug log during a panic when the filesystem is usable.
static void KiTryFlushPanicDebugLog(void) {
if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
return;
if (!KeCanWait() || !KeSystemRootReady())
return;
char path[1024];
if (!KeBuildSystemRootPath("kernel-panic.log", path, sizeof(path)))
return;
KSTATUS st = KeFlushDebugLogToFile(SysSvFromCstr(path));
if (!KSUCCESS(st))
DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
}
// KePanic.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
KiBeginFatalReport();
@@ -279,15 +456,39 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
va_end(va);
}
KiTryFlushPanicDebugLog();
for (;;) __halt();
}
// KePrintConsole - write user-visible console text using the default console palette.
// Panic output keeps a distinct color so fatal paths remain visually obvious.
// User text mirrors to COM1 immediately, then queues expensive screen drawing
// through the passive debug-console worker when that path is available.
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
if (!buf || !PrintChar) return;
for (size_t i = 0; i < len; i++)
PrintChar(buf[i], 0xCCCCCC, 0x012456);
if (!buf || len == 0) return;
bool queueConsole = false;
bool writeConsoleSync = false;
bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer();
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(buf, len);
if (PrintChar) {
if (!KiPanicIsActive() && canQueueScreen) {
KiDebugScreenAppendLocked(buf, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(buf, len);
}
// KeBugCheckProcess - user-mode fault handler
@@ -613,8 +814,22 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
// Prints crash Char.
static void KiPrintCrashChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic
PrintChar(c, 0xFFAA00, 0x000000);
}
static KSTATUS KiFaultExitStatus(int vector) {
switch (vector) {
case 0: return STATUS_PROCESS_DIVIDE_BY_ZERO;
case 3: return STATUS_PROCESS_BREAKPOINT;
case 6: return STATUS_PROCESS_ILLEGAL_OPCODE;
case 13: return STATUS_PROCESS_GENERAL_PROTECTION;
case 14: return STATUS_PROCESS_PAGE_FAULT;
case 17: return STATUS_PROCESS_ALIGNMENT_FAULT;
default: return STATUS_PROCESS_UNKNOWN_FAULT;
}
}
static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
@@ -648,13 +863,23 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
}
// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread) {
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
printf_set_putchar(KiKernelPutChar);
if (proc) KeExitProcess(proc, -1);
if (thread) KeExitThread(thread, -1);
int exitCode = (int)exitStatus;
if (proc) (void)KeTerminateProcess(proc, exitCode);
if (thread) KeExitThread(thread, exitCode);
while (1) __halt();
// KeExitThread normally switches away and never resumes this context. If
// a fault path re-enters here during scheduler bring-up or with no runnable
// peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
// dead thread to the scheduler with interrupts enabled so waiters such as
// the foreground shell can observe the process exit event.
while (true) {
_enable();
KeYield();
__halt();
}
}
_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
@@ -664,11 +889,11 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
KiPrintCrashHeader(proc, vector, rip, errCode, cr2);
KiDumpModules(proc);
KiFinishFaultedProcess(proc, thread);
KiFinishFaultedProcess(proc, thread, KiFaultExitStatus(vector));
}
// Implements the fault Dispatch Frame operation.
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Dispatches a CPU fault frame and returns only when a user #PF was repaired.
__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
// the interrupted context on a secondary CPU once another CPU owns panic.
if (KiPanicIsActive())
@@ -682,6 +907,12 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
(unsigned long long)(uintptr_t)frame);
}
if (frame->Vector == 14) {
KSTATUS handled = MmVadResolvePageFaultByAddress((uintptr_t)frame->Cr2, frame->ErrorCode);
if (KSUCCESS(handled))
return STATUS_SUCCESS;
}
if ((frame->Cs & 3) == 0) {
KiBeginFatalReport();
@@ -702,7 +933,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
if (vector == 14) {
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
MmDebugDumpUserRegions(proc);
MmVadDebugDumpProcess(proc);
}
if (vector == 14) {
@@ -743,7 +974,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
KiDumpFaultFrame(proc, frame);
KiDumpModules(proc);
KiFinishFaultedProcess(proc, thread);
KiFinishFaultedProcess(proc, thread, KiFaultExitStatus((int)frame->Vector));
}
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
@@ -762,5 +993,6 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
frame.Cr2 = cr2;
frame.Rip = rip;
frame.Cs = cs;
KeFaultDispatchFrame(&frame);
(void)KeFaultDispatchFrame(&frame);
while (1) __halt();
}
code/kernel/ke_display.c
+1150 -282
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_event.c
+204 -6
View File
@@ -1,17 +1,100 @@
// Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
#include <stdbool.h>
#include <intrin.h>
#include <kernel/ke_event.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_apc.h>
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
#include <ntlite/syscall.h>
// Multi-wait context shared by the stack wait blocks for one wait call.
typedef struct KE_EVENT_MULTI_WAIT_CONTEXT KE_EVENT_MULTI_WAIT_CONTEXT;
typedef struct KE_EVENT_WAIT_BLOCK {
LIST_ENTRY Link;
KE_EVENT_MULTI_WAIT_CONTEXT* Context;
EVENT* Event;
uint32_t Index;
uint8_t Armed;
} KE_EVENT_WAIT_BLOCK;
struct KE_EVENT_MULTI_WAIT_CONTEXT {
THREAD* Thread;
KE_EVENT_WAIT_BLOCK* Blocks;
uint32_t Count;
volatile long Completed;
KSTATUS Status;
uint32_t SelectedIndex;
};
// Implements the event Destroy operation.
static void KiEventDestroy(OBJECT* obj) { (void)obj; }
// Implements the event Create operation.
// Completes a multi-wait block while the source event lock is held.
static void KiCompleteMultiWaitBlockLocked(KE_EVENT_WAIT_BLOCK* block) {
if (!block || !block->Armed)
return;
ListRemove(&block->Link);
block->Armed = 0;
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
if (!ctx)
return;
if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) != 0)
return;
ctx->Status = STATUS_SUCCESS;
ctx->SelectedIndex = block->Index;
KeUnblockThreadFromWaitListWithStatus(ctx->Thread, STATUS_SUCCESS);
}
// Removes any wait blocks that are still linked after the wait completed.
static void KiCleanupMultiWaitBlocks(KE_EVENT_MULTI_WAIT_CONTEXT* ctx) {
if (!ctx || !ctx->Blocks)
return;
for (uint32_t i = 0; i < ctx->Count; i++) {
KE_EVENT_WAIT_BLOCK* block = &ctx->Blocks[i];
EVENT* event = block->Event;
if (!event)
continue;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (block->Armed) {
ListRemove(&block->Link);
block->Armed = 0;
}
KeReleaseSpinlock(&event->Lock, flags);
}
}
// Removes a multi-wait from every event list during abort or thread teardown.
static void KiRundownMultiWait(THREAD* thread, void* context, KSTATUS status) {
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = (KE_EVENT_MULTI_WAIT_CONTEXT*)context;
if (!ctx)
return;
if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) == 0) {
ctx->Status = status;
ctx->SelectedIndex = 0;
}
KiCleanupMultiWaitBlocks(ctx);
// Abort wakes a truly blocked waiter. Thread teardown may call this after a
// signal already made the thread ready; in that case only the list cleanup
// above is needed.
if (thread && thread->State == ThreadWaiting)
KeUnblockThreadFromWaitListWithStatus(thread, status);
}
// Creates event.
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
if (!out) return STATUS_INVALID_PARAMETER;
EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
@@ -21,6 +104,7 @@ __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialSta
e->Type = type;
e->Signaled = initialState ? 1 : 0;
LIST_INIT(&e->WaitList);
LIST_INIT(&e->MultiWaitList);
*out = e;
return STATUS_SUCCESS;
}
@@ -35,22 +119,39 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
while (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(t, (LIST_ENTRY*)0);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
flags = KeAcquireSpinlock(&event->Lock);
}
while (!LIST_IS_EMPTY(&event->MultiWaitList)) {
LIST_ENTRY* node = event->MultiWaitList.Fwd;
KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link);
KiCompleteMultiWaitBlockLocked(block);
}
} else {
if (!LIST_IS_EMPTY(&event->WaitList)) {
// Auto-reset handoff: transfer the signal directly to exactly one
// waiter. Do not leave Signaled=1, because a later waiter can steal
// the disk/APC completion before the selected thread resumes.
while (!LIST_IS_EMPTY(&event->WaitList)) {
event->Signaled = 0;
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(t, (LIST_ENTRY*)0);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
return;
}
while (!LIST_IS_EMPTY(&event->MultiWaitList)) {
LIST_ENTRY* node = event->MultiWaitList.Fwd;
KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link);
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
if (ctx && _InterlockedCompareExchange(&ctx->Completed, 0, 0) == 0) {
event->Signaled = 0;
KiCompleteMultiWaitBlockLocked(block);
KeReleaseSpinlock(&event->Lock, flags);
return;
}
KiCompleteMultiWaitBlockLocked(block);
}
event->Signaled = 1;
}
KeReleaseSpinlock(&event->Lock, flags);
@@ -64,7 +165,22 @@ __declspec(dllexport) void KeClearEvent(EVENT* event) {
KeReleaseSpinlock(&event->Lock, flags);
}
// Waits for for Event.
// Attempts to consume an event signal without blocking.
__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event) {
if (!event) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (!event->Signaled) {
KeReleaseSpinlock(&event->Lock, flags);
return STATUS_TIMEOUT;
}
if (event->Type == EventAutoReset)
event->Signaled = 0;
KeReleaseSpinlock(&event->Lock, flags);
return STATUS_SUCCESS;
}
// Waits for Event.
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
if (!event) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -85,3 +201,85 @@ __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
return KeCommitBlockThread(self, alertable);
}
// Waits for any of a set of events, returning the selected index.
__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count,
uint8_t alertable, uint64_t timeoutMs,
uint32_t* signaledIndexOut) {
if (!events || count == 0 || count > SYS_WAIT_MANY_MAX_HANDLES)
return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (alertable && KeAreKernelApcsDisabled()) return STATUS_INVALID_PARAMETER;
for (uint32_t i = 0; i < count; i++) {
if (!events[i]) return STATUS_INVALID_PARAMETER;
KSTATUS s = KeTryWaitEvent(events[i]);
if (KSUCCESS(s)) {
if (signaledIndexOut) *signaledIndexOut = i;
return STATUS_SUCCESS;
}
}
if (timeoutMs == 0)
return STATUS_TIMEOUT;
KE_EVENT_WAIT_BLOCK stackBlocks[SYS_WAIT_MANY_MAX_HANDLES];
KE_EVENT_MULTI_WAIT_CONTEXT ctx;
ctx.Thread = KeGetCurrentThread();
ctx.Blocks = stackBlocks;
ctx.Count = count;
ctx.Completed = 0;
ctx.Status = STATUS_PENDING;
ctx.SelectedIndex = 0;
for (uint32_t i = 0; i < count; i++) {
stackBlocks[i].Context = &ctx;
stackBlocks[i].Event = events[i];
stackBlocks[i].Index = i;
stackBlocks[i].Armed = 0;
LIST_INIT(&stackBlocks[i].Link);
}
KePrepareBlockThread(ctx.Thread, (OBJECT*)0, alertable);
KeSetThreadWaitRundown(ctx.Thread, KiRundownMultiWait, &ctx);
if (timeoutMs != SYS_WAIT_TIMEOUT_INFINITE) {
KSTATUS ts = KeArmCurrentThreadTimeout(timeoutMs, STATUS_TIMEOUT);
if (!KSUCCESS(ts)) {
KeClearThreadWaitRundown(ctx.Thread);
KeUnblockThreadFromWaitListWithStatus(ctx.Thread, ts);
return ts;
}
}
for (uint32_t i = 0; i < count; i++) {
EVENT* event = events[i];
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0 && event->Signaled) {
if (event->Type == EventAutoReset)
event->Signaled = 0;
ctx.Status = STATUS_SUCCESS;
ctx.SelectedIndex = i;
_InterlockedExchange(&ctx.Completed, 1);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadFromWaitListWithStatus(ctx.Thread, STATUS_SUCCESS);
break;
}
if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0) {
stackBlocks[i].Armed = 1;
ListInsertTail(&event->MultiWaitList, &stackBlocks[i].Link);
}
KeReleaseSpinlock(&event->Lock, flags);
}
KSTATUS waitStatus = KeCommitBlockThread(ctx.Thread, alertable);
if (_InterlockedCompareExchange(&ctx.Completed, 1, 0) == 0)
ctx.Status = waitStatus;
KiCleanupMultiWaitBlocks(&ctx);
KeClearThreadWaitRundown(ctx.Thread);
if (ctx.Status == STATUS_SUCCESS && signaledIndexOut)
*signaledIndexOut = ctx.SelectedIndex;
return ctx.Status;
}
code/kernel/ke_input.c
+94 -10
View File
@@ -8,6 +8,8 @@
#include <kernel/ke_input.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_display.h>
#include <kernel/ke_event.h>
#include <kernel/ob_object.h>
#define KEY_QUEUE_SIZE 256u
#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u)
@@ -32,6 +34,7 @@ static volatile uint32_t gModifiers;
typedef struct INPUT_KEY_QUEUE_ENTRY {
INPUT_KEY key;
char ch;
uint32_t modifiers;
} INPUT_KEY_QUEUE_ENTRY;
static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE];
@@ -50,6 +53,8 @@ static volatile int gMouseBoundsH = 1080;
static volatile uint8_t gMouseButtons;
static volatile uint32_t gNextInputDeviceId = 1;
static EVENT* gKeyboardEvent;
static EVENT* gInputStateEvent;
static char KiInputKeyToChar(INPUT_KEY key);
@@ -77,14 +82,18 @@ static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Implements the input Queue Key operation.
// Queues a keyboard event and wakes one blocked reader when the event was accepted.
static void KiInputQueueKey(INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
if (next == gKeyHead) return;
gKeyQueue[gKeyTail].key = key;
gKeyQueue[gKeyTail].ch = ch;
gKeyQueue[gKeyTail].modifiers = gModifiers;
gKeyTail = next;
if (gKeyboardEvent)
KeSignalEvent(gKeyboardEvent);
}
// Implements the input Queue Has Space operation.
@@ -211,6 +220,17 @@ void KeInputInitialize(const HAL_API* hal) {
gMouseHead = gMouseTail = 0;
gMouseButtons = 0;
if (!gKeyboardEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gKeyboardEvent);
if (!KSUCCESS(s))
DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
}
if (!gInputStateEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gInputStateEvent);
if (!KSUCCESS(s))
DbgPrint("Input: state wait event create failed st=0x%x\n", s);
}
int w = 1920, h = 1080;
DISPLAY_MODE mode;
if (KSUCCESS(KeDisplayGetMode(&mode))) {
@@ -252,6 +272,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 1u;
KiInputSetModifier(key, true);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputQueueKey(key, KiInputKeyToChar(key));
KiInputStartRepeat(key, KiInputNowTicks());
} else {
@@ -260,6 +281,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 0u;
KiInputSetModifier(key, false);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputStopRepeat(key);
}
}
@@ -341,27 +363,60 @@ INPUT_KEY KeInputKbPoll(void) {
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
}
// Waits until at least one keyboard event is available.
KSTATUS KeInputWaitKey(uint8_t alertable) {
if (KeInputKbPoll() != INPUT_KEY_NONE)
return STATUS_SUCCESS;
if (!gKeyboardEvent)
return STATUS_FAILURE;
for (;;) {
KSTATUS s = KeWaitForEvent(gKeyboardEvent, alertable);
if (!KSUCCESS(s))
return s;
if (KeInputKbPoll() != INPUT_KEY_NONE)
return STATUS_SUCCESS;
}
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
return gKeyDown[key] != 0;
}
INPUT_KEY KeInputReadKey(void) {
// Implements the input Peek Event operation.
bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
if (!record) return false;
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return INPUT_KEY_NONE;
INPUT_KEY key = gKeyQueue[gKeyHead].key;
if (gKeyHead == gKeyTail) return false;
INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead];
record->Key = entry.key;
record->Char = entry.ch;
record->Reserved = 0;
record->Modifiers = entry.modifiers;
return true;
}
// Implements the input Read Event operation.
bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
if (!KeInputPeekEvent(record)) return false;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
return key;
return true;
}
// Implements the input Read Key operation.
INPUT_KEY KeInputReadKey(void) {
INPUT_KEY_RECORD record;
return KeInputReadEvent(&record) ? record.Key : INPUT_KEY_NONE;
}
// Implements the input Read Char operation.
char KeInputReadChar(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return 0;
char ch = gKeyQueue[gKeyHead].ch;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
return ch;
INPUT_KEY_RECORD record;
return KeInputReadEvent(&record) ? record.Char : 0;
}
// Implements the input Get Modifiers operation.
@@ -416,3 +471,32 @@ void KeInputMouseSetBounds(int w, int h) {
gMouseBoundsH = h;
KeInputMouseSetPosition(gMouseX, gMouseY);
}
// Destroys an input queue handle.
static void KiInputQueueDestroy(OBJECT* object) {
INPUT_QUEUE* queue = CONTAINING_RECORD(object, INPUT_QUEUE, Header);
if (queue->WaitEvent) {
ObDereferenceObject(&queue->WaitEvent->Header);
queue->WaitEvent = (EVENT*)0;
}
}
// Opens the default keyboard input queue as a waitable object.
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
if (!out) return STATUS_INVALID_PARAMETER;
*out = (INPUT_QUEUE*)0;
if (!gInputStateEvent) return STATUS_FAILURE;
INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
if (!queue) return STATUS_NO_MEMORY;
queue->Header.Destroy = KiInputQueueDestroy;
ObReferenceObject(&gInputStateEvent->Header);
queue->WaitEvent = gInputStateEvent;
*out = queue;
return STATUS_SUCCESS;
}
// Returns the event that wakes when keyboard state changes.
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
return queue ? queue->WaitEvent : (EVENT*)0;
}
code/kernel/ke_irp.c
+1 -1
View File
@@ -138,7 +138,7 @@ static size_t IoIrpSize(uint8_t stackCount) {
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
if (!outIrp) return STATUS_INVALID_PARAMETER;
if (stackCount == 0) stackCount = 1;
IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount));
IRP* irp = (IRP*)MmAllocatePoolTag(IoIrpSize(stackCount), MM_POOL_TAG('i', 'r', 'p', ' '));
if (!irp) return STATUS_NO_MEMORY;
IoInitializeIrp(irp, stackCount);
*outIrp = irp;
code/kernel/ke_mutex.c
+5 -4
View File
@@ -12,7 +12,7 @@
static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
// Implements the mutex Create operation.
KSTATUS KeMutexCreate(MUTEX **out) {
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) {
if (!out) return STATUS_INVALID_PARAMETER;
MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
if (!m) return STATUS_NO_MEMORY;
@@ -25,7 +25,7 @@ KSTATUS KeMutexCreate(MUTEX **out) {
}
// Implements the acquire Mutex operation.
KSTATUS KeAcquireMutex(MUTEX* mutex) {
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
@@ -45,7 +45,7 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) {
}
// Implements the try Acquire Mutex operation.
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
@@ -59,13 +59,14 @@ KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
}
// Releases mutex.
void KeReleaseMutex(MUTEX* mutex) {
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
// Direct handoff: the selected waiter owns the mutex when its wait
// returns. Do not drop Owner to NULL, or another thread can steal it.
mutex->Owner = waiter;
code/kernel/ke_process.c
+148 -20
View File
@@ -6,7 +6,9 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_event.h>
#include <kernel/ke_system_root.h>
#include <kernel/ke_userenv.h>
#include <kernel/io_file.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_cpu.h>
@@ -140,7 +142,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
if (p == system) info->Flags |= SYS_PROCESS_FLAG_SYSTEM;
if (p == current) info->Flags |= SYS_PROCESS_FLAG_CURRENT;
MmQueryProcessUserMemory(p, &info->UserRegionCount,
MmVadQueryProcessMemory(p, &info->VadCount,
&info->UserReservedBytes,
&info->UserCommittedBytes);
@@ -158,27 +160,128 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
return count;
}
// Destructor (called by ObDereferenceObject when RefCount hits zero).
static void KiProcessDestroy(OBJECT* obj) {
PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
// Converts a kernel thread state to the public ABI state.
static uint32_t KiThreadStateToSys(THREAD_STATE state) {
switch (state) {
case ThreadReady: return SYS_THREAD_STATE_READY;
case ThreadRunning: return SYS_THREAD_STATE_RUNNING;
case ThreadWaiting: return SYS_THREAD_STATE_WAITING;
case ThreadDead: return SYS_THREAD_STATE_DEAD;
default: return SYS_THREAD_STATE_DEAD;
}
}
if (proc == gSystemProcess)
gSystemProcess = (PROCESS*)0;
// Converts a kernel object type to the public wait-object type.
static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
switch (type) {
case ObjectTypeProcess: return SYS_THREAD_WAIT_PROCESS;
case ObjectTypeThread: return SYS_THREAD_WAIT_THREAD;
case ObjectTypeMutex: return SYS_THREAD_WAIT_MUTEX;
case ObjectTypeSemaphore: return SYS_THREAD_WAIT_SEMAPHORE;
case ObjectTypeEvent: return SYS_THREAD_WAIT_EVENT;
case ObjectTypeQueue: return SYS_THREAD_WAIT_QUEUE;
case ObjectTypeFile: return SYS_THREAD_WAIT_FILE;
case ObjectTypeDir: return SYS_THREAD_WAIT_DIR;
case ObjectTypeAsyncIo: return SYS_THREAD_WAIT_ASYNC_IO;
case ObjectTypeSocket: return SYS_THREAD_WAIT_SOCKET;
case ObjectTypeDevice: return SYS_THREAD_WAIT_DEVICE;
case ObjectTypeDriver: return SYS_THREAD_WAIT_DRIVER;
case ObjectTypeUsbDevice: return SYS_THREAD_WAIT_USB_DEVICE;
case ObjectTypeUsbPipe: return SYS_THREAD_WAIT_USB_PIPE;
case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING;
case ObjectTypeSurface: return SYS_THREAD_WAIT_SURFACE;
default: return SYS_THREAD_WAIT_UNKNOWN;
}
}
// Release all remaining handles.
for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
if (proc->Handles[i].Object) {
ObDereferenceObject(proc->Handles[i].Object);
// Copies one process's thread telemetry into caller-provided storage.
KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut) {
if (!countOut) return STATUS_INVALID_PARAMETER;
if (maxCount && !infoOut) return STATUS_INVALID_PARAMETER;
PROCESS* process = KeGetProcessById(pid);
if (!process) return STATUS_NOT_FOUND;
THREAD* current = KeGetCurrentThread();
size_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
LIST_ENTRY* head = &process->ThreadList;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, ProcessLink);
if (infoOut && count < maxCount) {
SYS_THREAD_INFO* info = &infoOut[count];
memset(info, 0, sizeof(*info));
info->Size = sizeof(*info);
info->ProcessId = process->ProcessId;
info->ThreadId = thread->ThreadId;
info->State = KiThreadStateToSys(thread->State);
info->Priority = (uint32_t)thread->Priority;
info->RunningCpu = thread->RunningCpu;
info->LastCpu = thread->LastCpu;
info->QuantumRemaining = thread->QuantumRemaining;
info->QuantumTicks = thread->QuantumTicks;
info->CpuBurstTicks = thread->CpuBurstTicks;
info->LastRunTicks = thread->LastRunTicks;
info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE;
info->WaitStatus = (uint32_t)thread->WaitStatus;
info->KernelApcDisable = thread->KernelApcDisable;
if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT;
if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE;
if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL;
if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED;
if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT;
if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING;
if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING;
}
count++;
}
KeReleaseSpinlock(&process->HandleLock, flags);
*countOut = count;
ObDereferenceObject(&process->Header);
return STATUS_SUCCESS;
}
// Releases all kernel objects still reachable from a process handle table.
// This is part of process exit, not merely object destruction: a parent may keep
// a handle to an exited process for wait/query, but the dead process must not
// keep its own files, mappings, queues, sockets, or synchronization objects alive.
static void KiProcessReleaseAllHandles(PROCESS* proc) {
if (!proc) return;
for (uint32_t i = 1; i < HANDLE_TABLE_SIZE; ++i) {
OBJECT* obj = (OBJECT*)0;
uintptr_t flags = KeAcquireSpinlock(&proc->HandleLock);
obj = proc->Handles[i].Object;
if (obj) {
proc->Handles[i].Object = (OBJECT*)0;
proc->Handles[i].Rights = 0;
proc->Handles[i].Flags = 0;
}
KeReleaseSpinlock(&proc->HandleLock, flags);
if (obj)
ObDereferenceObject(obj);
}
}
// Releases the process-owned address space and per-process kernel resources.
// Idempotent because the normal exit path and the final OBJECT destructor can
// both observe a process whose threads are already gone.
static void KiProcessRundownResources(PROCESS* proc) {
if (!proc) return;
if (_InterlockedExchange(&proc->ResourcesReleased, 1) != 0)
return;
KiProcessReleaseAllHandles(proc);
// Release every user VA region owned by this process. This includes the
// main image, DLLs, TEBs/stacks that survived thread teardown, process
// environment strings, and leaked user allocations.
MmDestroyAllUserRegions(proc);
// environment strings, mmap views, and leaked user allocations.
MmVadDestroyAllForProcess(proc);
proc->ImageBase = 0;
proc->ImageSize = 0;
proc->ModuleCount = 0;
@@ -190,8 +293,21 @@ static void KiProcessDestroy(OBJECT* obj) {
proc->UserCmdlineSize = 0;
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
}
// Release the exit event.
// Destructor (called by ObDereferenceObject when RefCount hits zero).
static void KiProcessDestroy(OBJECT* obj) {
PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
if (proc == gSystemProcess)
gSystemProcess = (PROCESS*)0;
// If the process failed during launch, or no normal exit completion path
// ran, make sure its own handles and VADs are still reclaimed.
KiProcessRundownResources(proc);
// Release the exit event only when the process object itself dies. External
// process handles may still wait/query an exited process after rundown.
if (proc->ExitEvent) {
ObDereferenceObject(&proc->ExitEvent->Header);
proc->ExitEvent = (EVENT *)0;
@@ -227,6 +343,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->ExitCode = 0;
proc->Terminating = 0;
proc->Exited = 0;
proc->ResourcesReleased = 0;
proc->ImageBase = 0;
proc->ImageSize = 0;
proc->UserPebBase = 0;
@@ -240,10 +357,10 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->ModuleCount = 0;
proc->CurrentDirectory[0] = '\0';
proc->CurrentDirectoryLength = 0;
proc->CurrentDrive = 0;
proc->CurrentDrive = KeGetSystemRootDrive();
proc->CwdLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&proc->UserRegions);
proc->UserRegionLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&proc->VadList);
proc->VadLock = (SPINLOCK)SPINLOCK_INIT;
// Slot 0 is the invalid handle - leave it NULL.
memset(proc->Handles, 0, sizeof(proc->Handles));
@@ -427,7 +544,7 @@ fail:
}
// Process exit / termination.
static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
void KeCompleteProcessExitIfLastThread(PROCESS* process) {
if (!process) return;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
@@ -435,6 +552,7 @@ static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
KeReleaseSpinlock(&process->HandleLock, flags);
if (threadCount == 0 && _InterlockedExchange(&process->Exited, 1) == 0) {
KiProcessRundownResources(process);
if (process->ExitEvent) KeSignalEvent(process->ExitEvent);
}
}
@@ -448,7 +566,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
// The calling thread exits immediately after this in SysExitProcess. Do
// not drop the process object here: each thread owns one process reference,
// and KiThreadDestroy releases that reference after unlinking the thread.
KiCompleteProcessExitIfLastThread(process);
KeCompleteProcessExitIfLastThread(process);
}
// Implements the terminate Process operation.
@@ -469,7 +587,15 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
if (t == self)
continue;
(void)KeAlertThread(t);
// A terminating process must not be held open by worker threads that
// are blocked in non-alertable waits. User-mode DLLs such as Bink can
// leave IO/submit workers parked on semaphores; after the foreground
// thread crashes, those waiters would otherwise keep ThreadCount > 0
// and prevent the process ExitEvent from waking the shell.
if (t->State == ThreadWaiting)
KeAbortThreadWait(t, STATUS_ALERTED);
else
(void)KeAlertThread(t);
if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
KCPU* cpu = &gCpus[t->RunningCpu];
@@ -481,7 +607,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
}
KeReleaseSpinlock(&process->HandleLock, flags);
KiCompleteProcessExitIfLastThread(process);
KeCompleteProcessExitIfLastThread(process);
return STATUS_SUCCESS;
}
@@ -535,6 +661,8 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) {
process->Handles[handle].Rights = 0;
process->Handles[handle].Flags = 0;
KeReleaseSpinlock(&process->HandleLock, flags);
if (obj->Type == ObjectTypeFile)
(void)IoFlushFile((FILE_OBJECT*)obj);
ObDereferenceObject(obj);
return;
}
code/kernel/ke_sched.c
+133 -7
View File
@@ -17,6 +17,7 @@
#include <kernel/hal/hal_apic.h>
static LIST_ENTRY gRunQueue[PriorityCount];
static LIST_ENTRY gSleepList;
static SPINLOCK gSchedLock;
static uint32_t gAgingCounter = 0;
@@ -305,6 +306,7 @@ static void KiIdleLoop(void) {
// Initializes scheduler Global.
void KiInitializeSchedulerGlobal(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
LIST_INIT(&gSleepList);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
@@ -349,6 +351,8 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
idle->WaitStatus = STATUS_SUCCESS;
idle->WaitListHead = (LIST_ENTRY*)0;
idle->AlertableWait = 0;
idle->SleepUntilTick = 0;
idle->TimeoutStatus = STATUS_SUCCESS;
idle->UserStackBase = (void*)0;
idle->Teb = (void*)0;
idle->TebSize = 0;
@@ -358,7 +362,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
LIST_INIT(&idle->ProcessLink);
LIST_INIT(&idle->SchedulerLink);
void* kstack = MmAllocatePool(KSTACK_SIZE);
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
ObDereferenceObject(&idle->Header);
return STATUS_NO_MEMORY;
@@ -621,12 +625,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
}
// Implements the scheduler Wake Thread operation.
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
// Completes a wait while the scheduler lock is held.
static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
// Complete the wait while holding the scheduler lock. On SMP a wake can
// race the prepare->commit window on another CPU: the target thread is
// already on the object's wait list and has State=ThreadWaiting, but it is
@@ -634,7 +636,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
// That case must complete in-place, not enqueue the still-running thread.
if (thread->State != ThreadWaiting) {
thread->WaitStatus = status;
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
@@ -642,6 +643,8 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
thread->WaitStatus = status;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
thread->CpuBurstTicks = 0;
thread->QuantaConsumed = 0;
@@ -652,7 +655,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
// KeCommitBlockThread() will see State != ThreadWaiting and return the
// completed wait status without ever entering the ready queue.
thread->State = ThreadRunning;
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
@@ -662,10 +664,49 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
KiInsertReadyLocked(thread);
KiPokeRunnableCpusLocked(thread);
}
// Wakes a thread after first removing its scheduler-link wait membership.
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (thread->State == ThreadWaiting && thread->WaitListHead) {
ListRemove(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
}
KiWakeThreadLocked(thread, status);
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the scheduler Wake Thread operation.
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiWakeThreadLocked(thread, status);
KeReleaseSpinlock(&gSchedLock, flags);
}
// Wakes threads whose scheduler-tick sleep deadline has expired.
static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
LIST_ENTRY* node = gSleepList.Fwd;
while (node != &gSleepList) {
LIST_ENTRY* next = node->Fwd;
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (thread->State == ThreadWaiting &&
thread->WaitListHead == &gSleepList &&
thread->SleepUntilTick <= nowTick) {
ListRemove(node);
thread->WaitListHead = (LIST_ENTRY*)0;
KiWakeThreadLocked(thread, thread->TimeoutStatus);
}
node = next;
}
}
// Implements the scheduler Tick operation.
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
@@ -677,6 +718,7 @@ void KeSchedulerTick(void) {
KiArmPostSwitchUnlock(cur, flags);
gSchedStats.Ticks++;
KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
int requestReschedule = 0;
@@ -745,6 +787,90 @@ void KeSchedulerTick(void) {
KiRequestReschedule();
}
// Arms a timeout for the current thread after a wait primitive has prepared it to block.
KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) {
if (milliseconds == 0)
return STATUS_INVALID_PARAMETER;
if (!KeCanWait())
return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (!self || self->IsIdleThread)
return STATUS_INVALID_PARAMETER;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
if (ticks == 0)
ticks = 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (self->State != ThreadWaiting || self->WaitListHead != (LIST_ENTRY*)0 ||
self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
KeReleaseSpinlock(&gSchedLock, flags);
return STATUS_INVALID_PARAMETER;
}
uint64_t now = gSchedStats.Ticks;
uint64_t wakeTick = now + ticks;
if (wakeTick < now)
wakeTick = UINT64_MAX;
self->WaitListHead = &gSleepList;
self->SleepUntilTick = wakeTick;
self->TimeoutStatus = timeoutStatus;
ListInsertTail(&gSleepList, &self->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return STATUS_SUCCESS;
}
// Implements the delay Thread operation.
__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
if (milliseconds == 0) {
KeYield();
return STATUS_SUCCESS;
}
if (!KeCanWait())
return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (!self || self->IsIdleThread)
return STATUS_INVALID_PARAMETER;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
if (ticks == 0)
ticks = 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (self->Queued || self->State != ThreadRunning ||
self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
KeReleaseSpinlock(&gSchedLock, flags);
return STATUS_INVALID_PARAMETER;
}
uint64_t now = gSchedStats.Ticks;
uint64_t wakeTick = now + ticks;
if (wakeTick < now)
wakeTick = UINT64_MAX;
self->State = ThreadWaiting;
self->WaitObject = (OBJECT*)0;
self->WaitStatus = STATUS_PENDING;
self->WaitListHead = &gSleepList;
self->AlertableWait = 0;
self->SleepUntilTick = wakeTick;
self->TimeoutStatus = STATUS_SUCCESS;
ListInsertTail(&gSleepList, &self->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return KeCommitBlockThread(self, 0);
}
// Implements the yield operation.
__declspec(dllexport) void KeYield(void) {
KeAssertIrql(PASSIVE_LEVEL);
code/kernel/ke_selftest.c
+443 -3
View File
@@ -14,10 +14,13 @@
#include <kernel/ke_sched.h>
#include <ntlite/status.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_swap.h>
#include <kernel/mm_pfn.h>
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>
@@ -478,11 +481,118 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
// Virtual memory mapping invariants.
// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
static void KiRunPfnDatabaseSelfTest(void) {
MM_PFN_STATS before;
MM_PFN_STATS after;
MmPfnQueryStats(&before);
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n");
MM_PFN* pfn = MmPfnGet(frame);
if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"PFN selftest: allocated frame metadata invalid phys=%016llx\n",
(unsigned long long)frame);
if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount);
if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount);
if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount);
if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n");
if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount);
if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount);
if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n");
if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount);
if (!KSUCCESS(MmPfnFree(frame)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n");
PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma);
if (!run)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n");
MM_PFN* p0 = MmPfnGet(run);
MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE);
if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n");
MmPfnFree(run);
MmPfnFree(run + PAGE_SIZE);
// Accounting buckets are part of the VM contract. A missing increment here
// usually means a new allocation path is using an anonymous usage class.
PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache);
PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage);
PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile);
PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous);
PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow);
PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface);
if (!clean || !image || !mapped || !anon || !cow || !surface)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n");
MM_PFN_STATS bucketStats;
MmPfnQueryStats(&bucketStats);
if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 ||
bucketStats.ImageFrames < before.ImageFrames + 1 ||
bucketStats.MappedFileFrames < before.MappedFileFrames + 1 ||
bucketStats.AnonymousFrames < before.AnonymousFrames + 1 ||
bucketStats.CowFrames < before.CowFrames + 1 ||
bucketStats.SurfaceFrames < before.SurfaceFrames + 1) {
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n");
}
// Standby is the clean-page holding state used by later file-cache and image
// reclaim. It must not change the free count until the owning cache drops it.
if (!KSUCCESS(MmPfnMoveToStandby(clean)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n");
if (!KSUCCESS(MmPfnMoveToStandby(image)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n");
MM_PFN_STATS standbyStats;
MmPfnQueryStats(&standbyStats);
if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 ||
standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 ||
standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 ||
standbyStats.FreeFrames != bucketStats.FreeFrames) {
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n");
}
if (!KSUCCESS(MmPfnActivate(clean)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n");
if (!KSUCCESS(MmPfnActivate(image)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n");
(void)MmPfnFree(clean);
(void)MmPfnFree(image);
(void)MmPfnFree(mapped);
(void)MmPfnFree(anon);
(void)MmPfnFree(cow);
(void)MmPfnFree(surface);
MmPfnQueryStats(&after);
if (before.FreeFrames != after.FreeFrames)
KePanic(KE_PANIC_CODE_SELFTEST,
"PFN selftest: free count leak before=%llu after=%llu\n",
(unsigned long long)before.FreeFrames,
(unsigned long long)after.FreeFrames);
DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
}
// Verifies the page-table manager invariants that the higher VMM will depend on.
static void KiRunMemoryMappingSelfTest(void) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
KSTATUS s = MmQueryPage(0, &phys, &flags);
KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags);
if (s != STATUS_NOT_FOUND) {
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n",
(unsigned long long)phys,
@@ -490,7 +600,329 @@ static void KiRunMemoryMappingSelfTest(void) {
(unsigned)s);
}
DbgPrint("MM selftest: page zero unmapped\n");
uintptr_t testVa = 0xFFFFA00000000000ULL;
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (s != STATUS_NOT_FOUND) {
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n",
(unsigned long long)testVa,
(unsigned long long)phys,
(unsigned long long)flags);
}
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n");
s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent);
if (!KSUCCESS(s)) {
MmPfnFree(frame);
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s);
}
volatile uint64_t* probe = (volatile uint64_t*)testVa;
*probe = 0x1122334455667788ULL;
if (*probe != 0x1122334455667788ULL)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n");
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n",
s,
(unsigned long long)phys,
(unsigned long long)frame,
(unsigned long long)flags);
s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (!KSUCCESS(s) || (flags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n",
s,
(unsigned long long)flags);
s = MmPteUnmapPage(testVa);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s);
MmPfnFree(frame);
DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
}
// Verifies sparse VAD commitment and NOACCESS protection metadata.
static void KiRunVadSparseSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n");
size_t beforeVadCount = 0;
size_t beforeReserved = 0;
size_t beforeCommitted = 0;
MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted);
void* base = (void*)0;
KSTATUS s = MmVadReserve(process,
PAGE_SIZE * 4,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s);
uint8_t* p = (uint8_t*)base;
s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s);
s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s);
void* queryBase = (void*)0;
size_t querySize = 0;
uint64_t queryFlags = 0;
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=%p size=%llu flags=%016llx\n",
s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s);
s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s);
s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s);
s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s);
p[0] = 0x5A;
p[PAGE_SIZE * 2] = 0xA5;
s = MmVadProtect(process, p, PAGE_SIZE, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s);
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (!KSUCCESS(s) || queryFlags != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n",
s, (unsigned long long)queryFlags);
s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s);
if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n");
size_t afterVadCount = 0;
size_t afterReserved = 0;
size_t afterCommitted = 0;
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2)
KePanic(KE_PANIC_CODE_SELFTEST,
"VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n",
(unsigned long long)beforeCommitted,
(unsigned long long)afterCommitted,
(unsigned long long)beforeVadCount,
(unsigned long long)afterVadCount);
s = MmVadDestroy(process, base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s);
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n");
DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
}
// Verifies that writable copy-on-write views split before either alias can modify shared state.
static void KiRunVadCopyOnWriteSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n");
void* base = (void*)0;
KSTATUS s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s);
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s);
uint8_t* source = (uint8_t*)base;
source[0] = 0x31;
void* cowBase = (void*)0;
s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s);
PHYSICAL_ADDRESS sourcePhys = 0;
PHYSICAL_ADDRESS cowPhys = 0;
uint64_t sourceFlags = 0;
uint64_t cowFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags);
if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n",
s, (unsigned long long)sourceFlags);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags);
if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n",
s,
(unsigned long long)sourcePhys,
(unsigned long long)cowPhys,
(unsigned long long)cowFlags);
s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s);
uint8_t* copy = (uint8_t*)cowBase;
copy[0] = 0x62;
if (source[0] != 0x31 || copy[0] != 0x62)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n",
source[0], copy[0]);
PHYSICAL_ADDRESS splitPhys = 0;
uint64_t splitFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags);
if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
s,
(unsigned long long)sourcePhys,
(unsigned long long)splitPhys,
(unsigned long long)splitFlags);
(void)MmVadDestroy(process, cowBase);
(void)MmVadDestroy(process, base);
void* zeroBase = (void*)0;
s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&zeroBase);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s);
s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s);
PHYSICAL_ADDRESS zeroPhys = 0;
uint64_t zeroFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags);
if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n",
s, (unsigned long long)zeroFlags);
s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s);
PHYSICAL_ADDRESS privatePhys = 0;
uint64_t privateFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags);
if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
s,
(unsigned long long)zeroPhys,
(unsigned long long)privatePhys,
(unsigned long long)privateFlags);
((uint8_t*)zeroBase)[0] = 0x9C;
if (((uint8_t*)zeroBase)[0] != 0x9C)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n");
(void)MmVadDestroy(process, zeroBase);
DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
}
// Verifies that a private resident VAD page can move to swap and fault back in.
static void KiRunVadSwapSelfTest(void) {
if (!MmSwapIsEnabled()) {
DbgPrint("VAD swap selftest: SKIP, swap is disabled\n");
return;
}
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n");
void* base = (void*)0;
KSTATUS s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s);
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s);
uint8_t* p = (uint8_t*)base;
p[0] = 0xC1;
p[PAGE_SIZE - 1] = 0x5E;
void* trimmed = (void*)0;
s = MmVadTrimOnePageForTest(process, &trimmed);
if (!KSUCCESS(s) || trimmed != base)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=%p base=%p\n", s, trimmed, base);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s);
s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s);
if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]);
(void)MmVadDestroy(process, base);
DbgPrint("VAD swap selftest: private pageout/pagein PASS\n");
}
// String-view ABI self-test.
@@ -614,6 +1046,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
&st.Iosb,
st.CompletionEvent,
&st.Apc,
(MM_LOCKED_USER_BUFFER*)0,
(MM_LOCKED_USER_BUFFER*)0,
&readReq);
if (s != STATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s);
@@ -683,6 +1117,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
&wiosb,
wev,
(KAPC*)0,
(MM_LOCKED_USER_BUFFER*)0,
(MM_LOCKED_USER_BUFFER*)0,
&writeReq);
if (s != STATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s);
@@ -832,7 +1268,11 @@ void KiRunEarlyKernelSelfTests(void) {
void KiRunKernelSelfTests(const HAL_API* hal) {
DbgPrint("Kernel selftests: begin\n");
KiRunKernelApcSelfTest();
KiRunPfnDatabaseSelfTest();
KiRunMemoryMappingSelfTest();
KiRunVadSparseSelfTest();
KiRunVadCopyOnWriteSelfTest();
KiRunVadSwapSelfTest();
KiRunStringViewSelfTest();
KiRunIrpKernelApcSelfTest(hal);
KiRunAsyncFileIoSelfTest();
code/kernel/ke_semaphore.c
+1
View File
@@ -68,6 +68,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
// consuming a count later.
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
KeReleaseSpinlock(&sem->Lock, flags);
KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS);
return;
code/kernel/ke_smp.c
+9 -7
View File
@@ -20,6 +20,7 @@
#include <kernel/ke_event.h>
#include <kernel/ob_object.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
@@ -141,15 +142,17 @@ static KSTATUS KiBuildApTrampoline(void) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_PREFERRED_MIN,
AP_TRAMPOLINE_MAX_PHYS);
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_PREFERRED_MIN,
AP_TRAMPOLINE_MAX_PHYS,
MmPfnUseKernel);
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_FALLBACK_MIN,
AP_TRAMPOLINE_PREFERRED_MIN);
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_FALLBACK_MIN,
AP_TRAMPOLINE_PREFERRED_MIN,
MmPfnUseKernel);
}
if (!frame) {
DbgPrint("SMP: no free PMM frame available below 0x%016llX for AP trampoline\n",
@@ -589,7 +592,7 @@ void KeBuildProcessorTopology(void) {
// Creates ap Stack.
static KSTATUS KiAllocateApStack(KCPU* cpu) {
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
// MmPfnAllocateContiguous(8) made SMP startup depend on a DMA-style run being
// available, which is fragile on real machines and unrelated to AP startup.
size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE;
void* stack = MmAllocatePool(stackBytes);
@@ -967,12 +970,11 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
cpuIndex, st);
}
THREAD* thread = (THREAD*)0;
st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker,
(uintptr_t)arg,
PriorityNormal,
(1ULL << cpuIndex),
&thread);
(THREAD**)0);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u worker create failed status=0x%x\n",
code/kernel/ke_syscall.asm
+38 -1
View File
@@ -152,7 +152,7 @@ KeSyscallEntry ENDP
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
; void KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; ---------------------------------------------------------------------------
KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
@@ -379,8 +379,45 @@ pf_kernel_frame:
pf_call:
and rsp, -16
sub rsp, FAULT_CALL_SHADOW
mov rbx, rax ; Keep the KFAULT_FRAME pointer across C.
mov rcx, rax
call KeFaultDispatchFrame
test eax, eax
jnz pf_unhandled_return
mov rdx, rbx ; rdx = KFAULT_FRAME*.
mov rcx, [rdx + KF_Rcx]
mov r8, [rdx + KF_R8]
mov r9, [rdx + KF_R9]
mov rsi, [rdx + KF_Rsi]
mov rdi, [rdx + KF_Rdi]
mov rbp, [rdx + KF_Rbp]
mov r12, [rdx + KF_R12]
mov r13, [rdx + KF_R13]
mov r14, [rdx + KF_R14]
mov r15, [rdx + KF_R15]
mov rbx, [rdx + KF_Rbx]
lea rsp, [rdx + KFAULT_SIZE + 24] ; Skip saved RAX/R11 and the CPU error code.
mov rax, [rdx + KF_Rax]
mov r11, [rdx + KF_R11]
mov r10, [rdx + KF_Cs]
test r10, 3
jz pf_resume_no_swapgs
mov r10, [rdx + KF_R10]
mov rdx, [rdx + KF_Rdx]
swapgs
iretq
pf_resume_no_swapgs:
mov r10, [rdx + KF_R10]
mov rdx, [rdx + KF_Rdx]
iretq
pf_unhandled_return:
cli
pf_unhandled_halt:
hlt
jmp pf_unhandled_halt
KeFaultPF ENDP
code/kernel/ke_syscall.c
+1526 -275
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_system_root.c
+97
View File
@@ -0,0 +1,97 @@
// System root volume selection. This file keeps boot-volume identity in one place so loaders do not guess.
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <kernel/ke_system_root.h>
#include <kernel/ke_debug.h>
#include <kernel/io_file.h>
static uint8_t gSystemRootDrive;
static bool gSystemRootReady;
// Builds a drive-qualified path without depending on any process CWD.
static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) {
if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES)
return false;
size_t relLen = strlen(relative);
if (relLen + 4 > outSize)
return false;
out[0] = (char)('0' + drive);
out[1] = ':';
out[2] = '/';
memcpy(out + 3, relative, relLen + 1);
return true;
}
// Returns true when the candidate volume looks like the volume that booted the kernel.
static bool KiDriveLooksLikeSystemRoot(uint8_t drive) {
static const char* required[] = {
"system/colonel.exe",
"system/hal.dll",
"drivers/drivers.ini",
};
char path[1024];
for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) {
if (!KiBuildDrivePath(drive, required[i], path, sizeof(path)))
return false;
FILE_OBJECT* file = NULL;
KSTATUS s = IoOpenFile(SysSvFromCstr(path), &file);
if (!KSUCCESS(s))
return false;
IoCloseFile(file);
}
return true;
}
// Selects the mounted volume that contains the kernel/system tree.
KSTATUS KeInitializeSystemRootVolume(void) {
for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) {
if (!IoFilesystemPresent(drive))
continue;
if (!KiDriveLooksLikeSystemRoot(drive))
continue;
gSystemRootDrive = drive;
gSystemRootReady = true;
DbgPrint("System root: drive=%u\n", drive);
return STATUS_SUCCESS;
}
return STATUS_NOT_FOUND;
}
// Returns whether the system root volume has been selected.
bool KeSystemRootReady(void) {
return gSystemRootReady;
}
// Returns the selected system root drive, or zero before discovery.
uint8_t KeGetSystemRootDrive(void) {
return gSystemRootReady ? gSystemRootDrive : 0;
}
// Builds an absolute path rooted at the selected system volume.
bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize) {
if (!gSystemRootReady)
return false;
return KiBuildDrivePath(gSystemRootDrive, relative, out, outSize);
}
// Opens a file rooted at the selected system volume.
KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out) {
if (!out)
return STATUS_INVALID_PARAMETER;
*out = NULL;
char path[1024];
if (!KeBuildSystemRootPath(relative, path, sizeof(path)))
return STATUS_NOT_FOUND;
return IoOpenFile(SysSvFromCstr(path), out);
}
code/kernel/ke_thread.c
+180 -20
View File
@@ -8,6 +8,8 @@
#include <kernel/ke_cpu.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_event.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_userenv.h>
@@ -36,6 +38,11 @@ extern void KeThreadEntry(void);
static void KiThreadDestroy(OBJECT* obj) {
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
// A wait that uses auxiliary stack wait blocks must be removed before the
// kernel stack is released. A terminated thread may never return to the
// wait syscall to run normal cleanup.
KeRundownThreadWait(thread, STATUS_ALERTED);
KiRundownApcState(thread);
if (thread->ExitEvent) {
@@ -43,16 +50,16 @@ static void KiThreadDestroy(OBJECT* obj) {
thread->ExitEvent = (EVENT*)0;
}
PROCESS* ownerForUserRegions = thread->Owner;
if (ownerForUserRegions) {
PROCESS* ownerForVadList = thread->Owner;
if (ownerForVadList) {
if (thread->Teb && thread->TebSize) {
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb);
MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
}
if (thread->UserStackBase) {
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase);
MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase);
thread->UserStackBase = (void *)0;
}
}
@@ -66,10 +73,13 @@ static void KiThreadDestroy(OBJECT* obj) {
remaining = owner->ThreadCount;
KeReleaseSpinlock(&owner->HandleLock, flags);
if (remaining == 0 && owner->Terminating &&
// Implements the interlocked Exchange operation.
_InterlockedExchange(&owner->Exited, 1) == 0) {
if (owner->ExitEvent) KeSignalEvent(owner->ExitEvent);
if (remaining == 0) {
// The last thread leaving a process is process exit even if user
// code called ExitThread instead of ExitProcess. KeExitProcess /
// KeTerminateProcess set Terminating earlier; this covers the last
// thread voluntarily exiting without an explicit process-exit call.
_InterlockedExchange(&owner->Terminating, 1);
KeCompleteProcessExitIfLastThread(owner);
}
ObDereferenceObject(&owner->Header);
@@ -111,11 +121,13 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
void* kstack = MmAllocatePool(KSTACK_SIZE);
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
ObDereferenceObject(&thread->Header);
return STATUS_NO_MEMORY;
@@ -127,10 +139,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
KSTATUS s;
void* ustack = (void*)0;
s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE,
s = MmVadCreateCommitted(owner, USTACK_SIZE, PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
UserRegionStack,
MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
MmVadStack,
&ustack);
if (!KSUCCESS(s) || !ustack) {
MmFreePool(kstack);
@@ -139,11 +151,19 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
return STATUS_NO_MEMORY;
}
thread->UserStackBase = ustack;
// Ring-3 starts at the image entry via IRET, not CALL, so there is no
// pushed return address. The x64 ABI expects RSP % 16 == 8 at function
// entry; bias the initial stack as if a call had happened so compiler
// generated aligned SSE spills (movaps) are safe.
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8;
// Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
// nevertheless lets callees immediately use the caller-provided 32-byte
// home/shadow area above RSP, and MSVC commonly emits a first instruction
// like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
// entry stack as if a CALL with a normal home area had happened:
//
// RSP % 16 == 8 at function entry
// [RSP + 0] fake return-address slot
// [RSP + 8..39] home/shadow space
//
// Without the 32-byte home area, the first spill lands in the high stack
// guard page and every freshly-created MSVC thread faults immediately.
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8 - 32;
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
if (!teb) {
@@ -211,8 +231,16 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
KeReleaseSpinlock(&owner->HandleLock, flags);
KeInitThreadFpuContext(thread);
KeReadyThread(thread);
// Hand the creator a short-lived reference before making the thread
// runnable. On SMP a new user thread can run and exit before
// KeCreateThread() returns; without this reference, callers that merely
// read ThreadId for logging or syscall output can race the scheduler's
// dead-thread dereference and touch freed pool.
ObReferenceObject(&thread->Header);
*out = thread;
KeReadyThread(thread);
return STATUS_SUCCESS;
}
@@ -283,6 +311,8 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
@@ -290,7 +320,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx
thread->TebSize = 0;
void* kstack = MmAllocatePool(KSTACK_SIZE);
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
ObDereferenceObject(&thread->Header);
return STATUS_NO_MEMORY;
@@ -320,7 +350,16 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
owner->ThreadCount++;
KeReleaseSpinlock(&owner->HandleLock, flags);
if (out) *out = thread;
// Hand the creator a short-lived reference before making the kernel
// thread runnable. Kernel workers can run, signal their completion event,
// exit, and be reaped on another CPU before KeCreateKernelThread()'s
// caller drops its returned pointer. The live scheduler reference remains
// separate and is released by KeSchedulerPostSwitch() after the dead
// thread has switched away.
if (out) {
ObReferenceObject(&thread->Header);
*out = thread;
}
KeInitThreadFpuContext(thread);
KeReadyThread(thread);
@@ -352,6 +391,8 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable)
thread->WaitStatus = STATUS_PENDING;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = alertable ? 1 : 0;
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
thread->WaitRundownContext = (void*)0;
}
@@ -362,12 +403,41 @@ void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
thread->WaitListHead = waitListHead;
}
// Registers cleanup for waits with auxiliary wait blocks.
void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
thread->WaitRundownRoutine = routine;
thread->WaitRundownContext = context;
}
// Clears auxiliary wait cleanup after the owning wait has removed its blocks.
void KeClearThreadWaitRundown(THREAD* thread) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
thread->WaitRundownContext = (void*)0;
}
// Runs auxiliary wait cleanup for abort and thread teardown paths.
void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
if (!thread || !thread->WaitRundownRoutine) return;
KE_WAIT_RUNDOWN_ROUTINE routine = thread->WaitRundownRoutine;
void* context = thread->WaitRundownContext;
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
thread->WaitRundownContext = (void*)0;
routine(thread, context, status);
}
// Implements the finish Wait operation.
static void KiFinishWait(THREAD* thread, KSTATUS status) {
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = status;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
}
// Implements the commit Block Thread operation.
@@ -385,6 +455,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
if (thread->State != ThreadWaiting) {
KSTATUS status = thread->WaitStatus;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
return status;
}
@@ -406,6 +478,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
KiDeliverKernelApcs();
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
return thread->WaitStatus;
}
@@ -423,6 +497,92 @@ void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
KiSchedulerWakeThread(thread, status);
}
// Returns whether the scheduler link still belongs to an object wait list.
static int KiThreadWaitLinkIsLinked(const THREAD* thread) {
if (!thread) return 0;
return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
thread->SchedulerLink.Bwd != &thread->SchedulerLink;
}
// Removes a thread from one wait list while that wait object's lock is held.
static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
if (!thread || !waitListHead) return 0;
if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead)
return 0;
// The signaling side pops the wait-list entry before it wakes the thread.
// In that handoff window the scheduler link is already self-linked; abort
// must not remove it again or turn a successful mutex/semaphore handoff
// into an aborted wait for a thread that may already own the object.
if (!KiThreadWaitLinkIsLinked(thread))
return 0;
ListRemove(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
return 1;
}
// Removes a thread from an object-owned wait queue using that object's lock.
static int KiAbortObjectWait(THREAD* thread) {
if (!thread || !thread->WaitObject || !thread->WaitListHead)
return 0;
OBJECT* object = thread->WaitObject;
switch (object->Type) {
case ObjectTypeEvent: {
EVENT* event = CONTAINING_RECORD(object, EVENT, Header);
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &event->WaitList);
KeReleaseSpinlock(&event->Lock, flags);
return removed;
}
case ObjectTypeMutex: {
MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList);
KeReleaseSpinlock(&mutex->Lock, flags);
return removed;
}
case ObjectTypeSemaphore: {
SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
uintptr_t flags = KeAcquireSpinlock(&semaphore->Lock);
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &semaphore->WaitList);
KeReleaseSpinlock(&semaphore->Lock, flags);
return removed;
}
default:
return 0;
}
}
// Aborts a thread's current wait regardless of whether the wait was marked
// alertable. Process teardown uses this for crash/kill paths: a background
// worker blocked forever on a semaphore must not keep the process exit event
// unsignaled after the foreground thread has faulted. Object wait-list removal
// must use the object's own lock; otherwise signal/abort races can leave stale
// scheduler links in an event, mutex, or semaphore wait queue.
void KeAbortThreadWait(THREAD* thread, KSTATUS status) {
if (!thread) return;
if (thread->State != ThreadWaiting) return;
if (thread->WaitRundownRoutine) {
KeRundownThreadWait(thread, status);
return;
}
if (KiAbortObjectWait(thread)) {
KeUnblockThreadWithStatus(thread, status);
return;
}
// Sleep/timeouts and already-dequeued handoffs are owned by the scheduler.
// This path removes from the scheduler sleep list when present and otherwise
// completes the wait in place.
KeUnblockThreadFromWaitListWithStatus(thread, status);
}
// Implements the unblock Thread operation.
void KeUnblockThread(THREAD* thread) {
KeUnblockThreadWithStatus(thread, STATUS_SUCCESS);
code/kernel/ke_work.c
+117
View File
@@ -0,0 +1,117 @@
// Kernel passive-level work queue. This file owns the generic mechanism used
// to move ISR/DPC-originated work to a sleepable system thread context.
#include <stdbool.h>
#include <stddef.h>
#include <intrin.h>
#include <kernel/ke_work.h>
#include <kernel/ke_event.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_debug.h>
static SPINLOCK gWorkQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gWorkQueueList;
static EVENT* gWorkQueueEvent;
static volatile long gWorkQueueListReady;
static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started.
static void KiWorkQueueWorker(void* arg) {
(void)arg;
KeAssertIrql(PASSIVE_LEVEL);
for (;;) {
(void)KeWaitForEvent(gWorkQueueEvent, 0);
for (;;) {
KWORK_ITEM* item = (KWORK_ITEM*)0;
uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
if (!LIST_IS_EMPTY(&gWorkQueueList)) {
LIST_ENTRY* node = ListPopHead(&gWorkQueueList);
item = CONTAINING_RECORD(node, KWORK_ITEM, Link);
}
KeReleaseSpinlock(&gWorkQueueLock, flags);
if (!item)
break;
// Allow the item to be queued again while it is running. This keeps
// level-triggered users such as cursor updates naturally coalesced
// but does not lose an update that arrives during the callback.
_InterlockedExchange(&item->Queued, 0);
if (item->Routine)
item->Routine(item->Context);
}
}
}
void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context) {
if (!item)
return;
LIST_INIT(&item->Link);
item->Routine = routine;
item->Context = context;
item->Queued = 0;
}
bool KeIsWorkQueueStarted(void) {
return _InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 2;
}
bool KeQueueWorkItem(KWORK_ITEM* item) {
if (!item || !item->Routine)
return false;
if (_InterlockedCompareExchange(&item->Queued, 1, 0) != 0)
return false;
if (!_InterlockedCompareExchange(&gWorkQueueListReady, 1, 1)) {
_InterlockedExchange(&item->Queued, 0);
return false;
}
uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
ListInsertTail(&gWorkQueueList, &item->Link);
EVENT* event = gWorkQueueEvent;
KeReleaseSpinlock(&gWorkQueueLock, flags);
if (event)
KeSignalEvent(event);
return true;
}
KSTATUS KeStartWorkQueue(void) {
long state = _InterlockedCompareExchange(&gWorkQueueStartState, 1, 0);
if (state == 2)
return STATUS_SUCCESS;
if (state == 1) {
while (_InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 1)
KeYield();
return KeIsWorkQueueStarted() ? STATUS_SUCCESS : STATUS_FAILURE;
}
LIST_INIT(&gWorkQueueList);
_InterlockedExchange(&gWorkQueueListReady, 1);
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker,
0,
PriorityNormal,
&worker);
// Persistent system worker; keep the creator reference for kernel life.
(void)worker;
}
if (KSUCCESS(s)) {
_InterlockedExchange(&gWorkQueueStartState, 2);
KeSignalEvent(gWorkQueueEvent);
} else {
_InterlockedExchange(&gWorkQueueStartState, 0);
}
return s;
}
code/kernel/kmain.c
+178 -66
View File
@@ -7,7 +7,9 @@
#pragma intrinsic(_enable)
#include <ntlite/boot_info.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_swap.h>
#include <kernel/ob_object.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
@@ -28,6 +30,9 @@
#include <kernel/ke_input.h>
#include <kernel/ke_time.h>
#include <kernel/ke_display.h>
#include <kernel/ke_settings.h>
#include <kernel/ke_work.h>
#include <kernel/ke_system_root.h>
#include <common/ini_parser.h>
typedef struct {
@@ -35,6 +40,90 @@ typedef struct {
const HAL_API* hal;
} SYSTEM_INIT_ARGS;
#define KI_SWAP_MIN_MB 4ull
#define KI_SWAP_MAX_MB 8192ull
// Parses a required unsigned decimal integer from settings.ini.
// Configuration errors are treated as boot-time corruption instead of guessed defaults.
static uint64_t KiParseRequiredUInt64Setting(const char* section, const char* key) {
char value[64];
KSTATUS s = KeGetSettingString(section, key, value, sizeof(value));
if (!KSUCCESS(s)) {
KePanic(KE_PANIC_CODE_INIT,
"settings: missing required [%s] %s entry, status=0x%x\n",
section,
key,
s);
}
uint64_t result = 0;
const char* p = value;
if (!*p) {
KePanic(KE_PANIC_CODE_INIT,
"settings: empty numeric value for [%s] %s\n",
section,
key);
}
while (*p) {
if (*p < '0' || *p > '9') {
KePanic(KE_PANIC_CODE_INIT,
"settings: invalid numeric value for [%s] %s: %s\n",
section,
key,
value);
}
uint64_t digit = (uint64_t)(*p - '0');
if (result > (UINT64_MAX - digit) / 10ull) {
KePanic(KE_PANIC_CODE_INIT,
"settings: numeric value for [%s] %s is too large: %s\n",
section,
key,
value);
}
result = result * 10ull + digit;
++p;
}
return result;
}
// Initializes swap according to settings.ini.
// A size of zero is the explicit opt-out. Any missing or invalid size is a broken configuration.
static void KiInitializeSwapFromSettings(void) {
uint64_t swapMb = KiParseRequiredUInt64Setting("vm", "swap_size_mb");
if (swapMb == 0) {
DbgPrint("swap: disabled by settings\n");
return;
}
if (swapMb < KI_SWAP_MIN_MB || swapMb > KI_SWAP_MAX_MB) {
KePanic(KE_PANIC_CODE_INIT,
"settings: [vm] swap_size_mb=%llu is outside the allowed range %llu..%llu MB\n",
(unsigned long long)swapMb,
(unsigned long long)KI_SWAP_MIN_MB,
(unsigned long long)KI_SWAP_MAX_MB);
}
char pagefilePath[1024];
if (!KeBuildSystemRootPath("pagefile.sys", pagefilePath, sizeof(pagefilePath))) {
KePanic(KE_PANIC_CODE_INIT,
"swap: failed to build system-root pagefile path\n");
}
uint64_t swapBytes = swapMb * 1024ull * 1024ull;
KSTATUS s = MmSwapInitialize(SysSvFromCstr(pagefilePath), swapBytes);
if (!KSUCCESS(s)) {
KePanic(KE_PANIC_CODE_INIT,
"swap: failed to initialize %llu MB pagefile at %s, status=0x%x\n",
(unsigned long long)swapMb,
pagefilePath,
s);
}
}
typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *);
// HAL pointer cached at boot for subsystems that still need platform services.
@@ -68,7 +157,7 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
for (uintptr_t page = base; page < end; page += PAGE_SIZE) {
if (!KiPageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue;
MmFreeFrame((PHYSICAL_ADDRESS)page);
MmPfnFree((PHYSICAL_ADDRESS)page);
}
}
}
@@ -86,6 +175,7 @@ static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
}
}
// Phase Helper: Initialize dynamically loaded .sys drivers.
static void KiDriversIniCb(const char* section, const char* key,
const char* val, void* ud) {
@@ -120,7 +210,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
// Map the PE image into kernel space.
uintptr_t mappedBase = 0;
KSTATUS st = LdrLoadImage(rawPeBuffer, mod->ImageSize, &mappedBase, false);
KSTATUS st = LdrLoadKernelImage(rawPeBuffer, mod->ImageSize, &mappedBase);
if (!KSUCCESS(st)) {
DbgPrint("ERROR: Failed to map driver %s (0x%X)\n", mod->Name, st);
continue;
@@ -129,7 +219,9 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
// Update the struct so other modules can find it.
mod->VirtBase = mappedBase;
// Resolve imports (can now see Kernel, HAL, and previous drivers).
// Resolve imports while the image is still writable. Kernel driver images
// stay writable until the loader grows real section-backed mappings; the
// loader still clears NX for the whole image before returning it.
if (!KSUCCESS(LdrResolveImports(mappedBase))) {
DbgPrint("ERROR: Failed to resolve imports for %s\n", mod->Name);
continue;
@@ -152,9 +244,24 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
KSTATUS s;
FILE_OBJECT* f;
s = IoOpenFile(SYS_SV_LITERAL("drivers\\drivers.ini"), &f);
s = KeInitializeSystemRootVolume();
if (s != STATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini (status %d)\n", s);
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to find system root volume (status %d)\n", s);
}
char root[4];
root[0] = (char)('0' + KeGetSystemRootDrive());
root[1] = ':';
root[2] = '/';
root[3] = '\0';
s = IoSetCwd(SysSvFromCstr(root));
if (s != STATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to set system CWD to %s (status %d)\n", root, s);
}
s = KeOpenSystemRootFile("drivers\\drivers.ini", &f);
if (s != STATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini on system root (status %d)\n", s);
}
uint64_t size = IoGetFileSize(f) + 1;
@@ -174,26 +281,27 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]);
// Load the driver.
char path[1024];
char relative[512];
size_t len = strlen(runtimeKeys[i]);
char* path = MmAllocatePool(len + 10);
if (!path) {
DbgPrint("KernelMain: Failed to allocate memory for driver path. How?\n");
if (len + 9 > sizeof(relative)) {
DbgPrint("KiInitializeDrivers: runtime driver path too long: %s\n", runtimeKeys[i]);
continue;
}
memcpy(relative, "drivers\\", 8);
memcpy(relative + 8, runtimeKeys[i], len + 1);
// memcpy maneuvers.
memcpy(path, "drivers\\", 8);
memcpy(path + 8, runtimeKeys[i], len + 1);
if (!KeBuildSystemRootPath(relative, path, sizeof(path))) {
DbgPrint("KiInitializeDrivers: failed to build runtime driver path: %s\n", runtimeKeys[i]);
continue;
}
s = IoOpenFile(SysSvFromCstr(path), &f);
if (s != STATUS_SUCCESS) {
DbgPrint("KiInitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path);
continue; // Skip this driver but keep going with others
continue;
}
MmFreePool(path);
uint64_t drvSize = IoGetFileSize(f);
void* drvMem = MmAllocatePool(drvSize);
if (!KSUCCESS(IoReadFile(f, drvMem, drvSize, &got))) {
@@ -204,7 +312,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
IoCloseFile(f);
uintptr_t mappedBase = 0;
if (!KSUCCESS(LdrLoadImage(drvMem, drvSize, &mappedBase, false))) { // Load into kernel space
if (!KSUCCESS(LdrLoadKernelImage(drvMem, drvSize, &mappedBase))) { // Load into kernel space
DbgPrint("KiInitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]);
MmFreePool(drvMem);
continue;
@@ -215,7 +323,6 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
continue;
}
// Runtime modules may also export DDIs to drivers loaded later. Boot
// drivers were already registered above; keep the module model uniform.
LdrRegisterModule(runtimeKeys[i], mappedBase);
@@ -250,28 +357,15 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
KSTATUS s;
FILE_OBJECT* f;
// Read the executable file into a raw buffer.
s = IoOpenFile(SysSvFromCstr(filePath), &f);
// Kernel-launched user images are rooted at the selected system volume.
s = KeOpenSystemRootFile(filePath, &f);
if (s != STATUS_SUCCESS) {
DbgPrint("Failed to load file: \"%s\"!\n", filePath);
DbgPrint("Failed to load system file: \"%s\" (status %d)!\n", filePath, s);
return 0;
}
uint64_t size = IoGetFileSize(f);
void* mem = MmAllocatePool(size);
if (!mem) {
IoCloseFile(f);
return 0;
}
size_t got = 0;
s = IoReadFile(f, mem, size, &got);
IoCloseFile(f);
if (!KSUCCESS(s)) {
MmFreePool(mem);
return 0;
}
(void)size;
// Create the process object.
PROCESS* proc;
@@ -280,7 +374,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
s = KeCreateProcess(filePath, &proc);
if (!KSUCCESS(s)) {
DbgPrint("Failed to create process. System halted.\n");
MmFreePool(mem);
IoCloseFile(f);
return 0;
}
DbgPrint("Created process (PID %u) with command line: %s\n", proc->ProcessId, (args && args[0]) ? args : filePath);
@@ -291,17 +385,17 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
SysSvFromCstr(proc->CurrentDirectory));
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s);
MmFreePool(mem);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
}
// Parse the raw PE file and map it into executable memory.
// Map the executable through an image section backed by the open file.
uintptr_t imageBase = 0;
s = LdrLoadImageForProcess(proc, mem, size, &imageBase);
MmFreePool(mem);
s = LdrLoadImageFileForProcess(proc, f, &imageBase);
IoCloseFile(f);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s);
DbgPrint("KiLaunchUserProcess: LdrLoadImageFileForProcess failed with status: %d\n", s);
ObDereferenceObject(&proc->Header);
return 0;
}
@@ -313,6 +407,13 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
return 0;
}
s = LdrProtectImageForProcess(proc, imageBase, true);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to protect EXE image for %s (status %d)\n", filePath, s);
ObDereferenceObject(&proc->Header);
return 0;
}
proc->ImageBase = imageBase;
proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
@@ -327,7 +428,9 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
ObDereferenceObject(&proc->Header);
return 0;
}
DbgPrint("KiLaunchUserProcess: Thread created and queued!\n");
uint32_t tid = thread->ThreadId;
DbgPrint("KiLaunchUserProcess: Thread created and queued! TID=%u\n", tid);
ObDereferenceObject(&thread->Header);
// Caller owns the reference returned by KeCreateProcess.
return proc;
@@ -344,35 +447,39 @@ void SystemInitThread(void* arg) {
KiInitializeDrivers(args->bootInfo, args->hal);
KeTimeReloadSettings();
KiInitializeSwapFromSettings();
KiResolveSystemImports(args->bootInfo);
KiReclaimBootloaderMemory(args->bootInfo);
if (args->bootInfo->Debug & KE_DEBUG_SELFTEST)
KiRunKernelSelfTests(args->hal);
// Launch User-Mode init.exe and restart it until it exits cleanly.
while (true) {
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
if (!shellProc) {
DbgPrint("SystemInitThread: Failed to start system/init.exe; retrying.\n");
KeYield();
continue;
}
KeWaitForEvent(shellProc->ExitEvent, 0);
int exitCode = shellProc->ExitCode;
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
ObDereferenceObject(&shellProc->Header);
if (exitCode == 0) break;
DbgPrint("SystemInitThread: Restarting init.exe.\n");
// Launch User-Mode init.exe.
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
if (!shellProc) {
DbgPrint("SystemInitThread: Failed to start system/init.exe.\n");
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n");
}
DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n");
KeWaitForEvent(shellProc->ExitEvent, 0);
int exitCode = shellProc->ExitCode;
if (exitCode != 0) {
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init.exe exited with code %d.\n", exitCode);
}
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
ObDereferenceObject(&shellProc->Header);
if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log"));
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus);
char logPath[1024];
KSTATUS logStatus = STATUS_NOT_FOUND;
if (KeBuildSystemRootPath("kernel.log", logPath, sizeof(logPath))) {
logStatus = KeFlushDebugLogToFile(SysSvFromCstr(logPath));
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush to %s failed st=0x%x\n", logPath, logStatus);
} else {
DbgPrint("debuglog: shutdown flush skipped; system root path unavailable\n");
}
}
gKmainHal->Power.Shutdown();
MmFreePool(args);
@@ -459,6 +566,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeInitializeScheduler();
KSTATUS workStatus = KeStartWorkQueue();
if (!KSUCCESS(workStatus)) {
KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus);
}
KSTATUS smpSchedStatus = KeAdmitApplicationProcessorsToScheduler();
if (!KSUCCESS(smpSchedStatus)) {
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to admit APs to scheduler, status=0x%x\n", smpSchedStatus);
@@ -471,11 +583,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
initArgs->bootInfo = bootInfo;
initArgs->hal = hal;
// Spawn the Initialization Thread.
THREAD* initThread = (THREAD*)0;
KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
(uintptr_t)initArgs,
PriorityNormal, &initThread);
// Spawn the Initialization Thread. No creator reference is needed; the
// scheduler owns the live thread lifetime.
KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
(uintptr_t)initArgs,
PriorityNormal, (THREAD**)0);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st);
code/kernel/ldr_image.c
+164 -75
View File
@@ -3,11 +3,13 @@
#include <stdint.h>
#include <string.h>
#include <kernel/ldr_image.h>
#include <kernel/ldr_section.h>
#include <ntlite/boot_info.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_system_root.h>
#ifndef LDR_DEBUG_VERBOSE
#define LDR_DEBUG_VERBOSE 0
@@ -155,6 +157,8 @@ typedef struct {
#define IMAGE_REL_BASED_DIR64 10
#define IMAGE_SCN_CNT_CODE 0x00000020u
#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040u
#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u
#define IMAGE_SCN_MEM_EXECUTE 0x20000000u
#define IMAGE_SCN_MEM_READ 0x40000000u
#define IMAGE_SCN_MEM_WRITE 0x80000000u
@@ -171,6 +175,11 @@ typedef struct {
static MODULE_ENTRY gModules[MAX_MODULES];
static uint32_t gModuleCount = 0;
// Ensures a user image page is privately writable before the kernel patches it.
static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t address, size_t size) {
if (!user) return STATUS_SUCCESS;
return MmVadMakeImageRangePrivate(process, (void*)address, size);
}
// Implements the canonical Module Name operation.
static const char* KiCanonicalModuleName(const char* name) {
@@ -310,10 +319,10 @@ size_t LdrGetImageSize(uintptr_t base) {
size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
if (!base) return 0;
if (process) {
size_t regionSize = MmGetUserRegionSize(process, (const void*)base);
size_t regionSize = MmVadGetSize(process, (const void*)base);
if (regionSize) return regionSize;
}
size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base);
size_t globalRegionSize = MmVadGetSizeByBase((const void*)base);
if (globalRegionSize) return globalRegionSize;
return LdrGetImageSize(base);
}
@@ -327,13 +336,26 @@ static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, b
else MmFreePool((void*)imageBase);
}
// Releases loaded Image.
static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user);
// Returns the final page flags for a PE section.
static uint64_t KiSectionPageProtection(const SECTION_HEADER* section, bool user) {
uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
// Some old toolchains mark BSS-like sections as uninitialized data without
// setting MEM_WRITE. Treating those pages as read-only breaks programs that
// legitimately write their zero-fill storage.
if ((section->Characteristics & IMAGE_SCN_MEM_WRITE) ||
(section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)) {
prot |= PAGE_WRITE;
}
if (!((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
(section->Characteristics & IMAGE_SCN_CNT_CODE))) {
prot |= PAGE_NX;
}
return prot;
}
// Implements the ldr Protect Image For Process operation.
// Applies final page protections to a mapped PE image.
KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return STATUS_INVALID_PARAMETER;
if (user && !process) return STATUS_INVALID_PARAMETER;
@@ -353,6 +375,14 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset)
return STATUS_FAILURE;
size_t pageCount = (mappedSize + PAGE_SIZE - 1) / PAGE_SIZE;
uint64_t* pageProt = (uint64_t*)MmAllocatePoolTag(pageCount * sizeof(uint64_t), MM_POOL_TAG('i', 'p', 'r', 't'));
if (!pageProt) return STATUS_NO_MEMORY;
uint64_t headerProt = PAGE_PRESENT | (user ? PAGE_USER : 0) | PAGE_NX;
for (size_t i = 0; i < pageCount; ++i)
pageProt[i] = headerProt;
SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset);
for (int i = 0; i < pe->NumberOfSections; i++) {
size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData;
@@ -368,35 +398,56 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
(unsigned long long)imageBase, (unsigned long long)mappedSize);
continue;
}
if (secEnd < secBase) return STATUS_NO_MEMORY;
if (secEnd < secBase) {
MmFreePool(pageProt);
return STATUS_NO_MEMORY;
}
if (secEnd > mapEnd) {
LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n",
i, (unsigned long long)secEnd, (unsigned long long)mapEnd);
protectSize = (size_t)(mapEnd - secBase);
secEnd = mapEnd;
}
uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE)
prot |= PAGE_WRITE;
if (!((sec[i].Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
(sec[i].Characteristics & IMAGE_SCN_CNT_CODE)))
prot |= PAGE_NX;
KSTATUS s;
if (user) {
s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot);
} else {
s = MmSetPageProtect(secBase, protectSize, prot);
}
if (!KSUCCESS(s)) {
LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
i, (unsigned long long)secBase, (unsigned long long)protectSize,
(unsigned long long)prot, s,
(unsigned long long)imageBase, (unsigned long long)mappedSize);
return s;
size_t firstPage = (size_t)((secBase - imageBase) / PAGE_SIZE);
size_t lastPage = (size_t)((secEnd - 1 - imageBase) / PAGE_SIZE);
uint64_t prot = KiSectionPageProtection(&sec[i], user);
for (size_t page = firstPage; page <= lastPage && page < pageCount; ++page) {
// Protection is page-granular. If sections ever share a page, the
// hardware page must allow the union of their access requirements.
if (prot & PAGE_WRITE)
pageProt[page] |= PAGE_WRITE;
if ((prot & PAGE_NX) == 0)
pageProt[page] &= ~PAGE_NX;
}
}
return STATUS_SUCCESS;
size_t runStart = 0;
KSTATUS s = STATUS_SUCCESS;
while (runStart < pageCount) {
uint64_t prot = pageProt[runStart];
size_t runEnd = runStart + 1;
while (runEnd < pageCount && pageProt[runEnd] == prot)
runEnd++;
uintptr_t va = imageBase + runStart * PAGE_SIZE;
size_t bytes = (runEnd - runStart) * PAGE_SIZE;
if (user)
s = MmVadProtect(process, (void*)va, bytes, prot);
else
s = MmPteProtectRange(va, bytes, prot);
if (!KSUCCESS(s)) {
LdrDbg("LDR: protect failed va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
(unsigned long long)va, (unsigned long long)bytes,
(unsigned long long)prot, s,
(unsigned long long)imageBase, (unsigned long long)mappedSize);
break;
}
runStart = runEnd;
}
MmFreePool(pageProt);
return s;
}
@@ -408,7 +459,7 @@ uintptr_t LdrGetEntryPoint(uintptr_t base) {
}
// Base Relocation.
static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, uintptr_t delta, bool user) {
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
OPT_HEADER64* opt = &pe->OptionalHeader;
@@ -432,6 +483,8 @@ static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
// For 64-bit PEs, we mostly care about DIR64 (Type 10).
if (type == IMAGE_REL_BASED_DIR64) {
uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset);
KSTATUS ws = KiEnsureUserImageWrite(process, user, (uintptr_t)ptr, sizeof(*ptr));
if (!KSUCCESS(ws)) return ws;
*ptr += delta;
}
}
@@ -534,6 +587,8 @@ static KSTATUS LdrResolveImportsCommon(
return STATUS_NOT_FOUND;
}
KSTATUS ws = KiEnsureUserImageWrite(process, process != (PROCESS*)0, (uintptr_t)thunk, sizeof(*thunk));
if (!KSUCCESS(ws)) return ws;
*thunk = (uint64_t)addr;
thunk++;
@@ -566,6 +621,41 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
);
}
// Loads a user image from an open file through the image-section object model.
KSTATUS LdrLoadImageFileForProcess(PROCESS* process, FILE_OBJECT* file, uintptr_t* baseOut) {
if (!process || !file || !baseOut)
return STATUS_INVALID_PARAMETER;
*baseOut = 0;
IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)0;
KSTATUS s = LdrCreateImageSectionFromFile(file, &section);
if (!KSUCCESS(s))
return s;
void* view = (void*)0;
s = LdrMapImageSection(process, section, &view);
if (!KSUCCESS(s)) {
LdrDereferenceImageSection(section);
return s;
}
const IMAGE_SECTION_METADATA* metadata = LdrImageSectionMetadata(section);
uintptr_t imageBase = (uintptr_t)view;
uintptr_t delta = imageBase - metadata->PreferredBase;
if (delta != 0) {
s = LdrApplyRelocationsForProcess(process, imageBase, delta, true);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
LdrDereferenceImageSection(section);
return s;
}
}
LdrDereferenceImageSection(section);
*baseOut = imageBase;
return STATUS_SUCCESS;
}
// Implements the ldr Register Process Module operation.
static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) {
if (!process || !name || !base) return STATUS_INVALID_PARAMETER;
@@ -602,36 +692,27 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
FILE_OBJECT* f;
KSTATUS s = IoOpenFile(SysSvFromCstr(filename), &f);
if (!KSUCCESS(s)) {
char path[260];
memcpy(path, "0:/system/", 10);
memcpy(path + 10, filename, strnlen(filename, 248) + 1);
path[259] = '\0';
s = IoOpenFile(SysSvFromCstr(path), &f);
char relative[260];
size_t nameLen = strnlen(filename, sizeof(relative) - 8);
if (nameLen + 8 >= sizeof(relative))
return STATUS_BUFFER_TOO_SMALL;
memcpy(relative, "system/", 7);
memcpy(relative + 7, filename, nameLen);
relative[7 + nameLen] = '\0';
s = KeOpenSystemRootFile(relative, &f);
if (!KSUCCESS(s)) {
DbgPrint("LdrLoadModuleForProcess: Failed to open %s (%d)\n", filename, s);
DbgPrint("LdrLoadModuleForProcess: Failed to open %s on system root (%d)\n", filename, s);
return s;
}
}
uint64_t fileSize = IoGetFileSize(f);
LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize);
void* mem = MmAllocatePool((size_t)fileSize);
if (!mem) {
IoCloseFile(f);
return STATUS_NO_MEMORY;
}
size_t got = 0;
s = IoReadFile(f, mem, (size_t)fileSize, &got);
IoCloseFile(f);
if (!KSUCCESS(s)) {
MmFreePool(mem);
return s;
}
uintptr_t imageBase = 0;
s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase);
MmFreePool(mem);
s = LdrLoadImageFileForProcess(process, f, &imageBase);
IoCloseFile(f);
if (!KSUCCESS(s)) {
DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s);
return s;
@@ -665,10 +746,9 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
return STATUS_SUCCESS;
}
// LdrLoadImage - load a PE from a raw buffer into virtual space.
static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
// Loads a boot/kernel PE image from a raw memory buffer into kernel space.
static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, uintptr_t* baseOut) {
if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER;
if (user && !process) return STATUS_INVALID_PARAMETER;
if (size < 0x40) return STATUS_FAILURE;
const uint8_t* bytes = (const uint8_t*)buffer;
@@ -687,13 +767,13 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
uint16_t optionalMagic = *(const uint16_t *)(bytes + peOffset + 24);
if (machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
DbgPrint("LdrLoadImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
DbgPrint("LdrLoadKernelImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
if (machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
DbgPrint("LdrLoadImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
DbgPrint("LdrLoadKernelImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
@@ -704,17 +784,16 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize);
if (!KSUCCESS(sizeStatus)) return sizeStatus;
LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n",
LdrDbg("LDR: kernel image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u\n",
opt->SizeOfImage,
(unsigned long long)imageSize,
(unsigned long long)opt->ImageBase,
pe->NumberOfSections,
user ? 1 : 0,
process ? process->ProcessId : 0);
void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize);
pe->NumberOfSections);
void* mem = MmAllocatePoolTag(imageSize, MM_POOL_TAG('i', 'm', 'g', ' '));
if (!mem) {
DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n",
(unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0);
DbgPrint("LDR: kernel image allocation failed required=%llx\n",
(unsigned long long)imageSize);
return STATUS_NO_MEMORY;
}
memset(mem, 0, imageSize);
@@ -730,27 +809,37 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
memcpy(dst, src, copySize);
}
uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0);
MmSetPageProtect((uintptr_t)mem + sec[i].VirtualAddress,
MmPteProtectRange((uintptr_t)mem + sec[i].VirtualAddress,
sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData,
prot);
PAGE_PRESENT | PAGE_WRITE);
}
uintptr_t delta = (uintptr_t)mem - opt->ImageBase;
if (delta != 0) {
LdrApplyRelocations((uintptr_t)mem, delta);
KSTATUS rs = LdrApplyRelocationsForProcess((PROCESS*)0, (uintptr_t)mem, delta, false);
if (!KSUCCESS(rs)) {
MmFreePool(mem);
return rs;
}
}
// Kernel drivers are still pool-backed images. Section protections cannot
// safely make pages read-only yet because imports and driver-owned writable
// image data are not separated from callback code by a section object. What
// must be true is simpler: every page in the loaded image must be executable
// enough for driver callbacks, including section-alignment slack near the
// end of the image.
KSTATUS xs = MmPteProtectRange((uintptr_t)mem, imageSize, PAGE_PRESENT | PAGE_WRITE);
if (!KSUCCESS(xs)) {
MmFreePool(mem);
return xs;
}
*baseOut = (uintptr_t)mem;
return STATUS_SUCCESS;
}
// Implements the ldr Load Image For Process operation.
KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) {
return KiLdrLoadImageCommon(process, buffer, size, baseOut, true);
}
// Implements the ldr Load Image operation.
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user);
// Loads a kernel image from a bootloader or driver raw buffer.
KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut) {
return KiLdrLoadKernelImageFromBuffer(buffer, size, baseOut);
}
code/kernel/ldr_section.c
+736
View File
@@ -0,0 +1,736 @@
// Image section objects. This file owns parsed PE image identity, sparse prototype pages, and image-section caching.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <kernel/ldr_section.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_pte.h>
#include <kernel/ob_object.h>
#include <kernel/io_file.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_debug.h>
#define PE_SIGNATURE 0x00004550UL
#define IMAGE_FILE_MACHINE_AMD64 0x8664
#define IMAGE_FILE_MACHINE_I386 0x014c
#define IMAGE_NT_OPTIONAL_HDR32_MAGIC 0x10b
#define IMAGE_NT_OPTIONAL_HDR64_MAGIC 0x20b
#define IMAGE_SCN_CNT_CODE 0x00000020u
#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040u
#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u
#define IMAGE_SCN_MEM_EXECUTE 0x20000000u
#define IMAGE_SCN_MEM_READ 0x40000000u
#define IMAGE_SCN_MEM_WRITE 0x80000000u
#define LDR_IMAGE_SECTION_CACHE_MAX 32u
#pragma pack(push, 1)
typedef struct LDR_OPT_HEADER64 {
uint16_t Magic;
uint8_t MajorLinkerVersion;
uint8_t MinorLinkerVersion;
uint32_t SizeOfCode;
uint32_t SizeOfInitializedData;
uint32_t SizeOfUninitializedData;
uint32_t AddressOfEntryPoint;
uint32_t BaseOfCode;
uint64_t ImageBase;
uint32_t SectionAlignment;
uint32_t FileAlignment;
uint16_t MajorOSVersion;
uint16_t MinorOSVersion;
uint16_t MajorImageVersion;
uint16_t MinorImageVersion;
uint16_t MajorSubsystemVersion;
uint16_t MinorSubsystemVersion;
uint32_t Win32VersionValue;
uint32_t SizeOfImage;
uint32_t SizeOfHeaders;
uint32_t CheckSum;
uint16_t Subsystem;
uint16_t DllCharacteristics;
uint64_t SizeOfStackReserve;
uint64_t SizeOfStackCommit;
uint64_t SizeOfHeapReserve;
uint64_t SizeOfHeapCommit;
uint32_t LoaderFlags;
uint32_t NumberOfRvaAndSizes;
struct { uint32_t VirtualAddress; uint32_t Size; } DataDirectory[16];
} LDR_OPT_HEADER64;
typedef struct LDR_PE_HEADER {
uint32_t Signature;
uint16_t Machine;
uint16_t NumberOfSections;
uint32_t TimeDateStamp;
uint32_t PointerToSymbolTable;
uint32_t NumberOfSymbols;
uint16_t SizeOfOptionalHeader;
uint16_t Characteristics;
LDR_OPT_HEADER64 OptionalHeader;
} LDR_PE_HEADER;
typedef struct LDR_SECTION_HEADER {
char Name[8];
uint32_t VirtualSize;
uint32_t VirtualAddress;
uint32_t SizeOfRawData;
uint32_t PointerToRawData;
uint32_t PointerToRelocations;
uint32_t PointerToLinenumbers;
uint16_t NumberOfRelocations;
uint16_t NumberOfLinenumbers;
uint32_t Characteristics;
} LDR_SECTION_HEADER;
#pragma pack(pop)
struct IMAGE_SECTION_OBJECT {
OBJECT Header;
IMAGE_SECTION_METADATA Metadata;
IMAGE_SECTION_DESCRIPTOR* Sections;
PHYSICAL_ADDRESS* Frames;
FILE_OBJECT* BackingFile;
size_t HeadersFileSize;
SPINLOCK Lock;
};
static IMAGE_SECTION_OBJECT* gImageSectionCache[LDR_IMAGE_SECTION_CACHE_MAX];
static SPINLOCK gImageSectionCacheLock = SPINLOCK_INIT;
// Aligns a PE image size to the hardware page size.
static size_t KiImageSectionAlignSize(size_t value) {
return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
}
// Mixes filesystem identity into a compact cache key.
static uint64_t KiImageSectionIdentityKey(const IO_FILE_IDENTITY* identity) {
if (!identity)
return 0;
uint64_t h = 1469598103934665603ULL;
h ^= identity->VolumeId;
h *= 1099511628211ULL;
h ^= identity->FileId;
h *= 1099511628211ULL;
h ^= identity->Generation;
h *= 1099511628211ULL;
h ^= identity->Size;
h *= 1099511628211ULL;
return h ? h : 1;
}
// Converts PE section characteristics to the final user page protection union for that section.
static uint64_t KiImageSectionProtection(uint32_t characteristics) {
uint64_t prot = PAGE_PRESENT | PAGE_USER | PAGE_NX;
if (characteristics & IMAGE_SCN_MEM_WRITE)
prot |= PAGE_WRITE;
if (characteristics & IMAGE_SCN_MEM_EXECUTE)
prot &= ~PAGE_NX;
return prot;
}
// Computes the required image mapping size after validating raw section extents.
static KSTATUS KiImageSectionComputeSize(const LDR_PE_HEADER* pe, size_t fileSize, size_t* outSize) {
if (!pe || !outSize) return STATUS_INVALID_PARAMETER;
const LDR_OPT_HEADER64* opt = &pe->OptionalHeader;
size_t required = KiImageSectionAlignSize(opt->SizeOfImage);
size_t headers = KiImageSectionAlignSize(opt->SizeOfHeaders);
if (headers > required) required = headers;
const LDR_SECTION_HEADER* sec = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
for (uint32_t i = 0; i < pe->NumberOfSections; i++) {
size_t va = (size_t)sec[i].VirtualAddress;
size_t virt = (size_t)sec[i].VirtualSize;
size_t raw = (size_t)sec[i].SizeOfRawData;
size_t span = virt ? virt : raw;
if (raw > span) span = raw;
if (!span) continue;
if (va > (size_t)-1 - span) return STATUS_NO_MEMORY;
size_t end = KiImageSectionAlignSize(va + span);
if (end > required) required = end;
if (raw) {
size_t rawBase = (size_t)sec[i].PointerToRawData;
if (rawBase > fileSize || raw > fileSize - rawBase)
return STATUS_FAILURE;
}
}
if (!required) return STATUS_FAILURE;
*outSize = required;
return STATUS_SUCCESS;
}
// Parses and validates the PE header that backs an image section object.
static KSTATUS KiImageSectionParseHeader(const void* buffer,
size_t bufferSize,
size_t fileSize,
const LDR_PE_HEADER** peOut,
const LDR_SECTION_HEADER** sectionsOut,
size_t* imageSizeOut) {
if (!buffer || !bufferSize || !peOut || !sectionsOut || !imageSizeOut)
return STATUS_INVALID_PARAMETER;
if (bufferSize < 0x40) return STATUS_FAILURE;
const uint8_t* bytes = (const uint8_t*)buffer;
uint32_t peOffset = *(const uint32_t*)(bytes + 0x3C);
if ((size_t)peOffset > bufferSize || bufferSize - (size_t)peOffset < 24)
return STATUS_FAILURE;
const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)(bytes + peOffset);
if (pe->Signature != PE_SIGNATURE)
return STATUS_FAILURE;
uint16_t optionalSize = pe->SizeOfOptionalHeader;
if (optionalSize < sizeof(LDR_OPT_HEADER64) ||
(size_t)peOffset + 24 + optionalSize > bufferSize) {
return STATUS_FAILURE;
}
uint16_t optionalMagic = pe->OptionalHeader.Magic;
if (pe->Machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
DbgPrint("LdrImageSection: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
pe->Machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
if (pe->Machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
DbgPrint("LdrImageSection: unsupported PE image (machine=0x%X magic=0x%X)\n",
pe->Machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
if (!pe->NumberOfSections)
return STATUS_FAILURE;
const LDR_SECTION_HEADER* sections = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader;
size_t sectionTableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER);
if (sectionTableOffset > bufferSize || sectionTableBytes > bufferSize - sectionTableOffset)
return STATUS_FAILURE;
size_t imageSize = 0;
KSTATUS s = KiImageSectionComputeSize(pe, fileSize, &imageSize);
if (!KSUCCESS(s)) return s;
*peOut = pe;
*sectionsOut = sections;
*imageSizeOut = imageSize;
return STATUS_SUCCESS;
}
// Releases all resources owned by an image section object.
static void KiDestroyImageSectionObject(OBJECT* object) {
IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)object;
if (!section) return;
if (section->Frames) {
for (size_t i = 0; i < section->Metadata.PageCount; ++i) {
if (section->Frames[i])
(void)MmPfnFree(section->Frames[i]);
}
MmFreePool(section->Frames);
section->Frames = (PHYSICAL_ADDRESS*)0;
}
if (section->Sections) {
MmFreePool(section->Sections);
section->Sections = (IMAGE_SECTION_DESCRIPTOR*)0;
}
if (section->BackingFile) {
IoCloseFile(section->BackingFile);
section->BackingFile = (FILE_OBJECT*)0;
}
}
// Finds a cached image section while the image-section cache lock is held.
static IMAGE_SECTION_OBJECT* KiFindImageSectionLocked(const IO_FILE_IDENTITY* identity, uint64_t key) {
if (!identity || !key)
return (IMAGE_SECTION_OBJECT*)0;
for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) {
IMAGE_SECTION_OBJECT* section = gImageSectionCache[i];
if (section &&
section->Metadata.ContentKey == key &&
section->Metadata.VolumeId == identity->VolumeId &&
section->Metadata.FileId == identity->FileId &&
section->Metadata.FileGeneration == identity->Generation &&
section->Metadata.RawSize == identity->Size) {
return section;
}
}
return (IMAGE_SECTION_OBJECT*)0;
}
// Verifies that a cached image section is still backed by the same file generation.
static KSTATUS KiValidateImageSectionBackingFile(IMAGE_SECTION_OBJECT* section) {
if (!section || !section->BackingFile)
return STATUS_INVALID_PARAMETER;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(section->BackingFile, &identity);
if (!KSUCCESS(s))
return s;
if (identity.VolumeId != section->Metadata.VolumeId ||
identity.FileId != section->Metadata.FileId ||
identity.Generation != section->Metadata.FileGeneration ||
identity.Size != section->Metadata.RawSize) {
return STATUS_CHECKSUM_MISMATCH;
}
return STATUS_SUCCESS;
}
// Returns true when a virtual image page has initialized bytes in the backing file.
static bool KiImageSectionPageHasFileBytes(const IMAGE_SECTION_OBJECT* section, size_t pageIndex) {
if (!section || pageIndex >= section->Metadata.PageCount)
return false;
size_t pageStart = pageIndex * PAGE_SIZE;
size_t pageEnd = pageStart + PAGE_SIZE;
if (pageStart < section->HeadersFileSize && pageEnd > 0)
return true;
for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) {
const IMAGE_SECTION_DESCRIPTOR* desc = &section->Sections[i];
size_t raw = desc->RawDataSize;
if (desc->VirtualSize && desc->VirtualSize < raw)
raw = desc->VirtualSize;
if (!raw)
continue;
size_t start = desc->VirtualAddress;
size_t end = start + raw;
if (end < start)
continue;
if (pageStart < end && pageEnd > start)
return true;
}
return false;
}
// Copies a file range into an image prototype page, leaving absent bytes zero-filled.
static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section,
uint64_t fileOffset,
void* dst,
size_t size) {
if (!section || !section->BackingFile || (!dst && size))
return STATUS_INVALID_PARAMETER;
if (!size)
return STATUS_SUCCESS;
size_t got = 0;
KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got);
if (!KSUCCESS(s))
return s;
if (got != size)
return STATUS_FAILURE;
return STATUS_SUCCESS;
}
// Fills one prototype frame from the image backing file.
static KSTATUS KiFillImageSectionFrameFromFile(IMAGE_SECTION_OBJECT* section,
size_t pageIndex,
PHYSICAL_ADDRESS frame) {
if (!section || !frame || pageIndex >= section->Metadata.PageCount)
return STATUS_INVALID_PARAMETER;
uint8_t* page = (uint8_t*)MmPfnToVirtual(frame);
if (!page)
return STATUS_FAILURE;
memset(page, 0, PAGE_SIZE);
size_t pageStart = pageIndex * PAGE_SIZE;
size_t pageEnd = pageStart + PAGE_SIZE;
if (pageStart < section->HeadersFileSize) {
size_t copyStart = pageStart;
size_t copyEnd = pageEnd;
if (copyEnd > section->HeadersFileSize)
copyEnd = section->HeadersFileSize;
KSTATUS s = KiReadImageSectionFileBytes(section,
(uint64_t)copyStart,
page,
copyEnd - copyStart);
if (!KSUCCESS(s))
return s;
}
for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) {
const IMAGE_SECTION_DESCRIPTOR* desc = &section->Sections[i];
size_t raw = desc->RawDataSize;
if (desc->VirtualSize && desc->VirtualSize < raw)
raw = desc->VirtualSize;
if (!raw)
continue;
size_t sectionStart = desc->VirtualAddress;
size_t sectionEnd = sectionStart + raw;
if (sectionEnd < sectionStart)
return STATUS_FAILURE;
if (pageStart >= sectionEnd || pageEnd <= sectionStart)
continue;
size_t copyStart = pageStart > sectionStart ? pageStart : sectionStart;
size_t copyEnd = pageEnd < sectionEnd ? pageEnd : sectionEnd;
size_t dstOffset = copyStart - pageStart;
uint64_t srcOffset = (uint64_t)desc->RawDataOffset + (uint64_t)(copyStart - sectionStart);
KSTATUS s = KiReadImageSectionFileBytes(section,
srcOffset,
page + dstOffset,
copyEnd - copyStart);
if (!KSUCCESS(s))
return s;
}
return STATUS_SUCCESS;
}
// Ensures a prototype page exists for initialized image bytes.
static KSTATUS KiEnsureImageSectionFrame(IMAGE_SECTION_OBJECT* section, size_t pageIndex) {
if (!section || pageIndex >= section->Metadata.PageCount)
return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&section->Lock);
PHYSICAL_ADDRESS existing = section->Frames[pageIndex];
KeReleaseSpinlock(&section->Lock, flags);
if (existing)
return STATUS_SUCCESS;
if (section->BackingFile && !KiImageSectionPageHasFileBytes(section, pageIndex))
return STATUS_SUCCESS;
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseImage);
if (!frame)
return STATUS_NO_MEMORY;
void* va = MmPfnToVirtual(frame);
if (!va) {
MmPfnFree(frame);
return STATUS_FAILURE;
}
KSTATUS s = STATUS_SUCCESS;
if (section->BackingFile) {
s = KiValidateImageSectionBackingFile(section);
if (KSUCCESS(s))
s = KiFillImageSectionFrameFromFile(section, pageIndex, frame);
} else {
memset(va, 0, PAGE_SIZE);
}
if (!KSUCCESS(s)) {
MmPfnFree(frame);
return s;
}
flags = KeAcquireSpinlock(&section->Lock);
if (section->Frames[pageIndex]) {
KeReleaseSpinlock(&section->Lock, flags);
MmPfnFree(frame);
return STATUS_SUCCESS;
}
section->Frames[pageIndex] = frame;
KeReleaseSpinlock(&section->Lock, flags);
return STATUS_SUCCESS;
}
// Builds an uncached image section object backed directly by an open file.
static KSTATUS KiBuildImageSectionObjectFromFile(FILE_OBJECT* file,
const void* headerBuffer,
size_t headerBufferSize,
const IO_FILE_IDENTITY* identity,
uint64_t key,
IMAGE_SECTION_OBJECT** sectionOut) {
if (!file || !headerBuffer || !headerBufferSize || !identity || !sectionOut)
return STATUS_INVALID_PARAMETER;
*sectionOut = (IMAGE_SECTION_OBJECT*)0;
size_t fileSize = (size_t)identity->Size;
const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)0;
const LDR_SECTION_HEADER* rawSections = (const LDR_SECTION_HEADER*)0;
size_t imageSize = 0;
KSTATUS s = KiImageSectionParseHeader(headerBuffer,
headerBufferSize,
fileSize,
&pe,
&rawSections,
&imageSize);
if (!KSUCCESS(s))
return s;
size_t pageCount = (imageSize + PAGE_SIZE - 1) / PAGE_SIZE;
if (!pageCount || pageCount > ((size_t)-1 / sizeof(PHYSICAL_ADDRESS)))
return STATUS_FAILURE;
IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)ObAllocateObject(ObjectTypeImageSection,
sizeof(IMAGE_SECTION_OBJECT));
if (!section)
return STATUS_NO_MEMORY;
section->Header.Destroy = KiDestroyImageSectionObject;
PHYSICAL_ADDRESS* frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS),
MM_POOL_TAG('i', 's', 'f', 'r'));
if (!frames) {
ObDereferenceObject(&section->Header);
return STATUS_NO_MEMORY;
}
memset(frames, 0, pageCount * sizeof(PHYSICAL_ADDRESS));
IMAGE_SECTION_DESCRIPTOR* descriptors = (IMAGE_SECTION_DESCRIPTOR*)MmAllocatePoolTag(
(size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR),
MM_POOL_TAG('i', 's', 'e', 'c'));
if (!descriptors) {
MmFreePool(frames);
ObDereferenceObject(&section->Header);
return STATUS_NO_MEMORY;
}
memset(descriptors, 0, (size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR));
section->Frames = frames;
section->Sections = descriptors;
section->BackingFile = file;
section->HeadersFileSize = pe->OptionalHeader.SizeOfHeaders;
section->Lock = (SPINLOCK)SPINLOCK_INIT;
ObReferenceObject(&file->Header);
section->Metadata.ContentKey = key;
section->Metadata.VolumeId = identity->VolumeId;
section->Metadata.FileId = identity->FileId;
section->Metadata.FileGeneration = identity->Generation;
section->Metadata.PreferredBase = pe->OptionalHeader.ImageBase;
section->Metadata.EntryRva = pe->OptionalHeader.AddressOfEntryPoint;
section->Metadata.HeaderSize = pe->OptionalHeader.SizeOfHeaders;
section->Metadata.RawSize = fileSize;
section->Metadata.ImageSize = imageSize;
section->Metadata.PageCount = pageCount;
section->Metadata.SectionCount = pe->NumberOfSections;
for (uint32_t i = 0; i < pe->NumberOfSections; ++i) {
IMAGE_SECTION_DESCRIPTOR* dst = &section->Sections[i];
memcpy(dst->Name, rawSections[i].Name, sizeof(rawSections[i].Name));
dst->Name[8] = '\0';
dst->VirtualAddress = rawSections[i].VirtualAddress;
dst->VirtualSize = rawSections[i].VirtualSize;
dst->RawDataOffset = rawSections[i].PointerToRawData;
dst->RawDataSize = rawSections[i].SizeOfRawData;
dst->Characteristics = rawSections[i].Characteristics;
dst->PageProtection = KiImageSectionProtection(rawSections[i].Characteristics);
}
*sectionOut = section;
return STATUS_SUCCESS;
}
// Creates or references an image section object backed by an open file.
KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file,
IMAGE_SECTION_OBJECT** sectionOut) {
if (!file || !sectionOut)
return STATUS_INVALID_PARAMETER;
*sectionOut = (IMAGE_SECTION_OBJECT*)0;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(file, &identity);
if (!KSUCCESS(s))
return s;
size_t fileSize = (size_t)identity.Size;
if (!fileSize)
return STATUS_FAILURE;
size_t firstRead = fileSize < 4096u ? fileSize : 4096u;
uint8_t stackHeader[4096];
size_t got = 0;
s = IoReadFileAt(file, 0, stackHeader, firstRead, &got);
if (!KSUCCESS(s) || got != firstRead)
return KSUCCESS(s) ? STATUS_FAILURE : s;
if (firstRead < 0x40)
return STATUS_FAILURE;
uint32_t peOffset = *(const uint32_t*)(stackHeader + 0x3C);
if ((size_t)peOffset > fileSize || fileSize - (size_t)peOffset < 24)
return STATUS_FAILURE;
size_t minimumHeader = (size_t)peOffset + 24u;
if (minimumHeader > firstRead) {
if (minimumHeader > 65536u || minimumHeader > fileSize)
return STATUS_FAILURE;
void* tmp = MmAllocatePoolTag(minimumHeader, MM_POOL_TAG('i', 's', 'h', 'd'));
if (!tmp)
return STATUS_NO_MEMORY;
s = IoReadFileAt(file, 0, tmp, minimumHeader, &got);
if (!KSUCCESS(s) || got != minimumHeader) {
MmFreePool(tmp);
return KSUCCESS(s) ? STATUS_FAILURE : s;
}
uint16_t optSize = *(const uint16_t*)((const uint8_t*)tmp + peOffset + 20u);
size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize;
MmFreePool(tmp);
if (fullHeader > firstRead) {
firstRead = fullHeader;
}
} else {
uint16_t optSize = *(const uint16_t*)(stackHeader + peOffset + 20u);
size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize;
if (fullHeader > firstRead)
firstRead = fullHeader;
}
if (firstRead > fileSize || firstRead > 65536u)
return STATUS_FAILURE;
void* header = stackHeader;
bool heapHeader = false;
if (firstRead > sizeof(stackHeader)) {
header = MmAllocatePoolTag(firstRead, MM_POOL_TAG('i', 's', 'h', 'd'));
if (!header)
return STATUS_NO_MEMORY;
heapHeader = true;
s = IoReadFileAt(file, 0, header, firstRead, &got);
if (!KSUCCESS(s) || got != firstRead) {
MmFreePool(header);
return KSUCCESS(s) ? STATUS_FAILURE : s;
}
}
const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)((const uint8_t*)header + peOffset);
size_t tableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER);
size_t tableEnd = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader + tableBytes;
if (tableEnd > firstRead) {
if (tableEnd > fileSize || tableEnd > 65536u) {
if (heapHeader) MmFreePool(header);
return STATUS_FAILURE;
}
void* bigger = MmAllocatePoolTag(tableEnd, MM_POOL_TAG('i', 's', 'h', 'd'));
if (!bigger) {
if (heapHeader) MmFreePool(header);
return STATUS_NO_MEMORY;
}
s = IoReadFileAt(file, 0, bigger, tableEnd, &got);
if (heapHeader) MmFreePool(header);
if (!KSUCCESS(s) || got != tableEnd) {
MmFreePool(bigger);
return KSUCCESS(s) ? STATUS_FAILURE : s;
}
header = bigger;
firstRead = tableEnd;
heapHeader = true;
}
uint64_t key = KiImageSectionIdentityKey(&identity);
uintptr_t flags = KeAcquireSpinlock(&gImageSectionCacheLock);
IMAGE_SECTION_OBJECT* cached = KiFindImageSectionLocked(&identity, key);
if (cached) {
ObReferenceObject(&cached->Header);
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
if (heapHeader) MmFreePool(header);
*sectionOut = cached;
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
IMAGE_SECTION_OBJECT* built = (IMAGE_SECTION_OBJECT*)0;
s = KiBuildImageSectionObjectFromFile(file, header, firstRead, &identity, key, &built);
if (heapHeader) MmFreePool(header);
if (!KSUCCESS(s))
return s;
flags = KeAcquireSpinlock(&gImageSectionCacheLock);
cached = KiFindImageSectionLocked(&identity, key);
if (cached) {
ObReferenceObject(&cached->Header);
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
ObDereferenceObject(&built->Header);
*sectionOut = cached;
return STATUS_SUCCESS;
}
for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) {
if (!gImageSectionCache[i]) {
ObReferenceObject(&built->Header);
gImageSectionCache[i] = built;
break;
}
}
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
*sectionOut = built;
return STATUS_SUCCESS;
}
// Adds a reference to an existing image section object.
void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section) {
if (!section) return;
ObReferenceObject(&section->Header);
}
// Releases a reference returned by LdrCreateImageSectionFromFile.
void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section) {
if (!section) return;
ObDereferenceObject(&section->Header);
}
// Returns immutable metadata for an image section object.
const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section) {
if (!section) return (const IMAGE_SECTION_METADATA*)0;
return &section->Metadata;
}
// Returns the immutable PE section table owned by the image section object.
const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section,
uint32_t* countOut) {
if (countOut) *countOut = section ? section->Metadata.SectionCount : 0;
if (!section) return (const IMAGE_SECTION_DESCRIPTOR*)0;
return section->Sections;
}
// Retains one prototype page frame owned by an image section object.
KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section,
size_t pageIndex,
PHYSICAL_ADDRESS* frameOut) {
if (!section || !frameOut) return STATUS_INVALID_PARAMETER;
*frameOut = 0;
if (pageIndex >= section->Metadata.PageCount)
return STATUS_INVALID_PARAMETER;
KSTATUS es = KiEnsureImageSectionFrame(section, pageIndex);
if (!KSUCCESS(es))
return es;
uintptr_t flags = KeAcquireSpinlock(&section->Lock);
PHYSICAL_ADDRESS frame = section->Frames[pageIndex];
KeReleaseSpinlock(&section->Lock, flags);
if (!frame)
return STATUS_SUCCESS;
KSTATUS s = MmPfnRetain(frame);
if (!KSUCCESS(s))
return s;
*frameOut = frame;
return STATUS_SUCCESS;
}
// Maps an image section into a process as an image-backed VAD.
KSTATUS LdrMapImageSection(PROCESS* process,
IMAGE_SECTION_OBJECT* section,
void** baseOut) {
if (!process || !section || !baseOut) return STATUS_INVALID_PARAMETER;
*baseOut = (void*)0;
return MmVadCreateImageSectionMapping(process,
section,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
baseOut);
}
code/kernel/mm_file_cache.c
+1273
View File
File diff suppressed because it is too large. Load diff
code/kernel/mm_phys.c
+711 -176
View File
File diff suppressed because it is too large. Load diff
code/kernel/mm_pte.c
+575
View File
@@ -0,0 +1,575 @@
// Kernel page-table manager. This file owns x86-64 PTE manipulation, TLB invalidation, and shared-CR3 activation.
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/mm_pte.h>
#include <kernel/mm_pfn.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <kernel/hal/hal_apic.h>
// Page-table frames are reached through the physical direct map while the kernel still owns one shared CR3.
static uint64_t gPhysBase = PHYS_MAP_BASE;
// Converts a physical address to the kernel physical-window alias used for page-table editing.
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
return (void *)(p + gPhysBase);
}
// Returns the entry pointer for a page-table frame that is already mapped in the physical window.
static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
}
// The bootloader builds the initial PML4, and this layer owns all later edits to it.
static PHYSICAL_ADDRESS gPml4Phys = 0;
static SPINLOCK gPageTableLock = SPINLOCK_INIT;
static volatile long gTlbShootdownLock = 0;
static volatile long gTlbShootdownActive = 0;
static volatile long gTlbShootdownAcks = 0;
static uintptr_t gTlbShootdownVirt = 0;
static MM_PTE_STATS gPteStats;
#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u
// These macros extract the hardware page-table index for each level.
#define PML4_IDX(va) (((va) >> 39) & 0x1FF)
#define PDPT_IDX(va) (((va) >> 30) & 0x1FF)
#define PDT_IDX(va) (((va) >> 21) & 0x1FF)
#define PT_IDX(va) (((va) >> 12) & 0x1FF)
#define PAGE_PS (1ULL << 7)
#define PAGE_FLAGS_MASK (0xFFFULL | PAGE_NX)
#define PAGE_FRAME_4K_MASK 0x000FFFFFFFFFF000ULL
#define PAGE_FRAME_2M_MASK 0x000FFFFFFFE00000ULL
#define PAGE_FRAME_1G_MASK 0x000FFFFFC0000000ULL
// Bumps a telemetry counter. These counters are diagnostic only; exactness is
// less important than avoiding extra locks in hot page-table paths.
static inline void KiPteStatInc(uint64_t* counter) {
(*counter)++;
}
// Clamps query attribution so corrupt callers cannot index outside telemetry arrays.
static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose) {
if ((uint32_t)purpose >= (uint32_t)MmPteQueryPurposeCount)
return MmPteQueryGeneric;
return purpose;
}
// Ensures that one parent entry points to a child page table.
// New child tables are zeroed before publication so no stale mappings become visible.
static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
unsigned index, uint64_t extraFlags) {
uint64_t* entry = KiPtePtr(parentPhys, index);
if (*entry & PAGE_PRESENT) {
*entry |= extraFlags;
return *entry & PAGE_FRAME_4K_MASK;
}
PHYSICAL_ADDRESS childPhys = MmPfnAllocate(MmPfnUsePageTable);
if (!childPhys) return 0;
KiPteStatInc(&gPteStats.PageTablePagesAllocated);
memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE);
*entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags;
return childPhys;
}
// Initializes the page-table manager around the bootloader-created PML4.
void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys) {
gPml4Phys = bootPml4Phys;
gPageTableLock = (SPINLOCK)SPINLOCK_INIT;
gTlbShootdownLock = 0;
gTlbShootdownActive = 0;
gTlbShootdownAcks = 0;
gTlbShootdownVirt = 0;
memset(&gPteStats, 0, sizeof(gPteStats));
}
// Installs the shared PML4 into CR3.
void MmPteActivate(void) {
__writecr3(gPml4Phys);
}
// Flushes one virtual address from the local TLB.
void MmPteFlushLocal(uintptr_t virt) {
KiPteStatInc(&gPteStats.LocalFlushes);
__invlpg((void*)virt);
}
// Acquires the shootdown state lock without disabling interrupts.
static void KiAcquireTlbShootdownLock(void) {
// Do not use the normal spinlock here: it masks interrupts while spinning.
// A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
// concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is
// meant to avoid.
while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0)
_mm_pause();
_ReadWriteBarrier();
}
// Releases the shootdown state lock after all shared fields are stable.
static void KiReleaseTlbShootdownLock(void) {
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownLock, 0);
}
// Flushes one kernel mapping on every online CPU that shares this PML4.
void MmPteFlushKernelAddress(uintptr_t virt) {
KiPteStatInc(&gPteStats.GlobalFlushes);
MmPteFlushLocal(virt);
if (gCpuCount <= 1) {
KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets);
return;
}
KCPU* self = KeGetCurrentCpu();
uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU;
KiAcquireTlbShootdownLock();
long targets = 0;
gTlbShootdownVirt = virt;
_InterlockedExchange(&gTlbShootdownAcks, 0);
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownActive, 1);
_ReadWriteBarrier();
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (i == selfIndex)
continue;
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
targets++;
KiPteStatInc(&gPteStats.ShootdownTargetCpus);
HalApicSendTlbShootdownIpi(cpu->LapicId);
}
if (!targets) {
KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets);
_InterlockedExchange(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
return;
}
KiPteStatInc(&gPteStats.ShootdownRequests);
for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets)
break;
_mm_pause();
}
long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0);
_InterlockedExchange(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
gPteStats.ShootdownAcks += (uint64_t)acks;
if (acks < targets) {
KiPteStatInc(&gPteStats.ShootdownTimeouts);
KePanic(KE_PANIC_CODE_GENERIC,
"TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n",
(uint64_t)virt, targets, acks, selfIndex);
}
}
// Handles an incoming shootdown request on the current CPU.
static int KiHandleTlbShootdown(void) {
if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
return 0;
uintptr_t va = gTlbShootdownVirt;
_ReadWriteBarrier();
MmPteFlushLocal(va);
_ReadWriteBarrier();
_InterlockedIncrement(&gTlbShootdownAcks);
return 1;
}
// Handles the NMI vector reserved for TLB shootdowns.
int MmPteHandleShootdownNmi(void) {
return KiHandleTlbShootdown();
}
// Maps or replaces one 4 KiB leaf PTE in the shared page tables.
__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) {
KiPteStatInc(&gPteStats.MapCalls);
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
if (!pdpt) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
if (!pdt) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag);
if (!pt) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
if ((*pte & PAGE_PRESENT) && mode == MmPteMapFailIfPresent) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_ALREADY_EXISTS;
}
int wasPresent = ((*pte & PAGE_PRESENT) != 0);
*pte = phys | flags;
if (!wasPresent) {
KiPteStatInc(&gPteStats.CurrentLeafMappings);
KiPteStatInc(&gPteStats.NewMappingNoFlushes);
}
KeReleaseSpinlock(&gPageTableLock, lockFlags);
if (wasPresent) {
KiPteStatInc(&gPteStats.ReplacedMappingFlushes);
MmPteFlushKernelAddress(virt);
}
return STATUS_SUCCESS;
}
// Removes one 4 KiB leaf PTE from the shared page tables.
__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
KiPteStatInc(&gPteStats.UnmapCalls);
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
if (!(*e4 & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt));
if (!(*e3 & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt));
if (!(*e2 & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt));
if (!(*pte & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
*pte = 0;
if (gPteStats.CurrentLeafMappings)
gPteStats.CurrentLeafMappings--;
KeReleaseSpinlock(&gPageTableLock, lockFlags);
MmPteFlushKernelAddress(virt);
return STATUS_SUCCESS;
}
// Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) {
KiPteStatInc(&gPteStats.ProtectCalls);
if (!size) return STATUS_SUCCESS;
// The protection operation is page-granular even when callers pass an interior address.
uintptr_t base = virt & PAGE_MASK;
uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
KSTATUS status = STATUS_SUCCESS;
uintptr_t flushEnd = base;
for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
if (*e3 & PAGE_PS) {
status = STATUS_NOT_SUPPORTED;
break;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
if (*e2 & PAGE_PS) {
status = STATUS_NOT_SUPPORTED;
break;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
// The physical frame stays attached while access and cache attributes change.
PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK;
*pte = phys | flags;
flushEnd = va + PAGE_SIZE;
}
if (!KSUCCESS(status))
KiPteStatInc(&gPteStats.ProtectFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE)
MmPteFlushKernelAddress(va);
return status;
}
// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf. This is
// used by the pressure trimmer as a second-chance test for clean file-backed
// pages. If the page was accessed recently, reclaim skips it once and flushes
// the TLB entry so future accesses can set PAGE_ACCESSED again.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt) {
if (!gPml4Phys)
return 0;
uintptr_t va = virt & PAGE_MASK;
int wasAccessed = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT))
goto out;
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS))
goto out;
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS))
goto out;
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT))
goto out;
if (*pte & PAGE_ACCESSED) {
*pte &= ~PAGE_ACCESSED;
wasAccessed = 1;
}
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
if (wasAccessed)
MmPteFlushKernelAddress(va);
return wasAccessed;
}
// Tests and clears the hardware dirty bit on a mapped 4 KiB leaf.
__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt) {
if (!gPml4Phys)
return 0;
uintptr_t va = virt & PAGE_MASK;
int wasDirty = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT))
goto out;
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS))
goto out;
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS))
goto out;
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT))
goto out;
if (*pte & PAGE_DIRTY) {
*pte &= ~PAGE_DIRTY;
wasDirty = 1;
}
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
if (wasDirty)
MmPteFlushKernelAddress(va);
return wasDirty;
}
// Translates a mapped virtual address to a physical address.
__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt) {
KiPteStatInc(&gPteStats.TranslateCalls);
if (!gPml4Phys) return 0;
PHYSICAL_ADDRESS result = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t pml4_idx = (virt >> 39) & 0x1FF;
uint64_t pdpt_idx = (virt >> 30) & 0x1FF;
uint64_t pd_idx = (virt >> 21) & 0x1FF;
uint64_t pt_idx = (virt >> 12) & 0x1FF;
uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out;
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK);
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out;
// A present PDPT leaf maps a 1 GiB huge page.
if (pdpt[pdpt_idx] & PAGE_PS) {
result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF);
goto out;
}
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK);
if (!(pd[pd_idx] & PAGE_PRESENT)) goto out;
// A present PDT leaf maps a 2 MiB huge page.
if (pd[pd_idx] & PAGE_PS) {
result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF);
goto out;
}
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK);
if (!(pt[pt_idx] & PAGE_PRESENT)) goto out;
// A normal PT leaf maps a 4 KiB page.
result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return result;
}
// Queries one virtual address through the shared page-table hierarchy.
KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose,
uintptr_t virt,
PHYSICAL_ADDRESS* physOut,
uint64_t* flagsOut) {
purpose = KiNormalizeQueryPurpose(purpose);
KiPteStatInc(&gPteStats.QueryCalls);
KiPteStatInc(&gPteStats.QueryByPurpose[purpose]);
if (!gPml4Phys) {
KiPteStatInc(&gPteStats.QueryFailures);
KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]);
return STATUS_NOT_FOUND;
}
KSTATUS status = STATUS_NOT_FOUND;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys);
uint64_t e4 = pml4[PML4_IDX(virt)];
if (!(e4 & PAGE_PRESENT)) goto out;
uint64_t effectiveWrite = e4 & PAGE_WRITE;
uint64_t effectiveUser = e4 & PAGE_USER;
uint64_t effectiveNx = e4 & PAGE_NX;
uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK);
uint64_t e3 = pdpt[PDPT_IDX(virt)];
if (!(e3 & PAGE_PRESENT)) goto out;
effectiveWrite &= e3 & PAGE_WRITE;
effectiveUser &= e3 & PAGE_USER;
effectiveNx |= e3 & PAGE_NX;
if (e3 & PAGE_PS) {
if (physOut)
*physOut = (e3 & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFFULL);
if (flagsOut) {
uint64_t flags = (e3 & PAGE_FLAGS_MASK) | effectiveNx;
if (!effectiveWrite) flags &= ~PAGE_WRITE;
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK);
uint64_t e2 = pd[PDT_IDX(virt)];
if (!(e2 & PAGE_PRESENT)) goto out;
effectiveWrite &= e2 & PAGE_WRITE;
effectiveUser &= e2 & PAGE_USER;
effectiveNx |= e2 & PAGE_NX;
if (e2 & PAGE_PS) {
if (physOut)
*physOut = (e2 & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFFULL);
if (flagsOut) {
uint64_t flags = (e2 & PAGE_FLAGS_MASK) | effectiveNx;
if (!effectiveWrite) flags &= ~PAGE_WRITE;
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK);
uint64_t e1 = pt[PT_IDX(virt)];
if (!(e1 & PAGE_PRESENT)) goto out;
effectiveWrite &= e1 & PAGE_WRITE;
effectiveUser &= e1 & PAGE_USER;
effectiveNx |= e1 & PAGE_NX;
if (physOut)
*physOut = (e1 & PAGE_FRAME_4K_MASK) + (virt & 0xFFFULL);
if (flagsOut) {
uint64_t flags = (e1 & PAGE_FLAGS_MASK) | effectiveNx;
if (!effectiveWrite) flags &= ~PAGE_WRITE;
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
status = STATUS_SUCCESS;
out:
if (!KSUCCESS(status)) {
KiPteStatInc(&gPteStats.QueryFailures);
KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]);
}
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return status;
}
// Queries one virtual address without caller-specific attribution.
__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt,
PHYSICAL_ADDRESS* physOut,
uint64_t* flagsOut) {
return MmPteQueryPageFor(MmPteQueryGeneric, virt, physOut, flagsOut);
}
// Returns page-table telemetry for diagnostics.
void MmPteQueryStats(MM_PTE_STATS* out) {
if (!out)
return;
*out = gPteStats;
}
code/kernel/mm_swap.c
+243
View File
@@ -0,0 +1,243 @@
// Kernel swap manager.
// The first backend is a fixed-size pagefile accessed through VFS offset I/O.
// Kernel memory and filesystem code remain non-pageable, so this layer is safe
// to use before the VMM has direct block-extents for pageout.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <kernel/mm_swap.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_pfn.h>
#include <kernel/io_file.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_debug.h>
typedef struct MM_SWAP_STATE {
SPINLOCK Lock;
FILE_OBJECT* File;
uint8_t* Bitmap;
uint32_t SlotCount;
uint32_t UsedSlots;
uint32_t HighWaterSlots;
int Enabled;
MM_SWAP_STATS Stats;
} MM_SWAP_STATE;
static MM_SWAP_STATE gSwap = { .Lock = SPINLOCK_INIT };
// Returns whether a swap slot is marked in use.
static int KiSwapTestSlot(uint32_t slot) {
return (gSwap.Bitmap[slot >> 3] & (uint8_t)(1u << (slot & 7))) != 0;
}
// Marks a swap slot as allocated or free.
static void KiSwapSetSlot(uint32_t slot, int used) {
uint8_t mask = (uint8_t)(1u << (slot & 7));
if (used)
gSwap.Bitmap[slot >> 3] |= mask;
else
gSwap.Bitmap[slot >> 3] &= (uint8_t)~mask;
}
// Allocates one pagefile slot.
static KSTATUS KiSwapAllocateSlot(uint32_t* slotOut) {
if (!slotOut) return STATUS_INVALID_PARAMETER;
*slotOut = 0;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
if (!gSwap.Enabled || !gSwap.Bitmap || !gSwap.SlotCount) {
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_NOT_SUPPORTED;
}
for (uint32_t slot = 0; slot < gSwap.SlotCount; ++slot) {
if (!KiSwapTestSlot(slot)) {
KiSwapSetSlot(slot, 1);
gSwap.UsedSlots++;
if (gSwap.UsedSlots > gSwap.HighWaterSlots)
gSwap.HighWaterSlots = gSwap.UsedSlots;
gSwap.Stats.SlotAllocations++;
*slotOut = slot;
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_SUCCESS;
}
}
gSwap.Stats.SlotAllocationFailures++;
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_NO_MEMORY;
}
// Initializes the fixed pagefile backend.
KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
if (!path.Data || !path.Length || sizeBytes < PAGE_SIZE)
return STATUS_INVALID_PARAMETER;
if (sizeBytes & (PAGE_SIZE - 1))
sizeBytes = (sizeBytes + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1);
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
if (gSwap.Enabled) {
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&gSwap.Lock, flags);
FILE_OBJECT* file = (FILE_OBJECT*)0;
KSTATUS s = IoOpenFile(path, &file);
if (!KSUCCESS(s)) {
s = IoCreateFile(path, &file);
if (!KSUCCESS(s))
return s;
}
// Swap slots are never read before MmSwapPageOut overwrites the whole
// page, so the filesystem may allocate backing clusters without zeroing
// them. Normal files still get zero-on-extend semantics.
file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;
s = IoTruncateFile(file, sizeBytes);
if (!KSUCCESS(s)) {
IoCloseFile(file);
return s;
}
uint64_t slots64 = sizeBytes / PAGE_SIZE;
if (!slots64 || slots64 > 0xFFFFFFFFull) {
IoCloseFile(file);
return STATUS_INVALID_PARAMETER;
}
uint32_t slotCount = (uint32_t)slots64;
size_t bitmapBytes = (slotCount + 7u) / 8u;
uint8_t* bitmap = (uint8_t*)MmAllocatePoolTag(bitmapBytes, MM_POOL_TAG('s', 'w', 'a', 'p'));
if (!bitmap) {
IoCloseFile(file);
return STATUS_NO_MEMORY;
}
memset(bitmap, 0, bitmapBytes);
flags = KeAcquireSpinlock(&gSwap.Lock);
if (gSwap.Enabled) {
KeReleaseSpinlock(&gSwap.Lock, flags);
MmFreePool(bitmap);
IoCloseFile(file);
return STATUS_SUCCESS;
}
gSwap.File = file;
gSwap.Bitmap = bitmap;
gSwap.SlotCount = slotCount;
gSwap.UsedSlots = 0;
gSwap.HighWaterSlots = 0;
gSwap.Enabled = 1;
memset(&gSwap.Stats, 0, sizeof(gSwap.Stats));
gSwap.Stats.Enabled = 1;
gSwap.Stats.TotalSlots = slotCount;
gSwap.Stats.FreeSlots = slotCount;
KeReleaseSpinlock(&gSwap.Lock, flags);
DbgPrint("swap: initialized %u slots (%llu KiB) at %v\n",
slotCount,
(unsigned long long)(sizeBytes / 1024),
path);
return STATUS_SUCCESS;
}
// Returns whether swap is available for pageout.
int MmSwapIsEnabled(void) {
return gSwap.Enabled;
}
// Writes one physical page to a newly allocated swap slot.
KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut) {
if (!slotOut || !frame)
return STATUS_INVALID_PARAMETER;
*slotOut = 0;
if (!gSwap.Enabled || !gSwap.File)
return STATUS_NOT_SUPPORTED;
uint32_t slot = 0;
KSTATUS s = KiSwapAllocateSlot(&slot);
if (!KSUCCESS(s)) {
gSwap.Stats.PageOutFailures++;
return s;
}
void* page = MmPfnToVirtual(frame);
if (!page) {
MmSwapFreeSlot(slot);
gSwap.Stats.PageOutFailures++;
return STATUS_FAILURE;
}
size_t written = 0;
s = IoWriteFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &written);
if (!KSUCCESS(s) || written != PAGE_SIZE) {
MmSwapFreeSlot(slot);
gSwap.Stats.PageOutFailures++;
return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s;
}
gSwap.Stats.PageOuts++;
*slotOut = slot;
return STATUS_SUCCESS;
}
// Reads one swap slot into an already allocated physical page.
KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame) {
if (!frame)
return STATUS_INVALID_PARAMETER;
if (!gSwap.Enabled || !gSwap.File)
return STATUS_NOT_SUPPORTED;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
int valid = slot < gSwap.SlotCount && KiSwapTestSlot(slot);
KeReleaseSpinlock(&gSwap.Lock, flags);
if (!valid) {
gSwap.Stats.PageInFailures++;
return STATUS_NOT_FOUND;
}
void* page = MmPfnToVirtual(frame);
if (!page) {
gSwap.Stats.PageInFailures++;
return STATUS_FAILURE;
}
size_t read = 0;
KSTATUS s = IoReadFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &read);
if (!KSUCCESS(s) || read != PAGE_SIZE) {
gSwap.Stats.PageInFailures++;
return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s;
}
gSwap.Stats.PageIns++;
return STATUS_SUCCESS;
}
// Releases a swap slot after the owning VAD page no longer references it.
void MmSwapFreeSlot(uint32_t slot) {
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
if (gSwap.Enabled && slot < gSwap.SlotCount && KiSwapTestSlot(slot)) {
KiSwapSetSlot(slot, 0);
if (gSwap.UsedSlots)
gSwap.UsedSlots--;
gSwap.Stats.SlotFrees++;
}
KeReleaseSpinlock(&gSwap.Lock, flags);
}
// Copies swap telemetry for user diagnostics.
void MmSwapQueryStats(MM_SWAP_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
MM_SWAP_STATS stats = gSwap.Stats;
stats.Enabled = gSwap.Enabled ? 1 : 0;
stats.TotalSlots = gSwap.SlotCount;
stats.UsedSlots = gSwap.UsedSlots;
stats.FreeSlots = gSwap.SlotCount >= gSwap.UsedSlots ? gSwap.SlotCount - gSwap.UsedSlots : 0;
stats.HighWaterSlots = gSwap.HighWaterSlots;
KeReleaseSpinlock(&gSwap.Lock, flags);
*out = stats;
}
code/kernel/mm_virt.c
+2723 -889
View File
File diff suppressed because it is too large. Load diff
code/kernel/ob_object.c
+27 -2
View File
@@ -5,11 +5,36 @@
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
// Creates object.
// Returns a diagnostic pool tag for the object type.
static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
switch (type) {
case ObjectTypeProcess: return MM_POOL_TAG('p', 'r', 'o', 'c');
case ObjectTypeThread: return MM_POOL_TAG('t', 'h', 'r', 'd');
case ObjectTypeMutex: return MM_POOL_TAG('m', 't', 'x', ' ');
case ObjectTypeSemaphore: return MM_POOL_TAG('s', 'e', 'm', ' ');
case ObjectTypeEvent: return MM_POOL_TAG('e', 'v', 'n', 't');
case ObjectTypeQueue: return MM_POOL_TAG('q', 'u', 'e', 'u');
case ObjectTypeFile: return MM_POOL_TAG('f', 'i', 'l', 'e');
case ObjectTypeDir: return MM_POOL_TAG('d', 'i', 'r', ' ');
case ObjectTypeAsyncIo: return MM_POOL_TAG('a', 'i', 'o', ' ');
case ObjectTypeSocket: return MM_POOL_TAG('s', 'o', 'c', 'k');
case ObjectTypeDevice: return MM_POOL_TAG('d', 'e', 'v', ' ');
case ObjectTypeDriver: return MM_POOL_TAG('d', 'r', 'v', 'r');
case ObjectTypeUsbDevice: return MM_POOL_TAG('u', 's', 'b', 'd');
case ObjectTypeUsbPipe: return MM_POOL_TAG('u', 's', 'b', 'p');
case ObjectTypeFileMapping: return MM_POOL_TAG('m', 'a', 'p', 'f');
case ObjectTypeSurface: return MM_POOL_TAG('s', 'u', 'r', 'f');
case ObjectTypeInputQueue: return MM_POOL_TAG('i', 'n', 'p', 'q');
case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}
// Creates a zeroed, tagged object with one initial reference.
__declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) {
if (size < sizeof(OBJECT)) size = sizeof(OBJECT);
OBJECT* obj = (OBJECT *)MmAllocatePool(size);
OBJECT* obj = (OBJECT *)MmAllocatePoolTag(size, KiObjectPoolTag(type));
if (!obj) return (OBJECT *)0;
memset(obj, 0, size);
code/public/ntlite/ddi/hal_api.h
+1 -1
View File
@@ -120,7 +120,7 @@ typedef struct HAL_INIT_PARAMS {
void* (*AllocatePool)(size_t size);
} HAL_INIT_PARAMS;
#define HAL_API_VERSION 5
#define HAL_API_VERSION // do not change during development
__declspec(dllexport) const HAL_API* HalGetApi(void);
__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void));
code/public/ntlite/ddi/io_driver.h
+22
View File
@@ -179,6 +179,27 @@ typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex,
DEVICE_OBJECT* device,
void* context);
typedef struct IO_DISK_STATS {
uint64_t RegisteredDisks;
uint64_t ReadRequests;
uint64_t ReadSectors;
uint64_t WriteRequests;
uint64_t WriteSectors;
uint64_t MaxReadSectors;
uint64_t MaxWriteSectors;
uint64_t ReadaheadHits;
uint64_t ReadaheadMisses;
uint64_t ReadaheadStores;
uint64_t ReadaheadInvalidations;
uint64_t ReadaheadBytesServed;
uint64_t ReadaheadSectorsLoaded;
uint64_t ReadaheadSequentialDetections;
uint64_t ReadaheadSuppressed;
uint64_t ReadaheadAllocFailures;
uint64_t ReadaheadReadFailures;
uint64_t MaxReadaheadSectors;
} IO_DISK_STATS;
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex);
@@ -203,3 +224,4 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
const void* buffer);
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out);
code/public/ntlite/ddi/pci.h
+3
View File
@@ -118,3 +118,6 @@ PCI_API KSTATUS PciGetInterrupt(DEVICE_OBJECT* pdo,
PCI_API KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo,
uint8_t irqSlot);
PCI_API KSTATUS PciEnableMsiAuto(DEVICE_OBJECT* pdo,
uint8_t* outIrqSlot);
code/public/ntlite/display.h
+37
View File
@@ -32,6 +32,43 @@ typedef struct DISPLAY_RECT {
int32_t H;
} DISPLAY_RECT;
#define SYS_SURFACE_FORMAT_XRGB8888 1u
#define SYS_SURFACE_FLAG_NONE 0u
#define SYS_SURFACE_MAX_RECTS 64u
typedef struct SYS_SURFACE_DESC {
uint32_t Width;
uint32_t Height;
uint32_t PitchBytes;
uint32_t Format;
void* Base;
uint64_t PayloadBytes;
uint64_t PageCount;
uint64_t MapCount;
uint64_t PresentCount;
uint64_t PresentRectCount;
uint64_t PresentClippedRectCount;
uint64_t PresentEmptyClipCount;
uint64_t PresentRowsCopied;
uint64_t PresentBytesCopied;
} SYS_SURFACE_DESC;
typedef struct SYS_SURFACE_STATS {
uint64_t CreatedSurfaces;
uint64_t DestroyedSurfaces;
uint64_t LiveSurfaces;
uint64_t LivePages;
uint64_t LiveBytes;
uint64_t MapCalls;
uint64_t PresentCalls;
uint64_t PresentRectInputs;
uint64_t PresentClippedRects;
uint64_t PresentEmptyClips;
uint64_t PresentRowsCopied;
uint64_t PresentBytesCopied;
} SYS_SURFACE_STATS;
typedef struct DISPLAY_STATS {
uint64_t BlitCount;
uint64_t FillRectCount;
code/public/ntlite/input.h
+7
View File
@@ -136,6 +136,13 @@ typedef struct INPUT_KEY_EVENT {
bool Pressed;
} INPUT_KEY_EVENT;
typedef struct INPUT_KEY_RECORD {
INPUT_KEY Key;
char Char;
uint8_t Reserved;
uint32_t Modifiers;
} INPUT_KEY_RECORD;
typedef struct INPUT_MOUSE_EVENT {
int dx;
int dy;
code/public/ntlite/status.h
+22
View File
@@ -45,4 +45,26 @@ typedef uint32_t KSTATUS;
// --- Execution level (0x0060) ---.
#define STATUS_INVALID_IRQL ((KSTATUS)0x0060)
// --- Process termination reasons ---.
// Low values are reserved for ordinary KSTATUS returns. Abnormal process
// termination uses a high-bit private range so exit statuses cannot collide
// with normal kernel/API result codes.
//
// Layout:
// 31 : set => abnormal/private status
// 30..24 : subsystem group
// 23..0 : group-local reason
//
// Group 0x61 is process termination/fault status.
#define STATUS_PRIVATE_BASE ((KSTATUS)0x80000000u)
#define STATUS_PROCESS_BASE ((KSTATUS)0xE1000000u)
#define STATUS_PROCESS_KILLED ((KSTATUS)(STATUS_PROCESS_BASE | 0x0000u))
#define STATUS_PROCESS_PAGE_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0010u))
#define STATUS_PROCESS_GENERAL_PROTECTION ((KSTATUS)(STATUS_PROCESS_BASE | 0x0011u))
#define STATUS_PROCESS_ILLEGAL_OPCODE ((KSTATUS)(STATUS_PROCESS_BASE | 0x0012u))
#define STATUS_PROCESS_DIVIDE_BY_ZERO ((KSTATUS)(STATUS_PROCESS_BASE | 0x0013u))
#define STATUS_PROCESS_BREAKPOINT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0014u))
#define STATUS_PROCESS_ALIGNMENT_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0015u))
#define STATUS_PROCESS_UNKNOWN_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x001Fu))
#define KSUCCESS(s) ((s) == STATUS_SUCCESS)
code/public/ntlite/syscall.h
+376 -13
View File
@@ -18,7 +18,11 @@
#define SYS_STATUS_PENDING 4u
#define SYS_STATUS_ALERTED 5u
#define SYS_STATUS_INVALID_PARAMETER 0x10u
#define SYS_STATUS_INVALID_HANDLE 0x11u
#define SYS_STATUS_BUFFER_TOO_SMALL 0x12u
#define SYS_STATUS_NOT_SUPPORTED 0x02u
#define SYS_STATUS_NO_MEMORY 0x20u
#define SYS_STATUS_ACCESS_DENIED 0x21u
#define SYS_STATUS_KERNEL_APC 6u
#define SYS_STATUS_USER_APC 7u
#define SYS_STATUS_CANCELLED 0x33u
@@ -30,6 +34,7 @@
#define SYS_TEB_MAGIC 0x30424554u // "TEB0"
#define SYS_ENV_VERSION 1u
#define SYS_CWD_MAX 520u
#define SYS_PATH_MAX 1024u
#ifndef SYS_USER_ENV_TYPES_DEFINED
#define SYS_USER_ENV_TYPES_DEFINED 1
@@ -52,9 +57,7 @@ typedef struct SYS_TEB {
uint32_t ThreadId;
uint32_t Magic;
// Native per-thread extension pointer. The kernel leaves this NULL;
// user-mode personalities/libraries may hang their own thread-local
// state here without baking subsystem semantics into SYS_TEB.
// =The kernel leaves this NULL, userspace can modify and use.
void* SubsystemData;
} SYS_TEB;
@@ -80,6 +83,27 @@ typedef void (*SYS_USER_APC_ROUTINE)(void* normalContext,
SYS_IO_STATUS_BLOCK* iosb,
uint64_t information);
typedef struct SYS_FILE_IO_REQUEST {
uint32_t FileHandle;
uint32_t Flags;
void* Buffer;
size_t Size;
size_t* TransferredOut;
} SYS_FILE_IO_REQUEST;
static inline SYS_FILE_IO_REQUEST SysMakeFileIoRequest(uint32_t fileHandle,
const void* buffer,
size_t size,
size_t* transferredOut) {
SYS_FILE_IO_REQUEST request;
request.FileHandle = fileHandle;
request.Flags = 0;
request.Buffer = (void*)(uintptr_t)buffer;
request.Size = size;
request.TransferredOut = transferredOut;
return request;
}
typedef struct SYS_ASYNC_FILE_REQUEST {
uint32_t FileHandle;
uint32_t EventHandle; // optional; 0 means no event signal
@@ -181,11 +205,40 @@ typedef struct SYS_NAMED_QUEUE_OWNER_REQUEST {
uint32_t* PidOut;
} SYS_NAMED_QUEUE_OWNER_REQUEST;
#define SYS_WAIT_FLAG_NONE 0x00000000u
#define SYS_WAIT_FLAG_ALERTABLE 0x00000001u
#define SYS_WAIT_TIMEOUT_INFINITE UINT64_MAX
#define SYS_WAIT_MANY_MAX_HANDLES 32u
typedef struct SYS_WAIT_MANY_REQUEST {
uint32_t Size;
uint32_t Flags;
const uint32_t* Handles;
uint32_t Count;
uint32_t* SignaledIndexOut;
uint64_t TimeoutMilliseconds;
} SYS_WAIT_MANY_REQUEST;
typedef struct SYS_CREATE_THREAD_REQUEST {
uintptr_t Entry;
uintptr_t Argument;
uint32_t Priority;
uint32_t Flags;
uint32_t* ThreadHandleOut;
uint32_t* ThreadIdOut;
} SYS_CREATE_THREAD_REQUEST;
// Process & thread lifecycle (0-14).
void SysExitProcess(int exitCode);
void SysExitThread(int exitCode);
uint32_t SysCreateProcess(const SYS_CREATE_PROCESS_REQUEST* request);
uint32_t SysWaitForProcess(uint32_t handle);
uint32_t SysWaitSingle(uint32_t handle, uint32_t flags, uint64_t timeoutMilliseconds);
uint32_t SysWaitMany(const SYS_WAIT_MANY_REQUEST* request);
uint32_t SysOpenInputQueue(uint32_t flags, uint32_t* handleOut);
static inline uint32_t SysWait(uint32_t handle, uint32_t flags) {
return SysWaitSingle(handle, flags, SYS_WAIT_TIMEOUT_INFINITE);
}
uint32_t SysGetProcessExitCode(uint32_t handle, int* exitCodeOut);
uint32_t SysKillProcess(uint32_t pid, int exitCode);
@@ -200,6 +253,61 @@ uint32_t SysKillProcess(uint32_t pid, int exitCode);
#define SYS_PROCESS_FLAG_CURRENT 0x00000002u
#define SYS_PROCESS_FLAG_TERMINATING 0x00000004u
#define SYS_PROCESS_FLAG_EXITED 0x00000008u
#define SYS_PROCESS_FLAG_IDLE 0x00000010u
#define SYS_THREAD_STATE_READY 0u
#define SYS_THREAD_STATE_RUNNING 1u
#define SYS_THREAD_STATE_WAITING 2u
#define SYS_THREAD_STATE_DEAD 3u
#define SYS_THREAD_FLAG_CURRENT 0x00000001u
#define SYS_THREAD_FLAG_IDLE 0x00000002u
#define SYS_THREAD_FLAG_KERNEL 0x00000004u
#define SYS_THREAD_FLAG_QUEUED 0x00000008u
#define SYS_THREAD_FLAG_ALERTABLE_WAIT 0x00000010u
#define SYS_THREAD_FLAG_KERNEL_APC_PENDING 0x00000020u
#define SYS_THREAD_FLAG_USER_APC_PENDING 0x00000040u
#define SYS_THREAD_WAIT_NONE 0u
#define SYS_THREAD_WAIT_PROCESS 1u
#define SYS_THREAD_WAIT_THREAD 2u
#define SYS_THREAD_WAIT_MUTEX 3u
#define SYS_THREAD_WAIT_SEMAPHORE 4u
#define SYS_THREAD_WAIT_EVENT 5u
#define SYS_THREAD_WAIT_QUEUE 6u
#define SYS_THREAD_WAIT_FILE 7u
#define SYS_THREAD_WAIT_DIR 8u
#define SYS_THREAD_WAIT_ASYNC_IO 9u
#define SYS_THREAD_WAIT_SOCKET 10u
#define SYS_THREAD_WAIT_DEVICE 11u
#define SYS_THREAD_WAIT_DRIVER 12u
#define SYS_THREAD_WAIT_USB_DEVICE 13u
#define SYS_THREAD_WAIT_USB_PIPE 14u
#define SYS_THREAD_WAIT_FILE_MAPPING 15u
#define SYS_THREAD_WAIT_SURFACE 16u
#define SYS_THREAD_WAIT_UNKNOWN 0xffffffffu
typedef struct SYS_THREAD_INFO {
uint32_t Size;
uint32_t ProcessId;
uint32_t ThreadId;
uint32_t State;
uint32_t Priority;
uint32_t Flags;
uint32_t RunningCpu;
uint32_t LastCpu;
uint32_t QuantumRemaining;
uint32_t QuantumTicks;
uint32_t CpuBurstTicks;
uint32_t LastRunTicks;
uint32_t WaitObjectType;
uint32_t WaitStatus;
int32_t KernelApcDisable;
uint32_t Reserved0;
} SYS_THREAD_INFO;
typedef struct SYS_PROCESS_INFO {
uint32_t Size;
@@ -219,7 +327,7 @@ typedef struct SYS_PROCESS_INFO {
size_t ImageSize;
uintptr_t PebBase;
size_t UserRegionCount;
size_t VadCount;
size_t UserReservedBytes;
size_t UserCommittedBytes;
@@ -229,9 +337,9 @@ typedef struct SYS_PROCESS_INFO {
uint32_t SysGetProcessList(SYS_PROCESS_INFO* processesOut, size_t maxCount, size_t* countOut);
uint32_t SysGetProcessPid(uint32_t handle, uint32_t* pidOut);
uint32_t SysGetThreadList(uint32_t pid, SYS_THREAD_INFO* threadsOut, size_t maxCount, size_t* countOut);
uint32_t SysCreateThread(uintptr_t entry, uintptr_t arg, uint32_t priority, uint32_t* tidOut);
uint32_t SysWaitThread(uint32_t tid);
uint32_t SysCreateThread(const SYS_CREATE_THREAD_REQUEST* request);
uint32_t SysGetCurrentTid(void);
uint32_t SysGetCurrentPid(void);
uint32_t SysSetThreadPriority(uint32_t priority);
@@ -266,23 +374,32 @@ uint32_t SysAllocVm(void* preferredBase, size_t pageCount, uint32_t protect, uin
uint32_t SysFreeVm(void* base);
uint32_t SysProtectVm(void* base, size_t pageCount, uint32_t protect, uint32_t* oldProtectOut);
uint32_t SysQueryVm(void* address, SYS_VM_REGION_INFO* out);
#define SYS_MAP_ACCESS_READ 0x00000001u
#define SYS_MAP_ACCESS_WRITE 0x00000002u
#define SYS_MAP_ACCESS_COPY 0x00000004u
#define SYS_MAP_ACCESS_EXECUTE 0x00000008u
#define SYS_MAP_FLAG_NONE 0x00000000u
uint32_t SysCreateFileMapping(uint32_t fileHandle, uint32_t protect, uint32_t flags,
uint64_t maximumSize, uint32_t* mappingHandleOut);
uint32_t SysMapFile(uint32_t mappingHandle, uint64_t offset, size_t size,
uint32_t access, uint32_t flags, void** baseOut);
uint32_t SysCloseHandle(uint32_t handle);
// Synchronization primitives (19-31).
uint32_t SysCreateMutex(uint32_t* handleOut);
uint32_t SysAcquireMutex(uint32_t handle);
uint32_t SysTryAcquireMutex(uint32_t handle);
uint32_t SysReleaseMutex(uint32_t handle);
uint32_t SysCreateSem(int32_t initial, int32_t max, uint32_t* handleOut);
uint32_t SysWaitSem(uint32_t handle);
uint32_t SysTryWaitSem(uint32_t handle);
uint32_t SysSignalSem(uint32_t handle);
uint32_t SysCreateEvent(uint32_t type, uint32_t initialState, uint32_t* handleOut);
uint32_t SysSignalEvent(uint32_t handle);
uint32_t SysClearEvent(uint32_t handle);
uint32_t SysWaitEvent(uint32_t handle, uint32_t alertable);
// Queues and named queues (32-38).
uint32_t SysCreateQueue(uint32_t* handleOut);
@@ -296,8 +413,8 @@ uint32_t SysGetNamedQueueOwnerPid(const SYS_NAMED_QUEUE_OWNER_REQUEST* request);
// Files, directories, and current directory (39-56).
uint32_t SysOpenFile(const SYS_PATH_HANDLE_REQUEST* request);
uint32_t SysCreateFile(const SYS_PATH_HANDLE_REQUEST* request);
uint32_t SysReadFile(uint32_t handle, void* buf, size_t size, size_t* bytesRead);
uint32_t SysWriteFile(uint32_t handle, const void* buf, size_t size, size_t* written);
uint32_t SysReadFile(const SYS_FILE_IO_REQUEST* request);
uint32_t SysWriteFile(const SYS_FILE_IO_REQUEST* request);
uint32_t SysReadFileAsync(const SYS_ASYNC_FILE_REQUEST* request);
uint32_t SysWriteFileAsync(const SYS_ASYNC_FILE_REQUEST* request);
uint32_t SysCancelIo(uint32_t asyncRequestHandle);
@@ -305,6 +422,7 @@ uint32_t SysSeekFile(uint32_t handle, uint64_t offset);
uint64_t SysGetFilePos(uint32_t handle);
uint64_t SysGetFileSize(uint32_t handle);
uint32_t SysTruncateFile(uint32_t handle, uint64_t newSize);
uint32_t SysFlushFile(uint32_t handle);
uint32_t SysDeleteFile(const SYS_PATH_REQUEST* request);
uint32_t SysRenameFile(const SYS_RENAME_FILE_REQUEST* request);
@@ -330,7 +448,10 @@ uint32_t SysSetConsoleCursor(int x, int y);
uint32_t SysFillConsoleCharacter(char c, int x, int y, size_t length, size_t* writtenOut);
uint32_t SysFillConsoleAttribute(uint16_t attr, int x, int y, size_t length, size_t* writtenOut);
uint32_t SysPutScreenChar(char c, int x, int y, uint32_t fg, uint32_t bg);
uint32_t SysBlitScreen(int x, int y, int w, int h, const uint32_t* pixels);
uint32_t SysCreateSurface(uint32_t width, uint32_t height, uint32_t format, uint32_t flags, uint32_t* handleOut);
uint32_t SysMapSurface(uint32_t surfaceHandle, void** baseOut, uint32_t* pitchBytesOut);
uint32_t SysQuerySurface(uint32_t surfaceHandle, SYS_SURFACE_DESC* descOut);
uint32_t SysPresentSurface(uint32_t surfaceHandle, const DISPLAY_RECT* rects, uint32_t rectCount);
uint32_t SysClearScreen(uint32_t color);
uint32_t SysGetScreenSize(int* widthOut, int* heightOut);
uint32_t SysQueryDisplayInfo(SYS_DISPLAY_INFO* infoOut);
@@ -341,6 +462,8 @@ uint64_t SysIsKeyDown(INPUT_KEY key);
char SysReadKeyboardChar(void);
INPUT_KEY SysReadKey(void);
uint32_t SysGetKeyModifiers(void);
uint32_t SysReadInputEvent(INPUT_KEY_RECORD* recordOut);
uint32_t SysPeekInputEvent(INPUT_KEY_RECORD* recordOut);
uint64_t SysPollMouse(void);
uint64_t SysReadMouse(int* dx, int* dy, uint8_t* buttons);
@@ -437,11 +560,25 @@ typedef struct SYS_MEMORY_STATUS {
uint64_t FreePhysicalBytes;
uint64_t KernelPoolBytes;
uint64_t UserHeapBytes;
uint64_t ReclaimablePhysicalBytes;
uint64_t AvailablePhysicalBytes;
uint64_t PinnedPhysicalBytes;
uint64_t NonEvictablePhysicalBytes;
uint64_t UserPhysicalBytes;
uint64_t PageTablePhysicalBytes;
} SYS_MEMORY_STATUS;
#define SYS_KERNEL_STATS_SCHEDULER 1u
#define SYS_KERNEL_STATS_PMM 2u
#define SYS_KERNEL_STATS_CPU 3u
#define SYS_KERNEL_STATS_PFN 4u
#define SYS_KERNEL_STATS_PTE 5u
#define SYS_KERNEL_STATS_VAD 6u
#define SYS_KERNEL_STATS_SWAP 7u
#define SYS_KERNEL_STATS_FILE_CACHE 8u
#define SYS_KERNEL_STATS_SURFACE 9u
#define SYS_KERNEL_STATS_FILESYSTEM 10u
#define SYS_KERNEL_STATS_DISK 11u
#define SYS_KERNEL_STATS_MAX_CPUS 32u
@@ -469,11 +606,15 @@ typedef struct SYS_PMM_STATS {
uint64_t FreeRunHistogram[8];
} SYS_PMM_STATS;
#define SYS_CPU_THREAD_FLAG_IDLE 0x00000001u
typedef struct SYS_CPU_STAT {
uint32_t CpuIndex;
uint32_t LapicId;
uint32_t Flags;
uint32_t CurrentTid;
uint32_t CurrentThreadFlags;
uint32_t Reserved0;
uint64_t SchedulerTicks;
uint64_t ContextSwitches;
} SYS_CPU_STAT;
@@ -484,6 +625,228 @@ typedef struct SYS_CPU_STATS {
SYS_CPU_STAT Cpu[SYS_KERNEL_STATS_MAX_CPUS];
} SYS_CPU_STATS;
typedef struct SYS_PFN_STATS {
uint64_t TotalFrames;
uint64_t FreeFrames;
uint64_t UsedFrames;
uint64_t ReservedFrames;
uint64_t ZeroedFrames;
uint64_t ActiveFrames;
uint64_t StandbyFrames;
uint64_t ModifiedFrames;
uint64_t PinnedFrames;
uint64_t EvictableFrames;
uint64_t NonEvictableFrames;
uint64_t ReclaimableFrames;
uint64_t AvailableFrames;
uint64_t MappedFrames;
uint64_t KernelFrames;
uint64_t PageTableFrames;
uint64_t UserFrames;
uint64_t CleanFileCacheFrames;
uint64_t ActiveCleanFileCacheFrames;
uint64_t StandbyCleanFileCacheFrames;
uint64_t ImageFrames;
uint64_t ActiveImageFrames;
uint64_t StandbyImageFrames;
uint64_t MappedFileFrames;
uint64_t AnonymousFrames;
uint64_t CowFrames;
uint64_t SurfaceFrames;
uint64_t DriverFrames;
uint64_t DmaFrames;
uint64_t BootloaderFrames;
uint64_t PfnDatabaseFrames;
uint64_t MmioFrames;
uint64_t AllocationCount;
uint64_t FreeCount;
uint64_t RetainCount;
uint64_t ReleaseCount;
uint64_t PinCount;
uint64_t UnpinCount;
uint64_t FailedAllocations;
uint64_t FailedContiguousAllocations;
uint64_t FailedInvalidRange;
uint64_t FailedEmptyRange;
uint64_t FailedReserveDenied;
uint64_t FailedNoFrame;
uint64_t FailedContiguousInvalidRequest;
uint64_t FailedContiguousReserveDenied;
uint64_t FailedContiguousNoRun;
uint64_t ReclaimAttempts;
uint64_t ReclaimPagesRequested;
uint64_t ReclaimPagesSucceeded;
uint64_t ReclaimPagesFailed;
uint64_t ReclaimReentrantSkips;
} SYS_PFN_STATS;
typedef struct SYS_VAD_STATS {
uint64_t ResolveFaultCalls;
uint64_t ResolvedFaults;
uint64_t FailedFaults;
uint64_t ProtectionFaults;
uint64_t NoVadFaults;
uint64_t UncommittedFaults;
uint64_t NoAccessFaults;
uint64_t DemandZeroPages;
uint64_t ZeroPageMaps;
uint64_t ZeroPageCowSplits;
uint64_t CowFaults;
uint64_t CowPagesCopied;
uint64_t CowViewsCreated;
uint64_t CowPagesShared;
uint64_t EnsureResidentCalls;
uint64_t EnsureResidentPages;
uint64_t AlreadyResidentPages;
uint64_t FileBackedFaults;
uint64_t FileBackedReads;
uint64_t FileBackedReadFailures;
uint64_t FileBackedDiscards;
uint64_t FileBackedSharedWritableFaults;
uint64_t FileBackedDirtyFaults;
uint64_t FileBackedDirtySyncs;
uint64_t ImageBackedFaults;
uint64_t ImageBackedZeroFaults;
uint64_t ImageBackedPrivateFaults;
uint64_t ImageFixupPrivateCalls;
uint64_t ImageFixupPrivatePages;
uint64_t SwapPageOuts;
uint64_t SwapPageIns;
} SYS_VAD_STATS;
typedef struct SYS_FILE_CACHE_STATS {
uint64_t Lookups;
uint64_t Hits;
uint64_t Misses;
uint64_t Loads;
uint64_t LoadFailures;
uint64_t RaceDiscards;
uint64_t InflightWaits;
uint64_t Invalidations;
uint64_t DirtyPages;
uint64_t MaxDirtyPages;
uint64_t DirtyMarks;
uint64_t WritebackRequests;
uint64_t WritebackPages;
uint64_t WritebackBytes;
uint64_t WritebackFailures;
uint64_t FlushFileCalls;
uint64_t FlushFileFailures;
uint64_t MappedDirtySyncs;
uint64_t MappedDirtyPages;
uint64_t Drops;
uint64_t DropFailures;
uint64_t StandbyTransitions;
uint64_t StandbyHits;
uint64_t ReadAheadRequests;
uint64_t ReadAheadPagesRequested;
uint64_t ReadAheadPagesQueued;
uint64_t ReadAheadPagesLoaded;
uint64_t ReadAheadPagesSkipped;
uint64_t ReadAheadFailures;
uint64_t ReadAheadPagesThrottled;
uint64_t MaxReadAheadPagesPerRequest;
uint64_t BatchReads;
uint64_t BatchPagesRequested;
uint64_t BatchPagesLoaded;
uint64_t MaxBatchPagesRequested;
uint64_t MaxBatchPagesLoaded;
uint64_t ResidentPages;
uint64_t ActivePages;
uint64_t StandbyPages;
uint64_t MaxResidentPages;
} SYS_FILE_CACHE_STATS;
typedef struct SYS_FILESYSTEM_STATS {
uint64_t PathLookups;
uint64_t PathCacheHits;
uint64_t PathCacheMisses;
uint64_t PathCacheStores;
uint64_t PathCacheInvalidations;
uint64_t PathCacheGeneration;
uint64_t DirectorySearches;
uint64_t DirectoryClustersRead;
uint64_t DirectoryEntriesScanned;
uint64_t FatChainCacheLookups;
uint64_t FatChainCacheHits;
uint64_t FatChainCacheMisses;
uint64_t FatChainCacheStores;
uint64_t FatChainCacheInvalidations;
uint64_t FatChainWalkSteps;
uint64_t DataReadRuns;
uint64_t DataReadSectors;
uint64_t DataReadRunMerges;
uint64_t MaxDataReadSpanSectors;
} SYS_FILESYSTEM_STATS;
typedef struct SYS_DISK_STATS {
uint64_t RegisteredDisks;
uint64_t ReadRequests;
uint64_t ReadSectors;
uint64_t WriteRequests;
uint64_t WriteSectors;
uint64_t MaxReadSectors;
uint64_t MaxWriteSectors;
uint64_t ReadaheadHits;
uint64_t ReadaheadMisses;
uint64_t ReadaheadStores;
uint64_t ReadaheadInvalidations;
uint64_t ReadaheadBytesServed;
uint64_t ReadaheadSectorsLoaded;
uint64_t ReadaheadSequentialDetections;
uint64_t ReadaheadSuppressed;
uint64_t ReadaheadAllocFailures;
uint64_t ReadaheadReadFailures;
uint64_t MaxReadaheadSectors;
} SYS_DISK_STATS;
typedef struct SYS_SWAP_STATS {
uint64_t Enabled;
uint64_t TotalSlots;
uint64_t UsedSlots;
uint64_t FreeSlots;
uint64_t HighWaterSlots;
uint64_t PageOuts;
uint64_t PageIns;
uint64_t PageOutFailures;
uint64_t PageInFailures;
uint64_t SlotAllocations;
uint64_t SlotFrees;
uint64_t SlotAllocationFailures;
} SYS_SWAP_STATS;
typedef struct SYS_PTE_STATS {
uint64_t MapCalls;
uint64_t MapFailures;
uint64_t UnmapCalls;
uint64_t UnmapFailures;
uint64_t ProtectCalls;
uint64_t ProtectFailures;
uint64_t QueryCalls;
uint64_t QueryFailures;
uint64_t QueryByPurpose[6];
uint64_t QueryFailuresByPurpose[6];
uint64_t TranslateCalls;
uint64_t LocalFlushes;
uint64_t GlobalFlushes;
uint64_t ShootdownRequests;
uint64_t ShootdownTargetCpus;
uint64_t ShootdownSkippedNoTargets;
uint64_t ShootdownAcks;
uint64_t ShootdownTimeouts;
uint64_t NewMappingNoFlushes;
uint64_t ReplacedMappingFlushes;
uint64_t PageTablePagesAllocated;
uint64_t CurrentLeafMappings;
} SYS_PTE_STATS;
#define SYS_CPU_FLAG_PRESENT 0x00000001u
#define SYS_CPU_FLAG_STARTED 0x00000002u
#define SYS_CPU_FLAG_PARKED 0x00000004u
code/settings.ini
+4
View File
@@ -24,3 +24,7 @@ daylight_saving=auto
hardware_clock=utc
; Used when timezone is a custom/fixed policy; minutes east of UTC.
standard_offset_minutes=60
[vm]
; 0 disables swap. Nonzero swap must be between 4 MB and 8192 MB.
swap_size_mb=32
code/user/apps/doom/doom.c
+47 -11
View File
@@ -88,7 +88,8 @@ static int doom_read_wrapper(void* handle, void* buf, int count)
if (!handle || count <= 0) return 0;
DOOM_FILE_HANDLE* f = (DOOM_FILE_HANDLE*)handle;
size_t read = 0;
SysReadFile(f->Handle, buf, (size_t)count, &read);
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(f->Handle, buf, (size_t)count, &read);
SysReadFile(&req);
f->Pos += read;
if (f->Pos > f->Size) f->Size = f->Pos;
return (int)read;
@@ -99,7 +100,8 @@ static int doom_write_wrapper(void* handle, const void* buf, int count)
if (!handle || count <= 0) return 0;
DOOM_FILE_HANDLE* f = (DOOM_FILE_HANDLE*)handle;
size_t written = 0;
SysWriteFile(f->Handle, buf, (size_t)count, &written);
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(f->Handle, buf, (size_t)count, &written);
SysWriteFile(&req);
f->Pos += written;
if (f->Pos > f->Size) f->Size = f->Pos;
return (int)written;
@@ -262,7 +264,7 @@ static void poll_input(void)
}
// ---------------------------------------------------------------------------
// Render — Performs lightweight software upscale and blits to the screen
// Render — writes directly into a mapped kernel surface and presents it.
// ---------------------------------------------------------------------------
#define DOOM_W 320
#define DOOM_H 200
@@ -270,32 +272,65 @@ static void poll_input(void)
#define DISP_W (DOOM_W * SCALE)
#define DISP_H (DOOM_H * SCALE)
static uint32_t g_blit_buf[DISP_W * DISP_H];
static uint32_t g_surface_handle;
static uint32_t* g_surface_pixels;
static uint32_t g_surface_pitch_pixels;
static void init_surface(void)
{
uint32_t s = SysCreateSurface(DISP_W, DISP_H, SYS_SURFACE_FORMAT_XRGB8888,
SYS_SURFACE_FLAG_NONE, &g_surface_handle);
if (s != SYS_STATUS_SUCCESS || !g_surface_handle) {
doom_print_wrapper("doom: SysCreateSurface failed\n");
SysExitProcess(1);
}
void* base = 0;
uint32_t pitchBytes = 0;
s = SysMapSurface(g_surface_handle, &base, &pitchBytes);
if (s != SYS_STATUS_SUCCESS || !base || pitchBytes < DISP_W * sizeof(uint32_t)) {
doom_print_wrapper("doom: SysMapSurface failed\n");
SysExitProcess(1);
}
g_surface_pixels = (uint32_t*)base;
g_surface_pitch_pixels = pitchBytes / sizeof(uint32_t);
}
static void render_frame(const uint8_t* rgba)
{
const uint32_t* src = (const uint32_t*)rgba;
// Software upscale loop mapping 320x200 buffer directly to scaled window buffer
uint32_t* dstBase = g_surface_pixels;
uint32_t dstPitch = g_surface_pitch_pixels;
if (!dstBase || dstPitch < DISP_W) {
doom_print_wrapper("doom: surface not initialized\n");
SysExitProcess(1);
}
for (int y = 0; y < DOOM_H; y++) {
for (int x = 0; x < DOOM_W; x++) {
uint32_t p = src[y * DOOM_W + x];
// RGBA -> BGRA conversion (standard for linear x86/64 framebuffers)
// RGBA -> BGRA/XRGB8888 conversion for the current display path.
uint32_t bgra = (p & 0xFF00FF00u)
| ((p & 0x00FF0000u) >> 16)
| ((p & 0x000000FFu) << 16);
// Populate SCALE * SCALE pixel block
uint32_t* block = dstBase + (size_t)(y * SCALE) * dstPitch + (size_t)(x * SCALE);
for (int sy = 0; sy < SCALE; sy++) {
for (int sx = 0; sx < SCALE; sx++) {
g_blit_buf[(y * SCALE + sy) * DISP_W + (x * SCALE + sx)] = bgra;
block[(size_t)sy * dstPitch + sx] = bgra;
}
}
}
}
SysBlitScreen(0, 0, DISP_W, DISP_H, g_blit_buf);
DISPLAY_RECT rect = { 0, 0, DISP_W, DISP_H };
uint32_t code = SysPresentSurface(g_surface_handle, &rect, 1);
if (code != SYS_STATUS_SUCCESS) {
SysExitProcess(code);
}
}
static int build_argv_from_command_line(const char* cmdline, char* storage, size_t storageSize,
@@ -373,6 +408,7 @@ int main(void)
char argvStorage[512];
const char* argv[32];
int argc = build_argv_from_command_line(cmdline, argvStorage, sizeof(argvStorage), argv, 32);
init_surface();
doom_init(argc, argv, 0);
do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: entering game loop\n") }; SysWriteConsole(&req); } while (0);
code/user/apps/edit/edit.c
+7 -3
View File
@@ -380,12 +380,14 @@ static void save_file(void) {
for (int i = 0; i < gLineCount; i++) {
size_t written = 0;
if (gLines[i].len) {
st = SysWriteFile(h, gLines[i].text, (size_t)gLines[i].len, &written);
SYS_FILE_IO_REQUEST writeReq = SysMakeFileIoRequest(h, gLines[i].text, (size_t)gLines[i].len, &written);
st = SysWriteFile(&writeReq);
if (st != 0) { SysCloseHandle(h); msg_set("save failed: write error"); return; }
}
if (i + 1 < gLineCount) {
const char nl = '\n';
st = SysWriteFile(h, &nl, 1, &written);
SYS_FILE_IO_REQUEST newlineReq = SysMakeFileIoRequest(h, &nl, 1, &written);
st = SysWriteFile(&newlineReq);
if (st != 0) { SysCloseHandle(h); msg_set("save failed: write error"); return; }
}
}
@@ -418,7 +420,9 @@ static void load_file(void) {
char buf[512];
size_t got = 0;
bool truncated = false;
while (SysReadFile(h, buf, sizeof(buf), &got) == 0 && got > 0) {
for (;;) {
SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(h, buf, sizeof(buf), &got);
if (SysReadFile(&readReq) != 0 || got == 0) break;
for (size_t i = 0; i < got; i++) {
char c = buf[i];
if (c == '\r') continue;
code/user/apps/memtop/memtop.c
+71
View File
@@ -0,0 +1,71 @@
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <ntlite/syscall.h>
static int u64_to_string(uint64_t n, char* buf) {
int i = 0;
if (n == 0) {
buf[i++] = '0';
buf[i] = '\0';
return i;
}
char rev[24];
int r_idx = 0;
while (n > 0) {
rev[r_idx++] = (char)('0' + (n % 10));
n /= 10;
}
while (r_idx > 0) {
buf[i++] = rev[--r_idx];
}
buf[i] = '\0';
return i;
}
int main(void) {
SYS_CONSOLE_INFO console;
SYS_MEMORY_STATUS mem;
char metricsBuf[128];
while (true) {
if (SysGetConsoleInfo(&console) == SYS_STATUS_SUCCESS &&
SysGetMemoryStatus(&mem) == SYS_STATUS_SUCCESS) {
// If the system isn't reporting columns correctly, default to a safe standard width
int columns = (console.Columns > 0) ? (int)console.Columns : 80;
char freeStr[24], poolStr[24], heapStr[24];
u64_to_string(mem.FreePhysicalBytes / 1024u, freeStr);
u64_to_string(mem.KernelPoolBytes / 1024u, poolStr);
u64_to_string(mem.UserHeapBytes / 1024u, heapStr);
metricsBuf[0] = '\0';
strcat(metricsBuf, "[ Free: ");
strcat(metricsBuf, freeStr);
strcat(metricsBuf, "K | Pool: ");
strcat(metricsBuf, poolStr);
strcat(metricsBuf, "K | Heap: ");
strcat(metricsBuf, heapStr);
strcat(metricsBuf, "K ]");
size_t textLen = strlen(metricsBuf);
int targetX = columns - (int)textLen;
int targetY = 0;
if (targetX >= 0) {
size_t written = 0;
// Loop through the string and use the console's dedicated text filling API
// This utilizes the console's native font configuration instead of drawing raw screen bits
for (size_t i = 0; i < textLen; i++) {
SysFillConsoleCharacter(metricsBuf[i], targetX + (int)i, targetY, 1, &written);
}
}
}
SysSleep(500);
}
return 0;
}
code/user/apps/shell_native/shell_native.c
+1029 -37
View File
File diff suppressed because it is too large. Load diff
code/user/apps/shell_win32/shell_win32.c
+2
View File
@@ -147,6 +147,8 @@ static int runCommand(const char* cmd) {
WaitForSingleObject(pi.hProcess, INFINITE);
if (pi.hThread && pi.hThread != INVALID_HANDLE_VALUE)
CloseHandle(pi.hThread);
if (pi.hProcess && pi.hProcess != INVALID_HANDLE_VALUE)
CloseHandle(pi.hProcess);
FlushConsoleInputBuffer(hIn);
} else {
print("shell: launch failed\n");
code/user/apps/video/video.c
+209 -95
View File
@@ -1,110 +1,231 @@
// User application entry points for video.
// Native NTLite Bink video player.
//
// This application intentionally uses the native syscall ABI directly. Do not
// include <windows.h> here; Win32 compatibility belongs in the Win32 personality
// layer, not in native apps.
#include <windows.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/syscall.h>
#include "bink.h"
// Implements the command Line Second Token operation.
static const char* commandLineSecondToken(const char* cmdline, char* out, size_t outSize) {
if (!out || outSize == 0) return "video.bk2";
strcpy(out, "video.bk2");
if (!cmdline) return out;
#define VIDEO_DEFAULT_PATH "video.bk2"
while (*cmdline == ' ' || *cmdline == '\t') cmdline++;
if (*cmdline == '\0') return out;
static void print_span(const char* s, size_t len) {
if (!s || !len) return;
SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromParts(s, len) };
SysWriteConsole(&req);
}
if (*cmdline == '"') {
cmdline++;
while (*cmdline && *cmdline != '"') cmdline++;
if (*cmdline == '"') cmdline++;
static void print(const char* s) {
if (!s) return;
SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) };
SysWriteConsole(&req);
}
static void print_u64(uint64_t v) {
char buf[32];
int i = 0;
if (v == 0) {
print("0");
return;
}
while (v && i < (int)sizeof(buf)) {
buf[i++] = (char)('0' + (v % 10));
v /= 10;
}
while (i) {
char c = buf[--i];
print_span(&c, 1);
}
}
static void fail_status(const char* what, uint32_t status) {
print("video: ");
print(what);
print(" failed status=");
print_u64(status);
print("\n");
for (;;) SysExitProcess(1);
}
static void exit_now(int code) {
for (;;) SysExitProcess(code);
}
static SYS_STRING_VIEW command_line_view(void) {
SYS_PEB* peb = SysGetPeb();
if (!peb || peb->Magic != SYS_PEB_MAGIC)
return SYS_SV_NULL;
return peb->CommandLine;
}
static int sv_is_space(char c) {
return c == ' ' || c == '\t';
}
static SYS_STRING_VIEW skip_one_token(SYS_STRING_VIEW s) {
size_t i = 0;
while (i < s.Length && sv_is_space(s.Data[i]))
i++;
if (i < s.Length && s.Data[i] == '"') {
i++;
while (i < s.Length && s.Data[i] != '"')
i++;
if (i < s.Length && s.Data[i] == '"')
i++;
} else {
while (*cmdline && *cmdline != ' ' && *cmdline != '\t') cmdline++;
while (i < s.Length && !sv_is_space(s.Data[i]))
i++;
}
while (*cmdline == ' ' || *cmdline == '\t') cmdline++;
if (*cmdline == '\0') return out;
while (i < s.Length && sv_is_space(s.Data[i]))
i++;
size_t pos = 0;
if (*cmdline == '"') {
cmdline++;
while (*cmdline && *cmdline != '"' && pos + 1 < outSize)
out[pos++] = *cmdline++;
return SysSvFromParts(s.Data + i, s.Length - i);
}
static SYS_STRING_VIEW command_line_second_token(SYS_STRING_VIEW cmdline) {
SYS_STRING_VIEW rest = skip_one_token(cmdline);
if (!rest.Data || rest.Length == 0)
return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
size_t begin = 0;
size_t end = 0;
if (rest.Data[0] == '"') {
begin = 1;
end = begin;
while (end < rest.Length && rest.Data[end] != '"')
end++;
} else {
while (*cmdline && *cmdline != ' ' && *cmdline != '\t' && pos + 1 < outSize)
out[pos++] = *cmdline++;
begin = 0;
end = 0;
while (end < rest.Length && !sv_is_space(rest.Data[end]))
end++;
}
out[pos] = '\0';
if (end <= begin)
return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
return SysSvFromParts(rest.Data + begin, end - begin);
}
static const char* path_to_cstr(SYS_STRING_VIEW path, char* out, size_t outSize) {
if (!out || outSize == 0)
return VIDEO_DEFAULT_PATH;
if (!path.Data || path.Length == 0)
path = SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
size_t n = path.Length;
if (n + 1 > outSize)
n = outSize - 1;
memcpy(out, path.Data, n);
out[n] = '\0';
return out;
}
// Runs the requested operation.
static uint32_t open_file_view(SYS_STRING_VIEW path, uint32_t* handleOut) {
SYS_PATH_HANDLE_REQUEST req = { path, handleOut };
return SysOpenFile(&req);
}
int main(void) {
const char* cmdline = GetCommandLineA();
char videoPath[260];
const char* path = commandLineSecondToken(cmdline, videoPath, sizeof(videoPath));
SYS_STRING_VIEW cmdline = command_line_view();
SYS_STRING_VIEW pathView = command_line_second_token(cmdline);
void* h = GetStdHandle(STD_OUTPUT_HANDLE);
WriteConsoleA(h, "video: starting\n", 16, 0, 0);
WriteConsoleA(h, cmdline, (unsigned int)strlen(cmdline), 0, 0);
WriteConsoleA(h, "\n", 1, 0, 0);
void* f = CreateFileA(path, 0, 0, 0, 3, 0, 0);
if (!f || f == (void*)(unsigned long long)0xFFFFFFFF) { // fuckass
WriteConsoleA(h, "video: failed to open input file\n", 33, 0, 0);
ExitProcess(1);
print("video: starting\n");
if (cmdline.Data && cmdline.Length) {
print_span(cmdline.Data, cmdline.Length);
print("\n");
}
unsigned long s = GetFileSize(f, 0);
if (!s || s == 0xFFFFFFFF) {
WriteConsoleA(h, "video: GetFileSize failed!\n", 27, 0, 0);
ExitProcess(1);
}
void* buf = HeapAlloc(0, 0, s);
if (!buf) {
WriteConsoleA(h, "video: HeapAlloc failed\n", 24, 0, 0);
ExitProcess(1);
uint32_t file = 0;
uint32_t status = open_file_view(pathView, &file);
if (status != SYS_STATUS_SUCCESS) {
char pathBuf[260];
print("video: failed to open input file: ");
print(path_to_cstr(pathView, pathBuf, sizeof(pathBuf)));
print("\n");
fail_status("SysOpenFile", status);
}
int r = ReadFile(f, buf, s, 0, 0);
if (!r) {
WriteConsoleA(h, "video: ReadFile failed\n", 23, 0, 0);
ExitProcess(1);
uint64_t fileSize = SysGetFileSize(file);
if (fileSize == 0) {
SysCloseHandle(file);
print("video: input file is empty\n");
exit_now(1);
}
WriteConsoleA(h, "video: input file loaded\n", 25, 0, 0);
CloseHandle(h);
HBINK b = BinkOpen(buf, BINKFROMMEMORY);
uint32_t mapping = 0;
status = SysCreateFileMapping(file, SYS_VM_READ, SYS_MAP_FLAG_NONE, fileSize, &mapping);
if (status != SYS_STATUS_SUCCESS || !mapping) {
SysCloseHandle(file);
fail_status("SysCreateFileMapping", status);
}
void* buf = 0;
status = SysMapFile(mapping, 0, (size_t)fileSize,
SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &buf);
if (status != SYS_STATUS_SUCCESS || !buf) {
SysCloseHandle(mapping);
SysCloseHandle(file);
fail_status("SysMapFile", status);
}
// The mapped VAD owns the file reference now; keep only the mapped pointer
// alive while Bink is open. This avoids a second whole-file heap allocation
// and lets the VM fault file pages in on demand.
SysCloseHandle(mapping);
SysCloseHandle(file);
print("video: input file mapped\n");
HBINK b = BinkOpen((const char*)buf, BINKFROMMEMORY);
if (!b) {
WriteConsoleA(h, "video: BinkOpen failed: ", 24, 0, 0);
print("video: BinkOpen failed: ");
const char* err = BinkGetError();
WriteConsoleA(h, err, (unsigned int)strlen(err), 0, 0);
WriteConsoleA(h, "\n", 1, 0, 0);
ExitProcess(1);
print(err ? err : "unknown");
print("\n");
SysFreeVm(buf);
exit_now(1);
}
uint32_t width = b->Width;
uint32_t width = b->Width;
uint32_t height = b->Height;
uint32_t* fb = HeapAlloc(0, 0, width * height * sizeof(uint32_t));
if (!fb) {
WriteConsoleA(h, "video: framebuffer alloc failed\n", 32, 0, 0);
ExitProcess(1);
if (!width || !height) {
print("video: invalid video dimensions\n");
BinkClose(b);
SysFreeVm(buf);
exit_now(1);
}
// Prepare the Win32 BITMAPINFO structure.
BITMAPINFO bmi;
memset(&bmi, 0, sizeof(bmi));
bmi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
bmi.bmiHeader.biWidth = width;
bmi.bmiHeader.biHeight = -(LONG)height; // Negative means top-down drawing!
bmi.bmiHeader.biPlanes = 1;
bmi.bmiHeader.biBitCount = 32;
bmi.bmiHeader.biCompression = BI_RGB;
uint32_t surface = 0;
status = SysCreateSurface(width, height, SYS_SURFACE_FORMAT_XRGB8888,
SYS_SURFACE_FLAG_NONE, &surface);
if (status != SYS_STATUS_SUCCESS || !surface) {
BinkClose(b);
SysFreeVm(buf);
fail_status("SysCreateSurface", status);
}
void* surfaceBase = 0;
uint32_t pitchBytes = 0;
status = SysMapSurface(surface, &surfaceBase, &pitchBytes);
if (status != SYS_STATUS_SUCCESS || !surfaceBase || pitchBytes < width * sizeof(uint32_t)) {
SysCloseHandle(surface);
BinkClose(b);
SysFreeVm(buf);
fail_status("SysMapSurface", status);
}
print("video: native surface ready\n");
DISPLAY_RECT fullFrame = { 0, 0, (int32_t)width, (int32_t)height };
while (1) {
if (BinkWait(b)) {
@@ -120,33 +241,26 @@ int main(void) {
continue;
}
BinkCopyToBuffer(b, fb, width * 4, height, 0, 0,
BINKSURFACE32);
BinkCopyToBuffer(b, surfaceBase, pitchBytes, height, 0, 0, BINKSURFACE32);
// Welcome to the GDI pipeline!
PAINTSTRUCT ps;
// Using NULL/0 as the HWND since we don't have a real window handle yet.
HDC hdc = BeginPaint((HWND)0, &ps);
StretchDIBits(
hdc,
0, 0, width, height, // Destination rect
0, 0, width, height, // Source rect
fb,
&bmi,
DIB_RGB_COLORS,
SRCCOPY
);
EndPaint((HWND)0, &ps);
status = SysPresentSurface(surface, &fullFrame, 1);
if (status != SYS_STATUS_SUCCESS) {
SysCloseHandle(surface);
BinkClose(b);
SysFreeVm(buf);
fail_status("SysPresentSurface", status);
}
if (b->FrameNum == b->Frames)
break;
BinkNextFrame(b);
}
WriteConsoleA(h, "video: goodbye\n", 15, 0, 0);
ExitProcess(0);
SysCloseHandle(surface);
BinkClose(b);
SysFreeVm(buf);
print("video: goodbye\n");
exit_now(0);
}
code/user/personalities/posix/posix.h
+16 -1
View File
@@ -72,12 +72,27 @@ extern int errno; // defined in posix_fd.c; single global in v1
#define SEEK_CUR 1
#define SEEK_END 2
// mmap() protection and mapping flags.
#define PROT_NONE 0x0
#define PROT_READ 0x1
#define PROT_WRITE 0x2
#define PROT_EXEC 0x4
#define MAP_SHARED 0x01
#define MAP_PRIVATE 0x02
#define MAP_FAILED ((void*)(intptr_t)-1)
#define MS_SYNC 0x04
#define MS_ASYNC 0x01
#define MS_INVALIDATE 0x02
// Scalar types.
typedef int32_t pid_t;
typedef int32_t off_t;
typedef uint32_t mode_t;
typedef int64_t ssize_t;
__declspec(dllexport) void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset);
__declspec(dllexport) int munmap(void* addr, size_t length);
__declspec(dllexport) int msync(void* addr, size_t length, int flags);
// Sockets.
typedef uint16_t sa_family_t;
@@ -188,7 +203,7 @@ __declspec(dllexport) int getaddrinfo(const char* node, const char* service, con
__declspec(dllexport) void freeaddrinfo(struct addrinfo* res);
// pthread types.
typedef struct { uint32_t Tid; } pthread_t;
typedef struct { uint32_t Handle; uint32_t Tid; } pthread_t;
typedef struct { uint32_t Handle; } pthread_mutex_t;
typedef void* pthread_attr_t; // unused in v1
typedef void* pthread_mutexattr_t;
code/user/personalities/posix/posix_io.c
+4 -2
View File
@@ -81,7 +81,8 @@ __declspec(dllexport) ssize_t read(int fd, void* buf, size_t count) {
return (ssize_t)count;
case FD_FILE: {
size_t got = 0;
uint32_t s = SysReadFile(e->KHandle, buf, count, &got);
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(e->KHandle, buf, count, &got);
uint32_t s = SysReadFile(&req);
if (s != 0 && got == 0) { errno = PosixStatusToErrno(s); return -1; }
return (ssize_t)got;
}
@@ -115,7 +116,8 @@ __declspec(dllexport) ssize_t write(int fd, const void* buf, size_t count) {
return (ssize_t)count;
case FD_FILE: {
size_t written = 0;
uint32_t s = SysWriteFile(e->KHandle, buf, count, &written);
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(e->KHandle, buf, count, &written);
uint32_t s = SysWriteFile(&req);
if (s != 0 && written == 0) { errno = PosixStatusToErrno(s); return -1; }
return (ssize_t)written;
}
code/user/personalities/posix/posix_mman.c
+85
View File
@@ -0,0 +1,85 @@
// POSIX compatibility support for mmap-style file mappings.
#include <stdint.h>
#include <stddef.h>
#include <ntlite/syscall.h>
#include "posix.h"
#include "posix_fd.h"
static uint32_t PosixProtToSysMapAccess(int prot, int flags) {
uint32_t access = 0;
if (prot & PROT_EXEC) access |= SYS_MAP_ACCESS_EXECUTE;
if (prot & PROT_WRITE) {
access |= (flags & MAP_SHARED) ? SYS_MAP_ACCESS_WRITE : SYS_MAP_ACCESS_COPY;
} else if (flags & MAP_PRIVATE) {
access |= SYS_MAP_ACCESS_COPY;
}
if (prot & PROT_READ) access |= SYS_MAP_ACCESS_READ;
if (!access) access = SYS_MAP_ACCESS_READ;
return access;
}
__declspec(dllexport) void* mmap(void* addr, size_t length, int prot,
int flags, int fd, off_t offset) {
if (addr || length == 0 || offset < 0 || ((uint64_t)offset & 0xFFFu)) {
errno = EINVAL;
return MAP_FAILED;
}
if (!(flags & MAP_PRIVATE) && !(flags & MAP_SHARED)) {
errno = EINVAL;
return MAP_FAILED;
}
uint32_t pid = SysGetCurrentPid();
FD_ENTRY* e = FdGet(pid, fd);
if (!e) return MAP_FAILED;
if (e->Type != FD_FILE) {
errno = EBADF;
return MAP_FAILED;
}
uint64_t maxSize = (uint64_t)offset + (uint64_t)length;
if (maxSize < (uint64_t)offset) {
errno = EINVAL;
return MAP_FAILED;
}
uint32_t protect = (prot & PROT_WRITE) ? SYS_VM_READWRITE : SYS_VM_READ;
if (prot & PROT_EXEC)
protect = (prot & PROT_WRITE) ? SYS_VM_EXECUTE_READWRITE : SYS_VM_EXECUTE_READ;
uint32_t mapping = 0;
uint32_t s = SysCreateFileMapping(e->KHandle, protect, SYS_MAP_FLAG_NONE,
maxSize, &mapping);
if (s != SYS_STATUS_SUCCESS) {
errno = PosixStatusToErrno(s);
return MAP_FAILED;
}
void* base = (void*)0;
s = SysMapFile(mapping, (uint64_t)offset, length,
PosixProtToSysMapAccess(prot, flags),
SYS_MAP_FLAG_NONE, &base);
SysCloseHandle(mapping);
if (s != SYS_STATUS_SUCCESS) {
errno = PosixStatusToErrno(s);
return MAP_FAILED;
}
return base;
}
__declspec(dllexport) int munmap(void* addr, size_t length) {
(void)length;
if (!addr) { errno = EINVAL; return -1; }
uint32_t s = SysFreeVm(addr);
if (s != SYS_STATUS_SUCCESS) { errno = PosixStatusToErrno(s); return -1; }
return 0;
}
__declspec(dllexport) int msync(void* addr, size_t length, int flags) {
(void)addr;
(void)length;
(void)flags;
return 0;
}
code/user/personalities/posix/posix_proc.c
+1 -1
View File
@@ -126,7 +126,7 @@ __declspec(dllexport) pid_t waitpid(pid_t pid, int* status, int options) {
uint32_t handle = ProcTableFind(pid);
if (!handle) { errno = ESRCH; return -1; }
SysWaitForProcess(handle);
SysWait(handle, SYS_WAIT_FLAG_NONE);
int exitCode = 0;
SysGetProcessExitCode(handle, &exitCode);
code/user/personalities/posix/posix_thread.c
+20 -7
View File
@@ -84,28 +84,41 @@ __declspec(dllexport) int pthread_create(pthread_t* t, const pthread_attr_t* att
tr->Pid = pid;
uint32_t tid = 0;
uint32_t s = SysCreateThread((uint64_t)PthreadTrampoline,
(uint64_t)tr, 1, &tid);
uint32_t handle = 0;
SYS_CREATE_THREAD_REQUEST req = {
(uintptr_t)PthreadTrampoline,
(uintptr_t)tr,
1,
0,
&handle,
&tid
};
uint32_t s = SysCreateThread(&req);
if (s != 0) {
free(tr);
return EAGAIN;
}
// The trampoline allocates the retval slot itself after it knows its real
// TID, avoiding the creator/new-thread race.
if (t) t->Tid = tid;
if (t) {
t->Handle = handle;
t->Tid = tid;
} else {
SysCloseHandle(handle);
}
return 0;
}
// Implements the pthread join operation.
__declspec(dllexport) int pthread_join(pthread_t t, void** retval) {
uint32_t pid = SysGetCurrentPid();
uint32_t s = SysWaitThread(t.Tid);
// STATUS_NOT_FOUND (0x22) means already dead - treat as success.
if (s != 0 && s != 0x0022) return ESRCH;
uint32_t s = SysWait(t.Handle, SYS_WAIT_FLAG_NONE);
if (s != 0) return ESRCH;
PTHREAD_SLOT* slot = PthreadSlotFind(pid, t.Tid);
if (retval) *retval = slot ? slot->RetVal : (void*)0;
if (slot) slot->Used = false;
SysCloseHandle(t.Handle);
return 0;
}
@@ -138,7 +151,7 @@ __declspec(dllexport) int pthread_mutex_destroy(pthread_mutex_t* m) {
// Implements the pthread mutex lock operation.
__declspec(dllexport) int pthread_mutex_lock(pthread_mutex_t* m) {
uint32_t s = SysAcquireMutex(m->Handle);
uint32_t s = SysWait(m->Handle, SYS_WAIT_FLAG_NONE);
return s == 0 ? 0 : EINVAL;
}
code/user/personalities/win32/gdi32.c
+7 -92
View File
@@ -63,101 +63,16 @@ static int* srcX_map = NULL;
static int currentDestCapacity = 0;
static int currentXMapCapacity = 0;
__declspec(dllexport) int StretchDIBits(
HDC hdc,
int xDest, int yDest, int DestWidth, int DestHeight,
__declspec(dllexport) int StretchDIBits(HDC hdc, int xDest, int yDest, int DestWidth, int DestHeight,
int xSrc, int ySrc, int SrcWidth, int SrcHeight,
const void *lpBits, const BITMAPINFO *lpbmi,
UINT iUsage, DWORD rop)
{
(void)hdc; (void)lpbmi; (void)iUsage; (void)rop;
(void)hdc; (void)xDest; (void)yDest; (void)DestWidth; (void)DestHeight;
(void)xSrc; (void)ySrc; (void)SrcWidth; (void)SrcHeight;
(void)lpBits; (void)lpbmi; (void)iUsage; (void)rop;
// Safety check to prevent division by zero.
if (DestWidth <= 0 || DestHeight <= 0 || SrcWidth <= 0 || SrcHeight <= 0) {
return 0;
}
// Handle trivial case (1:1 copy) to bypass allocation and scaling overhead.
if (DestWidth == SrcWidth && DestHeight == SrcHeight && xSrc == 0 && ySrc == 0) {
SysBlitScreen(xDest, yDest, DestWidth, DestHeight, (const uint32_t*)lpBits);
return DestHeight;
}
const uint32_t* srcPixels = (const uint32_t*)lpBits;
// Only reallocate if the new size is LARGER than what we already have.
int requiredPixels = DestWidth * DestHeight;
if (requiredPixels > currentDestCapacity) {
free(destBuffer);
destBuffer = (uint32_t*)malloc(requiredPixels * sizeof(uint32_t));
currentDestCapacity = requiredPixels;
}
if (DestWidth > currentXMapCapacity) {
free(srcX_map);
srcX_map = (int*)malloc(DestWidth * sizeof(int));
currentXMapCapacity = DestWidth;
}
for (int dx = 0; dx < DestWidth; ++dx) {
// Nearest neighbor coordinate mapping.
srcX_map[dx] = xSrc + (dx * SrcWidth) / DestWidth;
// Clamp to safe bounds.
if (srcX_map[dx] < 0) srcX_map[dx] = 0;
if (srcX_map[dx] >= SrcWidth) srcX_map[dx] = SrcWidth - 1;
}
// Process the image row by row.
for (int dy = 0; dy < DestHeight; ++dy) {
// Map destination Y to source Y.
int sy = ySrc + (dy * SrcHeight) / DestHeight;
if (sy < 0) sy = 0;
if (sy >= SrcHeight) sy = SrcHeight - 1;
// Assumes stride == SrcWidth (tightly packed 32bpp).
const uint32_t* srcRow = srcPixels + (sy * SrcWidth);
uint32_t* dstRow = destBuffer + (dy * DestWidth);
int dx = 0;
#if defined(__SSE2__) || defined(_M_X64)
// Process 4 pixels (16 bytes) per loop iteration using SSE2.
for (; dx <= DestWidth - 4; dx += 4) {
// Gather 4 source pixels using our precomputed X map.
// Note: _mm_set_epi32 takes arguments in reverse order (3, 2, 1, 0).
__m128i pixels = _mm_set_epi32(
srcRow[srcX_map[dx + 3]],
srcRow[srcX_map[dx + 2]],
srcRow[srcX_map[dx + 1]],
srcRow[srcX_map[dx + 0]]
);
// Issue a single 128-bit unaligned store.
_mm_storeu_si128((__m128i*)&dstRow[dx], pixels);
}
#elif defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM64)
// Process 4 pixels per loop iteration using ARM NEON.
for (; dx <= DestWidth - 4; dx += 4) {
// Initialize a vector and populate lanes (avoids stack-memory hop).
uint32x4_t pixels = vdupq_n_u32(0);
pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 0]], pixels, 0);
pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 1]], pixels, 1);
pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 2]], pixels, 2);
pixels = vsetq_lane_u32(srcRow[srcX_map[dx + 3]], pixels, 3);
// Issue a single 128-bit store.
vst1q_u32(&dstRow[dx], pixels);
}
#endif
// Epilogue: Handle remaining pixels if DestWidth isn't a multiple of 4.
for (; dx < DestWidth; ++dx) {
dstRow[dx] = srcRow[srcX_map[dx]];
}
}
// Direct mapping to kernel screen blit using the stretched buffer.
SysBlitScreen(xDest, yDest, DestWidth, DestHeight, destBuffer);
return DestHeight;
// There is no implicit display blit ABI. Native graphics clients own a
// surface and present it explicitly.
return 0;
}
code/user/personalities/win32/kernel32.c
+321 -97
View File
@@ -27,6 +27,8 @@
#include <intrin.h>
#include <ntlite/syscall.h>
extern size_t _malloc_usable_size(const void* ptr);
// Types.
typedef void* HANDLE;
typedef unsigned long DWORD;
@@ -58,6 +60,7 @@ typedef struct _CONSOLE_SCREEN_BUFFER_INFO {
#define MEM_RELEASE 0x00008000UL
#define MEM_FREE 0x00010000UL
#define MEM_PRIVATE 0x00020000UL
#define MEM_MAPPED 0x00040000UL
#define PAGE_NOACCESS 0x01UL
#define PAGE_READONLY 0x02UL
#define PAGE_READWRITE 0x04UL
@@ -65,6 +68,13 @@ typedef struct _CONSOLE_SCREEN_BUFFER_INFO {
#define PAGE_EXECUTE 0x10UL
#define PAGE_EXECUTE_READ 0x20UL
#define PAGE_EXECUTE_READWRITE 0x40UL
#define SEC_COMMIT 0x08000000UL
#define FILE_MAP_COPY 0x00000001UL
#define FILE_MAP_WRITE 0x00000002UL
#define FILE_MAP_READ 0x00000004UL
#define FILE_MAP_EXECUTE 0x00000020UL
#define FILE_MAP_ALL_ACCESS 0x000F001FUL
typedef struct _MEMORY_BASIC_INFORMATION {
void* BaseAddress;
@@ -90,6 +100,26 @@ static uint32_t WinProtectToSysVm(DWORD protect) {
}
}
static uint32_t WinFileMappingProtectToSysVm(DWORD protect) {
// Strip section attributes; the low byte carries PAGE_* protection.
DWORD pageProtect = protect & 0xFFu;
if (pageProtect == PAGE_WRITECOPY)
return SYS_VM_READ; // MapViewOfFile(FILE_MAP_COPY) asks for the private writable view.
return WinProtectToSysVm(pageProtect ? pageProtect : PAGE_READONLY);
}
static uint32_t WinFileMapAccessToSys(DWORD desiredAccess) {
uint32_t access = 0;
if (desiredAccess & FILE_MAP_EXECUTE) access |= SYS_MAP_ACCESS_EXECUTE;
if (desiredAccess & FILE_MAP_COPY) access |= SYS_MAP_ACCESS_COPY;
if (desiredAccess & FILE_MAP_WRITE) access |= SYS_MAP_ACCESS_WRITE;
if (desiredAccess & FILE_MAP_READ) access |= SYS_MAP_ACCESS_READ;
if ((desiredAccess & FILE_MAP_ALL_ACCESS) == FILE_MAP_ALL_ACCESS)
access |= SYS_MAP_ACCESS_READ | SYS_MAP_ACCESS_COPY;
if (!access) access = SYS_MAP_ACCESS_READ;
return access;
}
// Implements the protect Vm To Win operation.
static DWORD SysProtectVmToWin(uint32_t protect) {
switch (protect) {
@@ -153,24 +183,9 @@ typedef struct _WIN32_FIND_DATAA {
static inline HANDLE KH_TO_HANDLE(uint32_t kh) { return (HANDLE)(uintptr_t)kh; }
static inline uint32_t HANDLE_TO_KH(HANDLE h) { return (uint32_t)(uintptr_t)h; }
// Thread IDs are not kernel handles. Keep Win32 thread pseudo-handles out of
// the low numeric handle namespace so CloseHandle/WaitForSingleObject cannot
// accidentally operate on an unrelated file/process handle with the same value.
#define WIN32_THREAD_HANDLE_TAG ((uintptr_t)1ULL << 63)
#define WIN32_MUTEX_SENTINEL ((uintptr_t)0x4D555458UL) // 'MUTX'
#define WIN32_SEMA_SENTINEL ((uintptr_t)0x53454D41UL) // 'SEMA'
static inline HANDLE TID_TO_THREAD_HANDLE(uint32_t tid) {
return (HANDLE)(WIN32_THREAD_HANDLE_TAG | (uintptr_t)tid);
}
static inline BOOL IS_THREAD_HANDLE(HANDLE h) {
return h && h != INVALID_HANDLE_VALUE && (((uintptr_t)h & WIN32_THREAD_HANDLE_TAG) != 0);
}
static inline uint32_t THREAD_HANDLE_TO_TID(HANDLE h) {
return (uint32_t)((uintptr_t)h & 0xFFFFFFFFu);
}
static inline BOOL IS_NOOP_SYNC_HANDLE(HANDLE h) {
static inline BOOL IS_VALID_KERNEL_HANDLE(HANDLE h) {
uintptr_t v = (uintptr_t)h;
return v == WIN32_MUTEX_SENTINEL || v == WIN32_SEMA_SENTINEL;
return v != 0 && v != (uintptr_t)INVALID_HANDLE_VALUE;
}
// Path translator.
@@ -268,11 +283,14 @@ static void K32DebugPrintA(const char* s) {
}
#define WIN32_TLS_SLOTS 64
#define FLS_MAX_SLOTS 64
#define FLS_OUT_OF_INDEXES 0xFFFFFFFFUL
typedef struct WIN32_THREAD_ENV {
DWORD LastError;
DWORD Reserved0;
void* TlsSlots[WIN32_TLS_SLOTS];
void* FlsSlots[FLS_MAX_SLOTS];
} WIN32_THREAD_ENV;
static WIN32_THREAD_ENV gFallbackThreadEnv;
@@ -418,11 +436,27 @@ __declspec(dllexport) uint64_t GetTickCount64(void) {
}
// Memory (heap argument is ignored - single user-mode heap).
#define HEAP_ZERO_MEMORY 0x00000008UL
#define HEAP_REALLOC_IN_PLACE_ONLY 0x00000010UL
__declspec(dllexport) void* HeapAlloc(void* heap, DWORD flags, size_t size) {
(void)heap; (void)flags;
(void)heap;
if (!size) return NULL;
void* ptr = malloc(size);
if (!ptr) gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY
if (!ptr) {
gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY
return NULL;
}
// Bink and the Microsoft CRT routinely allocate internal state with
// HEAP_ZERO_MEMORY. Ignoring this flag only works accidentally while the
// heap keeps receiving fresh demand-zero pages; under low memory / swap the
// allocator reuses dirty freed blocks and stale pitch/pointer fields can
// corrupt the decoder.
if (flags & HEAP_ZERO_MEMORY)
memset(ptr, 0, size);
return ptr;
}
@@ -436,9 +470,28 @@ __declspec(dllexport) BOOL HeapFree(void* heap, DWORD flags, void* ptr) {
__declspec(dllexport) void* HeapReAlloc(void* heap, DWORD flags,
void* ptr, size_t size) {
(void)heap; (void)flags;
(void)heap;
if (flags & HEAP_REALLOC_IN_PLACE_ONLY) {
// The user heap does not expose an in-place grow primitive. Preserve
// Windows semantics by failing instead of silently moving the block.
size_t oldSize = _malloc_usable_size(ptr);
if (oldSize != (size_t)-1 && oldSize >= size)
return ptr;
gLastError = 8; // ERROR_NOT_ENOUGH_MEMORY
return NULL;
}
size_t oldSize = ptr ? _malloc_usable_size(ptr) : 0;
void* newptr = realloc(ptr, size);
if (!newptr && size) gLastError = 8;
if (!newptr && size) {
gLastError = 8;
return NULL;
}
if (newptr && (flags & HEAP_ZERO_MEMORY) && oldSize != (size_t)-1 && size > oldSize)
memset((unsigned char*)newptr + oldSize, 0, size - oldSize);
return newptr;
}
@@ -508,10 +561,88 @@ __declspec(dllexport) size_t VirtualQuery(const void* lpAddress,
lpBuffer->RegionSize = info.PageCount * 4096u;
lpBuffer->State = MEM_COMMIT;
lpBuffer->Protect = protect;
lpBuffer->Type = MEM_PRIVATE;
lpBuffer->Type = (info.Type == 7u) ? MEM_MAPPED : MEM_PRIVATE; // 7 == MmVadMappedFile
return sizeof(MEMORY_BASIC_INFORMATION);
}
__declspec(dllexport) HANDLE CreateFileMappingA(HANDLE hFile,
void* lpFileMappingAttributes,
DWORD flProtect,
DWORD dwMaximumSizeHigh,
DWORD dwMaximumSizeLow,
const char* lpName) {
(void)lpFileMappingAttributes;
(void)lpName;
if (!hFile || hFile == INVALID_HANDLE_VALUE) {
gLastError = 6; // ERROR_INVALID_HANDLE
return NULL;
}
uint64_t maximumSize = ((uint64_t)dwMaximumSizeHigh << 32) | (uint64_t)dwMaximumSizeLow;
uint32_t kh = 0;
uint32_t s = SysCreateFileMapping(HANDLE_TO_KH(hFile),
WinFileMappingProtectToSysVm(flProtect),
SYS_MAP_FLAG_NONE,
maximumSize,
&kh);
if (s != SYS_STATUS_SUCCESS) {
gLastError = (s == SYS_STATUS_NOT_SUPPORTED) ? 120 : ((s == SYS_STATUS_NO_MEMORY) ? 8 : 87);
return NULL;
}
return KH_TO_HANDLE(kh);
}
__declspec(dllexport) HANDLE CreateFileMappingW(HANDLE hFile,
void* lpFileMappingAttributes,
DWORD flProtect,
DWORD dwMaximumSizeHigh,
DWORD dwMaximumSizeLow,
const unsigned short* lpName) {
(void)lpName;
return CreateFileMappingA(hFile, lpFileMappingAttributes, flProtect,
dwMaximumSizeHigh, dwMaximumSizeLow, NULL);
}
__declspec(dllexport) void* MapViewOfFile(HANDLE hFileMappingObject,
DWORD dwDesiredAccess,
DWORD dwFileOffsetHigh,
DWORD dwFileOffsetLow,
size_t dwNumberOfBytesToMap) {
if (!hFileMappingObject || hFileMappingObject == INVALID_HANDLE_VALUE) {
gLastError = 6;
return NULL;
}
uint64_t offset = ((uint64_t)dwFileOffsetHigh << 32) | (uint64_t)dwFileOffsetLow;
void* base = NULL;
uint32_t s = SysMapFile(HANDLE_TO_KH(hFileMappingObject),
offset,
dwNumberOfBytesToMap,
WinFileMapAccessToSys(dwDesiredAccess),
SYS_MAP_FLAG_NONE,
&base);
if (s != SYS_STATUS_SUCCESS) {
gLastError = (s == SYS_STATUS_NOT_SUPPORTED) ? 120 : ((s == SYS_STATUS_NO_MEMORY) ? 8 : 87);
return NULL;
}
return base;
}
__declspec(dllexport) BOOL UnmapViewOfFile(const void* lpBaseAddress) {
if (!lpBaseAddress) { gLastError = 87; return FALSE; }
uint32_t s = SysFreeVm((void*)lpBaseAddress);
if (s != SYS_STATUS_SUCCESS) { gLastError = 87; return FALSE; }
return TRUE;
}
__declspec(dllexport) BOOL FlushViewOfFile(const void* lpBaseAddress, size_t dwNumberOfBytesToFlush) {
(void)dwNumberOfBytesToFlush;
if (!lpBaseAddress) { gLastError = 87; return FALSE; }
// Native flush is file-handle based for now; unmap/close also harvests dirty
// PTEs, so FlushViewOfFile remains a successful cache-coherency hint.
return TRUE;
}
// --- Win32 API ---.
// GetAsyncKeyState implementation over the canonical input key namespace.
@@ -720,7 +851,7 @@ __declspec(dllexport) HANDLE CreateFileA(const char* name,
// GetFileSize - fills optional high-DWORD pointer, returns low DWORD.
// Returns 0xFFFFFFFF on error, matching Win32 behaviour.
__declspec(dllexport) DWORD GetFileSize(HANDLE hFile, DWORD* fileSizeHigh) {
if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) { gLastError = 6; return 0xFFFFFFFFUL; }
if (!hFile || hFile == INVALID_HANDLE_VALUE) { gLastError = 6; return 0xFFFFFFFFUL; }
uint64_t sz = SysGetFileSize(HANDLE_TO_KH(hFile));
if (fileSizeHigh) *fileSizeHigh = (DWORD)(sz >> 32);
return (DWORD)(sz & 0xFFFFFFFFULL);
@@ -731,25 +862,27 @@ __declspec(dllexport) BOOL ReadFile(HANDLE hFile, void* buf, DWORD toRead,
DWORD* bytesRead, void* overlapped) {
(void)overlapped;
if (bytesRead) *bytesRead = 0;
if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile) || !buf) {
if (!hFile || hFile == INVALID_HANDLE_VALUE || !buf) {
gLastError = 6;
return FALSE;
}
size_t got = 0;
uint32_t s = SysReadFile(HANDLE_TO_KH(hFile), buf, (size_t)toRead, &got);
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(HANDLE_TO_KH(hFile), buf, (size_t)toRead, &got);
uint32_t s = SysReadFile(&req);
if (bytesRead) *bytesRead = (DWORD)got;
// Harden callers that accidentally look past *bytesRead. EOF/short reads
// are legal for files, but consuming stale stack/heap bytes after them makes
// the failure mode look like random game-data corruption.
if (got < (size_t)toRead)
memset((unsigned char*)buf + got, 0, (size_t)toRead - got);
if (s != 0) {
gLastError = 1;
return FALSE;
}
// Harden callers that accidentally look past *bytesRead. EOF/short reads
// are legal for files, but do not touch the caller's remaining buffer after
// a failed read because that can fault in megabytes under low memory.
if (got < (size_t)toRead)
memset((unsigned char*)buf + got, 0, (size_t)toRead - got);
return TRUE;
}
@@ -758,9 +891,10 @@ __declspec(dllexport) BOOL ReadFile(HANDLE hFile, void* buf, DWORD toRead,
__declspec(dllexport) BOOL WriteFile(HANDLE hFile, const void* buf, DWORD toWrite,
DWORD* bytesWritten, void* overlapped) {
(void)overlapped;
if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile) || !buf) { gLastError = 6; return FALSE; }
if (!hFile || hFile == INVALID_HANDLE_VALUE || !buf) { gLastError = 6; return FALSE; }
size_t written = 0;
uint32_t s = SysWriteFile(HANDLE_TO_KH(hFile), buf, (size_t)toWrite, &written);
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(HANDLE_TO_KH(hFile), buf, (size_t)toWrite, &written);
uint32_t s = SysWriteFile(&req);
if (bytesWritten) *bytesWritten = (DWORD)written;
if (s != 0) { gLastError = 29; return FALSE; } // ERROR_WRITE_FAULT
return TRUE;
@@ -770,7 +904,7 @@ __declspec(dllexport) BOOL WriteFile(HANDLE hFile, const void* buf, DWORD toWrit
// Returns new position low DWORD, or 0xFFFFFFFF on error.
__declspec(dllexport) DWORD SetFilePointer(HANDLE hFile, long dist,
long* distHigh, DWORD method) {
if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) {
if (!hFile || hFile == INVALID_HANDLE_VALUE) {
gLastError = 6;
return 0xFFFFFFFFUL;
}
@@ -813,7 +947,7 @@ __declspec(dllexport) DWORD SetFilePointer(HANDLE hFile, long dist,
// SetEndOfFile - truncate or extend the file to the current seek position.
__declspec(dllexport) BOOL SetEndOfFile(HANDLE hFile) {
if (!hFile || hFile == INVALID_HANDLE_VALUE || IS_THREAD_HANDLE(hFile)) { gLastError = 6; return FALSE; }
if (!hFile || hFile == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; }
uint32_t kh = HANDLE_TO_KH(hFile);
uint64_t pos = SysGetFilePos(kh);
if (pos == (uint64_t)-1) { gLastError = 6; return FALSE; }
@@ -834,14 +968,10 @@ __declspec(dllexport) BOOL DeleteFileA(const char* lpFileName) {
return TRUE;
}
// CloseHandle - unified close for files, mutexes, semaphores, events, queues.
// CloseHandle - unified close for files, threads, mutexes, semaphores, events, queues, and processes.
__declspec(dllexport) BOOL CloseHandle(HANDLE hObject) {
if (!hObject || hObject == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; }
// Our thread/sentinel sync handles are not entries in the kernel handle table.
// There is nothing to close; the thread object is reaped by the scheduler.
if (IS_THREAD_HANDLE(hObject) || IS_NOOP_SYNC_HANDLE(hObject)) return TRUE;
SysCloseHandle(HANDLE_TO_KH(hObject));
return TRUE;
}
@@ -870,8 +1000,11 @@ __declspec(dllexport) BOOL HeapDestroy(HANDLE hHeap) {
// Implements the heap Size operation.
__declspec(dllexport) size_t HeapSize(HANDLE heap, DWORD flags, const void* ptr) {
(void)heap; (void)flags; (void)ptr;
return (size_t)-1; // Win32 returns (SIZE_T)-1 on error / unsupported
(void)heap; (void)flags;
size_t size = _malloc_usable_size(ptr);
if (size == (size_t)-1)
gLastError = 87; // ERROR_INVALID_PARAMETER
return size;
}
__declspec(dllexport) BOOL HeapSetInformation(HANDLE HeapHandle,
@@ -1050,9 +1183,29 @@ __declspec(dllexport) DWORD GetModuleFileNameA(HANDLE hModule,
// Implements the terminate Process operation.
__declspec(dllexport) BOOL TerminateProcess(HANDLE hProcess, unsigned int uExitCode) {
(void)hProcess;
SysExitProcess((int)uExitCode);
while (true) ;
if (hProcess == GetCurrentProcess()) {
SysExitProcess((int)uExitCode);
while (true) ;
}
if (!hProcess) {
gLastError = 6; // ERROR_INVALID_HANDLE
return FALSE;
}
uint32_t pid = 0;
uint32_t s = SysGetProcessPid(HANDLE_TO_KH(hProcess), &pid);
if (s != 0 || pid == 0) {
gLastError = 6; // ERROR_INVALID_HANDLE
return FALSE;
}
s = SysKillProcess(pid, (int)uExitCode);
if (s != 0) {
gLastError = 5; // ERROR_ACCESS_DENIED / generic failure for now
return FALSE;
}
return TRUE;
}
// STARTUPINFOA - only cb matters for our stub.
@@ -1099,35 +1252,42 @@ __declspec(dllexport) HANDLE CreateThread(void* lpThreadAttributes,
DWORD* lpThreadId) {
(void)lpThreadAttributes; (void)dwStackSize; (void)dwCreationFlags;
uint32_t tid = 0;
uint32_t s = SysCreateThread((uint64_t)(uintptr_t)lpStartAddress,
(uint64_t)(uintptr_t)lpParameter,
0, &tid);
uint32_t kh = 0;
SYS_CREATE_THREAD_REQUEST req = {
(uintptr_t)lpStartAddress,
(uintptr_t)lpParameter,
0,
0,
&kh,
&tid
};
uint32_t s = SysCreateThread(&req);
if (s != 0) { gLastError = 11; return INVALID_HANDLE_VALUE; }
if (lpThreadId) *lpThreadId = (DWORD)tid;
return TID_TO_THREAD_HANDLE(tid);
return KH_TO_HANDLE(kh);
}
// WaitForSingleObject - blocks on process handles via SysWaitForProcess;
// thread and other handle types are not yet waitable so return immediately.
// WaitForSingleObject - blocks on native waitable handles. A zero timeout is
// only a probe for mutexes/semaphores until native timeout waits exist.
__declspec(dllexport) DWORD WaitForSingleObject(HANDLE hHandle, DWORD dwMilliseconds) {
(void)dwMilliseconds;
if (!hHandle || hHandle == INVALID_HANDLE_VALUE) { gLastError = 6; return 0xFFFFFFFFUL; }
if (IS_THREAD_HANDLE(hHandle)) {
uint32_t tid = THREAD_HANDLE_TO_TID(hHandle);
uint32_t s = SysWaitThread(tid);
if (s != 0 && s != 0x0022) { gLastError = 6; return 0xFFFFFFFFUL; }
return WAIT_OBJECT_0;
}
if (IS_NOOP_SYNC_HANDLE(hHandle)) return WAIT_OBJECT_0;
if (!IS_VALID_KERNEL_HANDLE(hHandle)) { gLastError = 6; return 0xFFFFFFFFUL; }
uint32_t kh = HANDLE_TO_KH(hHandle);
if (kh) {
uint32_t s = SysWaitForProcess(kh);
if (s != 0) { gLastError = 6; return 0xFFFFFFFFUL; }
if (dwMilliseconds == 0) {
uint32_t s = SysTryAcquireMutex(kh);
if (s == 0) return WAIT_OBJECT_0;
s = SysTryWaitSem(kh);
if (s == 0) return WAIT_OBJECT_0;
return WAIT_TIMEOUT;
}
return WAIT_OBJECT_0;
uint32_t s = SysWait(kh, SYS_WAIT_FLAG_NONE);
if (s == 0) return WAIT_OBJECT_0;
gLastError = 6;
return 0xFFFFFFFFUL;
}
@@ -1250,7 +1410,7 @@ __declspec(dllexport) BOOL CreateProcessA(
__declspec(dllexport) BOOL GetExitCodeProcess(HANDLE hProcess, DWORD* lpExitCode) {
if (!lpExitCode) { gLastError = 87; return FALSE; }
uint32_t kh = HANDLE_TO_KH(hProcess);
if (!kh || IS_THREAD_HANDLE(hProcess)) { gLastError = 6; return FALSE; } // ERROR_INVALID_HANDLE
if (!kh) { gLastError = 6; return FALSE; } // ERROR_INVALID_HANDLE
int code = 0;
uint32_t s = SysGetProcessExitCode(kh, &code);
if (s != 0) { gLastError = 6; return FALSE; }
@@ -1791,57 +1951,123 @@ __declspec(dllexport) DWORD GetCurrentDirectoryA(DWORD nBufferLength, char* lpBu
__declspec(dllexport) HANDLE CreateMutexA(void* lpMutexAttributes, BOOL bInitialOwner,
const char* lpName) {
(void)lpMutexAttributes; (void)bInitialOwner; (void)lpName;
return (HANDLE)WIN32_MUTEX_SENTINEL;
(void)lpMutexAttributes; (void)lpName;
uint32_t kh = 0;
uint32_t s = SysCreateMutex(&kh);
if (s != 0 || !kh) { gLastError = 8; return NULL; }
if (bInitialOwner) {
s = SysWait(kh, SYS_WAIT_FLAG_NONE);
if (s != 0) {
SysCloseHandle(kh);
gLastError = 1;
return NULL;
}
}
return KH_TO_HANDLE(kh);
}
// Releases mutex.
__declspec(dllexport) BOOL ReleaseMutex(HANDLE hMutex) {
(void)hMutex;
if (!IS_VALID_KERNEL_HANDLE(hMutex)) { gLastError = 6; return FALSE; }
uint32_t s = SysReleaseMutex(HANDLE_TO_KH(hMutex));
if (s != 0) { gLastError = 6; return FALSE; }
return TRUE;
}
__declspec(dllexport) HANDLE CreateSemaphoreA(void* lpSemaphoreAttributes,
long lInitialCount, long lMaximumCount,
const char* lpName) {
(void)lpSemaphoreAttributes; (void)lInitialCount;
(void)lMaximumCount; (void)lpName;
return (HANDLE)WIN32_SEMA_SENTINEL;
(void)lpSemaphoreAttributes; (void)lpName;
uint32_t kh = 0;
uint32_t s = SysCreateSem((int32_t)lInitialCount, (int32_t)lMaximumCount, &kh);
if (s != 0 || !kh) { gLastError = 8; return NULL; }
return KH_TO_HANDLE(kh);
}
__declspec(dllexport) BOOL ReleaseSemaphore(HANDLE hSemaphore, long lReleaseCount,
long* lpPreviousCount) {
(void)hSemaphore; (void)lReleaseCount;
if (!IS_VALID_KERNEL_HANDLE(hSemaphore) || lReleaseCount < 0) { gLastError = 6; return FALSE; }
if (lpPreviousCount) *lpPreviousCount = 0;
for (long i = 0; i < lReleaseCount; ++i) {
uint32_t s = SysSignalSem(HANDLE_TO_KH(hSemaphore));
if (s != 0) { gLastError = 6; return FALSE; }
}
return TRUE;
}
// Critical Sections & Condition Variables
// Single-threaded: all locks are no-ops.
// Minimal opaque types; callers allocate these from their Windows headers -
// we just need the pointer to be accepted by our stubs.
// Critical Sections & Condition Variables.
// Match the size of Win64 RTL_CRITICAL_SECTION (5 pointer-sized fields) while
// using the storage as a small recursive spin lock. Bink uses these from its
// worker/copy paths, so the old no-op implementation allowed internal state
// races as soon as CreateThread started working.
typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION;
typedef struct { void* _ptr; } CONDITION_VARIABLE;
#define K32_CS_LOCK 0
#define K32_CS_OWNER 1
#define K32_CS_RECURSION 2
static volatile long* K32CsLockWord(CRITICAL_SECTION* cs) {
return (volatile long*)&cs->_opaque[K32_CS_LOCK];
}
static DWORD K32CurrentTidOrOne(void) {
DWORD tid = (DWORD)SysGetCurrentTid();
return tid ? tid : 1;
}
// Initializes critical Section.
__declspec(dllexport) void InitializeCriticalSection(CRITICAL_SECTION* lpCS) {
(void)lpCS;
if (!lpCS) return;
memset(lpCS, 0, sizeof(*lpCS));
}
__declspec(dllexport) BOOL InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION* lpCS,
DWORD dwSpinCount) {
(void)lpCS; (void)dwSpinCount;
(void)dwSpinCount;
InitializeCriticalSection(lpCS);
return TRUE;
}
// Releases critical Section.
__declspec(dllexport) void DeleteCriticalSection(CRITICAL_SECTION* lpCS) { (void)lpCS; }
// Implements the enter Critical Section operation.
__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS) { (void)lpCS; }
__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS) {
if (!lpCS) return;
DWORD tid = K32CurrentTidOrOne();
if ((DWORD)lpCS->_opaque[K32_CS_OWNER] == tid) {
lpCS->_opaque[K32_CS_RECURSION]++;
return;
}
while (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0)
_mm_pause();
lpCS->_opaque[K32_CS_OWNER] = tid;
lpCS->_opaque[K32_CS_RECURSION] = 1;
}
// Implements the leave Critical Section operation.
__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION* lpCS) { (void)lpCS; }
__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION* lpCS) {
if (!lpCS) return;
DWORD tid = K32CurrentTidOrOne();
if ((DWORD)lpCS->_opaque[K32_CS_OWNER] != tid || lpCS->_opaque[K32_CS_RECURSION] == 0)
return;
lpCS->_opaque[K32_CS_RECURSION]--;
if (lpCS->_opaque[K32_CS_RECURSION] == 0) {
lpCS->_opaque[K32_CS_OWNER] = 0;
_ReadWriteBarrier();
_InterlockedExchange(K32CsLockWord(lpCS), 0);
}
}
// Implements the try Enter Critical Section operation.
__declspec(dllexport) BOOL TryEnterCriticalSection(CRITICAL_SECTION* lpCS) {
(void)lpCS; return TRUE;
if (!lpCS) return FALSE;
DWORD tid = K32CurrentTidOrOne();
if ((DWORD)lpCS->_opaque[K32_CS_OWNER] == tid) {
lpCS->_opaque[K32_CS_RECURSION]++;
return TRUE;
}
if (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0)
return FALSE;
lpCS->_opaque[K32_CS_OWNER] = tid;
lpCS->_opaque[K32_CS_RECURSION] = 1;
return TRUE;
}
// Initializes condition Variable.
@@ -2357,14 +2583,12 @@ __declspec(dllexport) int GetLocaleInfoA(unsigned int Locale, DWORD LCType,
return 0;
}
// Fiber Local Storage (FLS)
// We implement a minimal 64-slot static table so callers that use FLS for
// thread-local-ish state don't immediately fail.
// Fiber Local Storage (FLS).
// Allocation state is process-global, but the values themselves are per-thread,
// matching the contract that Bink and the CRT expect when worker threads are
// enabled. The old single global value array let unrelated threads overwrite
// each other's decoder-local state.
#define FLS_MAX_SLOTS 64
#define FLS_OUT_OF_INDEXES 0xFFFFFFFFUL
static void* gFlsSlots[FLS_MAX_SLOTS];
static int gFlsUsed[FLS_MAX_SLOTS]; // 0 = free, 1 = allocated
// Implements the fls Alloc operation.
@@ -2372,8 +2596,8 @@ __declspec(dllexport) DWORD FlsAlloc(void* lpCallback) {
(void)lpCallback;
for (int i = 0; i < FLS_MAX_SLOTS; i++) {
if (!gFlsUsed[i]) {
gFlsUsed[i] = 1;
gFlsSlots[i] = NULL;
gFlsUsed[i] = 1;
K32ThreadEnv()->FlsSlots[i] = NULL;
return (DWORD)i;
}
}
@@ -2387,7 +2611,7 @@ __declspec(dllexport) void* FlsGetValue(DWORD dwFlsIndex) {
gLastError = 87; // ERROR_INVALID_PARAMETER
return NULL;
}
return gFlsSlots[dwFlsIndex];
return K32ThreadEnv()->FlsSlots[dwFlsIndex];
}
// Implements the fls Set Value operation.
@@ -2396,7 +2620,7 @@ __declspec(dllexport) BOOL FlsSetValue(DWORD dwFlsIndex, void* lpFlsData) {
gLastError = 87;
return FALSE;
}
gFlsSlots[dwFlsIndex] = lpFlsData;
K32ThreadEnv()->FlsSlots[dwFlsIndex] = lpFlsData;
return TRUE;
}
@@ -2406,8 +2630,8 @@ __declspec(dllexport) BOOL FlsFree(DWORD dwFlsIndex) {
gLastError = 87;
return FALSE;
}
gFlsUsed[dwFlsIndex] = 0;
gFlsSlots[dwFlsIndex] = NULL;
gFlsUsed[dwFlsIndex] = 0;
K32ThreadEnv()->FlsSlots[dwFlsIndex] = NULL;
return TRUE;
}
code/user/runtime/heap.c
+23
View File
@@ -210,6 +210,29 @@ void* _calloc(size_t count, size_t size) {
return ptr;
}
// Returns the usable payload size of an allocation made by this heap.
size_t _malloc_usable_size(const void* ptr) {
if (!ptr) return (size_t)-1;
HeapLock();
HEAP_SEGMENT* seg = SegmentForPointer((void*)ptr);
if (!seg) {
HeapUnlock();
return (size_t)-1;
}
HEAP_BLOCK* block = PayloadBlock((void*)ptr);
if (block->Magic != USER_HEAP_MAGIC || block->Free) {
HeapUnlock();
return (size_t)-1;
}
size_t size = block->Size;
HeapUnlock();
return size;
}
// Implements the realloc operation.
void* _realloc(void* ptr, size_t newSize) {
if (!ptr) return _malloc(newSize);
code/user/runtime/heap_exports.c
+5
View File
@@ -12,6 +12,7 @@ void* _malloc(size_t size);
void _free(void* ptr);
void* _calloc(size_t count, size_t size);
void* _realloc(void* ptr, size_t size);
size_t _malloc_usable_size(const void* ptr);
HEAP_EXPORT void* malloc(size_t size) {
return _malloc(size);
@@ -28,3 +29,7 @@ HEAP_EXPORT void* calloc(size_t count, size_t size) {
HEAP_EXPORT void* realloc(void* ptr, size_t size) {
return _realloc(ptr, size);
}
HEAP_EXPORT size_t malloc_usable_size(const void* ptr) {
return _malloc_usable_size(ptr);
}
code/user/syscall/syscall_nr.inc
+49 -38
View File
@@ -3,13 +3,13 @@
SYS_EXIT_PROCESS EQU 0
SYS_EXIT_THREAD EQU 1
SYS_CREATE_PROCESS EQU 2
SYS_WAIT_FOR_PROCESS EQU 3
SYS_WAIT_SINGLE EQU 3
SYS_GET_PROCESS_EXIT_CODE EQU 4
SYS_KILL_PROCESS EQU 5
SYS_GET_PROCESS_LIST EQU 6
SYS_GET_PROCESS_PID EQU 7
SYS_CREATE_THREAD EQU 8
SYS_WAIT_THREAD EQU 9
SYS_RETIRED_WAIT_THREAD EQU 9
SYS_GET_CURRENT_TID EQU 10
SYS_GET_CURRENT_PID EQU 11
SYS_SET_THREAD_PRIORITY EQU 12
@@ -23,17 +23,17 @@ SYS_PROTECT_VM EQU 19
SYS_QUERY_VM EQU 20
SYS_CLOSE_HANDLE EQU 21
SYS_CREATE_MUTEX EQU 22
SYS_ACQUIRE_MUTEX EQU 23
SYS_RETIRED_ACQUIRE_MUTEX EQU 23
SYS_TRY_ACQUIRE_MUTEX EQU 24
SYS_RELEASE_MUTEX EQU 25
SYS_CREATE_SEM EQU 26
SYS_WAIT_SEM EQU 27
SYS_RETIRED_WAIT_SEM EQU 27
SYS_TRY_WAIT_SEM EQU 28
SYS_SIGNAL_SEM EQU 29
SYS_CREATE_EVENT EQU 30
SYS_SIGNAL_EVENT EQU 31
SYS_CLEAR_EVENT EQU 32
SYS_WAIT_EVENT EQU 33
SYS_RETIRED_WAIT_EVENT EQU 33
SYS_CREATE_QUEUE EQU 34
SYS_SEND_QUEUE EQU 35
SYS_RECV_QUEUE EQU 36
@@ -66,36 +66,47 @@ SYS_SET_CONSOLE_CURSOR EQU 62
SYS_FILL_CONSOLE_CHARACTER EQU 63
SYS_FILL_CONSOLE_ATTRIBUTE EQU 64
SYS_PUT_SCREEN_CHAR EQU 65
SYS_BLIT_SCREEN EQU 66
SYS_CLEAR_SCREEN EQU 67
SYS_GET_SCREEN_SIZE EQU 68
SYS_QUERY_DISPLAY_INFO EQU 69
SYS_POLL_KEYBOARD EQU 70
SYS_IS_KEY_DOWN EQU 71
SYS_READ_KEYBOARD_CHAR EQU 72
SYS_READ_KEY EQU 73
SYS_GET_KEY_MODIFIERS EQU 74
SYS_POLL_MOUSE EQU 75
SYS_READ_MOUSE EQU 76
SYS_GET_MOUSE_POS EQU 77
SYS_GET_TICK EQU 78
SYS_PRINT_DEBUG EQU 79
SYS_QUERY_TRAP_FRAME EQU 80
SYS_CREATE_SOCKET EQU 81
SYS_BIND_SOCKET EQU 82
SYS_LISTEN_SOCKET EQU 83
SYS_ACCEPT_SOCKET EQU 84
SYS_CONNECT_SOCKET EQU 85
SYS_SEND_SOCKET EQU 86
SYS_RECV_SOCKET EQU 87
SYS_SHUTDOWN_SOCKET EQU 88
SYS_SEND_TO_SOCKET EQU 89
SYS_RECV_FROM_SOCKET EQU 90
SYS_RESOLVE_HOST EQU 91
SYS_SET_SOCKET_OPTION EQU 92
SYS_GET_SOCKET_OPTION EQU 93
SYS_POLL EQU 94
SYS_GET_MEMORY_STATUS EQU 95
SYS_QUERY_KERNEL_STATS EQU 96
SYS_GET_SYSTEM_TIME EQU 97
SYS_MAX EQU 98
SYS_CLEAR_SCREEN EQU 66
SYS_GET_SCREEN_SIZE EQU 67
SYS_QUERY_DISPLAY_INFO EQU 68
SYS_POLL_KEYBOARD EQU 69
SYS_IS_KEY_DOWN EQU 70
SYS_READ_KEYBOARD_CHAR EQU 71
SYS_READ_KEY EQU 72
SYS_GET_KEY_MODIFIERS EQU 73
SYS_POLL_MOUSE EQU 74
SYS_READ_MOUSE EQU 75
SYS_GET_MOUSE_POS EQU 76
SYS_GET_TICK EQU 77
SYS_PRINT_DEBUG EQU 78
SYS_QUERY_TRAP_FRAME EQU 79
SYS_CREATE_SOCKET EQU 80
SYS_BIND_SOCKET EQU 81
SYS_LISTEN_SOCKET EQU 82
SYS_ACCEPT_SOCKET EQU 83
SYS_CONNECT_SOCKET EQU 84
SYS_SEND_SOCKET EQU 85
SYS_RECV_SOCKET EQU 86
SYS_SHUTDOWN_SOCKET EQU 87
SYS_SEND_TO_SOCKET EQU 88
SYS_RECV_FROM_SOCKET EQU 89
SYS_RESOLVE_HOST EQU 90
SYS_SET_SOCKET_OPTION EQU 91
SYS_GET_SOCKET_OPTION EQU 92
SYS_POLL EQU 93
SYS_GET_MEMORY_STATUS EQU 94
SYS_QUERY_KERNEL_STATS EQU 95
SYS_GET_SYSTEM_TIME EQU 96
SYS_CREATE_FILE_MAPPING EQU 97
SYS_MAP_FILE EQU 98
SYS_CREATE_SURFACE EQU 99
SYS_MAP_SURFACE EQU 100
SYS_QUERY_SURFACE EQU 101
SYS_PRESENT_SURFACE EQU 102
SYS_GET_THREAD_LIST EQU 103
SYS_READ_INPUT_EVENT EQU 104
SYS_PEEK_INPUT_EVENT EQU 105
SYS_WAIT_MANY EQU 106
SYS_OPEN_INPUT_QUEUE EQU 107
SYS_FLUSH_FILE EQU 108
SYS_MAX EQU 109
code/user/syscall/syscall_stubs.asm
+36 -7
View File
@@ -12,7 +12,8 @@
; User-mode callers follow standard MS x64: args 1-4 in RCX/RDX/R8/R9,
; args 5+ on the stack above the home area ([rsp+28h], [rsp+30h], ...).
;
; SYSCALL_STUB2 - 4 or fewer args: pure register passing, no stack read.
; SYSCALL_STUB2/3 - 4 or fewer args: pure register passing, no stack read.
; SYSCALL_STUB5 - exactly 5 args: loads [rsp+28h] -> R10 only.
; SYSCALL_STUB6 - 5 or 6 args: loads [rsp+28h] -> R10 and [rsp+30h] -> R11.
;
; Handle lifecycle:
@@ -44,6 +45,27 @@ Sys&name PROC EXPORT
Sys&name ENDP
ENDM
; ---------------------------------------------------------------------------
SYSCALL_STUB3 MACRO name, nr
PUBLIC Sys&name
Sys&name PROC EXPORT
mov rax, nr
int 40h
ret
Sys&name ENDP
ENDM
; ---------------------------------------------------------------------------
SYSCALL_STUB5 MACRO name, nr
PUBLIC Sys&name
Sys&name PROC EXPORT
mov r10, [rsp+28h] ; 5th argument
mov rax, nr
int 40h
ret
Sys&name ENDP
ENDM
; ---------------------------------------------------------------------------
SYSCALL_STUB6 MACRO name, nr
PUBLIC Sys&name
@@ -62,14 +84,16 @@ ENDM
SYSCALL_STUB2 ExitProcess, SYS_EXIT_PROCESS
SYSCALL_STUB2 ExitThread, SYS_EXIT_THREAD
SYSCALL_STUB2 CreateProcess, SYS_CREATE_PROCESS
SYSCALL_STUB2 WaitForProcess, SYS_WAIT_FOR_PROCESS
SYSCALL_STUB3 WaitSingle, SYS_WAIT_SINGLE
SYSCALL_STUB2 WaitMany, SYS_WAIT_MANY
SYSCALL_STUB2 OpenInputQueue, SYS_OPEN_INPUT_QUEUE
SYSCALL_STUB2 GetProcessExitCode, SYS_GET_PROCESS_EXIT_CODE
SYSCALL_STUB2 KillProcess, SYS_KILL_PROCESS
SYSCALL_STUB2 GetProcessList, SYS_GET_PROCESS_LIST
SYSCALL_STUB2 GetProcessPid, SYS_GET_PROCESS_PID
SYSCALL_STUB2 GetThreadList, SYS_GET_THREAD_LIST
SYSCALL_STUB2 CreateThread, SYS_CREATE_THREAD
SYSCALL_STUB2 WaitThread, SYS_WAIT_THREAD
SYSCALL_STUB2 GetCurrentTid, SYS_GET_CURRENT_TID
SYSCALL_STUB2 GetCurrentPid, SYS_GET_CURRENT_PID
SYSCALL_STUB2 SetThreadPriority, SYS_SET_THREAD_PRIORITY
@@ -120,19 +144,16 @@ SYSCALL_STUB2 CloseHandle, SYS_CLOSE_HANDLE
; Synchronization objects
; ---------------------------------------------------------------------------
SYSCALL_STUB2 CreateMutex, SYS_CREATE_MUTEX
SYSCALL_STUB2 AcquireMutex, SYS_ACQUIRE_MUTEX
SYSCALL_STUB2 TryAcquireMutex, SYS_TRY_ACQUIRE_MUTEX
SYSCALL_STUB2 ReleaseMutex, SYS_RELEASE_MUTEX
SYSCALL_STUB2 CreateSem, SYS_CREATE_SEM
SYSCALL_STUB2 WaitSem, SYS_WAIT_SEM
SYSCALL_STUB2 TryWaitSem, SYS_TRY_WAIT_SEM
SYSCALL_STUB2 SignalSem, SYS_SIGNAL_SEM
SYSCALL_STUB2 CreateEvent, SYS_CREATE_EVENT
SYSCALL_STUB2 SignalEvent, SYS_SIGNAL_EVENT
SYSCALL_STUB2 ClearEvent, SYS_CLEAR_EVENT
SYSCALL_STUB2 WaitEvent, SYS_WAIT_EVENT
; ---------------------------------------------------------------------------
; Queues and named queues
@@ -159,6 +180,7 @@ SYSCALL_STUB2 SeekFile, SYS_SEEK_FILE
SYSCALL_STUB2 GetFilePos, SYS_GET_FILE_POS
SYSCALL_STUB2 GetFileSize, SYS_GET_FILE_SIZE
SYSCALL_STUB2 TruncateFile, SYS_TRUNCATE_FILE
SYSCALL_STUB2 FlushFile, SYS_FLUSH_FILE
SYSCALL_STUB2 DeleteFile, SYS_DELETE_FILE
SYSCALL_STUB2 RenameFile, SYS_RENAME_FILE
@@ -178,7 +200,10 @@ SYSCALL_STUB2 SetConsoleCursor, SYS_SET_CONSOLE_CURSOR
SYSCALL_STUB6 FillConsoleCharacter, SYS_FILL_CONSOLE_CHARACTER
SYSCALL_STUB6 FillConsoleAttribute, SYS_FILL_CONSOLE_ATTRIBUTE
SYSCALL_STUB6 PutScreenChar, SYS_PUT_SCREEN_CHAR
SYSCALL_STUB6 BlitScreen, SYS_BLIT_SCREEN
SYSCALL_STUB5 CreateSurface, SYS_CREATE_SURFACE
SYSCALL_STUB2 MapSurface, SYS_MAP_SURFACE
SYSCALL_STUB2 QuerySurface, SYS_QUERY_SURFACE
SYSCALL_STUB2 PresentSurface, SYS_PRESENT_SURFACE
SYSCALL_STUB2 ClearScreen, SYS_CLEAR_SCREEN
SYSCALL_STUB2 GetScreenSize, SYS_GET_SCREEN_SIZE
SYSCALL_STUB2 QueryDisplayInfo, SYS_QUERY_DISPLAY_INFO
@@ -191,6 +216,8 @@ SYSCALL_STUB2 IsKeyDown, SYS_IS_KEY_DOWN
SYSCALL_STUB2 ReadKeyboardChar, SYS_READ_KEYBOARD_CHAR
SYSCALL_STUB2 ReadKey, SYS_READ_KEY
SYSCALL_STUB2 GetKeyModifiers, SYS_GET_KEY_MODIFIERS
SYSCALL_STUB2 ReadInputEvent, SYS_READ_INPUT_EVENT
SYSCALL_STUB2 PeekInputEvent, SYS_PEEK_INPUT_EVENT
SYSCALL_STUB2 PollMouse, SYS_POLL_MOUSE
SYSCALL_STUB2 ReadMouse, SYS_READ_MOUSE
@@ -224,5 +251,7 @@ SYSCALL_STUB2 Poll, SYS_POLL
SYSCALL_STUB2 GetMemoryStatus, SYS_GET_MEMORY_STATUS
SYSCALL_STUB2 QueryKernelStats, SYS_QUERY_KERNEL_STATS
SYSCALL_STUB2 GetSystemTime, SYS_GET_SYSTEM_TIME
SYSCALL_STUB5 CreateFileMapping, SYS_CREATE_FILE_MAPPING
SYSCALL_STUB6 MapFile, SYS_MAP_FILE
END
code/user/system/init/init.c
+2 -2
View File
@@ -76,9 +76,9 @@ void _start(void)
}
debug_print("[init] waiting\n");
s = SysWaitForProcess(h);
s = SysWait(h, SYS_WAIT_FLAG_NONE);
if (s != STATUS_SUCCESS)
debug_print("[init] WARNING: SysWaitForProcess returned error\n");
debug_print("[init] WARNING: SysWait returned error\n");
int exitCode = 0;
if (SysGetProcessExitCode(h, &exitCode) != STATUS_SUCCESS) {
code/user/tests/async_apc/async_apc.c
+36 -6
View File
@@ -15,10 +15,35 @@ static void print(const char* s) {
SysWriteConsole(&req);
}
// Opens file.
static uint32_t open_file(const char* path, uint32_t* handleOut) {
SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), handleOut };
return SysOpenFile(&req);
// Opens the first available executable payload for async I/O.
// Tests are usually launched from /apps, so a bare "system/init.exe"
// resolves as /apps/system/init.exe and fails. Prefer the test binary itself,
// then fall back to absolute/drive-qualified paths.
static uint32_t open_test_file(uint32_t* handleOut, const char** pathOut) {
static const char* paths[] = {
"async_apc.exe",
"/apps/async_apc.exe",
"1:/apps/async_apc.exe",
"0:/apps/async_apc.exe",
"/system/init.exe",
"0:/system/init.exe",
"1:/system/init.exe",
"system/init.exe",
};
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
uint32_t h = 0;
SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h };
uint32_t s = SysOpenFile(&req);
if (s == SYS_STATUS_SUCCESS) {
*handleOut = h;
if (pathOut)
*pathOut = paths[i];
return SYS_STATUS_SUCCESS;
}
}
return SYS_STATUS_NOT_FOUND;
}
// Prints hex8.
@@ -74,7 +99,8 @@ int main(void) {
}
uint32_t file = 0;
s = open_file("system/init.exe", &file);
const char* openedPath = 0;
s = open_test_file(&file, &openedPath);
if (s != SYS_STATUS_SUCCESS) {
print("async_apc: open failed status=");
print_u64(s);
@@ -82,6 +108,10 @@ int main(void) {
SysExitProcess(1);
}
print("async_apc: using ");
print(openedPath ? openedPath : "<unknown>");
print("\n");
uint32_t event = 0;
s = SysCreateEvent(EVENT_AUTO_RESET, 0, &event);
if (s != SYS_STATUS_SUCCESS) {
@@ -125,7 +155,7 @@ int main(void) {
}
while (!state.Delivered) {
s = SysWaitEvent(event, 1);
s = SysWait(event, SYS_WAIT_FLAG_ALERTABLE);
print("async_apc: alertable wait returned ");
print_u64(s);
print("\n");
code/user/tests/async_cancel/async_cancel.c
+36 -6
View File
@@ -14,10 +14,35 @@ static void print(const char* s) {
SysWriteConsole(&req);
}
// Opens file.
static uint32_t open_file(const char* path, uint32_t* handleOut) {
SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), handleOut };
return SysOpenFile(&req);
// Opens the first available executable payload for async I/O.
// Tests are usually launched from /apps, so a bare "system/init.exe"
// resolves as /apps/system/init.exe and fails. Prefer the test binary itself,
// then fall back to absolute/drive-qualified paths.
static uint32_t open_test_file(uint32_t* handleOut, const char** pathOut) {
static const char* paths[] = {
"async_cancel.exe",
"/apps/async_cancel.exe",
"1:/apps/async_cancel.exe",
"0:/apps/async_cancel.exe",
"/system/init.exe",
"0:/system/init.exe",
"1:/system/init.exe",
"system/init.exe",
};
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
uint32_t h = 0;
SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h };
uint32_t s = SysOpenFile(&req);
if (s == SYS_STATUS_SUCCESS) {
*handleOut = h;
if (pathOut)
*pathOut = paths[i];
return SYS_STATUS_SUCCESS;
}
}
return SYS_STATUS_NOT_FOUND;
}
// Prints u64.
@@ -46,12 +71,17 @@ int main(void) {
uint32_t event = 0;
uint32_t request = 0;
uint32_t s = open_file("system/init.exe", &file);
const char* openedPath = 0;
uint32_t s = open_test_file(&file, &openedPath);
if (s != SYS_STATUS_SUCCESS) {
print("async_cancel: open failed status="); print_u64(s); print("\n");
SysExitProcess(1);
}
print("async_cancel: using ");
print(openedPath ? openedPath : "<unknown>");
print("\n");
s = SysCreateEvent(EVENT_AUTO_RESET, 0, &event);
if (s != SYS_STATUS_SUCCESS) {
print("async_cancel: event create failed status="); print_u64(s); print("\n");
@@ -90,7 +120,7 @@ int main(void) {
print("async_cancel: cancel returned "); print_u64(cs); print("\n");
while (iosb.Status == SYS_STATUS_PENDING) {
s = SysWaitEvent(event, 1);
s = SysWait(event, SYS_WAIT_FLAG_ALERTABLE);
print("async_cancel: wait returned "); print_u64(s); print("\n");
if (s != SYS_STATUS_SUCCESS && s != SYS_STATUS_KERNEL_APC && s != SYS_STATUS_USER_APC && s != SYS_STATUS_ALERTED)
break;
code/user/tests/dirty_writeback/dirty_writeback.c
+227
View File
@@ -0,0 +1,227 @@
// Dirty file-cache writeback regression test.
// Verifies buffered writes dirty cache pages, explicit flush writes them back, and close flushes synchronously.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <ntlite/syscall.h>
#define PAGE_SIZE_BYTES 4096u
#define TEST_SIZE (PAGE_SIZE_BYTES * 4u)
#define REWRITE_SIZE (PAGE_SIZE_BYTES * 2u)
#define CLOSE_SIZE (PAGE_SIZE_BYTES * 3u)
static uint8_t gPatternA[TEST_SIZE];
static uint8_t gPatternB[TEST_SIZE];
static uint8_t gReadback[TEST_SIZE];
static void print(const char* s) {
SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) };
SysWriteConsole(&req);
}
static void print_u64(uint64_t v) {
char buf[32];
int i = 0;
if (v == 0) {
print("0");
return;
}
while (v && i < (int)sizeof(buf)) {
buf[i++] = (char)('0' + (v % 10));
v /= 10;
}
while (i--) {
char c[2] = { buf[i], 0 };
print(c);
}
}
static void exit_now(int code) {
for (;;) SysExitProcess(code);
}
static void fail(const char* msg) {
print("dirty_writeback: ");
print(msg);
print("\n");
exit_now(1);
}
static void fail_status(const char* what, uint32_t status) {
print("dirty_writeback: ");
print(what);
print(" failed status=");
print_u64(status);
print("\n");
exit_now(1);
}
static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) {
memset(out, 0, sizeof(*out));
return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0);
}
static void delete_path(const char* path) {
SYS_PATH_REQUEST req = { SysSvFromCstr(path) };
(void)SysDeleteFile(&req);
}
static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) {
static const char* paths[] = {
"dirty_writeback.tmp",
"/dirty_writeback.tmp",
"1:/dirty_writeback.tmp",
"0:/dirty_writeback.tmp",
};
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
uint32_t h = 0;
delete_path(paths[i]);
SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(paths[i]), &h };
uint32_t s = SysCreateFile(&req);
if (s == SYS_STATUS_SUCCESS) {
*fileOut = h;
if (pathOut) *pathOut = paths[i];
return SYS_STATUS_SUCCESS;
}
}
return SYS_STATUS_NOT_FOUND;
}
static uint32_t open_path(const char* path, uint32_t* fileOut) {
uint32_t h = 0;
SYS_PATH_HANDLE_REQUEST req = { SysSvFromCstr(path), &h };
uint32_t s = SysOpenFile(&req);
if (s == SYS_STATUS_SUCCESS)
*fileOut = h;
return s;
}
static void fill_patterns(void) {
for (size_t i = 0; i < sizeof(gPatternA); ++i) {
gPatternA[i] = (uint8_t)(0x11u + (uint8_t)(i * 13u + (i >> 6)));
gPatternB[i] = (uint8_t)(0x80u + (uint8_t)(i * 31u + (i >> 4)));
}
}
static void checked_write(uint32_t file, const void* buffer, size_t size, const char* what) {
size_t done = 0;
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &done);
uint32_t s = SysWriteFile(&req);
if (s != SYS_STATUS_SUCCESS) fail_status(what, s);
if (done != size) fail("short write");
}
static void checked_read(uint32_t file, void* buffer, size_t size, size_t* gotOut, const char* what) {
size_t got = 0;
memset(buffer, 0xCC, size);
SYS_FILE_IO_REQUEST req = SysMakeFileIoRequest(file, buffer, size, &got);
uint32_t s = SysReadFile(&req);
if (s != SYS_STATUS_SUCCESS) fail_status(what, s);
if (gotOut) *gotOut = got;
}
static void checked_seek(uint32_t file, uint64_t offset, const char* what) {
uint32_t s = SysSeekFile(file, offset);
if (s != SYS_STATUS_SUCCESS) fail_status(what, s);
}
static void expect_writeback_progress(const SYS_FILE_CACHE_STATS* before,
const SYS_FILE_CACHE_STATS* after,
const char* what) {
if (after->FlushFileFailures != before->FlushFileFailures)
fail("flush failure counter increased");
if (after->WritebackFailures != before->WritebackFailures)
fail("writeback failure counter increased");
if (after->WritebackBytes <= before->WritebackBytes) {
print("dirty_writeback: ");
print(what);
print(" did not write back bytes\n");
exit_now(1);
}
}
int main(void) {
print("dirty_writeback: start\n");
fill_patterns();
uint32_t file = 0;
const char* path = 0;
uint32_t s = create_temp(&file, &path);
if (s != SYS_STATUS_SUCCESS) fail_status("create temp", s);
print("dirty_writeback: using ");
print(path ? path : "<unknown>");
print("\n");
SYS_FILE_CACHE_STATS before;
SYS_FILE_CACHE_STATS afterWrite;
SYS_FILE_CACHE_STATS afterFlush;
SYS_FILE_CACHE_STATS beforeClose;
SYS_FILE_CACHE_STATS afterClose;
if (query_cache(&before) != SYS_STATUS_SUCCESS) fail("initial stats query failed");
checked_write(file, gPatternA, sizeof(gPatternA), "write pattern A");
if (query_cache(&afterWrite) != SYS_STATUS_SUCCESS) fail("post-write stats query failed");
if (afterWrite.DirtyMarks <= before.DirtyMarks) fail("write did not mark dirty cache pages");
if (afterWrite.DirtyPages <= before.DirtyPages) fail("write did not leave dirty pages resident");
checked_seek(file, 0, "seek readback before flush");
size_t got = 0;
checked_read(file, gReadback, sizeof(gReadback), &got, "read before flush");
if (got != sizeof(gPatternA)) fail("read before flush size mismatch");
if (memcmp(gReadback, gPatternA, sizeof(gPatternA)) != 0) fail("cached read before flush mismatch");
s = SysFlushFile(file);
if (s != SYS_STATUS_SUCCESS) fail_status("explicit flush", s);
if (query_cache(&afterFlush) != SYS_STATUS_SUCCESS) fail("post-flush stats query failed");
if (afterFlush.FlushFileCalls <= afterWrite.FlushFileCalls) fail("flush call counter did not increase");
expect_writeback_progress(&afterWrite, &afterFlush, "explicit flush");
checked_seek(file, 0, "seek rewrite");
checked_write(file, gPatternB, REWRITE_SIZE, "rewrite pattern B");
checked_seek(file, REWRITE_SIZE, "seek close-tail");
checked_write(file, gPatternA + REWRITE_SIZE, CLOSE_SIZE - REWRITE_SIZE, "write close tail");
if (query_cache(&beforeClose) != SYS_STATUS_SUCCESS) fail("pre-close stats query failed");
if (beforeClose.DirtyMarks <= afterFlush.DirtyMarks) fail("rewrite did not mark dirty pages");
s = SysCloseHandle(file);
file = 0;
if (s != SYS_STATUS_SUCCESS) fail_status("close", s);
if (query_cache(&afterClose) != SYS_STATUS_SUCCESS) fail("post-close stats query failed");
if (afterClose.FlushFileCalls <= beforeClose.FlushFileCalls) fail("close did not flush file");
expect_writeback_progress(&beforeClose, &afterClose, "close flush");
s = open_path(path, &file);
if (s != SYS_STATUS_SUCCESS) fail_status("reopen", s);
checked_read(file, gReadback, CLOSE_SIZE, &got, "read after reopen");
if (got != CLOSE_SIZE) fail("read after reopen size mismatch");
if (memcmp(gReadback, gPatternB, REWRITE_SIZE) != 0) fail("rewrite data not durable after close");
if (memcmp(gReadback + REWRITE_SIZE, gPatternA + REWRITE_SIZE,
CLOSE_SIZE - REWRITE_SIZE) != 0) fail("tail data not durable after close");
s = SysCloseHandle(file);
file = 0;
if (s != SYS_STATUS_SUCCESS) fail_status("close reopened", s);
delete_path(path);
print("dirty_writeback: dirty-marks+=");
print_u64(afterClose.DirtyMarks - before.DirtyMarks);
print(" writeback-bytes+=");
print_u64(afterClose.WritebackBytes - before.WritebackBytes);
print(" flushes+=");
print_u64(afterClose.FlushFileCalls - before.FlushFileCalls);
print("\n");
print("dirty_writeback: PASS\n");
exit_now(0);
return 0;
}
Loaded 100 of 114 files, more files were not shown because too many files have changed in this diff. Show more