maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace the VHD/diskpart disk image pipeline with a directory-based vvfat root, we'll see when it breaks, fix /O2, add an all_tests, generalize the fat32 driver into a fat 16+32 driver, change policy on volume drivers, collapse KCPU's fields into a single atomic KE_CPU_STATE state machine, move scheduler ticking to per-CPU LAPIC timers, gate verbose SMP topology logging behind selftest diagnostics and add SMP pool-checkpoint validation around AP admission, add checked intrusive list helpers behind a new KE_DEBUG_LIST_CHECKS flag, add ObInitializeObjectHeader/ObTryReferenceObject, add waitable display clock objects and a SYS_OPEN_DISPLAY_CLOCK syscall for vsync-style waits, refactor kernel input into a per-queue KE_INPUT_CONTEXT with its own key/mouse ring buffers and DPC-driven key repeat, add KeInputQueuePeekEvent/ReadEvent/WaitKey and a SYS_QUERY_GLOBAL_INPUT_STATE syscall, split the kernel pool allocator out of mm_virt, new video files!

Browse Source
This commit is contained in:
maksw committed 2026-07-12 00:10:21 +02:00
1 parent 485a6bbfe7
commit 8b254f84f4
74 files changed
+4488 -2503
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/boot.ini
+1 -1
View File
@@ -1,5 +1,5 @@
[kernel]
; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file
; bitmask: 1 = debug prints, 2 = self check, 4 = file log, 4096 = checked intrusive lists
debug = 0xffff
[init]
commandline = apps/shell_native.exe
code/build/build.c
+100 -151
View File
@@ -6,7 +6,7 @@
// Compiler and linker flags stay in macros so they can be used in file-scope aggregate initializers.
// MSVC C rejects initializing a static object from another object, even when the source object is const.
// Drivers and user binaries do not inherit kernel include paths so private dependencies fail at compile time.
#define COMMON_CFLAGS_LITERAL "/nologo /W3 /WX /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi- /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include"
#define COMMON_CFLAGS_LITERAL "/nologo /W3 /WX /O2 /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi- /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include"
#define KERNEL_CFLAGS_LITERAL COMMON_CFLAGS_LITERAL " /Icode\\kernel\\include"
#define DRIVER_CFLAGS_LITERAL COMMON_CFLAGS_LITERAL
#define USER_CFLAGS_LITERAL COMMON_CFLAGS_LITERAL
@@ -48,7 +48,7 @@ static const Driver_Target STANDALONE_DRIVERS[] = {
DRIVER_TARGET("serial", "obj\\drivers\\serial", "code\\drivers\\input\\serial\\serial.c"),
DRIVER_TARGET_LIBS("ata", "obj\\drivers\\ata", "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"),
DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
DRIVER_TARGET("fat", "obj\\drivers\\fat", "code\\drivers\\fs\\fat\\fat.c code\\drivers\\fs\\fat\\fat_mount.c"),
DRIVER_TARGET("ntfs", "obj\\drivers\\ntfs", "code\\drivers\\fs\\ntfs\\ntfs.c"),
DRIVER_TARGET_LIBS("vmwaresvga", "obj\\drivers\\vmwaresvga", "code\\drivers\\display\\vmwaresvga\\vmwaresvga.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_bus.c code\\drivers\\bus\\usb\\usb_composite.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c code\\drivers\\bus\\usb\\usb_hub.c", "bin\\pci.lib"),
@@ -66,7 +66,7 @@ void create_layout() {
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop",
"obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke"
"obj\\all_tests", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke"
};
for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) nob_mkdir_if_not_exists(dirs[i]);
nob_mkdir_if_not_exists(PCI_BUS_DRIVER.obj_dir);
@@ -91,67 +91,32 @@ Nob_Proc run_async(const char *cmd_str) {
return proc;
}
// Deletes stale object files from a target directory before compiling into it.
// Link commands use directory wildcards, so removed translation units can otherwise
// survive as old .obj files and keep unresolved references alive.
bool clean_object_directory(const char *obj_dir) {
const char *cmd = nob_temp_sprintf("if exist %s\\*.obj del /q %s\\*.obj", obj_dir, obj_dir);
return run_sync(cmd);
// Cleans all object output by replacing obj/ as a whole.
// Only obj/ is disposable. fatroot/ and usbfatroot/ are developer-owned vvfat roots.
bool clean_build_object_directories(void) {
const char *cmd = "if exist obj rmdir /s /q obj";
if (!run_sync(cmd)) return false;
return nob_mkdir_if_not_exists("obj");
}
// Cleans object directories that are linked through wildcards.
// The build is intentionally simple and does not maintain per-file dependency state,
// so stale objects are more dangerous than recompiling a few extra files.
bool clean_build_object_directories(void) {
// Creates the FAT root layout used by QEMU vvfat without deleting existing files.
// The vvfat root is also a convenient developer workspace, so packaging must be non-destructive.
bool create_fatroot_layout(void) {
const char *dirs[] = {
"obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
"obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix",
"obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping",
"obj\\doom", "obj\\video", "obj\\memtop", "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke"
"fatroot\\EFI", "fatroot\\EFI\\BOOT",
"fatroot\\system", "fatroot\\drivers", "fatroot\\apps", "fatroot\\tests"
};
for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) {
if (!clean_object_directory(dirs[i])) return false;
}
if (!clean_object_directory(PCI_BUS_DRIVER.obj_dir)) return false;
for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) {
if (!clean_object_directory(STANDALONE_DRIVERS[i].obj_dir)) return false;
if (!nob_mkdir_if_not_exists(dirs[i])) return false;
}
return true;
}
// Creates an empty USB image the first time the build sees a missing usb.vhd.
// The image is intentionally left unmounted and unpopulated after formatting so
// callers can use it as removable FAT32 storage without the build overwriting it.
bool create_usb_vhd_if_missing(void) {
if (nob_file_exists("usb.vhd")) return true;
char vhd_path[MAX_PATH];
GetFullPathNameA("usb.vhd", MAX_PATH, vhd_path, NULL);
const char *diskpart_script = nob_temp_sprintf(
"create vdisk file=\"%s\" maximum=256 type=fixed\n"
"select vdisk file=\"%s\"\n"
"attach vdisk\n"
"convert gpt\n"
"create partition primary\n"
"format fs=ntfs quick\n"
"detach vdisk\n"
"exit\n",
vhd_path, vhd_path
);
nob_log(NOB_INFO, "usb.vhd missing, creating empty 256 MB FAT32 GPT image...");
nob_write_entire_file("usb_mkimage.txt", diskpart_script, strlen(diskpart_script));
bool ok = run_sync("diskpart /s usb_mkimage.txt");
remove("usb_mkimage.txt");
return ok;
// Creates the USB vvfat root once and leaves its contents alone between builds.
bool create_usbfatroot_if_missing(void) {
return nob_mkdir_if_not_exists("usbfatroot");
}
// Runs the requested operation.
@@ -159,14 +124,9 @@ int main(int argc, char **argv) {
// The build tool rebuilds itself before doing any real work so the graph below matches the source.
NOB_GO_REBUILD_URSELF(argc, argv);
// The boot image must exist before the build can copy fresh binaries into it.
if (!nob_file_exists("disk.vhd")) {
nob_log(NOB_INFO, "disk.vhd missing, calling mkimage.bat...");
if (!run_sync("mkimage.bat")) return 1;
}
if (!create_usb_vhd_if_missing()) return 1;
create_layout();
if (!clean_build_object_directories()) return 1;
create_layout();
if (!create_usbfatroot_if_missing()) return 1;
// Core images build sequentially because later images link against earlier import libraries.
nob_log(NOB_INFO, "--- Building Core Subsystems ---");
@@ -250,6 +210,7 @@ int main(int argc, char **argv) {
const char *apps[][4] = {
// Folder, Libs, Name, Optional Extra Link Args.
{"system\\init", NATIVE_INIT_LIBS_LITERAL, "init", "/ENTRY:_start"},
{"tests\\all_tests", NATIVE_EXE_LIBS_LITERAL, "all_tests", NATIVE_EXE_ENTRY_LITERAL},
{"apps\\shell_win32", WIN32_EXE_LIBS_LITERAL, "shell_win32", NATIVE_EXE_ENTRY_LITERAL},
{"apps\\shell_posix", POSIX_EXE_LIBS_LITERAL, "shell_posix", "/ENTRY:PosixStart"},
{"apps\\shell_native", NATIVE_EXE_LIBS_LITERAL, "shell_native", NATIVE_EXE_ENTRY_LITERAL},
@@ -315,103 +276,91 @@ int main(int argc, char **argv) {
return 1;
}
// 4. Mounting & Packing Image.
nob_log(NOB_INFO, "--- Mounting disk.vhd ---");
// Resolve the absolute path to disk.vhd for diskpart.
char vhd_path[MAX_PATH];
GetFullPathNameA("disk.vhd", MAX_PATH, vhd_path, NULL);
const char *diskpart_mnt = nob_temp_sprintf(
"select vdisk file=\"%s\"\nattach vdisk\n"
"select partition 1\nformat fs=fat32 quick\nassign letter=Z\n"
"select partition 2\nformat fs=fat32 quick\nassign letter=Y\nexit\n",
vhd_path
);
nob_write_entire_file("mnt.txt", diskpart_mnt, strlen(diskpart_mnt));
if (!run_sync("diskpart /s mnt.txt")) return 1;
nob_log(NOB_INFO, "--- Assembling ESP ---");
_mkdir("Z:\\apps"); _mkdir("Y:\\apps");
_mkdir("Z:\\system"); _mkdir("Z:\\drivers");
_mkdir("Z:\\EFI"); _mkdir("Z:\\EFI\\BOOT");
// 4. Packing vvfat root.
nob_log(NOB_INFO, "--- Assembling fatroot ---");
if (!create_fatroot_layout()) return 1;
const char *copies[][2] = {
{"bin\\bootx64.efi", "Z:\\EFI\\BOOT\\bootx64.efi"}, {"bin\\kernel.exe", "Z:\\system\\colonel.exe"},
{"bin\\hal.dll", "Z:\\system\\hal.dll"}, {"bin\\syscall.dll", "Z:\\system\\syscall.dll"},
{"bin\\win32.dll", "Z:\\system\\win32.dll"}, {"bin\\posix.dll", "Z:\\system\\posix.dll"},
{"code\\settings.ini", "Z:\\system\\settings.ini"}, {"bin\\init.exe", "Z:\\system\\init.exe"},
{"bin\\pci.sys", "Z:\\drivers\\pci.sys"}, {"bin\\ps2.sys", "Z:\\drivers\\ps2.sys"},
{"bin\\serial.sys", "Z:\\drivers\\serial.sys"},
{"bin\\ata.sys", "Z:\\drivers\\ata.sys"}, {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"},
{"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"}, {"bin\\ntfs.sys", "Z:\\drivers\\ntfs.sys"},
{"bin\\vmwaresvga.sys", "Z:\\drivers\\vmwaresvga.sys"},
{"bin\\lwip.sys", "Z:\\drivers\\lwip.sys"}, {"bin\\usb.sys", "Z:\\drivers\\usb.sys"},
{"bin\\rtl816x.sys", "Z:\\drivers\\rtl816x.sys"},
{"bin\\rtl8139.sys", "Z:\\drivers\\rtl8139.sys"}, {"code\\drivers\\drivers.ini", "Z:\\drivers\\drivers.ini"},
{"code\\boot.ini", "Z:\\boot.ini"},
{"bin\\bootx64.efi", "fatroot\\EFI\\BOOT\\bootx64.efi"},
{"bin\\kernel.exe", "fatroot\\system\\colonel.exe"},
{"bin\\hal.dll", "fatroot\\system\\hal.dll"},
{"bin\\syscall.dll", "fatroot\\system\\syscall.dll"},
{"bin\\win32.dll", "fatroot\\system\\win32.dll"},
{"bin\\posix.dll", "fatroot\\system\\posix.dll"},
{"code\\settings.ini", "fatroot\\system\\settings.ini"},
{"bin\\init.exe", "fatroot\\system\\init.exe"},
{"bin\\shell_win32.exe", "Z:\\apps\\shell_win32.exe"}, {"bin\\shell_posix.exe", "Z:\\apps\\shell_posix.exe"},
{"bin\\shell_native.exe", "Z:\\apps\\shell_native.exe"}, {"bin\\edit.exe", "Z:\\apps\\edit.exe"},
{"bin\\httpget.exe", "Z:\\apps\\httpget.exe"}, {"bin\\tcpserve.exe", "Z:\\apps\\tcpserve.exe"},
{"bin\\udp.exe", "Z:\\apps\\udp.exe"}, {"bin\\resolve.exe", "Z:\\apps\\resolve.exe"},
{"bin\\ping.exe", "Z:\\apps\\ping.exe"}, {"bin\\doom.exe", "Z:\\apps\\doom.exe"},
{"bin\\video.exe", "Z:\\apps\\video.exe"}, {"bin\\memtop.exe", "Z:\\apps\\memtop.exe"},
{"bin\\pci.sys", "fatroot\\drivers\\pci.sys"},
{"bin\\ps2.sys", "fatroot\\drivers\\ps2.sys"},
{"bin\\serial.sys", "fatroot\\drivers\\serial.sys"},
{"bin\\ata.sys", "fatroot\\drivers\\ata.sys"},
{"bin\\ahci.sys", "fatroot\\drivers\\ahci.sys"},
{"bin\\fat.sys", "fatroot\\drivers\\fat.sys"},
{"bin\\ntfs.sys", "fatroot\\drivers\\ntfs.sys"},
{"bin\\vmwaresvga.sys", "fatroot\\drivers\\vmwaresvga.sys"},
{"bin\\lwip.sys", "fatroot\\drivers\\lwip.sys"},
{"bin\\usb.sys", "fatroot\\drivers\\usb.sys"},
{"bin\\rtl816x.sys", "fatroot\\drivers\\rtl816x.sys"},
{"bin\\rtl8139.sys", "fatroot\\drivers\\rtl8139.sys"},
{"code\\drivers\\drivers.ini", "fatroot\\drivers\\drivers.ini"},
{"code\\boot.ini", "fatroot\\boot.ini"},
{"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"}, {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"},
{"bin\\apc_return_smoke.exe", "Y:\\apps\\apc_return_smoke.exe"},
{"bin\\file_stat_smoke.exe", "Y:\\apps\\file_stat_smoke.exe"},
{"bin\\handle_dup_smoke.exe", "Y:\\apps\\handle_dup_smoke.exe"},
{"bin\\named_queue_lifetime_smoke.exe", "Y:\\apps\\named_queue_lifetime_smoke.exe"},
{"bin\\input_api_smoke.exe", "Y:\\apps\\input_api_smoke.exe"},
{"bin\\input_model_smoke.exe", "Y:\\apps\\input_model_smoke.exe"},
{"bin\\timer_smoke.exe", "Y:\\apps\\timer_smoke.exe"},
{"bin\\stats_abi_smoke.exe", "Y:\\apps\\stats_abi_smoke.exe"},
{"bin\\realpath_smoke.exe", "Y:\\apps\\realpath_smoke.exe"},
{"bin\\file_lock_smoke.exe", "Y:\\apps\\file_lock_smoke.exe"},
{"bin\\process_protection_smoke.exe", "Y:\\apps\\process_protection_smoke.exe"},
{"bin\\socket_addr_smoke.exe", "Y:\\apps\\socket_addr_smoke.exe"},
{"bin\\net_irp_smoke.exe", "Y:\\apps\\net_irp_smoke.exe"},
{"bin\\module_load_smoke.exe", "Y:\\apps\\module_load_smoke.exe"},
{"bin\\posix_module_smoke.exe", "Y:\\apps\\posix_module_smoke.exe"},
{"bin\\exception_dispatcher_smoke.exe", "Y:\\apps\\exception_dispatcher_smoke.exe"},
{"bin\\native_exception_runtime_smoke.exe", "Y:\\apps\\native_exception_runtime_smoke.exe"},
{"bin\\native_raise_exception_smoke.exe", "Y:\\apps\\native_raise_exception_smoke.exe"},
{"bin\\exception_smoke.exe", "Y:\\apps\\exception_smoke.exe"},
{"bin\\win32_exception_smoke.exe", "Y:\\apps\\win32_exception_smoke.exe"},
{"bin\\win32_seh_smoke.exe", "Y:\\apps\\win32_seh_smoke.exe"},
{"bin\\syscall_contract_smoke.exe", "Y:\\apps\\syscall_contract_smoke.exe"},
{"bin\\would_block_smoke.exe", "Y:\\apps\\would_block_smoke.exe"},
{"bin\\cwd_peb_smoke.exe", "Y:\\apps\\cwd_peb_smoke.exe"},
{"bin\\mmap_smoke.exe", "Y:\\apps\\mmap_smoke.exe"}, {"bin\\pcache_smoke.exe", "Y:\\apps\\pcache_smoke.exe"},
{"bin\\pcache_pressure.exe", "Y:\\apps\\pcache_pressure.exe"},
{"bin\\dirty_writeback.exe", "Y:\\apps\\dirty_writeback.exe"},
{"bin\\shared_mmap.exe", "Y:\\apps\\shared_mmap.exe"},
{"bin\\pcache_invalidate.exe", "Y:\\apps\\pcache_invalidate.exe"},
{"bin\\pcache_mmap_read_reuse.exe", "Y:\\apps\\pcache_mmap_read_reuse.exe"},
{"bin\\pcache_multithread.exe", "Y:\\apps\\pcache_multithread.exe"},
{"bin\\surface_smoke.exe", "Y:\\apps\\surface_smoke.exe"},
{"bin\\surface_invalid_rect.exe", "Y:\\apps\\surface_invalid_rect.exe"},
{"bin\\surface_multithread_present.exe", "Y:\\apps\\surface_multithread_present.exe"}
{"bin\\shell_win32.exe", "fatroot\\apps\\shell_win32.exe"},
{"bin\\shell_posix.exe", "fatroot\\apps\\shell_posix.exe"},
{"bin\\shell_native.exe", "fatroot\\apps\\shell_native.exe"},
{"bin\\edit.exe", "fatroot\\apps\\edit.exe"},
{"bin\\httpget.exe", "fatroot\\apps\\httpget.exe"},
{"bin\\tcpserve.exe", "fatroot\\apps\\tcpserve.exe"},
{"bin\\udp.exe", "fatroot\\apps\\udp.exe"},
{"bin\\resolve.exe", "fatroot\\apps\\resolve.exe"},
{"bin\\ping.exe", "fatroot\\apps\\ping.exe"},
{"bin\\doom.exe", "fatroot\\apps\\doom.exe"},
{"bin\\video.exe", "fatroot\\apps\\video.exe"},
{"bin\\memtop.exe", "fatroot\\apps\\memtop.exe"},
{"bin\\all_tests.exe", "fatroot\\tests\\all_tests.exe"},
{"bin\\async_apc.exe", "fatroot\\tests\\async_apc.exe"},
{"bin\\async_cancel.exe", "fatroot\\tests\\async_cancel.exe"},
{"bin\\apc_return_smoke.exe", "fatroot\\tests\\apc_return_smoke.exe"},
{"bin\\file_stat_smoke.exe", "fatroot\\tests\\file_stat_smoke.exe"},
{"bin\\handle_dup_smoke.exe", "fatroot\\tests\\handle_dup_smoke.exe"},
{"bin\\named_queue_lifetime_smoke.exe", "fatroot\\tests\\named_queue_lifetime_smoke.exe"},
{"bin\\input_api_smoke.exe", "fatroot\\tests\\input_api_smoke.exe"},
{"bin\\input_model_smoke.exe", "fatroot\\tests\\input_model_smoke.exe"},
{"bin\\timer_smoke.exe", "fatroot\\tests\\timer_smoke.exe"},
{"bin\\stats_abi_smoke.exe", "fatroot\\tests\\stats_abi_smoke.exe"},
{"bin\\realpath_smoke.exe", "fatroot\\tests\\realpath_smoke.exe"},
{"bin\\file_lock_smoke.exe", "fatroot\\tests\\file_lock_smoke.exe"},
{"bin\\process_protection_smoke.exe", "fatroot\\tests\\process_protection_smoke.exe"},
{"bin\\socket_addr_smoke.exe", "fatroot\\tests\\socket_addr_smoke.exe"},
{"bin\\net_irp_smoke.exe", "fatroot\\tests\\net_irp_smoke.exe"},
{"bin\\module_load_smoke.exe", "fatroot\\tests\\module_load_smoke.exe"},
{"bin\\posix_module_smoke.exe", "fatroot\\tests\\posix_module_smoke.exe"},
{"bin\\exception_dispatcher_smoke.exe", "fatroot\\tests\\exception_dispatcher_smoke.exe"},
{"bin\\native_exception_runtime_smoke.exe", "fatroot\\tests\\native_exception_runtime_smoke.exe"},
{"bin\\native_raise_exception_smoke.exe", "fatroot\\tests\\native_raise_exception_smoke.exe"},
{"bin\\exception_smoke.exe", "fatroot\\tests\\exception_smoke.exe"},
{"bin\\win32_exception_smoke.exe", "fatroot\\tests\\win32_exception_smoke.exe"},
{"bin\\win32_seh_smoke.exe", "fatroot\\tests\\win32_seh_smoke.exe"},
{"bin\\syscall_contract_smoke.exe", "fatroot\\tests\\syscall_contract_smoke.exe"},
{"bin\\would_block_smoke.exe", "fatroot\\tests\\would_block_smoke.exe"},
{"bin\\cwd_peb_smoke.exe", "fatroot\\tests\\cwd_peb_smoke.exe"},
{"bin\\mmap_smoke.exe", "fatroot\\tests\\mmap_smoke.exe"},
{"bin\\pcache_smoke.exe", "fatroot\\tests\\pcache_smoke.exe"},
{"bin\\pcache_pressure.exe", "fatroot\\tests\\pcache_pressure.exe"},
{"bin\\dirty_writeback.exe", "fatroot\\tests\\dirty_writeback.exe"},
{"bin\\shared_mmap.exe", "fatroot\\tests\\shared_mmap.exe"},
{"bin\\pcache_invalidate.exe", "fatroot\\tests\\pcache_invalidate.exe"},
{"bin\\pcache_mmap_read_reuse.exe", "fatroot\\tests\\pcache_mmap_read_reuse.exe"},
{"bin\\pcache_multithread.exe", "fatroot\\tests\\pcache_multithread.exe"},
{"bin\\surface_smoke.exe", "fatroot\\tests\\surface_smoke.exe"},
{"bin\\surface_invalid_rect.exe", "fatroot\\tests\\surface_invalid_rect.exe"},
{"bin\\surface_multithread_present.exe", "fatroot\\tests\\surface_multithread_present.exe"}
};
for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) nob_copy_file(copies[i][0], copies[i][1]);
if (nob_file_exists("code\\user\\apps\\video\\bink2w64.dll")) nob_copy_file("code\\user\\apps\\video\\bink2w64.dll", "Z:\\apps\\bink2w64.dll");
if (nob_file_exists("code\\user\\apps\\video\\video.bk2")) nob_copy_file("code\\user\\apps\\video\\video.bk2", "Z:\\apps\\video.bk2");
if (nob_file_exists("code\\user\\apps\\doom\\doom1.wad")) nob_copy_file("code\\user\\apps\\doom\\doom1.wad", "Z:\\apps\\doom1.wad");
nob_log(NOB_INFO, "--- Unmounting disk.vhd ---");
const char *diskpart_umnt = nob_temp_sprintf(
"select vdisk file=\"%s\"\ndetach vdisk\nexit\n",
vhd_path
);
nob_write_entire_file("umnt.txt", diskpart_umnt, strlen(diskpart_umnt));
run_sync("diskpart /s umnt.txt");
remove("mnt.txt");
remove("umnt.txt");
for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) {
if (!nob_copy_file(copies[i][0], copies[i][1])) return 1;
}
nob_log(NOB_INFO, "Build complete!");
return 0;
code/drivers/drivers.ini
+1 -1
View File
@@ -5,7 +5,7 @@ serial.sys
ahci.sys
ata.sys
usb.sys
fat32.sys
fat.sys
ntfs.sys
[runtime]
code/drivers/fs/fat32/fat32.c → code/drivers/fs/fat/fat.c
+805 -654
View File
File diff suppressed because it is too large. Load diff
code/drivers/fs/fat/fat.h
+44
View File
@@ -0,0 +1,44 @@
// FAT driver internals shared by mount, directory, and file I/O code.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddk/ddk.h>
#include <ntlite/ddk/file.h>
#include <ntlite/ddk/memory.h>
#include <ntlite/ddk/debug.h>
#include <ntlite/ddk/irp.h>
#include <ntlite/ddk/semaphore.h>
#include <ntlite/ddk/thread.h>
#include <ntlite/ddk/scheduler.h>
#include <ntlite/ddk/time.h>
#include <ntlite/ddk/mutex.h>
#include <ntlite/ddk/irql.h>
typedef struct FAT_VOL_S FAT_VOL;
KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
void FatClose(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS FatRead(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead);
KSTATUS FatWrite(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten);
KSTATUS FatReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS FatWriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS FatCreate(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
KSTATUS FatCreateDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS FatRemoveDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS FatDelete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS FatTruncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags);
uint64_t FatGetSize(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS FatQueryInfo(IO_FILESYSTEM *fs, void *fileCtx, SYS_FILE_STAT *out);
KSTATUS FatStat(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, SYS_FILE_STAT *out);
KSTATUS FatQueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out);
KSTATUS FatOpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx);
KSTATUS FatReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out);
void FatCloseDir(IO_FILESYSTEM *fs, void *dirCtx);
void FatQueryStats(IO_FILESYSTEM *fs, IO_FILESYSTEM_STATS *out);
void FatInitVtable(IO_FILESYSTEM* fs, FAT_VOL* vol);
code/drivers/fs/fat/fat_disk.h
+119
View File
@@ -0,0 +1,119 @@
// FAT on-disk layout definitions shared by the FAT driver.
#pragma once
#include <stdint.h>
#pragma pack(push, 1)
typedef struct {
uint8_t Jump[3];
char OemId[8];
uint16_t BytesPerSector;
uint8_t SectorsPerCluster;
uint16_t ReservedSectors;
uint8_t NumFats;
uint16_t RootEntryCount;
uint16_t TotalSectors16;
uint8_t MediaType;
uint16_t FatSize16;
uint16_t SectorsPerTrack;
uint16_t NumHeads;
uint32_t HiddenSectors;
uint32_t TotalSectors32;
uint32_t FatSize32;
uint16_t ExtFlags;
uint16_t FsVersion;
uint32_t RootCluster;
uint16_t FsInfoSector;
uint16_t BackupBootSector;
uint8_t Reserved[12];
uint8_t DriveNumber;
uint8_t Reserved1;
uint8_t BootSignature;
uint32_t VolumeId;
char VolumeLabel[11];
char FsType[8];
} BPB;
typedef struct {
char Name[8];
char Ext[3];
uint8_t Attr;
uint8_t Reserved;
uint8_t CrtTimeTenth;
uint16_t CrtTime;
uint16_t CrtDate;
uint16_t LastAccDate;
uint16_t FirstClusterHi;
uint16_t WrtTime;
uint16_t WrtDate;
uint16_t FirstClusterLo;
uint32_t FileSize;
} DIRENT;
typedef struct {
uint8_t Order;
uint16_t Name1[5];
uint8_t Attr;
uint8_t Type;
uint8_t Checksum;
uint16_t Name2[6];
uint16_t FirstCluster;
uint16_t Name3[2];
} LFN_ENTRY;
typedef struct {
uint32_t Data1;
uint16_t Data2;
uint16_t Data3;
uint8_t Data4[8];
} EFI_GUID;
typedef struct {
char Signature[8];
uint32_t Revision;
uint32_t HeaderSize;
uint32_t HeaderCRC32;
uint32_t Reserved;
uint64_t MyLBA;
uint64_t AlternateLBA;
uint64_t FirstUsableLBA;
uint64_t LastUsableLBA;
EFI_GUID DiskGUID;
uint64_t PartitionEntryLBA;
uint32_t NumberOfPartitionEntries;
uint32_t SizeOfPartitionEntry;
uint32_t PartitionEntryArrayCRC32;
} GPT_HEADER;
typedef struct {
EFI_GUID PartitionTypeGUID;
EFI_GUID UniquePartitionGUID;
uint64_t StartingLBA;
uint64_t EndingLBA;
uint64_t Attributes;
uint16_t PartitionName[36];
} GPT_ENTRY;
#pragma pack(pop)
#define ATTR_VOLUME_ID 0x08
#define ATTR_DIRECTORY 0x10
#define ATTR_ARCHIVE 0x20
#define ATTR_LFN 0x0F
#define FAT_NTRES_BASE_LOWER 0x08u
#define FAT_NTRES_EXT_LOWER 0x10u
#define FAT_FREE 0x00000000u
#define FAT16_EOC 0x0000FFF8u
#define FAT16_EOC_MARK 0x0000FFFFu
#define FAT32_EOC 0x0FFFFFF8u
#define FAT32_EOC_MARK 0x0FFFFFFFu
#define FAT_FIXED_ROOT_CLUSTER 0xFFFFFFFEu
typedef enum FAT_VARIANT {
FatVariantUnsupported = 0,
FatVariant12,
FatVariant16,
FatVariant32,
} FAT_VARIANT;
code/drivers/fs/fat/fat_mount.c
+28
View File
@@ -0,0 +1,28 @@
// FAT mount logic. This file owns volume probing, BPB validation, and mount-table insertion.
#include "fat.h"
// Initializes vtable.
void FatInitVtable(IO_FILESYSTEM* fs, FAT_VOL* vol) {
memset(fs, 0, sizeof(*fs));
fs->Open = FatOpen;
fs->Read = FatRead;
fs->Write = FatWrite;
fs->ReadAsync = FatReadAsync;
fs->WriteAsync = FatWriteAsync;
fs->Create = FatCreate;
fs->Delete = FatDelete;
fs->CreateDir = FatCreateDir;
fs->RemoveDir = FatRemoveDir;
fs->Truncate = FatTruncate;
fs->GetSize = FatGetSize;
fs->QueryInfo = FatQueryInfo;
fs->Stat = FatStat;
fs->QueryIdentity = FatQueryIdentity;
fs->Close = FatClose;
fs->DriverCtx = vol;
fs->OpenDir = FatOpenDir;
fs->ReadDir = FatReadDir;
fs->CloseDir = FatCloseDir;
fs->QueryStats = FatQueryStats;
}
code/drivers/fs/fat32/fat32.h
-44
View File
@@ -1,44 +0,0 @@
// FAT32 driver internals shared by mount, directory, and file I/O code.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddk/ddk.h>
#include <ntlite/ddk/file.h>
#include <ntlite/ddk/memory.h>
#include <ntlite/ddk/debug.h>
#include <ntlite/ddk/irp.h>
#include <ntlite/ddk/semaphore.h>
#include <ntlite/ddk/thread.h>
#include <ntlite/ddk/scheduler.h>
#include <ntlite/ddk/time.h>
#include <ntlite/ddk/mutex.h>
#include <ntlite/ddk/irql.h>
typedef struct FAT32_VOL_S FAT32_VOL;
KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead);
KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten);
KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
KSTATUS Fat32CreateDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32RemoveDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags);
uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS Fat32QueryInfo(IO_FILESYSTEM *fs, void *fileCtx, SYS_FILE_STAT *out);
KSTATUS Fat32Stat(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, SYS_FILE_STAT *out);
KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out);
KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx);
KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out);
void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx);
void Fat32QueryStats(IO_FILESYSTEM *fs, IO_FILESYSTEM_STATS *out);
void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol);
code/drivers/fs/fat32/fat32_mount.c
-28
View File
@@ -1,28 +0,0 @@
// FAT32 mount logic. This file owns volume probing, BPB validation, and mount-table insertion.
#include "fat32.h"
// Initializes vtable.
void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
memset(fs, 0, sizeof(*fs));
fs->Open = Fat32Open;
fs->Read = Fat32Read;
fs->Write = Fat32Write;
fs->ReadAsync = Fat32ReadAsync;
fs->WriteAsync = Fat32WriteAsync;
fs->Create = Fat32Create;
fs->Delete = Fat32Delete;
fs->CreateDir = Fat32CreateDir;
fs->RemoveDir = Fat32RemoveDir;
fs->Truncate = Fat32Truncate;
fs->GetSize = Fat32GetSize;
fs->QueryInfo = Fat32QueryInfo;
fs->Stat = Fat32Stat;
fs->QueryIdentity = Fat32QueryIdentity;
fs->Close = Fat32Close;
fs->DriverCtx = vol;
fs->OpenDir = Fat32OpenDir;
fs->ReadDir = Fat32ReadDir;
fs->CloseDir = Fat32CloseDir;
fs->QueryStats = Fat32QueryStats;
}
code/drivers/fs/ntfs/ntfs.c
+15 -7
View File
@@ -1701,15 +1701,23 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
memset(gDiskScanned, 0, sizeof(gDiskScanned));
gMountedCount = 0;
KSTATUS s = IoRegisterDiskChangeCallback(NtfsDiskChange, (void*)0);
if (!KSUCCESS(s)) DbgPrint("NTFS: disk-change callback registration failed st=0x%x\n", s);
s = IoEnumerateDiskDevices(NtfsDiskArrival, (void*)0);
if (!KSUCCESS(s)) DbgPrint("NTFS: initial disk enumeration failed st=0x%x\n", s);
KSTATUS cbStatus = IoRegisterDiskChangeCallback(NtfsDiskChange, (void*)0);
bool callbackRegistered = KSUCCESS(cbStatus);
if (!callbackRegistered)
DbgPrint("NTFS: disk-change callback registration failed st=0x%x\n", cbStatus);
if (!gMountedCount) {
DbgPrint("NTFS: no mountable NTFS volumes found.\n");
return NTLSTATUS_NOT_SUPPORTED;
KSTATUS enumStatus = IoEnumerateDiskDevices(NtfsDiskArrival, (void*)0);
if (!KSUCCESS(enumStatus))
DbgPrint("NTFS: initial disk enumeration failed st=0x%x\n", enumStatus);
if (!gMountedCount && !callbackRegistered) {
DbgPrint("NTFS: no mountable NTFS volumes found and disk-change callback unavailable.\n");
return cbStatus;
}
if (!gMountedCount)
DbgPrint("NTFS: no mountable NTFS volumes found; driver resident for disk changes.\n");
DbgPrint("NTFS: Init complete; mounted %u volume(s).\n", gMountedCount);
return NTLSTATUS_SUCCESS;
}
code/kernel/include/kernel/ke_cpu.h
+12 -10
View File
@@ -15,6 +15,16 @@
struct THREAD;
typedef enum KE_CPU_STATE {
KeCpuAbsent = 0,
KeCpuPresent,
KeCpuStarted,
KeCpuParked,
KeCpuAdmissionRequested,
KeCpuSchedulerActive,
KeCpuOnline
} KE_CPU_STATE;
typedef struct KCPU {
// GS:0 points back at the owning KCPU while executing in ring 0.
// User mode keeps its TEB in GS_BASE; swapgs restores this pointer on
@@ -26,11 +36,8 @@ typedef struct KCPU {
uint32_t AcpiProcessorId;
uint32_t BootProcessorIndex;
uint8_t Present;
uint8_t Started;
uint8_t Parked;
uint8_t Online;
uint8_t IsBsp;
volatile long State;
uint8_t IsBsp;
// CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded
// from the APIC ID using CPUID leaf 0x1F/0x0B when available, with a
@@ -61,11 +68,6 @@ typedef struct KCPU {
uint32_t InterruptDepth;
volatile long ReschedulePending;
// AP scheduler admission handshake. APs set SchedulerActive just before
// they leave the raw kernel park loop and enter their per-CPU idle thread.
volatile long SchedulerCommand;
volatile long SchedulerActive;
TSS64 Tss;
uint64_t Gdt[KE_GDT_ENTRIES];
KGDTR Gdtr;
code/kernel/include/kernel/ke_debug.h
+1
View File
@@ -23,6 +23,7 @@
#define KE_DEBUG_INPUT_TRACE 0x00000200u
#define KE_DEBUG_VM_ASSERTS 0x00000400u
#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u
#define KE_DEBUG_LIST_CHECKS 0x00001000u
void KeInitDebug(const HAL_API* hal, uint32_t flags);
uint32_t KeGetDebugFlags(void);
code/kernel/include/kernel/ke_display.h
+18
View File
@@ -1,6 +1,7 @@
// Kernel ke display declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#ifndef DISPLAY_API
@@ -11,6 +12,23 @@
#endif
#endif
#include <kernel/ke_event.h>
#include <kernel/ke_list.h>
#include <kernel/ob_object.h>
#include <ntlite/ddk/display.h>
typedef struct DISPLAY_CLOCK_OBJECT {
OBJECT Header;
EVENT* Event;
LIST_ENTRY Link;
uint64_t PeriodTicks;
uint64_t NextTick;
uint64_t LastFrameGeneration;
uint8_t Linked;
} DISPLAY_CLOCK_OBJECT;
void KeDisplayInitialize(const HAL_API* hal);
DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock);
DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock);
DISPLAY_API void KeDisplayTick(void);
DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device);
code/kernel/include/kernel/ke_input.h
+41 -1
View File
@@ -1,4 +1,4 @@
// Kernel ke input declarations shared by implementation files and kernel clients.
// Kernel input declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
@@ -8,9 +8,45 @@
#include <ntlite/status.h>
#include <kernel/ob_object.h>
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
typedef struct SYS_GLOBAL_INPUT_STATE SYS_GLOBAL_INPUT_STATE;
typedef struct INPUT_KEY_QUEUE_ENTRY {
INPUT_KEY Key;
char Character;
uint32_t Modifiers;
} INPUT_KEY_QUEUE_ENTRY;
typedef struct KE_INPUT_CONTEXT {
const HAL_API* Hal;
volatile uint8_t KeyDown[256];
volatile uint32_t Modifiers;
volatile long long Generation;
INPUT_KEY_QUEUE_ENTRY KeyQueue[256];
volatile int KeyHead;
volatile int KeyTail;
volatile INPUT_KEY RepeatKey;
volatile uint64_t RepeatNextTick;
KDPC RepeatDpc;
INPUT_MOUSE_EVENT MouseQueue[128];
volatile int MouseHead;
volatile int MouseTail;
volatile int MouseX;
volatile int MouseY;
volatile int MouseBoundsWidth;
volatile int MouseBoundsHeight;
volatile int MouseWheel;
volatile uint8_t MouseButtons;
EVENT* WaitEvent;
} KE_INPUT_CONTEXT;
typedef struct INPUT_QUEUE {
OBJECT Header;
KE_INPUT_CONTEXT* Context;
EVENT* WaitEvent;
} INPUT_QUEUE;
@@ -29,10 +65,14 @@ bool KeInputIsKeyDown(INPUT_KEY key);
bool KeInputPeekEvent(INPUT_KEY_RECORD* record);
bool KeInputReadEvent(INPUT_KEY_RECORD* record);
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out);
bool KeInputQueuePeekEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record);
bool KeInputQueueReadEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record);
KSTATUS KeInputQueueWaitKey(INPUT_QUEUE* queue, uint8_t alertable, uint64_t timeoutMs);
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue);
INPUT_KEY KeInputReadKey(void);
char KeInputReadChar(void);
uint32_t KeInputGetModifiers(void);
KSTATUS KeInputQueryGlobalState(SYS_GLOBAL_INPUT_STATE* out);
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event);
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event);
code/kernel/include/kernel/ke_list.h
+19 -4
View File
@@ -1,9 +1,24 @@
// Kernel ke list declarations shared by implementation files and kernel clients.
// Kernel checked-list declarations shared by implementation files and kernel clients.
#pragma once
// ke_list.h - kernel-private include path for the shared intrusive list type.
// The type currently remains DDK-visible because legacy driver objects still
// expose embedded lists and IRPs. Opaque request/device work removes that need.
// The list layout remains DDK-visible, while checked mutation policy stays in
// the kernel so public LIST_ENTRY does not gain debug-only fields.
#include <stdbool.h>
#include <ntlite/ddk/list.h>
void KeListInsertTailChecked(LIST_ENTRY* head, LIST_ENTRY* node);
void KeListInsertHeadChecked(LIST_ENTRY* head, LIST_ENTRY* node);
void KeListRemoveChecked(LIST_ENTRY* node);
void KeListValidateHead(const LIST_ENTRY* head);
// Removes the first entry through the checked removal path.
static inline LIST_ENTRY* KeListPopHeadChecked(LIST_ENTRY* head) {
if (LIST_IS_EMPTY(head))
return (LIST_ENTRY*)0;
LIST_ENTRY* node = head->Fwd;
KeListRemoveChecked(node);
return node;
}
code/kernel/include/kernel/ke_sched.h
+6 -6
View File
@@ -2,7 +2,7 @@
#pragma once
// ke_sched.h - Multi-Level Feedback Queue scheduler
// Three run queues (Low / Normal / High). IRQ 0 drives KeSchedulerTick.
// Three run queues (Low / Normal / High). LAPIC timer IRQs drive KeSchedulerTick.
#include <kernel/ke_thread.h>
struct KCPU;
@@ -41,11 +41,11 @@ KSTATUS KiCreateIdleThreadForCpu(struct KCPU* cpu);
void KeReadyThread(THREAD* thread);
void KeUnreadyThread(THREAD* thread);
// Called from the timer ISR every SCHED_TICK_MS. This performs timer
// accounting and requests a reschedule only when the current thread has
// exhausted its scheduler budget, or when idle work becomes available. The
// actual context switch happens from
// KiExitInterrupt after the IRQ handler has returned and EOI has been sent.
// Called from each CPU timer ISR every SCHED_TICK_MS. Every CPU performs local
// quantum accounting and may request a local reschedule. Only the BSP advances
// global scheduler time and services sleepers, waitable timers, input repeat,
// and priority aging. The actual context switch happens from KiExitInterrupt
// after the IRQ handler has returned and EOI has been sent.
void KeSchedulerTick(void);
// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule
code/kernel/include/kernel/mm_pool.h
+38
View File
@@ -0,0 +1,38 @@
// Kernel pool allocator declarations.
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <ntlite/status.h>
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
#define MM_POOL_TAG(a, b, c, d) \
((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24))
// Initializes the kernel pool and optionally commits its initial extent.
void MmPoolInitialize(size_t poolSize);
// Allocates zeroed kernel-pool memory from the non-user heap.
__declspec(dllexport) void* MmAllocatePool(size_t size);
// Allocates zeroed kernel-pool memory with a diagnostic tag.
__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag);
// Releases kernel-pool memory back to the non-user heap.
__declspec(dllexport) void MmFreePool(void* ptr);
// Reallocates kernel-pool memory while preserving existing contents.
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// Verifies kernel-pool boundary tags and free-list links.
KSTATUS MmValidateKernelPool(const char* reason);
// Describes a virtual address that falls inside the kernel-pool aperture.
void MmPoolDebugDescribeAddress(uintptr_t addr);
// Returns the currently committed kernel-pool bytes.
size_t MmGetKernelPoolUsage(void);
code/kernel/include/kernel/mm_virt.h
+1 -15
View File
@@ -7,6 +7,7 @@
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pte.h>
#include <kernel/mm_pool.h>
#include <ntlite/status.h>
struct PROCESS;
@@ -39,9 +40,6 @@ typedef enum MM_VAD_TYPE {
#define USER_LIMIT 0x00007FFFFFFFF000ULL
#define KERNEL_BASE 0xFFFF800000000000ULL
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
// Initialisation
// Called by KernelMain after MmInitializePhysical.
// bootPml4Phys: the PML4 set up by the bootloader.
@@ -83,17 +81,6 @@ __declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
// Kernel pool - ring 0 only, never PAGE_USER.
// Best-fit free-list allocator, thread-safe via spinlock.
#define MM_POOL_TAG(a, b, c, d) \
((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24))
__declspec(dllexport) void* MmAllocatePool(size_t size);
__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag);
__declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// Virtual address descriptors.
// The single shared address space still has per-process VAD ownership so process
// teardown can reclaim leaked mappings. VADs track sparse per-page commitment,
@@ -165,7 +152,6 @@ __declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* pt
__declspec(dllexport) void* MmAllocateUserImage(size_t size);
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
size_t MmGetKernelPoolUsage(void);
size_t MmGetUserHeapUsage(void);
// Demand-paging telemetry used by the shell stats command.
code/kernel/include/kernel/ob_object.h
+12 -5
View File
@@ -2,16 +2,23 @@
#pragma once
// ob_object.h - executive object model.
// Every resource the kernel tracks is a typed, reference-counted OBJECT. The
// legacy OBJECT layout lives in the DDK while driver/device objects are still
// public; object-manager routines remain kernel-private.
// Every tracked kernel resource has a typed, reference-counted object header.
// The DDK exposes the header because driver and device objects embed it, while
// lifetime policy and transition helpers remain owned by the object manager.
#include <stddef.h>
#include <stdbool.h>
#include <ntlite/status.h>
#include <ntlite/ddk/object.h>
// Increment the reference count. Call whenever ownership is shared.
// Initialize an embedded object header with one live reference.
__declspec(dllexport) void ObInitializeObjectHeader(OBJECT* object, OBJECT_TYPE type);
// Acquire a reference only if the object is still alive and inserted.
// The caller must stabilize the pointer through its owning lock or reference.
__declspec(dllexport) bool ObTryReferenceObject(OBJECT* object);
// Increment the reference count. Stale-object use is fatal.
__declspec(dllexport) void ObReferenceObject(OBJECT* object);
// Decrement the reference count and destroy the object when it reaches zero.
code/kernel/include/kernel/syscall_nr.h
+53 -51
View File
@@ -92,61 +92,63 @@
#define SYS_REAL_PATH 73
// Console, display, and surfaces.
#define SYS_WRITE_CONSOLE 74
#define SYS_GET_CONSOLE_INFO 75
#define SYS_SET_CONSOLE_CURSOR 76
#define SYS_FILL_CONSOLE_CHARACTER 77
#define SYS_FILL_CONSOLE_ATTRIBUTE 78
#define SYS_PUT_SCREEN_CHAR 79
#define SYS_CLEAR_SCREEN 80
#define SYS_GET_SCREEN_SIZE 81
#define SYS_QUERY_DISPLAY_INFO 82
#define SYS_CREATE_SURFACE 83
#define SYS_MAP_SURFACE 84
#define SYS_QUERY_SURFACE 85
#define SYS_PRESENT_SURFACE 86
#define SYS_WRITE_CONSOLE 74
#define SYS_GET_CONSOLE_INFO 75
#define SYS_SET_CONSOLE_CURSOR 76
#define SYS_FILL_CONSOLE_CHARACTER 77
#define SYS_FILL_CONSOLE_ATTRIBUTE 78
#define SYS_PUT_SCREEN_CHAR 79
#define SYS_CLEAR_SCREEN 80
#define SYS_GET_SCREEN_SIZE 81
#define SYS_QUERY_DISPLAY_INFO 82
#define SYS_OPEN_DISPLAY_CLOCK 83
#define SYS_CREATE_SURFACE 84
#define SYS_MAP_SURFACE 85
#define SYS_QUERY_SURFACE 86
#define SYS_PRESENT_SURFACE 87
// Input.
#define SYS_POLL_KEYBOARD 87
#define SYS_IS_KEY_DOWN 88
#define SYS_READ_KEYBOARD_CHAR 89
#define SYS_READ_KEY 90
#define SYS_GET_KEY_MODIFIERS 91
#define SYS_OPEN_INPUT_QUEUE 92
#define SYS_READ_INPUT_EVENT 93
#define SYS_PEEK_INPUT_EVENT 94
#define SYS_PEEK_MOUSE 95
#define SYS_READ_MOUSE 96
#define SYS_GET_MOUSE_POS 97
#define SYS_POLL_KEYBOARD 88
#define SYS_IS_KEY_DOWN 89
#define SYS_READ_KEYBOARD_CHAR 90
#define SYS_READ_KEY 91
#define SYS_GET_KEY_MODIFIERS 92
#define SYS_QUERY_GLOBAL_INPUT_STATE 93
#define SYS_OPEN_INPUT_QUEUE 94
#define SYS_READ_INPUT_EVENT 95
#define SYS_PEEK_INPUT_EVENT 96
#define SYS_PEEK_MOUSE 97
#define SYS_READ_MOUSE 98
#define SYS_GET_MOUSE_POS 99
// Time, debug, and system information.
#define SYS_CREATE_TIMER 98
#define SYS_SET_TIMER 99
#define SYS_CANCEL_TIMER 100
#define SYS_GET_TICK 101
#define SYS_GET_TICK_FREQUENCY 102
#define SYS_GET_SYSTEM_TIME 103
#define SYS_PRINT_DEBUG 104
#define SYS_QUERY_TRAP_FRAME 105
#define SYS_GET_MEMORY_STATUS 106
#define SYS_QUERY_KERNEL_STATS 107
#define SYS_CREATE_TIMER 100
#define SYS_SET_TIMER 101
#define SYS_CANCEL_TIMER 102
#define SYS_GET_TICK 103
#define SYS_GET_TICK_FREQUENCY 104
#define SYS_GET_SYSTEM_TIME 105
#define SYS_PRINT_DEBUG 106
#define SYS_QUERY_TRAP_FRAME 107
#define SYS_GET_MEMORY_STATUS 108
#define SYS_QUERY_KERNEL_STATS 109
// Native sockets and polling.
#define SYS_CREATE_SOCKET 108
#define SYS_SOCKET_IO_ASYNC 109
#define SYS_BIND_SOCKET 110
#define SYS_LISTEN_SOCKET 111
#define SYS_ACCEPT_SOCKET 112
#define SYS_CONNECT_SOCKET 113
#define SYS_SEND_SOCKET 114
#define SYS_RECV_SOCKET 115
#define SYS_SHUTDOWN_SOCKET 116
#define SYS_SEND_TO_SOCKET 117
#define SYS_RECV_FROM_SOCKET 118
#define SYS_RESOLVE_HOST 119
#define SYS_SET_SOCKET_OPTION 120
#define SYS_GET_SOCKET_OPTION 121
#define SYS_POLL 122
#define SYS_UNPUBLISH_QUEUE 123
#define SYS_CREATE_SOCKET 110
#define SYS_SOCKET_IO_ASYNC 111
#define SYS_BIND_SOCKET 112
#define SYS_LISTEN_SOCKET 113
#define SYS_ACCEPT_SOCKET 114
#define SYS_CONNECT_SOCKET 115
#define SYS_SEND_SOCKET 116
#define SYS_RECV_SOCKET 117
#define SYS_SHUTDOWN_SOCKET 118
#define SYS_SEND_TO_SOCKET 119
#define SYS_RECV_FROM_SOCKET 120
#define SYS_RESOLVE_HOST 121
#define SYS_SET_SOCKET_OPTION 122
#define SYS_GET_SOCKET_OPTION 123
#define SYS_POLL 124
#define SYS_UNPUBLISH_QUEUE 125
#define SYS_MAX 124
#define SYS_MAX 126
code/kernel/io_device.c
+4 -6
View File
@@ -79,9 +79,7 @@ __declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
SYS_STRING_VIEW name) {
if (!driver) return;
memset(driver, 0, sizeof(*driver));
driver->Header.Type = ObjectTypeDriver;
driver->Header.RefCount = 1;
driver->Header.Destroy = (void*)0;
ObInitializeObjectHeader(&driver->Header, ObjectTypeDriver);
driver->Name = name;
}
@@ -91,9 +89,9 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
return NTLSTATUS_NO_MEMORY;
if (driver->Header.Type != ObjectTypeDriver) {
driver->Header.Type = ObjectTypeDriver;
driver->Header.RefCount = 1;
if (driver->Header.Type != ObjectTypeDriver ||
driver->Header.LifetimeState != ObjectAlive) {
ObInitializeObjectHeader(&driver->Header, ObjectTypeDriver);
}
gDriverObjects[gDriverObjectCount++] = driver;
code/kernel/ke_cpu.c
+2 -7
View File
@@ -48,10 +48,7 @@ void KeInitializeBootstrapCpu(void) {
cpu->Self = cpu;
cpu->CpuIndex = 0;
cpu->LapicId = 0;
cpu->Present = 1;
cpu->Started = 1;
cpu->Parked = 0;
cpu->Online = 1;
cpu->State = KeCpuStarted;
cpu->IsBsp = 1;
cpu->PackageId = 0;
cpu->CoreId = 0;
@@ -65,10 +62,8 @@ void KeInitializeBootstrapCpu(void) {
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
cpu->SchedulerCommand = 0;
cpu->SchedulerActive = 0;
cpu->KernelStack = gBootstrapCpuStack;
cpu->KernelStackTop = (uintptr_t)(gBootstrapCpuStack + sizeof(gBootstrapCpuStack));
cpu->KernelStackTop = ((uintptr_t)(gBootstrapCpuStack + sizeof(gBootstrapCpuStack))) & ~(uintptr_t)15;
memset(&cpu->Tss, 0, sizeof(cpu->Tss));
cpu->Tss.IoMapBase = sizeof(TSS64);
cpu->Tss.RSP0 = cpu->KernelStackTop;
code/kernel/ke_debug.c
+110 -30
View File
@@ -28,6 +28,8 @@ static const HAL_API* gDebugHal;
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_FATAL_OUTPUT_LINE_SIZE 2048u
#define KE_STACK_DUMP_SLOT_COUNT 12u
#define KE_RBP_CALL_STACK_DEPTH 16u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
@@ -412,6 +414,22 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
}
}
// Prints a resolved address without ending the current line.
static void KiPrintAddressValue(PROCESS* proc, uintptr_t addr) {
uintptr_t base = 0;
size_t size = 0;
const char* mod = KiResolveModuleEx(proc, addr, &base, &size);
if (base) {
KiFaultPrintf("%016llx <%s+0x%016llX>",
(unsigned long long)addr,
mod,
(unsigned long long)(addr - base));
} else {
KiFaultPrintf("%016llx", (unsigned long long)addr);
}
}
// Checks whether a kernel virtual range is mapped before diagnostic reads.
static int KiKernelRangeReadable(uintptr_t addr, size_t size) {
if (!addr || size == 0) return 0;
@@ -562,44 +580,103 @@ static void KiDumpModules(PROCESS* proc) {
}
}
// Dumps user Stack.
static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
KiFaultPrintf("User stack @ %016llx:\n", (unsigned long long)rsp);
#if 0
// Returns whether a range is inside the current kernel stack.
static int KiKernelStackContains(const THREAD* thread, uintptr_t addr, size_t size) {
if (!thread || !thread->KernelStackBase || !thread->KernelStackTop || !addr || size == 0)
return 0;
if (size - 1u > (size_t)(UINTPTR_MAX - addr))
return 0;
uintptr_t base = (uintptr_t)thread->KernelStackBase;
uintptr_t top = thread->KernelStackTop;
uintptr_t end = addr + size;
return addr >= base && end <= top;
}
// Returns whether the fault frame is using the current kernel stack.
static int KiFaultFrameOnKernelStack(const THREAD* thread, const KFAULT_FRAME* f) {
if (!f || !thread)
return 0;
return KiKernelStackContains(thread, (uintptr_t)f->Rsp, sizeof(uint64_t));
}
#endif
// Dumps stack words near RSP.
static void KiDumpStack(PROCESS* proc, uintptr_t rsp) {
KiFaultPrintf("Stack @ %016llx:\n", (unsigned long long)rsp);
if (!rsp) {
KiFaultPrintf(" <no user RSP captured>\n");
KiFaultPrintf(" <no RSP captured>\n");
return;
}
for (int i = 0; i < 12; i++) {
uintptr_t slot = rsp + (uintptr_t)i * 8u;
for (uint32_t i = 0; i < KE_STACK_DUMP_SLOT_COUNT; i++) {
uintptr_t slot = rsp + (uintptr_t)i * sizeof(uint64_t);
uint64_t value = 0;
KiFaultPrintf(" [%02u] %016llx : ",
(unsigned)i,
(unsigned long long)slot);
if (KiUserReadU64(slot, &value)) {
uintptr_t base = 0;
size_t size = 0;
const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size);
if (base) {
KiFaultPrintf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n",
i,
(unsigned long long)slot,
(unsigned long long)value,
mod,
(unsigned long long)((uintptr_t)value - base));
} else {
KiFaultPrintf(" [%02d] %016llx : %016llx\n",
i,
(unsigned long long)slot,
(unsigned long long)value);
}
KiPrintAddressValue(proc, (uintptr_t)value);
KiFaultPrintf("\n");
} else {
KiFaultPrintf(" [%02d] %016llx : <unreadable>\n",
i,
(unsigned long long)slot);
KiFaultPrintf("<unreadable>\n");
}
}
}
// Dumps fault Frame.
static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
#if 0
// Dumps a best-effort frame-pointer call stack.
static void KiDumpRbpCallStack(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
uintptr_t rbp = f ? (uintptr_t)f->Rbp : 0;
KiFaultPrintf("RBP call stack @ %016llx:\n", (unsigned long long)rbp);
if (!rbp) {
KiFaultPrintf(" <no RBP captured>\n");
return;
}
int kernelStack = KiFaultFrameOnKernelStack(thread, f);
for (uint32_t depth = 0; depth < KE_RBP_CALL_STACK_DEPTH; depth++) {
if (rbp & (sizeof(uint64_t) - 1u)) {
KiFaultPrintf(" stop: unaligned RBP %016llx\n",
(unsigned long long)rbp);
return;
}
if (kernelStack && !KiKernelStackContains(thread, rbp, sizeof(uint64_t) * 2u)) {
KiFaultPrintf(" stop: RBP outside kernel stack %016llx\n",
(unsigned long long)rbp);
return;
}
uint64_t next = 0;
uint64_t ret = 0;
if (!KiUserReadU64(rbp, &next) || !KiUserReadU64(rbp + sizeof(uint64_t), &ret)) {
KiFaultPrintf(" stop: frame unreadable at %016llx\n",
(unsigned long long)rbp);
return;
}
KiFaultPrintf(" #%02u rbp=%016llx ret=",
(unsigned)depth,
(unsigned long long)rbp);
KiPrintAddressValue(proc, (uintptr_t)ret);
KiFaultPrintf(" next=%016llx\n", (unsigned long long)next);
if (!next)
return;
if ((uintptr_t)next <= rbp) {
KiFaultPrintf(" stop: non-forward RBP chain %016llx -> %016llx\n",
(unsigned long long)rbp,
(unsigned long long)next);
return;
}
rbp = (uintptr_t)next;
}
}
#endif
// Dumps a fault frame.
static void KiDumpFaultFrame(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
if (!f) return;
KiFaultPrintf("Regs:\n");
@@ -634,7 +711,10 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
KiDumpBytes(proc, "Bytes near RIP", (uintptr_t)f->Rip);
}
KiDumpUserStack(proc, (uintptr_t)f->Rsp);
KiDumpStack(proc, (uintptr_t)f->Rsp);
#if 0
KiDumpRbpCallStack(proc, thread, f);
#endif
}
static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
@@ -710,7 +790,7 @@ static _Noreturn void KiPanicKernelFault(PROCESS* proc, THREAD* thread, const KF
thread,
proc ? proc->ProcessId : 0,
proc ? proc->Name : "<none>");
KiDumpFaultFrame(proc, frame);
KiDumpFaultFrame(proc, thread, frame);
if (vector == 14) {
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
if (frame->Rip) MmDebugDescribeAddress((uintptr_t)frame->Rip);
@@ -755,7 +835,7 @@ static _Noreturn void KiTerminateUserFault(PROCESS* proc, THREAD* thread, const
(uintptr_t)frame->Rip,
(uintptr_t)frame->ErrorCode,
(uintptr_t)frame->Cr2);
KiDumpFaultFrame(proc, frame);
KiDumpFaultFrame(proc, thread, frame);
KiDumpModules(proc);
KiFinishFaultedProcess(proc, thread, KeExceptionProcessExitStatus((uint32_t)frame->Vector));
code/kernel/ke_display.c
+193
View File
@@ -15,6 +15,9 @@
#include <kernel/ke_work.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_event.h>
#include <kernel/ke_list.h>
#include <kernel/ob_object.h>
#include <kernel/ke_irql.h>
#define DISPLAY_DEFAULT_FG 0x00CCCCCCu
@@ -32,6 +35,8 @@
#define DISPLAY_CURSOR_POS_INVALID (-2147483647 - 1)
#define DISPLAY_OUTPUT_BUFFER_CAPACITY 4096u
#define DISPLAY_OUTPUT_DRAIN_COUNT 128u
#define DISPLAY_CLOCK_DEFAULT_PERIOD_MS 1ull
#define DISPLAY_CLOCK_SIGNAL_BATCH 32u
typedef struct DISPLAY_OUTPUT_GLYPH {
char Character;
@@ -58,6 +63,10 @@ static KWORK_ITEM gDisplayCursorWorkItem;
static volatile long gDisplayCursorWorkInitialized;
static bool gDisplayTransitionInProgress;
static uint64_t gDisplayGeneration;
static LIST_ENTRY gDisplayClockList;
static SPINLOCK gDisplayClockLock = SPINLOCK_INIT;
static volatile long gDisplayClockInitialized;
static uint64_t gDisplayFrameGeneration;
static DISPLAY_DEVICE gBootFbDevice;
static BOOTFB_CONTEXT gBootFb;
@@ -125,6 +134,43 @@ static bool KiDisplayOutputSuspended(void);
static void KiDisplayCursorWorker(void* context);
static void KiDisplayDirtyAll(void);
// Converts a display-clock period to scheduler ticks with minimum one-tick progress.
static uint64_t KiDisplayClockMillisecondsToTicks(uint64_t milliseconds) {
if (milliseconds == 0)
milliseconds = DISPLAY_CLOCK_DEFAULT_PERIOD_MS;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Reads the global scheduler tick used by display-clock deadlines.
static uint64_t KiDisplayClockReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Destroys a display clock and removes it from the manager tick list.
static void KiDisplayClockDestroy(OBJECT* object) {
DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)object;
if (!clock)
return;
uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
if (clock->Linked) {
ListRemove(&clock->Link);
clock->Linked = 0;
}
KeReleaseSpinlock(&gDisplayClockLock, flags);
if (clock->Event) {
ObDereferenceObject(&clock->Event->Header);
clock->Event = (EVENT*)0;
}
}
static bool KiDisplayHasCurrentThread(void) {
return KeGetCurrentThread() != (THREAD*)0;
}
@@ -1105,6 +1151,10 @@ void KeDisplayInitialize(const HAL_API* hal) {
gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID;
gDisplayTransitionInProgress = false;
gDisplayGeneration = 1;
gDisplayClockLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&gDisplayClockList);
gDisplayFrameGeneration = 0;
_InterlockedExchange(&gDisplayClockInitialized, 1);
if (!gDisplayProviderMutex)
(void)KeMutexCreate(&gDisplayProviderMutex);
KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
@@ -1178,6 +1228,149 @@ void KeDisplayInitialize(const HAL_API* hal) {
KiDisplayRegisterOutputEndpoint();
}
DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock) {
if (!outClock)
return NTLSTATUS_INVALID_PARAMETER;
*outClock = (DISPLAY_CLOCK_OBJECT*)0;
if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
return NTLSTATUS_NOT_SUPPORTED;
uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
bool hasDevice = KiDisplayHasDevice();
KeReleaseSpinlock(&gDisplayLock, displayFlags);
if (!hasDevice)
return NTLSTATUS_NOT_SUPPORTED;
DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)ObAllocateObject(ObjectTypeDisplayClock,
sizeof(DISPLAY_CLOCK_OBJECT));
if (!clock)
return NTLSTATUS_NO_MEMORY;
clock->Header.Destroy = KiDisplayClockDestroy;
LIST_INIT(&clock->Link);
clock->PeriodTicks = KiDisplayClockMillisecondsToTicks(periodMilliseconds);
uint64_t now = KiDisplayClockReadSchedulerTick();
clock->NextTick = (UINT64_MAX - now < clock->PeriodTicks) ? UINT64_MAX : now + clock->PeriodTicks;
KSTATUS s = KeEventCreate(EventAutoReset, 0, &clock->Event);
if (!KSUCCESS(s)) {
ObDereferenceObject(&clock->Header);
return s;
}
uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
clock->LastFrameGeneration = gDisplayFrameGeneration;
clock->Linked = 1;
ListInsertTail(&gDisplayClockList, &clock->Link);
KeReleaseSpinlock(&gDisplayClockLock, flags);
*outClock = clock;
return NTLSTATUS_SUCCESS;
}
DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) {
if (!clock)
return (EVENT*)0;
return clock->Event;
}
// Signals every display clock once for a provider-owned frame generation.
static void KiDisplaySignalFrameGeneration(uint64_t generation) {
for (;;) {
EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
uint32_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
LIST_ENTRY* node = gDisplayClockList.Fwd;
while (node != &gDisplayClockList && count < DISPLAY_CLOCK_SIGNAL_BATCH) {
DISPLAY_CLOCK_OBJECT* clock = CONTAINING_RECORD(node, DISPLAY_CLOCK_OBJECT, Link);
node = node->Fwd;
if (!clock->Linked || !clock->Event || clock->LastFrameGeneration == generation)
continue;
if (!ObTryReferenceObject(&clock->Event->Header))
continue;
clock->LastFrameGeneration = generation;
events[count++] = clock->Event;
}
KeReleaseSpinlock(&gDisplayClockLock, flags);
for (uint32_t i = 0; i < count; i++) {
KeSignalEvent(events[i]);
ObDereferenceObject(&events[i]->Header);
}
if (count < DISPLAY_CLOCK_SIGNAL_BATCH)
break;
}
}
// Publishes a frame-ready event from the current primary display provider.
DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device) {
if (!device || !_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
return;
uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
bool accepted = device == gPrimaryDisplay &&
(device->Caps & DISPLAY_CAP_FRAME_EVENT) != 0;
KeReleaseSpinlock(&gDisplayLock, displayFlags);
if (!accepted)
return;
uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
uint64_t generation = ++gDisplayFrameGeneration;
if (generation == 0)
generation = ++gDisplayFrameGeneration;
KeReleaseSpinlock(&gDisplayClockLock, flags);
KiDisplaySignalFrameGeneration(generation);
}
DISPLAY_API void KeDisplayTick(void) {
if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
return;
uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
bool providerOwnsFrames = gPrimaryDisplay &&
(gPrimaryDisplay->Caps & DISPLAY_CAP_FRAME_EVENT) != 0;
KeReleaseSpinlock(&gDisplayLock, displayFlags);
if (providerOwnsFrames)
return;
uint64_t now = KiDisplayClockReadSchedulerTick();
for (;;) {
EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
uint32_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
LIST_ENTRY* node = gDisplayClockList.Fwd;
while (node != &gDisplayClockList && count < DISPLAY_CLOCK_SIGNAL_BATCH) {
DISPLAY_CLOCK_OBJECT* clock = CONTAINING_RECORD(node, DISPLAY_CLOCK_OBJECT, Link);
node = node->Fwd;
if (!clock->Linked || clock->NextTick > now || !clock->Event)
continue;
if (!ObTryReferenceObject(&clock->Event->Header))
continue;
uint64_t next = now + clock->PeriodTicks;
clock->NextTick = next < now ? UINT64_MAX : next;
events[count++] = clock->Event;
}
KeReleaseSpinlock(&gDisplayClockLock, flags);
for (uint32_t i = 0; i < count; i++) {
KeSignalEvent(events[i]);
ObDereferenceObject(&events[i]->Header);
}
if (count < DISPLAY_CLOCK_SIGNAL_BATCH)
break;
}
}
DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
if (!device || !device->Name || !device->Ops || !device->Ops->GetMode)
return NTLSTATUS_INVALID_PARAMETER;
code/kernel/ke_dpc.c
+3 -2
View File
@@ -1,6 +1,7 @@
// Deferred procedure call infrastructure. This file owns DPC queuing and execution outside hard IRQ context.
#include <kernel/ke_dpc.h>
#include <kernel/ke_list.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_debug.h>
#include <intrin.h>
@@ -58,7 +59,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
dpc->SystemArgument2 = systemArgument2;
uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
ListInsertTail(&gDpcQueue, &dpc->Link);
KeListInsertTailChecked(&gDpcQueue, &dpc->Link);
KeReleaseSpinlock(&gDpcLock, flags);
return true;
}
@@ -82,7 +83,7 @@ void KiDrainDpcQueue(void) {
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
LIST_ENTRY* node = ListPopHead(&gDpcQueue);
LIST_ENTRY* node = KeListPopHeadChecked(&gDpcQueue);
if (!node) {
KeReleaseSpinlock(&gDpcLock, flags);
break;
code/kernel/ke_event.c
+7 -6
View File
@@ -1,6 +1,7 @@
// Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <intrin.h>
#include <kernel/ke_event.h>
#include <kernel/ke_thread.h>
@@ -39,7 +40,7 @@ static void KiCompleteMultiWaitBlockLocked(KE_EVENT_WAIT_BLOCK* block) {
if (!block || !block->Armed)
return;
ListRemove(&block->Link);
KeListRemoveChecked(&block->Link);
block->Armed = 0;
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
@@ -67,7 +68,7 @@ static void KiCleanupMultiWaitBlocks(KE_EVENT_MULTI_WAIT_CONTEXT* ctx) {
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (block->Armed) {
ListRemove(&block->Link);
KeListRemoveChecked(&block->Link);
block->Armed = 0;
}
KeReleaseSpinlock(&event->Lock, flags);
@@ -117,7 +118,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
if (event->Type == EventManualReset) {
event->Signaled = 1;
while (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
LIST_ENTRY* node = KeListPopHeadChecked(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(t, (LIST_ENTRY*)0);
KeReleaseSpinlock(&event->Lock, flags);
@@ -132,7 +133,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
} else {
while (!LIST_IS_EMPTY(&event->WaitList)) {
event->Signaled = 0;
LIST_ENTRY* node = ListPopHead(&event->WaitList);
LIST_ENTRY* node = KeListPopHeadChecked(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(t, (LIST_ENTRY*)0);
KeReleaseSpinlock(&event->Lock, flags);
@@ -195,7 +196,7 @@ __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
}
KePrepareBlockThread(self, &event->Header, alertable);
ListInsertTail(&event->WaitList, &self->SchedulerLink);
KeListInsertTailChecked(&event->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &event->WaitList);
KeReleaseSpinlock(&event->Lock, flags);
return KeCommitBlockThread(self, alertable);
@@ -267,7 +268,7 @@ __declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t c
if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0) {
stackBlocks[i].Armed = 1;
ListInsertTail(&event->MultiWaitList, &stackBlocks[i].Link);
KeListInsertTailChecked(&event->MultiWaitList, &stackBlocks[i].Link);
}
KeReleaseSpinlock(&event->Lock, flags);
}
code/kernel/ke_input.c
+243 -166
View File
@@ -4,6 +4,7 @@
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/ke_input.h>
#include <kernel/ke_debug.h>
@@ -11,6 +12,7 @@
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
#include <kernel/ob_object.h>
#include <ntlite/syscall.h>
#define KEY_QUEUE_SIZE 256u
#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u)
@@ -27,38 +29,16 @@ typedef struct KEY_CHAR_PAIR {
char shifted;
} KEY_CHAR_PAIR;
static const HAL_API* gInputHal;
static volatile uint8_t gKeyDown[MAX_INPUT_KEYS];
static volatile uint32_t gModifiers;
typedef struct INPUT_KEY_QUEUE_ENTRY {
INPUT_KEY key;
char ch;
uint32_t modifiers;
} INPUT_KEY_QUEUE_ENTRY;
static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE];
static volatile int gKeyHead;
static volatile int gKeyTail;
static volatile INPUT_KEY gRepeatKey;
static volatile uint64_t gRepeatNextTick;
static KDPC gRepeatDpc;
static INPUT_MOUSE_EVENT gMouseQueue[MOUSE_QUEUE_SIZE];
static volatile int gMouseHead;
static volatile int gMouseTail;
static volatile int gMouseX;
static volatile int gMouseY;
static volatile int gMouseBoundsW = 1920;
static volatile int gMouseBoundsH = 1080;
static volatile uint8_t gMouseButtons;
static KE_INPUT_CONTEXT gDefaultInputContext;
static volatile uint32_t gNextInputDeviceId = 1;
static EVENT* gKeyboardEvent;
static char KiInputKeyToChar(INPUT_KEY key);
static void KiInputServiceRepeat(void);
static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key);
static void KiInputServiceRepeat(KE_INPUT_CONTEXT* context);
// Advances the physical input snapshot generation after visible state changes.
static void KiInputBumpGeneration(KE_INPUT_CONTEXT* context) {
_InterlockedIncrement64(&context->Generation);
}
static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' },
@@ -85,28 +65,28 @@ static const KEY_CHAR_PAIR gCharMap[] = {
};
// Updates the waitable keyboard event to match queued keyboard input.
static void KiInputRefreshKeyboardEvent(void) {
if (!gKeyboardEvent) return;
if (gKeyHead != gKeyTail) KeSignalEvent(gKeyboardEvent);
else KeClearEvent(gKeyboardEvent);
static void KiInputRefreshKeyboardEvent(KE_INPUT_CONTEXT* context) {
if (!context->WaitEvent) return;
if (context->KeyHead != context->KeyTail) KeSignalEvent(context->WaitEvent);
else KeClearEvent(context->WaitEvent);
}
// Queues a keyboard event and wakes waiters when the event was accepted.
static void KiInputQueueKey(INPUT_KEY key, char ch) {
static void KiInputQueueKey(KE_INPUT_CONTEXT* context, INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
if (next == gKeyHead) return;
gKeyQueue[gKeyTail].key = key;
gKeyQueue[gKeyTail].ch = ch;
gKeyQueue[gKeyTail].modifiers = gModifiers;
gKeyTail = next;
KiInputRefreshKeyboardEvent();
int next = (context->KeyTail + 1) & (int)KEY_QUEUE_MASK;
if (next == context->KeyHead) return;
context->KeyQueue[context->KeyTail].Key = key;
context->KeyQueue[context->KeyTail].Character = ch;
context->KeyQueue[context->KeyTail].Modifiers = context->Modifiers;
context->KeyTail = next;
KiInputRefreshKeyboardEvent(context);
}
// Implements the input Queue Has Space operation.
static bool KiInputQueueHasSpace(void) {
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
return next != gKeyHead;
static bool KiInputQueueHasSpace(KE_INPUT_CONTEXT* context) {
int next = (context->KeyTail + 1) & (int)KEY_QUEUE_MASK;
return next != context->KeyHead;
}
// Implements the input Key Is Repeatable operation.
@@ -131,61 +111,60 @@ static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
}
// Implements the input Now Ticks operation.
static uint64_t KiInputNowTicks(void) {
if (gInputHal && gInputHal->Timer.QueryTicks)
return gInputHal->Timer.QueryTicks();
static uint64_t KiInputNowTicks(KE_INPUT_CONTEXT* context) {
if (context->Hal && context->Hal->Timer.QueryTicks)
return context->Hal->Timer.QueryTicks();
return 0;
}
// Implements the input Start Repeat operation.
static void KiInputStartRepeat(INPUT_KEY key, uint64_t now) {
static void KiInputStartRepeat(KE_INPUT_CONTEXT* context, INPUT_KEY key, uint64_t now) {
if (!KiInputKeyIsRepeatable(key)) return;
gRepeatKey = key;
gRepeatNextTick = now + KEY_REPEAT_DELAY_US;
context->RepeatKey = key;
context->RepeatNextTick = now + KEY_REPEAT_DELAY_US;
}
// Implements the input Stop Repeat operation.
static void KiInputStopRepeat(INPUT_KEY key) {
if (gRepeatKey == key) {
gRepeatKey = INPUT_KEY_NONE;
gRepeatNextTick = 0;
static void KiInputStopRepeat(KE_INPUT_CONTEXT* context, INPUT_KEY key) {
if (context->RepeatKey == key) {
context->RepeatKey = INPUT_KEY_NONE;
context->RepeatNextTick = 0;
}
}
// Implements the input Service Repeat operation.
static void KiInputServiceRepeat(void) {
INPUT_KEY key = gRepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key]) {
gRepeatKey = INPUT_KEY_NONE;
gRepeatNextTick = 0;
static void KiInputServiceRepeat(KE_INPUT_CONTEXT* context) {
INPUT_KEY key = context->RepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !context->KeyDown[key]) {
context->RepeatKey = INPUT_KEY_NONE;
context->RepeatNextTick = 0;
return;
}
uint64_t now = KiInputNowTicks();
if (now == 0 || now < gRepeatNextTick)
uint64_t now = KiInputNowTicks(context);
if (now == 0 || now < context->RepeatNextTick)
return;
if (!KiInputQueueHasSpace()) {
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
if (!KiInputQueueHasSpace(context)) {
context->RepeatNextTick = now + KEY_REPEAT_PERIOD_US;
return;
}
KiInputQueueKey(key, KiInputKeyToChar(key));
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
KiInputQueueKey(context, key, KiInputKeyToChar(context, key));
context->RepeatNextTick = now + KEY_REPEAT_PERIOD_US;
}
// Implements the input Repeat Dpc Routine operation.
static void KiInputRepeatDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
KiInputServiceRepeat();
KiInputServiceRepeat((KE_INPUT_CONTEXT*)context);
}
// Implements the input Set Modifier operation.
static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
static void KiInputSetModifier(KE_INPUT_CONTEXT* context, INPUT_KEY key, bool pressed) {
uint32_t bit = 0;
switch (key) {
case INPUT_KEY_LEFT_SHIFT: bit = INPUT_MOD_LSHIFT; break;
@@ -200,25 +179,25 @@ static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
}
if (bit) {
if (pressed) gModifiers |= bit;
else gModifiers &= ~bit;
if (pressed) context->Modifiers |= bit;
else context->Modifiers &= ~bit;
}
if (pressed && key == INPUT_KEY_CAPS_LOCK) gModifiers ^= INPUT_MOD_CAPS_LOCK;
if (pressed && key == INPUT_KEY_NUM_LOCK) gModifiers ^= INPUT_MOD_NUM_LOCK;
if (pressed && key == INPUT_KEY_CAPS_LOCK) context->Modifiers ^= INPUT_MOD_CAPS_LOCK;
if (pressed && key == INPUT_KEY_NUM_LOCK) context->Modifiers ^= INPUT_MOD_NUM_LOCK;
}
// Implements the input Key To Char operation.
static char KiInputKeyToChar(INPUT_KEY key) {
static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key) {
if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) {
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
bool caps = (gModifiers & INPUT_MOD_CAPS_LOCK) != 0;
bool shifted = (context->Modifiers & INPUT_MOD_SHIFT) != 0;
bool caps = (context->Modifiers & INPUT_MOD_CAPS_LOCK) != 0;
char c = (char)('a' + (key - INPUT_KEY_A));
if (shifted ^ caps) c = (char)(c - 'a' + 'A');
return c;
}
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
bool shifted = (context->Modifiers & INPUT_MOD_SHIFT) != 0;
for (size_t i = 0; i < sizeof(gCharMap) / sizeof(gCharMap[0]); i++) {
if (gCharMap[i].key == key)
return shifted ? gCharMap[i].shifted : gCharMap[i].normal;
@@ -228,22 +207,27 @@ static char KiInputKeyToChar(INPUT_KEY key) {
// Implements the input Initialize operation.
void KeInputInitialize(const HAL_API* hal) {
gInputHal = hal;
memset((void*)gKeyDown, 0, sizeof(gKeyDown));
gModifiers = 0;
gKeyHead = gKeyTail = 0;
gRepeatKey = INPUT_KEY_NONE;
gRepeatNextTick = 0;
gMouseHead = gMouseTail = 0;
gMouseButtons = 0;
KeInitializeDpc(&gRepeatDpc, KiInputRepeatDpcRoutine, (void*)0);
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
context->Hal = hal;
memset((void*)context->KeyDown, 0, sizeof(context->KeyDown));
context->Modifiers = 0;
context->Generation = 0;
context->KeyHead = context->KeyTail = 0;
context->RepeatKey = INPUT_KEY_NONE;
context->RepeatNextTick = 0;
context->MouseHead = context->MouseTail = 0;
context->MouseBoundsWidth = 1920;
context->MouseBoundsHeight = 1080;
context->MouseWheel = 0;
context->MouseButtons = 0;
KeInitializeDpc(&context->RepeatDpc, KiInputRepeatDpcRoutine, context);
if (!gKeyboardEvent) {
KSTATUS s = KeEventCreate(EventManualReset, 0, &gKeyboardEvent);
if (!context->WaitEvent) {
KSTATUS s = KeEventCreate(EventManualReset, 0, &context->WaitEvent);
if (!KSUCCESS(s))
DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
}
KiInputRefreshKeyboardEvent();
KiInputRefreshKeyboardEvent(context);
int w = 1920, h = 1080;
DISPLAY_MODE mode;
@@ -271,11 +255,12 @@ __declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t f
// Implements the input Submit Key operation.
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
INPUT_KEY key = event->Key;
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return;
bool wasDown = gKeyDown[key] != 0;
bool wasDown = context->KeyDown[key] != 0;
bool pressed = event->Pressed;
if (pressed) {
@@ -284,24 +269,26 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
if (wasDown)
return;
gKeyDown[key] = 1u;
KiInputSetModifier(key, true);
KiInputQueueKey(key, KiInputKeyToChar(key));
KiInputStartRepeat(key, KiInputNowTicks());
context->KeyDown[key] = 1u;
KiInputSetModifier(context, key, true);
KiInputBumpGeneration(context);
KiInputQueueKey(context, key, KiInputKeyToChar(context, key));
KiInputStartRepeat(context, key, KiInputNowTicks(context));
} else {
if (!wasDown)
return;
gKeyDown[key] = 0u;
KiInputSetModifier(key, false);
KiInputStopRepeat(key);
context->KeyDown[key] = 0u;
KiInputSetModifier(context, key, false);
KiInputBumpGeneration(context);
KiInputStopRepeat(context, key);
}
}
// Implements the input Commit Mouse Position operation.
static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
static bool KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int nx, int ny, uint8_t buttons) {
int bw = context->MouseBoundsWidth;
int bh = context->MouseBoundsHeight;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
@@ -310,55 +297,70 @@ static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
if (nx >= bw) nx = bw - 1;
if (ny >= bh) ny = bh - 1;
gMouseX = nx;
gMouseY = ny;
gMouseButtons = buttons;
bool changed = (nx != context->MouseX) || (ny != context->MouseY) || (buttons != context->MouseButtons);
context->MouseX = nx;
context->MouseY = ny;
context->MouseButtons = buttons;
KeDisplaySetCursorPosition(nx, ny);
return changed;
}
// Implements the input Submit Mouse operation.
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
int oldX = gMouseX;
int oldY = gMouseY;
int oldX = context->MouseX;
int oldY = context->MouseY;
int nx = oldX + event->dx;
int ny = oldY + event->dy;
KiInputCommitMousePosition(nx, ny, event->buttons);
bool changed = KiInputCommitMousePosition(context, nx, ny, event->buttons);
if (event->wheel != 0) {
context->MouseWheel += event->wheel;
changed = true;
}
if (changed) KiInputBumpGeneration(context);
int next = (gMouseTail + 1) & (int)MOUSE_QUEUE_MASK;
if (next != gMouseHead) {
gMouseQueue[gMouseTail] = *event;
gMouseTail = next;
int next = (context->MouseTail + 1) & (int)MOUSE_QUEUE_MASK;
if (next != context->MouseHead) {
context->MouseQueue[context->MouseTail] = *event;
context->MouseTail = next;
}
}
// Implements the input Submit Mouse Absolute operation.
__declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
int oldX = gMouseX;
int oldY = gMouseY;
KiInputCommitMousePosition(event->x, event->y, event->buttons);
int oldX = context->MouseX;
int oldY = context->MouseY;
bool changed = KiInputCommitMousePosition(context, event->x, event->y, event->buttons);
if (event->wheel != 0) {
context->MouseWheel += event->wheel;
changed = true;
}
if (changed) KiInputBumpGeneration(context);
INPUT_MOUSE_EVENT rel;
rel.dx = gMouseX - oldX;
rel.dy = gMouseY - oldY;
rel.dx = context->MouseX - oldX;
rel.dy = context->MouseY - oldY;
rel.wheel = event->wheel;
rel.buttons = event->buttons;
int next = (gMouseTail + 1) & (int)MOUSE_QUEUE_MASK;
if (next != gMouseHead) {
gMouseQueue[gMouseTail] = rel;
gMouseTail = next;
int next = (context->MouseTail + 1) & (int)MOUSE_QUEUE_MASK;
if (next != context->MouseHead) {
context->MouseQueue[context->MouseTail] = rel;
context->MouseTail = next;
}
}
// Implements the input Submit Mouse Absolute Scaled operation.
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
int bw = context->MouseBoundsWidth;
int bh = context->MouseBoundsHeight;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
@@ -370,65 +372,90 @@ __declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint3
KeInputSubmitMouseAbsolute(&ev);
}
// Returns the next queued key without consuming it.
INPUT_KEY KeInputKbPoll(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (context->KeyHead == context->KeyTail) KiInputServiceRepeat(context);
return (context->KeyHead == context->KeyTail) ? INPUT_KEY_NONE : context->KeyQueue[context->KeyHead].Key;
}
// Waits until at least one keyboard event is available or the timeout expires.
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs) {
if (KeInputKbPoll() != INPUT_KEY_NONE)
// Waits for keyboard input in one context or until the timeout expires.
static KSTATUS KiInputWaitKey(KE_INPUT_CONTEXT* context, uint8_t alertable, uint64_t timeoutMs) {
if (!context)
return NTLSTATUS_INVALID_PARAMETER;
KiInputServiceRepeat(context);
if (context->KeyHead != context->KeyTail)
return NTLSTATUS_SUCCESS;
if (!gKeyboardEvent)
if (!context->WaitEvent)
return NTLSTATUS_FAILURE;
for (;;) {
EVENT* event = gKeyboardEvent;
EVENT* event = context->WaitEvent;
uint32_t selected = 0;
KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected);
if (!KSUCCESS(s))
return s;
if (KeInputKbPoll() != INPUT_KEY_NONE)
KiInputServiceRepeat(context);
if (context->KeyHead != context->KeyTail)
return NTLSTATUS_SUCCESS;
}
}
// Waits for keyboard input in the default input context.
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs) {
return KiInputWaitKey(&gDefaultInputContext, alertable, timeoutMs);
}
// Services typematic repeat from the scheduler timer bottom half.
void KeInputTimerTick(void) {
INPUT_KEY key = gRepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key])
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
INPUT_KEY key = context->RepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !context->KeyDown[key])
return;
uint64_t now = KiInputNowTicks();
if (now != 0 && now >= gRepeatNextTick)
KeInsertQueueDpc(&gRepeatDpc, (void*)0, (void*)0);
uint64_t now = KiInputNowTicks(context);
if (now != 0 && now >= context->RepeatNextTick)
KeInsertQueueDpc(&context->RepeatDpc, (void*)0, (void*)0);
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
return gKeyDown[key] != 0;
return context->KeyDown[key] != 0;
}
// Implements the input Peek Event operation.
// Peeks one keyboard event from an input context without consuming it.
static bool KiInputPeekEvent(KE_INPUT_CONTEXT* context, INPUT_KEY_RECORD* record) {
if (!context || !record) return false;
if (context->KeyHead == context->KeyTail) KiInputServiceRepeat(context);
if (context->KeyHead == context->KeyTail) return false;
INPUT_KEY_QUEUE_ENTRY entry = context->KeyQueue[context->KeyHead];
record->Key = entry.Key;
record->Char = entry.Character;
record->Modifiers = entry.Modifiers;
return true;
}
// Reads one keyboard event from an input context.
static bool KiInputReadEvent(KE_INPUT_CONTEXT* context, INPUT_KEY_RECORD* record) {
if (!KiInputPeekEvent(context, record)) return false;
context->KeyHead = (context->KeyHead + 1) & (int)KEY_QUEUE_MASK;
KiInputRefreshKeyboardEvent(context);
return true;
}
// Peeks one keyboard event from the default input context.
bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
if (!record) return false;
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return false;
INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead];
record->Key = entry.key;
record->Char = entry.ch;
record->Modifiers = entry.modifiers;
return true;
return KiInputPeekEvent(&gDefaultInputContext, record);
}
// Implements the input Read Event operation.
// Reads one keyboard event from the default input context.
bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
if (!KeInputPeekEvent(record)) return false;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
KiInputRefreshKeyboardEvent();
return true;
return KiInputReadEvent(&gDefaultInputContext, record);
}
// Implements the input Read Key operation.
@@ -445,53 +472,84 @@ char KeInputReadChar(void) {
// Implements the input Get Modifiers operation.
uint32_t KeInputGetModifiers(void) {
return gModifiers;
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
return context->Modifiers;
}
// Snapshots the current physical global input state without consuming events.
KSTATUS KeInputQueryGlobalState(SYS_GLOBAL_INPUT_STATE* out) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!out) return NTLSTATUS_INVALID_PARAMETER;
out->Size = sizeof(*out);
out->Flags = SYS_GLOBAL_INPUT_STATE_FLAG_NONE;
out->Generation = (uint64_t)context->Generation;
out->Modifiers = context->Modifiers;
memset(out->KeyDown, 0, sizeof(out->KeyDown));
for (uint32_t key = 0; key < MAX_INPUT_KEYS; key++) {
if (context->KeyDown[key])
out->KeyDown[key >> 3] |= (uint8_t)(1u << (key & 7u));
}
out->MouseX = context->MouseX;
out->MouseY = context->MouseY;
out->MouseWheel = context->MouseWheel;
out->MouseButtons = context->MouseButtons;
return NTLSTATUS_SUCCESS;
}
// Implements the input Mouse Peek operation.
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return false;
if (gMouseHead == gMouseTail) return false;
if (context->MouseHead == context->MouseTail) return false;
*event = gMouseQueue[gMouseHead];
*event = context->MouseQueue[context->MouseHead];
return true;
}
// Implements the input Mouse Read operation.
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!KeInputMousePeek(event)) return false;
gMouseHead = (gMouseHead + 1) & (int)MOUSE_QUEUE_MASK;
context->MouseHead = (context->MouseHead + 1) & (int)MOUSE_QUEUE_MASK;
return true;
}
// Implements the input Mouse Get Position operation.
void KeInputMouseGetPosition(int* x, int* y) {
if (x) *x = gMouseX;
if (y) *y = gMouseY;
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (x) *x = context->MouseX;
if (y) *y = context->MouseY;
}
// Implements the input Mouse Set Position operation.
void KeInputMouseSetPosition(int x, int y) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
int bw = context->MouseBoundsWidth;
int bh = context->MouseBoundsHeight;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
if (x < 0) x = 0;
if (y < 0) y = 0;
if (x >= bw) x = bw - 1;
if (y >= bh) y = bh - 1;
gMouseX = x;
gMouseY = y;
bool changed = (x != context->MouseX) || (y != context->MouseY);
context->MouseX = x;
context->MouseY = y;
KeDisplaySetCursorPosition(x, y);
if (changed) KiInputBumpGeneration(context);
}
// Implements the input Mouse Set Bounds operation.
void KeInputMouseSetBounds(int w, int h) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (w < 1) w = 1;
if (h < 1) h = 1;
gMouseBoundsW = w;
gMouseBoundsH = h;
KeInputMouseSetPosition(gMouseX, gMouseY);
context->MouseBoundsWidth = w;
context->MouseBoundsHeight = h;
KeInputMouseSetPosition(context->MouseX, context->MouseY);
}
// Destroys an input queue handle.
@@ -501,23 +559,42 @@ static void KiInputQueueDestroy(OBJECT* object) {
ObDereferenceObject(&queue->WaitEvent->Header);
queue->WaitEvent = (EVENT*)0;
}
queue->Context = (KE_INPUT_CONTEXT*)0;
}
// Opens the default keyboard input queue as a waitable object.
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!out) return NTLSTATUS_INVALID_PARAMETER;
*out = (INPUT_QUEUE*)0;
if (!gKeyboardEvent) return NTLSTATUS_FAILURE;
if (!context->WaitEvent) return NTLSTATUS_FAILURE;
INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
if (!queue) return NTLSTATUS_NO_MEMORY;
queue->Header.Destroy = KiInputQueueDestroy;
ObReferenceObject(&gKeyboardEvent->Header);
queue->WaitEvent = gKeyboardEvent;
queue->Context = context;
ObReferenceObject(&context->WaitEvent->Header);
queue->WaitEvent = context->WaitEvent;
*out = queue;
return NTLSTATUS_SUCCESS;
}
// Peeks one keyboard event from the queue's owning context.
bool KeInputQueuePeekEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record) {
return queue ? KiInputPeekEvent(queue->Context, record) : false;
}
// Reads one keyboard event from the queue's owning context.
bool KeInputQueueReadEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record) {
return queue ? KiInputReadEvent(queue->Context, record) : false;
}
// Waits for keyboard input in the queue's owning context.
KSTATUS KeInputQueueWaitKey(INPUT_QUEUE* queue, uint8_t alertable, uint64_t timeoutMs) {
return queue ? KiInputWaitKey(queue->Context, alertable, timeoutMs)
: NTLSTATUS_INVALID_PARAMETER;
}
// Returns the event that wakes when keyboard input can be read.
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
return queue ? queue->WaitEvent : (EVENT*)0;
code/kernel/ke_list.c
+86
View File
@@ -0,0 +1,86 @@
// Kernel checked intrusive-list helpers. This file detects stale or duplicate membership at the mutation site.
#include <stdint.h>
#include <kernel/ke_list.h>
#include <kernel/ke_debug.h>
#define KE_LIST_VALIDATION_LIMIT 1048576u
// Returns whether checked list validation is enabled for this boot.
static bool KiListChecksEnabled(void) {
return (KeGetDebugFlags() & KE_DEBUG_LIST_CHECKS) != 0;
}
// Stops on a checked-list invariant violation with the involved addresses.
static _Noreturn void KiListPanic(const char* operation,
const LIST_ENTRY* head,
const LIST_ENTRY* node) {
KePanic(KE_PANIC_CODE_GENERIC,
"list: %s invariant failed head=0x%016llX node=0x%016llX fwd=0x%016llX bwd=0x%016llX\n",
operation ? operation : "unknown",
(uint64_t)(uintptr_t)head,
(uint64_t)(uintptr_t)node,
(uint64_t)(uintptr_t)(node ? node->Fwd : (LIST_ENTRY*)0),
(uint64_t)(uintptr_t)(node ? node->Bwd : (LIST_ENTRY*)0));
}
// Validates a list head and every forward/backward linkage reachable from it.
void KeListValidateHead(const LIST_ENTRY* head) {
if (!KiListChecksEnabled())
return;
if (!head || !head->Fwd || !head->Bwd)
KiListPanic("validate-head", head, head);
const LIST_ENTRY* node = head;
for (uint32_t count = 0; count < KE_LIST_VALIDATION_LIMIT; count++) {
if (!node->Fwd || !node->Bwd)
KiListPanic("validate-link", head, node);
if (node->Fwd->Bwd != node || node->Bwd->Fwd != node)
KiListPanic("validate-neighbor", head, node);
node = node->Fwd;
if (node == head)
return;
}
KiListPanic("validate-cycle", head, node);
}
// Inserts a detached node at the tail after validating both invariants.
void KeListInsertTailChecked(LIST_ENTRY* head, LIST_ENTRY* node) {
if (KiListChecksEnabled()) {
KeListValidateHead(head);
if (!node || node->Fwd != node || node->Bwd != node)
KiListPanic("insert-tail", head, node);
}
ListInsertTail(head, node);
KeListValidateHead(head);
}
// Inserts a detached node at the head after validating both invariants.
void KeListInsertHeadChecked(LIST_ENTRY* head, LIST_ENTRY* node) {
if (KiListChecksEnabled()) {
KeListValidateHead(head);
if (!node || node->Fwd != node || node->Bwd != node)
KiListPanic("insert-head", head, node);
}
ListInsertHead(head, node);
KeListValidateHead(head);
}
// Removes a linked node and rejects detached or locally corrupt entries.
void KeListRemoveChecked(LIST_ENTRY* node) {
if (KiListChecksEnabled()) {
if (!node || !node->Fwd || !node->Bwd ||
(node->Fwd == node && node->Bwd == node) ||
node->Fwd->Bwd != node || node->Bwd->Fwd != node) {
KiListPanic("remove", (LIST_ENTRY*)0, node);
}
}
ListRemove(node);
if (KiListChecksEnabled() && (node->Fwd != node || node->Bwd != node))
KiListPanic("remove-detach", (LIST_ENTRY*)0, node);
}
code/kernel/ke_mutex.c
+3 -2
View File
@@ -1,6 +1,7 @@
// Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups.
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
@@ -38,7 +39,7 @@ __declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) {
}
KePrepareBlockThread(self, &mutex->Header, 0);
ListInsertTail(&mutex->WaitList, &self->SchedulerLink);
KeListInsertTailChecked(&mutex->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &mutex->WaitList);
KeReleaseSpinlock(&mutex->Lock, flags);
return KeCommitBlockThread(self, 0);
@@ -64,7 +65,7 @@ __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
LIST_ENTRY* node = KeListPopHeadChecked(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
// Direct handoff: the selected waiter owns the mutex when its wait
code/kernel/ke_process.c
+16 -8
View File
@@ -1,6 +1,7 @@
// Kernel process model. This file owns address-space metadata, handle tables, and process lifetime transitions.
#include <string.h>
#include <kernel/ke_list.h>
#include <intrin.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
@@ -240,6 +241,7 @@ static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING;
case ObjectTypeSurface: return SYS_THREAD_WAIT_SURFACE;
case ObjectTypeTimer: return SYS_THREAD_WAIT_TIMER;
case ObjectTypeDisplayClock: return SYS_THREAD_WAIT_DISPLAY_CLOCK;
default: return SYS_THREAD_WAIT_UNKNOWN;
}
}
@@ -377,7 +379,7 @@ static void KiProcessDestroy(OBJECT* obj) {
// Remove from global list.
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
ListRemove(&proc->GlobalLink);
KeListRemoveChecked(&proc->GlobalLink);
KeReleaseSpinlock(&gProcessListLock, flags);
// Memory for the struct itself is freed by ObDereferenceObject after
@@ -453,7 +455,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
}
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
ListInsertTail(&gProcessList, &proc->GlobalLink);
KeListInsertTailChecked(&gProcessList, &proc->GlobalLink);
KeReleaseSpinlock(&gProcessListLock, flags);
*out = proc;
@@ -681,7 +683,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
KCPU* cpu = &gCpus[t->RunningCpu];
if (cpu->Present && cpu->Started && cpu->Online) {
if (cpu->State == KeCpuOnline) {
_InterlockedExchange(&cpu->ReschedulePending, 1);
HalApicSendRescheduleIpi(cpu->LapicId);
}
@@ -718,7 +720,10 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri
for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
if (!process->Handles[i].Object) {
ObReferenceObject(object);
if (!ObTryReferenceObject(object)) {
KeReleaseSpinlock(&process->HandleLock, flags);
return NTLSTATUS_OBJECT_DEAD;
}
process->Handles[i].Object = object;
process->Handles[i].Rights = rights;
process->Handles[i].Flags = 0;
@@ -767,7 +772,8 @@ OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t req
OBJECT* obj = entry->Object;
if (obj && requiredRights && ((entry->Rights & requiredRights) != requiredRights))
obj = (OBJECT*)0;
if (obj) ObReferenceObject(obj);
if (obj && !ObTryReferenceObject(obj))
obj = (OBJECT*)0;
KeReleaseSpinlock(&process->HandleLock, flags);
return obj;
}
@@ -819,7 +825,10 @@ KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle
obj = entry->Object;
sourceRights = entry->Rights;
ObReferenceObject(obj);
if (!ObTryReferenceObject(obj)) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return NTLSTATUS_OBJECT_DEAD;
}
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
uint32_t targetRights = sourceRights;
@@ -844,8 +853,7 @@ PROCESS* KeGetProcessById(uint32_t pid) {
LIST_ENTRY* node = head->Fwd;
while (node != head) {
PROCESS* p = CONTAINING_RECORD(node, PROCESS, GlobalLink);
if (p->ProcessId == pid) {
ObReferenceObject(&p->Header);
if (p->ProcessId == pid && ObTryReferenceObject(&p->Header)) {
KeReleaseSpinlock(&gProcessListLock, flags);
return p;
}
code/kernel/ke_sched.c
+91 -45
View File
@@ -13,6 +13,7 @@
#include <kernel/ke_debug.h>
#include <kernel/ke_input.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_display.h>
#include <kernel/ke_list.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
@@ -48,7 +49,7 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
// Implements the cpu Is Idle Locked operation.
static int KiCpuIsIdleLocked(const KCPU* cpu) {
if (!cpu || !cpu->Online)
if (!cpu || cpu->State != KeCpuOnline)
return 0;
THREAD* cur = cpu->CurrentThread;
return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
@@ -61,7 +62,7 @@ static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
if (cpu->State != KeCpuOnline)
continue;
if (!cpu->IsPrimaryThread)
continue;
@@ -89,7 +90,7 @@ static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
// Implements the request Reschedule Cpu Locked operation.
static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || !target->Online)
if (!target || target->State != KeCpuOnline)
return;
_InterlockedExchange(&target->ReschedulePending, 1);
@@ -110,7 +111,7 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
if (cpu->State != KeCpuOnline)
continue;
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
@@ -135,7 +136,7 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
if (!requestedIdle && haveIdlePrimary == 0) {
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
if (cpu->State != KeCpuOnline)
continue;
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
@@ -265,7 +266,7 @@ static void KiInsertReadyLocked(THREAD* thread) {
thread->State = ThreadReady;
thread->Queued = 1;
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
KeListInsertTailChecked(&gRunQueue[p], &thread->SchedulerLink);
}
static int KiAnyRunnableForCpuLocked(KCPU* cpu);
@@ -367,13 +368,24 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
return NTLSTATUS_NO_MEMORY;
}
idle->KernelStackBase = kstack;
idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
idle->KernelStackTop = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
idle->TssRsp0 = idle->KernelStackTop;
uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop;
uintptr_t* ksp = (uintptr_t*)idle->KernelStackTop;
// Kernel stacks may come from allocators with weaker alignment than the
// MS x64 call ABI requires. Bias the top down before building synthetic
// entry frames so optimized C code can use aligned stack spills safely.
// KiIdleLoop is a C function reached by a synthetic RET, not by CALL.
// Leave the fake return slot and the MS x64 32-byte home area below the
// top of stack so optimized prologues may spill above RSP without writing
// into the pool canary or footer.
ksp -= 5;
__stosq((unsigned __int64*)ksp, 0, 5);
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
__stosq((unsigned __int64 *)ctx, 0, sizeof(KCONTEXT) / 8);
KCONTEXT* ctx = (KCONTEXT*)ksp;
__stosq((unsigned __int64*)ctx, 0, sizeof(KCONTEXT) / 8);
ctx->Rip = (uint64_t)KiIdleLoop;
idle->SavedContext = ctx;
KeInitThreadFpuContext(idle);
@@ -383,7 +395,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
// as the real idle process while kernel worker threads live under PID 1.
ObReferenceObject(&gIdleProcess->Header);
uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock);
ListInsertTail(&gIdleProcess->ThreadList, &idle->ProcessLink);
KeListInsertTailChecked(&gIdleProcess->ThreadList, &idle->ProcessLink);
gIdleProcess->ThreadCount++;
KeReleaseSpinlock(&gIdleProcess->HandleLock, idleFlags);
@@ -403,6 +415,8 @@ void KeInitializeScheduler(void) {
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create CPU%u idle thread, status=%u\n",
(unsigned)cpu->CpuIndex, s);
_InterlockedExchange(&cpu->State, KeCpuOnline);
}
// Safely release the lock and clean up any dead threads natively post-switch.
@@ -456,7 +470,7 @@ void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (thread->Queued) {
ListRemove(&thread->SchedulerLink);
KeListRemoveChecked(&thread->SchedulerLink);
thread->Queued = 0;
}
KeReleaseSpinlock(&gSchedLock, flags);
@@ -495,7 +509,7 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
continue;
}
ListRemove(node);
KeListRemoveChecked(node);
thread->Queued = 0;
KiAssertPickedThreadLocked("KiPickNextForCpu", thread);
return thread;
@@ -666,7 +680,7 @@ void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (thread->State == ThreadWaiting && thread->WaitListHead) {
ListRemove(&thread->SchedulerLink);
KeListRemoveChecked(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
@@ -692,7 +706,7 @@ static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
if (thread->State == ThreadWaiting &&
thread->WaitListHead == &gSleepList &&
thread->SleepUntilTick <= nowTick) {
ListRemove(node);
KeListRemoveChecked(node);
thread->WaitListHead = (LIST_ENTRY*)0;
KiWakeThreadLocked(thread, thread->TimeoutStatus);
}
@@ -701,20 +715,49 @@ static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
}
}
// Implements the scheduler Tick operation.
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
THREAD* idle = cpu->IdleThread;
if (!cur) return;
// Checks whether this CPU owns global scheduler tick duties.
static int KiCpuOwnsGlobalSchedulerTick(const KCPU* cpu) {
// LAPIC timer interrupts are per-CPU, but sleep deadlines, waitable timers,
// input repeat, and priority aging are global scheduler services. Keep the
// global clock single-sourced on the BSP so enabling AP timers does not
// make those services run once per online CPU.
return cpu && cpu->IsBsp;
}
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiArmPostSwitchUnlock(cur, flags);
// Ages ready threads after enough global scheduler ticks have elapsed.
static void KiAgeReadyThreadsLocked(void) {
// Periodic aging prevents indefinite starvation of PriorityLow and
// PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
// burst history; they earned the boost by waiting, not by burning CPU.
gAgingCounter++;
if (gAgingCounter < SCHED_AGING_INTERVAL_TICKS)
return;
gAgingCounter = 0;
for (int p = 0; p < PriorityHigh; p++) {
while (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = KeListPopHeadChecked(&gRunQueue[p]);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
t->Priority = PriorityHigh;
t->CpuBurstTicks = 0;
t->QuantaConsumed = 0;
t->QuantumRemaining = 0;
gSchedStats.AgingPromotions++;
KeListInsertTailChecked(&gRunQueue[PriorityHigh], &t->SchedulerLink);
}
}
}
// Runs scheduler work that must be single-sourced for the whole system.
static void KiRunGlobalSchedulerTickLocked(void) {
gSchedStats.Ticks++;
KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
KeInputTimerTick();
KiAgeReadyThreadsLocked();
}
// Services the current CPU's local scheduler timer interrupt.
static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) {
int requestReschedule = 0;
if (cur == idle) {
@@ -756,29 +799,32 @@ void KeSchedulerTick(void) {
}
}
// Periodic aging pass to prevent indefinite starvation of PriorityLow and
// PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
// burst history; they earned the boost by waiting, not by burning CPU.
gAgingCounter++;
if (gAgingCounter >= SCHED_AGING_INTERVAL_TICKS) {
gAgingCounter = 0;
for (int p = 0; p < PriorityHigh; p++) {
while (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = ListPopHead(&gRunQueue[p]);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
t->Priority = PriorityHigh;
t->CpuBurstTicks = 0;
t->QuantaConsumed = 0;
t->QuantumRemaining = 0;
gSchedStats.AgingPromotions++;
ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink);
}
}
}
return requestReschedule;
}
// Implements the scheduler Tick operation.
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
THREAD* idle = cpu->IdleThread;
if (!cur) return;
int ownsGlobalTick = KiCpuOwnsGlobalSchedulerTick(cpu);
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiArmPostSwitchUnlock(cur, flags);
if (ownsGlobalTick)
KiRunGlobalSchedulerTickLocked();
int requestReschedule = KiRunLocalSchedulerTickLocked(cpu, cur, idle);
KeReleaseSpinlock(&gSchedLock, flags);
KeTimerTick();
if (ownsGlobalTick) {
KeTimerTick();
KeDisplayTick();
}
if (requestReschedule)
KiRequestReschedule();
@@ -816,7 +862,7 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus)
self->WaitListHead = &gSleepList;
self->SleepUntilTick = wakeTick;
self->TimeoutStatus = timeoutStatus;
ListInsertTail(&gSleepList, &self->SchedulerLink);
KeListInsertTailChecked(&gSleepList, &self->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return NTLSTATUS_SUCCESS;
}
@@ -861,7 +907,7 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
self->AlertableWait = 0;
self->SleepUntilTick = wakeTick;
self->TimeoutStatus = NTLSTATUS_SUCCESS;
ListInsertTail(&gSleepList, &self->SchedulerLink);
KeListInsertTailChecked(&gSleepList, &self->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return KeCommitBlockThread(self, 0);
code/kernel/ke_selftest.c
+7 -13
View File
@@ -1189,11 +1189,9 @@ void KiRunCoreHardeningSelfTests(void) {
(uint64_t)cpu->Self, (uint64_t)cpu);
if (KeGetCurrentCpuIndex() != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n");
if (!cpu->Present || !cpu->Started || !cpu->Online || !cpu->IsBsp)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle flags invalid P/S/O/B=%u/%u/%u/%u\n",
cpu->Present, cpu->Started, cpu->Online, cpu->IsBsp);
if (cpu->Parked)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP is marked parked\n");
if (cpu->State != KeCpuOnline || !cpu->IsBsp)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle invalid state=%ld bsp=%u\n",
cpu->State, cpu->IsBsp);
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql());
if (KeGetCurrentTss() != &cpu->Tss)
@@ -1207,16 +1205,12 @@ void KiRunCoreHardeningSelfTests(void) {
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* ap = &gCpus[i];
if (!ap->Present)
if (ap->State == KeCpuAbsent)
continue;
if (!ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
if (ap->State != KeCpuOnline || ap->IsBsp)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n",
i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp);
if (!ap->SchedulerCommand || !ap->SchedulerActive)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u scheduler admission invalid command=%ld active=%ld\n",
i, ap->SchedulerCommand, ap->SchedulerActive);
"CORE selftest: AP%u lifecycle invalid state=%ld bsp=%u\n",
i, ap->State, ap->IsBsp);
if (!ap->KernelStack || !ap->KernelStackTop)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
code/kernel/ke_semaphore.c
+3 -2
View File
@@ -1,6 +1,7 @@
// Kernel semaphore primitive. This file owns counting wakeups and bounded resource acquisition.
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
@@ -39,7 +40,7 @@ __declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
}
KePrepareBlockThread(self, &sem->Header, 0);
ListInsertTail(&sem->WaitList, &self->SchedulerLink);
KeListInsertTailChecked(&sem->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &sem->WaitList);
KeReleaseSpinlock(&sem->Lock, flags);
return KeCommitBlockThread(self, 0);
@@ -66,7 +67,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
// Transfer this semaphore release directly to the selected waiter. The
// count stays unchanged; the waiter returns success without looping and
// consuming a count later.
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
LIST_ENTRY* node = KeListPopHeadChecked(&sem->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
KeReleaseSpinlock(&sem->Lock, flags);
code/kernel/ke_smp.c
+107 -91
View File
@@ -6,6 +6,7 @@
// to enter KiApKernelEntry() and park in kernel-owned state.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/status.h>
@@ -80,6 +81,20 @@ static PHYSICAL_ADDRESS gApTrampolinePhys;
static PHYSICAL_ADDRESS gReservedApTrampolinePhys;
static AP_BOOT_BLOCK* gApBootBlock;
// Returns whether investigation-only SMP diagnostics are enabled.
static bool KiSmpDiagnosticsEnabled(void) {
return (KeGetDebugFlags() & KE_DEBUG_SELFTEST) != 0;
}
// Validates pool state around AP admission so SMP races fail near their trigger.
static void KiSmpPoolCheckpoint(const char* phase, uint32_t cpuIndex) {
if (!KiSmpDiagnosticsEnabled())
return;
DbgPrint("SMP: pool checkpoint %s AP%u\n", phase ? phase : "unknown", cpuIndex);
(void)MmValidateKernelPool(phase ? phase : "smp");
}
// Implements the emit8 operation.
static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
// Implements the emit16 operation.
@@ -127,27 +142,35 @@ static KSTATUS KiBuildApTrampoline(void) {
if (frame < AP_TRAMPOLINE_FALLBACK_MIN ||
frame >= AP_TRAMPOLINE_MAX_PHYS ||
(frame & (PAGE_SIZE - 1))) {
DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
}
frame = 0;
} else {
DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
}
}
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
}
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_PREFERRED_MIN,
AP_TRAMPOLINE_MAX_PHYS,
MmPfnUseKernel);
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
}
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_FALLBACK_MIN,
AP_TRAMPOLINE_PREFERRED_MIN,
MmPfnUseKernel);
@@ -158,7 +181,8 @@ static KSTATUS KiBuildApTrampoline(void) {
return NTLSTATUS_NO_MEMORY;
}
DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame);
memset(page, 0, PAGE_SIZE);
@@ -285,13 +309,12 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
KeLoadIdtForCurrentCpu();
if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_IDT;
cpu->Started = 1;
cpu->Parked = 1;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuStarted);
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
_InterlockedExchange(&cpu->State, KeCpuParked);
if (gApBootBlock) {
gApBootBlock->Stage = AP_STAGE_PARK;
gApBootBlock->State = AP_STATE_KERNEL_PARKED;
@@ -300,32 +323,30 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
for (;;) {
cpu->SchedulerTicks++;
if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
THREAD* idle = cpu->IdleThread;
if (idle) {
cpu->CurrentThread = idle;
idle->State = ThreadRunning;
idle->RunningCpu = cpu->CpuIndex;
idle->Queued = 0;
cpu->Parked = 0;
// Do not arm the AP's periodic LAPIC timer while it is still in
// the raw park loop. Timer IRQs are scheduler-owned events;
// the AP may take them only after it has a current idle thread.
HalTimerConfigureCurrentCpu();
_InterlockedExchange(&cpu->SchedulerActive, 1);
_InterlockedExchange(&cpu->State, KeCpuSchedulerActive);
HalEnableInterrupts();
KeStartInitialThread(idle);
}
}
// The BSP admits parked APs by storing SchedulerCommand and sending a
// wake IPI. Avoid the classic lost wakeup where the command appears
// between the poll and HLT: check once with interrupts disabled, then
// STI;HLT as a single sleep transition.
// The BSP admits parked APs by publishing AdmissionRequested and
// sending a wake IPI. Avoid the classic lost wakeup where the state
// changes between the poll and HLT: check once with interrupts
// disabled, then use STI;HLT as a single sleep transition.
uintptr_t interruptFlags = HalDisableInterrupts();
if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
HalRestoreInterrupts(interruptFlags);
continue;
}
@@ -342,10 +363,7 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->LapicId = info->LapicId;
cpu->AcpiProcessorId = info->ProcessorUid;
cpu->BootProcessorIndex = cpuIndex;
cpu->Present = info->Enabled ? 1 : 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
cpu->State = info->Enabled ? KeCpuPresent : KeCpuAbsent;
cpu->IsBsp = info->IsBsp ? 1 : 0;
cpu->PackageId = 0;
cpu->CoreId = cpuIndex;
@@ -359,8 +377,6 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
cpu->SchedulerCommand = 0;
cpu->SchedulerActive = 0;
}
// Implements the ceil Log2 U32 operation.
@@ -496,7 +512,7 @@ void KeBuildProcessorTopology(void) {
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
if (cpu->State == KeCpuAbsent)
continue;
uint32_t apic = cpu->LapicId;
@@ -515,7 +531,7 @@ void KeBuildProcessorTopology(void) {
uint32_t nextCoreIndex = 0;
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
if (cpu->State == KeCpuAbsent)
continue;
if (cpu->PhysicalCoreIndex != KE_INVALID_CPU)
continue;
@@ -528,7 +544,7 @@ void KeBuildProcessorTopology(void) {
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
if (peer->State == KeCpuAbsent)
continue;
if (peer->PackageId == cpu->PackageId) {
if (j < 64)
@@ -549,7 +565,7 @@ void KeBuildProcessorTopology(void) {
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
if (peer->State == KeCpuAbsent)
continue;
if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
peer->CoreSiblingMask = coreMaskBits;
@@ -568,19 +584,21 @@ void KeBuildProcessorTopology(void) {
topo.LogicalPerPackage,
nextCoreIndex);
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
i,
cpu->LapicId,
cpu->PackageId,
cpu->CoreId,
cpu->ThreadId,
cpu->IsPrimaryThread,
(unsigned long long)cpu->CoreSiblingMask,
(unsigned long long)cpu->PackageMask);
if (KiSmpDiagnosticsEnabled()) {
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (cpu->State == KeCpuAbsent)
continue;
DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
i,
cpu->LapicId,
cpu->PackageId,
cpu->CoreId,
cpu->ThreadId,
cpu->IsPrimaryThread,
(unsigned long long)cpu->CoreSiblingMask,
(unsigned long long)cpu->PackageMask);
}
}
}
@@ -598,7 +616,7 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
}
cpu->KernelStack = stack;
cpu->KernelStackTop = (uintptr_t)stack + stackBytes;
cpu->KernelStackTop = ((uintptr_t)stack + stackBytes) & ~(uintptr_t)15;
memset(cpu->KernelStack, 0, stackBytes);
memset(&cpu->Tss, 0, sizeof(cpu->Tss));
cpu->Tss.IoMapBase = sizeof(TSS64);
@@ -647,12 +665,7 @@ KSTATUS KeStartApplicationProcessors(void) {
KCPU* bsp = KeGetCurrentCpu();
bsp->LapicId = bspLapic;
bsp->IsBsp = 1;
bsp->Present = 1;
bsp->Started = 1;
bsp->Online = 1;
bsp->Parked = 0;
bsp->SchedulerCommand = 0;
bsp->SchedulerActive = 1;
bsp->State = KeCpuStarted;
DbgPrint("SMP: APIC mode=%s BSP LAPIC=%u MADT CPUs=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
@@ -661,9 +674,11 @@ KSTATUS KeStartApplicationProcessors(void) {
uint32_t count = 1;
for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) {
HAL_CPU_INFO* info = &topo.Cpus[i];
DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
i, info->ProcessorUid, info->LapicId, info->Flags,
info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
i, info->ProcessorUid, info->LapicId, info->Flags,
info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
}
if (!info->Enabled) continue;
if (info->LapicId == bspLapic) {
@@ -693,8 +708,10 @@ KSTATUS KeStartApplicationProcessors(void) {
return NTLSTATUS_SUCCESS;
}
DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
(uint64_t)gReservedApTrampolinePhys);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
(uint64_t)gReservedApTrampolinePhys);
}
st = KiBuildApTrampoline();
if (!KSUCCESS(st)) {
DbgPrint("SMP: disabled; AP trampoline allocation/build failed status=0x%x\n", st);
@@ -707,30 +724,25 @@ KSTATUS KeStartApplicationProcessors(void) {
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present) continue;
if (cpu->State == KeCpuAbsent) continue;
st = KiAllocateApStack(cpu);
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
KiFillBootBlockForCpu(cpu, i);
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
st = HalStartApplicationProcessor(cpu->LapicId, sipiVector);
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
@@ -752,18 +764,18 @@ KSTATUS KeStartApplicationProcessors(void) {
DbgPrint("SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
cpu->Started = 1;
cpu->Parked = 1;
cpu->Online = 0;
if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u boot block parked without CPU state transition state=%ld\n",
i, cpu->State);
}
claimed++;
DbgPrint("SMP: AP%u reached kernel park\n", i);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: AP%u reached kernel park\n", i);
}
if (claimed == 0 && gCpuCount > 1)
@@ -771,6 +783,7 @@ KSTATUS KeStartApplicationProcessors(void) {
KeBuildProcessorTopology();
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
KiSmpPoolCheckpoint("smp after AP park", 0);
return NTLSTATUS_SUCCESS;
}
@@ -784,32 +797,33 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
uint32_t admitted = 0;
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present) continue;
if (cpu->State == KeCpuAbsent) continue;
if (!cpu->Started || !cpu->Parked || cpu->Online) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u cannot enter scheduler P/S/Park/O=%u/%u/%u/%u\n",
i, cpu->Present, cpu->Started, cpu->Parked, cpu->Online);
"SMP: AP%u cannot enter scheduler from state=%ld\n",
i, cpu->State);
}
KiSmpPoolCheckpoint("smp before AP idle create", i);
KSTATUS st = KiCreateIdleThreadForCpu(cpu);
if (!KSUCCESS(st))
return st;
KiSmpPoolCheckpoint("smp after AP idle create", i);
cpu->Online = 1;
_ReadWriteBarrier();
_InterlockedExchange(&cpu->SchedulerCommand, 1);
_InterlockedExchange(&cpu->State, KeCpuAdmissionRequested);
KiSmpPoolCheckpoint("smp before AP wake", i);
// Parked APs sleep with HLT and have no LAPIC timer armed yet. The
// command store alone is not sufficient; send a fixed-vector IPI only
// to wake the parked loop. ReschedulePending is intentionally left
// state transition alone is not sufficient; send a fixed-vector IPI
// only to wake the parked loop. ReschedulePending is intentionally left
// clear, so the interrupt exit path will not try to schedule before
// the AP has installed its idle thread.
HalApicSendRescheduleIpi(cpu->LapicId);
int active = 0;
for (uint32_t spin = 0; spin < AP_SCHED_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&cpu->SchedulerActive, 0, 0) != 0) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuSchedulerActive) {
active = 1;
break;
}
@@ -818,13 +832,15 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
if (!active) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u failed scheduler admission lapic=%u parked=%u online=%u command=%ld active=%ld ticks=%llu\n",
i, cpu->LapicId, cpu->Parked, cpu->Online,
cpu->SchedulerCommand, cpu->SchedulerActive, cpu->SchedulerTicks);
"SMP: AP%u failed scheduler admission lapic=%u state=%ld ticks=%llu\n",
i, cpu->LapicId, cpu->State, cpu->SchedulerTicks);
}
_InterlockedExchange(&cpu->State, KeCpuOnline);
KiSmpPoolCheckpoint("smp after AP scheduler active", i);
admitted++;
DbgPrint("SMP: AP%u entered scheduler idle thread\n", i);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: AP%u entered scheduler idle thread\n", i);
}
DbgPrint("SMP: scheduler admitted %u APs\n", admitted);
@@ -946,7 +962,7 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
uint32_t workerCount = 0;
for (uint32_t cpuIndex = 1; cpuIndex < gCpuCount; cpuIndex++) {
KCPU* cpu = &gCpus[cpuIndex];
if (!cpu->Present || !cpu->Started || !cpu->Online || cpu->Parked)
if (cpu->State != KeCpuOnline)
continue;
if (cpuIndex >= 64)
continue;
code/kernel/ke_switch.asm
+15 -11
View File
@@ -1,17 +1,17 @@
; ke_switch.asm - context switch and thread entry stubs (MASM64 / ML64)
;
; Struct offsets:
; sizeof(OBJECT) = 16 (type:4, refcount:4, destroy:8)
; THREAD.SavedContext offset = 88
; THREAD.TssRsp0 offset = 104
; THREAD.Teb offset = 112
; THREAD.FpuContext offset = 168
; sizeof(OBJECT) = 24 (type:4, refcount:4, state:4, flags:4, destroy:8)
; THREAD.SavedContext offset = 96
; THREAD.TssRsp0 offset = 112
; THREAD.Teb offset = 120
; THREAD.FpuContext offset = 176
; TSS64.RSP0 offset = 4
;
THREAD_SavedContext EQU 88
THREAD_TssRsp0 EQU 104
THREAD_Teb EQU 112
THREAD_FpuContext EQU 168
THREAD_SavedContext EQU 96
THREAD_TssRsp0 EQU 112
THREAD_Teb EQU 120
THREAD_FpuContext EQU 176
TSS_RSP0_OFFSET EQU 4
EXTERN KeSetCurrentTssRsp0 : PROC
@@ -135,9 +135,9 @@ EXTERN KeGetCurrentThread : PROC
EXTERN KeExitThread : PROC
KeThreadEntry PROC
sub rsp, 32 ; Reserve shadow space
sub rsp, 40 ; Reserve shadow space and restore MS x64 call alignment.
call KeSchedulerPostSwitch
add rsp, 32
add rsp, 40
pop rcx ; load user arg
swapgs ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU
iretq ; -> ring 3
@@ -160,6 +160,10 @@ KeKernelThreadEntry PROC
pop rbx ; rbx = fn ptr (non-volatile, survives the call below)
pop r12 ; r12 = arg (non-volatile)
; The synthetic entry frame is not reached by CALL. Drop any residual
; allocator-induced stack skew before making normal MS x64 calls.
and rsp, -16
sub rsp, 32
call KeSchedulerPostSwitch ; release scheduler spinlock, reap dead threads
add rsp, 32
code/kernel/ke_syscall.c
+105 -23
View File
@@ -616,6 +616,9 @@ static KSTATUS KiHandleToWaitEvent(PROCESS* proc, uint32_t handle, EVENT** event
case ObjectTypeTimer:
event = KeTimerWaitEvent(CONTAINING_RECORD(obj, KTIMER, Header));
break;
case ObjectTypeDisplayClock:
event = KeDisplayClockWaitEvent((DISPLAY_CLOCK_OBJECT*)obj);
break;
default:
ObDereferenceObject(obj);
return NTLSTATUS_NOT_SUPPORTED;
@@ -2189,6 +2192,35 @@ static uint64_t SyscallStatFile(uint64_t a0, uint64_t a1, uint64_t a2,
return CopyToUser(req.StatOut, &stat, sizeof(stat));
}
// SYS_OPEN_DISPLAY_CLOCK a0=*SYS_OPEN_DISPLAY_CLOCK_REQUEST.
static uint64_t SyscallOpenDisplayClock(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_OPEN_DISPLAY_CLOCK_REQUEST req;
KSTATUS s = KiCopyUserRequest(a0, &req, sizeof(req));
if (!KSUCCESS(s)) return s;
if (req.Size != sizeof(req) || req.Flags != SYS_DISPLAY_CLOCK_FLAG_NONE || !req.HandleOut)
return NTLSTATUS_INVALID_PARAMETER;
DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)0;
s = KeDisplayOpenClock(req.PeriodMilliseconds, &clock);
if (!KSUCCESS(s))
return s;
uint32_t handle = 0;
s = KiAllocHandleForCurrentWithRights(&clock->Header, HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY, &handle);
if (!KSUCCESS(s))
return s;
s = KiCopyHandleToUser((uint64_t)(uintptr_t)req.HandleOut, handle);
if (!KSUCCESS(s)) {
PROCESS* proc = KeGetCurrentProcess();
if (proc) KeCloseHandle(proc, handle);
}
return s;
}
// SYS_CREATE_SURFACE a0=*SYS_CREATE_SURFACE_REQUEST.
static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
@@ -2569,13 +2601,15 @@ static uint64_t SyscallFillConsoleAttribute(uint64_t a0, uint64_t a1, uint64_t a
return s;
}
// Validates a user input queue handle. Handle zero means the default queue.
static KSTATUS KiValidateInputQueueHandle(uint32_t handle) {
// References a user input queue handle. Handle zero selects the default context.
static KSTATUS KiReferenceInputQueueHandle(uint32_t handle, INPUT_QUEUE** queueOut) {
if (!queueOut) return NTLSTATUS_INVALID_PARAMETER;
*queueOut = (INPUT_QUEUE*)0;
if (handle == 0) return NTLSTATUS_SUCCESS;
OBJECT* obj = KiHandleToObject(handle, ObjectTypeInputQueue, HANDLE_RIGHT_READ);
if (!obj) return NTLSTATUS_INVALID_HANDLE;
ObDereferenceObject(obj);
OBJECT* object = KiHandleToObject(handle, ObjectTypeInputQueue, HANDLE_RIGHT_READ);
if (!object) return NTLSTATUS_INVALID_HANDLE;
*queueOut = CONTAINING_RECORD(object, INPUT_QUEUE, Header);
return NTLSTATUS_SUCCESS;
}
@@ -2592,7 +2626,7 @@ static KSTATUS KiCopyInputEventRequest(uint64_t arg, SYS_INPUT_EVENT_REQUEST* re
if (req->Size != sizeof(*req)) return NTLSTATUS_INVALID_PARAMETER;
if (req->Flags != SYS_INPUT_EVENT_FLAG_NONE) return NTLSTATUS_INVALID_PARAMETER;
if (!req->RecordOut) return NTLSTATUS_INVALID_PARAMETER;
return KiValidateInputQueueHandle(req->QueueHandle);
return NTLSTATUS_SUCCESS;
}
// SYS_POLL_KEYBOARD a0=*INPUT_KEY -> KSTATUS.
@@ -2658,6 +2692,22 @@ static uint64_t SyscallKbGetModifiers(uint64_t a0, uint64_t a1, uint64_t a2,
return (uint64_t)KeInputGetModifiers();
}
// SYS_QUERY_GLOBAL_INPUT_STATE a0=*SYS_GLOBAL_INPUT_STATE -> KSTATUS.
static uint64_t SyscallQueryGlobalInputState(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_GLOBAL_INPUT_STATE state;
KSTATUS s = KiCopyUserRequest(a0, &state, sizeof(state));
if (!KSUCCESS(s)) return s;
if (state.Size != sizeof(state)) return NTLSTATUS_INVALID_PARAMETER;
if (state.Flags != SYS_GLOBAL_INPUT_STATE_FLAG_NONE) return NTLSTATUS_INVALID_PARAMETER;
s = KeInputQueryGlobalState(&state);
if (!KSUCCESS(s)) return s;
return CopyToUser((void*)(uintptr_t)a0, &state, sizeof(state));
}
// SYS_READ_INPUT_EVENT a0=*SYS_INPUT_EVENT_REQUEST -> KSTATUS.
static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
@@ -2667,18 +2717,32 @@ static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
KSTATUS s = KiCopyInputEventRequest(a0, &req);
if (!KSUCCESS(s)) return s;
INPUT_QUEUE* queue = (INPUT_QUEUE*)0;
s = KiReferenceInputQueueHandle(req.QueueHandle, &queue);
if (!KSUCCESS(s)) return s;
INPUT_KEY_RECORD record;
for (;;) {
if (KeInputReadEvent(&record))
return KiCopyInputRecordToUser(req.RecordOut, &record);
bool available = queue ? KeInputQueueReadEvent(queue, &record)
: KeInputReadEvent(&record);
if (available) {
s = KiCopyInputRecordToUser(req.RecordOut, &record);
break;
}
if (req.TimeoutMilliseconds == 0)
return NTLSTATUS_WOULD_BLOCK;
if (req.TimeoutMilliseconds == 0) {
s = NTLSTATUS_WOULD_BLOCK;
break;
}
s = KeInputWaitKey(0, req.TimeoutMilliseconds);
s = queue ? KeInputQueueWaitKey(queue, 0, req.TimeoutMilliseconds)
: KeInputWaitKey(0, req.TimeoutMilliseconds);
if (!KSUCCESS(s))
return s;
break;
}
if (queue) ObDereferenceObject(&queue->Header);
return s;
}
// SYS_PEEK_INPUT_EVENT a0=*SYS_INPUT_EVENT_REQUEST -> KSTATUS.
@@ -2690,18 +2754,32 @@ static uint64_t SyscallKbPeekInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
KSTATUS s = KiCopyInputEventRequest(a0, &req);
if (!KSUCCESS(s)) return s;
INPUT_QUEUE* queue = (INPUT_QUEUE*)0;
s = KiReferenceInputQueueHandle(req.QueueHandle, &queue);
if (!KSUCCESS(s)) return s;
INPUT_KEY_RECORD record;
for (;;) {
if (KeInputPeekEvent(&record))
return KiCopyInputRecordToUser(req.RecordOut, &record);
bool available = queue ? KeInputQueuePeekEvent(queue, &record)
: KeInputPeekEvent(&record);
if (available) {
s = KiCopyInputRecordToUser(req.RecordOut, &record);
break;
}
if (req.TimeoutMilliseconds == 0)
return NTLSTATUS_WOULD_BLOCK;
if (req.TimeoutMilliseconds == 0) {
s = NTLSTATUS_WOULD_BLOCK;
break;
}
s = KeInputWaitKey(0, req.TimeoutMilliseconds);
s = queue ? KeInputQueueWaitKey(queue, 0, req.TimeoutMilliseconds)
: KeInputWaitKey(0, req.TimeoutMilliseconds);
if (!KSUCCESS(s))
return s;
break;
}
if (queue) ObDereferenceObject(&queue->Header);
return s;
}
// SYS_PEEK_MOUSE a0=*INPUT_MOUSE_EVENT -> KSTATUS.
@@ -3345,7 +3423,8 @@ static uint64_t SyscallOpenNamedQueue(uint64_t a0, uint64_t a1, uint64_t a2,
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
if (KiNamedQueueNameEquals(&gNamedQueues[i], nameView)) {
q = gNamedQueues[i].Queue;
ObReferenceObject(&q->Header);
if (q && !ObTryReferenceObject(&q->Header))
q = (QUEUE*)0;
break;
}
}
@@ -5274,10 +5353,11 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
SYS_CPU_STAT* out = &stats.Records[i];
out->CpuIndex = cpu->CpuIndex;
out->LapicId = cpu->LapicId;
if (cpu->Present) out->Flags |= SYS_CPU_FLAG_PRESENT;
if (cpu->Started) out->Flags |= SYS_CPU_FLAG_STARTED;
if (cpu->Parked) out->Flags |= SYS_CPU_FLAG_PARKED;
if (cpu->Online) out->Flags |= SYS_CPU_FLAG_ONLINE;
KE_CPU_STATE state = (KE_CPU_STATE)cpu->State;
if (state >= KeCpuPresent) out->Flags |= SYS_CPU_FLAG_PRESENT;
if (state >= KeCpuStarted) out->Flags |= SYS_CPU_FLAG_STARTED;
if (state == KeCpuParked) out->Flags |= SYS_CPU_FLAG_PARKED;
if (state == KeCpuOnline) out->Flags |= SYS_CPU_FLAG_ONLINE;
if (cpu->IsBsp) out->Flags |= SYS_CPU_FLAG_BSP;
if (cpu->IsPrimaryThread) out->Flags |= SYS_CPU_FLAG_PRIMARY;
out->CurrentTid = cpu->CurrentThread ? cpu->CurrentThread->ThreadId : 0;
@@ -5394,6 +5474,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
[SYS_CLEAR_SCREEN] = SyscallClearScreen,
[SYS_GET_SCREEN_SIZE] = SyscallGetScreenSize,
[SYS_QUERY_DISPLAY_INFO] = SyscallQueryDisplayInfo,
[SYS_OPEN_DISPLAY_CLOCK] = SyscallOpenDisplayClock,
[SYS_CREATE_SURFACE] = SyscallCreateSurface,
[SYS_MAP_SURFACE] = SyscallMapSurface,
[SYS_QUERY_SURFACE] = SyscallQuerySurface,
@@ -5405,6 +5486,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
[SYS_READ_KEYBOARD_CHAR] = SyscallKbReadChar,
[SYS_READ_KEY] = SyscallKbReadKey,
[SYS_GET_KEY_MODIFIERS] = SyscallKbGetModifiers,
[SYS_QUERY_GLOBAL_INPUT_STATE] = SyscallQueryGlobalInputState,
[SYS_OPEN_INPUT_QUEUE] = SyscallOpenInputQueue,
[SYS_READ_INPUT_EVENT] = SyscallKbReadInputEvent,
[SYS_PEEK_INPUT_EVENT] = SyscallKbPeekInputEvent,
code/kernel/ke_thread.c
+13 -12
View File
@@ -1,6 +1,7 @@
// Thread manager. This file owns thread allocation, kernel/user startup contexts, and lifecycle transitions.
#include <string.h>
#include <kernel/ke_list.h>
#include <stddef.h>
#include <intrin.h>
#include <kernel/ke_thread.h>
@@ -26,10 +27,10 @@
_Static_assert(offsetof(type, field) == (expected), \
#type "." #field " offset must match assembly")
STATIC_ASSERT_OFFSET(THREAD, SavedContext, 88);
STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 104);
STATIC_ASSERT_OFFSET(THREAD, Teb, 112);
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 168);
STATIC_ASSERT_OFFSET(THREAD, SavedContext, 96);
STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 112);
STATIC_ASSERT_OFFSET(THREAD, Teb, 120);
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 176);
static uint32_t gNextThreadId = 1;
extern void KeThreadEntry(void);
@@ -68,7 +69,7 @@ static void KiThreadDestroy(OBJECT* obj) {
PROCESS* owner = thread->Owner;
uint32_t remaining = 0;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
ListRemove(&thread->ProcessLink);
KeListRemoveChecked(&thread->ProcessLink);
if (owner->ThreadCount) owner->ThreadCount--;
remaining = owner->ThreadCount;
KeReleaseSpinlock(&owner->HandleLock, flags);
@@ -138,7 +139,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
}
memset(kstack, 0, KSTACK_SIZE);
thread->KernelStackBase = kstack;
thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
thread->KernelStackTop = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
thread->TssRsp0 = thread->KernelStackTop;
KSTATUS s;
@@ -230,7 +231,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->ExitEvent = exitEvent;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
ListInsertTail(&owner->ThreadList, &thread->ProcessLink);
KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
owner->ThreadCount++;
KeReleaseSpinlock(&owner->HandleLock, flags);
@@ -335,7 +336,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
}
memset(kstack, 0, KSTACK_SIZE);
thread->KernelStackBase = kstack;
thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
thread->KernelStackTop = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
thread->TssRsp0 = thread->KernelStackTop;
// Build the initial stack: arg, fn, then KCONTEXT.
@@ -354,7 +355,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
thread->Owner = owner;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
ListInsertTail(&owner->ThreadList, &thread->ProcessLink);
KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
owner->ThreadCount++;
KeReleaseSpinlock(&owner->HandleLock, flags);
@@ -471,7 +472,7 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
// consume the alertable wait immediately instead of going to sleep.
if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
if (thread->WaitListHead)
ListRemove(&thread->SchedulerLink);
KeListRemoveChecked(&thread->SchedulerLink);
KiFinishWait(thread, NTLSTATUS_USER_APC);
thread->State = ThreadRunning;
return NTLSTATUS_USER_APC;
@@ -522,7 +523,7 @@ static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* wa
if (!KiThreadWaitLinkIsLinked(thread))
return 0;
ListRemove(&thread->SchedulerLink);
KeListRemoveChecked(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
@@ -601,7 +602,7 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
return NTLSTATUS_PENDING;
if (thread->WaitListHead)
ListRemove(&thread->SchedulerLink);
KeListRemoveChecked(&thread->SchedulerLink);
KeUnblockThreadWithStatus(thread, status);
return NTLSTATUS_SUCCESS;
code/kernel/ke_timer.c
+3 -2
View File
@@ -1,6 +1,7 @@
// Kernel waitable timer objects. This file owns timer deadlines and timer wait signaling.
#include <stdint.h>
#include <kernel/ke_list.h>
#include <string.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_sched.h>
@@ -52,7 +53,7 @@ static uint8_t KiTimerRemoveLocked(KTIMER* timer) {
if (!timer || !timer->Linked)
return 0;
ListRemove(&timer->Link);
KeListRemoveChecked(&timer->Link);
timer->Linked = 0;
timer->Active = 0;
timer->DueTick = 0;
@@ -132,7 +133,7 @@ KSTATUS KeTimerSet(KTIMER* timer, uint64_t dueTimeMilliseconds, uint64_t periodM
timer->PeriodTicks = periodTicks;
timer->Active = 1;
timer->Linked = 1;
ListInsertTail(&gTimerList, &timer->Link);
KeListInsertTailChecked(&gTimerList, &timer->Link);
addListReference = 1;
}
code/kernel/ke_work.c
+3 -2
View File
@@ -2,6 +2,7 @@
// to move ISR/DPC-originated work to a sleepable system thread context.
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <stddef.h>
#include <intrin.h>
@@ -30,7 +31,7 @@ static void KiWorkQueueWorker(void* arg) {
KWORK_ITEM* item = (KWORK_ITEM*)0;
uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
if (!LIST_IS_EMPTY(&gWorkQueueList)) {
LIST_ENTRY* node = ListPopHead(&gWorkQueueList);
LIST_ENTRY* node = KeListPopHeadChecked(&gWorkQueueList);
item = CONTAINING_RECORD(node, KWORK_ITEM, Link);
}
KeReleaseSpinlock(&gWorkQueueLock, flags);
@@ -74,7 +75,7 @@ bool KeQueueWorkItem(KWORK_ITEM* item) {
}
uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
ListInsertTail(&gWorkQueueList, &item->Link);
KeListInsertTailChecked(&gWorkQueueList, &item->Link);
EVENT* event = gWorkQueueEvent;
KeReleaseSpinlock(&gWorkQueueLock, flags);
code/kernel/mm_pool.c
+957
View File
@@ -0,0 +1,957 @@
// Kernel pool allocator and corruption diagnostics.
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/mm_pool.h>
#include <kernel/mm_pte.h>
#include <kernel/mm_pfn.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
// The boundary-tag allocator stores a header before each allocation and a
// footer after it. Mirrored size and state fields permit constant-time
// coalescing while retaining enough metadata for corruption diagnostics.
#define BLOCK_MAGIC 0xB10CB10CUL
#define BLOCK_STATE_USED 0u
#define BLOCK_STATE_FREE 1u
#define BLOCK_STATE_QUARANTINE 2u
#define BLOCK_FLAG_CANARY 0x00000001u
#define BLOCK_FLAG_POISONED 0x00000002u
#define POOL_TAIL_CANARY_BASE 0xC0FFEE1234ABCDEFULL
#define POOL_POISON_FREE 0xDD
#define POOL_QUARANTINE_SLOTS 256u
#define POOL_ALIGNMENT 16u
#define POOL_TAG_DEFAULT 0x6C6F6F70u // 'pool'.
typedef struct BLOCK_HDR {
uint32_t Magic;
uint32_t Free; // BLOCK_STATE_*
size_t Size; // usable allocator bytes (excludes header + footer)
struct BLOCK_HDR* PrevFree;
struct BLOCK_HDR* NextFree;
// Debug metadata. Present in all builds so boot.ini bits can enable
// diagnostics without changing allocator layout or ABI.
size_t RequestedSize;
uint32_t Tag;
uint32_t Flags;
uint64_t Sequence;
uintptr_t AllocCaller;
uintptr_t FreeCaller;
uintptr_t FreeListInsertCaller;
uintptr_t FreeListRemoveCaller;
uintptr_t CoalesceCaller;
uint64_t FreeListSequence;
// Keep user data 16-byte aligned for MSVC optimized code and image constants.
uint64_t AlignmentPadding;
} BLOCK_HDR;
typedef struct BLOCK_FTR {
size_t Size;
uint32_t Free;
uint32_t Magic;
} BLOCK_FTR;
_Static_assert((sizeof(BLOCK_HDR) % POOL_ALIGNMENT) == 0, "pool header must preserve 16-byte user alignment");
_Static_assert((sizeof(BLOCK_FTR) % POOL_ALIGNMENT) == 0, "pool footer must preserve 16-byte block alignment");
#define HDR_SIZE (sizeof(BLOCK_HDR))
#define FTR_SIZE (sizeof(BLOCK_FTR))
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
// Implements the hdr To Ftr operation.
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
}
// Implements the ftr To Hdr operation.
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
}
// Implements the next Block operation.
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
}
// Implements the prev Ftr operation.
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
}
typedef struct HEAP HEAP;
struct HEAP {
BLOCK_HDR* FreeList; // doubly-linked list of free blocks
uintptr_t Start; // virtual base of the arena
uintptr_t Brk; // current committed end (page-aligned)
uintptr_t Limit; // hard upper bound
uint64_t MapFlags; // PAGE_* flags for new committed pages
SPINLOCK Lock;
int DecommitTailOnFree;
uint64_t NextSequence;
uint64_t NextFreeListSequence;
BLOCK_HDR* Quarantine[POOL_QUARANTINE_SLOTS];
size_t QuarantineCursor;
size_t QuarantineCount;
};
// Returns the active allocator diagnostic options.
static inline uint32_t KiPoolDebugFlags(void) {
return KeGetDebugFlags() & (KE_DEBUG_POOL_CANARY |
KE_DEBUG_POOL_POISON |
KE_DEBUG_POOL_QUARANTINE |
KE_DEBUG_POOL_CALLERS);
}
// Derives a per-allocation tail canary from stable block metadata.
static inline uint64_t KiPoolTailCanaryValue(const BLOCK_HDR* blk) {
return POOL_TAIL_CANARY_BASE ^
((uint64_t)(uintptr_t)blk << 13) ^
((uint64_t)blk->RequestedSize << 1) ^
blk->Sequence;
}
// Returns the first caller-visible byte of an allocator block.
static inline uint8_t* KiBlockUserBase(BLOCK_HDR* blk) {
return (uint8_t*)blk + HDR_SIZE;
}
// Returns the canary slot immediately after the requested payload.
static inline uint64_t* KiBlockTailCanary(BLOCK_HDR* blk) {
return (uint64_t*)(void*)(KiBlockUserBase(blk) + blk->RequestedSize);
}
// Returns a diagnostic name for an allocator block state.
static const char* KiPoolStateName(uint32_t state) {
switch (state) {
case BLOCK_STATE_USED: return "used";
case BLOCK_STATE_FREE: return "free";
case BLOCK_STATE_QUARANTINE: return "quarantine";
default: return "bad-state";
}
}
// Stops with allocator metadata that identifies the corrupt block and callers.
static _Noreturn void KiPoolPanic(const char* reason, HEAP* h, BLOCK_HDR* blk, const void* ptr) {
KePanic(KE_PANIC_CODE_GENERIC,
"MM: pool corruption: %s ptr=0x%016llX hdr=0x%016llX heap=[0x%016llX,0x%016llX) magic=%08x state=%u(%s) size=%llu req=%llu tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX fl_ins=0x%016llX fl_rem=0x%016llX coal=0x%016llX fl_seq=%llu\n",
reason ? reason : "unknown",
(unsigned long long)(uintptr_t)ptr,
(unsigned long long)(uintptr_t)blk,
(unsigned long long)(h ? h->Start : 0),
(unsigned long long)(h ? h->Brk : 0),
blk ? blk->Magic : 0,
blk ? blk->Free : 0,
blk ? KiPoolStateName(blk->Free) : "<null>",
(unsigned long long)(blk ? blk->Size : 0),
(unsigned long long)(blk ? blk->RequestedSize : 0),
blk ? blk->Tag : 0,
blk ? blk->Flags : 0,
(unsigned long long)(blk ? blk->Sequence : 0),
(unsigned long long)(blk ? blk->AllocCaller : 0),
(unsigned long long)(blk ? blk->FreeCaller : 0),
(unsigned long long)(blk ? blk->FreeListInsertCaller : 0),
(unsigned long long)(blk ? blk->FreeListRemoveCaller : 0),
(unsigned long long)(blk ? blk->CoalesceCaller : 0),
(unsigned long long)(blk ? blk->FreeListSequence : 0));
}
// Tests whether an allocator header pointer is inside the committed heap.
static int KiPoolHeaderInCommittedHeap(const HEAP* h, const BLOCK_HDR* blk) {
if (!h || !blk)
return 0;
uintptr_t addr = (uintptr_t)blk;
return addr >= h->Start && addr < h->Brk;
}
// Validates the mirrored footer and committed-heap bounds for one block.
static void KiPoolValidateFooterLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!h || !blk)
KiPoolPanic(where, h, blk, blk ? KiBlockUserBase(blk) : (void*)0);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
if ((uintptr_t)blk < h->Start || (uintptr_t)blk >= h->Brk)
KiPoolPanic("block outside heap", h, blk, KiBlockUserBase(blk));
if (blk->Size > (size_t)(h->Brk - (uintptr_t)blk - OVERHEAD))
KiPoolPanic("block size outside heap", h, blk, KiBlockUserBase(blk));
BLOCK_FTR* ftr = KiHdrToFtr(blk);
if ((uintptr_t)ftr + FTR_SIZE > h->Brk)
KiPoolPanic("footer outside heap", h, blk, KiBlockUserBase(blk));
if (ftr->Magic != BLOCK_MAGIC || ftr->Size != blk->Size || ftr->Free != blk->Free)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
// Writes the optional tail canary after the requested payload.
static void KiPoolWriteTailCanary(BLOCK_HDR* blk) {
if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
return;
*KiBlockTailCanary(blk) = KiPoolTailCanaryValue(blk);
}
// Validates an enabled tail canary while the heap lock is held.
static void KiPoolValidateTailCanaryLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
return;
if (blk->RequestedSize + sizeof(uint64_t) > blk->Size)
KiPoolPanic("invalid canary layout", h, blk, KiBlockUserBase(blk));
uint64_t actual = *KiBlockTailCanary(blk);
uint64_t expect = KiPoolTailCanaryValue(blk);
if (actual != expect)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
// Poisons the requested payload before a freed block becomes reusable.
static void KiPoolPoisonFreedPayload(BLOCK_HDR* blk) {
if (!blk)
return;
size_t bytes = blk->RequestedSize;
if (bytes > blk->Size)
bytes = blk->Size;
memset(KiBlockUserBase(blk), POOL_POISON_FREE, bytes);
blk->Flags |= BLOCK_FLAG_POISONED;
KiPoolWriteTailCanary(blk);
}
// Detects writes into a poisoned free or quarantined payload.
static void KiPoolValidateFreedPoisonLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!blk || !(blk->Flags & BLOCK_FLAG_POISONED))
return;
size_t bytes = blk->RequestedSize;
if (bytes > blk->Size)
bytes = blk->Size;
const uint8_t* p = KiBlockUserBase(blk);
for (size_t i = 0; i < bytes; i++) {
if (p[i] != POOL_POISON_FREE)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
}
// Rolls back a failed heap grow attempt. KiHeapGrow maps pages before the
// new free block is linked into the allocator; if a later frame allocation or
// page-table allocation fails, those earlier mappings must be returned or the
// pool silently leaks physical frames and leaves stale mappings below Brk.
static void KiHeapRollbackGrow(uintptr_t start, uintptr_t end) {
for (uintptr_t va = start; va < end; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPteTranslate(va);
if (!frame)
continue;
if (KSUCCESS(MmPteUnmapPage(va)))
(void)MmPfnFree(frame);
}
}
static void KiFreeListInsertHeadLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller);
static void KiFreeListRemoveLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller);
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (h->Brk + grow > h->Limit) return 0;
uintptr_t start = h->Brk;
uintptr_t mappedEnd = start;
for (uintptr_t va = start; va < start + grow; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame) {
KiHeapRollbackGrow(start, mappedEnd);
return 0;
}
if (!KSUCCESS(MmPteMapPage(va, frame, h->MapFlags, MmPteMapFailIfPresent))) {
MmPfnFree(frame);
KiHeapRollbackGrow(start, mappedEnd);
return 0;
}
mappedEnd = va + PAGE_SIZE;
memset((void *)va, 0, PAGE_SIZE);
}
// The grow path owns the only transition from reserved heap VA to mapped heap
// pages, so a successful loop above is the heap's committed-frontier proof.
// Re-querying every heap page on later allocations only burns page-table locks.
// Missing pool pages are corruption now, not something the allocator repairs.
uintptr_t oldBrk = h->Brk;
uintptr_t newBrk = oldBrk + grow;
// If the current tail block is already free, extend it in-place instead of
// leaving adjacent free blocks. This matters once tail decommit is enabled:
// a later grow should rebuild the same tail block, not fragment the arena
// into many page-sized free runs.
if (oldBrk > h->Start) {
BLOCK_FTR* prevFtr = (BLOCK_FTR*)(oldBrk - FTR_SIZE);
if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free == BLOCK_STATE_FREE) {
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
if ((uintptr_t)prev >= h->Start && prev->Magic == BLOCK_MAGIC && prev->Free == BLOCK_STATE_FREE) {
prev->CoalesceCaller = caller;
prev->Size += grow;
h->Brk = newBrk;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(prev)->Size = prev->Size;
return 1;
}
}
}
// Make the new pages a single free block.
BLOCK_HDR* blk = (BLOCK_HDR *)oldBrk;
blk->Magic = BLOCK_MAGIC;
blk->Free = BLOCK_STATE_FREE;
blk->Size = grow - OVERHEAD;
blk->PrevFree = (BLOCK_HDR *)0;
blk->NextFree = (BLOCK_HDR *)0;
blk->RequestedSize = 0;
blk->Tag = 0;
blk->Flags = 0;
blk->Sequence = 0;
blk->AllocCaller = 0;
blk->FreeCaller = 0;
blk->FreeListInsertCaller = 0;
blk->FreeListRemoveCaller = 0;
blk->CoalesceCaller = 0;
blk->FreeListSequence = 0;
h->Brk = newBrk;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Size = blk->Size;
KiFreeListInsertHeadLocked(h, blk, caller);
return 1;
}
// Validates a free-list node before the allocator mutates list links.
static void KiFreeListValidateNodeLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
KiPoolValidateFooterLocked(h, blk, where);
if (blk->Free != BLOCK_STATE_FREE)
KiPoolPanic("free list node is not free", h, blk, KiBlockUserBase(blk));
if (blk->PrevFree) {
if (!KiPoolHeaderInCommittedHeap(h, blk->PrevFree))
KiPoolPanic("free list previous pointer outside heap", h, blk, KiBlockUserBase(blk));
if (blk->PrevFree->Magic != BLOCK_MAGIC)
KiPoolPanic("free list previous pointer bad magic", h, blk->PrevFree, KiBlockUserBase(blk->PrevFree));
if (blk->PrevFree->NextFree != blk)
KiPoolPanic("free list previous backlink corrupt", h, blk, KiBlockUserBase(blk));
}
if (blk->NextFree) {
if (!KiPoolHeaderInCommittedHeap(h, blk->NextFree))
KiPoolPanic("free list next pointer outside heap", h, blk, KiBlockUserBase(blk));
if (blk->NextFree->Magic != BLOCK_MAGIC)
KiPoolPanic("free list next pointer bad magic", h, blk->NextFree, KiBlockUserBase(blk->NextFree));
if (blk->NextFree->PrevFree != blk)
KiPoolPanic("free list next backlink corrupt", h, blk, KiBlockUserBase(blk));
}
}
// Inserts a free block at the head of the free list and records the mutation site.
static void KiFreeListInsertHeadLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
KiPoolValidateFooterLocked(h, blk, "free list insert block corrupt");
if (blk->Free != BLOCK_STATE_FREE)
KiPoolPanic("free list insert of non-free block", h, blk, KiBlockUserBase(blk));
if (blk->PrevFree || blk->NextFree)
KiPoolPanic("free list insert of linked block", h, blk, KiBlockUserBase(blk));
blk->FreeListInsertCaller = caller;
blk->FreeListSequence = ++h->NextFreeListSequence;
blk->PrevFree = (BLOCK_HDR*)0;
blk->NextFree = h->FreeList;
if (h->FreeList) {
KiFreeListValidateNodeLocked(h, h->FreeList, "free list head corrupt before insert");
h->FreeList->PrevFree = blk;
}
h->FreeList = blk;
}
// Unlinks a free block from the free list and records the mutation site.
static void KiFreeListRemoveLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
KiFreeListValidateNodeLocked(h, blk, "free list remove block corrupt");
blk->FreeListRemoveCaller = caller;
blk->FreeListSequence = ++h->NextFreeListSequence;
if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
else h->FreeList = blk->NextFree;
if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
}
// Implements the heap Try Decommit Tail operation.
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk || blk->Free == BLOCK_STATE_USED) return;
// Do not eagerly decommit the kernel pool on every free. Large transient
// syscall bounce buffers, especially SysBlitScreen's frame-sized buffer,
// are often allocated/freed once per frame. If the tail block is unmapped
// immediately, the next frame has to map the same pages again, producing
// massive page-table churn and global TLB shootdowns. Keep the committed
// pool high-water resident during normal operation; add an explicit idle or
// pressure pool trim later if needed.
if (!h->DecommitTailOnFree) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
uintptr_t blockBase = (uintptr_t)blk;
uintptr_t oldBrk = h->Brk;
// Only the last free block can return committed heap pages. Interior free
// blocks must remain mapped because the allocator's boundary tags and free
// list links are stored inside those blocks.
if ((uintptr_t)KiNextBlock(blk) != oldBrk) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
// Keep enough mapped space for the tail block's header, footer, and a small
// usable payload, then decommit whole pages above that. The footer moves to
// the new Brk so future coalescing can still walk the boundary tag chain.
uintptr_t minEnd = blockBase + MIN_BLOCK;
uintptr_t newBrk = (minEnd + PAGE_SIZE - 1) & ~(uintptr_t)(PAGE_SIZE - 1);
if (newBrk >= oldBrk) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
for (uintptr_t va = newBrk; va < oldBrk; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPteTranslate(va);
if (!frame)
continue;
if (KSUCCESS(MmPteUnmapPage(va)))
(void)MmPfnFree(frame);
}
h->Brk = newBrk;
blk->Size = newBrk - blockBase - OVERHEAD;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
}
// Releases a previously quarantined block into the real free list. Caller holds h->Lock.
static void KiHeapReleaseFreeBlockLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
if (!h || !blk) return;
KiPoolValidateFooterLocked(h, blk, "release free footer corrupt");
KiPoolValidateTailCanaryLocked(h, blk, "release free tail canary corrupt");
KiPoolValidateFreedPoisonLocked(h, blk, "use-after-free payload write detected");
blk->Free = BLOCK_STATE_FREE;
blk->RequestedSize = 0;
blk->Tag = 0;
blk->Flags = 0;
blk->AllocCaller = 0;
// Keep FreeCaller/Sequence for diagnostics until the block is reused.
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
// Coalesce with next block if it is really free. Quarantined neighbors are
// intentionally not coalesced; their payload must remain intact for UAF checks.
BLOCK_HDR* next = KiNextBlock(blk);
if ((uintptr_t)next < h->Brk) {
KiPoolValidateFooterLocked(h, next, "next footer corrupt while freeing");
if (next->Free == BLOCK_STATE_FREE) {
KiFreeListRemoveLocked(h, next, caller);
blk->CoalesceCaller = caller;
next->CoalesceCaller = caller;
blk->Size += OVERHEAD + next->Size;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
}
}
// Coalesce with previous block if it is really free.
if ((uintptr_t)blk > h->Start) {
BLOCK_FTR* prevFtr = KiPrevFtr(blk);
if (prevFtr->Magic == BLOCK_MAGIC) {
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
KiPoolValidateFooterLocked(h, prev, "previous footer corrupt while freeing");
if (prev->Free == BLOCK_STATE_FREE) {
KiFreeListRemoveLocked(h, prev, caller);
prev->CoalesceCaller = caller;
blk->CoalesceCaller = caller;
prev->Size += OVERHEAD + blk->Size;
KiHdrToFtr(prev)->Size = prev->Size;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE;
blk = prev;
}
}
}
KiFreeListInsertHeadLocked(h, blk, caller);
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
KiHeapTryDecommitTail(h, blk);
}
// Releases one quarantine slot into the normal free-list path.
static void KiHeapReleaseQuarantineSlotLocked(HEAP* h, size_t index, uintptr_t caller) {
if (!h || index >= POOL_QUARANTINE_SLOTS)
return;
BLOCK_HDR* old = h->Quarantine[index];
if (!old)
return;
h->Quarantine[index] = (BLOCK_HDR*)0;
if (h->QuarantineCount)
h->QuarantineCount--;
if (old->Free != BLOCK_STATE_QUARANTINE)
KiPoolPanic("quarantine slot state corrupt", h, old, KiBlockUserBase(old));
KiHeapReleaseFreeBlockLocked(h, old, caller);
}
// Drains every quarantine slot when allocation pressure requires reuse.
static void KiHeapDrainQuarantineLocked(HEAP* h, uintptr_t caller) {
if (!h)
return;
for (size_t i = 0; i < POOL_QUARANTINE_SLOTS; i++)
KiHeapReleaseQuarantineSlotLocked(h, i, caller);
h->QuarantineCursor = 0;
}
// Places a freed block in the bounded quarantine before it can be reused.
static void KiHeapQuarantineBlockLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
if (!h || !blk)
return;
size_t idx = h->QuarantineCursor % POOL_QUARANTINE_SLOTS;
KiHeapReleaseQuarantineSlotLocked(h, idx, caller);
blk->Free = BLOCK_STATE_QUARANTINE;
KiHdrToFtr(blk)->Free = BLOCK_STATE_QUARANTINE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
h->Quarantine[idx] = blk;
h->QuarantineCursor = (idx + 1u) % POOL_QUARANTINE_SLOTS;
h->QuarantineCount++;
}
// Allocates a block from the heap.
static void* KiHeapAlloc(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t caller) {
if (!requestedSize) return (void *)0;
uint32_t dbg = KiPoolDebugFlags();
size_t size = requestedSize;
if (dbg & KE_DEBUG_POOL_CANARY) {
if (size > (size_t)-1 - sizeof(uint64_t))
return (void*)0;
size += sizeof(uint64_t);
}
if (size > (size_t)-1 - (POOL_ALIGNMENT - 1u))
return (void*)0;
size = (size + (POOL_ALIGNMENT - 1u)) & ~(size_t)(POOL_ALIGNMENT - 1u);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
// Best-fit search.
BLOCK_HDR* best = (BLOCK_HDR *)0;
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
KiPoolValidateFooterLocked(h, cur, "free list block corrupt");
if (cur->Free != BLOCK_STATE_FREE)
KiPoolPanic("free list contains non-free block", h, cur, KiBlockUserBase(cur));
if (cur->Size >= size) {
if (!best || cur->Size < best->Size) best = cur;
if (best->Size == size) break;
}
}
if (!best && h->QuarantineCount) {
KiHeapDrainQuarantineLocked(h, caller);
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
KiPoolValidateFooterLocked(h, cur, "free list block corrupt after quarantine drain");
if (cur->Size >= size) {
if (!best || cur->Size < best->Size) best = cur;
if (best->Size == size) break;
}
}
}
if (!best) {
if (!KiHeapGrow(h, size + OVERHEAD, caller)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
// Re-search; KiHeapGrow inserted at the front.
best = h->FreeList;
while (best && best->Size < size) best = best->NextFree;
if (!best) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
KiPoolValidateFooterLocked(h, best, "grown free block corrupt");
}
KiFreeListRemoveLocked(h, best, caller);
// Split the block if the remainder is big enough.
if (best->Size >= size + OVERHEAD + POOL_ALIGNMENT) {
BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE);
rem->Magic = BLOCK_MAGIC;
rem->Free = BLOCK_STATE_FREE;
rem->Size = best->Size - size - OVERHEAD;
rem->PrevFree = (BLOCK_HDR *)0;
rem->NextFree = (BLOCK_HDR *)0;
rem->RequestedSize = 0;
rem->Tag = 0;
rem->Flags = 0;
rem->Sequence = 0;
rem->AllocCaller = 0;
rem->FreeCaller = 0;
rem->FreeListInsertCaller = 0;
rem->FreeListRemoveCaller = 0;
rem->CoalesceCaller = 0;
rem->FreeListSequence = 0;
KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
KiHdrToFtr(rem)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(rem)->Size = rem->Size;
KiFreeListInsertHeadLocked(h, rem, caller);
best->Size = size;
}
best->Free = BLOCK_STATE_USED;
best->RequestedSize = requestedSize;
best->Tag = tag ? tag : POOL_TAG_DEFAULT;
best->Flags = 0;
best->Sequence = ++h->NextSequence;
best->AllocCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
best->FreeCaller = 0;
best->CoalesceCaller = 0;
if (dbg & KE_DEBUG_POOL_CANARY)
best->Flags |= BLOCK_FLAG_CANARY;
KiHdrToFtr(best)->Free = BLOCK_STATE_USED;
KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
KiHdrToFtr(best)->Size = best->Size;
memset(KiBlockUserBase(best), 0, requestedSize);
if (best->Size > requestedSize)
memset(KiBlockUserBase(best) + requestedSize, 0, best->Size - requestedSize);
KiPoolWriteTailCanary(best);
KeReleaseSpinlock(&h->Lock, flags);
return KiBlockUserBase(best);
}
// Releases a block back to the heap.
static void KiHeapFree(HEAP* h, void* ptr, uintptr_t caller) {
if (!ptr) return;
uintptr_t u = (uintptr_t)ptr;
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid free pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic("invalid free pointer/bad header magic", h, blk, ptr);
KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on free");
if (blk->Free == BLOCK_STATE_FREE)
KiPoolPanic("double free", h, blk, ptr);
if (blk->Free == BLOCK_STATE_QUARANTINE)
KiPoolPanic("double free/quarantined UAF", h, blk, ptr);
if (blk->Free != BLOCK_STATE_USED)
KiPoolPanic("invalid block state on free", h, blk, ptr);
KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on free");
uint32_t dbg = KiPoolDebugFlags();
blk->FreeCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
if (dbg & KE_DEBUG_POOL_POISON)
KiPoolPoisonFreedPayload(blk);
if (dbg & KE_DEBUG_POOL_QUARANTINE) {
KiHeapQuarantineBlockLocked(h, blk, caller);
} else {
KiHeapReleaseFreeBlockLocked(h, blk, caller);
}
KeReleaseSpinlock(&h->Lock, flags);
}
// Resizes a heap allocation while preserving its contents.
static void* KiHeapRealloc(HEAP* h, void* ptr, size_t newSize, uintptr_t caller) {
// If ptr is NULL, realloc acts like malloc.
if (!ptr) {
return KiHeapAlloc(h, newSize, POOL_TAG_DEFAULT, caller);
}
// If size is 0, realloc acts like free.
if (newSize == 0) {
KiHeapFree(h, ptr, caller);
return (void*)0;
}
uintptr_t u = (uintptr_t)ptr;
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid realloc pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic("invalid realloc pointer/bad header magic", h, blk, ptr);
KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on realloc");
if (blk->Free != BLOCK_STATE_USED)
KiPoolPanic("realloc of non-allocated block", h, blk, ptr);
KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on realloc");
size_t oldRequested = blk->RequestedSize;
size_t oldCopy = oldRequested < newSize ? oldRequested : newSize;
if (newSize <= oldRequested) {
blk->RequestedSize = newSize;
if (blk->Flags & BLOCK_FLAG_CANARY)
KiPoolWriteTailCanary(blk);
KeReleaseSpinlock(&h->Lock, flags);
return ptr;
}
KeReleaseSpinlock(&h->Lock, flags);
void* newPtr = KiHeapAlloc(h, newSize, POOL_TAG_DEFAULT, caller);
if (!newPtr)
return (void*)0;
memcpy(newPtr, ptr, oldCopy);
KiHeapFree(h, ptr, caller);
return newPtr;
}
// Kernel pool.
static HEAP gPoolHeap;
typedef struct KI_POOL_VALIDATE_STATS {
uint64_t Blocks;
uint64_t FreeBlocks;
uint64_t FreeListBlocks;
uint64_t QuarantineBlocks;
size_t UsedBytes;
size_t FreeBytes;
} KI_POOL_VALIDATE_STATS;
// Validates the heap boundary-tag walk and free-list topology while the heap is locked.
static KSTATUS KiPoolValidateHeapLocked(HEAP* h, const char* reason, KI_POOL_VALIDATE_STATS* out) {
KI_POOL_VALIDATE_STATS stats;
memset(&stats, 0, sizeof(stats));
if (!h || h->Start == 0 || h->Limit == 0)
return NTLSTATUS_NOT_SUPPORTED;
if (h->Brk < h->Start || h->Brk > h->Limit)
KiPoolPanic("heap break outside limits", h, (BLOCK_HDR*)0, (void*)0);
if (h->Brk == h->Start) {
if (out) *out = stats;
return NTLSTATUS_SUCCESS;
}
uintptr_t cursor = h->Start;
while (cursor < h->Brk) {
BLOCK_HDR* blk = (BLOCK_HDR*)cursor;
KiPoolValidateFooterLocked(h, blk, reason ? reason : "heap block corrupt");
if (blk->Free > BLOCK_STATE_QUARANTINE)
KiPoolPanic("heap block has invalid state", h, blk, KiBlockUserBase(blk));
stats.Blocks++;
if (blk->Free == BLOCK_STATE_USED) {
stats.UsedBytes += blk->RequestedSize;
KiPoolValidateTailCanaryLocked(h, blk, "heap used tail canary corrupt");
} else if (blk->Free == BLOCK_STATE_FREE) {
stats.FreeBlocks++;
stats.FreeBytes += blk->Size;
KiPoolValidateFreedPoisonLocked(h, blk, "heap free payload poison corrupt");
} else {
stats.QuarantineBlocks++;
KiPoolValidateFreedPoisonLocked(h, blk, "heap quarantine payload poison corrupt");
}
BLOCK_HDR* next = KiNextBlock(blk);
if ((uintptr_t)next <= cursor)
KiPoolPanic("heap block walk did not advance", h, blk, KiBlockUserBase(blk));
cursor = (uintptr_t)next;
}
if (cursor != h->Brk)
KiPoolPanic("heap block walk missed break", h, (BLOCK_HDR*)0, (void*)cursor);
BLOCK_HDR* prev = (BLOCK_HDR*)0;
uint64_t guard = stats.Blocks + 1u;
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
if (stats.FreeListBlocks >= guard)
KiPoolPanic("free list cycle detected", h, cur, KiBlockUserBase(cur));
stats.FreeListBlocks++;
KiFreeListValidateNodeLocked(h, cur, reason ? reason : "free list corrupt");
if (cur->PrevFree != prev)
KiPoolPanic("free list previous chain corrupt", h, cur, KiBlockUserBase(cur));
prev = cur;
}
if (stats.FreeListBlocks != stats.FreeBlocks)
KiPoolPanic("free-list/free-block count mismatch", h, h->FreeList, h->FreeList ? KiBlockUserBase(h->FreeList) : (void*)0);
if (out) *out = stats;
return NTLSTATUS_SUCCESS;
}
// Validates the kernel pool and prints a compact checkpoint line.
KSTATUS MmValidateKernelPool(const char* reason) {
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
KI_POOL_VALIDATE_STATS stats;
KSTATUS status = KiPoolValidateHeapLocked(&gPoolHeap, reason, &stats);
uintptr_t brk = gPoolHeap.Brk;
uintptr_t start = gPoolHeap.Start;
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
if (KSUCCESS(status)) {
DbgPrint("MM: pool checkpoint '%s' ok blocks=%llu free=%llu free-list=%llu quarantine=%llu used=%llu free-bytes=%llu brk=0x%016llX usage=%llu KiB\n",
reason ? reason : "pool",
(unsigned long long)stats.Blocks,
(unsigned long long)stats.FreeBlocks,
(unsigned long long)stats.FreeListBlocks,
(unsigned long long)stats.QuarantineBlocks,
(unsigned long long)stats.UsedBytes,
(unsigned long long)stats.FreeBytes,
(unsigned long long)brk,
(unsigned long long)((brk - start) / 1024));
}
return status;
}
// Initializes the kernel pool and optionally commits its initial extent.
void MmPoolInitialize(size_t poolSize) {
gPoolHeap.FreeList = (BLOCK_HDR*)0;
gPoolHeap.Start = POOL_BASE;
gPoolHeap.Brk = POOL_BASE;
gPoolHeap.Limit = POOL_LIMIT;
gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
gPoolHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
gPoolHeap.DecommitTailOnFree = 0;
gPoolHeap.NextSequence = 0;
gPoolHeap.NextFreeListSequence = 0;
gPoolHeap.QuarantineCursor = 0;
gPoolHeap.QuarantineCount = 0;
memset(gPoolHeap.Quarantine, 0, sizeof(gPoolHeap.Quarantine));
if (poolSize) {
poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
(void)KiHeapGrow(&gPoolHeap, poolSize, 0);
}
}
// Allocates zeroed kernel-pool memory from the non-user heap.
__declspec(dllexport) void* MmAllocatePool(size_t size) {
return KiHeapAlloc(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
}
// Allocates zeroed kernel-pool memory with a diagnostic tag.
__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag) {
return KiHeapAlloc(&gPoolHeap, size, tag, (uintptr_t)_ReturnAddress());
}
// Releases kernel-pool memory back to the non-user heap.
__declspec(dllexport) void MmFreePool(void* ptr) {
KiHeapFree(&gPoolHeap, ptr, (uintptr_t)_ReturnAddress());
}
// Reallocates kernel-pool memory while preserving existing contents.
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) {
return KiHeapRealloc(&gPoolHeap, ptr, newSize, (uintptr_t)_ReturnAddress());
}
// Describes a virtual address that falls inside the kernel-pool aperture.
void MmPoolDebugDescribeAddress(uintptr_t addr) {
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
(unsigned long long)gPoolHeap.Start,
(unsigned long long)gPoolHeap.Brk,
(unsigned long long)gPoolHeap.Limit,
(unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
if (addr >= gPoolHeap.Brk) {
DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
(unsigned long long)addr,
(unsigned long long)(addr - gPoolHeap.Brk));
return;
}
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
(uintptr_t)cur < gPoolHeap.Brk;
cur = KiNextBlock(cur)) {
if (cur->Magic != BLOCK_MAGIC) {
DbgPrint("MM: pool walk corrupt at hdr=0x%016llX magic=%08x\n",
(unsigned long long)(uintptr_t)cur,
cur->Magic);
break;
}
uintptr_t hdr = (uintptr_t)cur;
uintptr_t data = hdr + HDR_SIZE;
uintptr_t end = data + cur->Size;
uintptr_t ftr = end;
BLOCK_FTR* footer = KiHdrToFtr(cur);
if ((uintptr_t)footer + FTR_SIZE > gPoolHeap.Brk ||
footer->Magic != BLOCK_MAGIC || footer->Size != cur->Size || footer->Free != cur->Free) {
DbgPrint("MM: pool walk corrupt footer hdr=0x%016llX state=%u size=%llx footer=0x%016llX fmagic=%08x fstate=%u fsize=%llx\n",
(unsigned long long)(uintptr_t)cur,
cur->Free,
(unsigned long long)cur->Size,
(unsigned long long)(uintptr_t)footer,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Magic : 0,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Free : 0,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? (unsigned long long)footer->Size : 0ULL);
break;
}
if (addr >= hdr && addr < ftr + FTR_SIZE) {
DbgPrint("MM: address 0x%016llX in pool block hdr=0x%016llX data=0x%016llX-0x%016llX size=%llx req=%llx state=%u(%s) tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX fl_ins=0x%016llX fl_rem=0x%016llX coal=0x%016llX fl_seq=%llu\n",
(unsigned long long)addr,
(unsigned long long)hdr,
(unsigned long long)data,
(unsigned long long)end,
(unsigned long long)cur->Size,
(unsigned long long)cur->RequestedSize,
cur->Free,
KiPoolStateName(cur->Free),
cur->Tag,
cur->Flags,
(unsigned long long)cur->Sequence,
(unsigned long long)cur->AllocCaller,
(unsigned long long)cur->FreeCaller,
(unsigned long long)cur->FreeListInsertCaller,
(unsigned long long)cur->FreeListRemoveCaller,
(unsigned long long)cur->CoalesceCaller,
(unsigned long long)cur->FreeListSequence);
break;
}
if (KiNextBlock(cur) <= cur) {
DbgPrint("MM: pool walk stopped: non-advancing block hdr=0x%016llX size=%llx\n",
(unsigned long long)(uintptr_t)cur,
(unsigned long long)cur->Size);
break;
}
}
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
}
// Returns the currently committed kernel-pool bytes.
size_t MmGetKernelPoolUsage(void) {
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
}
code/kernel/mm_pte.c
+1 -1
View File
@@ -142,7 +142,7 @@ void MmPteFlushKernelAddress(uintptr_t virt) {
KCPU* cpu = &gCpus[i];
if (i == selfIndex)
continue;
if (!cpu->Present || !cpu->Started || !cpu->Online)
if (cpu->State != KeCpuOnline)
continue;
targets++;
KiPteStatInc(&gPteStats.ShootdownTargetCpus);
code/kernel/mm_virt.c
+5 -757
View File
@@ -1,5 +1,5 @@
// Kernel virtual-memory manager. This file owns kernel pool allocation, vad bookkeeping, and user-copy validation.
// Page-table construction lives in mm_pte.c so the higher VMM can grow toward VADs, faults, COW, and swap without mixing those policies with raw PTE edits.
// Kernel virtual-memory manager. This file owns VAD bookkeeping, faults, and user-copy validation.
// Pool allocation lives in mm_pool.c and page-table construction lives in mm_pte.c.
// Virtual address layout:
// 0000000000001000-00007FFFFFFFF000 is the lower-half user range.
@@ -12,9 +12,6 @@
// The current VM model uses one shared PML4, so user pointers are globally
// meaningful while protection and lifetime are still tracked per process VAD.
// Kernel pool:
// A boundary-tag allocator starts at POOL_BASE and commits pages on demand.
// User mappings:
// VADs separate reservation from commitment. Committed pages become resident
// when a user fault or kernel prefault path materializes them.
@@ -22,6 +19,7 @@
#include <string.h>
#include <intrin.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_pool.h>
#include <kernel/mm_pte.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_swap.h>
@@ -86,649 +84,6 @@ __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, s
return KiCopyToUserPaged(userDst, kernelSrc, size);
}
// Boundary-tag allocator (used for both pool and user heap)
// Each allocation has a BLOCK_HDR immediately before the user data, and a
// BLOCK_FTR immediately after. Both carry the block size and a 'free' flag.
// This allows O(1) coalescing of adjacent free blocks.
#define BLOCK_MAGIC 0xB10CB10CUL
#define BLOCK_STATE_USED 0u
#define BLOCK_STATE_FREE 1u
#define BLOCK_STATE_QUARANTINE 2u
#define BLOCK_FLAG_CANARY 0x00000001u
#define BLOCK_FLAG_POISONED 0x00000002u
#define POOL_TAIL_CANARY_BASE 0xC0FFEE1234ABCDEFULL
#define POOL_POISON_FREE 0xDD
#define POOL_QUARANTINE_SLOTS 256u
#define POOL_TAG_DEFAULT 0x6C6F6F70u /* 'pool' */
typedef struct BLOCK_HDR {
uint32_t Magic;
uint32_t Free; // BLOCK_STATE_*
size_t Size; // usable allocator bytes (excludes header + footer)
struct BLOCK_HDR* PrevFree;
struct BLOCK_HDR* NextFree;
// Debug metadata. Present in all builds so boot.ini bits can enable
// diagnostics without changing allocator layout or ABI.
size_t RequestedSize;
uint32_t Tag;
uint32_t Flags;
uint64_t Sequence;
uintptr_t AllocCaller;
uintptr_t FreeCaller;
} BLOCK_HDR;
typedef struct BLOCK_FTR {
size_t Size;
uint32_t Free;
uint32_t Magic;
} BLOCK_FTR;
#define HDR_SIZE (sizeof(BLOCK_HDR))
#define FTR_SIZE (sizeof(BLOCK_FTR))
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
// Implements the hdr To Ftr operation.
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
}
// Implements the ftr To Hdr operation.
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
}
// Implements the next Block operation.
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
}
// Implements the prev Ftr operation.
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
}
typedef struct HEAP HEAP;
struct HEAP {
BLOCK_HDR* FreeList; // doubly-linked list of free blocks
uintptr_t Start; // virtual base of the arena
uintptr_t Brk; // current committed end (page-aligned)
uintptr_t Limit; // hard upper bound
uint64_t MapFlags; // PAGE_* flags for new committed pages
SPINLOCK Lock;
int DecommitTailOnFree;
uint64_t NextSequence;
BLOCK_HDR* Quarantine[POOL_QUARANTINE_SLOTS];
size_t QuarantineCursor;
size_t QuarantineCount;
};
static inline uint32_t KiPoolDebugFlags(void) {
return KeGetDebugFlags() & (KE_DEBUG_POOL_CANARY |
KE_DEBUG_POOL_POISON |
KE_DEBUG_POOL_QUARANTINE |
KE_DEBUG_POOL_CALLERS);
}
static inline uint64_t KiPoolTailCanaryValue(const BLOCK_HDR* blk) {
return POOL_TAIL_CANARY_BASE ^
((uint64_t)(uintptr_t)blk << 13) ^
((uint64_t)blk->RequestedSize << 1) ^
blk->Sequence;
}
static inline uint8_t* KiBlockUserBase(BLOCK_HDR* blk) {
return (uint8_t*)blk + HDR_SIZE;
}
static inline uint64_t* KiBlockTailCanary(BLOCK_HDR* blk) {
return (uint64_t*)(void*)(KiBlockUserBase(blk) + blk->RequestedSize);
}
static const char* KiPoolStateName(uint32_t state) {
switch (state) {
case BLOCK_STATE_USED: return "used";
case BLOCK_STATE_FREE: return "free";
case BLOCK_STATE_QUARANTINE: return "quarantine";
default: return "bad-state";
}
}
static _Noreturn void KiPoolPanic(const char* reason, HEAP* h, BLOCK_HDR* blk, const void* ptr) {
KePanic(KE_PANIC_CODE_GENERIC,
"MM: pool corruption: %s ptr=0x%016llX hdr=0x%016llX heap=[%016llx,%016llx) magic=%08x state=%u(%s) size=%llu req=%llu tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX\n",
reason ? reason : "unknown",
ptr,
blk,
(unsigned long long)(h ? h->Start : 0),
(unsigned long long)(h ? h->Brk : 0),
blk ? blk->Magic : 0,
blk ? blk->Free : 0,
blk ? KiPoolStateName(blk->Free) : "<null>",
(unsigned long long)(blk ? blk->Size : 0),
(unsigned long long)(blk ? blk->RequestedSize : 0),
blk ? blk->Tag : 0,
blk ? blk->Flags : 0,
(unsigned long long)(blk ? blk->Sequence : 0),
blk ? (void*)blk->AllocCaller : (void*)0,
blk ? (void*)blk->FreeCaller : (void*)0);
}
static void KiPoolValidateFooterLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!h || !blk)
KiPoolPanic(where, h, blk, blk ? KiBlockUserBase(blk) : (void*)0);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
if ((uintptr_t)blk < h->Start || (uintptr_t)blk >= h->Brk)
KiPoolPanic("block outside heap", h, blk, KiBlockUserBase(blk));
if (blk->Size > (size_t)(h->Brk - (uintptr_t)blk - OVERHEAD))
KiPoolPanic("block size outside heap", h, blk, KiBlockUserBase(blk));
BLOCK_FTR* ftr = KiHdrToFtr(blk);
if ((uintptr_t)ftr + FTR_SIZE > h->Brk)
KiPoolPanic("footer outside heap", h, blk, KiBlockUserBase(blk));
if (ftr->Magic != BLOCK_MAGIC || ftr->Size != blk->Size || ftr->Free != blk->Free)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
static void KiPoolWriteTailCanary(BLOCK_HDR* blk) {
if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
return;
*KiBlockTailCanary(blk) = KiPoolTailCanaryValue(blk);
}
static void KiPoolValidateTailCanaryLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
return;
if (blk->RequestedSize + sizeof(uint64_t) > blk->Size)
KiPoolPanic("invalid canary layout", h, blk, KiBlockUserBase(blk));
uint64_t actual = *KiBlockTailCanary(blk);
uint64_t expect = KiPoolTailCanaryValue(blk);
if (actual != expect)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
static void KiPoolPoisonFreedPayload(BLOCK_HDR* blk) {
if (!blk)
return;
size_t bytes = blk->RequestedSize;
if (bytes > blk->Size)
bytes = blk->Size;
memset(KiBlockUserBase(blk), POOL_POISON_FREE, bytes);
blk->Flags |= BLOCK_FLAG_POISONED;
KiPoolWriteTailCanary(blk);
}
static void KiPoolValidateFreedPoisonLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
if (!blk || !(blk->Flags & BLOCK_FLAG_POISONED))
return;
size_t bytes = blk->RequestedSize;
if (bytes > blk->Size)
bytes = blk->Size;
const uint8_t* p = KiBlockUserBase(blk);
for (size_t i = 0; i < bytes; i++) {
if (p[i] != POOL_POISON_FREE)
KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
}
}
// Rolls back a failed heap grow attempt. KiHeapGrow maps pages before the
// new free block is linked into the allocator; if a later frame allocation or
// page-table allocation fails, those earlier mappings must be returned or the
// pool silently leaks physical frames and leaves stale mappings below Brk.
static void KiHeapRollbackGrow(uintptr_t start, uintptr_t end) {
for (uintptr_t va = start; va < end; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPteTranslate(va);
if (!frame)
continue;
if (KSUCCESS(MmPteUnmapPage(va)))
(void)MmPfnFree(frame);
}
}
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int KiHeapGrow(HEAP* h, size_t need) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (h->Brk + grow > h->Limit) return 0;
uintptr_t start = h->Brk;
uintptr_t mappedEnd = start;
for (uintptr_t va = start; va < start + grow; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame) {
KiHeapRollbackGrow(start, mappedEnd);
return 0;
}
if (!KSUCCESS(MmPteMapPage(va, frame, h->MapFlags, MmPteMapFailIfPresent))) {
MmPfnFree(frame);
KiHeapRollbackGrow(start, mappedEnd);
return 0;
}
mappedEnd = va + PAGE_SIZE;
memset((void *)va, 0, PAGE_SIZE);
}
// The grow path owns the only transition from reserved heap VA to mapped heap
// pages, so a successful loop above is the heap's committed-frontier proof.
// Re-querying every heap page on later allocations only burns page-table locks.
// Missing pool pages are corruption now, not something the allocator repairs.
uintptr_t oldBrk = h->Brk;
uintptr_t newBrk = oldBrk + grow;
// If the current tail block is already free, extend it in-place instead of
// leaving adjacent free blocks. This matters once tail decommit is enabled:
// a later grow should rebuild the same tail block, not fragment the arena
// into many page-sized free runs.
if (oldBrk > h->Start) {
BLOCK_FTR* prevFtr = (BLOCK_FTR*)(oldBrk - FTR_SIZE);
if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free == BLOCK_STATE_FREE) {
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
if ((uintptr_t)prev >= h->Start && prev->Magic == BLOCK_MAGIC && prev->Free == BLOCK_STATE_FREE) {
prev->Size += grow;
h->Brk = newBrk;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(prev)->Size = prev->Size;
return 1;
}
}
}
// Make the new pages a single free block.
BLOCK_HDR* blk = (BLOCK_HDR *)oldBrk;
blk->Magic = BLOCK_MAGIC;
blk->Free = BLOCK_STATE_FREE;
blk->Size = grow - OVERHEAD;
blk->PrevFree = (BLOCK_HDR *)0;
blk->NextFree = h->FreeList;
blk->RequestedSize = 0;
blk->Tag = 0;
blk->Flags = 0;
blk->Sequence = 0;
blk->AllocCaller = 0;
blk->FreeCaller = 0;
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
h->Brk = newBrk;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Size = blk->Size;
return 1;
}
// Unlink a free block from the free list.
static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
else h->FreeList = blk->NextFree;
if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
}
// Implements the heap Try Decommit Tail operation.
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk || blk->Free == BLOCK_STATE_USED) return;
// Do not eagerly decommit the kernel pool on every free. Large transient
// syscall bounce buffers, especially SysBlitScreen's frame-sized buffer,
// are often allocated/freed once per frame. If the tail block is unmapped
// immediately, the next frame has to map the same pages again, producing
// massive page-table churn and global TLB shootdowns. Keep the committed
// pool high-water resident during normal operation; add an explicit idle or
// pressure pool trim later if needed.
if (!h->DecommitTailOnFree) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
uintptr_t blockBase = (uintptr_t)blk;
uintptr_t oldBrk = h->Brk;
// Only the last free block can return committed heap pages. Interior free
// blocks must remain mapped because the allocator's boundary tags and free
// list links are stored inside those blocks.
if ((uintptr_t)KiNextBlock(blk) != oldBrk) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
// Keep enough mapped space for the tail block's header, footer, and a small
// usable payload, then decommit whole pages above that. The footer moves to
// the new Brk so future coalescing can still walk the boundary tag chain.
uintptr_t minEnd = blockBase + MIN_BLOCK;
uintptr_t newBrk = (minEnd + PAGE_SIZE - 1) & ~(uintptr_t)(PAGE_SIZE - 1);
if (newBrk >= oldBrk) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
return;
}
for (uintptr_t va = newBrk; va < oldBrk; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmPteTranslate(va);
if (!frame)
continue;
if (KSUCCESS(MmPteUnmapPage(va)))
(void)MmPfnFree(frame);
}
h->Brk = newBrk;
blk->Size = newBrk - blockBase - OVERHEAD;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
}
// Releases a previously quarantined block into the real free list. Caller holds h->Lock.
static void KiHeapReleaseFreeBlockLocked(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk) return;
KiPoolValidateFooterLocked(h, blk, "release free footer corrupt");
KiPoolValidateTailCanaryLocked(h, blk, "release free tail canary corrupt");
KiPoolValidateFreedPoisonLocked(h, blk, "use-after-free payload write detected");
blk->Free = BLOCK_STATE_FREE;
blk->RequestedSize = 0;
blk->Tag = 0;
blk->Flags = 0;
blk->AllocCaller = 0;
// Keep FreeCaller/Sequence for diagnostics until the block is reused.
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
// Coalesce with next block if it is really free. Quarantined neighbors are
// intentionally not coalesced; their payload must remain intact for UAF checks.
BLOCK_HDR* next = KiNextBlock(blk);
if ((uintptr_t)next < h->Brk) {
KiPoolValidateFooterLocked(h, next, "next footer corrupt while freeing");
if (next->Free == BLOCK_STATE_FREE) {
KiFreeListRemove(h, next);
blk->Size += OVERHEAD + next->Size;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
}
}
// Coalesce with previous block if it is really free.
if ((uintptr_t)blk > h->Start) {
BLOCK_FTR* prevFtr = KiPrevFtr(blk);
if (prevFtr->Magic == BLOCK_MAGIC) {
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
KiPoolValidateFooterLocked(h, prev, "previous footer corrupt while freeing");
if (prev->Free == BLOCK_STATE_FREE) {
KiFreeListRemove(h, prev);
prev->Size += OVERHEAD + blk->Size;
KiHdrToFtr(prev)->Size = prev->Size;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE;
blk = prev;
}
}
}
// Insert at head of free list.
blk->PrevFree = (BLOCK_HDR *)0;
blk->NextFree = h->FreeList;
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
KiHdrToFtr(blk)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
KiHeapTryDecommitTail(h, blk);
}
static void KiHeapReleaseQuarantineSlotLocked(HEAP* h, size_t index) {
if (!h || index >= POOL_QUARANTINE_SLOTS)
return;
BLOCK_HDR* old = h->Quarantine[index];
if (!old)
return;
h->Quarantine[index] = (BLOCK_HDR*)0;
if (h->QuarantineCount)
h->QuarantineCount--;
if (old->Free != BLOCK_STATE_QUARANTINE)
KiPoolPanic("quarantine slot state corrupt", h, old, KiBlockUserBase(old));
KiHeapReleaseFreeBlockLocked(h, old);
}
static void KiHeapDrainQuarantineLocked(HEAP* h) {
if (!h)
return;
for (size_t i = 0; i < POOL_QUARANTINE_SLOTS; i++)
KiHeapReleaseQuarantineSlotLocked(h, i);
h->QuarantineCursor = 0;
}
static void KiHeapQuarantineBlockLocked(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk)
return;
size_t idx = h->QuarantineCursor % POOL_QUARANTINE_SLOTS;
KiHeapReleaseQuarantineSlotLocked(h, idx);
blk->Free = BLOCK_STATE_QUARANTINE;
KiHdrToFtr(blk)->Free = BLOCK_STATE_QUARANTINE;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Size = blk->Size;
h->Quarantine[idx] = blk;
h->QuarantineCursor = (idx + 1u) % POOL_QUARANTINE_SLOTS;
h->QuarantineCount++;
}
// Implements the heap Alloc operation.
static void* KiHeapAllocEx(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t caller) {
if (!requestedSize) return (void *)0;
uint32_t dbg = KiPoolDebugFlags();
size_t size = requestedSize;
if (dbg & KE_DEBUG_POOL_CANARY) {
if (size > (size_t)-1 - sizeof(uint64_t))
return (void*)0;
size += sizeof(uint64_t);
}
size = (size + 7) & ~7ULL; // 8-byte align
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
// Best-fit search.
BLOCK_HDR* best = (BLOCK_HDR *)0;
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
KiPoolValidateFooterLocked(h, cur, "free list block corrupt");
if (cur->Free != BLOCK_STATE_FREE)
KiPoolPanic("free list contains non-free block", h, cur, KiBlockUserBase(cur));
if (cur->Size >= size) {
if (!best || cur->Size < best->Size) best = cur;
if (best->Size == size) break;
}
}
if (!best && h->QuarantineCount) {
KiHeapDrainQuarantineLocked(h);
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
KiPoolValidateFooterLocked(h, cur, "free list block corrupt after quarantine drain");
if (cur->Size >= size) {
if (!best || cur->Size < best->Size) best = cur;
if (best->Size == size) break;
}
}
}
if (!best) {
if (!KiHeapGrow(h, size + OVERHEAD)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
// Re-search; KiHeapGrow inserted at the front.
best = h->FreeList;
while (best && best->Size < size) best = best->NextFree;
if (!best) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
KiPoolValidateFooterLocked(h, best, "grown free block corrupt");
}
KiFreeListRemove(h, best);
// Split the block if the remainder is big enough.
if (best->Size >= size + OVERHEAD + sizeof(void*)) {
BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE);
rem->Magic = BLOCK_MAGIC;
rem->Free = BLOCK_STATE_FREE;
rem->Size = best->Size - size - OVERHEAD;
rem->PrevFree = (BLOCK_HDR *)0;
rem->NextFree = h->FreeList;
rem->RequestedSize = 0;
rem->Tag = 0;
rem->Flags = 0;
rem->Sequence = 0;
rem->AllocCaller = 0;
rem->FreeCaller = 0;
if (h->FreeList) h->FreeList->PrevFree = rem;
h->FreeList = rem;
KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
KiHdrToFtr(rem)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(rem)->Size = rem->Size;
best->Size = size;
}
best->Free = BLOCK_STATE_USED;
best->RequestedSize = requestedSize;
best->Tag = tag ? tag : POOL_TAG_DEFAULT;
best->Flags = 0;
best->Sequence = ++h->NextSequence;
best->AllocCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
best->FreeCaller = 0;
if (dbg & KE_DEBUG_POOL_CANARY)
best->Flags |= BLOCK_FLAG_CANARY;
KiHdrToFtr(best)->Free = BLOCK_STATE_USED;
KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
KiHdrToFtr(best)->Size = best->Size;
memset(KiBlockUserBase(best), 0, requestedSize);
if (best->Size > requestedSize)
memset(KiBlockUserBase(best) + requestedSize, 0, best->Size - requestedSize);
KiPoolWriteTailCanary(best);
KeReleaseSpinlock(&h->Lock, flags);
return KiBlockUserBase(best);
}
static void* KiHeapAlloc(HEAP* h, size_t size) {
return KiHeapAllocEx(h, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
}
// Implements the heap Free operation.
static void KiHeapFreeEx(HEAP* h, void* ptr, uintptr_t caller) {
if (!ptr) return;
uintptr_t u = (uintptr_t)ptr;
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid free pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic("invalid free pointer/bad header magic", h, blk, ptr);
KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on free");
if (blk->Free == BLOCK_STATE_FREE)
KiPoolPanic("double free", h, blk, ptr);
if (blk->Free == BLOCK_STATE_QUARANTINE)
KiPoolPanic("double free/quarantined UAF", h, blk, ptr);
if (blk->Free != BLOCK_STATE_USED)
KiPoolPanic("invalid block state on free", h, blk, ptr);
KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on free");
uint32_t dbg = KiPoolDebugFlags();
blk->FreeCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
if (dbg & KE_DEBUG_POOL_POISON)
KiPoolPoisonFreedPayload(blk);
if (dbg & KE_DEBUG_POOL_QUARANTINE) {
KiHeapQuarantineBlockLocked(h, blk);
} else {
KiHeapReleaseFreeBlockLocked(h, blk);
}
KeReleaseSpinlock(&h->Lock, flags);
}
static void KiHeapFree(HEAP* h, void* ptr) {
KiHeapFreeEx(h, ptr, (uintptr_t)_ReturnAddress());
}
// Implements the heap Realloc operation.
static void* KiHeapReallocEx(HEAP* h, void* ptr, size_t newSize, uintptr_t caller) {
// If ptr is NULL, realloc acts like malloc.
if (!ptr) {
return KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller);
}
// If size is 0, realloc acts like free.
if (newSize == 0) {
KiHeapFreeEx(h, ptr, caller);
return (void*)0;
}
uintptr_t u = (uintptr_t)ptr;
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid realloc pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (blk->Magic != BLOCK_MAGIC)
KiPoolPanic("invalid realloc pointer/bad header magic", h, blk, ptr);
KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on realloc");
if (blk->Free != BLOCK_STATE_USED)
KiPoolPanic("realloc of non-allocated block", h, blk, ptr);
KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on realloc");
size_t oldRequested = blk->RequestedSize;
size_t oldCopy = oldRequested < newSize ? oldRequested : newSize;
if (newSize <= oldRequested) {
blk->RequestedSize = newSize;
if (blk->Flags & BLOCK_FLAG_CANARY)
KiPoolWriteTailCanary(blk);
KeReleaseSpinlock(&h->Lock, flags);
return ptr;
}
KeReleaseSpinlock(&h->Lock, flags);
void* newPtr = KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller);
if (!newPtr)
return (void*)0;
memcpy(newPtr, ptr, oldCopy);
KiHeapFreeEx(h, ptr, caller);
return newPtr;
}
static void* KiHeapRealloc(HEAP* h, void* ptr, size_t newSize) {
return KiHeapReallocEx(h, ptr, newSize, (uintptr_t)_ReturnAddress());
}
// Kernel pool.
static HEAP gPoolHeap;
// VAD arena.
typedef enum MM_VAD_PAGE_STATE {
MmVadPageFree = 0,
@@ -803,22 +158,7 @@ static void KiVadInitialize(void) {
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
MmPteInitialize(bootPml4Phys);
gPoolHeap.FreeList = (BLOCK_HDR *)0;
gPoolHeap.Start = POOL_BASE;
gPoolHeap.Brk = POOL_BASE;
gPoolHeap.Limit = POOL_LIMIT;
gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
gPoolHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
gPoolHeap.DecommitTailOnFree = 0;
gPoolHeap.NextSequence = 0;
gPoolHeap.QuarantineCursor = 0;
gPoolHeap.QuarantineCount = 0;
memset(gPoolHeap.Quarantine, 0, sizeof(gPoolHeap.Quarantine));
if (poolSize) {
poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
KiHeapGrow(&gPoolHeap, poolSize);
}
MmPoolInitialize(poolSize);
KiVadInitialize();
@@ -830,26 +170,6 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
(void)MmPfnPin(gVadZeroPageFrame);
}
// Allocates zeroed kernel-pool memory from the non-user heap.
__declspec(dllexport) void* MmAllocatePool(size_t size) {
return KiHeapAllocEx(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
}
// Allocates zeroed kernel-pool memory with a diagnostic tag.
__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag) {
return KiHeapAllocEx(&gPoolHeap, size, tag, (uintptr_t)_ReturnAddress());
}
// Releases kernel-pool memory back to the non-user heap.
__declspec(dllexport) void MmFreePool(void* ptr) {
KiHeapFreeEx(&gPoolHeap, ptr, (uintptr_t)_ReturnAddress());
}
// Reallocates kernel-pool memory while preserving existing contents.
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) {
return KiHeapReallocEx(&gPoolHeap, ptr, newSize, (uintptr_t)_ReturnAddress());
}
// VAD utility helpers.
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
@@ -3122,72 +2442,6 @@ void MmVadDebugDumpProcess(PROCESS* process) {
if (!count) DbgPrint(" <none>\n");
}
// Implements the debug Describe Pool Address operation.
static void KiDebugDescribePoolAddress(uintptr_t addr) {
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
(unsigned long long)gPoolHeap.Start,
(unsigned long long)gPoolHeap.Brk,
(unsigned long long)gPoolHeap.Limit,
(unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
if (addr >= gPoolHeap.Brk) {
DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
(unsigned long long)addr,
(unsigned long long)(addr - gPoolHeap.Brk));
return;
}
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
(uintptr_t)cur < gPoolHeap.Brk;
cur = KiNextBlock(cur)) {
if (cur->Magic != BLOCK_MAGIC) {
DbgPrint("MM: pool walk corrupt at hdr=0x%016llX magic=%08x\n", cur, cur->Magic);
break;
}
uintptr_t hdr = (uintptr_t)cur;
uintptr_t data = hdr + HDR_SIZE;
uintptr_t end = data + cur->Size;
uintptr_t ftr = end;
BLOCK_FTR* footer = KiHdrToFtr(cur);
if ((uintptr_t)footer + FTR_SIZE > gPoolHeap.Brk ||
footer->Magic != BLOCK_MAGIC || footer->Size != cur->Size || footer->Free != cur->Free) {
DbgPrint("MM: pool walk corrupt footer hdr=0x%016llX state=%u size=%llx footer=0x%016llX fmagic=%08x fstate=%u fsize=%llx\n",
cur,
cur->Free,
(unsigned long long)cur->Size,
footer,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Magic : 0,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Free : 0,
((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? (unsigned long long)footer->Size : 0ULL);
break;
}
if (addr >= hdr && addr < ftr + FTR_SIZE) {
DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx req=%llx state=%u(%s) tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX\n",
(unsigned long long)addr,
(unsigned long long)hdr,
(unsigned long long)data,
(unsigned long long)end,
(unsigned long long)cur->Size,
(unsigned long long)cur->RequestedSize,
cur->Free,
KiPoolStateName(cur->Free),
cur->Tag,
cur->Flags,
(unsigned long long)cur->Sequence,
(void*)cur->AllocCaller,
(void*)cur->FreeCaller);
break;
}
if (KiNextBlock(cur) <= cur) {
DbgPrint("MM: pool walk stopped: non-advancing block hdr=0x%016llX size=%llx\n", cur, (unsigned long long)cur->Size);
break;
}
}
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
}
// Implements the debug Describe Address operation.
void MmDebugDescribeAddress(uintptr_t addr) {
PHYSICAL_ADDRESS phys = 0;
@@ -3200,7 +2454,7 @@ void MmDebugDescribeAddress(uintptr_t addr) {
(unsigned long long)flags);
if (addr >= POOL_BASE && addr < POOL_LIMIT) {
KiDebugDescribePoolAddress(addr);
MmPoolDebugDescribeAddress(addr);
return;
}
@@ -3334,12 +2588,6 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
MmFreeUserImageForProcess(process, ptr, size);
}
// Returns kernel Pool Usage.
size_t MmGetKernelPoolUsage(void) {
// Returns bytes committed to the kernel pool.
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
}
// Returns user Heap Usage.
size_t MmGetUserHeapUsage(void) {
size_t total = 0;
code/kernel/ob_object.c
+88 -23
View File
@@ -1,8 +1,10 @@
// Object manager. This file owns reference-counted kernel objects and handle-table lifetime rules.
// Object manager. This file owns reference-counted kernel objects and their terminal lifetime transitions.
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <kernel/ob_object.h>
#include <kernel/ke_debug.h>
#include <kernel/mm_virt.h>
// Returns a diagnostic pool tag for the object type.
@@ -28,39 +30,102 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' ');
case ObjectTypeModule: return MM_POOL_TAG('m', 'o', 'd', 'u');
case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}
// Initializes a caller-owned object header in the live, inserted state.
__declspec(dllexport) void ObInitializeObjectHeader(OBJECT* object, OBJECT_TYPE type) {
if (!object)
return;
object->Type = type;
object->RefCount = 1;
object->LifetimeState = ObjectAlive;
object->Flags = OBJECT_FLAG_INSERTED;
object->Destroy = (void*)0;
}
// Creates a zeroed, tagged object with one initial reference.
__declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) {
if (size < sizeof(OBJECT)) size = sizeof(OBJECT);
OBJECT* obj = (OBJECT *)MmAllocatePoolTag(size, KiObjectPoolTag(type));
if (!obj) return (OBJECT *)0;
OBJECT* object = (OBJECT*)MmAllocatePoolTag(size, KiObjectPoolTag(type));
if (!object) return (OBJECT*)0;
memset(obj, 0, size);
obj->Type = type;
obj->RefCount = 1;
obj->Destroy = (void *)0; // caller fills this in
return obj;
memset(object, 0, size);
ObInitializeObjectHeader(object, type);
return object;
}
// Implements the reference Object operation.
__declspec(dllexport) void ObReferenceObject(OBJECT* object) {
if (!object) return;
_InterlockedIncrement(&object->RefCount);
}
// Acquires a reference without allowing zero-to-one resurrection.
__declspec(dllexport) bool ObTryReferenceObject(OBJECT* object) {
if (!object)
return false;
// Implements the dereference Object operation.
__declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
if (!object) return;
long prev = _InterlockedDecrement(&object->RefCount);
if (prev == 0) {
// Reference count hit zero - run the destructor then free.
if (object->Destroy) {
object->Destroy(object);
}
MmFreePool(object);
for (;;) {
if (_InterlockedCompareExchange(&object->LifetimeState, 0, 0) != ObjectAlive)
return false;
if ((_InterlockedCompareExchange(&object->Flags, 0, 0) & OBJECT_FLAG_INSERTED) == 0)
return false;
long refs = _InterlockedCompareExchange(&object->RefCount, 0, 0);
if (refs <= 0 || refs == 0x7fffffffL)
return false;
if (_InterlockedCompareExchange(&object->RefCount, refs + 1, refs) == refs)
return true;
}
}
// Acquires a required reference and catches stale pointers at the caller site.
__declspec(dllexport) void ObReferenceObject(OBJECT* object) {
if (!object)
return;
if (ObTryReferenceObject(object))
return;
KePanic(KE_PANIC_CODE_GENERIC,
"object: stale reference object=0x%016llX type=%u refs=%ld state=%ld flags=0x%08x\n",
(uint64_t)(uintptr_t)object,
(unsigned)object->Type,
object->RefCount,
object->LifetimeState,
(unsigned)object->Flags);
}
// Releases a reference and performs the one-way Alive-to-Dying-to-Dead transition.
__declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
if (!object)
return;
long refs = _InterlockedDecrement(&object->RefCount);
if (refs > 0)
return;
if (refs < 0) {
KePanic(KE_PANIC_CODE_GENERIC,
"object: reference underflow object=0x%016llX type=%u refs=%ld state=%ld flags=0x%08x\n",
(uint64_t)(uintptr_t)object,
(unsigned)object->Type,
refs,
object->LifetimeState,
(unsigned)object->Flags);
}
if (_InterlockedCompareExchange(&object->LifetimeState, ObjectDying, ObjectAlive) != ObjectAlive) {
KePanic(KE_PANIC_CODE_GENERIC,
"object: duplicate terminal transition object=0x%016llX type=%u state=%ld flags=0x%08x\n",
(uint64_t)(uintptr_t)object,
(unsigned)object->Type,
object->LifetimeState,
(unsigned)object->Flags);
}
_InterlockedExchange(&object->Flags, OBJECT_FLAG_UNLINKED);
if (object->Destroy)
object->Destroy(object);
_InterlockedExchange(&object->LifetimeState, ObjectDead);
MmFreePool(object);
}
code/public/ntlite/ddi/display.h
+1
View File
@@ -67,3 +67,4 @@ struct DISPLAY_DEVICE {
DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device);
DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device);
DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device);
code/public/ntlite/ddk/ddk.h
+1 -1
View File
@@ -25,7 +25,7 @@
#include <ntlite/ddk/time.h>
#include <ntlite/ddk/timer.h>
#define NTL_DDK_VERSION 1u
#define NTL_DDK_VERSION 2u
// Root DDK table passed to kernel drivers by the driver loader.
typedef struct NTL_DDK {
code/public/ntlite/ddk/object.h
+23 -3
View File
@@ -4,8 +4,9 @@
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
// Kernel object type identifiers used by the legacy object header.
// Kernel object type identifiers used by the shared object header.
typedef enum OBJECT_TYPE {
ObjectTypeProcess = 1,
ObjectTypeThread = 2,
@@ -27,16 +28,35 @@ typedef enum OBJECT_TYPE {
ObjectTypeImageSection = 18,
ObjectTypeTimer = 19,
ObjectTypeModule = 20,
ObjectTypeDisplayClock = 21,
} OBJECT_TYPE;
// Legacy object header embedded by current driver/device objects.
typedef enum OBJECT_LIFETIME_STATE {
ObjectAlive = 1,
ObjectDying,
ObjectDead
} OBJECT_LIFETIME_STATE;
#define OBJECT_FLAG_INSERTED 0x00000001u
#define OBJECT_FLAG_UNLINKED 0x00000002u
// Object header embedded by kernel and driver-owned reference-counted objects.
typedef struct OBJECT {
OBJECT_TYPE Type;
volatile long RefCount;
volatile long LifetimeState;
volatile long Flags;
void (*Destroy)(struct OBJECT* self);
} OBJECT;
// Increments an object's reference count.
// Initializes an embedded object header with one live reference.
__declspec(dllexport) void ObInitializeObjectHeader(OBJECT* object, OBJECT_TYPE type);
// Acquires a reference only while the object is alive and inserted.
// The caller must stabilize the pointer through its owning lock or reference.
__declspec(dllexport) bool ObTryReferenceObject(OBJECT* object);
// Increments an object's reference count and stops on stale-object misuse.
__declspec(dllexport) void ObReferenceObject(OBJECT* object);
// Decrements an object's reference count and destroys it when it reaches zero.
code/public/ntlite/display.h
+1
View File
@@ -94,6 +94,7 @@ typedef struct DISPLAY_STATS {
#define DISPLAY_CAP_PRESENT_SYNC 0x00000020u
#define DISPLAY_CAP_VBLANK 0x00000040u
#define DISPLAY_CAP_3D 0x00000080u
#define DISPLAY_CAP_FRAME_EVENT 0x00000100u
#define SYS_DISPLAY_NAME_MAX 32u
code/public/ntlite/syscall.h
+35 -1
View File
@@ -560,6 +560,7 @@ KSTATUS SysKillProcess(uint32_t processHandle, int32_t exitCode);
#define SYS_THREAD_WAIT_FILE_MAPPING 15u
#define SYS_THREAD_WAIT_SURFACE 16u
#define SYS_THREAD_WAIT_TIMER 17u
#define SYS_THREAD_WAIT_DISPLAY_CLOCK 18u
#define SYS_THREAD_WAIT_UNKNOWN 0xffffffffu
typedef struct SYS_THREAD_INFO {
@@ -863,9 +864,41 @@ KSTATUS SysClearScreen(uint32_t color);
KSTATUS SysGetScreenSize(int* widthOut, int* heightOut);
KSTATUS SysQueryDisplayInfo(SYS_DISPLAY_INFO* infoOut);
// Display clocks are waitable pacing objects. Providers with frame-event
// support signal them once per submitted frame. Other providers use the BSP
// scheduler tick, with PeriodMilliseconds rounded up to one or more ticks.
#define SYS_DISPLAY_CLOCK_FLAG_NONE 0u
#define SYS_DISPLAY_CLOCK_DEFAULT_PERIOD_MS 1ull
typedef struct SYS_OPEN_DISPLAY_CLOCK_REQUEST {
uint32_t Size;
uint32_t Flags;
uint64_t PeriodMilliseconds;
uint32_t* HandleOut;
} SYS_OPEN_DISPLAY_CLOCK_REQUEST;
KSTATUS SysOpenDisplayClock(const SYS_OPEN_DISPLAY_CLOCK_REQUEST* request);
// Input.
#define SYS_INPUT_QUEUE_FLAG_NONE 0x00000000u
#define SYS_INPUT_EVENT_FLAG_NONE 0x00000000u
#define SYS_GLOBAL_INPUT_STATE_FLAG_NONE 0x00000000u
#define SYS_GLOBAL_INPUT_KEY_BYTES 32u
// SysQueryGlobalInputState snapshots the current physical input state. It does
// not consume queued keyboard or mouse events and is intentionally global; a
// later routed input context API should use a context handle instead.
typedef struct SYS_GLOBAL_INPUT_STATE {
uint32_t Size;
uint32_t Flags;
uint64_t Generation;
uint32_t Modifiers;
uint8_t KeyDown[SYS_GLOBAL_INPUT_KEY_BYTES];
int32_t MouseX;
int32_t MouseY;
int32_t MouseWheel;
uint32_t MouseButtons;
} SYS_GLOBAL_INPUT_STATE;
// A zero QueueHandle means the default keyboard input queue. A
// nonzero QueueHandle must be a handle returned by SysOpenInputQueue. A zero
@@ -884,6 +917,7 @@ KSTATUS SysIsKeyDown(INPUT_KEY key, bool* downOut);
KSTATUS SysReadKeyboardChar(char* charOut);
KSTATUS SysReadKey(INPUT_KEY* keyOut);
uint32_t SysGetKeyModifiers(void);
KSTATUS SysQueryGlobalInputState(SYS_GLOBAL_INPUT_STATE* stateOut);
KSTATUS SysReadInputEvent(const SYS_INPUT_EVENT_REQUEST* request);
KSTATUS SysPeekInputEvent(const SYS_INPUT_EVENT_REQUEST* request);
@@ -1046,7 +1080,7 @@ KSTATUS SysRecvSocket(uint32_t socket, void* buffer, size_t length, uint32_t fla
KSTATUS SysShutdownSocket(uint32_t socket, uint32_t how);
KSTATUS SysSendToSocket(uint32_t socket, const SYS_SOCKET_SEND_TO_REQUEST* request);
KSTATUS SysRecvFromSocket(uint32_t socket, const SYS_SOCKET_RECV_FROM_REQUEST* request);
KSTATUS SysResolveHost(const SYS_RESOLVE_HOST_REQUEST* request);
KSTATUS SysResolveHost(const SYS_RESOLVE_HOST_REQUEST* request); // Cry some more, microkernel purists!
KSTATUS SysSetSocketOption(uint32_t socket, const SYS_SOCKET_OPTION_REQUEST* request);
KSTATUS SysGetSocketOption(uint32_t socket, const SYS_SOCKET_OPTION_REQUEST* request);
KSTATUS SysPoll(const SYS_POLL_REQUEST* request);
code/user/apps/doom/doom.c
+61 -6
View File
@@ -274,17 +274,25 @@ static const KeyMap g_keymap[] = {
static bool g_prev_key[KEYMAP_COUNT];
// Returns whether one NTLite input key is currently held.
static bool ntlite_key_down(INPUT_KEY key)
// Returns whether one key bit is set in a global input snapshot.
static bool ntlite_snapshot_key_down(const SYS_GLOBAL_INPUT_STATE* state, INPUT_KEY key)
{
bool down = false;
return SysIsKeyDown(key, &down) == NTLSTATUS_SUCCESS && down;
if (!state || key >= 256u) return false;
return (state->KeyDown[key >> 3] & (uint8_t)(1u << (key & 7u))) != 0;
}
static void poll_input(void)
{
SYS_GLOBAL_INPUT_STATE input;
memset(&input, 0, sizeof(input));
input.Size = sizeof(input);
input.Flags = SYS_GLOBAL_INPUT_STATE_FLAG_NONE;
if (SysQueryGlobalInputState(&input) != NTLSTATUS_SUCCESS)
return;
for (int i = 0; i < (int)KEYMAP_COUNT; i++) {
bool cur_down = ntlite_key_down(g_keymap[i].inputKey);
bool cur_down = ntlite_snapshot_key_down(&input, g_keymap[i].inputKey);
bool prev_down = g_prev_key[i];
if (cur_down && !prev_down) doom_key_down(g_keymap[i].key);
@@ -304,6 +312,7 @@ static void poll_input(void)
#define DISP_H (DOOM_H * SCALE)
static uint32_t g_surface_handle;
static uint32_t g_display_clock_handle;
static uint32_t* g_surface_pixels;
static uint32_t g_surface_pitch_pixels;
@@ -335,6 +344,22 @@ static void init_surface(void)
g_surface_pitch_pixels = pitchBytes / sizeof(uint32_t);
}
static void init_display_clock(void)
{
SYS_OPEN_DISPLAY_CLOCK_REQUEST req;
memset(&req, 0, sizeof(req));
req.Size = sizeof(req);
req.Flags = SYS_DISPLAY_CLOCK_FLAG_NONE;
req.PeriodMilliseconds = 1;
req.HandleOut = &g_display_clock_handle;
uint32_t s = SysOpenDisplayClock(&req);
if (s != NTLSTATUS_SUCCESS) {
g_display_clock_handle = 0;
doom_print_wrapper("doom: display clock unavailable, using sleep pacing\n");
}
}
static void render_frame(const uint8_t* rgba)
{
const uint32_t* src = (const uint32_t*)rgba;
@@ -409,6 +434,30 @@ static int build_argv_from_command_line(const char* cmdline, char* storage, size
return argc;
}
// ---------------------------------------------------------------------------
// Timing
// ---------------------------------------------------------------------------
static int doom_now_tic(void)
{
return I_GetTime();
}
// Waits until Doom has a new simulation tic to render.
static int wait_for_next_doom_tic(int lastTic)
{
for (;;) {
int now = doom_now_tic();
if (now != lastTic)
return now;
if (g_display_clock_handle) {
(void)SysWaitSingle(g_display_clock_handle, SYS_WAIT_FLAG_NONE, SYS_WAIT_TIMEOUT_INFINITE);
} else {
SysSleep(1);
}
}
}
// ---------------------------------------------------------------------------
// main
// ---------------------------------------------------------------------------
@@ -447,12 +496,18 @@ int main(void)
const char* argv[32];
int argc = build_argv_from_command_line(cmdline, argvStorage, sizeof(argvStorage), argv, 32);
init_surface();
init_display_clock();
doom_init(argc, argv, 0);
do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: entering game loop\n") }; SysWriteConsole(&req); } while (0);
// Core execution loop
// Core execution loop. Doom advances at TICRATE, so rendering duplicate
// frames as fast as possible only amplifies scheduler and input jitter.
int lastFrameTic = doom_now_tic();
while (1) {
int frameTic = wait_for_next_doom_tic(lastFrameTic);
lastFrameTic = frameTic;
poll_input();
doom_update();
render_frame((const uint8_t*)doom_get_framebuffer(4 /* RGBA */));
code/user/apps/doom/doom1.wad
BIN
View File
Binary file not shown.
code/user/apps/shell_native/shell_native.c
+142 -20
View File
@@ -1416,36 +1416,158 @@ static void cmd_write(const char* args) {
print("File written successfully.\n");
}
#define SHELL_DIR_INITIAL_CAPACITY 64u
// Returns an ASCII lower-case value for case-insensitive directory sorting.
static char shellAsciiLower(char c) {
if (c >= 'A' && c <= 'Z')
return (char)(c - 'A' + 'a');
return c;
}
// Compares two names using case-insensitive order with a stable case tie-breaker.
static int shellCompareNames(const char* a, const char* b) {
size_t i = 0;
if (!a) a = "";
if (!b) b = "";
for (;;) {
char ca = shellAsciiLower(a[i]);
char cb = shellAsciiLower(b[i]);
if (ca != cb)
return (ca < cb) ? -1 : 1;
if (ca == '\0')
break;
i++;
}
return strcmp(a, b);
}
// Compares two directory entries by display name.
static int shellCompareDirEntries(const SYS_DIR_ENTRY* a, const SYS_DIR_ENTRY* b) {
return shellCompareNames(a->Name, b->Name);
}
// Allocates a directory-entry array from VM so large listings do not consume stack.
static uint32_t shellAllocDirEntries(uint32_t capacity, SYS_DIR_ENTRY** entriesOut) {
void* base = 0;
size_t bytes = (size_t)capacity * sizeof(SYS_DIR_ENTRY);
size_t pages = (bytes + 4095u) / 4096u;
if (pages == 0) pages = 1;
SYS_ALLOC_VM_REQUEST req;
memset(&req, 0, sizeof(req));
req.Size = sizeof(req);
req.Protect = SYS_VM_READWRITE;
req.Flags = SYS_VM_FLAG_NONE;
req.PageCount = pages;
req.BaseOut = &base;
uint32_t status = SysAllocVm(&req);
if (status != 0 || !base) {
*entriesOut = 0;
return status ? status : 1;
}
*entriesOut = (SYS_DIR_ENTRY*)base;
return 0;
}
// Grows a directory-entry array while preserving already-read entries.
static uint32_t shellGrowDirEntries(SYS_DIR_ENTRY** entries, uint32_t* capacity, uint32_t count) {
uint32_t oldCapacity = *capacity;
uint32_t newCapacity = oldCapacity ? oldCapacity * 2u : SHELL_DIR_INITIAL_CAPACITY;
SYS_DIR_ENTRY* next = 0;
uint32_t status = shellAllocDirEntries(newCapacity, &next);
if (status != 0)
return status;
if (*entries && count)
memcpy(next, *entries, (size_t)count * sizeof(SYS_DIR_ENTRY));
if (*entries)
SysFreeVm(*entries);
*entries = next;
*capacity = newCapacity;
return 0;
}
// Sorts directory entries alphabetically.
static void shellSortDirEntries(SYS_DIR_ENTRY* entries, uint32_t count) {
for (uint32_t i = 1; i < count; i++) {
SYS_DIR_ENTRY current = entries[i];
uint32_t j = i;
while (j > 0 && shellCompareDirEntries(&current, &entries[j - 1]) < 0) {
entries[j] = entries[j - 1];
j--;
}
entries[j] = current;
}
}
// Prints one directory entry in cmd dir format.
static void shellPrintDirEntry(const SYS_DIR_ENTRY* ke) {
printDirTime(&ke->LastWriteTime);
if (ke->Attr & 0x10) {
print("<DIR> ");
print(ke->Name);
} else {
print(" ");
print(ke->Name);
print(" (");
printUInt64(ke->FileSize);
print(" bytes)");
}
print("\n");
}
// Implements the cmd dir operation.
static void cmd_dir(const char* path) {
uint32_t hDir;
const char* target = (path && *path != '\0') ? path : ".";
uint32_t st = open_dir(target, &hDir);
if (st != 0) {
if (st != 0) {
print("dir: failed to open directory, status=0x");
printHex32(st);
print("\n");
return;
}
SYS_DIR_ENTRY ke;
while (SysReadDir(hDir, &ke) == 0) {
printDirTime(&ke.LastWriteTime);
if (ke.Attr & 0x10) {
print("<DIR> ");
print(ke.Name);
} else {
print(" ");
print(ke.Name);
print(" (");
printUInt64(ke.FileSize);
print(" bytes)");
}
print("\n");
return;
}
SYS_DIR_ENTRY* entries = 0;
uint32_t count = 0;
uint32_t capacity = 0;
for (;;) {
SYS_DIR_ENTRY ke;
st = SysReadDir(hDir, &ke);
if (st != 0)
break;
if (count == capacity) {
st = shellGrowDirEntries(&entries, &capacity, count);
if (st != 0) {
print("dir: out of memory, status=0x");
printHex32(st);
print("\n");
SysCloseHandle(hDir);
if (entries) SysFreeVm(entries);
return;
}
}
entries[count++] = ke;
}
SysCloseHandle(hDir);
shellSortDirEntries(entries, count);
for (uint32_t i = 0; i < count; i++)
shellPrintDirEntry(&entries[i]);
if (entries)
SysFreeVm(entries);
}
// Implements the cmd type operation.
code/user/apps/shell_posix/shell_posix.c
+77 -2
View File
@@ -23,6 +23,9 @@ extern int closedir(DIR* d);
extern int unlink(const char *pathname);
extern int posix_spawn(int *pid, const char *path, const void *act, const void *attr, char *const argv[], char *const envp[]);
extern int waitpid(int pid, int *status, int options);
extern void* malloc(size_t size);
extern void* realloc(void* ptr, size_t size);
extern void free(void* ptr);
extern void exit(int status);
extern void clearscreen(void);
extern void clearkeyboard(void);
@@ -121,17 +124,89 @@ static void cmd_write(const char* args) {
print("File written successfully.\n");
}
typedef struct SHELL_POSIX_DIR_ENTRY {
char Name[NAME_MAX + 1];
unsigned char Type;
} SHELL_POSIX_DIR_ENTRY;
// Returns an ASCII lower-case value for case-insensitive directory sorting.
static char shellAsciiLower(char c) {
if (c >= 'A' && c <= 'Z')
return (char)(c - 'A' + 'a');
return c;
}
// Compares two names using case-insensitive order with a stable case tie-breaker.
static int shellCompareNames(const char* a, const char* b) {
size_t i = 0;
if (!a) a = "";
if (!b) b = "";
for (;;) {
char ca = shellAsciiLower(a[i]);
char cb = shellAsciiLower(b[i]);
if (ca != cb)
return (ca < cb) ? -1 : 1;
if (ca == '\0')
break;
i++;
}
return strcmp(a, b);
}
// Sorts directory entries alphabetically.
static void shellSortDirEntries(SHELL_POSIX_DIR_ENTRY* entries, size_t count) {
for (size_t i = 1; i < count; i++) {
SHELL_POSIX_DIR_ENTRY current = entries[i];
size_t j = i;
while (j > 0 && shellCompareNames(current.Name, entries[j - 1].Name) < 0) {
entries[j] = entries[j - 1];
j--;
}
entries[j] = current;
}
}
// Implements the cmd dir operation.
static void cmd_dir(const char* path) {
const char* target = (path && *path != '\0') ? path : ".";
DIR* d = opendir(target);
if (!d) { print("dir: cannot access directory\n"); return; }
SHELL_POSIX_DIR_ENTRY* entries = NULL;
size_t count = 0;
size_t capacity = 0;
dirent* entry;
while ((entry = readdir(d)) != NULL) {
print(entry->d_type == 1 ? "<DIR> " : " ");
print(entry->d_name); print("\n");
if (count == capacity) {
size_t nextCapacity = capacity ? capacity * 2u : 64u;
SHELL_POSIX_DIR_ENTRY* next = (SHELL_POSIX_DIR_ENTRY*)realloc(entries, nextCapacity * sizeof(SHELL_POSIX_DIR_ENTRY));
if (!next) {
print("dir: out of memory\n");
free(entries);
closedir(d);
return;
}
entries = next;
capacity = nextCapacity;
}
strncpy(entries[count].Name, entry->d_name, NAME_MAX);
entries[count].Name[NAME_MAX] = '\0';
entries[count].Type = entry->d_type;
count++;
}
closedir(d);
shellSortDirEntries(entries, count);
for (size_t i = 0; i < count; i++) {
print(entries[i].Type == 1 ? "<DIR> " : " ");
print(entries[i].Name);
print("\n");
}
free(entries);
}
// Implements the cmd type operation.
code/user/apps/shell_win32/shell_win32.c
+82 -3
View File
@@ -94,12 +94,56 @@ static void cmd_write(const char* args) {
print("File written successfully.\n");
}
typedef struct SHELL_WIN32_DIR_ENTRY {
char Name[MAX_PATH];
DWORD Attributes;
} SHELL_WIN32_DIR_ENTRY;
// Returns an ASCII lower-case value for case-insensitive directory sorting.
static char shellAsciiLower(char c) {
if (c >= 'A' && c <= 'Z')
return (char)(c - 'A' + 'a');
return c;
}
// Compares two names using case-insensitive order with a stable case tie-breaker.
static int shellCompareNames(const char* a, const char* b) {
size_t i = 0;
if (!a) a = "";
if (!b) b = "";
for (;;) {
char ca = shellAsciiLower(a[i]);
char cb = shellAsciiLower(b[i]);
if (ca != cb)
return (ca < cb) ? -1 : 1;
if (ca == '\0')
break;
i++;
}
return strcmp(a, b);
}
// Sorts directory entries alphabetically.
static void shellSortDirEntries(SHELL_WIN32_DIR_ENTRY* entries, size_t count) {
for (size_t i = 1; i < count; i++) {
SHELL_WIN32_DIR_ENTRY current = entries[i];
size_t j = i;
while (j > 0 && shellCompareNames(current.Name, entries[j - 1].Name) < 0) {
entries[j] = entries[j - 1];
j--;
}
entries[j] = current;
}
}
// Implements the cmd dir operation.
static void cmd_dir(const char* path) {
WIN32_FIND_DATAA fd;
char search_path[MAX_PATH];
const char* target = (path && *path != '\0') ? path : ".";
size_t len = strlen(target);
if (len + 3 > MAX_PATH) { print("dir: path too long\n"); return; }
strcpy(search_path, target);
@@ -107,11 +151,46 @@ static void cmd_dir(const char* path) {
HANDLE hFind = FindFirstFileA(search_path, &fd);
if (hFind == INVALID_HANDLE_VALUE) { print("dir: failed to read directory\n"); return; }
void* heap = GetProcessHeap();
SHELL_WIN32_DIR_ENTRY* entries = NULL;
size_t count = 0;
size_t capacity = 0;
do {
print(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ? "<DIR> " : " ");
print(fd.cFileName); print("\n");
if (count == capacity) {
size_t nextCapacity = capacity ? capacity * 2u : 64u;
size_t bytes = nextCapacity * sizeof(SHELL_WIN32_DIR_ENTRY);
SHELL_WIN32_DIR_ENTRY* next;
if (entries)
next = (SHELL_WIN32_DIR_ENTRY*)HeapReAlloc(heap, 0, entries, bytes);
else
next = (SHELL_WIN32_DIR_ENTRY*)HeapAlloc(heap, 0, bytes);
if (!next) {
print("dir: out of memory\n");
if (entries) HeapFree(heap, 0, entries);
FindClose(hFind);
return;
}
entries = next;
capacity = nextCapacity;
}
strncpy(entries[count].Name, fd.cFileName, MAX_PATH - 1);
entries[count].Name[MAX_PATH - 1] = '\0';
entries[count].Attributes = fd.dwFileAttributes;
count++;
} while (FindNextFileA(hFind, &fd));
FindClose(hFind);
shellSortDirEntries(entries, count);
for (size_t i = 0; i < count; i++) {
print(entries[i].Attributes & FILE_ATTRIBUTE_DIRECTORY ? "<DIR> " : " ");
print(entries[i].Name);
print("\n");
}
if (entries)
HeapFree(heap, 0, entries);
}
// Implements the cmd type operation.
code/user/apps/video/bink2w64.dll
BIN
View File
Binary file not shown.
code/user/apps/video/video.bk2
BIN
View File
Binary file not shown.
code/user/apps/video/video.c
+41 -8
View File
@@ -11,7 +11,9 @@
#include <ntlite/syscall.h>
#include "bink.h"
#define VIDEO_DEFAULT_PATH "video.bk2"
#define VIDEO_USAGE "usage: video <file.bk2>\n" \
" video --help\n" \
"Plays a Bink video file from disk.\n"
static void print_span(const char* s, size_t len) {
if (!s || !len) return;
@@ -88,10 +90,10 @@ static SYS_STRING_VIEW skip_one_token(SYS_STRING_VIEW s) {
return SysSvFromParts(s.Data + i, s.Length - i);
}
static SYS_STRING_VIEW command_line_second_token(SYS_STRING_VIEW cmdline) {
static SYS_STRING_VIEW command_line_first_argument(SYS_STRING_VIEW cmdline) {
SYS_STRING_VIEW rest = skip_one_token(cmdline);
if (!rest.Data || rest.Length == 0)
return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
return SYS_SV_NULL;
size_t begin = 0;
size_t end = 0;
@@ -108,17 +110,43 @@ static SYS_STRING_VIEW command_line_second_token(SYS_STRING_VIEW cmdline) {
}
if (end <= begin)
return SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
return SYS_SV_NULL;
return SysSvFromParts(rest.Data + begin, end - begin);
}
static int sv_equals_cstr(SYS_STRING_VIEW s, const char* cstr) {
size_t len = 0;
if (!s.Data || !cstr)
return 0;
while (cstr[len])
len++;
if (s.Length != len)
return 0;
return memcmp(s.Data, cstr, len) == 0;
}
static int video_argument_is_help(SYS_STRING_VIEW arg) {
return sv_equals_cstr(arg, "-h") ||
sv_equals_cstr(arg, "--help") ||
sv_equals_cstr(arg, "/?");
}
static void print_usage(void) {
print(VIDEO_USAGE);
}
static const char* path_to_cstr(SYS_STRING_VIEW path, char* out, size_t outSize) {
if (!out || outSize == 0)
return VIDEO_DEFAULT_PATH;
return "";
if (!path.Data || path.Length == 0)
path = SYS_SV_LITERAL(VIDEO_DEFAULT_PATH);
if (!path.Data || path.Length == 0) {
out[0] = '\0';
return out;
}
size_t n = path.Length;
if (n + 1 > outSize)
@@ -140,7 +168,12 @@ static uint32_t open_file_view(SYS_STRING_VIEW path, uint32_t* handleOut) {
int main(void) {
SYS_STRING_VIEW cmdline = command_line_view();
SYS_STRING_VIEW pathView = command_line_second_token(cmdline);
SYS_STRING_VIEW pathView = command_line_first_argument(cmdline);
if (!pathView.Data || pathView.Length == 0 || video_argument_is_help(pathView)) {
print_usage();
exit_now(0);
}
print("video: starting\n");
if (cmdline.Data && cmdline.Length) {
code/user/syscall/syscall_nr.inc
+53 -51
View File
@@ -85,61 +85,63 @@ SYS_SET_CWD EQU 72
SYS_REAL_PATH EQU 73
; Console, display, and surfaces.
SYS_WRITE_CONSOLE EQU 74
SYS_GET_CONSOLE_INFO EQU 75
SYS_SET_CONSOLE_CURSOR EQU 76
SYS_FILL_CONSOLE_CHARACTER EQU 77
SYS_FILL_CONSOLE_ATTRIBUTE EQU 78
SYS_PUT_SCREEN_CHAR EQU 79
SYS_CLEAR_SCREEN EQU 80
SYS_GET_SCREEN_SIZE EQU 81
SYS_QUERY_DISPLAY_INFO EQU 82
SYS_CREATE_SURFACE EQU 83
SYS_MAP_SURFACE EQU 84
SYS_QUERY_SURFACE EQU 85
SYS_PRESENT_SURFACE EQU 86
SYS_WRITE_CONSOLE EQU 74
SYS_GET_CONSOLE_INFO EQU 75
SYS_SET_CONSOLE_CURSOR EQU 76
SYS_FILL_CONSOLE_CHARACTER EQU 77
SYS_FILL_CONSOLE_ATTRIBUTE EQU 78
SYS_PUT_SCREEN_CHAR EQU 79
SYS_CLEAR_SCREEN EQU 80
SYS_GET_SCREEN_SIZE EQU 81
SYS_QUERY_DISPLAY_INFO EQU 82
SYS_OPEN_DISPLAY_CLOCK EQU 83
SYS_CREATE_SURFACE EQU 84
SYS_MAP_SURFACE EQU 85
SYS_QUERY_SURFACE EQU 86
SYS_PRESENT_SURFACE EQU 87
; Input.
SYS_POLL_KEYBOARD EQU 87
SYS_IS_KEY_DOWN EQU 88
SYS_READ_KEYBOARD_CHAR EQU 89
SYS_READ_KEY EQU 90
SYS_GET_KEY_MODIFIERS EQU 91
SYS_OPEN_INPUT_QUEUE EQU 92
SYS_READ_INPUT_EVENT EQU 93
SYS_PEEK_INPUT_EVENT EQU 94
SYS_PEEK_MOUSE EQU 95
SYS_READ_MOUSE EQU 96
SYS_GET_MOUSE_POS EQU 97
SYS_POLL_KEYBOARD EQU 88
SYS_IS_KEY_DOWN EQU 89
SYS_READ_KEYBOARD_CHAR EQU 90
SYS_READ_KEY EQU 91
SYS_GET_KEY_MODIFIERS EQU 92
SYS_QUERY_GLOBAL_INPUT_STATE EQU 93
SYS_OPEN_INPUT_QUEUE EQU 94
SYS_READ_INPUT_EVENT EQU 95
SYS_PEEK_INPUT_EVENT EQU 96
SYS_PEEK_MOUSE EQU 97
SYS_READ_MOUSE EQU 98
SYS_GET_MOUSE_POS EQU 99
; Time, debug, and system information.
SYS_CREATE_TIMER EQU 98
SYS_SET_TIMER EQU 99
SYS_CANCEL_TIMER EQU 100
SYS_GET_TICK EQU 101
SYS_GET_TICK_FREQUENCY EQU 102
SYS_GET_SYSTEM_TIME EQU 103
SYS_PRINT_DEBUG EQU 104
SYS_QUERY_TRAP_FRAME EQU 105
SYS_GET_MEMORY_STATUS EQU 106
SYS_QUERY_KERNEL_STATS EQU 107
SYS_CREATE_TIMER EQU 100
SYS_SET_TIMER EQU 101
SYS_CANCEL_TIMER EQU 102
SYS_GET_TICK EQU 103
SYS_GET_TICK_FREQUENCY EQU 104
SYS_GET_SYSTEM_TIME EQU 105
SYS_PRINT_DEBUG EQU 106
SYS_QUERY_TRAP_FRAME EQU 107
SYS_GET_MEMORY_STATUS EQU 108
SYS_QUERY_KERNEL_STATS EQU 109
; Native sockets and polling.
SYS_CREATE_SOCKET EQU 108
SYS_SOCKET_IO_ASYNC EQU 109
SYS_BIND_SOCKET EQU 110
SYS_LISTEN_SOCKET EQU 111
SYS_ACCEPT_SOCKET EQU 112
SYS_CONNECT_SOCKET EQU 113
SYS_SEND_SOCKET EQU 114
SYS_RECV_SOCKET EQU 115
SYS_SHUTDOWN_SOCKET EQU 116
SYS_SEND_TO_SOCKET EQU 117
SYS_RECV_FROM_SOCKET EQU 118
SYS_RESOLVE_HOST EQU 119
SYS_SET_SOCKET_OPTION EQU 120
SYS_GET_SOCKET_OPTION EQU 121
SYS_POLL EQU 122
SYS_UNPUBLISH_QUEUE EQU 123
SYS_CREATE_SOCKET EQU 110
SYS_SOCKET_IO_ASYNC EQU 111
SYS_BIND_SOCKET EQU 112
SYS_LISTEN_SOCKET EQU 113
SYS_ACCEPT_SOCKET EQU 114
SYS_CONNECT_SOCKET EQU 115
SYS_SEND_SOCKET EQU 116
SYS_RECV_SOCKET EQU 117
SYS_SHUTDOWN_SOCKET EQU 118
SYS_SEND_TO_SOCKET EQU 119
SYS_RECV_FROM_SOCKET EQU 120
SYS_RESOLVE_HOST EQU 121
SYS_SET_SOCKET_OPTION EQU 122
SYS_GET_SOCKET_OPTION EQU 123
SYS_POLL EQU 124
SYS_UNPUBLISH_QUEUE EQU 125
SYS_MAX EQU 124
SYS_MAX EQU 126
code/user/syscall/syscall_stubs.asm
+2
View File
@@ -218,6 +218,7 @@ SYSCALL_STUB2 PresentSurface, SYS_PRESENT_SURFACE
SYSCALL_STUB2 ClearScreen, SYS_CLEAR_SCREEN
SYSCALL_STUB2 GetScreenSize, SYS_GET_SCREEN_SIZE
SYSCALL_STUB2 QueryDisplayInfo, SYS_QUERY_DISPLAY_INFO
SYSCALL_STUB2 OpenDisplayClock, SYS_OPEN_DISPLAY_CLOCK
; ---------------------------------------------------------------------------
; Input
@@ -227,6 +228,7 @@ SYSCALL_STUB2 IsKeyDown, SYS_IS_KEY_DOWN
SYSCALL_STUB2 ReadKeyboardChar, SYS_READ_KEYBOARD_CHAR
SYSCALL_STUB2 ReadKey, SYS_READ_KEY
SYSCALL_STUB2 GetKeyModifiers, SYS_GET_KEY_MODIFIERS
SYSCALL_STUB2 QueryGlobalInputState, SYS_QUERY_GLOBAL_INPUT_STATE
SYSCALL_STUB2 ReadInputEvent, SYS_READ_INPUT_EVENT
SYSCALL_STUB2 PeekInputEvent, SYS_PEEK_INPUT_EVENT
code/user/tests/all_tests/all_tests.c
+356
View File
@@ -0,0 +1,356 @@
// Dynamic test runner for every executable in the runner directory.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <ntlite/syscall.h>
#define ALL_TESTS_INITIAL_CAPACITY 32u
typedef struct TEST_ENTRY {
char Name[SYS_DIR_NAME_MAX];
} TEST_ENTRY;
// Writes one NUL-terminated string to the console.
static void print(const char* s) {
SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) };
SysWriteConsole(&req);
}
// Writes one unsigned integer to the console.
static void print_u64(uint64_t v) {
char buf[32];
int i = 0;
if (v == 0) {
print("0");
return;
}
while (v && i < (int)sizeof(buf)) {
buf[i++] = (char)('0' + (v % 10));
v /= 10;
}
while (i--) {
char c[2] = { buf[i], 0 };
print(c);
}
}
// Writes one signed integer to the console.
static void print_i32(int32_t v) {
if (v < 0) {
print("-");
print_u64((uint32_t)(-(v + 1)) + 1u);
return;
}
print_u64((uint32_t)v);
}
// Writes one 32-bit value as hexadecimal.
static void print_hex32(uint32_t v) {
static const char hex[] = "0123456789abcdef";
for (int i = 7; i >= 0; i--) {
char c[2] = { hex[(v >> (i * 4)) & 0xfu], 0 };
print(c);
}
}
// Exits the process without returning to the runtime.
static void exit_now(int code) {
for (;;) SysExitProcess(code);
}
// Converts ASCII to lowercase for stable case-insensitive comparisons.
static char ascii_lower(char c) {
if (c >= 'A' && c <= 'Z') return (char)(c - 'A' + 'a');
return c;
}
// Compares two names case-insensitively, with a case-sensitive tie-breaker.
static int compare_names(const char* a, const char* b) {
size_t i = 0;
for (;;) {
char ca = ascii_lower(a[i]);
char cb = ascii_lower(b[i]);
if (ca != cb) return (unsigned char)ca < (unsigned char)cb ? -1 : 1;
if (ca == '\0') break;
i++;
}
return strcmp(a, b);
}
// Returns whether one character is a path separator.
static bool is_separator(char c) {
return c == '/' || c == '\\';
}
// Copies a SYS_STRING_VIEW into a NUL-terminated fixed buffer.
static bool copy_view(char* out, size_t outSize, SYS_STRING_VIEW view) {
if (!out || outSize == 0 || !view.Data || view.Length + 1 > outSize) return false;
memcpy(out, view.Data, view.Length);
out[view.Length] = '\0';
return true;
}
// Splits an executable path into its directory and basename components.
static bool split_self_path(const char* path, char* dir, size_t dirSize, char* name, size_t nameSize) {
if (!path || !dir || !name || dirSize == 0 || nameSize == 0) return false;
size_t len = strlen(path);
size_t last = (size_t)-1;
for (size_t i = 0; i < len; i++) {
if (is_separator(path[i])) last = i;
}
const char* base = path;
size_t dirLen = 1;
if (last == (size_t)-1) {
if (dirSize < 2) return false;
dir[0] = '.';
dir[1] = '\0';
} else {
dirLen = last;
if (last == 0) dirLen = 1;
if (last >= 2 && path[1] == ':' && is_separator(path[2])) dirLen = last + 1;
if (dirLen + 1 > dirSize) return false;
memcpy(dir, path, dirLen);
dir[dirLen] = '\0';
base = path + last + 1;
}
size_t baseLen = strlen(base);
if (baseLen + 1 > nameSize) return false;
memcpy(name, base, baseLen + 1);
return true;
}
// Returns whether one filename ends in .exe, case-insensitively.
static bool has_exe_suffix(const char* name) {
size_t len = strlen(name);
if (len < 4) return false;
const char* ext = name + len - 4;
return ascii_lower(ext[0]) == '.' &&
ascii_lower(ext[1]) == 'e' &&
ascii_lower(ext[2]) == 'x' &&
ascii_lower(ext[3]) == 'e';
}
// Allocates a test-entry array from user VM.
static uint32_t alloc_entries(uint32_t capacity, TEST_ENTRY** entriesOut) {
void* base = 0;
size_t bytes = (size_t)capacity * sizeof(TEST_ENTRY);
size_t pages = (bytes + 4095u) / 4096u;
if (pages == 0) pages = 1;
SYS_ALLOC_VM_REQUEST req;
memset(&req, 0, sizeof(req));
req.Size = sizeof(req);
req.Protect = SYS_VM_READWRITE;
req.Flags = SYS_VM_FLAG_NONE;
req.PageCount = pages;
req.BaseOut = &base;
uint32_t status = SysAllocVm(&req);
if (status != NTLSTATUS_SUCCESS || !base) {
*entriesOut = 0;
return status ? status : NTLSTATUS_NO_MEMORY;
}
*entriesOut = (TEST_ENTRY*)base;
return NTLSTATUS_SUCCESS;
}
// Grows the test-entry array.
static uint32_t grow_entries(TEST_ENTRY** entries, uint32_t* capacity, uint32_t count) {
uint32_t oldCapacity = *capacity;
uint32_t newCapacity = oldCapacity ? oldCapacity * 2u : ALL_TESTS_INITIAL_CAPACITY;
TEST_ENTRY* next = 0;
uint32_t status = alloc_entries(newCapacity, &next);
if (status != NTLSTATUS_SUCCESS) return status;
if (*entries && count) memcpy(next, *entries, (size_t)count * sizeof(TEST_ENTRY));
if (*entries) SysFreeVm(*entries);
*entries = next;
*capacity = newCapacity;
return NTLSTATUS_SUCCESS;
}
// Sorts test entries in deterministic execution order.
static void sort_entries(TEST_ENTRY* entries, uint32_t count) {
for (uint32_t i = 1; i < count; i++) {
TEST_ENTRY current = entries[i];
uint32_t j = i;
while (j > 0 && compare_names(current.Name, entries[j - 1].Name) < 0) {
entries[j] = entries[j - 1];
j--;
}
entries[j] = current;
}
}
// Builds one child path from the runner directory and test filename.
static bool build_child_path(const char* dir, const char* name, char* out, size_t outSize) {
if (!dir || !name || !out || outSize == 0) return false;
size_t dirLen = strlen(dir);
size_t nameLen = strlen(name);
bool needSep = dirLen > 0 && !is_separator(dir[dirLen - 1]);
size_t total = dirLen + (needSep ? 1u : 0u) + nameLen;
if (total + 1 > outSize) return false;
memcpy(out, dir, dirLen);
size_t pos = dirLen;
if (needSep) out[pos++] = '\\';
memcpy(out + pos, name, nameLen);
out[total] = '\0';
return true;
}
// Collects executable tests from the runner directory.
static uint32_t collect_tests(const char* dir, const char* selfName, TEST_ENTRY** entriesOut, uint32_t* countOut) {
*entriesOut = 0;
*countOut = 0;
uint32_t hDir = 0;
SYS_PATH_HANDLE_REQUEST openReq = { SysSvFromCstr(dir), &hDir };
uint32_t status = SysOpenDir(&openReq);
if (status != NTLSTATUS_SUCCESS) return status;
TEST_ENTRY* entries = 0;
uint32_t count = 0;
uint32_t capacity = 0;
for (;;) {
SYS_DIR_ENTRY entry;
status = SysReadDir(hDir, &entry);
if (status != NTLSTATUS_SUCCESS) break;
if (!entry.Name[0]) continue;
if ((entry.Attr & 0x10u) != 0) continue;
if (!has_exe_suffix(entry.Name)) continue;
if (compare_names(entry.Name, selfName) == 0) continue;
if (count == capacity) {
status = grow_entries(&entries, &capacity, count);
if (status != NTLSTATUS_SUCCESS) {
SysCloseHandle(hDir);
if (entries) SysFreeVm(entries);
return status;
}
}
strncpy(entries[count].Name, entry.Name, sizeof(entries[count].Name) - 1);
entries[count].Name[sizeof(entries[count].Name) - 1] = '\0';
count++;
}
SysCloseHandle(hDir);
sort_entries(entries, count);
*entriesOut = entries;
*countOut = count;
return NTLSTATUS_SUCCESS;
}
// Runs one test process and returns the child's exit code through exitCodeOut.
static uint32_t run_test(const char* path, int32_t* exitCodeOut) {
uint32_t process = 0;
SYS_CREATE_PROCESS_REQUEST req;
req.Path = SysSvFromCstr(path);
req.CommandLine = SysSvFromCstr(path);
req.HandleOut = &process;
uint32_t status = SysCreateProcess(&req);
if (status != NTLSTATUS_SUCCESS) return status;
status = SysWait(process, SYS_WAIT_FLAG_NONE);
if (status != NTLSTATUS_SUCCESS) {
SysCloseHandle(process);
return status;
}
int32_t exitCode = 1;
status = SysGetProcessExitCode(process, &exitCode);
SysCloseHandle(process);
if (status != NTLSTATUS_SUCCESS) return status;
*exitCodeOut = exitCode;
return NTLSTATUS_SUCCESS;
}
// Program entry point.
int main(void) {
print("all_tests: start\n");
char selfPath[SYS_PATH_MAX];
char selfDir[SYS_PATH_MAX];
char selfName[SYS_DIR_NAME_MAX];
SYS_PEB* peb = SysGetPeb();
if (!peb || peb->Magic != SYS_PEB_MAGIC || peb->Version != SYS_ENV_VERSION ||
!copy_view(selfPath, sizeof(selfPath), peb->ImagePath) ||
!split_self_path(selfPath, selfDir, sizeof(selfDir), selfName, sizeof(selfName))) {
print("all_tests: failed to resolve runner path\n");
exit_now(1);
}
TEST_ENTRY* entries = 0;
uint32_t count = 0;
uint32_t status = collect_tests(selfDir, selfName, &entries, &count);
if (status != NTLSTATUS_SUCCESS) {
print("all_tests: failed to enumerate tests status=0x");
print_hex32(status);
print("\n");
exit_now(1);
}
if (count == 0) {
print("all_tests: no test executables found\n");
if (entries) SysFreeVm(entries);
exit_now(1);
}
for (uint32_t i = 0; i < count; i++) {
char childPath[SYS_PATH_MAX];
if (!build_child_path(selfDir, entries[i].Name, childPath, sizeof(childPath))) {
print("all_tests: child path too long: ");
print(entries[i].Name);
print("\n");
SysFreeVm(entries);
exit_now(1);
}
print("all_tests: RUN ");
print(childPath);
print("\n");
int32_t exitCode = 1;
status = run_test(childPath, &exitCode);
if (status != NTLSTATUS_SUCCESS) {
print("all_tests: FAIL ");
print(childPath);
print(" launch/wait status=0x");
print_hex32(status);
print("\n");
SysFreeVm(entries);
exit_now(1);
}
if (exitCode != 0) {
print("all_tests: FAIL ");
print(childPath);
print(" exit=");
print_i32(exitCode);
print("\n");
SysFreeVm(entries);
exit_now(exitCode ? exitCode : 1);
}
print("all_tests: PASS ");
print(childPath);
print("\n");
}
print("all_tests: PASS count=");
print_u64(count);
print("\n");
SysFreeVm(entries);
exit_now(0);
return 0;
}
code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c
+57 -18
View File
@@ -54,6 +54,51 @@ static void fail(const char* what) {
exit_now(1);
}
// Copies a string view into a NUL-terminated buffer.
static int copy_view(char* dst, size_t cap, SYS_STRING_VIEW view) {
if (!dst || cap == 0 || !view.Data || view.Length + 1 > cap)
return 0;
memcpy(dst, view.Data, view.Length);
dst[view.Length] = '\0';
return 1;
}
// Computes the parent of one canonical native current-directory string.
static int parent_of_cwd(const char* cwd, char* out, size_t cap) {
if (!cwd || !out || cap == 0)
return 0;
size_t len = strlen(cwd);
if (len < 3 || cwd[1] != ':' || (cwd[2] != '/' && cwd[2] != '\\'))
return 0;
if (len + 1 > cap)
return 0;
memcpy(out, cwd, len + 1);
while (len > 3 && (out[len - 1] == '/' || out[len - 1] == '\\'))
out[--len] = '\0';
if (len <= 3) {
out[0] = cwd[0];
out[1] = ':';
out[2] = '/';
out[3] = '\0';
return 1;
}
size_t pos = len;
while (pos > 3 && out[pos - 1] != '/' && out[pos - 1] != '\\')
pos--;
if (pos <= 3) {
out[0] = cwd[0];
out[1] = ':';
out[2] = '/';
out[3] = '\0';
} else {
out[pos - 1] = '\0';
}
return 1;
}
// Sets the current directory to one native path.
static void set_cwd_or_fail(const char* path, const char* what) {
SYS_PATH_REQUEST req = { SysSvFromCstr(path) };
@@ -72,8 +117,8 @@ static void expect_cwd(const char* expected) {
// Program entry point.
int main(void) {
static const char rootPath[] = "0:/";
static const char tempDir[] = "0:/cwd_peb_smoke_dir";
char original[SYS_CWD_MAX];
char parent[SYS_CWD_MAX];
print("cwd_peb_smoke: start\n");
@@ -82,25 +127,19 @@ int main(void) {
if (peb->Magic != SYS_PEB_MAGIC) fail("peb magic mismatch");
if (peb->Version != SYS_ENV_VERSION) fail("peb version mismatch");
set_cwd_or_fail(rootPath, "set root cwd");
expect_cwd(rootPath);
if (!copy_view(original, sizeof(original), SysGetCurrentDirectoryView()))
fail("initial cwd missing");
if (!parent_of_cwd(original, parent, sizeof(parent)))
fail("initial cwd malformed");
SYS_PATH_REQUEST dirReq = { SysSvFromCstr(tempDir) };
uint32_t status = SysRemoveDir(&dirReq);
if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_NOT_FOUND)
fail_status("remove stale temp dir", status);
set_cwd_or_fail(".", "set dot cwd");
expect_cwd(original);
status = SysCreateDir(&dirReq);
if (status != NTLSTATUS_SUCCESS) fail_status("create temp dir", status);
set_cwd_or_fail("..", "set parent cwd");
expect_cwd(parent);
set_cwd_or_fail(tempDir, "set temp cwd");
expect_cwd(tempDir);
set_cwd_or_fail(rootPath, "restore root cwd");
expect_cwd(rootPath);
status = SysRemoveDir(&dirReq);
if (status != NTLSTATUS_SUCCESS) fail_status("remove temp dir", status);
set_cwd_or_fail(original, "restore original cwd");
expect_cwd(original);
print("cwd_peb_smoke: PASS\n");
exit_now(0);
code/user/tests/dirty_writeback/dirty_writeback.c
+7 -4
View File
@@ -68,11 +68,14 @@ static void delete_path(const char* path) {
}
static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) {
// Writable QEMU vvfat maps long-name creation through a host rename and
// can abort before the guest receives an I/O status. Keep this file in
// 8.3 form so the test measures writeback rather than the host bridge.
static const char* paths[] = {
"dirty_writeback.tmp",
"/dirty_writeback.tmp",
"1:/dirty_writeback.tmp",
"0:/dirty_writeback.tmp",
"dwrite.tmp",
"/dwrite.tmp",
"1:/dwrite.tmp",
"0:/dwrite.tmp",
};
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
code/user/tests/file_stat_smoke/file_stat_smoke.c
+77 -100
View File
@@ -1,14 +1,13 @@
// Native file metadata smoke test.
// Verifies path stat, handle stat, directory metadata, missing paths, and invalid handles.
// Verifies path stat, handle stat, directory metadata, missing paths, and invalid handles without mutating the filesystem.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <ntlite/syscall.h>
#define TEST_FILE_SIZE 1537u
static uint8_t gPattern[TEST_FILE_SIZE];
static char gSelfPath[SYS_PATH_MAX];
static char gSelfDir[SYS_PATH_MAX];
// Prints one NUL-terminated string to the kernel console.
static void print(const char* s) {
@@ -58,84 +57,77 @@ static void fail_status(const char* what, uint32_t status) {
exit_now(1);
}
// Removes a file path if it already exists.
static void delete_file_path(const char* path) {
SYS_PATH_REQUEST req = { SysSvFromCstr(path) };
(void)SysDeleteFile(&req);
// Copies one string view into a fixed NUL-terminated buffer.
static int copy_view(char* out, size_t outSize, SYS_STRING_VIEW view) {
if (!out || outSize == 0 || !view.Data || view.Length + 1 > outSize)
return 0;
memcpy(out, view.Data, view.Length);
out[view.Length] = '\0';
return 1;
}
// Removes a directory path if it already exists.
static void remove_dir_path(const char* path) {
SYS_PATH_REQUEST req = { SysSvFromCstr(path) };
(void)SysRemoveDir(&req);
}
// Opens a new temporary file on the first writable path that works.
static uint32_t create_temp_file(uint32_t* fileOut, const char** pathOut) {
static const char* paths[] = {
"file_stat_smoke.tmp",
"/file_stat_smoke.tmp",
"1:/file_stat_smoke.tmp",
"0:/file_stat_smoke.tmp",
};
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
uint32_t h = 0;
delete_file_path(paths[i]);
SYS_OPEN_FILE_REQUEST req = SysMakeOpenFileRequest(SysSvFromCstr(paths[i]),
SYS_FILE_ACCESS_READ | SYS_FILE_ACCESS_WRITE | SYS_FILE_ACCESS_QUERY,
SYS_FILE_CREATE_NEW,
SYS_FILE_FLAG_NONE,
&h);
uint32_t s = SysOpenFile(&req);
if (s == NTLSTATUS_SUCCESS) {
*fileOut = h;
*pathOut = paths[i];
return NTLSTATUS_SUCCESS;
}
// Builds the containing directory path for one process image path.
static void build_parent_dir(char* out, size_t outSize, const char* path) {
size_t len = strlen(path);
size_t slash = (size_t)-1;
for (size_t i = 0; i < len; ++i) {
if (path[i] == '/' || path[i] == '\\')
slash = i;
}
return NTLSTATUS_NOT_FOUND;
}
// Creates a temporary directory on the first writable path that works.
static uint32_t create_temp_dir(const char** pathOut) {
static const char* paths[] = {
"file_stat_smoke.dir",
"/file_stat_smoke.dir",
"1:/file_stat_smoke.dir",
"0:/file_stat_smoke.dir",
};
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) {
remove_dir_path(paths[i]);
SYS_PATH_REQUEST req = { SysSvFromCstr(paths[i]) };
uint32_t s = SysCreateDir(&req);
if (s == NTLSTATUS_SUCCESS) {
*pathOut = paths[i];
return NTLSTATUS_SUCCESS;
}
if (slash == (size_t)-1) {
if (outSize < 2) fail("parent directory buffer too small");
out[0] = '.';
out[1] = '\0';
return;
}
return NTLSTATUS_NOT_FOUND;
if (slash == 0) {
if (outSize < 2) fail("parent directory buffer too small");
out[0] = path[0];
out[1] = '\0';
return;
}
if (slash + 1 >= outSize)
fail("parent directory path too long");
memcpy(out, path, slash);
out[slash] = '\0';
}
// Fills the write pattern used by the test file.
static void fill_pattern(void) {
for (size_t i = 0; i < sizeof(gPattern); ++i)
gPattern[i] = (uint8_t)(0x31u + (uint8_t)(i * 17u + (i >> 3)));
}
// Opens this process image for read/query metadata checks.
static uint32_t open_self_image(uint32_t* fileOut, uint64_t* sizeOut) {
SYS_PEB* peb = SysGetPeb();
if (!peb || peb->Magic != SYS_PEB_MAGIC || peb->Version != SYS_ENV_VERSION)
return NTLSTATUS_INVALID_PARAMETER;
if (!copy_view(gSelfPath, sizeof(gSelfPath), peb->ImagePath))
return NTLSTATUS_INVALID_PARAMETER;
build_parent_dir(gSelfDir, sizeof(gSelfDir), gSelfPath);
// Writes the test file contents and flushes them to the filesystem.
static void write_test_file(uint32_t file) {
size_t done = 0;
SYS_FILE_IO_REQUEST writeReq = SysMakeFileIoRequest(file, gPattern, sizeof(gPattern), &done);
uint32_t s = SysWriteFile(&writeReq);
if (s != NTLSTATUS_SUCCESS) fail_status("write test file", s);
if (done != sizeof(gPattern)) fail("short write");
uint32_t h = 0;
SYS_OPEN_FILE_REQUEST req = SysMakeOpenFileRequest(SysSvFromCstr(gSelfPath),
SYS_FILE_ACCESS_READ | SYS_FILE_ACCESS_QUERY,
SYS_FILE_OPEN_EXISTING,
SYS_FILE_FLAG_NONE,
&h);
uint32_t s = SysOpenFile(&req);
if (s != NTLSTATUS_SUCCESS)
return s;
s = SysFlushFile(file);
if (s != NTLSTATUS_SUCCESS) fail_status("flush test file", s);
uint64_t size = 0;
s = SysGetFileSize(h, &size);
if (s != NTLSTATUS_SUCCESS) {
SysCloseHandle(h);
return s;
}
if (size == 0) {
SysCloseHandle(h);
return NTLSTATUS_NOT_FOUND;
}
*fileOut = h;
*sizeOut = size;
return NTLSTATUS_SUCCESS;
}
// Checks one regular-file metadata result.
@@ -171,13 +163,13 @@ static void check_directory_stat(const SYS_FILE_STAT* stat) {
if (stat->LinkCount == 0) fail("directory has zero links");
}
// Exercises handle and path stat for one regular file.
static void test_file_stat(uint32_t file, const char* path) {
// Exercises handle and path stat for this process image.
static void test_file_stat(uint32_t file, uint64_t expectedSize) {
SYS_FILE_STAT byHandle;
memset(&byHandle, 0xA5, sizeof(byHandle));
uint32_t s = SysGetFileStat(file, &byHandle);
if (s != NTLSTATUS_SUCCESS) fail_status("get file stat", s);
check_regular_stat(&byHandle, sizeof(gPattern), "handle stat");
check_regular_stat(&byHandle, expectedSize, "handle stat");
uint64_t size = 0;
s = SysGetFileSize(file, &size);
@@ -186,10 +178,10 @@ static void test_file_stat(uint32_t file, const char* path) {
SYS_FILE_STAT byPath;
memset(&byPath, 0x5A, sizeof(byPath));
SYS_STAT_FILE_REQUEST statReq = { SysSvFromCstr(path), &byPath };
SYS_STAT_FILE_REQUEST statReq = { SysSvFromCstr(gSelfPath), &byPath };
s = SysStatFile(&statReq);
if (s != NTLSTATUS_SUCCESS) fail_status("path stat", s);
check_regular_stat(&byPath, sizeof(gPattern), "path stat");
check_regular_stat(&byPath, expectedSize, "path stat");
if (byHandle.VolumeId != 0 && byPath.VolumeId != 0 && byHandle.VolumeId != byPath.VolumeId)
fail("handle and path volume ids differ");
@@ -197,11 +189,11 @@ static void test_file_stat(uint32_t file, const char* path) {
fail("handle and path file ids differ");
}
// Exercises path stat for one directory.
static void test_directory_stat(const char* path) {
// Exercises path stat for the executable's containing directory.
static void test_directory_stat(void) {
SYS_FILE_STAT stat;
memset(&stat, 0xCC, sizeof(stat));
SYS_STAT_FILE_REQUEST statReq = { SysSvFromCstr(path), &stat };
SYS_STAT_FILE_REQUEST statReq = { SysSvFromCstr(gSelfDir), &stat };
uint32_t s = SysStatFile(&statReq);
if (s != NTLSTATUS_SUCCESS) fail_status("directory stat", s);
check_directory_stat(&stat);
@@ -226,30 +218,15 @@ static void test_failure_paths(void) {
int main(void) {
print("file_stat_smoke: start\n");
fill_pattern();
uint32_t file = 0;
const char* filePath = 0;
uint32_t s = create_temp_file(&file, &filePath);
if (s != NTLSTATUS_SUCCESS) fail_status("create temp file", s);
uint64_t size = 0;
uint32_t s = open_self_image(&file, &size);
if (s != NTLSTATUS_SUCCESS) fail_status("open self image", s);
write_test_file(file);
test_file_stat(file, filePath);
test_file_stat(file, size);
SysCloseHandle(file);
delete_file_path(filePath);
const char* dirPath = 0;
s = create_temp_dir(&dirPath);
if (s != NTLSTATUS_SUCCESS) fail_status("create temp dir", s);
test_directory_stat(dirPath);
remove_dir_path(dirPath);
SYS_PATH_REQUEST dirReq = { SysSvFromCstr(dirPath) };
s = SysCreateDir(&dirReq);
if (s != NTLSTATUS_SUCCESS) fail_status("recreate temp dir", s);
test_directory_stat(dirPath);
remove_dir_path(dirPath);
test_directory_stat();
test_failure_paths();
print("file_stat_smoke: PASS\n");
code/user/tests/input_model_smoke/input_model_smoke.c
+21
View File
@@ -89,6 +89,27 @@ int main(void) {
badReq.QueueHandle = 0xdeadbeefu;
expect_status("read input bad handle", SysReadInputEvent(&badReq), NTLSTATUS_INVALID_HANDLE);
SYS_GLOBAL_INPUT_STATE state;
state.Size = sizeof(state);
state.Flags = SYS_GLOBAL_INPUT_STATE_FLAG_NONE;
expect_status("query global input null", SysQueryGlobalInputState((SYS_GLOBAL_INPUT_STATE*)0), NTLSTATUS_INVALID_PARAMETER);
SYS_GLOBAL_INPUT_STATE badState = state;
badState.Size = 0;
expect_status("query global input bad size", SysQueryGlobalInputState(&badState), NTLSTATUS_INVALID_PARAMETER);
badState = state;
badState.Flags = 1;
expect_status("query global input bad flags", SysQueryGlobalInputState(&badState), NTLSTATUS_INVALID_PARAMETER);
status = SysQueryGlobalInputState(&state);
if (status != NTLSTATUS_SUCCESS)
fail_status("query global input", status);
if (state.Size != sizeof(state))
fail_status("query global input size", state.Size);
if (state.Flags != SYS_GLOBAL_INPUT_STATE_FLAG_NONE)
fail_status("query global input flags", state.Flags);
status = SysPeekInputEvent(&req);
if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_WOULD_BLOCK)
fail_status("peek input immediate", status);
code/user/tests/pcache_pressure/pcache_pressure.c
+53 -46
View File
@@ -1,6 +1,6 @@
// Sequential file-cache pressure test.
// Reads a large read-only asset through SysReadFile and checks that the
// read-only page cache stays bounded by dropping clean pages.
// Reads this test executable through SysReadFile and checks cache behavior when
// the image is large enough to apply pressure.
#include <stdint.h>
#include <stddef.h>
@@ -56,47 +56,50 @@ static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) {
return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0);
}
static uint32_t open_first_large(uint32_t* fileOut, const char** pathOut, uint64_t* sizeOut) {
static const char* candidates[] = {
"1:/apps/video.bk2",
"0:/apps/video.bk2",
"/apps/video.bk2",
"video.bk2",
static char gSelfPath[SYS_PATH_MAX];
"1:/apps/doom1.wad",
"0:/apps/doom1.wad",
"/apps/doom1.wad",
"doom1.wad",
};
// Copies a string view into a fixed NUL-terminated buffer.
static int copy_view(char* out, size_t outSize, SYS_STRING_VIEW view) {
if (!out || outSize == 0 || !view.Data || view.Length + 1 > outSize)
return 0;
memcpy(out, view.Data, view.Length);
out[view.Length] = '\0';
return 1;
}
for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) {
uint32_t h = 0;
SYS_OPEN_FILE_REQUEST req = SysMakeOpenFileRequest(SysSvFromCstr(candidates[i]),
SYS_FILE_ACCESS_READ | SYS_FILE_ACCESS_QUERY,
SYS_FILE_OPEN_EXISTING,
SYS_FILE_FLAG_NONE,
&h);
uint32_t s = SysOpenFile(&req);
if (s != NTLSTATUS_SUCCESS)
continue;
// Opens this process image as the cache-pressure input file.
static uint32_t open_self_image(uint32_t* fileOut, const char** pathOut, uint64_t* sizeOut) {
SYS_PEB* peb = SysGetPeb();
if (!peb || peb->Magic != SYS_PEB_MAGIC || peb->Version != SYS_ENV_VERSION)
return NTLSTATUS_INVALID_PARAMETER;
if (!copy_view(gSelfPath, sizeof(gSelfPath), peb->ImagePath))
return NTLSTATUS_INVALID_PARAMETER;
uint64_t size = 0;
s = SysGetFileSize(h, &size);
if (s != NTLSTATUS_SUCCESS) {
SysCloseHandle(h);
continue;
}
if (size >= MIN_PRESSURE_BYTES) {
*fileOut = h;
*pathOut = candidates[i];
*sizeOut = size;
return NTLSTATUS_SUCCESS;
}
uint32_t h = 0;
SYS_OPEN_FILE_REQUEST req = SysMakeOpenFileRequest(SysSvFromCstr(gSelfPath),
SYS_FILE_ACCESS_READ | SYS_FILE_ACCESS_QUERY,
SYS_FILE_OPEN_EXISTING,
SYS_FILE_FLAG_NONE,
&h);
uint32_t s = SysOpenFile(&req);
if (s != NTLSTATUS_SUCCESS)
return s;
uint64_t size = 0;
s = SysGetFileSize(h, &size);
if (s != NTLSTATUS_SUCCESS) {
SysCloseHandle(h);
return s;
}
if (size == 0) {
SysCloseHandle(h);
return NTLSTATUS_NOT_FOUND;
}
return NTLSTATUS_NOT_FOUND;
*fileOut = h;
*pathOut = gSelfPath;
*sizeOut = size;
return NTLSTATUS_SUCCESS;
}
int main(void) {
@@ -110,9 +113,9 @@ int main(void) {
uint32_t file = 0;
const char* path = 0;
uint64_t size = 0;
uint32_t s = open_first_large(&file, &path, &size);
uint32_t s = open_self_image(&file, &path, &size);
if (s != NTLSTATUS_SUCCESS)
fail_status("open large asset", s);
fail_status("open self image", s);
print("pcache_pressure: using ");
print(path ? path : "<unknown>");
@@ -135,8 +138,8 @@ int main(void) {
SysCloseHandle(file);
if (total < MIN_PRESSURE_BYTES)
fail("asset read was too short to pressure the cache");
if (total == 0)
fail("self image read returned no bytes");
if (query_cache(&after) != NTLSTATUS_SUCCESS)
fail("final cache stats query failed");
@@ -165,12 +168,16 @@ int main(void) {
print(" batch-pages+="); print_u64(batchPages);
print("\n");
if (loads < 512)
fail("expected at least 512 cache page loads");
if (after.ResidentPages > RESIDENT_LIMIT_PAGES)
fail("resident page cache exceeded pressure bound");
if (loads > RESIDENT_LIMIT_PAGES && drops == 0)
fail("cache did not drop clean pages under pressure");
if (size >= MIN_PRESSURE_BYTES) {
if (loads < 512)
fail("expected at least 512 cache page loads");
if (after.ResidentPages > RESIDENT_LIMIT_PAGES)
fail("resident page cache exceeded pressure bound");
if (loads > RESIDENT_LIMIT_PAGES && drops == 0)
fail("cache did not drop clean pages under pressure");
} else {
print("pcache_pressure: self image below pressure size; pressure bounds skipped\n");
}
print("pcache_pressure: PASS\n");
exit_now(0);