replace the VHD/diskpart disk image pipeline with a directory-based vvfat root, we'll see when it breaks, fix /O2, add an all_tests, generalize the fat32 driver into a fat 16+32 driver, change policy on volume drivers, collapse KCPU's fields into a single atomic KE_CPU_STATE state machine, move scheduler ticking to per-CPU LAPIC timers, gate verbose SMP topology logging behind selftest diagnostics and add SMP pool-checkpoint validation around AP admission, add checked intrusive list helpers behind a new KE_DEBUG_LIST_CHECKS flag, add ObInitializeObjectHeader/ObTryReferenceObject, add waitable display clock objects and a SYS_OPEN_DISPLAY_CLOCK syscall for vsync-style waits, refactor kernel input into a per-queue KE_INPUT_CONTEXT with its own key/mouse ring buffers and DPC-driven key repeat, add KeInputQueuePeekEvent/ReadEvent/WaitKey and a SYS_QUERY_GLOBAL_INPUT_STATE syscall, split the kernel pool allocator out of mm_virt, new video files!

code/kernel/include/kernel/ke_cpu.h

@@ -15,6 +15,16 @@
 
 struct THREAD;
 
+typedef enum KE_CPU_STATE {
+    KeCpuAbsent = 0,
+    KeCpuPresent,
+    KeCpuStarted,
+    KeCpuParked,
+    KeCpuAdmissionRequested,
+    KeCpuSchedulerActive,
+    KeCpuOnline
+} KE_CPU_STATE;
+
 typedef struct KCPU {
     // GS:0 points back at the owning KCPU while executing in ring 0.
     // User mode keeps its TEB in GS_BASE; swapgs restores this pointer on
@@ -26,11 +36,8 @@ typedef struct KCPU {
     uint32_t AcpiProcessorId;
     uint32_t BootProcessorIndex;
 
-    uint8_t  Present;
-    uint8_t  Started;
-    uint8_t  Parked;
-    uint8_t  Online;
-    uint8_t  IsBsp;
+    volatile long State;
+    uint8_t IsBsp;
 
     // CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded
     // from the APIC ID using CPUID leaf 0x1F/0x0B when available, with a
@@ -61,11 +68,6 @@ typedef struct KCPU {
     uint32_t InterruptDepth;
     volatile long ReschedulePending;
 
-    // AP scheduler admission handshake. APs set SchedulerActive just before
-    // they leave the raw kernel park loop and enter their per-CPU idle thread.
-    volatile long SchedulerCommand;
-    volatile long SchedulerActive;
-
     TSS64    Tss;
     uint64_t Gdt[KE_GDT_ENTRIES];
     KGDTR    Gdtr;

code/kernel/include/kernel/ke_debug.h

@@ -23,6 +23,7 @@
 #define KE_DEBUG_INPUT_TRACE      0x00000200u
 #define KE_DEBUG_VM_ASSERTS       0x00000400u
 #define KE_DEBUG_SYSCALL_ASSERTS  0x00000800u
+#define KE_DEBUG_LIST_CHECKS      0x00001000u
 
 void KeInitDebug(const HAL_API* hal, uint32_t flags);
 uint32_t KeGetDebugFlags(void);

code/kernel/include/kernel/ke_display.h

@@ -1,6 +1,7 @@
 // Kernel ke display declarations shared by implementation files and kernel clients.
 
 #pragma once
+#include <stdint.h>
 #include <ntlite/ddi/hal_api.h>
 
 #ifndef DISPLAY_API
@@ -11,6 +12,23 @@
 #endif
 #endif
 
+#include <kernel/ke_event.h>
+#include <kernel/ke_list.h>
+#include <kernel/ob_object.h>
 #include <ntlite/ddk/display.h>
 
+typedef struct DISPLAY_CLOCK_OBJECT {
+    OBJECT Header;
+    EVENT* Event;
+    LIST_ENTRY Link;
+    uint64_t PeriodTicks;
+    uint64_t NextTick;
+    uint64_t LastFrameGeneration;
+    uint8_t Linked;
+} DISPLAY_CLOCK_OBJECT;
+
 void KeDisplayInitialize(const HAL_API* hal);
+DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock);
+DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock);
+DISPLAY_API void KeDisplayTick(void);
+DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device);

code/kernel/include/kernel/ke_input.h

@@ -1,4 +1,4 @@
-// Kernel ke input declarations shared by implementation files and kernel clients.
+// Kernel input declarations shared by implementation files and kernel clients.
 
 #pragma once
 #include <stdint.h>
@@ -8,9 +8,45 @@
 #include <ntlite/status.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_event.h>
+#include <kernel/ke_dpc.h>
+
+typedef struct SYS_GLOBAL_INPUT_STATE SYS_GLOBAL_INPUT_STATE;
+
+typedef struct INPUT_KEY_QUEUE_ENTRY {
+    INPUT_KEY Key;
+    char Character;
+    uint32_t Modifiers;
+} INPUT_KEY_QUEUE_ENTRY;
+
+typedef struct KE_INPUT_CONTEXT {
+    const HAL_API* Hal;
+    volatile uint8_t KeyDown[256];
+    volatile uint32_t Modifiers;
+    volatile long long Generation;
+
+    INPUT_KEY_QUEUE_ENTRY KeyQueue[256];
+    volatile int KeyHead;
+    volatile int KeyTail;
+    volatile INPUT_KEY RepeatKey;
+    volatile uint64_t RepeatNextTick;
+    KDPC RepeatDpc;
+
+    INPUT_MOUSE_EVENT MouseQueue[128];
+    volatile int MouseHead;
+    volatile int MouseTail;
+    volatile int MouseX;
+    volatile int MouseY;
+    volatile int MouseBoundsWidth;
+    volatile int MouseBoundsHeight;
+    volatile int MouseWheel;
+    volatile uint8_t MouseButtons;
+
+    EVENT* WaitEvent;
+} KE_INPUT_CONTEXT;
 
 typedef struct INPUT_QUEUE {
     OBJECT Header;
+    KE_INPUT_CONTEXT* Context;
     EVENT* WaitEvent;
 } INPUT_QUEUE;
 
@@ -29,10 +65,14 @@ bool      KeInputIsKeyDown(INPUT_KEY key);
 bool      KeInputPeekEvent(INPUT_KEY_RECORD* record);
 bool      KeInputReadEvent(INPUT_KEY_RECORD* record);
 KSTATUS   KeInputOpenQueue(INPUT_QUEUE** out);
+bool      KeInputQueuePeekEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record);
+bool      KeInputQueueReadEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record);
+KSTATUS   KeInputQueueWaitKey(INPUT_QUEUE* queue, uint8_t alertable, uint64_t timeoutMs);
 EVENT*    KeInputQueueWaitEvent(INPUT_QUEUE* queue);
 INPUT_KEY KeInputReadKey(void);
 char      KeInputReadChar(void);
 uint32_t  KeInputGetModifiers(void);
+KSTATUS   KeInputQueryGlobalState(SYS_GLOBAL_INPUT_STATE* out);
 
 bool KeInputMousePeek(INPUT_MOUSE_EVENT* event);
 bool KeInputMouseRead(INPUT_MOUSE_EVENT* event);

code/kernel/include/kernel/ke_list.h

@@ -1,9 +1,24 @@
-// Kernel ke list declarations shared by implementation files and kernel clients.
+// Kernel checked-list declarations shared by implementation files and kernel clients.
 
 #pragma once
 
-// ke_list.h - kernel-private include path for the shared intrusive list type.
-// The type currently remains DDK-visible because legacy driver objects still
-// expose embedded lists and IRPs. Opaque request/device work removes that need.
+// The list layout remains DDK-visible, while checked mutation policy stays in
+// the kernel so public LIST_ENTRY does not gain debug-only fields.
 
+#include <stdbool.h>
 #include <ntlite/ddk/list.h>
+
+void KeListInsertTailChecked(LIST_ENTRY* head, LIST_ENTRY* node);
+void KeListInsertHeadChecked(LIST_ENTRY* head, LIST_ENTRY* node);
+void KeListRemoveChecked(LIST_ENTRY* node);
+void KeListValidateHead(const LIST_ENTRY* head);
+
+// Removes the first entry through the checked removal path.
+static inline LIST_ENTRY* KeListPopHeadChecked(LIST_ENTRY* head) {
+    if (LIST_IS_EMPTY(head))
+        return (LIST_ENTRY*)0;
+
+    LIST_ENTRY* node = head->Fwd;
+    KeListRemoveChecked(node);
+    return node;
+}

code/kernel/include/kernel/ke_sched.h

@@ -2,7 +2,7 @@
 
 #pragma once
 // ke_sched.h - Multi-Level Feedback Queue scheduler
-// Three run queues (Low / Normal / High). IRQ 0 drives KeSchedulerTick.
+// Three run queues (Low / Normal / High). LAPIC timer IRQs drive KeSchedulerTick.
 #include <kernel/ke_thread.h>
 
 struct KCPU;
@@ -41,11 +41,11 @@ KSTATUS KiCreateIdleThreadForCpu(struct KCPU* cpu);
 void    KeReadyThread(THREAD* thread);
 void    KeUnreadyThread(THREAD* thread);
 
-// Called from the timer ISR every SCHED_TICK_MS. This performs timer
-// accounting and requests a reschedule only when the current thread has
-// exhausted its scheduler budget, or when idle work becomes available. The
-// actual context switch happens from
-// KiExitInterrupt after the IRQ handler has returned and EOI has been sent.
+// Called from each CPU timer ISR every SCHED_TICK_MS. Every CPU performs local
+// quantum accounting and may request a local reschedule. Only the BSP advances
+// global scheduler time and services sleepers, waitable timers, input repeat,
+// and priority aging. The actual context switch happens from KiExitInterrupt
+// after the IRQ handler has returned and EOI has been sent.
 void    KeSchedulerTick(void);
 
 // Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule

code/kernel/include/kernel/mm_pool.h

@@ -0,0 +1,38 @@
+// Kernel pool allocator declarations.
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include <ntlite/status.h>
+
+#define POOL_BASE  0xFFFFC00000000000ULL
+#define POOL_LIMIT 0xFFFFE00000000000ULL
+
+#define MM_POOL_TAG(a, b, c, d) \
+    ((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
+     ((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24))
+
+// Initializes the kernel pool and optionally commits its initial extent.
+void MmPoolInitialize(size_t poolSize);
+
+// Allocates zeroed kernel-pool memory from the non-user heap.
+__declspec(dllexport) void* MmAllocatePool(size_t size);
+
+// Allocates zeroed kernel-pool memory with a diagnostic tag.
+__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag);
+
+// Releases kernel-pool memory back to the non-user heap.
+__declspec(dllexport) void MmFreePool(void* ptr);
+
+// Reallocates kernel-pool memory while preserving existing contents.
+__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
+
+// Verifies kernel-pool boundary tags and free-list links.
+KSTATUS MmValidateKernelPool(const char* reason);
+
+// Describes a virtual address that falls inside the kernel-pool aperture.
+void MmPoolDebugDescribeAddress(uintptr_t addr);
+
+// Returns the currently committed kernel-pool bytes.
+size_t MmGetKernelPoolUsage(void);

code/kernel/include/kernel/mm_virt.h

@@ -7,6 +7,7 @@
 #include <stddef.h>
 #include <kernel/mm_phys.h>
 #include <kernel/mm_pte.h>
+#include <kernel/mm_pool.h>
 #include <ntlite/status.h>
 
 struct PROCESS;
@@ -39,9 +40,6 @@ typedef enum MM_VAD_TYPE {
 #define USER_LIMIT         0x00007FFFFFFFF000ULL
 
 #define KERNEL_BASE        0xFFFF800000000000ULL
-#define POOL_BASE          0xFFFFC00000000000ULL
-#define POOL_LIMIT         0xFFFFE00000000000ULL
-
 // Initialisation
 // Called by KernelMain after MmInitializePhysical.
 // bootPml4Phys: the PML4 set up by the bootloader.
@@ -83,17 +81,6 @@ __declspec(dllexport) void    MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
 __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
 __declspec(dllexport) void    MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
 
-// Kernel pool - ring 0 only, never PAGE_USER.
-// Best-fit free-list allocator, thread-safe via spinlock.
-#define MM_POOL_TAG(a, b, c, d) \
-    ((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
-     ((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24))
-
-__declspec(dllexport) void* MmAllocatePool(size_t size);
-__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag);
-__declspec(dllexport) void  MmFreePool(void* ptr);
-__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
-
 // Virtual address descriptors.
 // The single shared address space still has per-process VAD ownership so process
 // teardown can reclaim leaked mappings. VADs track sparse per-page commitment,
@@ -165,7 +152,6 @@ __declspec(dllexport) void  MmFreeUserImageForProcess(PROCESS* process, void* pt
 __declspec(dllexport) void* MmAllocateUserImage(size_t size);
 __declspec(dllexport) void  MmFreeUserImage(void* ptr, size_t size);
 
-size_t MmGetKernelPoolUsage(void);
 size_t MmGetUserHeapUsage(void);
 
 // Demand-paging telemetry used by the shell stats command.

code/kernel/include/kernel/ob_object.h

@@ -2,16 +2,23 @@
 
 #pragma once
 
-// ob_object.h - executive object model.
-// Every resource the kernel tracks is a typed, reference-counted OBJECT. The
-// legacy OBJECT layout lives in the DDK while driver/device objects are still
-// public; object-manager routines remain kernel-private.
+// Every tracked kernel resource has a typed, reference-counted object header.
+// The DDK exposes the header because driver and device objects embed it, while
+// lifetime policy and transition helpers remain owned by the object manager.
 
 #include <stddef.h>
+#include <stdbool.h>
 #include <ntlite/status.h>
 #include <ntlite/ddk/object.h>
 
-// Increment the reference count. Call whenever ownership is shared.
+// Initialize an embedded object header with one live reference.
+__declspec(dllexport) void ObInitializeObjectHeader(OBJECT* object, OBJECT_TYPE type);
+
+// Acquire a reference only if the object is still alive and inserted.
+// The caller must stabilize the pointer through its owning lock or reference.
+__declspec(dllexport) bool ObTryReferenceObject(OBJECT* object);
+
+// Increment the reference count. Stale-object use is fatal.
 __declspec(dllexport) void ObReferenceObject(OBJECT* object);
 
 // Decrement the reference count and destroy the object when it reaches zero.

code/kernel/include/kernel/syscall_nr.h

@@ -92,61 +92,63 @@
 #define SYS_REAL_PATH        73
 
 // Console, display, and surfaces.
-#define SYS_WRITE_CONSOLE          74
-#define SYS_GET_CONSOLE_INFO       75
-#define SYS_SET_CONSOLE_CURSOR     76
-#define SYS_FILL_CONSOLE_CHARACTER 77
-#define SYS_FILL_CONSOLE_ATTRIBUTE 78
-#define SYS_PUT_SCREEN_CHAR        79
-#define SYS_CLEAR_SCREEN           80
-#define SYS_GET_SCREEN_SIZE        81
-#define SYS_QUERY_DISPLAY_INFO     82
-#define SYS_CREATE_SURFACE         83
-#define SYS_MAP_SURFACE            84
-#define SYS_QUERY_SURFACE          85
-#define SYS_PRESENT_SURFACE        86
+#define SYS_WRITE_CONSOLE            74
+#define SYS_GET_CONSOLE_INFO         75
+#define SYS_SET_CONSOLE_CURSOR       76
+#define SYS_FILL_CONSOLE_CHARACTER   77
+#define SYS_FILL_CONSOLE_ATTRIBUTE   78
+#define SYS_PUT_SCREEN_CHAR          79
+#define SYS_CLEAR_SCREEN             80
+#define SYS_GET_SCREEN_SIZE          81
+#define SYS_QUERY_DISPLAY_INFO       82
+#define SYS_OPEN_DISPLAY_CLOCK       83
+#define SYS_CREATE_SURFACE           84
+#define SYS_MAP_SURFACE              85
+#define SYS_QUERY_SURFACE            86
+#define SYS_PRESENT_SURFACE          87
 
 // Input.
-#define SYS_POLL_KEYBOARD      87
-#define SYS_IS_KEY_DOWN        88
-#define SYS_READ_KEYBOARD_CHAR 89
-#define SYS_READ_KEY           90
-#define SYS_GET_KEY_MODIFIERS  91
-#define SYS_OPEN_INPUT_QUEUE   92
-#define SYS_READ_INPUT_EVENT   93
-#define SYS_PEEK_INPUT_EVENT   94
-#define SYS_PEEK_MOUSE         95
-#define SYS_READ_MOUSE         96
-#define SYS_GET_MOUSE_POS      97
+#define SYS_POLL_KEYBOARD            88
+#define SYS_IS_KEY_DOWN              89
+#define SYS_READ_KEYBOARD_CHAR       90
+#define SYS_READ_KEY                 91
+#define SYS_GET_KEY_MODIFIERS        92
+#define SYS_QUERY_GLOBAL_INPUT_STATE 93
+#define SYS_OPEN_INPUT_QUEUE         94
+#define SYS_READ_INPUT_EVENT         95
+#define SYS_PEEK_INPUT_EVENT         96
+#define SYS_PEEK_MOUSE               97
+#define SYS_READ_MOUSE               98
+#define SYS_GET_MOUSE_POS            99
 
 // Time, debug, and system information.
-#define SYS_CREATE_TIMER       98
-#define SYS_SET_TIMER          99
-#define SYS_CANCEL_TIMER       100
-#define SYS_GET_TICK           101
-#define SYS_GET_TICK_FREQUENCY 102
-#define SYS_GET_SYSTEM_TIME    103
-#define SYS_PRINT_DEBUG        104
-#define SYS_QUERY_TRAP_FRAME   105
-#define SYS_GET_MEMORY_STATUS  106
-#define SYS_QUERY_KERNEL_STATS 107
+#define SYS_CREATE_TIMER             100
+#define SYS_SET_TIMER                101
+#define SYS_CANCEL_TIMER             102
+#define SYS_GET_TICK                 103
+#define SYS_GET_TICK_FREQUENCY       104
+#define SYS_GET_SYSTEM_TIME          105
+#define SYS_PRINT_DEBUG              106
+#define SYS_QUERY_TRAP_FRAME         107
+#define SYS_GET_MEMORY_STATUS        108
+#define SYS_QUERY_KERNEL_STATS       109
 
 // Native sockets and polling.
-#define SYS_CREATE_SOCKET     108
-#define SYS_SOCKET_IO_ASYNC   109
-#define SYS_BIND_SOCKET       110
-#define SYS_LISTEN_SOCKET     111
-#define SYS_ACCEPT_SOCKET     112
-#define SYS_CONNECT_SOCKET    113
-#define SYS_SEND_SOCKET       114
-#define SYS_RECV_SOCKET       115
-#define SYS_SHUTDOWN_SOCKET   116
-#define SYS_SEND_TO_SOCKET    117
-#define SYS_RECV_FROM_SOCKET  118
-#define SYS_RESOLVE_HOST      119
-#define SYS_SET_SOCKET_OPTION 120
-#define SYS_GET_SOCKET_OPTION 121
-#define SYS_POLL              122
-#define SYS_UNPUBLISH_QUEUE   123
+#define SYS_CREATE_SOCKET            110
+#define SYS_SOCKET_IO_ASYNC          111
+#define SYS_BIND_SOCKET              112
+#define SYS_LISTEN_SOCKET            113
+#define SYS_ACCEPT_SOCKET            114
+#define SYS_CONNECT_SOCKET           115
+#define SYS_SEND_SOCKET              116
+#define SYS_RECV_SOCKET              117
+#define SYS_SHUTDOWN_SOCKET          118
+#define SYS_SEND_TO_SOCKET           119
+#define SYS_RECV_FROM_SOCKET         120
+#define SYS_RESOLVE_HOST             121
+#define SYS_SET_SOCKET_OPTION        122
+#define SYS_GET_SOCKET_OPTION        123
+#define SYS_POLL                     124
+#define SYS_UNPUBLISH_QUEUE          125
 
-#define SYS_MAX 124
+#define SYS_MAX 126

code/kernel/io_device.c

@@ -79,9 +79,7 @@ __declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
                                                     SYS_STRING_VIEW name) {
     if (!driver) return;
     memset(driver, 0, sizeof(*driver));
-    driver->Header.Type = ObjectTypeDriver;
-    driver->Header.RefCount = 1;
-    driver->Header.Destroy = (void*)0;
+    ObInitializeObjectHeader(&driver->Header, ObjectTypeDriver);
     driver->Name = name;
 }
 
@@ -91,9 +89,9 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
     if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
         return NTLSTATUS_NO_MEMORY;
 
-    if (driver->Header.Type != ObjectTypeDriver) {
-        driver->Header.Type = ObjectTypeDriver;
-        driver->Header.RefCount = 1;
+    if (driver->Header.Type != ObjectTypeDriver ||
+        driver->Header.LifetimeState != ObjectAlive) {
+        ObInitializeObjectHeader(&driver->Header, ObjectTypeDriver);
     }
 
     gDriverObjects[gDriverObjectCount++] = driver;

code/kernel/ke_cpu.c

@@ -48,10 +48,7 @@ void KeInitializeBootstrapCpu(void) {
     cpu->Self = cpu;
     cpu->CpuIndex = 0;
     cpu->LapicId = 0;
-    cpu->Present = 1;
-    cpu->Started = 1;
-    cpu->Parked = 0;
-    cpu->Online = 1;
+    cpu->State = KeCpuStarted;
     cpu->IsBsp = 1;
     cpu->PackageId = 0;
     cpu->CoreId = 0;
@@ -65,10 +62,8 @@ void KeInitializeBootstrapCpu(void) {
     cpu->CurrentIrql = PASSIVE_LEVEL;
     cpu->InterruptDepth = 0;
     cpu->ReschedulePending = 0;
-    cpu->SchedulerCommand = 0;
-    cpu->SchedulerActive = 0;
     cpu->KernelStack = gBootstrapCpuStack;
-    cpu->KernelStackTop = (uintptr_t)(gBootstrapCpuStack + sizeof(gBootstrapCpuStack));
+    cpu->KernelStackTop = ((uintptr_t)(gBootstrapCpuStack + sizeof(gBootstrapCpuStack))) & ~(uintptr_t)15;
     memset(&cpu->Tss, 0, sizeof(cpu->Tss));
     cpu->Tss.IoMapBase = sizeof(TSS64);
     cpu->Tss.RSP0 = cpu->KernelStackTop;

code/kernel/ke_debug.c

@@ -28,6 +28,8 @@ static const HAL_API* gDebugHal;
 #define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
 #define KE_DEBUG_LOG_LINE_SIZE   1024u
 #define KE_FATAL_OUTPUT_LINE_SIZE 2048u
+#define KE_STACK_DUMP_SLOT_COUNT 12u
+#define KE_RBP_CALL_STACK_DEPTH 16u
 
 static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
 static char* gDebugLogBuffer;
@@ -412,6 +414,22 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
     }
 }
 
+// Prints a resolved address without ending the current line.
+static void KiPrintAddressValue(PROCESS* proc, uintptr_t addr) {
+    uintptr_t base = 0;
+    size_t size = 0;
+    const char* mod = KiResolveModuleEx(proc, addr, &base, &size);
+
+    if (base) {
+        KiFaultPrintf("%016llx <%s+0x%016llX>",
+                      (unsigned long long)addr,
+                      mod,
+                      (unsigned long long)(addr - base));
+    } else {
+        KiFaultPrintf("%016llx", (unsigned long long)addr);
+    }
+}
+
 // Checks whether a kernel virtual range is mapped before diagnostic reads.
 static int KiKernelRangeReadable(uintptr_t addr, size_t size) {
     if (!addr || size == 0) return 0;
@@ -562,44 +580,103 @@ static void KiDumpModules(PROCESS* proc) {
     }
 }
 
-// Dumps user Stack.
-static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
-    KiFaultPrintf("User stack @ %016llx:\n", (unsigned long long)rsp);
+#if 0
+// Returns whether a range is inside the current kernel stack.
+static int KiKernelStackContains(const THREAD* thread, uintptr_t addr, size_t size) {
+    if (!thread || !thread->KernelStackBase || !thread->KernelStackTop || !addr || size == 0)
+        return 0;
+    if (size - 1u > (size_t)(UINTPTR_MAX - addr))
+        return 0;
+
+    uintptr_t base = (uintptr_t)thread->KernelStackBase;
+    uintptr_t top = thread->KernelStackTop;
+    uintptr_t end = addr + size;
+    return addr >= base && end <= top;
+}
+
+// Returns whether the fault frame is using the current kernel stack.
+static int KiFaultFrameOnKernelStack(const THREAD* thread, const KFAULT_FRAME* f) {
+    if (!f || !thread)
+        return 0;
+    return KiKernelStackContains(thread, (uintptr_t)f->Rsp, sizeof(uint64_t));
+}
+
+#endif
+// Dumps stack words near RSP.
+static void KiDumpStack(PROCESS* proc, uintptr_t rsp) {
+    KiFaultPrintf("Stack @ %016llx:\n", (unsigned long long)rsp);
     if (!rsp) {
-        KiFaultPrintf("  <no user RSP captured>\n");
+        KiFaultPrintf("  <no RSP captured>\n");
         return;
     }
 
-    for (int i = 0; i < 12; i++) {
-        uintptr_t slot = rsp + (uintptr_t)i * 8u;
+    for (uint32_t i = 0; i < KE_STACK_DUMP_SLOT_COUNT; i++) {
+        uintptr_t slot = rsp + (uintptr_t)i * sizeof(uint64_t);
         uint64_t value = 0;
+        KiFaultPrintf("  [%02u] %016llx : ",
+                      (unsigned)i,
+                      (unsigned long long)slot);
         if (KiUserReadU64(slot, &value)) {
-            uintptr_t base = 0;
-            size_t size = 0;
-            const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size);
-            if (base) {
-                KiFaultPrintf("  [%02d] %016llx : %016llx <%s+0x%016llX>\n",
-                       i,
-                       (unsigned long long)slot,
-                       (unsigned long long)value,
-                       mod,
-                       (unsigned long long)((uintptr_t)value - base));
-            } else {
-                KiFaultPrintf("  [%02d] %016llx : %016llx\n",
-                       i,
-                       (unsigned long long)slot,
-                       (unsigned long long)value);
-            }
+            KiPrintAddressValue(proc, (uintptr_t)value);
+            KiFaultPrintf("\n");
         } else {
-            KiFaultPrintf("  [%02d] %016llx : <unreadable>\n",
-                   i,
-                   (unsigned long long)slot);
+            KiFaultPrintf("<unreadable>\n");
         }
     }
 }
 
-// Dumps fault Frame.
-static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
+#if 0
+// Dumps a best-effort frame-pointer call stack.
+static void KiDumpRbpCallStack(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
+    uintptr_t rbp = f ? (uintptr_t)f->Rbp : 0;
+    KiFaultPrintf("RBP call stack @ %016llx:\n", (unsigned long long)rbp);
+    if (!rbp) {
+        KiFaultPrintf("  <no RBP captured>\n");
+        return;
+    }
+
+    int kernelStack = KiFaultFrameOnKernelStack(thread, f);
+    for (uint32_t depth = 0; depth < KE_RBP_CALL_STACK_DEPTH; depth++) {
+        if (rbp & (sizeof(uint64_t) - 1u)) {
+            KiFaultPrintf("  stop: unaligned RBP %016llx\n",
+                          (unsigned long long)rbp);
+            return;
+        }
+        if (kernelStack && !KiKernelStackContains(thread, rbp, sizeof(uint64_t) * 2u)) {
+            KiFaultPrintf("  stop: RBP outside kernel stack %016llx\n",
+                          (unsigned long long)rbp);
+            return;
+        }
+
+        uint64_t next = 0;
+        uint64_t ret = 0;
+        if (!KiUserReadU64(rbp, &next) || !KiUserReadU64(rbp + sizeof(uint64_t), &ret)) {
+            KiFaultPrintf("  stop: frame unreadable at %016llx\n",
+                          (unsigned long long)rbp);
+            return;
+        }
+
+        KiFaultPrintf("  #%02u rbp=%016llx ret=",
+                      (unsigned)depth,
+                      (unsigned long long)rbp);
+        KiPrintAddressValue(proc, (uintptr_t)ret);
+        KiFaultPrintf(" next=%016llx\n", (unsigned long long)next);
+
+        if (!next)
+            return;
+        if ((uintptr_t)next <= rbp) {
+            KiFaultPrintf("  stop: non-forward RBP chain %016llx -> %016llx\n",
+                          (unsigned long long)rbp,
+                          (unsigned long long)next);
+            return;
+        }
+        rbp = (uintptr_t)next;
+    }
+}
+
+#endif
+// Dumps a fault frame.
+static void KiDumpFaultFrame(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
     if (!f) return;
 
     KiFaultPrintf("Regs:\n");
@@ -634,7 +711,10 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
         KiDumpBytes(proc, "Bytes near RIP", (uintptr_t)f->Rip);
     }
 
-    KiDumpUserStack(proc, (uintptr_t)f->Rsp);
+    KiDumpStack(proc, (uintptr_t)f->Rsp);
+#if 0
+    KiDumpRbpCallStack(proc, thread, f);
+#endif
 }
 
 static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
@@ -710,7 +790,7 @@ static _Noreturn void KiPanicKernelFault(PROCESS* proc, THREAD* thread, const KF
              thread,
              proc ? proc->ProcessId : 0,
              proc ? proc->Name : "<none>");
-    KiDumpFaultFrame(proc, frame);
+    KiDumpFaultFrame(proc, thread, frame);
     if (vector == 14) {
         MmDebugDescribeAddress((uintptr_t)frame->Cr2);
         if (frame->Rip) MmDebugDescribeAddress((uintptr_t)frame->Rip);
@@ -755,7 +835,7 @@ static _Noreturn void KiTerminateUserFault(PROCESS* proc, THREAD* thread, const
                        (uintptr_t)frame->Rip,
                        (uintptr_t)frame->ErrorCode,
                        (uintptr_t)frame->Cr2);
-    KiDumpFaultFrame(proc, frame);
+    KiDumpFaultFrame(proc, thread, frame);
     KiDumpModules(proc);
 
     KiFinishFaultedProcess(proc, thread, KeExceptionProcessExitStatus((uint32_t)frame->Vector));

code/kernel/ke_display.c

@@ -15,6 +15,9 @@
 #include <kernel/ke_work.h>
 #include <kernel/ke_mutex.h>
 #include <kernel/ke_sched.h>
+#include <kernel/ke_event.h>
+#include <kernel/ke_list.h>
+#include <kernel/ob_object.h>
 #include <kernel/ke_irql.h>
 
 #define DISPLAY_DEFAULT_FG 0x00CCCCCCu
@@ -32,6 +35,8 @@
 #define DISPLAY_CURSOR_POS_INVALID (-2147483647 - 1)
 #define DISPLAY_OUTPUT_BUFFER_CAPACITY 4096u
 #define DISPLAY_OUTPUT_DRAIN_COUNT     128u
+#define DISPLAY_CLOCK_DEFAULT_PERIOD_MS 1ull
+#define DISPLAY_CLOCK_SIGNAL_BATCH      32u
 
 typedef struct DISPLAY_OUTPUT_GLYPH {
     char Character;
@@ -58,6 +63,10 @@ static KWORK_ITEM gDisplayCursorWorkItem;
 static volatile long gDisplayCursorWorkInitialized;
 static bool gDisplayTransitionInProgress;
 static uint64_t gDisplayGeneration;
+static LIST_ENTRY gDisplayClockList;
+static SPINLOCK gDisplayClockLock = SPINLOCK_INIT;
+static volatile long gDisplayClockInitialized;
+static uint64_t gDisplayFrameGeneration;
 
 static DISPLAY_DEVICE gBootFbDevice;
 static BOOTFB_CONTEXT gBootFb;
@@ -125,6 +134,43 @@ static bool KiDisplayOutputSuspended(void);
 static void KiDisplayCursorWorker(void* context);
 static void KiDisplayDirtyAll(void);
 
+// Converts a display-clock period to scheduler ticks with minimum one-tick progress.
+static uint64_t KiDisplayClockMillisecondsToTicks(uint64_t milliseconds) {
+    if (milliseconds == 0)
+        milliseconds = DISPLAY_CLOCK_DEFAULT_PERIOD_MS;
+
+    uint64_t ticks = milliseconds / SCHED_TICK_MS;
+    if ((milliseconds % SCHED_TICK_MS) != 0)
+        ticks++;
+    return ticks ? ticks : 1;
+}
+
+// Reads the global scheduler tick used by display-clock deadlines.
+static uint64_t KiDisplayClockReadSchedulerTick(void) {
+    KE_SCHED_STATS stats;
+    KeGetSchedulerStats(&stats);
+    return stats.Ticks;
+}
+
+// Destroys a display clock and removes it from the manager tick list.
+static void KiDisplayClockDestroy(OBJECT* object) {
+    DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)object;
+    if (!clock)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
+    if (clock->Linked) {
+        ListRemove(&clock->Link);
+        clock->Linked = 0;
+    }
+    KeReleaseSpinlock(&gDisplayClockLock, flags);
+
+    if (clock->Event) {
+        ObDereferenceObject(&clock->Event->Header);
+        clock->Event = (EVENT*)0;
+    }
+}
+
 static bool KiDisplayHasCurrentThread(void) {
     return KeGetCurrentThread() != (THREAD*)0;
 }
@@ -1105,6 +1151,10 @@ void KeDisplayInitialize(const HAL_API* hal) {
     gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID;
     gDisplayTransitionInProgress = false;
     gDisplayGeneration = 1;
+    gDisplayClockLock = (SPINLOCK)SPINLOCK_INIT;
+    LIST_INIT(&gDisplayClockList);
+    gDisplayFrameGeneration = 0;
+    _InterlockedExchange(&gDisplayClockInitialized, 1);
     if (!gDisplayProviderMutex)
         (void)KeMutexCreate(&gDisplayProviderMutex);
     KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
@@ -1178,6 +1228,149 @@ void KeDisplayInitialize(const HAL_API* hal) {
     KiDisplayRegisterOutputEndpoint();
 }
 
+DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock) {
+    if (!outClock)
+        return NTLSTATUS_INVALID_PARAMETER;
+    *outClock = (DISPLAY_CLOCK_OBJECT*)0;
+
+    if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
+        return NTLSTATUS_NOT_SUPPORTED;
+
+    uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
+    bool hasDevice = KiDisplayHasDevice();
+    KeReleaseSpinlock(&gDisplayLock, displayFlags);
+    if (!hasDevice)
+        return NTLSTATUS_NOT_SUPPORTED;
+
+    DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)ObAllocateObject(ObjectTypeDisplayClock,
+                                                                          sizeof(DISPLAY_CLOCK_OBJECT));
+    if (!clock)
+        return NTLSTATUS_NO_MEMORY;
+
+    clock->Header.Destroy = KiDisplayClockDestroy;
+    LIST_INIT(&clock->Link);
+    clock->PeriodTicks = KiDisplayClockMillisecondsToTicks(periodMilliseconds);
+    uint64_t now = KiDisplayClockReadSchedulerTick();
+    clock->NextTick = (UINT64_MAX - now < clock->PeriodTicks) ? UINT64_MAX : now + clock->PeriodTicks;
+
+    KSTATUS s = KeEventCreate(EventAutoReset, 0, &clock->Event);
+    if (!KSUCCESS(s)) {
+        ObDereferenceObject(&clock->Header);
+        return s;
+    }
+
+    uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
+    clock->LastFrameGeneration = gDisplayFrameGeneration;
+    clock->Linked = 1;
+    ListInsertTail(&gDisplayClockList, &clock->Link);
+    KeReleaseSpinlock(&gDisplayClockLock, flags);
+
+    *outClock = clock;
+    return NTLSTATUS_SUCCESS;
+}
+
+DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) {
+    if (!clock)
+        return (EVENT*)0;
+    return clock->Event;
+}
+
+// Signals every display clock once for a provider-owned frame generation.
+static void KiDisplaySignalFrameGeneration(uint64_t generation) {
+    for (;;) {
+        EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
+        uint32_t count = 0;
+
+        uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
+        LIST_ENTRY* node = gDisplayClockList.Fwd;
+        while (node != &gDisplayClockList && count < DISPLAY_CLOCK_SIGNAL_BATCH) {
+            DISPLAY_CLOCK_OBJECT* clock = CONTAINING_RECORD(node, DISPLAY_CLOCK_OBJECT, Link);
+            node = node->Fwd;
+
+            if (!clock->Linked || !clock->Event || clock->LastFrameGeneration == generation)
+                continue;
+
+            if (!ObTryReferenceObject(&clock->Event->Header))
+                continue;
+            clock->LastFrameGeneration = generation;
+            events[count++] = clock->Event;
+        }
+        KeReleaseSpinlock(&gDisplayClockLock, flags);
+
+        for (uint32_t i = 0; i < count; i++) {
+            KeSignalEvent(events[i]);
+            ObDereferenceObject(&events[i]->Header);
+        }
+
+        if (count < DISPLAY_CLOCK_SIGNAL_BATCH)
+            break;
+    }
+}
+
+// Publishes a frame-ready event from the current primary display provider.
+DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device) {
+    if (!device || !_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
+        return;
+
+    uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
+    bool accepted = device == gPrimaryDisplay &&
+                    (device->Caps & DISPLAY_CAP_FRAME_EVENT) != 0;
+    KeReleaseSpinlock(&gDisplayLock, displayFlags);
+    if (!accepted)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
+    uint64_t generation = ++gDisplayFrameGeneration;
+    if (generation == 0)
+        generation = ++gDisplayFrameGeneration;
+    KeReleaseSpinlock(&gDisplayClockLock, flags);
+
+    KiDisplaySignalFrameGeneration(generation);
+}
+
+DISPLAY_API void KeDisplayTick(void) {
+    if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
+        return;
+
+    uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
+    bool providerOwnsFrames = gPrimaryDisplay &&
+                              (gPrimaryDisplay->Caps & DISPLAY_CAP_FRAME_EVENT) != 0;
+    KeReleaseSpinlock(&gDisplayLock, displayFlags);
+    if (providerOwnsFrames)
+        return;
+
+    uint64_t now = KiDisplayClockReadSchedulerTick();
+    for (;;) {
+        EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
+        uint32_t count = 0;
+
+        uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
+        LIST_ENTRY* node = gDisplayClockList.Fwd;
+        while (node != &gDisplayClockList && count < DISPLAY_CLOCK_SIGNAL_BATCH) {
+            DISPLAY_CLOCK_OBJECT* clock = CONTAINING_RECORD(node, DISPLAY_CLOCK_OBJECT, Link);
+            node = node->Fwd;
+
+            if (!clock->Linked || clock->NextTick > now || !clock->Event)
+                continue;
+
+            if (!ObTryReferenceObject(&clock->Event->Header))
+                continue;
+            uint64_t next = now + clock->PeriodTicks;
+            clock->NextTick = next < now ? UINT64_MAX : next;
+            events[count++] = clock->Event;
+        }
+        KeReleaseSpinlock(&gDisplayClockLock, flags);
+
+        for (uint32_t i = 0; i < count; i++) {
+            KeSignalEvent(events[i]);
+            ObDereferenceObject(&events[i]->Header);
+        }
+
+        if (count < DISPLAY_CLOCK_SIGNAL_BATCH)
+            break;
+    }
+}
+
 DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
     if (!device || !device->Name || !device->Ops || !device->Ops->GetMode)
         return NTLSTATUS_INVALID_PARAMETER;

code/kernel/ke_dpc.c

@@ -1,6 +1,7 @@
 // Deferred procedure call infrastructure. This file owns DPC queuing and execution outside hard IRQ context.
 
 #include <kernel/ke_dpc.h>
+#include <kernel/ke_list.h>
 #include <kernel/ke_irql.h>
 #include <kernel/ke_debug.h>
 #include <intrin.h>
@@ -58,7 +59,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
     dpc->SystemArgument2 = systemArgument2;
 
     uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
-    ListInsertTail(&gDpcQueue, &dpc->Link);
+    KeListInsertTailChecked(&gDpcQueue, &dpc->Link);
     KeReleaseSpinlock(&gDpcLock, flags);
     return true;
 }
@@ -82,7 +83,7 @@ void KiDrainDpcQueue(void) {
 
     for (;;) {
         uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
-        LIST_ENTRY* node = ListPopHead(&gDpcQueue);
+        LIST_ENTRY* node = KeListPopHeadChecked(&gDpcQueue);
         if (!node) {
             KeReleaseSpinlock(&gDpcLock, flags);
             break;

code/kernel/ke_event.c

@@ -1,6 +1,7 @@
 // Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
 
 #include <stdbool.h>
+#include <kernel/ke_list.h>
 #include <intrin.h>
 #include <kernel/ke_event.h>
 #include <kernel/ke_thread.h>
@@ -39,7 +40,7 @@ static void KiCompleteMultiWaitBlockLocked(KE_EVENT_WAIT_BLOCK* block) {
     if (!block || !block->Armed)
         return;
 
-    ListRemove(&block->Link);
+    KeListRemoveChecked(&block->Link);
     block->Armed = 0;
 
     KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
@@ -67,7 +68,7 @@ static void KiCleanupMultiWaitBlocks(KE_EVENT_MULTI_WAIT_CONTEXT* ctx) {
 
         uintptr_t flags = KeAcquireSpinlock(&event->Lock);
         if (block->Armed) {
-            ListRemove(&block->Link);
+            KeListRemoveChecked(&block->Link);
             block->Armed = 0;
         }
         KeReleaseSpinlock(&event->Lock, flags);
@@ -117,7 +118,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
     if (event->Type == EventManualReset) {
         event->Signaled = 1;
         while (!LIST_IS_EMPTY(&event->WaitList)) {
-            LIST_ENTRY* node = ListPopHead(&event->WaitList);
+            LIST_ENTRY* node = KeListPopHeadChecked(&event->WaitList);
             THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
             KeSetThreadWaitList(t, (LIST_ENTRY*)0);
             KeReleaseSpinlock(&event->Lock, flags);
@@ -132,7 +133,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
     } else {
         while (!LIST_IS_EMPTY(&event->WaitList)) {
             event->Signaled = 0;
-            LIST_ENTRY* node = ListPopHead(&event->WaitList);
+            LIST_ENTRY* node = KeListPopHeadChecked(&event->WaitList);
             THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
             KeSetThreadWaitList(t, (LIST_ENTRY*)0);
             KeReleaseSpinlock(&event->Lock, flags);
@@ -195,7 +196,7 @@ __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
     }
 
     KePrepareBlockThread(self, &event->Header, alertable);
-    ListInsertTail(&event->WaitList, &self->SchedulerLink);
+    KeListInsertTailChecked(&event->WaitList, &self->SchedulerLink);
     KeSetThreadWaitList(self, &event->WaitList);
     KeReleaseSpinlock(&event->Lock, flags);
     return KeCommitBlockThread(self, alertable);
@@ -267,7 +268,7 @@ __declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t c
 
         if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0) {
             stackBlocks[i].Armed = 1;
-            ListInsertTail(&event->MultiWaitList, &stackBlocks[i].Link);
+            KeListInsertTailChecked(&event->MultiWaitList, &stackBlocks[i].Link);
         }
         KeReleaseSpinlock(&event->Lock, flags);
     }

code/kernel/ke_input.c

@@ -4,6 +4,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include <string.h>
+#include <intrin.h>
 
 #include <kernel/ke_input.h>
 #include <kernel/ke_debug.h>
@@ -11,6 +12,7 @@
 #include <kernel/ke_event.h>
 #include <kernel/ke_dpc.h>
 #include <kernel/ob_object.h>
+#include <ntlite/syscall.h>
 
 #define KEY_QUEUE_SIZE   256u
 #define KEY_QUEUE_MASK   (KEY_QUEUE_SIZE - 1u)
@@ -27,38 +29,16 @@ typedef struct KEY_CHAR_PAIR {
     char shifted;
 } KEY_CHAR_PAIR;
 
-static const HAL_API* gInputHal;
-
-static volatile uint8_t gKeyDown[MAX_INPUT_KEYS];
-static volatile uint32_t gModifiers;
-
-typedef struct INPUT_KEY_QUEUE_ENTRY {
-    INPUT_KEY key;
-    char      ch;
-    uint32_t  modifiers;
-} INPUT_KEY_QUEUE_ENTRY;
-
-static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE];
-static volatile int gKeyHead;
-static volatile int gKeyTail;
-static volatile INPUT_KEY gRepeatKey;
-static volatile uint64_t  gRepeatNextTick;
-static KDPC gRepeatDpc;
-
-static INPUT_MOUSE_EVENT gMouseQueue[MOUSE_QUEUE_SIZE];
-static volatile int      gMouseHead;
-static volatile int      gMouseTail;
-static volatile int      gMouseX;
-static volatile int      gMouseY;
-static volatile int      gMouseBoundsW = 1920;
-static volatile int      gMouseBoundsH = 1080;
-static volatile uint8_t  gMouseButtons;
-
+static KE_INPUT_CONTEXT gDefaultInputContext;
 static volatile uint32_t gNextInputDeviceId = 1;
-static EVENT* gKeyboardEvent;
 
-static char KiInputKeyToChar(INPUT_KEY key);
-static void KiInputServiceRepeat(void);
+static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key);
+static void KiInputServiceRepeat(KE_INPUT_CONTEXT* context);
+
+// Advances the physical input snapshot generation after visible state changes.
+static void KiInputBumpGeneration(KE_INPUT_CONTEXT* context) {
+    _InterlockedIncrement64(&context->Generation);
+}
 
 static const KEY_CHAR_PAIR gCharMap[] = {
     { INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' },
@@ -85,28 +65,28 @@ static const KEY_CHAR_PAIR gCharMap[] = {
 };
 
 // Updates the waitable keyboard event to match queued keyboard input.
-static void KiInputRefreshKeyboardEvent(void) {
-    if (!gKeyboardEvent) return;
-    if (gKeyHead != gKeyTail) KeSignalEvent(gKeyboardEvent);
-    else KeClearEvent(gKeyboardEvent);
+static void KiInputRefreshKeyboardEvent(KE_INPUT_CONTEXT* context) {
+    if (!context->WaitEvent) return;
+    if (context->KeyHead != context->KeyTail) KeSignalEvent(context->WaitEvent);
+    else KeClearEvent(context->WaitEvent);
 }
 
 // Queues a keyboard event and wakes waiters when the event was accepted.
-static void KiInputQueueKey(INPUT_KEY key, char ch) {
+static void KiInputQueueKey(KE_INPUT_CONTEXT* context, INPUT_KEY key, char ch) {
     if (key == INPUT_KEY_NONE) return;
-    int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
-    if (next == gKeyHead) return;
-    gKeyQueue[gKeyTail].key = key;
-    gKeyQueue[gKeyTail].ch = ch;
-    gKeyQueue[gKeyTail].modifiers = gModifiers;
-    gKeyTail = next;
-    KiInputRefreshKeyboardEvent();
+    int next = (context->KeyTail + 1) & (int)KEY_QUEUE_MASK;
+    if (next == context->KeyHead) return;
+    context->KeyQueue[context->KeyTail].Key = key;
+    context->KeyQueue[context->KeyTail].Character = ch;
+    context->KeyQueue[context->KeyTail].Modifiers = context->Modifiers;
+    context->KeyTail = next;
+    KiInputRefreshKeyboardEvent(context);
 }
 
 // Implements the input Queue Has Space operation.
-static bool KiInputQueueHasSpace(void) {
-    int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
-    return next != gKeyHead;
+static bool KiInputQueueHasSpace(KE_INPUT_CONTEXT* context) {
+    int next = (context->KeyTail + 1) & (int)KEY_QUEUE_MASK;
+    return next != context->KeyHead;
 }
 
 // Implements the input Key Is Repeatable operation.
@@ -131,61 +111,60 @@ static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
 }
 
 // Implements the input Now Ticks operation.
-static uint64_t KiInputNowTicks(void) {
-    if (gInputHal && gInputHal->Timer.QueryTicks)
-        return gInputHal->Timer.QueryTicks();
+static uint64_t KiInputNowTicks(KE_INPUT_CONTEXT* context) {
+    if (context->Hal && context->Hal->Timer.QueryTicks)
+        return context->Hal->Timer.QueryTicks();
     return 0;
 }
 
 // Implements the input Start Repeat operation.
-static void KiInputStartRepeat(INPUT_KEY key, uint64_t now) {
+static void KiInputStartRepeat(KE_INPUT_CONTEXT* context, INPUT_KEY key, uint64_t now) {
     if (!KiInputKeyIsRepeatable(key)) return;
-    gRepeatKey = key;
-    gRepeatNextTick = now + KEY_REPEAT_DELAY_US;
+    context->RepeatKey = key;
+    context->RepeatNextTick = now + KEY_REPEAT_DELAY_US;
 }
 
 // Implements the input Stop Repeat operation.
-static void KiInputStopRepeat(INPUT_KEY key) {
-    if (gRepeatKey == key) {
-        gRepeatKey = INPUT_KEY_NONE;
-        gRepeatNextTick = 0;
+static void KiInputStopRepeat(KE_INPUT_CONTEXT* context, INPUT_KEY key) {
+    if (context->RepeatKey == key) {
+        context->RepeatKey = INPUT_KEY_NONE;
+        context->RepeatNextTick = 0;
     }
 }
 
 // Implements the input Service Repeat operation.
-static void KiInputServiceRepeat(void) {
-    INPUT_KEY key = gRepeatKey;
-    if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key]) {
-        gRepeatKey = INPUT_KEY_NONE;
-        gRepeatNextTick = 0;
+static void KiInputServiceRepeat(KE_INPUT_CONTEXT* context) {
+    INPUT_KEY key = context->RepeatKey;
+    if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !context->KeyDown[key]) {
+        context->RepeatKey = INPUT_KEY_NONE;
+        context->RepeatNextTick = 0;
         return;
     }
 
-    uint64_t now = KiInputNowTicks();
-    if (now == 0 || now < gRepeatNextTick)
+    uint64_t now = KiInputNowTicks(context);
+    if (now == 0 || now < context->RepeatNextTick)
         return;
 
-    if (!KiInputQueueHasSpace()) {
-        gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
+    if (!KiInputQueueHasSpace(context)) {
+        context->RepeatNextTick = now + KEY_REPEAT_PERIOD_US;
         return;
     }
 
-    KiInputQueueKey(key, KiInputKeyToChar(key));
-    gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
+    KiInputQueueKey(context, key, KiInputKeyToChar(context, key));
+    context->RepeatNextTick = now + KEY_REPEAT_PERIOD_US;
 }
 
 // Implements the input Repeat Dpc Routine operation.
 static void KiInputRepeatDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
     (void)dpc;
-    (void)context;
     (void)systemArgument1;
     (void)systemArgument2;
 
-    KiInputServiceRepeat();
+    KiInputServiceRepeat((KE_INPUT_CONTEXT*)context);
 }
 
 // Implements the input Set Modifier operation.
-static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
+static void KiInputSetModifier(KE_INPUT_CONTEXT* context, INPUT_KEY key, bool pressed) {
     uint32_t bit = 0;
     switch (key) {
         case INPUT_KEY_LEFT_SHIFT:  bit = INPUT_MOD_LSHIFT; break;
@@ -200,25 +179,25 @@ static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
     }
 
     if (bit) {
-        if (pressed) gModifiers |= bit;
-        else         gModifiers &= ~bit;
+        if (pressed) context->Modifiers |= bit;
+        else         context->Modifiers &= ~bit;
     }
 
-    if (pressed && key == INPUT_KEY_CAPS_LOCK) gModifiers ^= INPUT_MOD_CAPS_LOCK;
-    if (pressed && key == INPUT_KEY_NUM_LOCK)  gModifiers ^= INPUT_MOD_NUM_LOCK;
+    if (pressed && key == INPUT_KEY_CAPS_LOCK) context->Modifiers ^= INPUT_MOD_CAPS_LOCK;
+    if (pressed && key == INPUT_KEY_NUM_LOCK)  context->Modifiers ^= INPUT_MOD_NUM_LOCK;
 }
 
 // Implements the input Key To Char operation.
-static char KiInputKeyToChar(INPUT_KEY key) {
+static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key) {
     if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) {
-        bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
-        bool caps    = (gModifiers & INPUT_MOD_CAPS_LOCK) != 0;
+        bool shifted = (context->Modifiers & INPUT_MOD_SHIFT) != 0;
+        bool caps    = (context->Modifiers & INPUT_MOD_CAPS_LOCK) != 0;
         char c = (char)('a' + (key - INPUT_KEY_A));
         if (shifted ^ caps) c = (char)(c - 'a' + 'A');
         return c;
     }
 
-    bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
+    bool shifted = (context->Modifiers & INPUT_MOD_SHIFT) != 0;
     for (size_t i = 0; i < sizeof(gCharMap) / sizeof(gCharMap[0]); i++) {
         if (gCharMap[i].key == key)
             return shifted ? gCharMap[i].shifted : gCharMap[i].normal;
@@ -228,22 +207,27 @@ static char KiInputKeyToChar(INPUT_KEY key) {
 
 // Implements the input Initialize operation.
 void KeInputInitialize(const HAL_API* hal) {
-    gInputHal = hal;
-    memset((void*)gKeyDown, 0, sizeof(gKeyDown));
-    gModifiers = 0;
-    gKeyHead = gKeyTail = 0;
-    gRepeatKey = INPUT_KEY_NONE;
-    gRepeatNextTick = 0;
-    gMouseHead = gMouseTail = 0;
-    gMouseButtons = 0;
-    KeInitializeDpc(&gRepeatDpc, KiInputRepeatDpcRoutine, (void*)0);
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    context->Hal = hal;
+    memset((void*)context->KeyDown, 0, sizeof(context->KeyDown));
+    context->Modifiers = 0;
+    context->Generation = 0;
+    context->KeyHead = context->KeyTail = 0;
+    context->RepeatKey = INPUT_KEY_NONE;
+    context->RepeatNextTick = 0;
+    context->MouseHead = context->MouseTail = 0;
+    context->MouseBoundsWidth = 1920;
+    context->MouseBoundsHeight = 1080;
+    context->MouseWheel = 0;
+    context->MouseButtons = 0;
+    KeInitializeDpc(&context->RepeatDpc, KiInputRepeatDpcRoutine, context);
 
-    if (!gKeyboardEvent) {
-        KSTATUS s = KeEventCreate(EventManualReset, 0, &gKeyboardEvent);
+    if (!context->WaitEvent) {
+        KSTATUS s = KeEventCreate(EventManualReset, 0, &context->WaitEvent);
         if (!KSUCCESS(s))
             DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
     }
-    KiInputRefreshKeyboardEvent();
+    KiInputRefreshKeyboardEvent(context);
 
     int w = 1920, h = 1080;
     DISPLAY_MODE mode;
@@ -271,11 +255,12 @@ __declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t f
 
 // Implements the input Submit Key operation.
 __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (!event) return;
     INPUT_KEY key = event->Key;
     if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return;
 
-    bool wasDown = gKeyDown[key] != 0;
+    bool wasDown = context->KeyDown[key] != 0;
     bool pressed = event->Pressed;
 
     if (pressed) {
@@ -284,24 +269,26 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
         if (wasDown)
             return;
 
-        gKeyDown[key] = 1u;
-        KiInputSetModifier(key, true);
-        KiInputQueueKey(key, KiInputKeyToChar(key));
-        KiInputStartRepeat(key, KiInputNowTicks());
+        context->KeyDown[key] = 1u;
+        KiInputSetModifier(context, key, true);
+        KiInputBumpGeneration(context);
+        KiInputQueueKey(context, key, KiInputKeyToChar(context, key));
+        KiInputStartRepeat(context, key, KiInputNowTicks(context));
     } else {
         if (!wasDown)
             return;
 
-        gKeyDown[key] = 0u;
-        KiInputSetModifier(key, false);
-        KiInputStopRepeat(key);
+        context->KeyDown[key] = 0u;
+        KiInputSetModifier(context, key, false);
+        KiInputBumpGeneration(context);
+        KiInputStopRepeat(context, key);
     }
 }
 
 // Implements the input Commit Mouse Position operation.
-static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
-    int bw = gMouseBoundsW;
-    int bh = gMouseBoundsH;
+static bool KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int nx, int ny, uint8_t buttons) {
+    int bw = context->MouseBoundsWidth;
+    int bh = context->MouseBoundsHeight;
 
     if (bw < 1) bw = 1;
     if (bh < 1) bh = 1;
@@ -310,55 +297,70 @@ static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
     if (nx >= bw) nx = bw - 1;
     if (ny >= bh) ny = bh - 1;
 
-    gMouseX = nx;
-    gMouseY = ny;
-    gMouseButtons = buttons;
+    bool changed = (nx != context->MouseX) || (ny != context->MouseY) || (buttons != context->MouseButtons);
+    context->MouseX = nx;
+    context->MouseY = ny;
+    context->MouseButtons = buttons;
     KeDisplaySetCursorPosition(nx, ny);
+    return changed;
 }
 
 // Implements the input Submit Mouse operation.
 __declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (!event) return;
 
-    int oldX = gMouseX;
-    int oldY = gMouseY;
+    int oldX = context->MouseX;
+    int oldY = context->MouseY;
     int nx = oldX + event->dx;
     int ny = oldY + event->dy;
 
-    KiInputCommitMousePosition(nx, ny, event->buttons);
+    bool changed = KiInputCommitMousePosition(context, nx, ny, event->buttons);
+    if (event->wheel != 0) {
+        context->MouseWheel += event->wheel;
+        changed = true;
+    }
+    if (changed) KiInputBumpGeneration(context);
 
-    int next = (gMouseTail + 1) & (int)MOUSE_QUEUE_MASK;
-    if (next != gMouseHead) {
-        gMouseQueue[gMouseTail] = *event;
-        gMouseTail = next;
+    int next = (context->MouseTail + 1) & (int)MOUSE_QUEUE_MASK;
+    if (next != context->MouseHead) {
+        context->MouseQueue[context->MouseTail] = *event;
+        context->MouseTail = next;
     }
 }
 
 // Implements the input Submit Mouse Absolute operation.
 __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE_EVENT* event) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (!event) return;
 
-    int oldX = gMouseX;
-    int oldY = gMouseY;
-    KiInputCommitMousePosition(event->x, event->y, event->buttons);
+    int oldX = context->MouseX;
+    int oldY = context->MouseY;
+    bool changed = KiInputCommitMousePosition(context, event->x, event->y, event->buttons);
+    if (event->wheel != 0) {
+        context->MouseWheel += event->wheel;
+        changed = true;
+    }
+    if (changed) KiInputBumpGeneration(context);
 
     INPUT_MOUSE_EVENT rel;
-    rel.dx = gMouseX - oldX;
-    rel.dy = gMouseY - oldY;
+    rel.dx = context->MouseX - oldX;
+    rel.dy = context->MouseY - oldY;
     rel.wheel = event->wheel;
     rel.buttons = event->buttons;
 
-    int next = (gMouseTail + 1) & (int)MOUSE_QUEUE_MASK;
-    if (next != gMouseHead) {
-        gMouseQueue[gMouseTail] = rel;
-        gMouseTail = next;
+    int next = (context->MouseTail + 1) & (int)MOUSE_QUEUE_MASK;
+    if (next != context->MouseHead) {
+        context->MouseQueue[context->MouseTail] = rel;
+        context->MouseTail = next;
     }
 }
 
 // Implements the input Submit Mouse Absolute Scaled operation.
 __declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons) {
-    int bw = gMouseBoundsW;
-    int bh = gMouseBoundsH;
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    int bw = context->MouseBoundsWidth;
+    int bh = context->MouseBoundsHeight;
     if (bw < 1) bw = 1;
     if (bh < 1) bh = 1;
 
@@ -370,65 +372,90 @@ __declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint3
     KeInputSubmitMouseAbsolute(&ev);
 }
 
+// Returns the next queued key without consuming it.
 INPUT_KEY KeInputKbPoll(void) {
-    if (gKeyHead == gKeyTail) KiInputServiceRepeat();
-    return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    if (context->KeyHead == context->KeyTail) KiInputServiceRepeat(context);
+    return (context->KeyHead == context->KeyTail) ? INPUT_KEY_NONE : context->KeyQueue[context->KeyHead].Key;
 }
 
-// Waits until at least one keyboard event is available or the timeout expires.
-KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs) {
-    if (KeInputKbPoll() != INPUT_KEY_NONE)
+// Waits for keyboard input in one context or until the timeout expires.
+static KSTATUS KiInputWaitKey(KE_INPUT_CONTEXT* context, uint8_t alertable, uint64_t timeoutMs) {
+    if (!context)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    KiInputServiceRepeat(context);
+    if (context->KeyHead != context->KeyTail)
         return NTLSTATUS_SUCCESS;
-    if (!gKeyboardEvent)
+    if (!context->WaitEvent)
         return NTLSTATUS_FAILURE;
 
     for (;;) {
-        EVENT* event = gKeyboardEvent;
+        EVENT* event = context->WaitEvent;
         uint32_t selected = 0;
         KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected);
         if (!KSUCCESS(s))
             return s;
-        if (KeInputKbPoll() != INPUT_KEY_NONE)
+
+        KiInputServiceRepeat(context);
+        if (context->KeyHead != context->KeyTail)
             return NTLSTATUS_SUCCESS;
     }
 }
 
+// Waits for keyboard input in the default input context.
+KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs) {
+    return KiInputWaitKey(&gDefaultInputContext, alertable, timeoutMs);
+}
+
 // Services typematic repeat from the scheduler timer bottom half.
 void KeInputTimerTick(void) {
-    INPUT_KEY key = gRepeatKey;
-    if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key])
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    INPUT_KEY key = context->RepeatKey;
+    if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !context->KeyDown[key])
         return;
 
-    uint64_t now = KiInputNowTicks();
-    if (now != 0 && now >= gRepeatNextTick)
-        KeInsertQueueDpc(&gRepeatDpc, (void*)0, (void*)0);
+    uint64_t now = KiInputNowTicks(context);
+    if (now != 0 && now >= context->RepeatNextTick)
+        KeInsertQueueDpc(&context->RepeatDpc, (void*)0, (void*)0);
 }
 
 // Implements the input Is Key Down operation.
 bool KeInputIsKeyDown(INPUT_KEY key) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
-    return gKeyDown[key] != 0;
+    return context->KeyDown[key] != 0;
 }
 
-// Implements the input Peek Event operation.
+// Peeks one keyboard event from an input context without consuming it.
+static bool KiInputPeekEvent(KE_INPUT_CONTEXT* context, INPUT_KEY_RECORD* record) {
+    if (!context || !record) return false;
+    if (context->KeyHead == context->KeyTail) KiInputServiceRepeat(context);
+    if (context->KeyHead == context->KeyTail) return false;
+
+    INPUT_KEY_QUEUE_ENTRY entry = context->KeyQueue[context->KeyHead];
+    record->Key = entry.Key;
+    record->Char = entry.Character;
+    record->Modifiers = entry.Modifiers;
+    return true;
+}
+
+// Reads one keyboard event from an input context.
+static bool KiInputReadEvent(KE_INPUT_CONTEXT* context, INPUT_KEY_RECORD* record) {
+    if (!KiInputPeekEvent(context, record)) return false;
+    context->KeyHead = (context->KeyHead + 1) & (int)KEY_QUEUE_MASK;
+    KiInputRefreshKeyboardEvent(context);
+    return true;
+}
+
+// Peeks one keyboard event from the default input context.
 bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
-    if (!record) return false;
-    if (gKeyHead == gKeyTail) KiInputServiceRepeat();
-    if (gKeyHead == gKeyTail) return false;
-
-    INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead];
-    record->Key = entry.key;
-    record->Char = entry.ch;
-    record->Modifiers = entry.modifiers;
-    return true;
+    return KiInputPeekEvent(&gDefaultInputContext, record);
 }
 
-// Implements the input Read Event operation.
+// Reads one keyboard event from the default input context.
 bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
-    if (!KeInputPeekEvent(record)) return false;
-    gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
-    KiInputRefreshKeyboardEvent();
-    return true;
+    return KiInputReadEvent(&gDefaultInputContext, record);
 }
 
 // Implements the input Read Key operation.
@@ -445,53 +472,84 @@ char KeInputReadChar(void) {
 
 // Implements the input Get Modifiers operation.
 uint32_t KeInputGetModifiers(void) {
-    return gModifiers;
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    return context->Modifiers;
+}
+
+// Snapshots the current physical global input state without consuming events.
+KSTATUS KeInputQueryGlobalState(SYS_GLOBAL_INPUT_STATE* out) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    if (!out) return NTLSTATUS_INVALID_PARAMETER;
+
+    out->Size = sizeof(*out);
+    out->Flags = SYS_GLOBAL_INPUT_STATE_FLAG_NONE;
+    out->Generation = (uint64_t)context->Generation;
+    out->Modifiers = context->Modifiers;
+    memset(out->KeyDown, 0, sizeof(out->KeyDown));
+
+    for (uint32_t key = 0; key < MAX_INPUT_KEYS; key++) {
+        if (context->KeyDown[key])
+            out->KeyDown[key >> 3] |= (uint8_t)(1u << (key & 7u));
+    }
+
+    out->MouseX = context->MouseX;
+    out->MouseY = context->MouseY;
+    out->MouseWheel = context->MouseWheel;
+    out->MouseButtons = context->MouseButtons;
+    return NTLSTATUS_SUCCESS;
 }
 
 // Implements the input Mouse Peek operation.
 bool KeInputMousePeek(INPUT_MOUSE_EVENT* event) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (!event) return false;
-    if (gMouseHead == gMouseTail) return false;
+    if (context->MouseHead == context->MouseTail) return false;
 
-    *event = gMouseQueue[gMouseHead];
+    *event = context->MouseQueue[context->MouseHead];
     return true;
 }
 
 // Implements the input Mouse Read operation.
 bool KeInputMouseRead(INPUT_MOUSE_EVENT* event) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (!KeInputMousePeek(event)) return false;
-    gMouseHead = (gMouseHead + 1) & (int)MOUSE_QUEUE_MASK;
+    context->MouseHead = (context->MouseHead + 1) & (int)MOUSE_QUEUE_MASK;
     return true;
 }
 
 // Implements the input Mouse Get Position operation.
 void KeInputMouseGetPosition(int* x, int* y) {
-    if (x) *x = gMouseX;
-    if (y) *y = gMouseY;
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    if (x) *x = context->MouseX;
+    if (y) *y = context->MouseY;
 }
 
 // Implements the input Mouse Set Position operation.
 void KeInputMouseSetPosition(int x, int y) {
-    int bw = gMouseBoundsW;
-    int bh = gMouseBoundsH;
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
+    int bw = context->MouseBoundsWidth;
+    int bh = context->MouseBoundsHeight;
     if (bw < 1) bw = 1;
     if (bh < 1) bh = 1;
     if (x < 0) x = 0;
     if (y < 0) y = 0;
     if (x >= bw) x = bw - 1;
     if (y >= bh) y = bh - 1;
-    gMouseX = x;
-    gMouseY = y;
+    bool changed = (x != context->MouseX) || (y != context->MouseY);
+    context->MouseX = x;
+    context->MouseY = y;
     KeDisplaySetCursorPosition(x, y);
+    if (changed) KiInputBumpGeneration(context);
 }
 
 // Implements the input Mouse Set Bounds operation.
 void KeInputMouseSetBounds(int w, int h) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (w < 1) w = 1;
     if (h < 1) h = 1;
-    gMouseBoundsW = w;
-    gMouseBoundsH = h;
-    KeInputMouseSetPosition(gMouseX, gMouseY);
+    context->MouseBoundsWidth = w;
+    context->MouseBoundsHeight = h;
+    KeInputMouseSetPosition(context->MouseX, context->MouseY);
 }
 
 // Destroys an input queue handle.
@@ -501,23 +559,42 @@ static void KiInputQueueDestroy(OBJECT* object) {
         ObDereferenceObject(&queue->WaitEvent->Header);
         queue->WaitEvent = (EVENT*)0;
     }
+    queue->Context = (KE_INPUT_CONTEXT*)0;
 }
 
 // Opens the default keyboard input queue as a waitable object.
 KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
+    KE_INPUT_CONTEXT* context = &gDefaultInputContext;
     if (!out) return NTLSTATUS_INVALID_PARAMETER;
     *out = (INPUT_QUEUE*)0;
-    if (!gKeyboardEvent) return NTLSTATUS_FAILURE;
+    if (!context->WaitEvent) return NTLSTATUS_FAILURE;
 
     INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
     if (!queue) return NTLSTATUS_NO_MEMORY;
     queue->Header.Destroy = KiInputQueueDestroy;
-    ObReferenceObject(&gKeyboardEvent->Header);
-    queue->WaitEvent = gKeyboardEvent;
+    queue->Context = context;
+    ObReferenceObject(&context->WaitEvent->Header);
+    queue->WaitEvent = context->WaitEvent;
     *out = queue;
     return NTLSTATUS_SUCCESS;
 }
 
+// Peeks one keyboard event from the queue's owning context.
+bool KeInputQueuePeekEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record) {
+    return queue ? KiInputPeekEvent(queue->Context, record) : false;
+}
+
+// Reads one keyboard event from the queue's owning context.
+bool KeInputQueueReadEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record) {
+    return queue ? KiInputReadEvent(queue->Context, record) : false;
+}
+
+// Waits for keyboard input in the queue's owning context.
+KSTATUS KeInputQueueWaitKey(INPUT_QUEUE* queue, uint8_t alertable, uint64_t timeoutMs) {
+    return queue ? KiInputWaitKey(queue->Context, alertable, timeoutMs)
+                 : NTLSTATUS_INVALID_PARAMETER;
+}
+
 // Returns the event that wakes when keyboard input can be read.
 EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
     return queue ? queue->WaitEvent : (EVENT*)0;

code/kernel/ke_list.c

@@ -0,0 +1,86 @@
+// Kernel checked intrusive-list helpers. This file detects stale or duplicate membership at the mutation site.
+
+#include <stdint.h>
+#include <kernel/ke_list.h>
+#include <kernel/ke_debug.h>
+
+#define KE_LIST_VALIDATION_LIMIT 1048576u
+
+// Returns whether checked list validation is enabled for this boot.
+static bool KiListChecksEnabled(void) {
+    return (KeGetDebugFlags() & KE_DEBUG_LIST_CHECKS) != 0;
+}
+
+// Stops on a checked-list invariant violation with the involved addresses.
+static _Noreturn void KiListPanic(const char* operation,
+                                 const LIST_ENTRY* head,
+                                 const LIST_ENTRY* node) {
+    KePanic(KE_PANIC_CODE_GENERIC,
+            "list: %s invariant failed head=0x%016llX node=0x%016llX fwd=0x%016llX bwd=0x%016llX\n",
+            operation ? operation : "unknown",
+            (uint64_t)(uintptr_t)head,
+            (uint64_t)(uintptr_t)node,
+            (uint64_t)(uintptr_t)(node ? node->Fwd : (LIST_ENTRY*)0),
+            (uint64_t)(uintptr_t)(node ? node->Bwd : (LIST_ENTRY*)0));
+}
+
+// Validates a list head and every forward/backward linkage reachable from it.
+void KeListValidateHead(const LIST_ENTRY* head) {
+    if (!KiListChecksEnabled())
+        return;
+    if (!head || !head->Fwd || !head->Bwd)
+        KiListPanic("validate-head", head, head);
+
+    const LIST_ENTRY* node = head;
+    for (uint32_t count = 0; count < KE_LIST_VALIDATION_LIMIT; count++) {
+        if (!node->Fwd || !node->Bwd)
+            KiListPanic("validate-link", head, node);
+        if (node->Fwd->Bwd != node || node->Bwd->Fwd != node)
+            KiListPanic("validate-neighbor", head, node);
+
+        node = node->Fwd;
+        if (node == head)
+            return;
+    }
+
+    KiListPanic("validate-cycle", head, node);
+}
+
+// Inserts a detached node at the tail after validating both invariants.
+void KeListInsertTailChecked(LIST_ENTRY* head, LIST_ENTRY* node) {
+    if (KiListChecksEnabled()) {
+        KeListValidateHead(head);
+        if (!node || node->Fwd != node || node->Bwd != node)
+            KiListPanic("insert-tail", head, node);
+    }
+
+    ListInsertTail(head, node);
+    KeListValidateHead(head);
+}
+
+// Inserts a detached node at the head after validating both invariants.
+void KeListInsertHeadChecked(LIST_ENTRY* head, LIST_ENTRY* node) {
+    if (KiListChecksEnabled()) {
+        KeListValidateHead(head);
+        if (!node || node->Fwd != node || node->Bwd != node)
+            KiListPanic("insert-head", head, node);
+    }
+
+    ListInsertHead(head, node);
+    KeListValidateHead(head);
+}
+
+// Removes a linked node and rejects detached or locally corrupt entries.
+void KeListRemoveChecked(LIST_ENTRY* node) {
+    if (KiListChecksEnabled()) {
+        if (!node || !node->Fwd || !node->Bwd ||
+            (node->Fwd == node && node->Bwd == node) ||
+            node->Fwd->Bwd != node || node->Bwd->Fwd != node) {
+            KiListPanic("remove", (LIST_ENTRY*)0, node);
+        }
+    }
+
+    ListRemove(node);
+    if (KiListChecksEnabled() && (node->Fwd != node || node->Bwd != node))
+        KiListPanic("remove-detach", (LIST_ENTRY*)0, node);
+}

code/kernel/ke_mutex.c

@@ -1,6 +1,7 @@
 // Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups.
 
 #include <stdbool.h>
+#include <kernel/ke_list.h>
 #include <kernel/ke_mutex.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
@@ -38,7 +39,7 @@ __declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) {
     }
 
     KePrepareBlockThread(self, &mutex->Header, 0);
-    ListInsertTail(&mutex->WaitList, &self->SchedulerLink);
+    KeListInsertTailChecked(&mutex->WaitList, &self->SchedulerLink);
     KeSetThreadWaitList(self, &mutex->WaitList);
     KeReleaseSpinlock(&mutex->Lock, flags);
     return KeCommitBlockThread(self, 0);
@@ -64,7 +65,7 @@ __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
     uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
 
     if (!LIST_IS_EMPTY(&mutex->WaitList)) {
-        LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
+        LIST_ENTRY* node = KeListPopHeadChecked(&mutex->WaitList);
         THREAD* waiter   = CONTAINING_RECORD(node, THREAD, SchedulerLink);
         KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
         // Direct handoff: the selected waiter owns the mutex when its wait

code/kernel/ke_process.c

@@ -1,6 +1,7 @@
 // Kernel process model. This file owns address-space metadata, handle tables, and process lifetime transitions.
 
 #include <string.h>
+#include <kernel/ke_list.h>
 #include <intrin.h>
 #include <kernel/ke_process.h>
 #include <kernel/ke_thread.h>
@@ -240,6 +241,7 @@ static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
         case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING;
         case ObjectTypeSurface:     return SYS_THREAD_WAIT_SURFACE;
         case ObjectTypeTimer:       return SYS_THREAD_WAIT_TIMER;
+        case ObjectTypeDisplayClock: return SYS_THREAD_WAIT_DISPLAY_CLOCK;
         default:                    return SYS_THREAD_WAIT_UNKNOWN;
     }
 }
@@ -377,7 +379,7 @@ static void KiProcessDestroy(OBJECT* obj) {
 
     // Remove from global list.
     uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
-    ListRemove(&proc->GlobalLink);
+    KeListRemoveChecked(&proc->GlobalLink);
     KeReleaseSpinlock(&gProcessListLock, flags);
 
     // Memory for the struct itself is freed by ObDereferenceObject after
@@ -453,7 +455,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
     }
 
     uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
-    ListInsertTail(&gProcessList, &proc->GlobalLink);
+    KeListInsertTailChecked(&gProcessList, &proc->GlobalLink);
     KeReleaseSpinlock(&gProcessListLock, flags);
 
     *out = proc;
@@ -681,7 +683,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
 
         if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
             KCPU* cpu = &gCpus[t->RunningCpu];
-            if (cpu->Present && cpu->Started && cpu->Online) {
+            if (cpu->State == KeCpuOnline) {
                 _InterlockedExchange(&cpu->ReschedulePending, 1);
                 HalApicSendRescheduleIpi(cpu->LapicId);
             }
@@ -718,7 +720,10 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri
 
     for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
         if (!process->Handles[i].Object) {
-            ObReferenceObject(object);
+            if (!ObTryReferenceObject(object)) {
+                KeReleaseSpinlock(&process->HandleLock, flags);
+                return NTLSTATUS_OBJECT_DEAD;
+            }
             process->Handles[i].Object = object;
             process->Handles[i].Rights = rights;
             process->Handles[i].Flags = 0;
@@ -767,7 +772,8 @@ OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t req
     OBJECT* obj = entry->Object;
     if (obj && requiredRights && ((entry->Rights & requiredRights) != requiredRights))
         obj = (OBJECT*)0;
-    if (obj) ObReferenceObject(obj);
+    if (obj && !ObTryReferenceObject(obj))
+        obj = (OBJECT*)0;
     KeReleaseSpinlock(&process->HandleLock, flags);
     return obj;
 }
@@ -819,7 +825,10 @@ KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle
 
     obj = entry->Object;
     sourceRights = entry->Rights;
-    ObReferenceObject(obj);
+    if (!ObTryReferenceObject(obj)) {
+        KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
+        return NTLSTATUS_OBJECT_DEAD;
+    }
     KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
 
     uint32_t targetRights = sourceRights;
@@ -844,8 +853,7 @@ PROCESS* KeGetProcessById(uint32_t pid) {
     LIST_ENTRY* node = head->Fwd;
     while (node != head) {
         PROCESS* p = CONTAINING_RECORD(node, PROCESS, GlobalLink);
-        if (p->ProcessId == pid) {
-            ObReferenceObject(&p->Header);
+        if (p->ProcessId == pid && ObTryReferenceObject(&p->Header)) {
             KeReleaseSpinlock(&gProcessListLock, flags);
             return p;
         }

code/kernel/ke_sched.c

@@ -13,6 +13,7 @@
 #include <kernel/ke_debug.h>
 #include <kernel/ke_input.h>
 #include <kernel/ke_timer.h>
+#include <kernel/ke_display.h>
 #include <kernel/ke_list.h>
 #include <kernel/mm_virt.h>
 #include <kernel/hal/hal_apic.h>
@@ -48,7 +49,7 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
 
 // Implements the cpu Is Idle Locked operation.
 static int KiCpuIsIdleLocked(const KCPU* cpu) {
-    if (!cpu || !cpu->Online)
+    if (!cpu || cpu->State != KeCpuOnline)
         return 0;
     THREAD* cur = cpu->CurrentThread;
     return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
@@ -61,7 +62,7 @@ static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
 
     for (uint32_t i = 0; i < gCpuCount; i++) {
         KCPU* cpu = &gCpus[i];
-        if (!cpu->Present || !cpu->Started || !cpu->Online)
+        if (cpu->State != KeCpuOnline)
             continue;
         if (!cpu->IsPrimaryThread)
             continue;
@@ -89,7 +90,7 @@ static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
 
 // Implements the request Reschedule Cpu Locked operation.
 static void KiRequestRescheduleCpuLocked(KCPU* target) {
-    if (!target || !target->Online)
+    if (!target || target->State != KeCpuOnline)
         return;
 
     _InterlockedExchange(&target->ReschedulePending, 1);
@@ -110,7 +111,7 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
 
     for (uint32_t i = 0; i < gCpuCount; i++) {
         KCPU* cpu = &gCpus[i];
-        if (!cpu->Present || !cpu->Started || !cpu->Online)
+        if (cpu->State != KeCpuOnline)
             continue;
         if (!KiThreadCanRunOnCpuLocked(thread, cpu))
             continue;
@@ -135,7 +136,7 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
     if (!requestedIdle && haveIdlePrimary == 0) {
         for (uint32_t i = 0; i < gCpuCount; i++) {
             KCPU* cpu = &gCpus[i];
-            if (!cpu->Present || !cpu->Started || !cpu->Online)
+            if (cpu->State != KeCpuOnline)
                 continue;
             if (!KiThreadCanRunOnCpuLocked(thread, cpu))
                 continue;
@@ -265,7 +266,7 @@ static void KiInsertReadyLocked(THREAD* thread) {
 
     thread->State = ThreadReady;
     thread->Queued = 1;
-    ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
+    KeListInsertTailChecked(&gRunQueue[p], &thread->SchedulerLink);
 }
 
 static int KiAnyRunnableForCpuLocked(KCPU* cpu);
@@ -367,13 +368,24 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
         return NTLSTATUS_NO_MEMORY;
     }
     idle->KernelStackBase = kstack;
-    idle->KernelStackTop  = (uintptr_t)kstack + KSTACK_SIZE;
+    idle->KernelStackTop  = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
     idle->TssRsp0         = idle->KernelStackTop;
 
-    uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop;
+    uintptr_t* ksp = (uintptr_t*)idle->KernelStackTop;
+
+    // Kernel stacks may come from allocators with weaker alignment than the
+    // MS x64 call ABI requires. Bias the top down before building synthetic
+    // entry frames so optimized C code can use aligned stack spills safely.
+    // KiIdleLoop is a C function reached by a synthetic RET, not by CALL.
+    // Leave the fake return slot and the MS x64 32-byte home area below the
+    // top of stack so optimized prologues may spill above RSP without writing
+    // into the pool canary or footer.
+    ksp -= 5;
+    __stosq((unsigned __int64*)ksp, 0, 5);
+
     ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
-    KCONTEXT* ctx = (KCONTEXT *)ksp;
-    __stosq((unsigned __int64 *)ctx, 0, sizeof(KCONTEXT) / 8);
+    KCONTEXT* ctx = (KCONTEXT*)ksp;
+    __stosq((unsigned __int64*)ctx, 0, sizeof(KCONTEXT) / 8);
     ctx->Rip = (uint64_t)KiIdleLoop;
     idle->SavedContext = ctx;
     KeInitThreadFpuContext(idle);
@@ -383,7 +395,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
     // as the real idle process while kernel worker threads live under PID 1.
     ObReferenceObject(&gIdleProcess->Header);
     uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock);
-    ListInsertTail(&gIdleProcess->ThreadList, &idle->ProcessLink);
+    KeListInsertTailChecked(&gIdleProcess->ThreadList, &idle->ProcessLink);
     gIdleProcess->ThreadCount++;
     KeReleaseSpinlock(&gIdleProcess->HandleLock, idleFlags);
 
@@ -403,6 +415,8 @@ void KeInitializeScheduler(void) {
     if (!KSUCCESS(s))
         KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create CPU%u idle thread, status=%u\n",
                 (unsigned)cpu->CpuIndex, s);
+
+    _InterlockedExchange(&cpu->State, KeCpuOnline);
 }
 
 // Safely release the lock and clean up any dead threads natively post-switch.
@@ -456,7 +470,7 @@ void KeUnreadyThread(THREAD* thread) {
     if (!thread) return;
     uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
     if (thread->Queued) {
-        ListRemove(&thread->SchedulerLink);
+        KeListRemoveChecked(&thread->SchedulerLink);
         thread->Queued = 0;
     }
     KeReleaseSpinlock(&gSchedLock, flags);
@@ -495,7 +509,7 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
                 continue;
             }
 
-            ListRemove(node);
+            KeListRemoveChecked(node);
             thread->Queued = 0;
             KiAssertPickedThreadLocked("KiPickNextForCpu", thread);
             return thread;
@@ -666,7 +680,7 @@ void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
     if (!thread) return;
     uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
     if (thread->State == ThreadWaiting && thread->WaitListHead) {
-        ListRemove(&thread->SchedulerLink);
+        KeListRemoveChecked(&thread->SchedulerLink);
         thread->WaitListHead = (LIST_ENTRY*)0;
         thread->SleepUntilTick = 0;
         thread->TimeoutStatus = NTLSTATUS_SUCCESS;
@@ -692,7 +706,7 @@ static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
         if (thread->State == ThreadWaiting &&
             thread->WaitListHead == &gSleepList &&
             thread->SleepUntilTick <= nowTick) {
-            ListRemove(node);
+            KeListRemoveChecked(node);
             thread->WaitListHead = (LIST_ENTRY*)0;
             KiWakeThreadLocked(thread, thread->TimeoutStatus);
         }
@@ -701,20 +715,49 @@ static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
     }
 }
 
-// Implements the scheduler Tick operation.
-void KeSchedulerTick(void) {
-    KCPU* cpu = KeGetCurrentCpu();
-    THREAD* cur = cpu->CurrentThread;
-    THREAD* idle = cpu->IdleThread;
-    if (!cur) return;
+// Checks whether this CPU owns global scheduler tick duties.
+static int KiCpuOwnsGlobalSchedulerTick(const KCPU* cpu) {
+    // LAPIC timer interrupts are per-CPU, but sleep deadlines, waitable timers,
+    // input repeat, and priority aging are global scheduler services. Keep the
+    // global clock single-sourced on the BSP so enabling AP timers does not
+    // make those services run once per online CPU.
+    return cpu && cpu->IsBsp;
+}
 
-    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
-    KiArmPostSwitchUnlock(cur, flags);
+// Ages ready threads after enough global scheduler ticks have elapsed.
+static void KiAgeReadyThreadsLocked(void) {
+    // Periodic aging prevents indefinite starvation of PriorityLow and
+    // PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
+    // burst history; they earned the boost by waiting, not by burning CPU.
+    gAgingCounter++;
+    if (gAgingCounter < SCHED_AGING_INTERVAL_TICKS)
+        return;
 
+    gAgingCounter = 0;
+    for (int p = 0; p < PriorityHigh; p++) {
+        while (!LIST_IS_EMPTY(&gRunQueue[p])) {
+            LIST_ENTRY* node = KeListPopHeadChecked(&gRunQueue[p]);
+            THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+            t->Priority = PriorityHigh;
+            t->CpuBurstTicks = 0;
+            t->QuantaConsumed = 0;
+            t->QuantumRemaining = 0;
+            gSchedStats.AgingPromotions++;
+            KeListInsertTailChecked(&gRunQueue[PriorityHigh], &t->SchedulerLink);
+        }
+    }
+}
+
+// Runs scheduler work that must be single-sourced for the whole system.
+static void KiRunGlobalSchedulerTickLocked(void) {
     gSchedStats.Ticks++;
     KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
     KeInputTimerTick();
+    KiAgeReadyThreadsLocked();
+}
 
+// Services the current CPU's local scheduler timer interrupt.
+static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) {
     int requestReschedule = 0;
 
     if (cur == idle) {
@@ -756,29 +799,32 @@ void KeSchedulerTick(void) {
         }
     }
 
-    // Periodic aging pass to prevent indefinite starvation of PriorityLow and
-    // PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
-    // burst history; they earned the boost by waiting, not by burning CPU.
-    gAgingCounter++;
-    if (gAgingCounter >= SCHED_AGING_INTERVAL_TICKS) {
-        gAgingCounter = 0;
-        for (int p = 0; p < PriorityHigh; p++) {
-            while (!LIST_IS_EMPTY(&gRunQueue[p])) {
-                LIST_ENTRY* node = ListPopHead(&gRunQueue[p]);
-                THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
-                t->Priority = PriorityHigh;
-                t->CpuBurstTicks = 0;
-                t->QuantaConsumed = 0;
-                t->QuantumRemaining = 0;
-                gSchedStats.AgingPromotions++;
-                ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink);
-            }
-        }
-    }
+    return requestReschedule;
+}
+
+// Implements the scheduler Tick operation.
+void KeSchedulerTick(void) {
+    KCPU* cpu = KeGetCurrentCpu();
+    THREAD* cur = cpu->CurrentThread;
+    THREAD* idle = cpu->IdleThread;
+    if (!cur) return;
+
+    int ownsGlobalTick = KiCpuOwnsGlobalSchedulerTick(cpu);
+
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    KiArmPostSwitchUnlock(cur, flags);
+
+    if (ownsGlobalTick)
+        KiRunGlobalSchedulerTickLocked();
+
+    int requestReschedule = KiRunLocalSchedulerTickLocked(cpu, cur, idle);
 
     KeReleaseSpinlock(&gSchedLock, flags);
 
-    KeTimerTick();
+    if (ownsGlobalTick) {
+        KeTimerTick();
+        KeDisplayTick();
+    }
 
     if (requestReschedule)
         KiRequestReschedule();
@@ -816,7 +862,7 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus)
     self->WaitListHead = &gSleepList;
     self->SleepUntilTick = wakeTick;
     self->TimeoutStatus = timeoutStatus;
-    ListInsertTail(&gSleepList, &self->SchedulerLink);
+    KeListInsertTailChecked(&gSleepList, &self->SchedulerLink);
     KeReleaseSpinlock(&gSchedLock, flags);
     return NTLSTATUS_SUCCESS;
 }
@@ -861,7 +907,7 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
     self->AlertableWait = 0;
     self->SleepUntilTick = wakeTick;
     self->TimeoutStatus = NTLSTATUS_SUCCESS;
-    ListInsertTail(&gSleepList, &self->SchedulerLink);
+    KeListInsertTailChecked(&gSleepList, &self->SchedulerLink);
 
     KeReleaseSpinlock(&gSchedLock, flags);
     return KeCommitBlockThread(self, 0);

code/kernel/ke_selftest.c

@@ -1189,11 +1189,9 @@ void KiRunCoreHardeningSelfTests(void) {
                 (uint64_t)cpu->Self, (uint64_t)cpu);
     if (KeGetCurrentCpuIndex() != 0)
         KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n");
-    if (!cpu->Present || !cpu->Started || !cpu->Online || !cpu->IsBsp)
-        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle flags invalid P/S/O/B=%u/%u/%u/%u\n",
-                cpu->Present, cpu->Started, cpu->Online, cpu->IsBsp);
-    if (cpu->Parked)
-        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP is marked parked\n");
+    if (cpu->State != KeCpuOnline || !cpu->IsBsp)
+        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle invalid state=%ld bsp=%u\n",
+                cpu->State, cpu->IsBsp);
     if (KeGetCurrentIrql() != PASSIVE_LEVEL)
         KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql());
     if (KeGetCurrentTss() != &cpu->Tss)
@@ -1207,16 +1205,12 @@ void KiRunCoreHardeningSelfTests(void) {
 
     for (uint32_t i = 1; i < gCpuCount; i++) {
         KCPU* ap = &gCpus[i];
-        if (!ap->Present)
+        if (ap->State == KeCpuAbsent)
             continue;
-        if (!ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
+        if (ap->State != KeCpuOnline || ap->IsBsp)
             KePanic(KE_PANIC_CODE_SELFTEST,
-                    "CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n",
-                    i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp);
-        if (!ap->SchedulerCommand || !ap->SchedulerActive)
-            KePanic(KE_PANIC_CODE_SELFTEST,
-                    "CORE selftest: AP%u scheduler admission invalid command=%ld active=%ld\n",
-                    i, ap->SchedulerCommand, ap->SchedulerActive);
+                    "CORE selftest: AP%u lifecycle invalid state=%ld bsp=%u\n",
+                    i, ap->State, ap->IsBsp);
         if (!ap->KernelStack || !ap->KernelStackTop)
             KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
         if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))

code/kernel/ke_semaphore.c

@@ -1,6 +1,7 @@
 // Kernel semaphore primitive. This file owns counting wakeups and bounded resource acquisition.
 
 #include <stdbool.h>
+#include <kernel/ke_list.h>
 #include <kernel/ke_semaphore.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
@@ -39,7 +40,7 @@ __declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
     }
 
     KePrepareBlockThread(self, &sem->Header, 0);
-    ListInsertTail(&sem->WaitList, &self->SchedulerLink);
+    KeListInsertTailChecked(&sem->WaitList, &self->SchedulerLink);
     KeSetThreadWaitList(self, &sem->WaitList);
     KeReleaseSpinlock(&sem->Lock, flags);
     return KeCommitBlockThread(self, 0);
@@ -66,7 +67,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
         // Transfer this semaphore release directly to the selected waiter. The
         // count stays unchanged; the waiter returns success without looping and
         // consuming a count later.
-        LIST_ENTRY* node = ListPopHead(&sem->WaitList);
+        LIST_ENTRY* node = KeListPopHeadChecked(&sem->WaitList);
         THREAD* waiter   = CONTAINING_RECORD(node, THREAD, SchedulerLink);
         KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
         KeReleaseSpinlock(&sem->Lock, flags);

code/kernel/ke_smp.c

@@ -6,6 +6,7 @@
 // to enter KiApKernelEntry() and park in kernel-owned state.
 #include <stdint.h>
 #include <stddef.h>
+#include <stdbool.h>
 #include <string.h>
 #include <intrin.h>
 #include <ntlite/status.h>
@@ -80,6 +81,20 @@ static PHYSICAL_ADDRESS gApTrampolinePhys;
 static PHYSICAL_ADDRESS gReservedApTrampolinePhys;
 static AP_BOOT_BLOCK* gApBootBlock;
 
+// Returns whether investigation-only SMP diagnostics are enabled.
+static bool KiSmpDiagnosticsEnabled(void) {
+    return (KeGetDebugFlags() & KE_DEBUG_SELFTEST) != 0;
+}
+
+// Validates pool state around AP admission so SMP races fail near their trigger.
+static void KiSmpPoolCheckpoint(const char* phase, uint32_t cpuIndex) {
+    if (!KiSmpDiagnosticsEnabled())
+        return;
+
+    DbgPrint("SMP: pool checkpoint %s AP%u\n", phase ? phase : "unknown", cpuIndex);
+    (void)MmValidateKernelPool(phase ? phase : "smp");
+}
+
 // Implements the emit8 operation.
 static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
 // Implements the emit16 operation.
@@ -127,27 +142,35 @@ static KSTATUS KiBuildApTrampoline(void) {
         if (frame < AP_TRAMPOLINE_FALLBACK_MIN ||
             frame >= AP_TRAMPOLINE_MAX_PHYS ||
             (frame & (PAGE_SIZE - 1))) {
-            DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
-                     (uint64_t)frame);
+            if (KiSmpDiagnosticsEnabled()) {
+                DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
+                         (uint64_t)frame);
+            }
             frame = 0;
         } else {
-            DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
-                     (uint64_t)frame);
+            if (KiSmpDiagnosticsEnabled()) {
+                DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
+                         (uint64_t)frame);
+            }
         }
     }
 
     if (!frame) {
-        DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
-                 (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
-                 (uint64_t)AP_TRAMPOLINE_MAX_PHYS);
+        if (KiSmpDiagnosticsEnabled()) {
+            DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
+                     (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
+                     (uint64_t)AP_TRAMPOLINE_MAX_PHYS);
+        }
         frame = MmPfnAllocateInRange(AP_TRAMPOLINE_PREFERRED_MIN,
                                        AP_TRAMPOLINE_MAX_PHYS,
                                        MmPfnUseKernel);
     }
     if (!frame) {
-        DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
-                 (uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
-                 (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
+        if (KiSmpDiagnosticsEnabled()) {
+            DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
+                     (uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
+                     (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
+        }
         frame = MmPfnAllocateInRange(AP_TRAMPOLINE_FALLBACK_MIN,
                                        AP_TRAMPOLINE_PREFERRED_MIN,
                                        MmPfnUseKernel);
@@ -158,7 +181,8 @@ static KSTATUS KiBuildApTrampoline(void) {
         return NTLSTATUS_NO_MEMORY;
     }
 
-    DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
+    if (KiSmpDiagnosticsEnabled())
+        DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
 
     uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame);
     memset(page, 0, PAGE_SIZE);
@@ -285,13 +309,12 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
     KeLoadIdtForCurrentCpu();
     if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_IDT;
 
-    cpu->Started = 1;
-    cpu->Parked = 1;
-    cpu->Online = 0;
+    _InterlockedExchange(&cpu->State, KeCpuStarted);
     cpu->CurrentIrql = PASSIVE_LEVEL;
     cpu->InterruptDepth = 0;
     cpu->ReschedulePending = 0;
 
+    _InterlockedExchange(&cpu->State, KeCpuParked);
     if (gApBootBlock) {
         gApBootBlock->Stage = AP_STAGE_PARK;
         gApBootBlock->State = AP_STATE_KERNEL_PARKED;
@@ -300,32 +323,30 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
     for (;;) {
         cpu->SchedulerTicks++;
 
-        if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
+        if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
             THREAD* idle = cpu->IdleThread;
             if (idle) {
                 cpu->CurrentThread = idle;
                 idle->State = ThreadRunning;
                 idle->RunningCpu = cpu->CpuIndex;
                 idle->Queued = 0;
-                cpu->Parked = 0;
-
                 // Do not arm the AP's periodic LAPIC timer while it is still in
                 // the raw park loop. Timer IRQs are scheduler-owned events;
                 // the AP may take them only after it has a current idle thread.
                 HalTimerConfigureCurrentCpu();
 
-                _InterlockedExchange(&cpu->SchedulerActive, 1);
+                _InterlockedExchange(&cpu->State, KeCpuSchedulerActive);
                 HalEnableInterrupts();
                 KeStartInitialThread(idle);
             }
         }
 
-        // The BSP admits parked APs by storing SchedulerCommand and sending a
-        // wake IPI. Avoid the classic lost wakeup where the command appears
-        // between the poll and HLT: check once with interrupts disabled, then
-        // STI;HLT as a single sleep transition.
+        // The BSP admits parked APs by publishing AdmissionRequested and
+        // sending a wake IPI. Avoid the classic lost wakeup where the state
+        // changes between the poll and HLT: check once with interrupts
+        // disabled, then use STI;HLT as a single sleep transition.
         uintptr_t interruptFlags = HalDisableInterrupts();
-        if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
+        if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
             HalRestoreInterrupts(interruptFlags);
             continue;
         }
@@ -342,10 +363,7 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
     cpu->LapicId = info->LapicId;
     cpu->AcpiProcessorId = info->ProcessorUid;
     cpu->BootProcessorIndex = cpuIndex;
-    cpu->Present = info->Enabled ? 1 : 0;
-    cpu->Started = 0;
-    cpu->Parked = 0;
-    cpu->Online = 0;
+    cpu->State = info->Enabled ? KeCpuPresent : KeCpuAbsent;
     cpu->IsBsp = info->IsBsp ? 1 : 0;
     cpu->PackageId = 0;
     cpu->CoreId = cpuIndex;
@@ -359,8 +377,6 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
     cpu->CurrentIrql = PASSIVE_LEVEL;
     cpu->InterruptDepth = 0;
     cpu->ReschedulePending = 0;
-    cpu->SchedulerCommand = 0;
-    cpu->SchedulerActive = 0;
 }
 
 // Implements the ceil Log2 U32 operation.
@@ -496,7 +512,7 @@ void KeBuildProcessorTopology(void) {
 
     for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
         KCPU* cpu = &gCpus[i];
-        if (!cpu->Present)
+        if (cpu->State == KeCpuAbsent)
             continue;
 
         uint32_t apic = cpu->LapicId;
@@ -515,7 +531,7 @@ void KeBuildProcessorTopology(void) {
     uint32_t nextCoreIndex = 0;
     for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
         KCPU* cpu = &gCpus[i];
-        if (!cpu->Present)
+        if (cpu->State == KeCpuAbsent)
             continue;
         if (cpu->PhysicalCoreIndex != KE_INVALID_CPU)
             continue;
@@ -528,7 +544,7 @@ void KeBuildProcessorTopology(void) {
 
         for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
             KCPU* peer = &gCpus[j];
-            if (!peer->Present)
+            if (peer->State == KeCpuAbsent)
                 continue;
             if (peer->PackageId == cpu->PackageId) {
                 if (j < 64)
@@ -549,7 +565,7 @@ void KeBuildProcessorTopology(void) {
 
         for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
             KCPU* peer = &gCpus[j];
-            if (!peer->Present)
+            if (peer->State == KeCpuAbsent)
                 continue;
             if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
                 peer->CoreSiblingMask = coreMaskBits;
@@ -568,19 +584,21 @@ void KeBuildProcessorTopology(void) {
              topo.LogicalPerPackage,
              nextCoreIndex);
 
-    for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
-        KCPU* cpu = &gCpus[i];
-        if (!cpu->Present)
-            continue;
-        DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
-                 i,
-                 cpu->LapicId,
-                 cpu->PackageId,
-                 cpu->CoreId,
-                 cpu->ThreadId,
-                 cpu->IsPrimaryThread,
-                 (unsigned long long)cpu->CoreSiblingMask,
-                 (unsigned long long)cpu->PackageMask);
+    if (KiSmpDiagnosticsEnabled()) {
+        for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
+            KCPU* cpu = &gCpus[i];
+            if (cpu->State == KeCpuAbsent)
+                continue;
+            DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
+                     i,
+                     cpu->LapicId,
+                     cpu->PackageId,
+                     cpu->CoreId,
+                     cpu->ThreadId,
+                     cpu->IsPrimaryThread,
+                     (unsigned long long)cpu->CoreSiblingMask,
+                     (unsigned long long)cpu->PackageMask);
+        }
     }
 }
 
@@ -598,7 +616,7 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
     }
 
     cpu->KernelStack = stack;
-    cpu->KernelStackTop = (uintptr_t)stack + stackBytes;
+    cpu->KernelStackTop = ((uintptr_t)stack + stackBytes) & ~(uintptr_t)15;
     memset(cpu->KernelStack, 0, stackBytes);
     memset(&cpu->Tss, 0, sizeof(cpu->Tss));
     cpu->Tss.IoMapBase = sizeof(TSS64);
@@ -647,12 +665,7 @@ KSTATUS KeStartApplicationProcessors(void) {
     KCPU* bsp = KeGetCurrentCpu();
     bsp->LapicId = bspLapic;
     bsp->IsBsp = 1;
-    bsp->Present = 1;
-    bsp->Started = 1;
-    bsp->Online = 1;
-    bsp->Parked = 0;
-    bsp->SchedulerCommand = 0;
-    bsp->SchedulerActive = 1;
+    bsp->State = KeCpuStarted;
 
     DbgPrint("SMP: APIC mode=%s BSP LAPIC=%u MADT CPUs=%u\n",
              HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
@@ -661,9 +674,11 @@ KSTATUS KeStartApplicationProcessors(void) {
     uint32_t count = 1;
     for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) {
         HAL_CPU_INFO* info = &topo.Cpus[i];
-        DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
-                 i, info->ProcessorUid, info->LapicId, info->Flags,
-                 info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
+        if (KiSmpDiagnosticsEnabled()) {
+            DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
+                     i, info->ProcessorUid, info->LapicId, info->Flags,
+                     info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
+        }
 
         if (!info->Enabled) continue;
         if (info->LapicId == bspLapic) {
@@ -693,8 +708,10 @@ KSTATUS KeStartApplicationProcessors(void) {
         return NTLSTATUS_SUCCESS;
     }
 
-    DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
-             (uint64_t)gReservedApTrampolinePhys);
+    if (KiSmpDiagnosticsEnabled()) {
+        DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
+                 (uint64_t)gReservedApTrampolinePhys);
+    }
     st = KiBuildApTrampoline();
     if (!KSUCCESS(st)) {
         DbgPrint("SMP: disabled; AP trampoline allocation/build failed status=0x%x\n", st);
@@ -707,30 +724,25 @@ KSTATUS KeStartApplicationProcessors(void) {
 
     for (uint32_t i = 1; i < gCpuCount; i++) {
         KCPU* cpu = &gCpus[i];
-        if (!cpu->Present) continue;
+        if (cpu->State == KeCpuAbsent) continue;
 
         st = KiAllocateApStack(cpu);
         if (!KSUCCESS(st)) {
             DbgPrint("SMP: AP%u LAPIC=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
                      i, cpu->LapicId, st);
-            cpu->Present = 0;
-            cpu->Started = 0;
-            cpu->Parked = 0;
-            cpu->Online = 0;
+            _InterlockedExchange(&cpu->State, KeCpuAbsent);
             continue;
         }
 
         KiFillBootBlockForCpu(cpu, i);
-        DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
+        if (KiSmpDiagnosticsEnabled())
+            DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
 
         st = HalStartApplicationProcessor(cpu->LapicId, sipiVector);
         if (!KSUCCESS(st)) {
             DbgPrint("SMP: AP%u LAPIC=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
                      i, cpu->LapicId, st);
-            cpu->Present = 0;
-            cpu->Started = 0;
-            cpu->Parked = 0;
-            cpu->Online = 0;
+            _InterlockedExchange(&cpu->State, KeCpuAbsent);
             continue;
         }
 
@@ -752,18 +764,18 @@ KSTATUS KeStartApplicationProcessors(void) {
             DbgPrint("SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
                      i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
                      KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
-            cpu->Present = 0;
-            cpu->Started = 0;
-            cpu->Parked = 0;
-            cpu->Online = 0;
+            _InterlockedExchange(&cpu->State, KeCpuAbsent);
             continue;
         }
 
-        cpu->Started = 1;
-        cpu->Parked = 1;
-        cpu->Online = 0;
+        if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
+            KePanic(KE_PANIC_CODE_INIT,
+                    "SMP: AP%u boot block parked without CPU state transition state=%ld\n",
+                    i, cpu->State);
+        }
         claimed++;
-        DbgPrint("SMP: AP%u reached kernel park\n", i);
+        if (KiSmpDiagnosticsEnabled())
+            DbgPrint("SMP: AP%u reached kernel park\n", i);
     }
 
     if (claimed == 0 && gCpuCount > 1)
@@ -771,6 +783,7 @@ KSTATUS KeStartApplicationProcessors(void) {
 
     KeBuildProcessorTopology();
     DbgPrint("SMP: kernel claimed %u APs\n", claimed);
+    KiSmpPoolCheckpoint("smp after AP park", 0);
     return NTLSTATUS_SUCCESS;
 }
 
@@ -784,32 +797,33 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
     uint32_t admitted = 0;
     for (uint32_t i = 1; i < gCpuCount; i++) {
         KCPU* cpu = &gCpus[i];
-        if (!cpu->Present) continue;
+        if (cpu->State == KeCpuAbsent) continue;
 
-        if (!cpu->Started || !cpu->Parked || cpu->Online) {
+        if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
             KePanic(KE_PANIC_CODE_INIT,
-                    "SMP: AP%u cannot enter scheduler P/S/Park/O=%u/%u/%u/%u\n",
-                    i, cpu->Present, cpu->Started, cpu->Parked, cpu->Online);
+                    "SMP: AP%u cannot enter scheduler from state=%ld\n",
+                    i, cpu->State);
         }
 
+        KiSmpPoolCheckpoint("smp before AP idle create", i);
         KSTATUS st = KiCreateIdleThreadForCpu(cpu);
         if (!KSUCCESS(st))
             return st;
+        KiSmpPoolCheckpoint("smp after AP idle create", i);
 
-        cpu->Online = 1;
-        _ReadWriteBarrier();
-        _InterlockedExchange(&cpu->SchedulerCommand, 1);
+        _InterlockedExchange(&cpu->State, KeCpuAdmissionRequested);
+        KiSmpPoolCheckpoint("smp before AP wake", i);
 
         // Parked APs sleep with HLT and have no LAPIC timer armed yet. The
-        // command store alone is not sufficient; send a fixed-vector IPI only
-        // to wake the parked loop. ReschedulePending is intentionally left
+        // state transition alone is not sufficient; send a fixed-vector IPI
+        // only to wake the parked loop. ReschedulePending is intentionally left
         // clear, so the interrupt exit path will not try to schedule before
         // the AP has installed its idle thread.
         HalApicSendRescheduleIpi(cpu->LapicId);
 
         int active = 0;
         for (uint32_t spin = 0; spin < AP_SCHED_TIMEOUT_SPINS; spin++) {
-            if (_InterlockedCompareExchange(&cpu->SchedulerActive, 0, 0) != 0) {
+            if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuSchedulerActive) {
                 active = 1;
                 break;
             }
@@ -818,13 +832,15 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
 
         if (!active) {
             KePanic(KE_PANIC_CODE_INIT,
-                    "SMP: AP%u failed scheduler admission lapic=%u parked=%u online=%u command=%ld active=%ld ticks=%llu\n",
-                    i, cpu->LapicId, cpu->Parked, cpu->Online,
-                    cpu->SchedulerCommand, cpu->SchedulerActive, cpu->SchedulerTicks);
+                    "SMP: AP%u failed scheduler admission lapic=%u state=%ld ticks=%llu\n",
+                    i, cpu->LapicId, cpu->State, cpu->SchedulerTicks);
         }
 
+        _InterlockedExchange(&cpu->State, KeCpuOnline);
+        KiSmpPoolCheckpoint("smp after AP scheduler active", i);
         admitted++;
-        DbgPrint("SMP: AP%u entered scheduler idle thread\n", i);
+        if (KiSmpDiagnosticsEnabled())
+            DbgPrint("SMP: AP%u entered scheduler idle thread\n", i);
     }
 
     DbgPrint("SMP: scheduler admitted %u APs\n", admitted);
@@ -946,7 +962,7 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
     uint32_t workerCount = 0;
     for (uint32_t cpuIndex = 1; cpuIndex < gCpuCount; cpuIndex++) {
         KCPU* cpu = &gCpus[cpuIndex];
-        if (!cpu->Present || !cpu->Started || !cpu->Online || cpu->Parked)
+        if (cpu->State != KeCpuOnline)
             continue;
         if (cpuIndex >= 64)
             continue;

code/kernel/ke_switch.asm

@@ -1,17 +1,17 @@
 ; ke_switch.asm - context switch and thread entry stubs (MASM64 / ML64)
 ;
 ; Struct offsets:
-;   sizeof(OBJECT)  = 16  (type:4, refcount:4, destroy:8)
-;   THREAD.SavedContext offset = 88
-;   THREAD.TssRsp0  offset     = 104
-;   THREAD.Teb      offset     = 112
-;   THREAD.FpuContext offset   = 168
+;   sizeof(OBJECT)  = 24  (type:4, refcount:4, state:4, flags:4, destroy:8)
+;   THREAD.SavedContext offset = 96
+;   THREAD.TssRsp0  offset     = 112
+;   THREAD.Teb      offset     = 120
+;   THREAD.FpuContext offset   = 176
 ;   TSS64.RSP0      offset     = 4
 ;
-THREAD_SavedContext EQU 88
-THREAD_TssRsp0      EQU 104
-THREAD_Teb          EQU 112
-THREAD_FpuContext   EQU 168
+THREAD_SavedContext EQU 96
+THREAD_TssRsp0      EQU 112
+THREAD_Teb          EQU 120
+THREAD_FpuContext   EQU 176
 TSS_RSP0_OFFSET     EQU 4
 
 EXTERN KeSetCurrentTssRsp0 : PROC
@@ -135,9 +135,9 @@ EXTERN KeGetCurrentThread        : PROC
 EXTERN KeExitThread           : PROC
 
 KeThreadEntry PROC
-    sub     rsp, 32    ; Reserve shadow space
+    sub     rsp, 40    ; Reserve shadow space and restore MS x64 call alignment.
     call    KeSchedulerPostSwitch
-    add     rsp, 32
+    add     rsp, 40
     pop     rcx        ; load user arg
     swapgs             ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU
     iretq              ; -> ring 3
@@ -160,6 +160,10 @@ KeKernelThreadEntry PROC
     pop     rbx            ; rbx = fn ptr  (non-volatile, survives the call below)
     pop     r12            ; r12 = arg     (non-volatile)
 
+    ; The synthetic entry frame is not reached by CALL. Drop any residual
+    ; allocator-induced stack skew before making normal MS x64 calls.
+    and     rsp, -16
+
     sub     rsp, 32
     call    KeSchedulerPostSwitch   ; release scheduler spinlock, reap dead threads
     add     rsp, 32

code/kernel/ke_syscall.c

@@ -616,6 +616,9 @@ static KSTATUS KiHandleToWaitEvent(PROCESS* proc, uint32_t handle, EVENT** event
     case ObjectTypeTimer:
         event = KeTimerWaitEvent(CONTAINING_RECORD(obj, KTIMER, Header));
         break;
+    case ObjectTypeDisplayClock:
+        event = KeDisplayClockWaitEvent((DISPLAY_CLOCK_OBJECT*)obj);
+        break;
     default:
         ObDereferenceObject(obj);
         return NTLSTATUS_NOT_SUPPORTED;
@@ -2189,6 +2192,35 @@ static uint64_t SyscallStatFile(uint64_t a0, uint64_t a1, uint64_t a2,
     return CopyToUser(req.StatOut, &stat, sizeof(stat));
 }
 
+// SYS_OPEN_DISPLAY_CLOCK a0=*SYS_OPEN_DISPLAY_CLOCK_REQUEST.
+static uint64_t SyscallOpenDisplayClock(uint64_t a0, uint64_t a1, uint64_t a2,
+                                        uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_OPEN_DISPLAY_CLOCK_REQUEST req;
+    KSTATUS s = KiCopyUserRequest(a0, &req, sizeof(req));
+    if (!KSUCCESS(s)) return s;
+    if (req.Size != sizeof(req) || req.Flags != SYS_DISPLAY_CLOCK_FLAG_NONE || !req.HandleOut)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)0;
+    s = KeDisplayOpenClock(req.PeriodMilliseconds, &clock);
+    if (!KSUCCESS(s))
+        return s;
+
+    uint32_t handle = 0;
+    s = KiAllocHandleForCurrentWithRights(&clock->Header, HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY, &handle);
+    if (!KSUCCESS(s))
+        return s;
+
+    s = KiCopyHandleToUser((uint64_t)(uintptr_t)req.HandleOut, handle);
+    if (!KSUCCESS(s)) {
+        PROCESS* proc = KeGetCurrentProcess();
+        if (proc) KeCloseHandle(proc, handle);
+    }
+    return s;
+}
+
 // SYS_CREATE_SURFACE a0=*SYS_CREATE_SURFACE_REQUEST.
 static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2,
                                       uint64_t a3, uint64_t a4, uint64_t a5) {
@@ -2569,13 +2601,15 @@ static uint64_t SyscallFillConsoleAttribute(uint64_t a0, uint64_t a1, uint64_t a
     return s;
 }
 
-// Validates a user input queue handle. Handle zero means the default queue.
-static KSTATUS KiValidateInputQueueHandle(uint32_t handle) {
+// References a user input queue handle. Handle zero selects the default context.
+static KSTATUS KiReferenceInputQueueHandle(uint32_t handle, INPUT_QUEUE** queueOut) {
+    if (!queueOut) return NTLSTATUS_INVALID_PARAMETER;
+    *queueOut = (INPUT_QUEUE*)0;
     if (handle == 0) return NTLSTATUS_SUCCESS;
 
-    OBJECT* obj = KiHandleToObject(handle, ObjectTypeInputQueue, HANDLE_RIGHT_READ);
-    if (!obj) return NTLSTATUS_INVALID_HANDLE;
-    ObDereferenceObject(obj);
+    OBJECT* object = KiHandleToObject(handle, ObjectTypeInputQueue, HANDLE_RIGHT_READ);
+    if (!object) return NTLSTATUS_INVALID_HANDLE;
+    *queueOut = CONTAINING_RECORD(object, INPUT_QUEUE, Header);
     return NTLSTATUS_SUCCESS;
 }
 
@@ -2592,7 +2626,7 @@ static KSTATUS KiCopyInputEventRequest(uint64_t arg, SYS_INPUT_EVENT_REQUEST* re
     if (req->Size != sizeof(*req)) return NTLSTATUS_INVALID_PARAMETER;
     if (req->Flags != SYS_INPUT_EVENT_FLAG_NONE) return NTLSTATUS_INVALID_PARAMETER;
     if (!req->RecordOut) return NTLSTATUS_INVALID_PARAMETER;
-    return KiValidateInputQueueHandle(req->QueueHandle);
+    return NTLSTATUS_SUCCESS;
 }
 
 // SYS_POLL_KEYBOARD a0=*INPUT_KEY -> KSTATUS.
@@ -2658,6 +2692,22 @@ static uint64_t SyscallKbGetModifiers(uint64_t a0, uint64_t a1, uint64_t a2,
     return (uint64_t)KeInputGetModifiers();
 }
 
+// SYS_QUERY_GLOBAL_INPUT_STATE a0=*SYS_GLOBAL_INPUT_STATE -> KSTATUS.
+static uint64_t SyscallQueryGlobalInputState(uint64_t a0, uint64_t a1, uint64_t a2,
+                                            uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_GLOBAL_INPUT_STATE state;
+    KSTATUS s = KiCopyUserRequest(a0, &state, sizeof(state));
+    if (!KSUCCESS(s)) return s;
+    if (state.Size != sizeof(state)) return NTLSTATUS_INVALID_PARAMETER;
+    if (state.Flags != SYS_GLOBAL_INPUT_STATE_FLAG_NONE) return NTLSTATUS_INVALID_PARAMETER;
+
+    s = KeInputQueryGlobalState(&state);
+    if (!KSUCCESS(s)) return s;
+    return CopyToUser((void*)(uintptr_t)a0, &state, sizeof(state));
+}
+
 // SYS_READ_INPUT_EVENT a0=*SYS_INPUT_EVENT_REQUEST -> KSTATUS.
 static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
                                         uint64_t a3, uint64_t a4, uint64_t a5) {
@@ -2667,18 +2717,32 @@ static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
     KSTATUS s = KiCopyInputEventRequest(a0, &req);
     if (!KSUCCESS(s)) return s;
 
+    INPUT_QUEUE* queue = (INPUT_QUEUE*)0;
+    s = KiReferenceInputQueueHandle(req.QueueHandle, &queue);
+    if (!KSUCCESS(s)) return s;
+
     INPUT_KEY_RECORD record;
     for (;;) {
-        if (KeInputReadEvent(&record))
-            return KiCopyInputRecordToUser(req.RecordOut, &record);
+        bool available = queue ? KeInputQueueReadEvent(queue, &record)
+                               : KeInputReadEvent(&record);
+        if (available) {
+            s = KiCopyInputRecordToUser(req.RecordOut, &record);
+            break;
+        }
 
-        if (req.TimeoutMilliseconds == 0)
-            return NTLSTATUS_WOULD_BLOCK;
+        if (req.TimeoutMilliseconds == 0) {
+            s = NTLSTATUS_WOULD_BLOCK;
+            break;
+        }
 
-        s = KeInputWaitKey(0, req.TimeoutMilliseconds);
+        s = queue ? KeInputQueueWaitKey(queue, 0, req.TimeoutMilliseconds)
+                  : KeInputWaitKey(0, req.TimeoutMilliseconds);
         if (!KSUCCESS(s))
-            return s;
+            break;
     }
+
+    if (queue) ObDereferenceObject(&queue->Header);
+    return s;
 }
 
 // SYS_PEEK_INPUT_EVENT a0=*SYS_INPUT_EVENT_REQUEST -> KSTATUS.
@@ -2690,18 +2754,32 @@ static uint64_t SyscallKbPeekInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
     KSTATUS s = KiCopyInputEventRequest(a0, &req);
     if (!KSUCCESS(s)) return s;
 
+    INPUT_QUEUE* queue = (INPUT_QUEUE*)0;
+    s = KiReferenceInputQueueHandle(req.QueueHandle, &queue);
+    if (!KSUCCESS(s)) return s;
+
     INPUT_KEY_RECORD record;
     for (;;) {
-        if (KeInputPeekEvent(&record))
-            return KiCopyInputRecordToUser(req.RecordOut, &record);
+        bool available = queue ? KeInputQueuePeekEvent(queue, &record)
+                               : KeInputPeekEvent(&record);
+        if (available) {
+            s = KiCopyInputRecordToUser(req.RecordOut, &record);
+            break;
+        }
 
-        if (req.TimeoutMilliseconds == 0)
-            return NTLSTATUS_WOULD_BLOCK;
+        if (req.TimeoutMilliseconds == 0) {
+            s = NTLSTATUS_WOULD_BLOCK;
+            break;
+        }
 
-        s = KeInputWaitKey(0, req.TimeoutMilliseconds);
+        s = queue ? KeInputQueueWaitKey(queue, 0, req.TimeoutMilliseconds)
+                  : KeInputWaitKey(0, req.TimeoutMilliseconds);
         if (!KSUCCESS(s))
-            return s;
+            break;
     }
+
+    if (queue) ObDereferenceObject(&queue->Header);
+    return s;
 }
 
 // SYS_PEEK_MOUSE a0=*INPUT_MOUSE_EVENT -> KSTATUS.
@@ -3345,7 +3423,8 @@ static uint64_t SyscallOpenNamedQueue(uint64_t a0, uint64_t a1, uint64_t a2,
     for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
         if (KiNamedQueueNameEquals(&gNamedQueues[i], nameView)) {
             q = gNamedQueues[i].Queue;
-            ObReferenceObject(&q->Header);
+            if (q && !ObTryReferenceObject(&q->Header))
+                q = (QUEUE*)0;
             break;
         }
     }
@@ -5274,10 +5353,11 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
             SYS_CPU_STAT* out = &stats.Records[i];
             out->CpuIndex = cpu->CpuIndex;
             out->LapicId = cpu->LapicId;
-            if (cpu->Present) out->Flags |= SYS_CPU_FLAG_PRESENT;
-            if (cpu->Started) out->Flags |= SYS_CPU_FLAG_STARTED;
-            if (cpu->Parked) out->Flags |= SYS_CPU_FLAG_PARKED;
-            if (cpu->Online) out->Flags |= SYS_CPU_FLAG_ONLINE;
+            KE_CPU_STATE state = (KE_CPU_STATE)cpu->State;
+            if (state >= KeCpuPresent) out->Flags |= SYS_CPU_FLAG_PRESENT;
+            if (state >= KeCpuStarted) out->Flags |= SYS_CPU_FLAG_STARTED;
+            if (state == KeCpuParked) out->Flags |= SYS_CPU_FLAG_PARKED;
+            if (state == KeCpuOnline) out->Flags |= SYS_CPU_FLAG_ONLINE;
             if (cpu->IsBsp) out->Flags |= SYS_CPU_FLAG_BSP;
             if (cpu->IsPrimaryThread) out->Flags |= SYS_CPU_FLAG_PRIMARY;
             out->CurrentTid = cpu->CurrentThread ? cpu->CurrentThread->ThreadId : 0;
@@ -5394,6 +5474,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
     [SYS_CLEAR_SCREEN] = SyscallClearScreen,
     [SYS_GET_SCREEN_SIZE] = SyscallGetScreenSize,
     [SYS_QUERY_DISPLAY_INFO] = SyscallQueryDisplayInfo,
+    [SYS_OPEN_DISPLAY_CLOCK] = SyscallOpenDisplayClock,
     [SYS_CREATE_SURFACE] = SyscallCreateSurface,
     [SYS_MAP_SURFACE] = SyscallMapSurface,
     [SYS_QUERY_SURFACE] = SyscallQuerySurface,
@@ -5405,6 +5486,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
     [SYS_READ_KEYBOARD_CHAR] = SyscallKbReadChar,
     [SYS_READ_KEY] = SyscallKbReadKey,
     [SYS_GET_KEY_MODIFIERS] = SyscallKbGetModifiers,
+    [SYS_QUERY_GLOBAL_INPUT_STATE] = SyscallQueryGlobalInputState,
     [SYS_OPEN_INPUT_QUEUE] = SyscallOpenInputQueue,
     [SYS_READ_INPUT_EVENT] = SyscallKbReadInputEvent,
     [SYS_PEEK_INPUT_EVENT] = SyscallKbPeekInputEvent,

code/kernel/ke_thread.c

@@ -1,6 +1,7 @@
 // Thread manager. This file owns thread allocation, kernel/user startup contexts, and lifecycle transitions.
 
 #include <string.h>
+#include <kernel/ke_list.h>
 #include <stddef.h>
 #include <intrin.h>
 #include <kernel/ke_thread.h>
@@ -26,10 +27,10 @@
     _Static_assert(offsetof(type, field) == (expected), \
                    #type "." #field " offset must match assembly")
 
-STATIC_ASSERT_OFFSET(THREAD, SavedContext, 88);
-STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 104);
-STATIC_ASSERT_OFFSET(THREAD, Teb, 112);
-STATIC_ASSERT_OFFSET(THREAD, FpuContext, 168);
+STATIC_ASSERT_OFFSET(THREAD, SavedContext, 96);
+STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 112);
+STATIC_ASSERT_OFFSET(THREAD, Teb, 120);
+STATIC_ASSERT_OFFSET(THREAD, FpuContext, 176);
 
 static uint32_t gNextThreadId = 1;
 extern void KeThreadEntry(void);
@@ -68,7 +69,7 @@ static void KiThreadDestroy(OBJECT* obj) {
         PROCESS* owner = thread->Owner;
         uint32_t remaining = 0;
         uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
-        ListRemove(&thread->ProcessLink);
+        KeListRemoveChecked(&thread->ProcessLink);
         if (owner->ThreadCount) owner->ThreadCount--;
         remaining = owner->ThreadCount;
         KeReleaseSpinlock(&owner->HandleLock, flags);
@@ -138,7 +139,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     }
     memset(kstack, 0, KSTACK_SIZE);
     thread->KernelStackBase = kstack;
-    thread->KernelStackTop  = (uintptr_t)kstack + KSTACK_SIZE;
+    thread->KernelStackTop  = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
     thread->TssRsp0         = thread->KernelStackTop;
 
     KSTATUS s;
@@ -230,7 +231,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     thread->ExitEvent = exitEvent;
 
     uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
-    ListInsertTail(&owner->ThreadList, &thread->ProcessLink);
+    KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
     owner->ThreadCount++;
     KeReleaseSpinlock(&owner->HandleLock, flags);
 
@@ -335,7 +336,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     }
     memset(kstack, 0, KSTACK_SIZE);
     thread->KernelStackBase = kstack;
-    thread->KernelStackTop  = (uintptr_t)kstack + KSTACK_SIZE;
+    thread->KernelStackTop  = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
     thread->TssRsp0         = thread->KernelStackTop;
 
     // Build the initial stack: arg, fn, then KCONTEXT.
@@ -354,7 +355,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     thread->Owner = owner;
 
     uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
-    ListInsertTail(&owner->ThreadList, &thread->ProcessLink);
+    KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
     owner->ThreadCount++;
     KeReleaseSpinlock(&owner->HandleLock, flags);
 
@@ -471,7 +472,7 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     // consume the alertable wait immediately instead of going to sleep.
     if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
         if (thread->WaitListHead)
-            ListRemove(&thread->SchedulerLink);
+            KeListRemoveChecked(&thread->SchedulerLink);
         KiFinishWait(thread, NTLSTATUS_USER_APC);
         thread->State = ThreadRunning;
         return NTLSTATUS_USER_APC;
@@ -522,7 +523,7 @@ static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* wa
     if (!KiThreadWaitLinkIsLinked(thread))
         return 0;
 
-    ListRemove(&thread->SchedulerLink);
+    KeListRemoveChecked(&thread->SchedulerLink);
     thread->WaitListHead = (LIST_ENTRY*)0;
     thread->SleepUntilTick = 0;
     thread->TimeoutStatus = NTLSTATUS_SUCCESS;
@@ -601,7 +602,7 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
         return NTLSTATUS_PENDING;
 
     if (thread->WaitListHead)
-        ListRemove(&thread->SchedulerLink);
+        KeListRemoveChecked(&thread->SchedulerLink);
 
     KeUnblockThreadWithStatus(thread, status);
     return NTLSTATUS_SUCCESS;

code/kernel/ke_timer.c

@@ -1,6 +1,7 @@
 // Kernel waitable timer objects. This file owns timer deadlines and timer wait signaling.
 
 #include <stdint.h>
+#include <kernel/ke_list.h>
 #include <string.h>
 #include <kernel/ke_timer.h>
 #include <kernel/ke_sched.h>
@@ -52,7 +53,7 @@ static uint8_t KiTimerRemoveLocked(KTIMER* timer) {
     if (!timer || !timer->Linked)
         return 0;
 
-    ListRemove(&timer->Link);
+    KeListRemoveChecked(&timer->Link);
     timer->Linked = 0;
     timer->Active = 0;
     timer->DueTick = 0;
@@ -132,7 +133,7 @@ KSTATUS KeTimerSet(KTIMER* timer, uint64_t dueTimeMilliseconds, uint64_t periodM
         timer->PeriodTicks = periodTicks;
         timer->Active = 1;
         timer->Linked = 1;
-        ListInsertTail(&gTimerList, &timer->Link);
+        KeListInsertTailChecked(&gTimerList, &timer->Link);
         addListReference = 1;
     }
 

code/kernel/ke_work.c

@@ -2,6 +2,7 @@
 // to move ISR/DPC-originated work to a sleepable system thread context.
 
 #include <stdbool.h>
+#include <kernel/ke_list.h>
 #include <stddef.h>
 #include <intrin.h>
 
@@ -30,7 +31,7 @@ static void KiWorkQueueWorker(void* arg) {
             KWORK_ITEM* item = (KWORK_ITEM*)0;
             uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
             if (!LIST_IS_EMPTY(&gWorkQueueList)) {
-                LIST_ENTRY* node = ListPopHead(&gWorkQueueList);
+                LIST_ENTRY* node = KeListPopHeadChecked(&gWorkQueueList);
                 item = CONTAINING_RECORD(node, KWORK_ITEM, Link);
             }
             KeReleaseSpinlock(&gWorkQueueLock, flags);
@@ -74,7 +75,7 @@ bool KeQueueWorkItem(KWORK_ITEM* item) {
     }
 
     uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
-    ListInsertTail(&gWorkQueueList, &item->Link);
+    KeListInsertTailChecked(&gWorkQueueList, &item->Link);
     EVENT* event = gWorkQueueEvent;
     KeReleaseSpinlock(&gWorkQueueLock, flags);
 

code/kernel/mm_pool.c

@@ -0,0 +1,957 @@
+// Kernel pool allocator and corruption diagnostics.
+
+#include <stddef.h>
+#include <string.h>
+#include <intrin.h>
+#include <kernel/mm_pool.h>
+#include <kernel/mm_pte.h>
+#include <kernel/mm_pfn.h>
+#include <kernel/ke_debug.h>
+#include <kernel/ke_spinlock.h>
+
+// The boundary-tag allocator stores a header before each allocation and a
+// footer after it. Mirrored size and state fields permit constant-time
+// coalescing while retaining enough metadata for corruption diagnostics.
+
+#define BLOCK_MAGIC  0xB10CB10CUL
+
+#define BLOCK_STATE_USED        0u
+#define BLOCK_STATE_FREE        1u
+#define BLOCK_STATE_QUARANTINE  2u
+
+#define BLOCK_FLAG_CANARY       0x00000001u
+#define BLOCK_FLAG_POISONED     0x00000002u
+
+#define POOL_TAIL_CANARY_BASE   0xC0FFEE1234ABCDEFULL
+#define POOL_POISON_FREE        0xDD
+#define POOL_QUARANTINE_SLOTS   256u
+#define POOL_ALIGNMENT          16u
+#define POOL_TAG_DEFAULT        0x6C6F6F70u  // 'pool'.
+
+typedef struct BLOCK_HDR {
+    uint32_t Magic;
+    uint32_t Free;      // BLOCK_STATE_*
+    size_t   Size;      // usable allocator bytes (excludes header + footer)
+    struct BLOCK_HDR* PrevFree;
+    struct BLOCK_HDR* NextFree;
+
+    // Debug metadata. Present in all builds so boot.ini bits can enable
+    // diagnostics without changing allocator layout or ABI.
+    size_t   RequestedSize;
+    uint32_t Tag;
+    uint32_t Flags;
+    uint64_t Sequence;
+    uintptr_t AllocCaller;
+    uintptr_t FreeCaller;
+    uintptr_t FreeListInsertCaller;
+    uintptr_t FreeListRemoveCaller;
+    uintptr_t CoalesceCaller;
+    uint64_t FreeListSequence;
+
+    // Keep user data 16-byte aligned for MSVC optimized code and image constants.
+    uint64_t AlignmentPadding;
+} BLOCK_HDR;
+
+typedef struct BLOCK_FTR {
+    size_t Size;
+    uint32_t Free;
+    uint32_t Magic;
+} BLOCK_FTR;
+
+_Static_assert((sizeof(BLOCK_HDR) % POOL_ALIGNMENT) == 0, "pool header must preserve 16-byte user alignment");
+_Static_assert((sizeof(BLOCK_FTR) % POOL_ALIGNMENT) == 0, "pool footer must preserve 16-byte block alignment");
+
+#define HDR_SIZE  (sizeof(BLOCK_HDR))
+#define FTR_SIZE  (sizeof(BLOCK_FTR))
+#define OVERHEAD  (HDR_SIZE + FTR_SIZE)
+#define MIN_BLOCK (OVERHEAD + sizeof(void*))
+
+// Implements the hdr To Ftr operation.
+static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
+    return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
+}
+// Implements the ftr To Hdr operation.
+static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
+    return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
+}
+// Implements the next Block operation.
+static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
+    return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
+}
+// Implements the prev Ftr operation.
+static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
+    return (BLOCK_FTR *)((char *)h - FTR_SIZE);
+}
+
+typedef struct HEAP HEAP;
+
+struct HEAP {
+    BLOCK_HDR* FreeList;    // doubly-linked list of free blocks
+    uintptr_t   Start;       // virtual base of the arena
+    uintptr_t   Brk;         // current committed end (page-aligned)
+    uintptr_t   Limit;       // hard upper bound
+    uint64_t    MapFlags;    // PAGE_* flags for new committed pages
+    SPINLOCK    Lock;
+    int         DecommitTailOnFree;
+
+    uint64_t    NextSequence;
+    uint64_t    NextFreeListSequence;
+    BLOCK_HDR*  Quarantine[POOL_QUARANTINE_SLOTS];
+    size_t      QuarantineCursor;
+    size_t      QuarantineCount;
+};
+
+// Returns the active allocator diagnostic options.
+static inline uint32_t KiPoolDebugFlags(void) {
+    return KeGetDebugFlags() & (KE_DEBUG_POOL_CANARY |
+                                KE_DEBUG_POOL_POISON |
+                                KE_DEBUG_POOL_QUARANTINE |
+                                KE_DEBUG_POOL_CALLERS);
+}
+
+// Derives a per-allocation tail canary from stable block metadata.
+static inline uint64_t KiPoolTailCanaryValue(const BLOCK_HDR* blk) {
+    return POOL_TAIL_CANARY_BASE ^
+           ((uint64_t)(uintptr_t)blk << 13) ^
+           ((uint64_t)blk->RequestedSize << 1) ^
+           blk->Sequence;
+}
+
+// Returns the first caller-visible byte of an allocator block.
+static inline uint8_t* KiBlockUserBase(BLOCK_HDR* blk) {
+    return (uint8_t*)blk + HDR_SIZE;
+}
+
+// Returns the canary slot immediately after the requested payload.
+static inline uint64_t* KiBlockTailCanary(BLOCK_HDR* blk) {
+    return (uint64_t*)(void*)(KiBlockUserBase(blk) + blk->RequestedSize);
+}
+
+// Returns a diagnostic name for an allocator block state.
+static const char* KiPoolStateName(uint32_t state) {
+    switch (state) {
+        case BLOCK_STATE_USED: return "used";
+        case BLOCK_STATE_FREE: return "free";
+        case BLOCK_STATE_QUARANTINE: return "quarantine";
+        default: return "bad-state";
+    }
+}
+
+// Stops with allocator metadata that identifies the corrupt block and callers.
+static _Noreturn void KiPoolPanic(const char* reason, HEAP* h, BLOCK_HDR* blk, const void* ptr) {
+    KePanic(KE_PANIC_CODE_GENERIC,
+            "MM: pool corruption: %s ptr=0x%016llX hdr=0x%016llX heap=[0x%016llX,0x%016llX) magic=%08x state=%u(%s) size=%llu req=%llu tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX fl_ins=0x%016llX fl_rem=0x%016llX coal=0x%016llX fl_seq=%llu\n",
+            reason ? reason : "unknown",
+            (unsigned long long)(uintptr_t)ptr,
+            (unsigned long long)(uintptr_t)blk,
+            (unsigned long long)(h ? h->Start : 0),
+            (unsigned long long)(h ? h->Brk : 0),
+            blk ? blk->Magic : 0,
+            blk ? blk->Free : 0,
+            blk ? KiPoolStateName(blk->Free) : "<null>",
+            (unsigned long long)(blk ? blk->Size : 0),
+            (unsigned long long)(blk ? blk->RequestedSize : 0),
+            blk ? blk->Tag : 0,
+            blk ? blk->Flags : 0,
+            (unsigned long long)(blk ? blk->Sequence : 0),
+            (unsigned long long)(blk ? blk->AllocCaller : 0),
+            (unsigned long long)(blk ? blk->FreeCaller : 0),
+            (unsigned long long)(blk ? blk->FreeListInsertCaller : 0),
+            (unsigned long long)(blk ? blk->FreeListRemoveCaller : 0),
+            (unsigned long long)(blk ? blk->CoalesceCaller : 0),
+            (unsigned long long)(blk ? blk->FreeListSequence : 0));
+}
+
+// Tests whether an allocator header pointer is inside the committed heap.
+static int KiPoolHeaderInCommittedHeap(const HEAP* h, const BLOCK_HDR* blk) {
+    if (!h || !blk)
+        return 0;
+    uintptr_t addr = (uintptr_t)blk;
+    return addr >= h->Start && addr < h->Brk;
+}
+
+// Validates the mirrored footer and committed-heap bounds for one block.
+static void KiPoolValidateFooterLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
+    if (!h || !blk)
+        KiPoolPanic(where, h, blk, blk ? KiBlockUserBase(blk) : (void*)0);
+    if (blk->Magic != BLOCK_MAGIC)
+        KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
+    if ((uintptr_t)blk < h->Start || (uintptr_t)blk >= h->Brk)
+        KiPoolPanic("block outside heap", h, blk, KiBlockUserBase(blk));
+    if (blk->Size > (size_t)(h->Brk - (uintptr_t)blk - OVERHEAD))
+        KiPoolPanic("block size outside heap", h, blk, KiBlockUserBase(blk));
+    BLOCK_FTR* ftr = KiHdrToFtr(blk);
+    if ((uintptr_t)ftr + FTR_SIZE > h->Brk)
+        KiPoolPanic("footer outside heap", h, blk, KiBlockUserBase(blk));
+    if (ftr->Magic != BLOCK_MAGIC || ftr->Size != blk->Size || ftr->Free != blk->Free)
+        KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
+}
+
+// Writes the optional tail canary after the requested payload.
+static void KiPoolWriteTailCanary(BLOCK_HDR* blk) {
+    if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
+        return;
+    *KiBlockTailCanary(blk) = KiPoolTailCanaryValue(blk);
+}
+
+// Validates an enabled tail canary while the heap lock is held.
+static void KiPoolValidateTailCanaryLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
+    if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
+        return;
+    if (blk->RequestedSize + sizeof(uint64_t) > blk->Size)
+        KiPoolPanic("invalid canary layout", h, blk, KiBlockUserBase(blk));
+    uint64_t actual = *KiBlockTailCanary(blk);
+    uint64_t expect = KiPoolTailCanaryValue(blk);
+    if (actual != expect)
+        KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
+}
+
+// Poisons the requested payload before a freed block becomes reusable.
+static void KiPoolPoisonFreedPayload(BLOCK_HDR* blk) {
+    if (!blk)
+        return;
+    size_t bytes = blk->RequestedSize;
+    if (bytes > blk->Size)
+        bytes = blk->Size;
+    memset(KiBlockUserBase(blk), POOL_POISON_FREE, bytes);
+    blk->Flags |= BLOCK_FLAG_POISONED;
+    KiPoolWriteTailCanary(blk);
+}
+
+// Detects writes into a poisoned free or quarantined payload.
+static void KiPoolValidateFreedPoisonLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
+    if (!blk || !(blk->Flags & BLOCK_FLAG_POISONED))
+        return;
+    size_t bytes = blk->RequestedSize;
+    if (bytes > blk->Size)
+        bytes = blk->Size;
+    const uint8_t* p = KiBlockUserBase(blk);
+    for (size_t i = 0; i < bytes; i++) {
+        if (p[i] != POOL_POISON_FREE)
+            KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
+    }
+}
+
+// Rolls back a failed heap grow attempt. KiHeapGrow maps pages before the
+// new free block is linked into the allocator; if a later frame allocation or
+// page-table allocation fails, those earlier mappings must be returned or the
+// pool silently leaks physical frames and leaves stale mappings below Brk.
+static void KiHeapRollbackGrow(uintptr_t start, uintptr_t end) {
+    for (uintptr_t va = start; va < end; va += PAGE_SIZE) {
+        PHYSICAL_ADDRESS frame = MmPteTranslate(va);
+        if (!frame)
+            continue;
+        if (KSUCCESS(MmPteUnmapPage(va)))
+            (void)MmPfnFree(frame);
+    }
+}
+
+static void KiFreeListInsertHeadLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller);
+static void KiFreeListRemoveLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller);
+
+// Extend heap by at least 'need' bytes (rounded up to page boundary).
+static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
+    size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
+    if (h->Brk + grow > h->Limit) return 0;
+
+    uintptr_t start = h->Brk;
+    uintptr_t mappedEnd = start;
+    for (uintptr_t va = start; va < start + grow; va += PAGE_SIZE) {
+        PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
+        if (!frame) {
+            KiHeapRollbackGrow(start, mappedEnd);
+            return 0;
+        }
+        if (!KSUCCESS(MmPteMapPage(va, frame, h->MapFlags, MmPteMapFailIfPresent))) {
+            MmPfnFree(frame);
+            KiHeapRollbackGrow(start, mappedEnd);
+            return 0;
+        }
+        mappedEnd = va + PAGE_SIZE;
+        memset((void *)va, 0, PAGE_SIZE);
+    }
+
+    // The grow path owns the only transition from reserved heap VA to mapped heap
+    // pages, so a successful loop above is the heap's committed-frontier proof.
+    // Re-querying every heap page on later allocations only burns page-table locks.
+    // Missing pool pages are corruption now, not something the allocator repairs.
+
+    uintptr_t oldBrk = h->Brk;
+    uintptr_t newBrk = oldBrk + grow;
+
+    // If the current tail block is already free, extend it in-place instead of
+    // leaving adjacent free blocks. This matters once tail decommit is enabled:
+    // a later grow should rebuild the same tail block, not fragment the arena
+    // into many page-sized free runs.
+    if (oldBrk > h->Start) {
+        BLOCK_FTR* prevFtr = (BLOCK_FTR*)(oldBrk - FTR_SIZE);
+        if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free == BLOCK_STATE_FREE) {
+            BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
+            if ((uintptr_t)prev >= h->Start && prev->Magic == BLOCK_MAGIC && prev->Free == BLOCK_STATE_FREE) {
+                prev->CoalesceCaller = caller;
+                prev->Size += grow;
+                h->Brk = newBrk;
+                KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
+                KiHdrToFtr(prev)->Free  = BLOCK_STATE_FREE;
+                KiHdrToFtr(prev)->Size  = prev->Size;
+                return 1;
+            }
+        }
+    }
+
+    // Make the new pages a single free block.
+    BLOCK_HDR* blk = (BLOCK_HDR *)oldBrk;
+    blk->Magic    = BLOCK_MAGIC;
+    blk->Free     = BLOCK_STATE_FREE;
+    blk->Size     = grow - OVERHEAD;
+    blk->PrevFree = (BLOCK_HDR *)0;
+    blk->NextFree = (BLOCK_HDR *)0;
+    blk->RequestedSize = 0;
+    blk->Tag = 0;
+    blk->Flags = 0;
+    blk->Sequence = 0;
+    blk->AllocCaller = 0;
+    blk->FreeCaller = 0;
+    blk->FreeListInsertCaller = 0;
+    blk->FreeListRemoveCaller = 0;
+    blk->CoalesceCaller = 0;
+    blk->FreeListSequence = 0;
+
+    h->Brk = newBrk;
+    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+    KiHdrToFtr(blk)->Size  = blk->Size;
+    KiFreeListInsertHeadLocked(h, blk, caller);
+
+    return 1;
+}
+
+// Validates a free-list node before the allocator mutates list links.
+static void KiFreeListValidateNodeLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
+    KiPoolValidateFooterLocked(h, blk, where);
+    if (blk->Free != BLOCK_STATE_FREE)
+        KiPoolPanic("free list node is not free", h, blk, KiBlockUserBase(blk));
+
+    if (blk->PrevFree) {
+        if (!KiPoolHeaderInCommittedHeap(h, blk->PrevFree))
+            KiPoolPanic("free list previous pointer outside heap", h, blk, KiBlockUserBase(blk));
+        if (blk->PrevFree->Magic != BLOCK_MAGIC)
+            KiPoolPanic("free list previous pointer bad magic", h, blk->PrevFree, KiBlockUserBase(blk->PrevFree));
+        if (blk->PrevFree->NextFree != blk)
+            KiPoolPanic("free list previous backlink corrupt", h, blk, KiBlockUserBase(blk));
+    }
+
+    if (blk->NextFree) {
+        if (!KiPoolHeaderInCommittedHeap(h, blk->NextFree))
+            KiPoolPanic("free list next pointer outside heap", h, blk, KiBlockUserBase(blk));
+        if (blk->NextFree->Magic != BLOCK_MAGIC)
+            KiPoolPanic("free list next pointer bad magic", h, blk->NextFree, KiBlockUserBase(blk->NextFree));
+        if (blk->NextFree->PrevFree != blk)
+            KiPoolPanic("free list next backlink corrupt", h, blk, KiBlockUserBase(blk));
+    }
+}
+
+// Inserts a free block at the head of the free list and records the mutation site.
+static void KiFreeListInsertHeadLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
+    KiPoolValidateFooterLocked(h, blk, "free list insert block corrupt");
+    if (blk->Free != BLOCK_STATE_FREE)
+        KiPoolPanic("free list insert of non-free block", h, blk, KiBlockUserBase(blk));
+    if (blk->PrevFree || blk->NextFree)
+        KiPoolPanic("free list insert of linked block", h, blk, KiBlockUserBase(blk));
+
+    blk->FreeListInsertCaller = caller;
+    blk->FreeListSequence = ++h->NextFreeListSequence;
+    blk->PrevFree = (BLOCK_HDR*)0;
+    blk->NextFree = h->FreeList;
+
+    if (h->FreeList) {
+        KiFreeListValidateNodeLocked(h, h->FreeList, "free list head corrupt before insert");
+        h->FreeList->PrevFree = blk;
+    }
+    h->FreeList = blk;
+}
+
+// Unlinks a free block from the free list and records the mutation site.
+static void KiFreeListRemoveLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
+    KiFreeListValidateNodeLocked(h, blk, "free list remove block corrupt");
+
+    blk->FreeListRemoveCaller = caller;
+    blk->FreeListSequence = ++h->NextFreeListSequence;
+
+    if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
+    else               h->FreeList = blk->NextFree;
+    if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
+    blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
+}
+
+// Implements the heap Try Decommit Tail operation.
+static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
+    if (!h || !blk || blk->Free == BLOCK_STATE_USED) return;
+
+    // Do not eagerly decommit the kernel pool on every free. Large transient
+    // syscall bounce buffers, especially SysBlitScreen's frame-sized buffer,
+    // are often allocated/freed once per frame. If the tail block is unmapped
+    // immediately, the next frame has to map the same pages again, producing
+    // massive page-table churn and global TLB shootdowns. Keep the committed
+    // pool high-water resident during normal operation; add an explicit idle or
+    // pressure pool trim later if needed.
+    if (!h->DecommitTailOnFree) {
+        KiHdrToFtr(blk)->Size  = blk->Size;
+        KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+        KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+        return;
+    }
+
+    uintptr_t blockBase = (uintptr_t)blk;
+    uintptr_t oldBrk = h->Brk;
+
+    // Only the last free block can return committed heap pages. Interior free
+    // blocks must remain mapped because the allocator's boundary tags and free
+    // list links are stored inside those blocks.
+    if ((uintptr_t)KiNextBlock(blk) != oldBrk) {
+        KiHdrToFtr(blk)->Size  = blk->Size;
+        KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+        KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+        return;
+    }
+
+    // Keep enough mapped space for the tail block's header, footer, and a small
+    // usable payload, then decommit whole pages above that. The footer moves to
+    // the new Brk so future coalescing can still walk the boundary tag chain.
+    uintptr_t minEnd = blockBase + MIN_BLOCK;
+    uintptr_t newBrk = (minEnd + PAGE_SIZE - 1) & ~(uintptr_t)(PAGE_SIZE - 1);
+
+    if (newBrk >= oldBrk) {
+        KiHdrToFtr(blk)->Size  = blk->Size;
+        KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+        KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+        return;
+    }
+
+    for (uintptr_t va = newBrk; va < oldBrk; va += PAGE_SIZE) {
+        PHYSICAL_ADDRESS frame = MmPteTranslate(va);
+        if (!frame)
+            continue;
+        if (KSUCCESS(MmPteUnmapPage(va)))
+            (void)MmPfnFree(frame);
+    }
+
+    h->Brk = newBrk;
+    blk->Size = newBrk - blockBase - OVERHEAD;
+    KiHdrToFtr(blk)->Size  = blk->Size;
+    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+}
+
+// Releases a previously quarantined block into the real free list. Caller holds h->Lock.
+static void KiHeapReleaseFreeBlockLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
+    if (!h || !blk) return;
+
+    KiPoolValidateFooterLocked(h, blk, "release free footer corrupt");
+    KiPoolValidateTailCanaryLocked(h, blk, "release free tail canary corrupt");
+    KiPoolValidateFreedPoisonLocked(h, blk, "use-after-free payload write detected");
+
+    blk->Free = BLOCK_STATE_FREE;
+    blk->RequestedSize = 0;
+    blk->Tag = 0;
+    blk->Flags = 0;
+    blk->AllocCaller = 0;
+    // Keep FreeCaller/Sequence for diagnostics until the block is reused.
+    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+    KiHdrToFtr(blk)->Size  = blk->Size;
+
+    // Coalesce with next block if it is really free. Quarantined neighbors are
+    // intentionally not coalesced; their payload must remain intact for UAF checks.
+    BLOCK_HDR* next = KiNextBlock(blk);
+    if ((uintptr_t)next < h->Brk) {
+        KiPoolValidateFooterLocked(h, next, "next footer corrupt while freeing");
+        if (next->Free == BLOCK_STATE_FREE) {
+            KiFreeListRemoveLocked(h, next, caller);
+            blk->CoalesceCaller = caller;
+            next->CoalesceCaller = caller;
+            blk->Size += OVERHEAD + next->Size;
+            KiHdrToFtr(blk)->Size  = blk->Size;
+            KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+            KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+        }
+    }
+
+    // Coalesce with previous block if it is really free.
+    if ((uintptr_t)blk > h->Start) {
+        BLOCK_FTR* prevFtr = KiPrevFtr(blk);
+        if (prevFtr->Magic == BLOCK_MAGIC) {
+            BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
+            KiPoolValidateFooterLocked(h, prev, "previous footer corrupt while freeing");
+            if (prev->Free == BLOCK_STATE_FREE) {
+                KiFreeListRemoveLocked(h, prev, caller);
+                prev->CoalesceCaller = caller;
+                blk->CoalesceCaller = caller;
+                prev->Size += OVERHEAD + blk->Size;
+                KiHdrToFtr(prev)->Size  = prev->Size;
+                KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
+                KiHdrToFtr(prev)->Free  = BLOCK_STATE_FREE;
+                blk = prev;
+            }
+        }
+    }
+
+    KiFreeListInsertHeadLocked(h, blk, caller);
+
+    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
+    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+    KiHdrToFtr(blk)->Size  = blk->Size;
+
+    KiHeapTryDecommitTail(h, blk);
+}
+
+// Releases one quarantine slot into the normal free-list path.
+static void KiHeapReleaseQuarantineSlotLocked(HEAP* h, size_t index, uintptr_t caller) {
+    if (!h || index >= POOL_QUARANTINE_SLOTS)
+        return;
+    BLOCK_HDR* old = h->Quarantine[index];
+    if (!old)
+        return;
+    h->Quarantine[index] = (BLOCK_HDR*)0;
+    if (h->QuarantineCount)
+        h->QuarantineCount--;
+    if (old->Free != BLOCK_STATE_QUARANTINE)
+        KiPoolPanic("quarantine slot state corrupt", h, old, KiBlockUserBase(old));
+    KiHeapReleaseFreeBlockLocked(h, old, caller);
+}
+
+// Drains every quarantine slot when allocation pressure requires reuse.
+static void KiHeapDrainQuarantineLocked(HEAP* h, uintptr_t caller) {
+    if (!h)
+        return;
+    for (size_t i = 0; i < POOL_QUARANTINE_SLOTS; i++)
+        KiHeapReleaseQuarantineSlotLocked(h, i, caller);
+    h->QuarantineCursor = 0;
+}
+
+// Places a freed block in the bounded quarantine before it can be reused.
+static void KiHeapQuarantineBlockLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
+    if (!h || !blk)
+        return;
+    size_t idx = h->QuarantineCursor % POOL_QUARANTINE_SLOTS;
+    KiHeapReleaseQuarantineSlotLocked(h, idx, caller);
+
+    blk->Free = BLOCK_STATE_QUARANTINE;
+    KiHdrToFtr(blk)->Free  = BLOCK_STATE_QUARANTINE;
+    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
+    KiHdrToFtr(blk)->Size  = blk->Size;
+
+    h->Quarantine[idx] = blk;
+    h->QuarantineCursor = (idx + 1u) % POOL_QUARANTINE_SLOTS;
+    h->QuarantineCount++;
+}
+
+// Allocates a block from the heap.
+static void* KiHeapAlloc(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t caller) {
+    if (!requestedSize) return (void *)0;
+
+    uint32_t dbg = KiPoolDebugFlags();
+    size_t size = requestedSize;
+    if (dbg & KE_DEBUG_POOL_CANARY) {
+        if (size > (size_t)-1 - sizeof(uint64_t))
+            return (void*)0;
+        size += sizeof(uint64_t);
+    }
+    if (size > (size_t)-1 - (POOL_ALIGNMENT - 1u))
+        return (void*)0;
+    size = (size + (POOL_ALIGNMENT - 1u)) & ~(size_t)(POOL_ALIGNMENT - 1u);
+
+    uintptr_t flags = KeAcquireSpinlock(&h->Lock);
+
+    // Best-fit search.
+    BLOCK_HDR* best = (BLOCK_HDR *)0;
+    for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
+        KiPoolValidateFooterLocked(h, cur, "free list block corrupt");
+        if (cur->Free != BLOCK_STATE_FREE)
+            KiPoolPanic("free list contains non-free block", h, cur, KiBlockUserBase(cur));
+        if (cur->Size >= size) {
+            if (!best || cur->Size < best->Size) best = cur;
+            if (best->Size == size) break;
+        }
+    }
+
+    if (!best && h->QuarantineCount) {
+        KiHeapDrainQuarantineLocked(h, caller);
+        for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
+            KiPoolValidateFooterLocked(h, cur, "free list block corrupt after quarantine drain");
+            if (cur->Size >= size) {
+                if (!best || cur->Size < best->Size) best = cur;
+                if (best->Size == size) break;
+            }
+        }
+    }
+
+    if (!best) {
+        if (!KiHeapGrow(h, size + OVERHEAD, caller)) {
+            KeReleaseSpinlock(&h->Lock, flags);
+            return (void *)0;
+        }
+        // Re-search; KiHeapGrow inserted at the front.
+        best = h->FreeList;
+        while (best && best->Size < size) best = best->NextFree;
+        if (!best) {
+            KeReleaseSpinlock(&h->Lock, flags);
+            return (void *)0;
+        }
+        KiPoolValidateFooterLocked(h, best, "grown free block corrupt");
+    }
+
+    KiFreeListRemoveLocked(h, best, caller);
+
+    // Split the block if the remainder is big enough.
+    if (best->Size >= size + OVERHEAD + POOL_ALIGNMENT) {
+        BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE);
+        rem->Magic     = BLOCK_MAGIC;
+        rem->Free      = BLOCK_STATE_FREE;
+        rem->Size      = best->Size - size - OVERHEAD;
+        rem->PrevFree  = (BLOCK_HDR *)0;
+        rem->NextFree  = (BLOCK_HDR *)0;
+        rem->RequestedSize = 0;
+        rem->Tag = 0;
+        rem->Flags = 0;
+        rem->Sequence = 0;
+        rem->AllocCaller = 0;
+        rem->FreeCaller = 0;
+        rem->FreeListInsertCaller = 0;
+        rem->FreeListRemoveCaller = 0;
+        rem->CoalesceCaller = 0;
+        rem->FreeListSequence = 0;
+        KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
+        KiHdrToFtr(rem)->Free  = BLOCK_STATE_FREE;
+        KiHdrToFtr(rem)->Size  = rem->Size;
+        KiFreeListInsertHeadLocked(h, rem, caller);
+
+        best->Size = size;
+    }
+
+    best->Free = BLOCK_STATE_USED;
+    best->RequestedSize = requestedSize;
+    best->Tag = tag ? tag : POOL_TAG_DEFAULT;
+    best->Flags = 0;
+    best->Sequence = ++h->NextSequence;
+    best->AllocCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
+    best->FreeCaller = 0;
+    best->CoalesceCaller = 0;
+    if (dbg & KE_DEBUG_POOL_CANARY)
+        best->Flags |= BLOCK_FLAG_CANARY;
+
+    KiHdrToFtr(best)->Free  = BLOCK_STATE_USED;
+    KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
+    KiHdrToFtr(best)->Size  = best->Size;
+
+    memset(KiBlockUserBase(best), 0, requestedSize);
+    if (best->Size > requestedSize)
+        memset(KiBlockUserBase(best) + requestedSize, 0, best->Size - requestedSize);
+    KiPoolWriteTailCanary(best);
+
+    KeReleaseSpinlock(&h->Lock, flags);
+    return KiBlockUserBase(best);
+}
+
+// Releases a block back to the heap.
+static void KiHeapFree(HEAP* h, void* ptr, uintptr_t caller) {
+    if (!ptr) return;
+
+    uintptr_t u = (uintptr_t)ptr;
+    if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
+        KiPoolPanic("invalid free pointer outside heap", h, (BLOCK_HDR*)0, ptr);
+
+    BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
+
+    uintptr_t flags = KeAcquireSpinlock(&h->Lock);
+
+    if (blk->Magic != BLOCK_MAGIC)
+        KiPoolPanic("invalid free pointer/bad header magic", h, blk, ptr);
+    KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on free");
+    if (blk->Free == BLOCK_STATE_FREE)
+        KiPoolPanic("double free", h, blk, ptr);
+    if (blk->Free == BLOCK_STATE_QUARANTINE)
+        KiPoolPanic("double free/quarantined UAF", h, blk, ptr);
+    if (blk->Free != BLOCK_STATE_USED)
+        KiPoolPanic("invalid block state on free", h, blk, ptr);
+
+    KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on free");
+
+    uint32_t dbg = KiPoolDebugFlags();
+    blk->FreeCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
+    if (dbg & KE_DEBUG_POOL_POISON)
+        KiPoolPoisonFreedPayload(blk);
+
+    if (dbg & KE_DEBUG_POOL_QUARANTINE) {
+        KiHeapQuarantineBlockLocked(h, blk, caller);
+    } else {
+        KiHeapReleaseFreeBlockLocked(h, blk, caller);
+    }
+
+    KeReleaseSpinlock(&h->Lock, flags);
+}
+
+// Resizes a heap allocation while preserving its contents.
+static void* KiHeapRealloc(HEAP* h, void* ptr, size_t newSize, uintptr_t caller) {
+    // If ptr is NULL, realloc acts like malloc.
+    if (!ptr) {
+        return KiHeapAlloc(h, newSize, POOL_TAG_DEFAULT, caller);
+    }
+
+    // If size is 0, realloc acts like free.
+    if (newSize == 0) {
+        KiHeapFree(h, ptr, caller);
+        return (void*)0;
+    }
+
+    uintptr_t u = (uintptr_t)ptr;
+    if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
+        KiPoolPanic("invalid realloc pointer outside heap", h, (BLOCK_HDR*)0, ptr);
+
+    BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
+
+    uintptr_t flags = KeAcquireSpinlock(&h->Lock);
+    if (blk->Magic != BLOCK_MAGIC)
+        KiPoolPanic("invalid realloc pointer/bad header magic", h, blk, ptr);
+    KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on realloc");
+    if (blk->Free != BLOCK_STATE_USED)
+        KiPoolPanic("realloc of non-allocated block", h, blk, ptr);
+    KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on realloc");
+
+    size_t oldRequested = blk->RequestedSize;
+    size_t oldCopy = oldRequested < newSize ? oldRequested : newSize;
+    if (newSize <= oldRequested) {
+        blk->RequestedSize = newSize;
+        if (blk->Flags & BLOCK_FLAG_CANARY)
+            KiPoolWriteTailCanary(blk);
+        KeReleaseSpinlock(&h->Lock, flags);
+        return ptr;
+    }
+    KeReleaseSpinlock(&h->Lock, flags);
+
+    void* newPtr = KiHeapAlloc(h, newSize, POOL_TAG_DEFAULT, caller);
+    if (!newPtr)
+        return (void*)0;
+    memcpy(newPtr, ptr, oldCopy);
+    KiHeapFree(h, ptr, caller);
+    return newPtr;
+}
+
+// Kernel pool.
+static HEAP gPoolHeap;
+
+typedef struct KI_POOL_VALIDATE_STATS {
+    uint64_t Blocks;
+    uint64_t FreeBlocks;
+    uint64_t FreeListBlocks;
+    uint64_t QuarantineBlocks;
+    size_t UsedBytes;
+    size_t FreeBytes;
+} KI_POOL_VALIDATE_STATS;
+
+// Validates the heap boundary-tag walk and free-list topology while the heap is locked.
+static KSTATUS KiPoolValidateHeapLocked(HEAP* h, const char* reason, KI_POOL_VALIDATE_STATS* out) {
+    KI_POOL_VALIDATE_STATS stats;
+    memset(&stats, 0, sizeof(stats));
+
+    if (!h || h->Start == 0 || h->Limit == 0)
+        return NTLSTATUS_NOT_SUPPORTED;
+    if (h->Brk < h->Start || h->Brk > h->Limit)
+        KiPoolPanic("heap break outside limits", h, (BLOCK_HDR*)0, (void*)0);
+    if (h->Brk == h->Start) {
+        if (out) *out = stats;
+        return NTLSTATUS_SUCCESS;
+    }
+
+    uintptr_t cursor = h->Start;
+    while (cursor < h->Brk) {
+        BLOCK_HDR* blk = (BLOCK_HDR*)cursor;
+        KiPoolValidateFooterLocked(h, blk, reason ? reason : "heap block corrupt");
+        if (blk->Free > BLOCK_STATE_QUARANTINE)
+            KiPoolPanic("heap block has invalid state", h, blk, KiBlockUserBase(blk));
+
+        stats.Blocks++;
+        if (blk->Free == BLOCK_STATE_USED) {
+            stats.UsedBytes += blk->RequestedSize;
+            KiPoolValidateTailCanaryLocked(h, blk, "heap used tail canary corrupt");
+        } else if (blk->Free == BLOCK_STATE_FREE) {
+            stats.FreeBlocks++;
+            stats.FreeBytes += blk->Size;
+            KiPoolValidateFreedPoisonLocked(h, blk, "heap free payload poison corrupt");
+        } else {
+            stats.QuarantineBlocks++;
+            KiPoolValidateFreedPoisonLocked(h, blk, "heap quarantine payload poison corrupt");
+        }
+
+        BLOCK_HDR* next = KiNextBlock(blk);
+        if ((uintptr_t)next <= cursor)
+            KiPoolPanic("heap block walk did not advance", h, blk, KiBlockUserBase(blk));
+        cursor = (uintptr_t)next;
+    }
+
+    if (cursor != h->Brk)
+        KiPoolPanic("heap block walk missed break", h, (BLOCK_HDR*)0, (void*)cursor);
+
+    BLOCK_HDR* prev = (BLOCK_HDR*)0;
+    uint64_t guard = stats.Blocks + 1u;
+    for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
+        if (stats.FreeListBlocks >= guard)
+            KiPoolPanic("free list cycle detected", h, cur, KiBlockUserBase(cur));
+        stats.FreeListBlocks++;
+        KiFreeListValidateNodeLocked(h, cur, reason ? reason : "free list corrupt");
+        if (cur->PrevFree != prev)
+            KiPoolPanic("free list previous chain corrupt", h, cur, KiBlockUserBase(cur));
+        prev = cur;
+    }
+
+    if (stats.FreeListBlocks != stats.FreeBlocks)
+        KiPoolPanic("free-list/free-block count mismatch", h, h->FreeList, h->FreeList ? KiBlockUserBase(h->FreeList) : (void*)0);
+
+    if (out) *out = stats;
+    return NTLSTATUS_SUCCESS;
+}
+
+// Validates the kernel pool and prints a compact checkpoint line.
+KSTATUS MmValidateKernelPool(const char* reason) {
+    uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
+    KI_POOL_VALIDATE_STATS stats;
+    KSTATUS status = KiPoolValidateHeapLocked(&gPoolHeap, reason, &stats);
+    uintptr_t brk = gPoolHeap.Brk;
+    uintptr_t start = gPoolHeap.Start;
+    KeReleaseSpinlock(&gPoolHeap.Lock, flags);
+
+    if (KSUCCESS(status)) {
+        DbgPrint("MM: pool checkpoint '%s' ok blocks=%llu free=%llu free-list=%llu quarantine=%llu used=%llu free-bytes=%llu brk=0x%016llX usage=%llu KiB\n",
+                 reason ? reason : "pool",
+                 (unsigned long long)stats.Blocks,
+                 (unsigned long long)stats.FreeBlocks,
+                 (unsigned long long)stats.FreeListBlocks,
+                 (unsigned long long)stats.QuarantineBlocks,
+                 (unsigned long long)stats.UsedBytes,
+                 (unsigned long long)stats.FreeBytes,
+                 (unsigned long long)brk,
+                 (unsigned long long)((brk - start) / 1024));
+    }
+
+    return status;
+}
+
+// Initializes the kernel pool and optionally commits its initial extent.
+void MmPoolInitialize(size_t poolSize) {
+    gPoolHeap.FreeList = (BLOCK_HDR*)0;
+    gPoolHeap.Start = POOL_BASE;
+    gPoolHeap.Brk = POOL_BASE;
+    gPoolHeap.Limit = POOL_LIMIT;
+    gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
+    gPoolHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
+    gPoolHeap.DecommitTailOnFree = 0;
+    gPoolHeap.NextSequence = 0;
+    gPoolHeap.NextFreeListSequence = 0;
+    gPoolHeap.QuarantineCursor = 0;
+    gPoolHeap.QuarantineCount = 0;
+    memset(gPoolHeap.Quarantine, 0, sizeof(gPoolHeap.Quarantine));
+
+    if (poolSize) {
+        poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
+        (void)KiHeapGrow(&gPoolHeap, poolSize, 0);
+    }
+}
+
+// Allocates zeroed kernel-pool memory from the non-user heap.
+__declspec(dllexport) void* MmAllocatePool(size_t size) {
+    return KiHeapAlloc(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
+}
+
+// Allocates zeroed kernel-pool memory with a diagnostic tag.
+__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag) {
+    return KiHeapAlloc(&gPoolHeap, size, tag, (uintptr_t)_ReturnAddress());
+}
+
+// Releases kernel-pool memory back to the non-user heap.
+__declspec(dllexport) void MmFreePool(void* ptr) {
+    KiHeapFree(&gPoolHeap, ptr, (uintptr_t)_ReturnAddress());
+}
+
+// Reallocates kernel-pool memory while preserving existing contents.
+__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) {
+    return KiHeapRealloc(&gPoolHeap, ptr, newSize, (uintptr_t)_ReturnAddress());
+}
+
+// Describes a virtual address that falls inside the kernel-pool aperture.
+void MmPoolDebugDescribeAddress(uintptr_t addr) {
+    DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
+             (unsigned long long)gPoolHeap.Start,
+             (unsigned long long)gPoolHeap.Brk,
+             (unsigned long long)gPoolHeap.Limit,
+             (unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
+
+    if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
+    if (addr >= gPoolHeap.Brk) {
+        DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
+                 (unsigned long long)addr,
+                 (unsigned long long)(addr - gPoolHeap.Brk));
+        return;
+    }
+
+    uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
+    for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
+         (uintptr_t)cur < gPoolHeap.Brk;
+         cur = KiNextBlock(cur)) {
+        if (cur->Magic != BLOCK_MAGIC) {
+            DbgPrint("MM: pool walk corrupt at hdr=0x%016llX magic=%08x\n",
+                     (unsigned long long)(uintptr_t)cur,
+                     cur->Magic);
+            break;
+        }
+        uintptr_t hdr = (uintptr_t)cur;
+        uintptr_t data = hdr + HDR_SIZE;
+        uintptr_t end = data + cur->Size;
+        uintptr_t ftr = end;
+        BLOCK_FTR* footer = KiHdrToFtr(cur);
+        if ((uintptr_t)footer + FTR_SIZE > gPoolHeap.Brk ||
+            footer->Magic != BLOCK_MAGIC || footer->Size != cur->Size || footer->Free != cur->Free) {
+            DbgPrint("MM: pool walk corrupt footer hdr=0x%016llX state=%u size=%llx footer=0x%016llX fmagic=%08x fstate=%u fsize=%llx\n",
+                     (unsigned long long)(uintptr_t)cur,
+                     cur->Free,
+                     (unsigned long long)cur->Size,
+                     (unsigned long long)(uintptr_t)footer,
+                     ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Magic : 0,
+                     ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Free : 0,
+                     ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? (unsigned long long)footer->Size : 0ULL);
+            break;
+        }
+        if (addr >= hdr && addr < ftr + FTR_SIZE) {
+            DbgPrint("MM: address 0x%016llX in pool block hdr=0x%016llX data=0x%016llX-0x%016llX size=%llx req=%llx state=%u(%s) tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX fl_ins=0x%016llX fl_rem=0x%016llX coal=0x%016llX fl_seq=%llu\n",
+                     (unsigned long long)addr,
+                     (unsigned long long)hdr,
+                     (unsigned long long)data,
+                     (unsigned long long)end,
+                     (unsigned long long)cur->Size,
+                     (unsigned long long)cur->RequestedSize,
+                     cur->Free,
+                     KiPoolStateName(cur->Free),
+                     cur->Tag,
+                     cur->Flags,
+                     (unsigned long long)cur->Sequence,
+                     (unsigned long long)cur->AllocCaller,
+                     (unsigned long long)cur->FreeCaller,
+                     (unsigned long long)cur->FreeListInsertCaller,
+                     (unsigned long long)cur->FreeListRemoveCaller,
+                     (unsigned long long)cur->CoalesceCaller,
+                     (unsigned long long)cur->FreeListSequence);
+            break;
+        }
+        if (KiNextBlock(cur) <= cur) {
+            DbgPrint("MM: pool walk stopped: non-advancing block hdr=0x%016llX size=%llx\n",
+                     (unsigned long long)(uintptr_t)cur,
+                     (unsigned long long)cur->Size);
+            break;
+        }
+    }
+    KeReleaseSpinlock(&gPoolHeap.Lock, flags);
+}
+
+// Returns the currently committed kernel-pool bytes.
+size_t MmGetKernelPoolUsage(void) {
+    return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
+}

code/kernel/mm_pte.c

@@ -142,7 +142,7 @@ void MmPteFlushKernelAddress(uintptr_t virt) {
         KCPU* cpu = &gCpus[i];
         if (i == selfIndex)
             continue;
-        if (!cpu->Present || !cpu->Started || !cpu->Online)
+        if (cpu->State != KeCpuOnline)
             continue;
         targets++;
         KiPteStatInc(&gPteStats.ShootdownTargetCpus);

code/kernel/mm_virt.c

@@ -1,5 +1,5 @@
-// Kernel virtual-memory manager. This file owns kernel pool allocation, vad bookkeeping, and user-copy validation.
-// Page-table construction lives in mm_pte.c so the higher VMM can grow toward VADs, faults, COW, and swap without mixing those policies with raw PTE edits.
+// Kernel virtual-memory manager. This file owns VAD bookkeeping, faults, and user-copy validation.
+// Pool allocation lives in mm_pool.c and page-table construction lives in mm_pte.c.
 
 // Virtual address layout:
 //   0000000000001000-00007FFFFFFFF000 is the lower-half user range.
@@ -12,9 +12,6 @@
 // The current VM model uses one shared PML4, so user pointers are globally
 // meaningful while protection and lifetime are still tracked per process VAD.
 
-// Kernel pool:
-//   A boundary-tag allocator starts at POOL_BASE and commits pages on demand.
-
 // User mappings:
 //   VADs separate reservation from commitment. Committed pages become resident
 //   when a user fault or kernel prefault path materializes them.
@@ -22,6 +19,7 @@
 #include <string.h>
 #include <intrin.h>
 #include <kernel/mm_virt.h>
+#include <kernel/mm_pool.h>
 #include <kernel/mm_pte.h>
 #include <kernel/mm_pfn.h>
 #include <kernel/mm_swap.h>
@@ -86,649 +84,6 @@ __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, s
     return KiCopyToUserPaged(userDst, kernelSrc, size);
 }
 
-// Boundary-tag allocator (used for both pool and user heap)
-// Each allocation has a BLOCK_HDR immediately before the user data, and a
-// BLOCK_FTR immediately after. Both carry the block size and a 'free' flag.
-// This allows O(1) coalescing of adjacent free blocks.
-
-#define BLOCK_MAGIC  0xB10CB10CUL
-
-#define BLOCK_STATE_USED        0u
-#define BLOCK_STATE_FREE        1u
-#define BLOCK_STATE_QUARANTINE  2u
-
-#define BLOCK_FLAG_CANARY       0x00000001u
-#define BLOCK_FLAG_POISONED     0x00000002u
-
-#define POOL_TAIL_CANARY_BASE   0xC0FFEE1234ABCDEFULL
-#define POOL_POISON_FREE        0xDD
-#define POOL_QUARANTINE_SLOTS   256u
-#define POOL_TAG_DEFAULT        0x6C6F6F70u  /* 'pool' */
-
-typedef struct BLOCK_HDR {
-    uint32_t Magic;
-    uint32_t Free;      // BLOCK_STATE_*
-    size_t   Size;      // usable allocator bytes (excludes header + footer)
-    struct BLOCK_HDR* PrevFree;
-    struct BLOCK_HDR* NextFree;
-
-    // Debug metadata. Present in all builds so boot.ini bits can enable
-    // diagnostics without changing allocator layout or ABI.
-    size_t   RequestedSize;
-    uint32_t Tag;
-    uint32_t Flags;
-    uint64_t Sequence;
-    uintptr_t AllocCaller;
-    uintptr_t FreeCaller;
-} BLOCK_HDR;
-
-typedef struct BLOCK_FTR {
-    size_t Size;
-    uint32_t Free;
-    uint32_t Magic;
-} BLOCK_FTR;
-
-#define HDR_SIZE  (sizeof(BLOCK_HDR))
-#define FTR_SIZE  (sizeof(BLOCK_FTR))
-#define OVERHEAD  (HDR_SIZE + FTR_SIZE)
-#define MIN_BLOCK (OVERHEAD + sizeof(void*))
-
-// Implements the hdr To Ftr operation.
-static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
-    return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
-}
-// Implements the ftr To Hdr operation.
-static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
-    return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
-}
-// Implements the next Block operation.
-static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
-    return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
-}
-// Implements the prev Ftr operation.
-static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
-    return (BLOCK_FTR *)((char *)h - FTR_SIZE);
-}
-
-typedef struct HEAP HEAP;
-
-struct HEAP {
-    BLOCK_HDR* FreeList;    // doubly-linked list of free blocks
-    uintptr_t   Start;       // virtual base of the arena
-    uintptr_t   Brk;         // current committed end (page-aligned)
-    uintptr_t   Limit;       // hard upper bound
-    uint64_t    MapFlags;    // PAGE_* flags for new committed pages
-    SPINLOCK    Lock;
-    int         DecommitTailOnFree;
-
-    uint64_t    NextSequence;
-    BLOCK_HDR*  Quarantine[POOL_QUARANTINE_SLOTS];
-    size_t      QuarantineCursor;
-    size_t      QuarantineCount;
-};
-
-static inline uint32_t KiPoolDebugFlags(void) {
-    return KeGetDebugFlags() & (KE_DEBUG_POOL_CANARY |
-                                KE_DEBUG_POOL_POISON |
-                                KE_DEBUG_POOL_QUARANTINE |
-                                KE_DEBUG_POOL_CALLERS);
-}
-
-static inline uint64_t KiPoolTailCanaryValue(const BLOCK_HDR* blk) {
-    return POOL_TAIL_CANARY_BASE ^
-           ((uint64_t)(uintptr_t)blk << 13) ^
-           ((uint64_t)blk->RequestedSize << 1) ^
-           blk->Sequence;
-}
-
-static inline uint8_t* KiBlockUserBase(BLOCK_HDR* blk) {
-    return (uint8_t*)blk + HDR_SIZE;
-}
-
-static inline uint64_t* KiBlockTailCanary(BLOCK_HDR* blk) {
-    return (uint64_t*)(void*)(KiBlockUserBase(blk) + blk->RequestedSize);
-}
-
-static const char* KiPoolStateName(uint32_t state) {
-    switch (state) {
-        case BLOCK_STATE_USED: return "used";
-        case BLOCK_STATE_FREE: return "free";
-        case BLOCK_STATE_QUARANTINE: return "quarantine";
-        default: return "bad-state";
-    }
-}
-
-static _Noreturn void KiPoolPanic(const char* reason, HEAP* h, BLOCK_HDR* blk, const void* ptr) {
-    KePanic(KE_PANIC_CODE_GENERIC,
-            "MM: pool corruption: %s ptr=0x%016llX hdr=0x%016llX heap=[%016llx,%016llx) magic=%08x state=%u(%s) size=%llu req=%llu tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX\n",
-            reason ? reason : "unknown",
-            ptr,
-            blk,
-            (unsigned long long)(h ? h->Start : 0),
-            (unsigned long long)(h ? h->Brk : 0),
-            blk ? blk->Magic : 0,
-            blk ? blk->Free : 0,
-            blk ? KiPoolStateName(blk->Free) : "<null>",
-            (unsigned long long)(blk ? blk->Size : 0),
-            (unsigned long long)(blk ? blk->RequestedSize : 0),
-            blk ? blk->Tag : 0,
-            blk ? blk->Flags : 0,
-            (unsigned long long)(blk ? blk->Sequence : 0),
-            blk ? (void*)blk->AllocCaller : (void*)0,
-            blk ? (void*)blk->FreeCaller : (void*)0);
-}
-
-static void KiPoolValidateFooterLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
-    if (!h || !blk)
-        KiPoolPanic(where, h, blk, blk ? KiBlockUserBase(blk) : (void*)0);
-    if (blk->Magic != BLOCK_MAGIC)
-        KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
-    if ((uintptr_t)blk < h->Start || (uintptr_t)blk >= h->Brk)
-        KiPoolPanic("block outside heap", h, blk, KiBlockUserBase(blk));
-    if (blk->Size > (size_t)(h->Brk - (uintptr_t)blk - OVERHEAD))
-        KiPoolPanic("block size outside heap", h, blk, KiBlockUserBase(blk));
-    BLOCK_FTR* ftr = KiHdrToFtr(blk);
-    if ((uintptr_t)ftr + FTR_SIZE > h->Brk)
-        KiPoolPanic("footer outside heap", h, blk, KiBlockUserBase(blk));
-    if (ftr->Magic != BLOCK_MAGIC || ftr->Size != blk->Size || ftr->Free != blk->Free)
-        KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
-}
-
-static void KiPoolWriteTailCanary(BLOCK_HDR* blk) {
-    if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
-        return;
-    *KiBlockTailCanary(blk) = KiPoolTailCanaryValue(blk);
-}
-
-static void KiPoolValidateTailCanaryLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
-    if (!blk || !(blk->Flags & BLOCK_FLAG_CANARY))
-        return;
-    if (blk->RequestedSize + sizeof(uint64_t) > blk->Size)
-        KiPoolPanic("invalid canary layout", h, blk, KiBlockUserBase(blk));
-    uint64_t actual = *KiBlockTailCanary(blk);
-    uint64_t expect = KiPoolTailCanaryValue(blk);
-    if (actual != expect)
-        KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
-}
-
-static void KiPoolPoisonFreedPayload(BLOCK_HDR* blk) {
-    if (!blk)
-        return;
-    size_t bytes = blk->RequestedSize;
-    if (bytes > blk->Size)
-        bytes = blk->Size;
-    memset(KiBlockUserBase(blk), POOL_POISON_FREE, bytes);
-    blk->Flags |= BLOCK_FLAG_POISONED;
-    KiPoolWriteTailCanary(blk);
-}
-
-static void KiPoolValidateFreedPoisonLocked(HEAP* h, BLOCK_HDR* blk, const char* where) {
-    if (!blk || !(blk->Flags & BLOCK_FLAG_POISONED))
-        return;
-    size_t bytes = blk->RequestedSize;
-    if (bytes > blk->Size)
-        bytes = blk->Size;
-    const uint8_t* p = KiBlockUserBase(blk);
-    for (size_t i = 0; i < bytes; i++) {
-        if (p[i] != POOL_POISON_FREE)
-            KiPoolPanic(where, h, blk, KiBlockUserBase(blk));
-    }
-}
-
-// Rolls back a failed heap grow attempt. KiHeapGrow maps pages before the
-// new free block is linked into the allocator; if a later frame allocation or
-// page-table allocation fails, those earlier mappings must be returned or the
-// pool silently leaks physical frames and leaves stale mappings below Brk.
-static void KiHeapRollbackGrow(uintptr_t start, uintptr_t end) {
-    for (uintptr_t va = start; va < end; va += PAGE_SIZE) {
-        PHYSICAL_ADDRESS frame = MmPteTranslate(va);
-        if (!frame)
-            continue;
-        if (KSUCCESS(MmPteUnmapPage(va)))
-            (void)MmPfnFree(frame);
-    }
-}
-
-// Extend heap by at least 'need' bytes (rounded up to page boundary).
-static int KiHeapGrow(HEAP* h, size_t need) {
-    size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
-    if (h->Brk + grow > h->Limit) return 0;
-
-    uintptr_t start = h->Brk;
-    uintptr_t mappedEnd = start;
-    for (uintptr_t va = start; va < start + grow; va += PAGE_SIZE) {
-        PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
-        if (!frame) {
-            KiHeapRollbackGrow(start, mappedEnd);
-            return 0;
-        }
-        if (!KSUCCESS(MmPteMapPage(va, frame, h->MapFlags, MmPteMapFailIfPresent))) {
-            MmPfnFree(frame);
-            KiHeapRollbackGrow(start, mappedEnd);
-            return 0;
-        }
-        mappedEnd = va + PAGE_SIZE;
-        memset((void *)va, 0, PAGE_SIZE);
-    }
-
-    // The grow path owns the only transition from reserved heap VA to mapped heap
-    // pages, so a successful loop above is the heap's committed-frontier proof.
-    // Re-querying every heap page on later allocations only burns page-table locks.
-    // Missing pool pages are corruption now, not something the allocator repairs.
-
-    uintptr_t oldBrk = h->Brk;
-    uintptr_t newBrk = oldBrk + grow;
-
-    // If the current tail block is already free, extend it in-place instead of
-    // leaving adjacent free blocks. This matters once tail decommit is enabled:
-    // a later grow should rebuild the same tail block, not fragment the arena
-    // into many page-sized free runs.
-    if (oldBrk > h->Start) {
-        BLOCK_FTR* prevFtr = (BLOCK_FTR*)(oldBrk - FTR_SIZE);
-        if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free == BLOCK_STATE_FREE) {
-            BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
-            if ((uintptr_t)prev >= h->Start && prev->Magic == BLOCK_MAGIC && prev->Free == BLOCK_STATE_FREE) {
-                prev->Size += grow;
-                h->Brk = newBrk;
-                KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
-                KiHdrToFtr(prev)->Free  = BLOCK_STATE_FREE;
-                KiHdrToFtr(prev)->Size  = prev->Size;
-                return 1;
-            }
-        }
-    }
-
-    // Make the new pages a single free block.
-    BLOCK_HDR* blk = (BLOCK_HDR *)oldBrk;
-    blk->Magic    = BLOCK_MAGIC;
-    blk->Free     = BLOCK_STATE_FREE;
-    blk->Size     = grow - OVERHEAD;
-    blk->PrevFree = (BLOCK_HDR *)0;
-    blk->NextFree = h->FreeList;
-    blk->RequestedSize = 0;
-    blk->Tag = 0;
-    blk->Flags = 0;
-    blk->Sequence = 0;
-    blk->AllocCaller = 0;
-    blk->FreeCaller = 0;
-    if (h->FreeList) h->FreeList->PrevFree = blk;
-    h->FreeList = blk;
-
-    h->Brk = newBrk;
-    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-    KiHdrToFtr(blk)->Size  = blk->Size;
-
-    return 1;
-}
-
-// Unlink a free block from the free list.
-static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
-    if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
-    else               h->FreeList = blk->NextFree;
-    if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
-    blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
-}
-
-// Implements the heap Try Decommit Tail operation.
-static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
-    if (!h || !blk || blk->Free == BLOCK_STATE_USED) return;
-
-    // Do not eagerly decommit the kernel pool on every free. Large transient
-    // syscall bounce buffers, especially SysBlitScreen's frame-sized buffer,
-    // are often allocated/freed once per frame. If the tail block is unmapped
-    // immediately, the next frame has to map the same pages again, producing
-    // massive page-table churn and global TLB shootdowns. Keep the committed
-    // pool high-water resident during normal operation; add an explicit idle or
-    // pressure pool trim later if needed.
-    if (!h->DecommitTailOnFree) {
-        KiHdrToFtr(blk)->Size  = blk->Size;
-        KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-        KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-        return;
-    }
-
-    uintptr_t blockBase = (uintptr_t)blk;
-    uintptr_t oldBrk = h->Brk;
-
-    // Only the last free block can return committed heap pages. Interior free
-    // blocks must remain mapped because the allocator's boundary tags and free
-    // list links are stored inside those blocks.
-    if ((uintptr_t)KiNextBlock(blk) != oldBrk) {
-        KiHdrToFtr(blk)->Size  = blk->Size;
-        KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-        KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-        return;
-    }
-
-    // Keep enough mapped space for the tail block's header, footer, and a small
-    // usable payload, then decommit whole pages above that. The footer moves to
-    // the new Brk so future coalescing can still walk the boundary tag chain.
-    uintptr_t minEnd = blockBase + MIN_BLOCK;
-    uintptr_t newBrk = (minEnd + PAGE_SIZE - 1) & ~(uintptr_t)(PAGE_SIZE - 1);
-
-    if (newBrk >= oldBrk) {
-        KiHdrToFtr(blk)->Size  = blk->Size;
-        KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-        KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-        return;
-    }
-
-    for (uintptr_t va = newBrk; va < oldBrk; va += PAGE_SIZE) {
-        PHYSICAL_ADDRESS frame = MmPteTranslate(va);
-        if (!frame)
-            continue;
-        if (KSUCCESS(MmPteUnmapPage(va)))
-            (void)MmPfnFree(frame);
-    }
-
-    h->Brk = newBrk;
-    blk->Size = newBrk - blockBase - OVERHEAD;
-    KiHdrToFtr(blk)->Size  = blk->Size;
-    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-}
-
-// Releases a previously quarantined block into the real free list. Caller holds h->Lock.
-static void KiHeapReleaseFreeBlockLocked(HEAP* h, BLOCK_HDR* blk) {
-    if (!h || !blk) return;
-
-    KiPoolValidateFooterLocked(h, blk, "release free footer corrupt");
-    KiPoolValidateTailCanaryLocked(h, blk, "release free tail canary corrupt");
-    KiPoolValidateFreedPoisonLocked(h, blk, "use-after-free payload write detected");
-
-    blk->Free = BLOCK_STATE_FREE;
-    blk->RequestedSize = 0;
-    blk->Tag = 0;
-    blk->Flags = 0;
-    blk->AllocCaller = 0;
-    // Keep FreeCaller/Sequence for diagnostics until the block is reused.
-    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-    KiHdrToFtr(blk)->Size  = blk->Size;
-
-    // Coalesce with next block if it is really free. Quarantined neighbors are
-    // intentionally not coalesced; their payload must remain intact for UAF checks.
-    BLOCK_HDR* next = KiNextBlock(blk);
-    if ((uintptr_t)next < h->Brk) {
-        KiPoolValidateFooterLocked(h, next, "next footer corrupt while freeing");
-        if (next->Free == BLOCK_STATE_FREE) {
-            KiFreeListRemove(h, next);
-            blk->Size += OVERHEAD + next->Size;
-            KiHdrToFtr(blk)->Size  = blk->Size;
-            KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-            KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-        }
-    }
-
-    // Coalesce with previous block if it is really free.
-    if ((uintptr_t)blk > h->Start) {
-        BLOCK_FTR* prevFtr = KiPrevFtr(blk);
-        if (prevFtr->Magic == BLOCK_MAGIC) {
-            BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
-            KiPoolValidateFooterLocked(h, prev, "previous footer corrupt while freeing");
-            if (prev->Free == BLOCK_STATE_FREE) {
-                KiFreeListRemove(h, prev);
-                prev->Size += OVERHEAD + blk->Size;
-                KiHdrToFtr(prev)->Size  = prev->Size;
-                KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
-                KiHdrToFtr(prev)->Free  = BLOCK_STATE_FREE;
-                blk = prev;
-            }
-        }
-    }
-
-    // Insert at head of free list.
-    blk->PrevFree  = (BLOCK_HDR *)0;
-    blk->NextFree  = h->FreeList;
-    if (h->FreeList) h->FreeList->PrevFree = blk;
-    h->FreeList = blk;
-
-    KiHdrToFtr(blk)->Free  = BLOCK_STATE_FREE;
-    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-    KiHdrToFtr(blk)->Size  = blk->Size;
-
-    KiHeapTryDecommitTail(h, blk);
-}
-
-static void KiHeapReleaseQuarantineSlotLocked(HEAP* h, size_t index) {
-    if (!h || index >= POOL_QUARANTINE_SLOTS)
-        return;
-    BLOCK_HDR* old = h->Quarantine[index];
-    if (!old)
-        return;
-    h->Quarantine[index] = (BLOCK_HDR*)0;
-    if (h->QuarantineCount)
-        h->QuarantineCount--;
-    if (old->Free != BLOCK_STATE_QUARANTINE)
-        KiPoolPanic("quarantine slot state corrupt", h, old, KiBlockUserBase(old));
-    KiHeapReleaseFreeBlockLocked(h, old);
-}
-
-static void KiHeapDrainQuarantineLocked(HEAP* h) {
-    if (!h)
-        return;
-    for (size_t i = 0; i < POOL_QUARANTINE_SLOTS; i++)
-        KiHeapReleaseQuarantineSlotLocked(h, i);
-    h->QuarantineCursor = 0;
-}
-
-static void KiHeapQuarantineBlockLocked(HEAP* h, BLOCK_HDR* blk) {
-    if (!h || !blk)
-        return;
-    size_t idx = h->QuarantineCursor % POOL_QUARANTINE_SLOTS;
-    KiHeapReleaseQuarantineSlotLocked(h, idx);
-
-    blk->Free = BLOCK_STATE_QUARANTINE;
-    KiHdrToFtr(blk)->Free  = BLOCK_STATE_QUARANTINE;
-    KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
-    KiHdrToFtr(blk)->Size  = blk->Size;
-
-    h->Quarantine[idx] = blk;
-    h->QuarantineCursor = (idx + 1u) % POOL_QUARANTINE_SLOTS;
-    h->QuarantineCount++;
-}
-
-// Implements the heap Alloc operation.
-static void* KiHeapAllocEx(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t caller) {
-    if (!requestedSize) return (void *)0;
-
-    uint32_t dbg = KiPoolDebugFlags();
-    size_t size = requestedSize;
-    if (dbg & KE_DEBUG_POOL_CANARY) {
-        if (size > (size_t)-1 - sizeof(uint64_t))
-            return (void*)0;
-        size += sizeof(uint64_t);
-    }
-    size = (size + 7) & ~7ULL;   // 8-byte align
-
-    uintptr_t flags = KeAcquireSpinlock(&h->Lock);
-
-    // Best-fit search.
-    BLOCK_HDR* best = (BLOCK_HDR *)0;
-    for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
-        KiPoolValidateFooterLocked(h, cur, "free list block corrupt");
-        if (cur->Free != BLOCK_STATE_FREE)
-            KiPoolPanic("free list contains non-free block", h, cur, KiBlockUserBase(cur));
-        if (cur->Size >= size) {
-            if (!best || cur->Size < best->Size) best = cur;
-            if (best->Size == size) break;
-        }
-    }
-
-    if (!best && h->QuarantineCount) {
-        KiHeapDrainQuarantineLocked(h);
-        for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
-            KiPoolValidateFooterLocked(h, cur, "free list block corrupt after quarantine drain");
-            if (cur->Size >= size) {
-                if (!best || cur->Size < best->Size) best = cur;
-                if (best->Size == size) break;
-            }
-        }
-    }
-
-    if (!best) {
-        if (!KiHeapGrow(h, size + OVERHEAD)) {
-            KeReleaseSpinlock(&h->Lock, flags);
-            return (void *)0;
-        }
-        // Re-search; KiHeapGrow inserted at the front.
-        best = h->FreeList;
-        while (best && best->Size < size) best = best->NextFree;
-        if (!best) {
-            KeReleaseSpinlock(&h->Lock, flags);
-            return (void *)0;
-        }
-        KiPoolValidateFooterLocked(h, best, "grown free block corrupt");
-    }
-
-    KiFreeListRemove(h, best);
-
-    // Split the block if the remainder is big enough.
-    if (best->Size >= size + OVERHEAD + sizeof(void*)) {
-        BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE);
-        rem->Magic     = BLOCK_MAGIC;
-        rem->Free      = BLOCK_STATE_FREE;
-        rem->Size      = best->Size - size - OVERHEAD;
-        rem->PrevFree  = (BLOCK_HDR *)0;
-        rem->NextFree  = h->FreeList;
-        rem->RequestedSize = 0;
-        rem->Tag = 0;
-        rem->Flags = 0;
-        rem->Sequence = 0;
-        rem->AllocCaller = 0;
-        rem->FreeCaller = 0;
-        if (h->FreeList) h->FreeList->PrevFree = rem;
-        h->FreeList    = rem;
-        KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
-        KiHdrToFtr(rem)->Free  = BLOCK_STATE_FREE;
-        KiHdrToFtr(rem)->Size  = rem->Size;
-
-        best->Size = size;
-    }
-
-    best->Free = BLOCK_STATE_USED;
-    best->RequestedSize = requestedSize;
-    best->Tag = tag ? tag : POOL_TAG_DEFAULT;
-    best->Flags = 0;
-    best->Sequence = ++h->NextSequence;
-    best->AllocCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
-    best->FreeCaller = 0;
-    if (dbg & KE_DEBUG_POOL_CANARY)
-        best->Flags |= BLOCK_FLAG_CANARY;
-
-    KiHdrToFtr(best)->Free  = BLOCK_STATE_USED;
-    KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
-    KiHdrToFtr(best)->Size  = best->Size;
-
-    memset(KiBlockUserBase(best), 0, requestedSize);
-    if (best->Size > requestedSize)
-        memset(KiBlockUserBase(best) + requestedSize, 0, best->Size - requestedSize);
-    KiPoolWriteTailCanary(best);
-
-    KeReleaseSpinlock(&h->Lock, flags);
-    return KiBlockUserBase(best);
-}
-
-static void* KiHeapAlloc(HEAP* h, size_t size) {
-    return KiHeapAllocEx(h, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
-}
-
-// Implements the heap Free operation.
-static void KiHeapFreeEx(HEAP* h, void* ptr, uintptr_t caller) {
-    if (!ptr) return;
-
-    uintptr_t u = (uintptr_t)ptr;
-    if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
-        KiPoolPanic("invalid free pointer outside heap", h, (BLOCK_HDR*)0, ptr);
-
-    BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
-
-    uintptr_t flags = KeAcquireSpinlock(&h->Lock);
-
-    if (blk->Magic != BLOCK_MAGIC)
-        KiPoolPanic("invalid free pointer/bad header magic", h, blk, ptr);
-    KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on free");
-    if (blk->Free == BLOCK_STATE_FREE)
-        KiPoolPanic("double free", h, blk, ptr);
-    if (blk->Free == BLOCK_STATE_QUARANTINE)
-        KiPoolPanic("double free/quarantined UAF", h, blk, ptr);
-    if (blk->Free != BLOCK_STATE_USED)
-        KiPoolPanic("invalid block state on free", h, blk, ptr);
-
-    KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on free");
-
-    uint32_t dbg = KiPoolDebugFlags();
-    blk->FreeCaller = (dbg & KE_DEBUG_POOL_CALLERS) ? caller : 0;
-    if (dbg & KE_DEBUG_POOL_POISON)
-        KiPoolPoisonFreedPayload(blk);
-
-    if (dbg & KE_DEBUG_POOL_QUARANTINE) {
-        KiHeapQuarantineBlockLocked(h, blk);
-    } else {
-        KiHeapReleaseFreeBlockLocked(h, blk);
-    }
-
-    KeReleaseSpinlock(&h->Lock, flags);
-}
-
-static void KiHeapFree(HEAP* h, void* ptr) {
-    KiHeapFreeEx(h, ptr, (uintptr_t)_ReturnAddress());
-}
-
-// Implements the heap Realloc operation.
-static void* KiHeapReallocEx(HEAP* h, void* ptr, size_t newSize, uintptr_t caller) {
-    // If ptr is NULL, realloc acts like malloc.
-    if (!ptr) {
-        return KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller);
-    }
-
-    // If size is 0, realloc acts like free.
-    if (newSize == 0) {
-        KiHeapFreeEx(h, ptr, caller);
-        return (void*)0;
-    }
-
-    uintptr_t u = (uintptr_t)ptr;
-    if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
-        KiPoolPanic("invalid realloc pointer outside heap", h, (BLOCK_HDR*)0, ptr);
-
-    BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
-
-    uintptr_t flags = KeAcquireSpinlock(&h->Lock);
-    if (blk->Magic != BLOCK_MAGIC)
-        KiPoolPanic("invalid realloc pointer/bad header magic", h, blk, ptr);
-    KiPoolValidateFooterLocked(h, blk, "allocated footer corrupt on realloc");
-    if (blk->Free != BLOCK_STATE_USED)
-        KiPoolPanic("realloc of non-allocated block", h, blk, ptr);
-    KiPoolValidateTailCanaryLocked(h, blk, "tail canary corrupt on realloc");
-
-    size_t oldRequested = blk->RequestedSize;
-    size_t oldCopy = oldRequested < newSize ? oldRequested : newSize;
-    if (newSize <= oldRequested) {
-        blk->RequestedSize = newSize;
-        if (blk->Flags & BLOCK_FLAG_CANARY)
-            KiPoolWriteTailCanary(blk);
-        KeReleaseSpinlock(&h->Lock, flags);
-        return ptr;
-    }
-    KeReleaseSpinlock(&h->Lock, flags);
-
-    void* newPtr = KiHeapAllocEx(h, newSize, POOL_TAG_DEFAULT, caller);
-    if (!newPtr)
-        return (void*)0;
-    memcpy(newPtr, ptr, oldCopy);
-    KiHeapFreeEx(h, ptr, caller);
-    return newPtr;
-}
-
-static void* KiHeapRealloc(HEAP* h, void* ptr, size_t newSize) {
-    return KiHeapReallocEx(h, ptr, newSize, (uintptr_t)_ReturnAddress());
-}
-
-// Kernel pool.
-static HEAP gPoolHeap;
-
 // VAD arena.
 typedef enum MM_VAD_PAGE_STATE {
     MmVadPageFree = 0,
@@ -803,22 +158,7 @@ static void KiVadInitialize(void) {
 void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
     MmPteInitialize(bootPml4Phys);
 
-    gPoolHeap.FreeList = (BLOCK_HDR *)0;
-    gPoolHeap.Start    = POOL_BASE;
-    gPoolHeap.Brk      = POOL_BASE;
-    gPoolHeap.Limit    = POOL_LIMIT;
-    gPoolHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
-    gPoolHeap.Lock     = (SPINLOCK)SPINLOCK_INIT;
-    gPoolHeap.DecommitTailOnFree = 0;
-    gPoolHeap.NextSequence = 0;
-    gPoolHeap.QuarantineCursor = 0;
-    gPoolHeap.QuarantineCount = 0;
-    memset(gPoolHeap.Quarantine, 0, sizeof(gPoolHeap.Quarantine));
-
-    if (poolSize) {
-        poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
-        KiHeapGrow(&gPoolHeap, poolSize);
-    }
+    MmPoolInitialize(poolSize);
 
     KiVadInitialize();
 
@@ -830,26 +170,6 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
     (void)MmPfnPin(gVadZeroPageFrame);
 }
 
-// Allocates zeroed kernel-pool memory from the non-user heap.
-__declspec(dllexport) void* MmAllocatePool(size_t size) {
-    return KiHeapAllocEx(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
-}
-
-// Allocates zeroed kernel-pool memory with a diagnostic tag.
-__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag) {
-    return KiHeapAllocEx(&gPoolHeap, size, tag, (uintptr_t)_ReturnAddress());
-}
-
-// Releases kernel-pool memory back to the non-user heap.
-__declspec(dllexport) void MmFreePool(void* ptr) {
-    KiHeapFreeEx(&gPoolHeap, ptr, (uintptr_t)_ReturnAddress());
-}
-
-// Reallocates kernel-pool memory while preserving existing contents.
-__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) {
-    return KiHeapReallocEx(&gPoolHeap, ptr, newSize, (uintptr_t)_ReturnAddress());
-}
-
 // VAD utility helpers.
 static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
     if (align < PAGE_SIZE) align = PAGE_SIZE;
@@ -3122,72 +2442,6 @@ void MmVadDebugDumpProcess(PROCESS* process) {
     if (!count) DbgPrint("  <none>\n");
 }
 
-// Implements the debug Describe Pool Address operation.
-static void KiDebugDescribePoolAddress(uintptr_t addr) {
-    DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
-             (unsigned long long)gPoolHeap.Start,
-             (unsigned long long)gPoolHeap.Brk,
-             (unsigned long long)gPoolHeap.Limit,
-             (unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
-
-    if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
-    if (addr >= gPoolHeap.Brk) {
-        DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
-                 (unsigned long long)addr,
-                 (unsigned long long)(addr - gPoolHeap.Brk));
-        return;
-    }
-
-    uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
-    for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
-         (uintptr_t)cur < gPoolHeap.Brk;
-         cur = KiNextBlock(cur)) {
-        if (cur->Magic != BLOCK_MAGIC) {
-            DbgPrint("MM: pool walk corrupt at hdr=0x%016llX magic=%08x\n", cur, cur->Magic);
-            break;
-        }
-        uintptr_t hdr = (uintptr_t)cur;
-        uintptr_t data = hdr + HDR_SIZE;
-        uintptr_t end = data + cur->Size;
-        uintptr_t ftr = end;
-        BLOCK_FTR* footer = KiHdrToFtr(cur);
-        if ((uintptr_t)footer + FTR_SIZE > gPoolHeap.Brk ||
-            footer->Magic != BLOCK_MAGIC || footer->Size != cur->Size || footer->Free != cur->Free) {
-            DbgPrint("MM: pool walk corrupt footer hdr=0x%016llX state=%u size=%llx footer=0x%016llX fmagic=%08x fstate=%u fsize=%llx\n",
-                     cur,
-                     cur->Free,
-                     (unsigned long long)cur->Size,
-                     footer,
-                     ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Magic : 0,
-                     ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? footer->Free : 0,
-                     ((uintptr_t)footer + FTR_SIZE <= gPoolHeap.Brk) ? (unsigned long long)footer->Size : 0ULL);
-            break;
-        }
-        if (addr >= hdr && addr < ftr + FTR_SIZE) {
-            DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx req=%llx state=%u(%s) tag=%08x flags=%08x seq=%llu alloc=0x%016llX free=0x%016llX\n",
-                     (unsigned long long)addr,
-                     (unsigned long long)hdr,
-                     (unsigned long long)data,
-                     (unsigned long long)end,
-                     (unsigned long long)cur->Size,
-                     (unsigned long long)cur->RequestedSize,
-                     cur->Free,
-                     KiPoolStateName(cur->Free),
-                     cur->Tag,
-                     cur->Flags,
-                     (unsigned long long)cur->Sequence,
-                     (void*)cur->AllocCaller,
-                     (void*)cur->FreeCaller);
-            break;
-        }
-        if (KiNextBlock(cur) <= cur) {
-            DbgPrint("MM: pool walk stopped: non-advancing block hdr=0x%016llX size=%llx\n", cur, (unsigned long long)cur->Size);
-            break;
-        }
-    }
-    KeReleaseSpinlock(&gPoolHeap.Lock, flags);
-}
-
 // Implements the debug Describe Address operation.
 void MmDebugDescribeAddress(uintptr_t addr) {
     PHYSICAL_ADDRESS phys = 0;
@@ -3200,7 +2454,7 @@ void MmDebugDescribeAddress(uintptr_t addr) {
              (unsigned long long)flags);
 
     if (addr >= POOL_BASE && addr < POOL_LIMIT) {
-        KiDebugDescribePoolAddress(addr);
+        MmPoolDebugDescribeAddress(addr);
         return;
     }
 
@@ -3334,12 +2588,6 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
     MmFreeUserImageForProcess(process, ptr, size);
 }
 
-// Returns kernel Pool Usage.
-size_t MmGetKernelPoolUsage(void) {
-    // Returns bytes committed to the kernel pool.
-    return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
-}
-
 // Returns user Heap Usage.
 size_t MmGetUserHeapUsage(void) {
     size_t total = 0;

code/kernel/ob_object.c

@@ -1,8 +1,10 @@
-// Object manager. This file owns reference-counted kernel objects and handle-table lifetime rules.
+// Object manager. This file owns reference-counted kernel objects and their terminal lifetime transitions.
 
+#include <stdbool.h>
 #include <string.h>
 #include <intrin.h>
 #include <kernel/ob_object.h>
+#include <kernel/ke_debug.h>
 #include <kernel/mm_virt.h>
 
 // Returns a diagnostic pool tag for the object type.
@@ -28,39 +30,102 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
     case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
     case ObjectTypeTimer:       return MM_POOL_TAG('t', 'm', 'r', ' ');
     case ObjectTypeModule:      return MM_POOL_TAG('m', 'o', 'd', 'u');
+    case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k');
     default:                    return MM_POOL_TAG('o', 'b', 'j', ' ');
     }
 }
 
+// Initializes a caller-owned object header in the live, inserted state.
+__declspec(dllexport) void ObInitializeObjectHeader(OBJECT* object, OBJECT_TYPE type) {
+    if (!object)
+        return;
+
+    object->Type = type;
+    object->RefCount = 1;
+    object->LifetimeState = ObjectAlive;
+    object->Flags = OBJECT_FLAG_INSERTED;
+    object->Destroy = (void*)0;
+}
+
 // Creates a zeroed, tagged object with one initial reference.
 __declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) {
     if (size < sizeof(OBJECT)) size = sizeof(OBJECT);
 
-    OBJECT* obj = (OBJECT *)MmAllocatePoolTag(size, KiObjectPoolTag(type));
-    if (!obj) return (OBJECT *)0;
+    OBJECT* object = (OBJECT*)MmAllocatePoolTag(size, KiObjectPoolTag(type));
+    if (!object) return (OBJECT*)0;
 
-    memset(obj, 0, size);
-    obj->Type     = type;
-    obj->RefCount = 1;
-    obj->Destroy  = (void *)0; // caller fills this in
-    return obj;
+    memset(object, 0, size);
+    ObInitializeObjectHeader(object, type);
+    return object;
 }
 
-// Implements the reference Object operation.
-__declspec(dllexport) void ObReferenceObject(OBJECT* object) {
-    if (!object) return;
-    _InterlockedIncrement(&object->RefCount);
-}
+// Acquires a reference without allowing zero-to-one resurrection.
+__declspec(dllexport) bool ObTryReferenceObject(OBJECT* object) {
+    if (!object)
+        return false;
 
-// Implements the dereference Object operation.
-__declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
-    if (!object) return;
-    long prev = _InterlockedDecrement(&object->RefCount);
-    if (prev == 0) {
-        // Reference count hit zero - run the destructor then free.
-        if (object->Destroy) {
-            object->Destroy(object);
-        }
-        MmFreePool(object);
+    for (;;) {
+        if (_InterlockedCompareExchange(&object->LifetimeState, 0, 0) != ObjectAlive)
+            return false;
+        if ((_InterlockedCompareExchange(&object->Flags, 0, 0) & OBJECT_FLAG_INSERTED) == 0)
+            return false;
+
+        long refs = _InterlockedCompareExchange(&object->RefCount, 0, 0);
+        if (refs <= 0 || refs == 0x7fffffffL)
+            return false;
+
+        if (_InterlockedCompareExchange(&object->RefCount, refs + 1, refs) == refs)
+            return true;
     }
 }
+
+// Acquires a required reference and catches stale pointers at the caller site.
+__declspec(dllexport) void ObReferenceObject(OBJECT* object) {
+    if (!object)
+        return;
+    if (ObTryReferenceObject(object))
+        return;
+
+    KePanic(KE_PANIC_CODE_GENERIC,
+            "object: stale reference object=0x%016llX type=%u refs=%ld state=%ld flags=0x%08x\n",
+            (uint64_t)(uintptr_t)object,
+            (unsigned)object->Type,
+            object->RefCount,
+            object->LifetimeState,
+            (unsigned)object->Flags);
+}
+
+// Releases a reference and performs the one-way Alive-to-Dying-to-Dead transition.
+__declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
+    if (!object)
+        return;
+
+    long refs = _InterlockedDecrement(&object->RefCount);
+    if (refs > 0)
+        return;
+    if (refs < 0) {
+        KePanic(KE_PANIC_CODE_GENERIC,
+                "object: reference underflow object=0x%016llX type=%u refs=%ld state=%ld flags=0x%08x\n",
+                (uint64_t)(uintptr_t)object,
+                (unsigned)object->Type,
+                refs,
+                object->LifetimeState,
+                (unsigned)object->Flags);
+    }
+
+    if (_InterlockedCompareExchange(&object->LifetimeState, ObjectDying, ObjectAlive) != ObjectAlive) {
+        KePanic(KE_PANIC_CODE_GENERIC,
+                "object: duplicate terminal transition object=0x%016llX type=%u state=%ld flags=0x%08x\n",
+                (uint64_t)(uintptr_t)object,
+                (unsigned)object->Type,
+                object->LifetimeState,
+                (unsigned)object->Flags);
+    }
+
+    _InterlockedExchange(&object->Flags, OBJECT_FLAG_UNLINKED);
+    if (object->Destroy)
+        object->Destroy(object);
+
+    _InterlockedExchange(&object->LifetimeState, ObjectDead);
+    MmFreePool(object);
+}