add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

code/kernel/ke_thread.c

@@ -8,6 +8,8 @@
 #include <kernel/ke_cpu.h>
 #include <kernel/ke_irql.h>
 #include <kernel/ke_event.h>
+#include <kernel/ke_mutex.h>
+#include <kernel/ke_semaphore.h>
 #include <kernel/ke_apc.h>
 #include <kernel/ke_debug.h>
 #include <kernel/ke_userenv.h>
@@ -36,6 +38,11 @@ extern void KeThreadEntry(void);
 static void KiThreadDestroy(OBJECT* obj) {
     THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
 
+    // A wait that uses auxiliary stack wait blocks must be removed before the
+    // kernel stack is released. A terminated thread may never return to the
+    // wait syscall to run normal cleanup.
+    KeRundownThreadWait(thread, STATUS_ALERTED);
+
     KiRundownApcState(thread);
 
     if (thread->ExitEvent) {
@@ -43,16 +50,16 @@ static void KiThreadDestroy(OBJECT* obj) {
         thread->ExitEvent = (EVENT*)0;
     }
 
-    PROCESS* ownerForUserRegions = thread->Owner;
-    if (ownerForUserRegions) {
+    PROCESS* ownerForVadList = thread->Owner;
+    if (ownerForVadList) {
         if (thread->Teb && thread->TebSize) {
-            MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb);
+            MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb);
             thread->Teb = (void*)0;
             thread->TebSize = 0;
         }
 
         if (thread->UserStackBase) {
-            MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase);
+            MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase);
             thread->UserStackBase = (void *)0;
         }
     }
@@ -66,10 +73,13 @@ static void KiThreadDestroy(OBJECT* obj) {
         remaining = owner->ThreadCount;
         KeReleaseSpinlock(&owner->HandleLock, flags);
 
-        if (remaining == 0 && owner->Terminating &&
-            // Implements the interlocked Exchange operation.
-            _InterlockedExchange(&owner->Exited, 1) == 0) {
-            if (owner->ExitEvent) KeSignalEvent(owner->ExitEvent);
+        if (remaining == 0) {
+            // The last thread leaving a process is process exit even if user
+            // code called ExitThread instead of ExitProcess.  KeExitProcess /
+            // KeTerminateProcess set Terminating earlier; this covers the last
+            // thread voluntarily exiting without an explicit process-exit call.
+            _InterlockedExchange(&owner->Terminating, 1);
+            KeCompleteProcessExitIfLastThread(owner);
         }
 
         ObDereferenceObject(&owner->Header);
@@ -111,11 +121,13 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     thread->WaitStatus       = STATUS_SUCCESS;
     thread->WaitListHead     = (LIST_ENTRY*)0;
     thread->AlertableWait    = 0;
+    thread->SleepUntilTick   = 0;
+    thread->TimeoutStatus    = STATUS_SUCCESS;
     thread->SwitchLockCookie = 0x202ULL;
     KiInitializeApcState(thread);
     thread->TrapFrame = (KTRAP_FRAME*)0;
 
-    void* kstack = MmAllocatePool(KSTACK_SIZE);
+    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
     if (!kstack) {
         ObDereferenceObject(&thread->Header);
         return STATUS_NO_MEMORY;
@@ -127,10 +139,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
 
     KSTATUS s;
     void* ustack = (void*)0;
-    s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE,
+    s = MmVadCreateCommitted(owner, USTACK_SIZE, PAGE_SIZE,
                            PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
-                           USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
-                           UserRegionStack,
+                           MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
+                           MmVadStack,
                            &ustack);
     if (!KSUCCESS(s) || !ustack) {
         MmFreePool(kstack);
@@ -139,11 +151,19 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
         return STATUS_NO_MEMORY;
     }
     thread->UserStackBase = ustack;
-    // Ring-3 starts at the image entry via IRET, not CALL, so there is no
-    // pushed return address. The x64 ABI expects RSP % 16 == 8 at function
-    // entry; bias the initial stack as if a call had happened so compiler
-    // generated aligned SSE spills (movaps) are safe.
-    uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8;
+    // Ring-3 starts at the image entry via IRET, not CALL.  The MS x64 ABI
+    // nevertheless lets callees immediately use the caller-provided 32-byte
+    // home/shadow area above RSP, and MSVC commonly emits a first instruction
+    // like "mov [rsp+8], rcx" in unoptimized test code.  Bias the synthetic
+    // entry stack as if a CALL with a normal home area had happened:
+    //
+    //   RSP % 16 == 8 at function entry
+    //   [RSP + 0]     fake return-address slot
+    //   [RSP + 8..39] home/shadow space
+    //
+    // Without the 32-byte home area, the first spill lands in the high stack
+    // guard page and every freshly-created MSVC thread faults immediately.
+    uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8 - 32;
 
     SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
     if (!teb) {
@@ -211,8 +231,16 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     KeReleaseSpinlock(&owner->HandleLock, flags);
 
     KeInitThreadFpuContext(thread);
-    KeReadyThread(thread);
+
+    // Hand the creator a short-lived reference before making the thread
+    // runnable.  On SMP a new user thread can run and exit before
+    // KeCreateThread() returns; without this reference, callers that merely
+    // read ThreadId for logging or syscall output can race the scheduler's
+    // dead-thread dereference and touch freed pool.
+    ObReferenceObject(&thread->Header);
     *out = thread;
+
+    KeReadyThread(thread);
     return STATUS_SUCCESS;
 }
 
@@ -283,6 +311,8 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     thread->WaitStatus      = STATUS_SUCCESS;
     thread->WaitListHead    = (LIST_ENTRY*)0;
     thread->AlertableWait   = 0;
+    thread->SleepUntilTick  = 0;
+    thread->TimeoutStatus   = STATUS_SUCCESS;
     thread->SwitchLockCookie = 0x202ULL;
     KiInitializeApcState(thread);
     thread->TrapFrame       = (KTRAP_FRAME*)0;
@@ -290,7 +320,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     thread->Teb             = (void*)0;   // no TEB; GS_BASE will be 0 in kernel ctx
     thread->TebSize         = 0;
 
-    void* kstack = MmAllocatePool(KSTACK_SIZE);
+    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
     if (!kstack) {
         ObDereferenceObject(&thread->Header);
         return STATUS_NO_MEMORY;
@@ -320,7 +350,16 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     owner->ThreadCount++;
     KeReleaseSpinlock(&owner->HandleLock, flags);
 
-    if (out) *out = thread;
+    // Hand the creator a short-lived reference before making the kernel
+    // thread runnable.  Kernel workers can run, signal their completion event,
+    // exit, and be reaped on another CPU before KeCreateKernelThread()'s
+    // caller drops its returned pointer.  The live scheduler reference remains
+    // separate and is released by KeSchedulerPostSwitch() after the dead
+    // thread has switched away.
+    if (out) {
+        ObReferenceObject(&thread->Header);
+        *out = thread;
+    }
 
     KeInitThreadFpuContext(thread);
     KeReadyThread(thread);
@@ -352,6 +391,8 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable)
     thread->WaitStatus    = STATUS_PENDING;
     thread->WaitListHead  = (LIST_ENTRY*)0;
     thread->AlertableWait = alertable ? 1 : 0;
+    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
+    thread->WaitRundownContext = (void*)0;
 }
 
 
@@ -362,12 +403,41 @@ void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
     thread->WaitListHead = waitListHead;
 }
 
+// Registers cleanup for waits with auxiliary wait blocks.
+void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context) {
+    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
+    if (!thread) return;
+    thread->WaitRundownRoutine = routine;
+    thread->WaitRundownContext = context;
+}
+
+// Clears auxiliary wait cleanup after the owning wait has removed its blocks.
+void KeClearThreadWaitRundown(THREAD* thread) {
+    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
+    if (!thread) return;
+    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
+    thread->WaitRundownContext = (void*)0;
+}
+
+// Runs auxiliary wait cleanup for abort and thread teardown paths.
+void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
+    if (!thread || !thread->WaitRundownRoutine) return;
+
+    KE_WAIT_RUNDOWN_ROUTINE routine = thread->WaitRundownRoutine;
+    void* context = thread->WaitRundownContext;
+    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
+    thread->WaitRundownContext = (void*)0;
+    routine(thread, context, status);
+}
+
 // Implements the finish Wait operation.
 static void KiFinishWait(THREAD* thread, KSTATUS status) {
     thread->WaitObject    = (OBJECT*)0;
     thread->WaitStatus    = status;
     thread->WaitListHead  = (LIST_ENTRY*)0;
     thread->AlertableWait = 0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
 }
 
 // Implements the commit Block Thread operation.
@@ -385,6 +455,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     if (thread->State != ThreadWaiting) {
         KSTATUS status = thread->WaitStatus;
         thread->AlertableWait = 0;
+        thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
         return status;
     }
 
@@ -406,6 +478,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     KiDeliverKernelApcs();
 
     thread->AlertableWait = 0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
     return thread->WaitStatus;
 }
 
@@ -423,6 +497,92 @@ void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
     KiSchedulerWakeThread(thread, status);
 }
 
+// Returns whether the scheduler link still belongs to an object wait list.
+static int KiThreadWaitLinkIsLinked(const THREAD* thread) {
+    if (!thread) return 0;
+    return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
+           thread->SchedulerLink.Bwd != &thread->SchedulerLink;
+}
+
+// Removes a thread from one wait list while that wait object's lock is held.
+static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
+    if (!thread || !waitListHead) return 0;
+    if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead)
+        return 0;
+
+    // The signaling side pops the wait-list entry before it wakes the thread.
+    // In that handoff window the scheduler link is already self-linked; abort
+    // must not remove it again or turn a successful mutex/semaphore handoff
+    // into an aborted wait for a thread that may already own the object.
+    if (!KiThreadWaitLinkIsLinked(thread))
+        return 0;
+
+    ListRemove(&thread->SchedulerLink);
+    thread->WaitListHead = (LIST_ENTRY*)0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
+    return 1;
+}
+
+// Removes a thread from an object-owned wait queue using that object's lock.
+static int KiAbortObjectWait(THREAD* thread) {
+    if (!thread || !thread->WaitObject || !thread->WaitListHead)
+        return 0;
+
+    OBJECT* object = thread->WaitObject;
+    switch (object->Type) {
+        case ObjectTypeEvent: {
+            EVENT* event = CONTAINING_RECORD(object, EVENT, Header);
+            uintptr_t flags = KeAcquireSpinlock(&event->Lock);
+            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &event->WaitList);
+            KeReleaseSpinlock(&event->Lock, flags);
+            return removed;
+        }
+        case ObjectTypeMutex: {
+            MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
+            uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
+            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList);
+            KeReleaseSpinlock(&mutex->Lock, flags);
+            return removed;
+        }
+        case ObjectTypeSemaphore: {
+            SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
+            uintptr_t flags = KeAcquireSpinlock(&semaphore->Lock);
+            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &semaphore->WaitList);
+            KeReleaseSpinlock(&semaphore->Lock, flags);
+            return removed;
+        }
+        default:
+            return 0;
+    }
+}
+
+// Aborts a thread's current wait regardless of whether the wait was marked
+// alertable. Process teardown uses this for crash/kill paths: a background
+// worker blocked forever on a semaphore must not keep the process exit event
+// unsignaled after the foreground thread has faulted. Object wait-list removal
+// must use the object's own lock; otherwise signal/abort races can leave stale
+// scheduler links in an event, mutex, or semaphore wait queue.
+void KeAbortThreadWait(THREAD* thread, KSTATUS status) {
+    if (!thread) return;
+    if (thread->State != ThreadWaiting) return;
+
+    if (thread->WaitRundownRoutine) {
+        KeRundownThreadWait(thread, status);
+        return;
+    }
+
+    if (KiAbortObjectWait(thread)) {
+        KeUnblockThreadWithStatus(thread, status);
+        return;
+    }
+
+    // Sleep/timeouts and already-dequeued handoffs are owned by the scheduler.
+    // This path removes from the scheduler sleep list when present and otherwise
+    // completes the wait in place.
+    KeUnblockThreadFromWaitListWithStatus(thread, status);
+}
+
 // Implements the unblock Thread operation.
 void KeUnblockThread(THREAD* thread) {
     KeUnblockThreadWithStatus(thread, STATUS_SUCCESS);