implement proper IRQL, DPCs, kernel and user APCs, alertable waits, async file I/O with cancellation, trap frames, selftests, a real driver/device model, native async FAT32, VFAT dotfile fixes, syscall ABI cleanup, async_apc and async_cancel tests, POSIX multi-volume paths, and pimp the readme.

code/kernel/ke_thread.c

@@ -1,9 +1,12 @@
 // ke_thread.c - thread model
 #include <string.h>
+#include <stddef.h>
 #include <intrin.h>
 #include <ke_thread.h>
 #include <ke_sched.h>
+#include <ke_irql.h>
 #include <ke_event.h>
+#include <ke_apc.h>
 #include <ke_userenv.h>
 #include <mm_virt.h>
 #include <ob_object.h>
@@ -11,12 +14,26 @@
 #define FPU_FCW_DEFAULT   0x037Fu
 #define FPU_MXCSR_DEFAULT 0x1F80u
 
+// These offsets are consumed directly by ke_switch.asm.  Keep the asserts
+// here so future THREAD layout edits fail at build time instead of corrupting
+// context switches at runtime.
+#define STATIC_ASSERT_OFFSET(type, field, expected) \
+    _Static_assert(offsetof(type, field) == (expected), \
+                   #type "." #field " offset must match assembly")
+
+STATIC_ASSERT_OFFSET(THREAD, SavedContext, 88);
+STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 104);
+STATIC_ASSERT_OFFSET(THREAD, Teb, 112);
+STATIC_ASSERT_OFFSET(THREAD, FpuContext, 168);
+
 static uint32_t gNextThreadId = 1;
 extern void KeThreadEntry(void);
 
-static void ThreadDestroy(OBJECT* obj) {
+static void KiThreadDestroy(OBJECT* obj) {
     THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
 
+    KiRundownApcState(thread);
+
     if (thread->ExitEvent) {
         ObDereferenceObject(&thread->ExitEvent->Header);
         thread->ExitEvent = (EVENT*)0;
@@ -37,13 +54,13 @@ static void ThreadDestroy(OBJECT* obj) {
     }
     
     if (thread->Teb && thread->TebSize) {
-        MmFreeUserMemory(thread->Teb, thread->TebSize);
+        MmFreeUserMemory(thread->Teb);
         thread->Teb = (void*)0;
         thread->TebSize = 0;
     }
 
     if (thread->UserStackBase) {
-        MmFreeUserMemory(thread->UserStackBase, USTACK_SIZE);
+        MmFreeUserMemory(thread->UserStackBase);
         thread->UserStackBase = (void *)0;
     }
 }
@@ -56,13 +73,18 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
     if (!thread) return STATUS_NO_MEMORY;
 
-    thread->Header.Destroy   = ThreadDestroy;
+    thread->Header.Destroy   = KiThreadDestroy;
     thread->ThreadId         = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
     thread->State            = ThreadReady;
     thread->Priority         = priority;
     thread->QuantaConsumed   = 0;
     thread->WaitObject       = (OBJECT *)0;
+    thread->WaitStatus       = STATUS_SUCCESS;
+    thread->WaitListHead     = (LIST_ENTRY*)0;
+    thread->AlertableWait    = 0;
     thread->SchedLockFlags   = 0x202ULL;
+    KiInitializeApcState(thread);
+    thread->TrapFrame = (KTRAP_FRAME*)0;
 
     void* kstack = MmAllocatePool(KSTACK_SIZE);
     if (!kstack) {
@@ -88,7 +110,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
 
     SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemory(sizeof(SYS_TEB));
     if (!teb) {
-        MmFreeUserMemory(ustack, USTACK_SIZE);
+        MmFreeUserMemory(ustack);
         thread->UserStackBase = (void*)0;
         MmFreePool(kstack);
         thread->KernelStackBase = (void*)0;
@@ -132,10 +154,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     EVENT* exitEvent;
     KSTATUS s = KeEventCreate(EventManualReset, 0, &exitEvent);
     if (!KSUCCESS(s)) {
-        if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb, thread->TebSize);
+        if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb);
         thread->Teb = (void*)0;
         thread->TebSize = 0;
-        MmFreeUserMemory(ustack, USTACK_SIZE);
+        MmFreeUserMemory(ustack);
         thread->UserStackBase = (void*)0;
         MmFreePool(kstack);
         thread->KernelStackBase = (void*)0;
@@ -161,6 +183,8 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
     (void)code;
     if (!thread) return;
 
+    KiRundownApcState(thread);
+
     // Wake any pthread_join waiters before the thread is marked dead and
     // potentially freed. Manual-reset means all concurrent waiters unblock.
     if (thread->ExitEvent)
@@ -197,13 +221,18 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
     THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
     if (!thread) return STATUS_NO_MEMORY;
 
-    thread->Header.Destroy  = ThreadDestroy;
+    thread->Header.Destroy  = KiThreadDestroy;
     thread->ThreadId        = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1);
     thread->State           = ThreadReady;
     thread->Priority        = priority;
     thread->QuantaConsumed  = 0;
     thread->WaitObject      = (OBJECT*)0;
+    thread->WaitStatus      = STATUS_SUCCESS;
+    thread->WaitListHead    = (LIST_ENTRY*)0;
+    thread->AlertableWait   = 0;
     thread->SchedLockFlags  = 0x202ULL;
+    KiInitializeApcState(thread);
+    thread->TrapFrame       = (KTRAP_FRAME*)0;
     thread->UserStackBase   = (void*)0;   // no user stack
     thread->Teb             = (void*)0;   // no TEB; GS_BASE will be 0 in kernel ctx
     thread->TebSize         = 0;
@@ -245,16 +274,101 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
     return STATUS_SUCCESS;
 }
 
-void KeBlockThread(THREAD* thread, OBJECT* waitObject) {
-    thread->State      = ThreadWaiting;
-    thread->WaitObject = waitObject;
+void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
+    // Called with the synchronization object's spinlock held, so the caller is
+    // normally at DISPATCH_LEVEL here.  This function only marks intent to
+    // block; the actual scheduling point is KeCommitBlockThread at PASSIVE.
+    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
+    if (!thread) return;
 
-    KeYield();
+    thread->State         = ThreadWaiting;
+    thread->WaitObject    = waitObject;
+    thread->WaitStatus    = STATUS_PENDING;
+    thread->WaitListHead  = (LIST_ENTRY*)0;
+    thread->AlertableWait = alertable ? 1 : 0;
 }
 
-void KeUnblockThread(THREAD* thread) {
-    thread->WaitObject = (OBJECT *)0;
-    thread->State      = ThreadReady;
+
+void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
+    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
+    if (!thread) return;
+    thread->WaitListHead = waitListHead;
+}
+
+static void KiFinishWait(THREAD* thread, KSTATUS status) {
+    thread->WaitObject    = (OBJECT*)0;
+    thread->WaitStatus    = status;
+    thread->WaitListHead  = (LIST_ENTRY*)0;
+    thread->AlertableWait = 0;
+}
+
+KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
+    KeAssertIrql(PASSIVE_LEVEL);
+    if (!thread) return STATUS_INVALID_PARAMETER;
+
+    if (alertable)
+        thread->AlertableWait = 1;
+
+    // A signal/APC may have arrived after the waiter was placed on the
+    // object's wait list but before this thread reached the scheduling point.
+    // In that case KeUnblockThreadWithStatus completed the wait in-place and
+    // there is nothing left to block on.
+    if (thread->State != ThreadWaiting) {
+        KSTATUS status = thread->WaitStatus;
+        thread->AlertableWait = 0;
+        return status;
+    }
+
+    // If a user APC was already pending before the thread fully blocked,
+    // consume the alertable wait immediately instead of going to sleep.
+    if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
+        if (thread->WaitListHead)
+            ListRemove(&thread->SchedulerLink);
+        KiFinishWait(thread, STATUS_USER_APC);
+        thread->State = ThreadRunning;
+        return STATUS_USER_APC;
+    }
+
+    KeYield();
+
+    // Kernel APCs are delivered before an alertable wait returns
+    // STATUS_KERNEL_APC to its caller.
+    KiDeliverKernelApcs();
+
+    thread->AlertableWait = 0;
+    return thread->WaitStatus;
+}
+
+
+KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
+    KePrepareBlockThread(thread, waitObject, alertable);
+    return KeCommitBlockThread(thread, alertable);
+}
+
+
+void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
+    if (!thread) return;
+
+    // If the selected waiter is still the currently executing thread, the
+    // signal landed in the prepare->commit window.  Complete the wait in-place
+    // instead of inserting the running thread into the ready queue.
+    if (thread == KeGetCurrentThread()) {
+        KiFinishWait(thread, status);
+        if (thread->State == ThreadWaiting)
+            thread->State = ThreadRunning;
+        return;
+    }
+
+    // The same object can be signaled after a timeout/cancel path exists, or a
+    // buggy caller can try to wake a non-waiting thread.  Do not enqueue such a
+    // thread twice.
+    if (thread->State != ThreadWaiting) {
+        thread->WaitStatus = status;
+        return;
+    }
+
+    KiFinishWait(thread, status);
+    thread->State = ThreadReady;
     
     if (thread->Priority < PriorityHigh) {
         thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1);
@@ -264,6 +378,35 @@ void KeUnblockThread(THREAD* thread) {
     KeReadyThread(thread);
 }
 
+void KeUnblockThread(THREAD* thread) {
+    KeUnblockThreadWithStatus(thread, STATUS_SUCCESS);
+}
+
+static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
+    if (!thread) return STATUS_INVALID_PARAMETER;
+
+    if (thread->State != ThreadWaiting || !thread->AlertableWait)
+        return STATUS_PENDING;
+
+    if (thread->WaitListHead)
+        ListRemove(&thread->SchedulerLink);
+
+    KeUnblockThreadWithStatus(thread, status);
+    return STATUS_SUCCESS;
+}
+
+KSTATUS KeAlertThread(THREAD* thread) {
+    return KiAlertThreadWithStatus(thread, STATUS_ALERTED);
+}
+
+void KiWakeThreadForKernelApc(THREAD* thread) {
+    (void)KiAlertThreadWithStatus(thread, STATUS_KERNEL_APC);
+}
+
+void KiWakeThreadForUserApc(THREAD* thread) {
+    (void)KiAlertThreadWithStatus(thread, STATUS_USER_APC);
+}
+
 void KeInitThreadFpuContext(THREAD* thread)
 {
     uintptr_t raw =