add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

code/kernel/ke_sched.c

@@ -17,6 +17,7 @@
 #include <kernel/hal/hal_apic.h>
 
 static LIST_ENTRY gRunQueue[PriorityCount];
+static LIST_ENTRY gSleepList;
 static SPINLOCK   gSchedLock;
 
 static uint32_t gAgingCounter = 0;
@@ -305,6 +306,7 @@ static void KiIdleLoop(void) {
 // Initializes scheduler Global.
 void KiInitializeSchedulerGlobal(void) {
     for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
+    LIST_INIT(&gSleepList);
     gSchedLock = (SPINLOCK)SPINLOCK_INIT;
 
     KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
@@ -349,6 +351,8 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
     idle->WaitStatus      = STATUS_SUCCESS;
     idle->WaitListHead    = (LIST_ENTRY*)0;
     idle->AlertableWait   = 0;
+    idle->SleepUntilTick  = 0;
+    idle->TimeoutStatus   = STATUS_SUCCESS;
     idle->UserStackBase   = (void*)0;
     idle->Teb             = (void*)0;
     idle->TebSize         = 0;
@@ -358,7 +362,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
     LIST_INIT(&idle->ProcessLink);
     LIST_INIT(&idle->SchedulerLink);
 
-    void* kstack = MmAllocatePool(KSTACK_SIZE);
+    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
     if (!kstack) {
         ObDereferenceObject(&idle->Header);
         return STATUS_NO_MEMORY;
@@ -621,12 +625,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
 }
 
 
-// Implements the scheduler Wake Thread operation.
-void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
+// Completes a wait while the scheduler lock is held.
+static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
     if (!thread) return;
 
-    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
-
     // Complete the wait while holding the scheduler lock. On SMP a wake can
     // race the prepare->commit window on another CPU: the target thread is
     // already on the object's wait list and has State=ThreadWaiting, but it is
@@ -634,7 +636,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
     // That case must complete in-place, not enqueue the still-running thread.
     if (thread->State != ThreadWaiting) {
         thread->WaitStatus = status;
-        KeReleaseSpinlock(&gSchedLock, flags);
         return;
     }
 
@@ -642,6 +643,8 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
     thread->WaitStatus    = status;
     thread->WaitListHead  = (LIST_ENTRY*)0;
     thread->AlertableWait = 0;
+    thread->SleepUntilTick = 0;
+    thread->TimeoutStatus = STATUS_SUCCESS;
 
     thread->CpuBurstTicks = 0;
     thread->QuantaConsumed = 0;
@@ -652,7 +655,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
         // KeCommitBlockThread() will see State != ThreadWaiting and return the
         // completed wait status without ever entering the ready queue.
         thread->State = ThreadRunning;
-        KeReleaseSpinlock(&gSchedLock, flags);
         return;
     }
 
@@ -662,10 +664,49 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
 
     KiInsertReadyLocked(thread);
     KiPokeRunnableCpusLocked(thread);
+}
 
+
+// Wakes a thread after first removing its scheduler-link wait membership.
+void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
+    if (!thread) return;
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    if (thread->State == ThreadWaiting && thread->WaitListHead) {
+        ListRemove(&thread->SchedulerLink);
+        thread->WaitListHead = (LIST_ENTRY*)0;
+        thread->SleepUntilTick = 0;
+        thread->TimeoutStatus = STATUS_SUCCESS;
+    }
+    KiWakeThreadLocked(thread, status);
     KeReleaseSpinlock(&gSchedLock, flags);
 }
 
+// Implements the scheduler Wake Thread operation.
+void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    KiWakeThreadLocked(thread, status);
+    KeReleaseSpinlock(&gSchedLock, flags);
+}
+
+// Wakes threads whose scheduler-tick sleep deadline has expired.
+static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
+    LIST_ENTRY* node = gSleepList.Fwd;
+    while (node != &gSleepList) {
+        LIST_ENTRY* next = node->Fwd;
+        THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+
+        if (thread->State == ThreadWaiting &&
+            thread->WaitListHead == &gSleepList &&
+            thread->SleepUntilTick <= nowTick) {
+            ListRemove(node);
+            thread->WaitListHead = (LIST_ENTRY*)0;
+            KiWakeThreadLocked(thread, thread->TimeoutStatus);
+        }
+
+        node = next;
+    }
+}
+
 // Implements the scheduler Tick operation.
 void KeSchedulerTick(void) {
     KCPU* cpu = KeGetCurrentCpu();
@@ -677,6 +718,7 @@ void KeSchedulerTick(void) {
     KiArmPostSwitchUnlock(cur, flags);
 
     gSchedStats.Ticks++;
+    KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
 
     int requestReschedule = 0;
 
@@ -745,6 +787,90 @@ void KeSchedulerTick(void) {
         KiRequestReschedule();
 }
 
+
+// Arms a timeout for the current thread after a wait primitive has prepared it to block.
+KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) {
+    if (milliseconds == 0)
+        return STATUS_INVALID_PARAMETER;
+    if (!KeCanWait())
+        return STATUS_INVALID_IRQL;
+
+    THREAD* self = KeGetCurrentThread();
+    if (!self || self->IsIdleThread)
+        return STATUS_INVALID_PARAMETER;
+
+    uint64_t ticks = milliseconds / SCHED_TICK_MS;
+    if ((milliseconds % SCHED_TICK_MS) != 0)
+        ticks++;
+    if (ticks == 0)
+        ticks = 1;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    if (self->State != ThreadWaiting || self->WaitListHead != (LIST_ENTRY*)0 ||
+        self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
+        KeReleaseSpinlock(&gSchedLock, flags);
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    uint64_t now = gSchedStats.Ticks;
+    uint64_t wakeTick = now + ticks;
+    if (wakeTick < now)
+        wakeTick = UINT64_MAX;
+
+    self->WaitListHead = &gSleepList;
+    self->SleepUntilTick = wakeTick;
+    self->TimeoutStatus = timeoutStatus;
+    ListInsertTail(&gSleepList, &self->SchedulerLink);
+    KeReleaseSpinlock(&gSchedLock, flags);
+    return STATUS_SUCCESS;
+}
+
+// Implements the delay Thread operation.
+__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
+    if (milliseconds == 0) {
+        KeYield();
+        return STATUS_SUCCESS;
+    }
+
+    if (!KeCanWait())
+        return STATUS_INVALID_IRQL;
+
+    THREAD* self = KeGetCurrentThread();
+    if (!self || self->IsIdleThread)
+        return STATUS_INVALID_PARAMETER;
+
+    uint64_t ticks = milliseconds / SCHED_TICK_MS;
+    if ((milliseconds % SCHED_TICK_MS) != 0)
+        ticks++;
+    if (ticks == 0)
+        ticks = 1;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+
+    if (self->Queued || self->State != ThreadRunning ||
+        self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
+        KeReleaseSpinlock(&gSchedLock, flags);
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    uint64_t now = gSchedStats.Ticks;
+    uint64_t wakeTick = now + ticks;
+    if (wakeTick < now)
+        wakeTick = UINT64_MAX;
+
+    self->State = ThreadWaiting;
+    self->WaitObject = (OBJECT*)0;
+    self->WaitStatus = STATUS_PENDING;
+    self->WaitListHead = &gSleepList;
+    self->AlertableWait = 0;
+    self->SleepUntilTick = wakeTick;
+    self->TimeoutStatus = STATUS_SUCCESS;
+    ListInsertTail(&gSleepList, &self->SchedulerLink);
+
+    KeReleaseSpinlock(&gSchedLock, flags);
+    return KeCommitBlockThread(self, 0);
+}
+
 // Implements the yield operation.
 __declspec(dllexport) void KeYield(void) {
     KeAssertIrql(PASSIVE_LEVEL);