add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

code/kernel/hal/hal_processor.c

@@ -37,6 +37,6 @@ __declspec(dllexport) void HalHalt(void) {
 // Masks interrupts and permanently stops the current processor.
 __declspec(dllexport) _Noreturn void HalStopProcessor(void) {
     _disable();
-    for (;;)
+    while (true)
         __halt();
 }

code/kernel/include/kernel/io_dma.h

@@ -0,0 +1,13 @@
+// Kernel DMA declarations shared by the I/O manager and DDK provider.
+
+#pragma once
+
+#include <ntlite/ddk/dma.h>
+
+struct DEVICE_OBJECT;
+
+// Returns the kernel-owned DDK DMA table.
+const NTL_DDK_DMA* IoGetDmaDdk(void);
+
+// Detaches the adapter owned by a device before the device object can die.
+void IoDmaDetachDevice(struct DEVICE_OBJECT* device);

code/kernel/include/kernel/io_file.h

@@ -12,6 +12,16 @@
 
 struct PROCESS;
 
+// Initializes filesystem namespace change notification before drivers register volumes.
+KSTATUS IoInitializeVolumeNamespace(void);
+
+// Returns the generation of the mounted-volume namespace.
+uint64_t IoGetVolumeNamespaceGeneration(void);
+
+// Waits until the mounted-volume namespace changes or the timeout expires.
+KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration,
+                                       uint64_t timeoutMilliseconds);
+
 __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file,
                                                     void* buf,
                                                     size_t size,

code/kernel/include/kernel/ke_audio.h

@@ -0,0 +1,183 @@
+// Kernel audio-core declarations shared by the mixer, syscall layer, and selftests.
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <ntlite/ddk/audio.h>
+#include <kernel/ob_object.h>
+#include <kernel/ke_event.h>
+
+#define KE_AUDIO_MAX_ENDPOINTS             32u
+#define KE_AUDIO_MAX_STREAMS_PER_ENDPOINT  64u
+#define KE_AUDIO_STREAM_RING_FRAMES      4096u
+#define KE_AUDIO_OUTPUT_SAMPLE_RATE     48000u
+#define KE_AUDIO_OUTPUT_CHANNELS            2u
+#define KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD 512u
+#define KE_AUDIO_OUTPUT_PERIOD_COUNT        4u
+#define KE_AUDIO_OUTPUT_BYTES_PER_FRAME     4u
+#define KE_AUDIO_OUTPUT_PERIOD_BYTES     2048u
+#define KE_AUDIO_GAIN_RAMP_FRAMES          480u
+#define KE_AUDIO_IDLE_STOP_MILLISECONDS    250u
+#define KE_AUDIO_MIXER_REALTIME_PRIORITY    16u
+
+#define KE_AUDIO_WAIT_WRITABLE      0x00000001u
+#define KE_AUDIO_WAIT_CONSUMED      0x00000002u
+#define KE_AUDIO_WAIT_PRESENTED     0x00000004u
+#define KE_AUDIO_WAIT_STOPPED       0x00000008u
+#define KE_AUDIO_WAIT_DRAINED       0x00000010u
+#define KE_AUDIO_WAIT_FAILED        0x00000020u
+#define KE_AUDIO_WAIT_DISCONTINUITY 0x00000040u
+#define KE_AUDIO_WAIT_VALID_MASK    0x0000007Fu
+
+typedef enum KE_AUDIO_ENDPOINT_STATE {
+    KeAudioEndpointStarting = 1,
+    KeAudioEndpointReady,
+    KeAudioEndpointStopping,
+    KeAudioEndpointFailed,
+    KeAudioEndpointRemoved
+} KE_AUDIO_ENDPOINT_STATE;
+
+typedef enum KE_AUDIO_STREAM_STATE {
+    KeAudioStreamCreated = 1,
+    KeAudioStreamRunning,
+    KeAudioStreamPaused,
+    KeAudioStreamDraining,
+    KeAudioStreamStopped,
+    KeAudioStreamFailed,
+    KeAudioStreamClosed
+} KE_AUDIO_STREAM_STATE;
+
+typedef struct KE_AUDIO_STREAM KE_AUDIO_STREAM;
+
+// Reports one endpoint snapshot without exposing provider-owned pointers.
+typedef struct KE_AUDIO_ENDPOINT_INFO {
+    uint64_t EndpointId;
+    IO_AUDIO_ENDPOINT_KIND Kind;
+    KE_AUDIO_ENDPOINT_STATE State;
+    uint32_t Capabilities;
+    IO_AUDIO_FORMAT HardwareFormat;
+    uint32_t PeriodCount;
+    uint32_t FramesPerPeriod;
+    uint8_t Available;
+    uint8_t IsDefault;
+    uint8_t HardwareRunning;
+    uint8_t MasterMuted;
+    int32_t MasterHardwareGainMilliDb;
+    float MasterRequestedVolume;
+    uint32_t StreamCount;
+    uint32_t ActiveStreamCount;
+    uint64_t PeriodsMixed;
+    uint64_t PeriodsCompleted;
+    uint64_t FramesPresented;
+    uint64_t HardwarePositionFrames;
+    uint64_t InterruptCount;
+    uint64_t LastPresentationTimestampTicks;
+    uint64_t SilenceFramesInserted;
+    uint64_t Underruns;
+    uint64_t DiscontinuityGeneration;
+    KSTATUS FailureStatus;
+} KE_AUDIO_ENDPOINT_INFO;
+
+// Selects the next condition that should signal one stream's wait event.
+typedef struct KE_AUDIO_STREAM_WAIT_CONDITION {
+    uint32_t Mask;
+    uint32_t WritableFrames;
+    uint64_t ConsumedFrames;
+    uint64_t PresentedFrames;
+    uint64_t DrainedGeneration;
+    uint64_t DiscontinuityGeneration;
+} KE_AUDIO_STREAM_WAIT_CONDITION;
+
+// Reports one logical stream snapshot.
+typedef struct KE_AUDIO_STREAM_INFO {
+    KE_AUDIO_STREAM_STATE State;
+    IO_AUDIO_FORMAT Format;
+    uint32_t CapacityFrames;
+    uint32_t BufferedFrames;
+    uint32_t WritableFrames;
+    uint64_t FramesSubmitted;
+    uint64_t FramesConsumed;
+    uint64_t FramesPresented;
+    uint64_t FramesDiscarded;
+    uint64_t OutputFramesMixed;
+    uint64_t SilenceFramesInserted;
+    uint64_t Underruns;
+    uint64_t DrainedGeneration;
+    uint64_t DiscontinuityGeneration;
+    KSTATUS FailureStatus;
+    float Volume;
+    float Pan;
+    uint8_t Muted;
+} KE_AUDIO_STREAM_INFO;
+
+// Initializes endpoint registry and mixer ownership before drivers load.
+void KeAudioInitialize(void);
+
+// Returns the provider-facing audio DDK table.
+const NTL_DDK_AUDIO* KeAudioGetDdk(void);
+
+// Copies current endpoint snapshots and reports the full available count.
+KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints,
+                              uint32_t capacity,
+                              uint32_t* countOut);
+
+// Returns the currently selected endpoint identifier, or zero when none is usable.
+uint64_t KeAudioGetDefaultEndpointId(void);
+
+// Creates one fixed-format logical render stream and returns a caller-owned reference.
+KSTATUS KeAudioCreateStream(uint64_t endpointId,
+                            const IO_AUDIO_FORMAT* format,
+                            KE_AUDIO_STREAM** outStream);
+
+// Records one process handle that owns the stream.
+void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream);
+
+// Releases one process-handle owner without blocking the handle-table path.
+void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream);
+
+// Returns a borrowed passive event valid while the caller retains the stream.
+EVENT* KeAudioStreamWaitEvent(KE_AUDIO_STREAM* stream);
+
+// Arms the stream event for one or more progress conditions without a lost-wakeup window.
+KSTATUS KeAudioArmStreamWait(KE_AUDIO_STREAM* stream,
+                             const KE_AUDIO_STREAM_WAIT_CONDITION* condition);
+
+// Copies as many complete source frames as fit without blocking.
+KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream,
+                           const void* frames,
+                           uint32_t requestedFrames,
+                           uint32_t* writtenFramesOut);
+
+// Starts one created or stopped stream.
+KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream);
+
+// Pauses future mixing while preserving source frames not yet consumed.
+KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream);
+
+// Resumes one paused stream.
+KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream);
+
+// Rejects new writes and stops after every submitted frame is presented or discarded.
+KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream);
+
+// Stops future mixing and discards source frames not yet committed to endpoint periods.
+KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream);
+
+// Invalidates queued presentation accounting and returns an empty stream to position zero.
+KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream);
+
+// Updates per-stream software gain with a ten-millisecond ramp.
+KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream,
+                               float volume,
+                               float pan,
+                               bool muted);
+
+// Updates endpoint master software gain and separate logical mute state.
+KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted);
+
+// Copies one coherent stream snapshot.
+KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut);
+
+// Runs the final render path selftest behind KE_DEBUG_AUDIO_SELFTEST.
+void KiRunAudioFinalPathSelfTest(void);

code/kernel/include/kernel/ke_cpu.h

@@ -7,6 +7,7 @@
 // owns CPU-local scheduler, IRQL, GDT, and TSS state.
 
 #include <stdint.h>
+#include <stdbool.h>
 #include <kernel/ke_irql.h>
 #include <kernel/ke_idt.h>
 
@@ -83,6 +84,8 @@ extern KCPU gCpus[KE_MAX_CPUS];
 extern uint32_t gCpuCount;
 
 void KeInitializeBootstrapCpu(void);
+void KeInitializeMemoryTypes(void);
+bool KeWriteCombiningAvailable(void);
 void KeLoadKernelGsBase(KCPU* cpu);
 void KeSetUserGsBase(uintptr_t userGsBase);
 uintptr_t KeGetUserGsBase(void);

code/kernel/include/kernel/ke_debug.h

@@ -10,21 +10,6 @@
 #include <ntlite/ddi/hal_api.h>
 #include <ntlite/ddk/debug.h>
 
-#define KE_DEBUG_CONSOLE   0x00000001u
-#define KE_DEBUG_SELFTEST  0x00000002u
-#define KE_DEBUG_FILE_LOG  0x00000004u
-
-#define KE_DEBUG_POOL_CANARY      0x00000008u
-#define KE_DEBUG_POOL_POISON      0x00000010u
-#define KE_DEBUG_POOL_QUARANTINE  0x00000020u
-#define KE_DEBUG_POOL_CALLERS     0x00000040u
-#define KE_DEBUG_DISPLAY_ASSERTS  0x00000080u
-#define KE_DEBUG_DISPLAY_TRACE    0x00000100u
-#define KE_DEBUG_INPUT_TRACE      0x00000200u
-#define KE_DEBUG_VM_ASSERTS       0x00000400u
-#define KE_DEBUG_SYSCALL_ASSERTS  0x00000800u
-#define KE_DEBUG_LIST_CHECKS      0x00001000u
-
 void KeInitDebug(const HAL_API* hal, uint32_t flags);
 uint32_t KeGetDebugFlags(void);
 __declspec(dllexport) void DbgPrint(const char* fmt, ...);

code/kernel/include/kernel/ke_display.h

@@ -28,6 +28,9 @@ typedef struct DISPLAY_CLOCK_OBJECT {
 } DISPLAY_CLOCK_OBJECT;
 
 void KeDisplayInitialize(const HAL_API* hal);
+
+// Completes every console glyph queued before the call.
+KSTATUS KeDisplayFlushConsoleOutput(void);
 DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock);
 DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock);
 DISPLAY_API void KeDisplayTick(void);

code/kernel/include/kernel/ke_mutex.h

@@ -1,20 +1,37 @@
-// Kernel ke mutex declarations shared by implementation files and kernel clients.
+// Kernel mutex declarations shared by implementation files and kernel clients.
 
 #pragma once
-// ke_mutex.h - binary ownership mutex.
+
 #include <kernel/ob_object.h>
 #include <kernel/ke_spinlock.h>
 #include <kernel/ke_list.h>
+#include <ntlite/ddk/thread.h>
 #include <ntlite/status.h>
 
+struct THREAD;
+
 typedef struct MUTEX {
-    OBJECT       Header;
-    SPINLOCK     Lock;
-    struct THREAD* Owner;  // NULL if unowned
-    LIST_ENTRY   WaitList;
+    OBJECT Header;
+    SPINLOCK Lock;
+    struct THREAD* Owner;
+    LIST_ENTRY WaitList;
+    LIST_ENTRY OwnerLink;
+    KE_SCHEDULING_CLASS DonationClass;
+    uint32_t DonationPriority;
+    uint8_t HasDonation;
+    uint8_t Abandoned;
 } MUTEX;
 
-__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out);
-__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex);
-__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex);
-__declspec(dllexport) void    KeReleaseMutex(MUTEX* mutex);
+// Tests whether a mutex acquisition transferred ownership to the caller.
+#define KE_MUTEX_ACQUIRE_SUCCEEDED(status) \
+    ((status) == NTLSTATUS_SUCCESS || (status) == NTLSTATUS_ABANDONED)
+
+__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out);
+__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds);
+__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex);
+
+// Removes one blocked waiter while preserving priority-inheritance state.
+int KiMutexAbortWait(MUTEX* mutex, struct THREAD* thread);
+
+// Releases every mutex owned by a terminating thread as abandoned.
+void KiMutexRundownThread(struct THREAD* thread);

code/kernel/include/kernel/ke_process.h

@@ -30,6 +30,7 @@
 #define HANDLE_RIGHT_QUERY      0x00000020u
 #define HANDLE_RIGHT_TERMINATE  0x00000040u
 #define HANDLE_RIGHT_LOCK       0x00000080u
+#define HANDLE_RIGHT_RESUME     0x00000100u
 #define HANDLE_RIGHT_ALL        0xFFFFFFFFu
 
 struct SYS_EXCEPTION_STATS;
@@ -62,7 +63,8 @@ typedef struct PROCESS {
     uint8_t      CurrentDrive;
     SPINLOCK     CwdLock;
 
-    // Thread list.
+    // Live thread set. Dead THREAD objects may outlive this membership while
+    // retained handles remain waitable and queryable.
     LIST_ENTRY   ThreadList;  // links THREAD.ProcessLink
     uint32_t     ThreadCount;
 

code/kernel/include/kernel/ke_sched.h

@@ -1,18 +1,20 @@
 // Kernel ke sched declarations shared by implementation files and kernel clients.
 
 #pragma once
-// ke_sched.h - Multi-Level Feedback Queue scheduler
-// Three run queues (Low / Normal / High). LAPIC timer IRQs drive KeSchedulerTick.
+// The scheduler combines dynamic normal queues with static real-time queues.
+// LAPIC timer IRQs drive KeSchedulerTick.
 #include <kernel/ke_thread.h>
 
 struct KCPU;
 
 #define SCHED_TICK_MS                  10u
+#define SCHED_REALTIME_QUANTUM_TICKS    1u
 #define SCHED_HIGH_QUANTUM_TICKS        2u
 #define SCHED_NORMAL_QUANTUM_TICKS      4u
 #define SCHED_LOW_QUANTUM_TICKS         8u
 #define SCHED_DEMOTE_AFTER_BURST_TICKS 16u
 #define SCHED_AGING_INTERVAL_TICKS    100u
+#define KE_PRIORITY_INHERITANCE_MAX_DEPTH 64u
 
 typedef struct KE_SCHED_STATS {
     uint64_t Ticks;
@@ -27,8 +29,61 @@ typedef struct KE_SCHED_STATS {
     uint64_t CpuBurstDemotions;
     uint64_t SmtSiblingDeferrals;
     uint64_t SmtPrimaryPokes;
+    uint64_t RealtimeDispatches;
+    uint64_t RealtimePreemptions;
+    uint64_t PriorityInheritanceBoosts;
+    uint64_t PriorityInheritanceDrops;
+    uint64_t PriorityInheritancePropagations;
+    uint64_t PriorityInheritanceMaxDepth;
 } KE_SCHED_STATS;
 
+
+struct MUTEX;
+
+typedef struct KE_THREAD_SCHEDULER_SNAPSHOT {
+    THREAD_STATE State;
+    KE_SCHEDULING_CLASS BaseSchedulingClass;
+    uint32_t BasePriority;
+    KE_SCHEDULING_CLASS EffectiveSchedulingClass;
+    uint32_t EffectivePriority;
+    uint32_t RunningCpu;
+    uint32_t LastCpu;
+    uint32_t QuantumRemaining;
+    uint32_t QuantumTicks;
+    uint32_t CpuBurstTicks;
+    uint32_t LastRunTicks;
+    uint32_t WaitObjectType;
+    KSTATUS WaitStatus;
+    struct MUTEX* WaitingMutex;
+    uint8_t Queued;
+    uint8_t AlertableWait;
+} KE_THREAD_SCHEDULER_SNAPSHOT;
+
+// Copies scheduler-owned thread state into a stable diagnostic snapshot.
+void KiSchedulerSnapshotThread(THREAD* thread, KE_THREAD_SCHEDULER_SNAPSHOT* snapshot);
+
+// Selects waiters and refreshes donations while the mutex lock is held.
+struct THREAD* KiSchedulerSelectMutexWaiter(struct MUTEX* mutex);
+int KiSchedulerRefreshMutexDonation(struct MUTEX* mutex);
+
+// Updates one thread's normal-class base priority.
+KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority);
+
+// Publishes one mutex dependency atomically with cycle detection.
+KSTATUS KiSchedulerPrepareMutexWait(THREAD* thread, struct MUTEX* mutex, THREAD* owner);
+void KiSchedulerCancelMutexWait(THREAD* thread, struct MUTEX* mutex);
+
+// Associates mutex ownership with priority-inheritance bookkeeping.
+void KiSchedulerAcquireMutexOwnership(THREAD* thread, struct MUTEX* mutex);
+void KiSchedulerTransferMutexOwnership(THREAD* oldOwner, THREAD* newOwner,
+                                       struct MUTEX* mutex);
+void KiSchedulerReleaseMutexOwnership(THREAD* thread, struct MUTEX* mutex);
+struct MUTEX* KiSchedulerFirstOwnedMutex(THREAD* thread);
+void KiSchedulerNotePriorityPropagation(uint32_t depth);
+
+// Recomputes effective rank and optionally returns a referenced upstream mutex.
+int KiSchedulerRefreshEffectivePriority(THREAD* thread, struct MUTEX** waitingMutexOut);
+
 // Called once by KernelMain before any threads are created.
 void    KeInitializeScheduler(void);
 

code/kernel/include/kernel/ke_semaphore.h

@@ -17,6 +17,5 @@ typedef struct SEMAPHORE {
 } SEMAPHORE;
 
 __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out);
-__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore);
-__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* semaphore);
+__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore, uint64_t timeoutMilliseconds);
 __declspec(dllexport) void    KeSignalSemaphore(SEMAPHORE* semaphore);

code/kernel/include/kernel/ke_thread.h

@@ -18,6 +18,7 @@
 #define KE_USER_EXCEPTION_MAX_DEPTH 8u
 
 typedef struct THREAD THREAD;
+struct MUTEX;
 
 #ifndef NTL_THREAD_PRIORITY_DEFINED
 typedef enum THREAD_PRIORITY {
@@ -34,6 +35,7 @@ typedef enum THREAD_STATE {
     ThreadRunning = 1,
     ThreadWaiting = 2,
     ThreadDead    = 3,
+    ThreadSuspended = 4,
 } THREAD_STATE;
 
 typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread,
@@ -51,17 +53,20 @@ typedef struct KCONTEXT {
 struct THREAD {
     OBJECT          Header;  // must be first
 
-    uint32_t        ThreadId;
-    THREAD_STATE    State;
-    THREAD_PRIORITY Priority;
+    uint32_t            ThreadId;
+    THREAD_STATE        State;
+    KE_SCHEDULING_CLASS BaseSchedulingClass;
+    uint32_t            BasePriority;
+    KE_SCHEDULING_CLASS EffectiveSchedulingClass;
+    uint32_t            EffectivePriority;
 
     // CPU-burst mirror kept here for the assembly-stable THREAD layout.
     // Dynamic quantum state lives later in the structure.
-    uint32_t        QuantaConsumed;
+    uint32_t            QuantaConsumed;
 
     PROCESS*        Owner;
     LIST_ENTRY      ProcessLink;  // node in Owner->ThreadList
-    LIST_ENTRY      SchedulerLink;  // node in run queue or wait list
+    LIST_ENTRY      SchedulerLink;  // node in run queue or object wait list
 
     // Kernel stack - never PAGE_USER, pointed to by TssRsp0.
     void*           KernelStackBase;
@@ -108,6 +113,12 @@ struct THREAD {
     uint32_t CpuBurstTicks;
     uint32_t LastRunTicks;
     uint32_t RunningCpu;
+    uint32_t SuspendCount;
+
+    // Priority inheritance ownership is scheduler-owned. Mutex wait-list
+    // membership remains protected by the mutex spinlock.
+    LIST_ENTRY OwnedMutexList;
+    struct MUTEX* WaitingMutex;
 
     // Scheduler hardening state. Queued means membership in a scheduler ready
     // queue only; wait-object membership is tracked by WaitListHead. Idle
@@ -125,7 +136,9 @@ struct THREAD {
     KSTATUS WaitStatus;
     LIST_ENTRY* WaitListHead;
     uint8_t AlertableWait;
-    uint64_t SleepUntilTick;
+    LIST_ENTRY TimeoutLink;
+    uint8_t TimeoutArmed;
+    uint64_t TimeoutTick;
     KSTATUS TimeoutStatus;
 
     // Optional cleanup hook for waits that use auxiliary wait blocks.
@@ -167,6 +180,11 @@ struct THREAD {
     uint32_t   UserExceptionNoncontinuableMask;
     uintptr_t  UserExceptionFrameUserAddress;
     KSTATUS    UserExceptionExitStatus[KE_USER_EXCEPTION_MAX_DEPTH];
+
+    // Synchronous kernel work can temporarily retain caller-owned stack or
+    // process memory. Process termination must wait until that ownership has
+    // returned instead of freeing the thread underneath an active driver.
+    volatile long TerminationDeferralCount;
 };
 
 // Creates and queues a user thread. On success, *out receives a referenced
@@ -174,24 +192,31 @@ struct THREAD {
 // after copying any needed IDs/state. The live scheduler/thread lifetime is
 // separate and remains valid until the thread exits and is reaped.
 __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
-                       THREAD_PRIORITY priority, THREAD** out);
+                       const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out);
 
-// Create a kernel-mode-only thread. fn is called directly in ring 0 with
+// Cancels a thread that has never left its initial suspended state. This drops
+// the scheduler-owned lifetime reference but leaves the caller's reference
+// intact.
+KSTATUS KeCancelSuspendedThread(THREAD* thread);
+
+// Creates a kernel-mode-only thread. fn is called directly in ring 0 with
 // arg; when fn returns the thread exits automatically. No user stack or TEB
-// is allocated. The thread is owned by the system process and runs at the
-// given priority. On success, non-NULL out receives a referenced THREAD
-// pointer owned by the caller; release it with ObDereferenceObject() after
-// copying any needed IDs/state or after joining/waiting on caller-owned
-// completion state. The scheduler's live thread reference is separate.
+// is allocated. The thread is owned by the system process. On success,
+// non-NULL out receives a referenced THREAD pointer owned by the caller;
+// release it with ObDereferenceObject() after copying any needed IDs/state or
+// after joining caller-owned completion state. The scheduler's live thread
+// reference is separate.
 __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
-                              THREAD_PRIORITY priority, THREAD** out);
-__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
-                              THREAD_PRIORITY priority, uint64_t affinityMask,
+                              const KE_THREAD_CREATE_PARAMETERS* parameters,
                               THREAD** out);
 
 // Mark a thread dead and deschedule it permanently.
 __declspec(dllexport) void    KeExitThread(THREAD* thread, int code);
 
+// Publishes completion, releases per-thread user memory, and removes one dead
+// thread from its process live-thread set. Retained handles remain waitable.
+void    KeFinalizeThreadExit(THREAD* thread);
+
 // Block / unblock primitives used by sync objects.
 
 // Correct wait pattern:
@@ -213,12 +238,17 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
 KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
 void    KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status);
 void    KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status);
-void    KeAbortThreadWait(THREAD* thread, KSTATUS status);
+int     KeAbortThreadWait(THREAD* thread, KSTATUS status);
 void    KeUnblockThread(THREAD* thread);
 KSTATUS KeAlertThread(THREAD* thread);
 void    KiWakeThreadForKernelApc(THREAD* thread);
 void    KiWakeThreadForUserApc(THREAD* thread);
 
+// Defers process termination while kernel work owns caller-backed storage.
+void    KeEnterTerminationDeferral(void);
+void    KeLeaveTerminationDeferral(void);
+int     KeThreadTerminationDeferred(const THREAD* thread);
+
 // Assembly stub - implemented in ke_switch.asm.
 void    KeSwitchContext(THREAD* from, THREAD* to);
 __declspec(noreturn) void KeStartInitialThread(THREAD* thread);

code/kernel/include/kernel/ke_time.h

@@ -11,3 +11,6 @@ void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal);
 KSTATUS KeTimeReloadSettings(void);
 __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime);
 __declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void);
+
+// Returns the monotonic platform counter in microseconds.
+uint64_t KeQueryMonotonicTicks(void);

code/kernel/include/kernel/mm_file_cache.h

@@ -43,19 +43,6 @@ typedef struct MM_FILE_CACHE_STATS {
     uint64_t DropFailures;
     uint64_t StandbyTransitions;
     uint64_t StandbyHits;
-    uint64_t ReadAheadRequests;
-    uint64_t ReadAheadPagesRequested;
-    uint64_t ReadAheadPagesQueued;
-    uint64_t ReadAheadPagesLoaded;
-    uint64_t ReadAheadPagesSkipped;
-    uint64_t ReadAheadFailures;
-    uint64_t ReadAheadPagesThrottled;
-    uint64_t MaxReadAheadPagesPerRequest;
-    uint64_t BatchReads;
-    uint64_t BatchPagesRequested;
-    uint64_t BatchPagesLoaded;
-    uint64_t MaxBatchPagesRequested;
-    uint64_t MaxBatchPagesLoaded;
     uint64_t ResidentPages;
     uint64_t ActivePages;
     uint64_t StandbyPages;
@@ -92,8 +79,4 @@ void    MmFileCacheInvalidateAll(FILE_OBJECT* file);
 // it never writes back because v1 stores only clean file data.
 KSTATUS MmFileCacheDropOnePageForPressure(void);
 
-// Queues best-effort read-ahead for clean file pages. The queue owns a file
-// reference until the worker has consumed the request.
-KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages);
-
 void    MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out);

code/kernel/include/kernel/mm_pfn.h

@@ -124,6 +124,12 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min
                                       PHYSICAL_ADDRESS maxExclusive,
                                       MM_PFN_USAGE usage);
 __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage);
+PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count,
+                                                 PHYSICAL_ADDRESS minInclusive,
+                                                 PHYSICAL_ADDRESS maxExclusive,
+                                                 size_t alignment,
+                                                 size_t boundary,
+                                                 MM_PFN_USAGE usage);
 __declspec(dllexport) KSTATUS          MmPfnFree(PHYSICAL_ADDRESS frame);
 __declspec(dllexport) KSTATUS          MmPfnRetain(PHYSICAL_ADDRESS frame);
 __declspec(dllexport) KSTATUS          MmPfnRelease(PHYSICAL_ADDRESS frame);

code/kernel/include/kernel/mm_pte.h

@@ -12,10 +12,11 @@
 #define PAGE_PRESENT      (1ULL << 0)
 #define PAGE_WRITE        (1ULL << 1)
 #define PAGE_USER         (1ULL << 2)
-#ifndef PAGE_WRITETHROUGH
-#define PAGE_WRITETHROUGH (1ULL << 3)
-#endif
-#define PAGE_NOCACHE      (1ULL << 4)
+// Cache policy is selected through the canonical PAT layout.
+#define PAGE_CACHE_WB     0ULL
+#define PAGE_CACHE_WC     (1ULL << 3)
+#define PAGE_CACHE_UC     (1ULL << 4)
+#define PAGE_CACHE_MASK   (PAGE_CACHE_WC | PAGE_CACHE_UC)
 #define PAGE_ACCESSED     (1ULL << 5)
 #define PAGE_DIRTY        (1ULL << 6)
 #define PAGE_NX           (1ULL << 63)

code/kernel/include/kernel/mm_virt.h

@@ -66,6 +66,15 @@ typedef struct MM_PINNED_USER_PAGE {
 } MM_PINNED_USER_PAGE;
 
 typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
+typedef struct MM_KERNEL_VIEW MM_KERNEL_VIEW;
+
+// Maps existing frames into a stable kernel-only virtual range. The view owns
+// virtual address space and PTEs, but it does not own the physical frames.
+KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames,
+                                     size_t pageCount,
+                                     MM_KERNEL_VIEW** outView);
+void* MmKernelViewBase(const MM_KERNEL_VIEW* view);
+void MmDestroyKernelView(MM_KERNEL_VIEW* view);
 
 // Pins one user page and returns a kernel direct-map alias for the requested
 // address. Callers may copy through KernelAddress + PageOffset while the page is
@@ -80,6 +89,8 @@ __declspec(dllexport) void    MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
 // unlocked. WriteAccess describes kernel writes to the user buffer.
 __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
 __declspec(dllexport) void    MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
+size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer);
+PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index);
 
 // Virtual address descriptors.
 // The single shared address space still has per-process VAD ownership so process

code/kernel/include/kernel/syscall_nr.h

@@ -17,138 +17,154 @@
 #define SYS_GET_PROCESS_LIST         6
 #define SYS_GET_PROCESS_PID          7
 #define SYS_CREATE_THREAD            8
-#define SYS_GET_THREAD_LIST          9
-#define SYS_LOAD_MODULE              10
-#define SYS_UNLOAD_MODULE            11
-#define SYS_GET_MODULE_INFO          12
-#define SYS_GET_MODULE_EXPORT        13
-#define SYS_GET_MODULE_LIST          14
-#define SYS_GET_CURRENT_TID          15
-#define SYS_GET_CURRENT_PID          16
-#define SYS_SET_THREAD_PRIORITY      17
-#define SYS_YIELD                    18
-#define SYS_SLEEP                    19
-#define SYS_WAIT_SINGLE              20
-#define SYS_WAIT_MANY                21
-#define SYS_SET_USER_APC_DISPATCHER  22
-#define SYS_APC_RETURN               23
-#define SYS_SET_EXCEPTION_DISPATCHER 24
-#define SYS_GET_EXCEPTION_DISPATCHER 25
-#define SYS_CONTINUE_EXCEPTION       26
-#define SYS_RAISE_EXCEPTION          27
+#define SYS_RESUME_THREAD            9
+#define SYS_GET_THREAD_LIST          10
+#define SYS_LOAD_MODULE              11
+#define SYS_UNLOAD_MODULE            12
+#define SYS_GET_MODULE_INFO          13
+#define SYS_GET_MODULE_EXPORT        14
+#define SYS_GET_MODULE_LIST          15
+#define SYS_GET_CURRENT_TID          16
+#define SYS_GET_CURRENT_PID          17
+#define SYS_SET_THREAD_PRIORITY      18
+#define SYS_YIELD                    19
+#define SYS_SLEEP                    20
+#define SYS_WAIT_SINGLE              21
+#define SYS_WAIT_MANY                22
+#define SYS_SET_USER_APC_DISPATCHER  23
+#define SYS_APC_RETURN               24
+#define SYS_SET_EXCEPTION_DISPATCHER 25
+#define SYS_GET_EXCEPTION_DISPATCHER 26
+#define SYS_CONTINUE_EXCEPTION       27
+#define SYS_RAISE_EXCEPTION          28
 
 // Virtual memory, mappings, and handles.
-#define SYS_ALLOC_VM            28
-#define SYS_FREE_VM             29
-#define SYS_PROTECT_VM          30
-#define SYS_QUERY_VM            31
-#define SYS_CLOSE_HANDLE        32
-#define SYS_DUPLICATE_HANDLE    33
-#define SYS_CREATE_FILE_MAPPING 34
-#define SYS_MAP_FILE            35
+#define SYS_ALLOC_VM            29
+#define SYS_FREE_VM             30
+#define SYS_PROTECT_VM          31
+#define SYS_QUERY_VM            32
+#define SYS_CLOSE_HANDLE        33
+#define SYS_DUPLICATE_HANDLE    34
+#define SYS_CREATE_FILE_MAPPING 35
+#define SYS_MAP_FILE            36
 
 // Synchronization objects.
-#define SYS_CREATE_MUTEX      36
-#define SYS_TRY_ACQUIRE_MUTEX 37
-#define SYS_RELEASE_MUTEX     38
-#define SYS_CREATE_SEM        39
-#define SYS_TRY_WAIT_SEM      40
-#define SYS_SIGNAL_SEM        41
-#define SYS_CREATE_EVENT      42
-#define SYS_SIGNAL_EVENT      43
-#define SYS_CLEAR_EVENT       44
+#define SYS_CREATE_MUTEX      37
+#define SYS_TRY_ACQUIRE_MUTEX 38
+#define SYS_RELEASE_MUTEX     39
+#define SYS_CREATE_SEM        40
+#define SYS_TRY_WAIT_SEM      41
+#define SYS_SIGNAL_SEM        42
+#define SYS_CREATE_EVENT      43
+#define SYS_SIGNAL_EVENT      44
+#define SYS_CLEAR_EVENT       45
 
 // Queues and named queues.
-#define SYS_CREATE_QUEUE                   45
-#define SYS_SEND_QUEUE                     46
-#define SYS_RECV_QUEUE                     47
-#define SYS_PUBLISH_QUEUE                  48
-#define SYS_OPEN_NAMED_QUEUE               49
-#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 50
+#define SYS_CREATE_QUEUE                   46
+#define SYS_SEND_QUEUE                     47
+#define SYS_RECV_QUEUE                     48
+#define SYS_PUBLISH_QUEUE                  49
+#define SYS_OPEN_NAMED_QUEUE               50
+#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 51
 
 // Files, directories, and current directory.
-#define SYS_OPEN_FILE        51
-#define SYS_READ_FILE        52
-#define SYS_WRITE_FILE       53
-#define SYS_READ_FILE_ASYNC  54
-#define SYS_WRITE_FILE_ASYNC 55
-#define SYS_CANCEL_IO        56
-#define SYS_SEEK_FILE        57
-#define SYS_GET_FILE_POS     58
-#define SYS_GET_FILE_SIZE    59
-#define SYS_GET_FILE_STAT    60
-#define SYS_STAT_FILE        61
-#define SYS_TRUNCATE_FILE    62
-#define SYS_FLUSH_FILE       63
-#define SYS_DELETE_FILE      64
-#define SYS_RENAME_FILE      65
-#define SYS_LOCK_FILE        66
-#define SYS_UNLOCK_FILE      67
-#define SYS_OPEN_DIR         68
-#define SYS_READ_DIR         69
-#define SYS_CREATE_DIR       70
-#define SYS_REMOVE_DIR       71
-#define SYS_SET_CWD          72
-#define SYS_REAL_PATH        73
+#define SYS_OPEN_FILE        52
+#define SYS_READ_FILE        53
+#define SYS_WRITE_FILE       54
+#define SYS_READ_FILE_ASYNC  55
+#define SYS_WRITE_FILE_ASYNC 56
+#define SYS_CANCEL_IO        57
+#define SYS_SEEK_FILE        58
+#define SYS_GET_FILE_POS     59
+#define SYS_GET_FILE_SIZE    60
+#define SYS_GET_FILE_STAT    61
+#define SYS_STAT_FILE        62
+#define SYS_TRUNCATE_FILE    63
+#define SYS_FLUSH_FILE       64
+#define SYS_DELETE_FILE      65
+#define SYS_RENAME_FILE      66
+#define SYS_LOCK_FILE        67
+#define SYS_UNLOCK_FILE      68
+#define SYS_OPEN_DIR         69
+#define SYS_READ_DIR         70
+#define SYS_CREATE_DIR       71
+#define SYS_REMOVE_DIR       72
+#define SYS_SET_CWD          73
+#define SYS_REAL_PATH        74
 
 // Console, display, and surfaces.
-#define SYS_WRITE_CONSOLE            74
-#define SYS_GET_CONSOLE_INFO         75
-#define SYS_SET_CONSOLE_CURSOR       76
-#define SYS_FILL_CONSOLE_CHARACTER   77
-#define SYS_FILL_CONSOLE_ATTRIBUTE   78
-#define SYS_PUT_SCREEN_CHAR          79
-#define SYS_CLEAR_SCREEN             80
-#define SYS_GET_SCREEN_SIZE          81
-#define SYS_QUERY_DISPLAY_INFO       82
-#define SYS_OPEN_DISPLAY_CLOCK       83
-#define SYS_CREATE_SURFACE           84
-#define SYS_MAP_SURFACE              85
-#define SYS_QUERY_SURFACE            86
-#define SYS_PRESENT_SURFACE          87
+#define SYS_WRITE_CONSOLE            75
+#define SYS_GET_CONSOLE_INFO         76
+#define SYS_SET_CONSOLE_CURSOR       77
+#define SYS_FILL_CONSOLE_CHARACTER   78
+#define SYS_FILL_CONSOLE_ATTRIBUTE   79
+#define SYS_PUT_SCREEN_CHAR          80
+#define SYS_CLEAR_SCREEN             81
+#define SYS_GET_SCREEN_SIZE          82
+#define SYS_QUERY_DISPLAY_INFO       83
+#define SYS_OPEN_DISPLAY_CLOCK       84
+#define SYS_CREATE_SURFACE           85
+#define SYS_MAP_SURFACE              86
+#define SYS_QUERY_SURFACE            87
+#define SYS_PRESENT_SURFACE          88
+
+// Native render audio.
+#define SYS_AUDIO_ENUMERATE_ENDPOINTS 89
+#define SYS_AUDIO_OPEN_STREAM         90
+#define SYS_AUDIO_WRITE               91
+#define SYS_AUDIO_ARM_WAIT            92
+#define SYS_AUDIO_QUERY_STREAM        93
+#define SYS_AUDIO_START               94
+#define SYS_AUDIO_PAUSE               95
+#define SYS_AUDIO_RESUME              96
+#define SYS_AUDIO_DRAIN               97
+#define SYS_AUDIO_STOP                98
+#define SYS_AUDIO_RESET               99
+#define SYS_AUDIO_SET_VOLUME          100
+#define SYS_AUDIO_SET_MASTER_VOLUME   101
 
 // Input.
-#define SYS_POLL_KEYBOARD            88
-#define SYS_IS_KEY_DOWN              89
-#define SYS_READ_KEYBOARD_CHAR       90
-#define SYS_READ_KEY                 91
-#define SYS_GET_KEY_MODIFIERS        92
-#define SYS_QUERY_GLOBAL_INPUT_STATE 93
-#define SYS_OPEN_INPUT_QUEUE         94
-#define SYS_READ_INPUT_EVENT         95
-#define SYS_PEEK_INPUT_EVENT         96
-#define SYS_PEEK_MOUSE               97
-#define SYS_READ_MOUSE               98
-#define SYS_GET_MOUSE_POS            99
+#define SYS_POLL_KEYBOARD            102
+#define SYS_IS_KEY_DOWN              103
+#define SYS_READ_KEYBOARD_CHAR       104
+#define SYS_READ_KEY                 105
+#define SYS_GET_KEY_MODIFIERS        106
+#define SYS_QUERY_GLOBAL_INPUT_STATE 107
+#define SYS_OPEN_INPUT_QUEUE         108
+#define SYS_READ_INPUT_EVENT         109
+#define SYS_PEEK_INPUT_EVENT         110
+#define SYS_PEEK_MOUSE               111
+#define SYS_READ_MOUSE               112
+#define SYS_GET_MOUSE_POS            113
 
 // Time, debug, and system information.
-#define SYS_CREATE_TIMER             100
-#define SYS_SET_TIMER                101
-#define SYS_CANCEL_TIMER             102
-#define SYS_GET_TICK                 103
-#define SYS_GET_TICK_FREQUENCY       104
-#define SYS_GET_SYSTEM_TIME          105
-#define SYS_PRINT_DEBUG              106
-#define SYS_QUERY_TRAP_FRAME         107
-#define SYS_GET_MEMORY_STATUS        108
-#define SYS_QUERY_KERNEL_STATS       109
+#define SYS_CREATE_TIMER             114
+#define SYS_SET_TIMER                115
+#define SYS_CANCEL_TIMER             116
+#define SYS_GET_TICK                 117
+#define SYS_GET_TICK_FREQUENCY       118
+#define SYS_GET_SYSTEM_TIME          119
+#define SYS_PRINT_DEBUG              120
+#define SYS_QUERY_TRAP_FRAME         121
+#define SYS_GET_MEMORY_STATUS        122
+#define SYS_QUERY_KERNEL_STATS       123
 
 // Native sockets and polling.
-#define SYS_CREATE_SOCKET            110
-#define SYS_SOCKET_IO_ASYNC          111
-#define SYS_BIND_SOCKET              112
-#define SYS_LISTEN_SOCKET            113
-#define SYS_ACCEPT_SOCKET            114
-#define SYS_CONNECT_SOCKET           115
-#define SYS_SEND_SOCKET              116
-#define SYS_RECV_SOCKET              117
-#define SYS_SHUTDOWN_SOCKET          118
-#define SYS_SEND_TO_SOCKET           119
-#define SYS_RECV_FROM_SOCKET         120
-#define SYS_RESOLVE_HOST             121
-#define SYS_SET_SOCKET_OPTION        122
-#define SYS_GET_SOCKET_OPTION        123
-#define SYS_POLL                     124
-#define SYS_UNPUBLISH_QUEUE          125
+#define SYS_CREATE_SOCKET            124
+#define SYS_SOCKET_IO_ASYNC          125
+#define SYS_BIND_SOCKET              126
+#define SYS_LISTEN_SOCKET            127
+#define SYS_ACCEPT_SOCKET            128
+#define SYS_CONNECT_SOCKET           129
+#define SYS_SEND_SOCKET              130
+#define SYS_RECV_SOCKET              131
+#define SYS_SHUTDOWN_SOCKET          132
+#define SYS_SEND_TO_SOCKET           133
+#define SYS_RECV_FROM_SOCKET         134
+#define SYS_RESOLVE_HOST             135
+#define SYS_SET_SOCKET_OPTION        136
+#define SYS_GET_SOCKET_OPTION        137
+#define SYS_POLL                     138
+#define SYS_UNPUBLISH_QUEUE          139
 
-#define SYS_MAX 126
+#define SYS_MAX 140

code/kernel/io_ddk.c

@@ -5,8 +5,12 @@
 #include <ntlite/ddi/hal_api.h>
 #include <kernel/hal/hal_apic.h>
 #include <kernel/io_ddk.h>
+#include <kernel/io_dma.h>
+#include <kernel/ke_audio.h>
+#include <kernel/ke_debug.h>
 #include <kernel/ke_irq.h>
 #include <kernel/ke_sched.h>
+#include <kernel/mm_pte.h>
 
 static const HAL_API* gIoDdkHal;
 
@@ -68,6 +72,7 @@ static KSTATUS IoDdkAllocateMsiVector(uint8_t* outIrq) {
 
 static const NTL_DDK_DEBUG gIoDdkDebug = {
     .Print = DbgPrint,
+    .GetFlags = KeGetDebugFlags,
 };
 
 static const NTL_DDK_INTERRUPT gIoDdkInterrupt = {
@@ -99,6 +104,8 @@ static NTL_DDK_TIMER gIoDdkTimer = {
 static NTL_DDK gIoDdk = {
     .Version = NTL_DDK_VERSION,
     .Debug = &gIoDdkDebug,
+    .Audio = (const NTL_DDK_AUDIO*)0,
+    .Dma = (const NTL_DDK_DMA*)0,
     .Interrupt = &gIoDdkInterrupt,
     .Memory = &gIoDdkMemory,
     .Platform = &gIoDdkPlatform,
@@ -109,6 +116,8 @@ static NTL_DDK gIoDdk = {
 // Initializes the kernel-owned DDK table from platform services.
 void IoInitializeDdk(const HAL_API* hal) {
     gIoDdkHal = hal;
+    gIoDdk.Audio = KeAudioGetDdk();
+    gIoDdk.Dma = IoGetDmaDdk();
     gIoDdkTimer.QueryTicks = hal ? hal->Timer.QueryTicks : 0;
     gIoDdkTimer.Stall = hal ? hal->Timer.Stall : 0;
 }

code/kernel/io_device.c

@@ -4,6 +4,7 @@
 #include <intrin.h>
 #include <ntlite/ddi/io_driver.h>
 #include <kernel/ke_irp.h>
+#include <kernel/io_dma.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ke_debug.h>
 #include <kernel/ob_object.h>
@@ -107,6 +108,7 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
 
 static void IoDeviceDestroy(OBJECT* object) {
     DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
+    IoDmaDetachDevice(dev);
 
     for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
         if (gPhysicalDevices[i] == dev) {
@@ -316,12 +318,19 @@ static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
 
 // Releases device.
 __declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
-    if (!device) return;
-    if (device->DeletePending)
+    if (!device)
         return;
 
+    uintptr_t flags = KeAcquireSpinlock(&device->Lock);
+    if (device->DeletePending) {
+        KeReleaseSpinlock(&device->Lock, flags);
+        return;
+    }
     device->DeletePending = true;
     device->PnpState = DevicePnpDeleted;
+    KeReleaseSpinlock(&device->Lock, flags);
+
+    IoDmaDetachDevice(device);
     IoUnlinkDeviceFromParent(device);
     IoRemovePhysicalDeviceRegistryEntry(device);
     IoDereferenceDeviceObject(device);
@@ -441,167 +450,50 @@ static uint32_t gDiskChangeCallbackCount;
 static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
 static uint32_t gDiskArrivalCallbackCount;
 
-#define IO_DISK_READAHEAD_MAX_SECTORS 128u
-#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u
-#define IO_DISK_READAHEAD_MIN_SECTORS 8u
 #define IO_DISK_SECTOR_BYTES 512u
 
-typedef struct IO_DISK_READAHEAD_STATE {
+typedef struct IO_DISK_STATS_STATE {
     SPINLOCK Lock;
-    uint8_t* Buffer;
-    uint8_t  Valid;
-    uint8_t  Loading;
-    uint64_t Generation;
-    uint64_t BaseLba;
-    uint32_t Count;
-    uint64_t NextLba;
-    uint32_t SequentialScore;
     IO_DISK_STATS Stats;
-} IO_DISK_READAHEAD_STATE;
+} IO_DISK_STATS_STATE;
 
-static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES];
+static IO_DISK_STATS_STATE gDiskStats[MAX_DISK_DEVICES];
 
-// Updates a maximum statistic while holding the disk read-ahead lock.
-static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) {
-    if (field && *field < value) *field = value;
-}
-
-// Prepares the per-disk read-ahead buffer.
-static void IoDiskReadAheadPrepare(uint8_t driveIndex) {
+// Initializes per-disk I/O statistics.
+static void IoDiskStatsPrepare(uint8_t driveIndex) {
     if (driveIndex >= MAX_DISK_DEVICES)
         return;
 
-    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
     state->Lock = (SPINLOCK)SPINLOCK_INIT;
-    state->Valid = 0;
-    state->Loading = 0;
-    state->Generation = 0;
-    state->BaseLba = 0;
-    state->Count = 0;
-    state->NextLba = 0;
-    state->SequentialScore = 0;
     memset(&state->Stats, 0, sizeof(state->Stats));
-
-    if (!state->Buffer) {
-        state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES,
-                                                    MM_POOL_TAG('d', 'r', 'a', 'h'));
-        if (!state->Buffer)
-            state->Stats.ReadaheadAllocFailures++;
-    }
 }
 
-// Releases the per-disk read-ahead buffer.
-static void IoDiskReadAheadRelease(uint8_t driveIndex) {
+// Records one disk read request.
+static void IoDiskStatsNoteRead(uint8_t driveIndex, uint32_t count) {
     if (driveIndex >= MAX_DISK_DEVICES)
         return;
 
-    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
-    uint8_t* buffer = (uint8_t*)0;
-    for (;;) {
-        uintptr_t flags = KeAcquireSpinlock(&state->Lock);
-        if (!state->Loading) {
-            buffer = state->Buffer;
-            state->Buffer = (uint8_t*)0;
-            state->Valid = 0;
-            state->Generation++;
-            state->Count = 0;
-            state->SequentialScore = 0;
-            KeReleaseSpinlock(&state->Lock, flags);
-            break;
-        }
-        KeReleaseSpinlock(&state->Lock, flags);
-        KeYield();
-    }
-
-    if (buffer)
-        MmFreePool(buffer);
-}
-
-// Invalidates cached disk read-ahead data for one disk.
-static void IoDiskReadAheadInvalidate(uint8_t driveIndex) {
-    if (driveIndex >= MAX_DISK_DEVICES)
-        return;
-
-    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
-    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
-    state->Valid = 0;
-    state->Generation++;
-    state->Count = 0;
-    state->Stats.ReadaheadInvalidations++;
-    KeReleaseSpinlock(&state->Lock, flags);
-}
-
-// Returns true when one range fully contains another range.
-static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount,
-                                uint64_t innerBase, uint32_t innerCount) {
-    uint64_t outerEnd = outerBase + outerCount;
-    uint64_t innerEnd = innerBase + innerCount;
-    if (outerEnd < outerBase || innerEnd < innerBase)
-        return false;
-    return innerBase >= outerBase && innerEnd <= outerEnd;
-}
-
-// Attempts to satisfy a disk read from the per-disk read-ahead buffer.
-static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) {
-    if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0)
-        return false;
-
-    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
-    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
-    bool hit = state->Buffer && state->Valid && !state->Loading &&
-               IoDiskRangeContains(state->BaseLba, state->Count, lba, count);
-    if (hit) {
-        size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES;
-        size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES;
-        memcpy(buffer, state->Buffer + offset, bytes);
-        state->Stats.ReadaheadHits++;
-        state->Stats.ReadaheadBytesServed += bytes;
-    } else {
-        state->Stats.ReadaheadMisses++;
-    }
-    KeReleaseSpinlock(&state->Lock, flags);
-    return hit;
-}
-
-// Records request size and sequential access state for one disk read.
-static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) {
-    if (driveIndex >= MAX_DISK_DEVICES || count == 0)
-        return 0;
-
-    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
     uintptr_t flags = KeAcquireSpinlock(&state->Lock);
     state->Stats.ReadRequests++;
     state->Stats.ReadSectors += count;
-    IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count);
-
-    if (lba == state->NextLba) {
-        if (state->SequentialScore < 32)
-            state->SequentialScore++;
-    } else {
-        state->SequentialScore = 1;
-    }
-    state->NextLba = lba + count;
-    uint32_t score = state->SequentialScore;
-    if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE)
-        state->Stats.ReadaheadSequentialDetections++;
+    if (state->Stats.MaxReadSectors < count)
+        state->Stats.MaxReadSectors = count;
     KeReleaseSpinlock(&state->Lock, flags);
-    return score;
 }
 
-// Records one disk write and invalidates stale read-ahead data.
-static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) {
+// Records one successful disk write request.
+static void IoDiskStatsNoteWrite(uint8_t driveIndex, uint32_t count) {
     if (driveIndex >= MAX_DISK_DEVICES)
         return;
 
-    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
+    IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
     uintptr_t flags = KeAcquireSpinlock(&state->Lock);
     state->Stats.WriteRequests++;
     state->Stats.WriteSectors += count;
-    IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count);
-    state->Valid = 0;
-    state->Generation++;
-    state->Count = 0;
-    state->Stats.ReadaheadInvalidations++;
+    if (state->Stats.MaxWriteSectors < count)
+        state->Stats.MaxWriteSectors = count;
     KeReleaseSpinlock(&state->Lock, flags);
 }
 
@@ -653,7 +545,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
     if (pnpStatus == NTLSTATUS_NOT_SUPPORTED)
         device->PnpState = DevicePnpStarted;
 
-    IoDiskReadAheadPrepare(driveIndex);
+    IoDiskStatsPrepare(driveIndex);
 
     IoReferenceDeviceObject(device);
     gDiskDevices[driveIndex] = device;
@@ -668,8 +560,6 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
     DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
     if (!dev) return NTLSTATUS_NOT_FOUND;
 
-    IoDiskReadAheadInvalidate(driveIndex);
-
     dev->PnpState = DevicePnpSurpriseRemoved;
     IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
     (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
@@ -698,7 +588,6 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
     (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
 
     gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
-    IoDiskReadAheadRelease(driveIndex);
     IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
     IoDereferenceDeviceObject(dev);
     return NTLSTATUS_SUCCESS;
@@ -846,83 +735,6 @@ static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
     return IoSubmitIrpSync(dev, &irp);
 }
 
-// Chooses a conservative read-ahead span from the current sequential score.
-static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) {
-    uint32_t sectors = currentCount;
-    if (sectors < IO_DISK_READAHEAD_MIN_SECTORS)
-        sectors = IO_DISK_READAHEAD_MIN_SECTORS;
-    if (score >= 8 && sectors < 128u)
-        sectors = 128u;
-    else if (score >= 4 && sectors < 64u)
-        sectors = 64u;
-    else if (score >= 2 && sectors < 32u)
-        sectors = 32u;
-    if (sectors > IO_DISK_READAHEAD_MAX_SECTORS)
-        sectors = IO_DISK_READAHEAD_MAX_SECTORS;
-    return sectors;
-}
-
-// Loads the next sequential disk range into the read-ahead buffer.
-static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev,
-                                 uint64_t nextLba, uint32_t currentCount, uint32_t score) {
-    if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE)
-        return;
-
-    IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
-    if (!state->Buffer) {
-        uintptr_t flags = KeAcquireSpinlock(&state->Lock);
-        state->Stats.ReadaheadSuppressed++;
-        KeReleaseSpinlock(&state->Lock, flags);
-        return;
-    }
-
-    uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score);
-    uint64_t diskSectors = dev->u.Disk.SectorCount;
-    if (nextLba >= diskSectors)
-        return;
-    if ((uint64_t)sectors > diskSectors - nextLba)
-        sectors = (uint32_t)(diskSectors - nextLba);
-    if (sectors == 0)
-        return;
-
-    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
-    bool covered = state->Valid && !state->Loading &&
-                   IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors);
-    if (covered) {
-        KeReleaseSpinlock(&state->Lock, flags);
-        return;
-    }
-    if (state->Loading) {
-        state->Stats.ReadaheadSuppressed++;
-        KeReleaseSpinlock(&state->Lock, flags);
-        return;
-    }
-    uint64_t generation = state->Generation;
-    state->Loading = 1;
-    state->Valid = 0;
-    state->Count = 0;
-    KeReleaseSpinlock(&state->Lock, flags);
-
-    KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer);
-
-    flags = KeAcquireSpinlock(&state->Lock);
-    state->Loading = 0;
-    if (KSUCCESS(st) && generation == state->Generation) {
-        state->BaseLba = nextLba;
-        state->Count = sectors;
-        state->Valid = 1;
-        state->Stats.ReadaheadStores++;
-        state->Stats.ReadaheadSectorsLoaded += sectors;
-        IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors);
-    } else {
-        state->Valid = 0;
-        state->Count = 0;
-        if (!KSUCCESS(st))
-            state->Stats.ReadaheadReadFailures++;
-    }
-    KeReleaseSpinlock(&state->Lock, flags);
-}
-
 __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
                                                 uint64_t lba,
                                                 uint32_t count,
@@ -938,17 +750,8 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
         return rangeStatus;
     }
 
-    if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) {
-        uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
-        IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
-        IoDereferenceDeviceObject(dev);
-        return NTLSTATUS_SUCCESS;
-    }
-
     KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
-    uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
-    if (KSUCCESS(st))
-        IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
+    IoDiskStatsNoteRead(driveIndex, count);
 
     IoDereferenceDeviceObject(dev);
     return st;
@@ -971,19 +774,39 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
 
     KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
     if (KSUCCESS(st))
-        IoDiskReadAheadNoteWrite(driveIndex, count);
+        IoDiskStatsNoteWrite(driveIndex, count);
 
     IoDereferenceDeviceObject(dev);
     return st;
 }
 
+// Flushes volatile write caches for one disk device.
+__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex) {
+    DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
+    if (!dev) return NTLSTATUS_NOT_FOUND;
+
+    IRP irp;
+    IoInitializeIrp(&irp, 1);
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(&irp);
+    if (!ios) {
+        IoDereferenceDeviceObject(dev);
+        return NTLSTATUS_FAILURE;
+    }
+    ios->MajorFunction = IRP_MJ_FLUSH_BUFFERS;
+    ios->MinorFunction = 0;
+
+    KSTATUS st = IoSubmitIrpSync(dev, &irp);
+    IoDereferenceDeviceObject(dev);
+    return st;
+}
+
 // Queries aggregate disk I/O statistics.
 __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
     if (!out) return;
     memset(out, 0, sizeof(*out));
 
     for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
-        IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i];
+        IO_DISK_STATS_STATE* state = &gDiskStats[i];
         uintptr_t flags = KeAcquireSpinlock(&state->Lock);
         if (gDiskDevices[i])
             out->RegisteredDisks++;
@@ -995,18 +818,6 @@ __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
             out->MaxReadSectors = state->Stats.MaxReadSectors;
         if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
             out->MaxWriteSectors = state->Stats.MaxWriteSectors;
-        out->ReadaheadHits += state->Stats.ReadaheadHits;
-        out->ReadaheadMisses += state->Stats.ReadaheadMisses;
-        out->ReadaheadStores += state->Stats.ReadaheadStores;
-        out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations;
-        out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed;
-        out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded;
-        out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections;
-        out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed;
-        out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures;
-        out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures;
-        if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors)
-            out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors;
         KeReleaseSpinlock(&state->Lock, flags);
     }
 }

code/kernel/io_file.c

@@ -26,6 +26,47 @@ typedef struct {
 
 static VOLUME_ENTRY gVolumes[MAX_DRIVES];
 
+static EVENT* gVolumeChangeEvent;
+static SPINLOCK gVolumeNamespaceLock = SPINLOCK_INIT;
+static uint64_t gVolumeNamespaceGeneration;
+
+// Publishes a mounted-volume namespace change to boot discovery and other waiters.
+static void KiIoNotifyVolumeNamespaceChange(void) {
+    uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock);
+    gVolumeNamespaceGeneration++;
+    KeReleaseSpinlock(&gVolumeNamespaceLock, flags);
+    if (gVolumeChangeEvent)
+        KeSignalEvent(gVolumeChangeEvent);
+}
+
+// Initializes the waitable mounted-volume namespace notification.
+KSTATUS IoInitializeVolumeNamespace(void) {
+    if (gVolumeChangeEvent)
+        return NTLSTATUS_SUCCESS;
+    return KeEventCreate(EventAutoReset, 0, &gVolumeChangeEvent);
+}
+
+// Returns the current mounted-volume namespace generation.
+uint64_t IoGetVolumeNamespaceGeneration(void) {
+    uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock);
+    uint64_t generation = gVolumeNamespaceGeneration;
+    KeReleaseSpinlock(&gVolumeNamespaceLock, flags);
+    return generation;
+}
+
+// Waits for a mounted-volume namespace change without polling driver discovery.
+KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration,
+                                       uint64_t timeoutMilliseconds) {
+    if (IoGetVolumeNamespaceGeneration() != observedGeneration)
+        return NTLSTATUS_SUCCESS;
+    if (!gVolumeChangeEvent)
+        return NTLSTATUS_OBJECT_DEAD;
+
+    EVENT* events[1] = { gVolumeChangeEvent };
+    uint32_t selected = 0;
+    return KeWaitForMultipleEvents(events, 1, 0, timeoutMilliseconds, &selected);
+}
+
 // Async file I/O worker.
 typedef enum IO_ASYNC_FILE_OP {
     IoAsyncFileRead  = 0,
@@ -275,8 +316,8 @@ static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
 static void KiIoAsyncFileWorker(void* arg) {
     (void)arg;
 
-    for (;;) {
-        KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore);
+    while (true) {
+        KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore, KE_WAIT_INFINITE);
         if (!KSUCCESS(s)) {
             KeYield();
             continue;
@@ -335,9 +376,10 @@ static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
     KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore);
     if (KSUCCESS(s)) {
         THREAD* worker = (THREAD*)0;
+        const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
         s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker,
                                  0,
-                                 PriorityNormal,
+                                 &parameters,
                                  &worker);
         // Keep the creator reference: this worker is intentionally persistent
         // for the lifetime of the kernel.
@@ -751,6 +793,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
         return NTLSTATUS_ALREADY_EXISTS;
     gVolumes[volumeIndex].Fs  = fs;
     gVolumes[volumeIndex].Vol = *vol;
+    KiIoNotifyVolumeNamespaceChange();
     return NTLSTATUS_SUCCESS;
 }
 
@@ -762,18 +805,23 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
         return NTLSTATUS_NOT_FOUND;
 
     memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex]));
+    KiIoNotifyVolumeNamespaceChange();
     return NTLSTATUS_SUCCESS;
 }
 
 // Implements the unregister Filesystems For Disk operation.
 __declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
     KSTATUS last = NTLSTATUS_NOT_FOUND;
+    bool changed = false;
     for (uint8_t i = 0; i < MAX_DRIVES; i++) {
         if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) {
             memset(&gVolumes[i], 0, sizeof(gVolumes[i]));
             last = NTLSTATUS_SUCCESS;
+            changed = true;
         }
     }
+    if (changed)
+        KiIoNotifyVolumeNamespaceChange();
     return last;
 }
 
@@ -819,7 +867,11 @@ __declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
 static void KiFileDestroy(OBJECT* obj) {
     FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
     if (f->Fs && f->FileCtx) {
-        (void)IoFlushFile(f);
+        // Close drains kernel-owned dirty pages, but stable-media semantics
+        // remain explicit through IoFlushFile so every close does not issue a
+        // whole-device cache flush.
+        (void)MmVadSyncFileMappings(f);
+        (void)MmFileCacheFlushFile(f);
         MmFileCacheInvalidateAll(f);
         f->Fs->Close(f->Fs, f->FileCtx);
     }
@@ -853,71 +905,11 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
     f->Size    = fs->GetSize(fs, fileCtx);
     f->Offset  = 0;
     f->Flags   = 0;
-    f->ReadaheadScore = 0;
-    f->ReadaheadNextOffset = 0;
-    f->ReadaheadIssuedUntil = 0;
-    f->ReadaheadSuppressedUntil = 0;
 
     *out = f;
     return NTLSTATUS_SUCCESS;
 }
 
-static uint64_t KiAlignUpPage64(uint64_t value) {
-    return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
-}
-
-static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) {
-    if (!file || bytesRead < 512)
-        return;
-    if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0)
-        return;
-
-    uint64_t end = offset + bytesRead;
-    uint64_t start = 0;
-    uint32_t pages = 0;
-
-    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
-    if (offset == file->ReadaheadNextOffset) {
-        if (file->ReadaheadScore < 32)
-            file->ReadaheadScore++;
-    } else {
-        file->ReadaheadScore = 1;
-        file->ReadaheadIssuedUntil = end;
-    }
-    file->ReadaheadNextOffset = end;
-
-    if (end < file->ReadaheadSuppressedUntil) {
-        pages = 0;
-    } else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) {
-        pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u);
-        start = KiAlignUpPage64(end);
-        if (start < file->ReadaheadIssuedUntil)
-            start = KiAlignUpPage64(file->ReadaheadIssuedUntil);
-        if (start >= fileSize) {
-            pages = 0;
-        } else {
-            uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE);
-            if (maxEnd > fileSize)
-                pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE);
-            file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE);
-        }
-    }
-    KeReleaseSpinlock(&file->Lock, flags);
-
-    if (pages) {
-        KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages);
-        if (ras == NTLSTATUS_QUEUE_FULL) {
-            flags = KeAcquireSpinlock(&file->Lock);
-            uint64_t suppressUntil = end + (1024u * 1024u);
-            if (suppressUntil < end || suppressUntil > fileSize)
-                suppressUntil = fileSize;
-            if (file->ReadaheadSuppressedUntil < suppressUntil)
-                file->ReadaheadSuppressedUntil = suppressUntil;
-            KeReleaseSpinlock(&file->Lock, flags);
-        }
-    }
-}
-
 // Explicit-offset file I/O. Async I/O is built on these routines so it does
 // not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
 __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -979,7 +971,6 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
     }
 
     *bytesRead = total;
-    KiIoMaybeScheduleReadAhead(file, offset, total, fileSize);
     return NTLSTATUS_SUCCESS;
 }
 
@@ -1022,8 +1013,13 @@ __declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) {
     if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
 
     KSTATUS sync = MmVadSyncFileMappings(file);
-    KSTATUS flush = MmFileCacheFlushFile(file);
-    return KSUCCESS(sync) ? flush : sync;
+    KSTATUS cache = MmFileCacheFlushFile(file);
+    KSTATUS media = NTLSTATUS_SUCCESS;
+    if (KSUCCESS(sync) && KSUCCESS(cache) && file->Fs && file->Fs->Flush)
+        media = file->Fs->Flush(file->Fs, file->FileCtx);
+    if (!KSUCCESS(sync)) return sync;
+    if (!KSUCCESS(cache)) return cache;
+    return media;
 }
 
 // IoReadFile - sequential read at current offset.
@@ -1091,10 +1087,6 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o
     f->Size    = 0;
     f->Offset  = 0;
     f->Flags   = 0;
-    f->ReadaheadScore = 0;
-    f->ReadaheadNextOffset = 0;
-    f->ReadaheadIssuedUntil = 0;
-    f->ReadaheadSuppressedUntil = 0;
 
     *out = f;
     return NTLSTATUS_SUCCESS;
@@ -1278,7 +1270,7 @@ __declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint
     IO_FILE_LOCK_WAITER waiter;
     memset(&waiter, 0, sizeof(waiter));
 
-    for (;;) {
+    while (true) {
         IO_FILE_LOCK_ENTRY* entry = (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k'));
         if (!entry) {
             if (waitEvent) ObDereferenceObject(&waitEvent->Header);

code/kernel/ke_apc.c

@@ -200,7 +200,7 @@ void KiDeliverKernelApcs(void) {
 
     thread->ApcDeliveryActive = 1;
 
-    for (;;) {
+    while (true) {
         if (KeGetCurrentIrql() != PASSIVE_LEVEL || thread->KernelApcDisable != 0)
             break;
 
@@ -375,7 +375,7 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
 void KiRundownApcState(THREAD* thread) {
     if (!thread) return;
 
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
         LIST_ENTRY* node = ListPopHead(&thread->ApcQueue);
         if (LIST_IS_EMPTY(&thread->ApcQueue))
@@ -389,7 +389,7 @@ void KiRundownApcState(THREAD* thread) {
             apc->RundownRoutine(apc);
     }
 
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
         LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue);
         if (LIST_IS_EMPTY(&thread->UserApcQueue))

code/kernel/ke_cpu.c

@@ -4,19 +4,148 @@
 #include <string.h>
 #include <intrin.h>
 #include <kernel/ke_cpu.h>
+#include <kernel/ke_debug.h>
+#include <ntlite/ddi/hal_api.h>
 
 #define BOOTSTRAP_CPU_STACK_SIZE 0x4000u
 #define IA32_GS_BASE             0xC0000101u
 #define IA32_KERNEL_GS_BASE      0xC0000102u
+#define IA32_MTRRCAP             0x000000FEu
+#define IA32_MTRR_PHYSBASE0      0x00000200u
+#define IA32_MTRR_PHYSMASK0      0x00000201u
+#define IA32_PAT                 0x00000277u
+#define IA32_MTRR_DEF_TYPE       0x000002FFu
+#define CPUID_FEATURE_MTRR       (1u << 12)
+#define CPUID_FEATURE_PAT        (1u << 16)
+#define MTRR_CAP_VARIABLE_COUNT  0x000000FFull
+#define MTRR_MASK_VALID          (1ull << 11)
+#define MTRR_DEF_TYPE_MASK       0x000000FFull
+#define MTRR_DEF_FIXED_ENABLE    (1ull << 10)
+#define MTRR_DEF_ENABLE          (1ull << 11)
+#define MEMORY_TYPE_UC           0x00ull
+#define MEMORY_TYPE_WC           0x01ull
+#define MEMORY_TYPE_WB           0x06ull
+#define CR0_CACHE_DISABLE        (1ull << 30)
+#define CR0_NOT_WRITE_THROUGH    (1ull << 29)
+#define CR4_PAGE_GLOBAL_ENABLE   (1ull << 7)
+#define CR4_PCID_ENABLE          (1ull << 17)
+#define PAT_ENTRY(type, index)   ((uint64_t)(type) << ((index) * 8u))
+#define CANONICAL_PAT            (PAT_ENTRY(MEMORY_TYPE_WB, 0) | \
+                                  PAT_ENTRY(MEMORY_TYPE_WC, 1) | \
+                                  PAT_ENTRY(MEMORY_TYPE_UC, 2) | \
+                                  PAT_ENTRY(MEMORY_TYPE_UC, 3) | \
+                                  PAT_ENTRY(MEMORY_TYPE_WB, 4) | \
+                                  PAT_ENTRY(MEMORY_TYPE_WC, 5) | \
+                                  PAT_ENTRY(MEMORY_TYPE_UC, 6) | \
+                                  PAT_ENTRY(MEMORY_TYPE_UC, 7))
 
 KCPU gCpus[KE_MAX_CPUS];
 uint32_t gCpuCount = 0;
 
 static uint8_t gBootstrapCpuStack[BOOTSTRAP_CPU_STACK_SIZE];
 static volatile long gKernelGsReady = 0;
+static volatile long gWriteCombiningAvailable = 0;
 
 extern KCPU* KeReadCurrentCpuFromGs(void);
 
+// Returns whether this CPU exposes the memory-type mechanisms required by the kernel policy.
+static bool KiMemoryTypeFeaturesAvailable(void) {
+    int regs[4];
+    __cpuid(regs, 1);
+    uint32_t features = (uint32_t)regs[3];
+    return (features & (CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT)) ==
+           (CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT);
+}
+
+// Flushes all translations before memory-type state changes and leaves global pages disabled.
+static uintptr_t KiDisableGlobalTranslations(void) {
+    uintptr_t cr4 = __readcr4();
+
+    // nt-lite does not enable PCID. Refuse inherited PCID state rather than
+    // pretending that a CR3 reload flushed every tagged translation.
+    if (cr4 & CR4_PCID_ENABLE)
+        HalStopProcessor();
+
+    if (cr4 & CR4_PAGE_GLOBAL_ENABLE)
+        __writecr4(cr4 & ~CR4_PAGE_GLOBAL_ENABLE);
+    else
+        __writecr3(__readcr3());
+
+    return cr4;
+}
+
+// Returns whether this CPU accepted the complete canonical PAT and MTRR policy.
+static bool KiMemoryTypePolicyInstalled(uint32_t variableCount) {
+    if (__readmsr(IA32_PAT) != CANONICAL_PAT)
+        return false;
+
+    uint64_t expectedDefault = MTRR_DEF_ENABLE | MEMORY_TYPE_WB;
+    uint64_t defaultType = __readmsr(IA32_MTRR_DEF_TYPE);
+    uint64_t relevantDefaultBits = MTRR_DEF_TYPE_MASK | MTRR_DEF_FIXED_ENABLE | MTRR_DEF_ENABLE;
+    if ((defaultType & relevantDefaultBits) != expectedDefault)
+        return false;
+
+    for (uint32_t i = 0; i < variableCount; i++) {
+        uint64_t base = __readmsr(IA32_MTRR_PHYSBASE0 + i * 2u);
+        uint64_t mask = __readmsr(IA32_MTRR_PHYSMASK0 + i * 2u);
+        if (base != 0 || (mask & MTRR_MASK_VALID) != 0)
+            return false;
+    }
+
+    return true;
+}
+
+// Programs the kernel-owned PAT policy over a uniform write-back MTRR baseline.
+void KeInitializeMemoryTypes(void) {
+    if (!KiMemoryTypeFeaturesAvailable())
+        HalStopProcessor();
+
+    uint32_t variableCount = (uint32_t)(__readmsr(IA32_MTRRCAP) & MTRR_CAP_VARIABLE_COUNT);
+    if (KiMemoryTypePolicyInstalled(variableCount)) {
+        _InterlockedExchange(&gWriteCombiningAvailable, 1);
+        return;
+    }
+
+    uintptr_t interruptFlags = HalDisableInterrupts();
+    uintptr_t cr0 = __readcr0();
+
+    // Every CPU runs this before entering normal kernel execution. The BSP
+    // installs the policy while it is alone, and each AP installs it before
+    // publishing its started state.
+    __writecr0((cr0 | CR0_CACHE_DISABLE) & ~CR0_NOT_WRITE_THROUGH);
+    __wbinvd();
+    uintptr_t cr4 = KiDisableGlobalTranslations();
+
+    // MTRRs provide only a uniform WB baseline. PAT is the sole owner of the
+    // page-granular WB, WC, and UC policy selected by kernel mappings.
+    __writemsr(IA32_MTRR_DEF_TYPE, 0);
+    for (uint32_t i = 0; i < variableCount; i++) {
+        __writemsr(IA32_MTRR_PHYSBASE0 + i * 2u, 0);
+        __writemsr(IA32_MTRR_PHYSMASK0 + i * 2u, 0);
+    }
+
+    __writemsr(IA32_PAT, CANONICAL_PAT);
+    __writemsr(IA32_MTRR_DEF_TYPE, MTRR_DEF_ENABLE | MEMORY_TYPE_WB);
+
+    __wbinvd();
+    __writecr3(__readcr3());
+    __writecr0(cr0);
+    if (cr4 & CR4_PAGE_GLOBAL_ENABLE)
+        __writecr4(cr4);
+
+    bool installed = KiMemoryTypePolicyInstalled(variableCount);
+    HalRestoreInterrupts(interruptFlags);
+    if (!installed)
+        HalStopProcessor();
+
+    _InterlockedExchange(&gWriteCombiningAvailable, 1);
+}
+
+// Reports whether PAT write-combining is available on the current platform.
+bool KeWriteCombiningAvailable(void) {
+    return _InterlockedCompareExchange(&gWriteCombiningAvailable, 0, 0) != 0;
+}
+
 // Loads kernel Gs Base.
 void KeLoadKernelGsBase(KCPU* cpu) {
     if (!cpu)
@@ -69,6 +198,7 @@ void KeInitializeBootstrapCpu(void) {
     cpu->Tss.RSP0 = cpu->KernelStackTop;
 
     gCpuCount = 1;
+    KeInitializeMemoryTypes();
     KeLoadKernelGsBase(cpu);
     KeSetUserGsBase(0);
 }

code/kernel/ke_debug.c

@@ -220,7 +220,29 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
                   line, length);
 }
 
-// Flushes debug Log To File.
+// Writes one debug-log segment and rejects successful zero-progress writes.
+static KSTATUS KiDebugLogWriteAll(FILE_OBJECT* file,
+                                  const char* data,
+                                  size_t length,
+                                  size_t* totalWritten) {
+    if (!file || (!data && length) || !totalWritten)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    while (length != 0) {
+        size_t written = 0;
+        KSTATUS st = IoWriteFile(file, data, length, &written);
+        if (!KSUCCESS(st))
+            return st;
+        if (written == 0)
+            return NTLSTATUS_DEVICE_ERROR;
+        data += written;
+        length -= written;
+        *totalWritten += written;
+    }
+    return NTLSTATUS_SUCCESS;
+}
+
+// Persists the in-memory debug ring and flushes it through the storage stack.
 __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
     if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
         return NTLSTATUS_INVALID_PARAMETER;
@@ -237,11 +259,6 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
     start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
     KeReleaseSpinlock(&gDebugPrintLock, flags);
 
-    if (used == 0) {
-        _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
-        return NTLSTATUS_SUCCESS;
-    }
-
     (void)IoDeleteFile(path);
 
     FILE_OBJECT* file = (FILE_OBJECT*)0;
@@ -251,23 +268,32 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
         return st;
     }
 
-    size_t written = 0;
-    size_t totalWritten = 0;
-    size_t first = gDebugLogSize - start;
-    if (first > used) first = used;
-    st = IoWriteFile(file, gDebugLogBuffer + start, first, &written);
-    totalWritten += written;
+    // Shutdown logging must not depend on deferred file-cache writeback. FAT
+    // still updates the directory entry and cluster chain through its normal
+    // write path, while IoFlushFile pushes metadata and the device cache.
+    file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;
 
-    if (KSUCCESS(st) && used > first) {
-        written = 0;
-        st = IoWriteFile(file, gDebugLogBuffer, used - first, &written);
-        totalWritten += written;
+    size_t totalWritten = 0;
+    if (used != 0) {
+        size_t first = gDebugLogSize - start;
+        if (first > used)
+            first = used;
+
+        st = KiDebugLogWriteAll(file, gDebugLogBuffer + start, first, &totalWritten);
+        if (KSUCCESS(st) && used > first) {
+            st = KiDebugLogWriteAll(file, gDebugLogBuffer,
+                                    used - first, &totalWritten);
+        }
     }
 
-    IoCloseFile(file);
+    if (KSUCCESS(st) && totalWritten != used)
+        st = NTLSTATUS_DEVICE_ERROR;
+    if (KSUCCESS(st))
+        st = IoFlushFile(file);
 
+    IoCloseFile(file);
     _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
-    return (KSUCCESS(st) && totalWritten == used) ? NTLSTATUS_SUCCESS : st;
+    return st;
 }
 
 // Attempts to persist the debug log during a panic when the filesystem is usable.
@@ -863,6 +889,18 @@ __declspec(dllexport) KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) {
     if (!KeFaultFrameFromUser(frame))
         KiPanicKernelFault(proc, thread, frame);
 
+    DbgPrint("USERFAULT: pid=%u tid=%u vector=%llu rip=0x%016llX rsp=0x%016llX cr2=0x%016llX error=0x%016llX rcx=0x%016llX rbx=0x%016llX rax=0x%016llX\n",
+             proc ? proc->ProcessId : 0,
+             thread ? thread->ThreadId : 0,
+             frame->Vector,
+             (uint64_t)frame->Rip,
+             (uint64_t)frame->Rsp,
+             (uint64_t)frame->Cr2,
+             (uint64_t)frame->ErrorCode,
+             (uint64_t)frame->Rcx,
+             (uint64_t)frame->Rbx,
+             (uint64_t)frame->Rax);
+
     KSTATUS delivered = KeDeliverUserException(frame);
     if (KSUCCESS(delivered))
         return NTLSTATUS_SUCCESS;

code/kernel/ke_display.c

@@ -19,6 +19,7 @@
 #include <kernel/ke_list.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_irql.h>
+#include <kernel/ke_cpu.h>
 
 #define DISPLAY_DEFAULT_FG 0x00CCCCCCu
 #define DISPLAY_DEFAULT_BG 0x00012456u
@@ -44,6 +45,15 @@ typedef struct DISPLAY_OUTPUT_GLYPH {
     uint32_t Background;
 } DISPLAY_OUTPUT_GLYPH;
 
+typedef struct DISPLAY_OUTPUT_DRAW {
+    char Character;
+    uint32_t Foreground;
+    uint32_t Background;
+    int Column;
+    int Row;
+    size_t ScrollCount;
+} DISPLAY_OUTPUT_DRAW;
+
 typedef struct BOOTFB_CONTEXT {
     const HAL_API* Hal;
     uint32_t* Framebuffer;
@@ -59,6 +69,7 @@ static DISPLAY_DEVICE* gDisplayList;
 static DISPLAY_DEVICE* gPrimaryDisplay;
 static SPINLOCK gDisplayLock = SPINLOCK_INIT;
 static MUTEX* gDisplayProviderMutex;
+static MUTEX* gDisplayOutputMutex;
 static KWORK_ITEM gDisplayCursorWorkItem;
 static volatile long gDisplayCursorWorkInitialized;
 static bool gDisplayTransitionInProgress;
@@ -104,8 +115,10 @@ static KWORK_ITEM gDisplayOutputWorkItem;
 static volatile long gDisplayOutputWorkInitialized;
 
 static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg);
+static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count);
 static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
-static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels);
+static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
+static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
 static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color);
 static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h);
 static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride);
@@ -119,6 +132,7 @@ static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats)
 static const DISPLAY_DEVICE_OPS gBootFbOps = {
     .GetMode = BootFbGetMode,
     .Blit = BootFbBlit,
+    .PresentPitched = BootFbPresentPitched,
     .FillRect = BootFbFillRect,
     .CopyRect = BootFbCopyRect,
     .ReadRect = BootFbReadRect,
@@ -192,10 +206,15 @@ static KSTATUS KiDisplayAcquireProviderMutex(bool* acquired) {
     if (KeGetDebugFlags() & KE_DEBUG_DISPLAY_ASSERTS)
         KeAssertIrql(PASSIVE_LEVEL);
 
-    KSTATUS s = KeAcquireMutex(gDisplayProviderMutex);
-    if (KSUCCESS(s) && acquired)
+    KSTATUS s = KeAcquireMutex(gDisplayProviderMutex, KE_WAIT_INFINITE);
+    if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s))
+        return s;
+
+    if (acquired)
         *acquired = true;
-    return s;
+
+    // Provider operations still own the mutex after an abandoned acquisition.
+    return NTLSTATUS_SUCCESS;
 }
 
 static void KiDisplayReleaseProviderMutex(bool acquired) {
@@ -449,6 +468,34 @@ static bool KiDisplayConsoleGridFromMode(const DISPLAY_MODE* mode, int* outCols,
     return true;
 }
 
+// Returns whether two dirty rectangles overlap or share an edge.
+static bool KiDisplayDirtyRectsMergeable(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
+    if (!a || !b)
+        return false;
+
+    int64_t aRight = (int64_t)a->X + a->W;
+    int64_t aBottom = (int64_t)a->Y + a->H;
+    int64_t bRight = (int64_t)b->X + b->W;
+    int64_t bBottom = (int64_t)b->Y + b->H;
+    return (int64_t)a->X <= bRight && (int64_t)b->X <= aRight &&
+           (int64_t)a->Y <= bBottom && (int64_t)b->Y <= aBottom;
+}
+
+// Unions two clipped dirty rectangles.
+static DISPLAY_RECT KiDisplayUnionDirtyRects(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
+    int64_t left = a->X < b->X ? a->X : b->X;
+    int64_t top = a->Y < b->Y ? a->Y : b->Y;
+    int64_t aRight = (int64_t)a->X + a->W;
+    int64_t bRight = (int64_t)b->X + b->W;
+    int64_t aBottom = (int64_t)a->Y + a->H;
+    int64_t bBottom = (int64_t)b->Y + b->H;
+    int64_t right = aRight > bRight ? aRight : bRight;
+    int64_t bottom = aBottom > bBottom ? aBottom : bBottom;
+    DISPLAY_RECT result = { (int32_t)left, (int32_t)top,
+                            (int32_t)(right - left), (int32_t)(bottom - top) };
+    return result;
+}
+
 static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
     if (!mode || gDirtyAll || w <= 0 || h <= 0)
         return;
@@ -457,6 +504,19 @@ static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int
     if (!KiDisplayClipRect(mode, &rect))
         return;
 
+    for (size_t i = 0; i < gDirtyRectCount;) {
+        if (!KiDisplayDirtyRectsMergeable(&rect, &gDirtyRects[i])) {
+            i++;
+            continue;
+        }
+
+        rect = KiDisplayUnionDirtyRects(&rect, &gDirtyRects[i]);
+        gDirtyRectCount--;
+        if (i != gDirtyRectCount)
+            gDirtyRects[i] = gDirtyRects[gDirtyRectCount];
+        i = 0;
+    }
+
     if (rect.X == 0 && rect.Y == 0 &&
         rect.W == (int32_t)mode->Width && rect.H == (int32_t)mode->Height) {
         KiDisplayDirtyAll();
@@ -478,6 +538,24 @@ static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, in
     KiDisplayAddDirtyForModeLocked(mode, x, y, w, h);
 }
 
+static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
+    if (!mode || w <= 0 || h <= 0)
+        return;
+
+    DISPLAY_RECT rect = { x, y, w, h };
+    if (!KiDisplayClipRect(mode, &rect))
+        return;
+
+    uint64_t bytes = (uint64_t)(uint32_t)rect.W *
+                     (uint64_t)(uint32_t)rect.H * sizeof(uint32_t);
+    gDisplayStats.BlitCount++;
+    gDisplayStats.PresentCount++;
+    gDisplayStats.PresentRectCount++;
+    gDisplayStats.DirectPresentCount++;
+    gDisplayStats.ScanoutBytes += bytes;
+    gDisplayStats.BytesCopied += bytes;
+}
+
 static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
     gDisplayStats.FillRectCount++;
     gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
@@ -660,7 +738,7 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
 
     uint32_t glyphPixels[FONT_W * FONT_H];
     KiDisplayBuildGlyphPixels(c, fg, bg, glyphPixels);
-    KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels);
+    KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels, FONT_W * sizeof(uint32_t));
     if (KSUCCESS(s)) {
         uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
         if (device == gPrimaryDisplay)
@@ -670,20 +748,26 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
     return s;
 }
 
-static KSTATUS KiDisplayDrawBlitSnapshot(DISPLAY_DEVICE* device,
-                                         const DISPLAY_DEVICE_OPS* ops,
-                                         const DISPLAY_MODE* mode,
-                                         int x,
-                                         int y,
-                                         int w,
-                                         int h,
-                                         const uint32_t* pixels) {
-    if (!pixels || w <= 0 || h <= 0)
+// Draws a pitched source rectangle and records it as one display operation.
+static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device,
+                                                const DISPLAY_DEVICE_OPS* ops,
+                                                const DISPLAY_MODE* mode,
+                                                int x,
+                                                int y,
+                                                int w,
+                                                int h,
+                                                const uint32_t* pixels,
+                                                uint32_t pitchBytes) {
+    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
         return NTLSTATUS_INVALID_PARAMETER;
-    if (!device || !ops || !ops->Blit || !mode)
+    if (!device || !ops || !mode)
         return NTLSTATUS_NOT_SUPPORTED;
 
-    KSTATUS s = ops->Blit(device, x, y, w, h, pixels);
+    if (!ops->Blit)
+        return NTLSTATUS_NOT_SUPPORTED;
+
+    KSTATUS s = ops->Blit(device, x, y, w, h, pixels, pitchBytes);
+
     if (KSUCCESS(s)) {
         uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
         if (device == gPrimaryDisplay)
@@ -867,7 +951,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3
     if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Blit)
         return NTLSTATUS_NOT_SUPPORTED;
 
-    KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels);
+    KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels, (uint32_t)w * sizeof(uint32_t));
     if (KSUCCESS(s)) {
         gDisplayStats.BlitCount++;
         gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
@@ -1043,11 +1127,18 @@ static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uin
 }
 
 // Drains buffered output through the active display provider.
-static void KiDisplayOutputWorker(void* context) {
-    (void)context;
+static KSTATUS KiDisplayDrainOutput(void) {
     DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT];
+    bool mutexHeld = false;
 
-    for (;;) {
+    if (gDisplayOutputMutex) {
+        KSTATUS acquireStatus = KeAcquireMutex(gDisplayOutputMutex, KE_WAIT_INFINITE);
+        if (!KE_MUTEX_ACQUIRE_SUCCEEDED(acquireStatus))
+            return acquireStatus;
+        mutexHeld = true;
+    }
+
+    while (true) {
         size_t count = 0;
         uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
         while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) {
@@ -1059,12 +1150,28 @@ static void KiDisplayOutputWorker(void* context) {
 
         if (count == 0)
             break;
-        for (size_t i = 0; i < count; i++) {
-            (void)KiDisplayPrintCharInternal(glyphs[i].Character,
-                                             glyphs[i].Foreground,
-                                             glyphs[i].Background);
+        KSTATUS drawStatus = KiDisplayPrintGlyphsInternal(glyphs, count);
+        if (!KSUCCESS(drawStatus)) {
+            if (mutexHeld)
+                KeReleaseMutex(gDisplayOutputMutex);
+            return drawStatus;
         }
     }
+
+    if (mutexHeld)
+        KeReleaseMutex(gDisplayOutputMutex);
+    return NTLSTATUS_SUCCESS;
+}
+
+// Drains buffered output from the passive work queue.
+static void KiDisplayOutputWorker(void* context) {
+    (void)context;
+    (void)KiDisplayDrainOutput();
+}
+
+// Completes buffered console output before a caller overlays console pixels.
+KSTATUS KeDisplayFlushConsoleOutput(void) {
+    return KiDisplayDrainOutput();
 }
 
 // Queues the runtime display output worker when passive work is available.
@@ -1157,6 +1264,8 @@ void KeDisplayInitialize(const HAL_API* hal) {
     _InterlockedExchange(&gDisplayClockInitialized, 1);
     if (!gDisplayProviderMutex)
         (void)KeMutexCreate(&gDisplayProviderMutex);
+    if (!gDisplayOutputMutex)
+        (void)KeMutexCreate(&gDisplayOutputMutex);
     KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
     _InterlockedExchange(&gDisplayCursorWorkInitialized, 1);
     KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0);
@@ -1212,8 +1321,9 @@ void KeDisplayInitialize(const HAL_API* hal) {
         (void)KeDisplaySetPrimary(&gBootFbDevice);
         DISPLAY_MODE mode;
         if (KSUCCESS(KeDisplayGetMode(&mode))) {
-            DbgPrint("Display: bootfb primary %ux%u stride=%u\n",
-                     mode.Width, mode.Height, mode.Stride);
+            DbgPrint("Display: bootfb primary %ux%u stride=%u cache=%s\n",
+                     mode.Width, mode.Height, mode.Stride,
+                     KeWriteCombiningAvailable() ? "WC" : "WT");
         }
 
         // The registration messages above were routed through the HAL boot
@@ -1277,7 +1387,7 @@ DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) {
 
 // Signals every display clock once for a provider-owned frame generation.
 static void KiDisplaySignalFrameGeneration(uint64_t generation) {
-    for (;;) {
+    while (true) {
         EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
         uint32_t count = 0;
 
@@ -1340,7 +1450,7 @@ DISPLAY_API void KeDisplayTick(void) {
         return;
 
     uint64_t now = KiDisplayClockReadSchedulerTick();
-    for (;;) {
+    while (true) {
         EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
         uint32_t count = 0;
 
@@ -1424,7 +1534,8 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE
         s = oldDevice->Ops->ReadRect(oldDevice, 0, (int)y, (int)newMode.Width, 1, row, newMode.Width);
         if (!KSUCCESS(s))
             break;
-        s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row);
+        s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row,
+                                  newMode.Width * sizeof(uint32_t));
         if (!KSUCCESS(s))
             break;
     }
@@ -1610,6 +1721,8 @@ DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) {
     for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next)
         count++;
     info.DeviceCount = count;
+    if (KeWriteCombiningAvailable())
+        info.Flags |= DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE;
     info.ManagerStats = gDisplayStats;
 
     DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
@@ -1660,7 +1773,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi
     s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
     if (KSUCCESS(s)) {
         (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
-        s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y, w, h, pixels);
+        s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels,
+                                               (uint32_t)w * sizeof(uint32_t));
         s = KiDisplayFinishProviderDraw(device, ops, caps, s);
     }
 
@@ -1668,8 +1782,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi
     return s;
 }
 
-DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch) {
-    if (!source || !scratch || w <= 0 || h <= 0)
+DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
+    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
         return NTLSTATUS_INVALID_PARAMETER;
 
     bool providerHeld = false;
@@ -1684,14 +1798,7 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW
     s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
     if (KSUCCESS(s)) {
         (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
-        for (int row = 0; row < h; ++row) {
-            s = source(context, row, scratch, (size_t)w);
-            if (!KSUCCESS(s))
-                break;
-            s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y + row, w, 1, scratch);
-            if (!KSUCCESS(s))
-                break;
-        }
+        s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, pitchBytes);
         s = KiDisplayFinishProviderDraw(device, ops, caps, s);
     }
 
@@ -1699,6 +1806,47 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW
     return s;
 }
 
+DISPLAY_API KSTATUS KeDisplayPresentPitched(int x,
+                                              int y,
+                                              int w,
+                                              int h,
+                                              const uint32_t* pixels,
+                                              uint32_t pitchBytes) {
+    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    bool providerHeld = false;
+    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
+    if (!KSUCCESS(s))
+        return s;
+
+    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
+    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
+    uint32_t caps = 0;
+    DISPLAY_MODE mode;
+    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
+    if (KSUCCESS(s)) {
+        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
+        if (ops->PresentPitched) {
+            s = ops->PresentPitched(device, x, y, w, h, pixels, pitchBytes);
+            if (KSUCCESS(s)) {
+                uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
+                if (device == gPrimaryDisplay)
+                    KiDisplayRecordDirectPresentLocked(&mode, x, y, w, h);
+                KeReleaseSpinlock(&gDisplayLock, flags);
+            }
+            KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps);
+        } else {
+            s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode,
+                                                 x, y, w, h, pixels, pitchBytes);
+            s = KiDisplayFinishProviderDraw(device, ops, caps, s);
+        }
+    }
+
+    KiDisplayReleaseProviderMutex(providerHeld);
+    return s;
+}
+
 DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) {
     bool providerHeld = false;
     KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
@@ -1835,18 +1983,28 @@ DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t
     return s;
 }
 
-static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
+// Renders one or more console glyphs as one provider transaction and one present.
+static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count) {
+    if (!glyphs && count != 0)
+        return NTLSTATUS_INVALID_PARAMETER;
+    if (count == 0)
+        return NTLSTATUS_SUCCESS;
+    if (count > DISPLAY_OUTPUT_DRAIN_COUNT)
+        return NTLSTATUS_INVALID_PARAMETER;
+
     if (KiDisplayOutputSuspended()) {
-        if (!KeCanWait())
-            return KiDisplayFallbackPrintChar(c, fg, bg);
+        if (!KeCanWait()) {
+            for (size_t i = 0; i < count; i++)
+                (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
+        }
         return NTLSTATUS_SUCCESS;
     }
 
-    // Once the scheduler/work-queue era begins, display provider callbacks are
-    // PASSIVE_LEVEL-only. The HAL output endpoint buffers normal text so this
-    // fallback is reserved for explicit display calls made at elevated IRQL.
-    if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait())
-        return KiDisplayFallbackPrintChar(c, fg, bg);
+    if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) {
+        for (size_t i = 0; i < count; i++)
+            (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
+        return NTLSTATUS_SUCCESS;
+    }
 
     bool providerHeld = false;
     KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
@@ -1861,8 +2019,10 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
     if (!KSUCCESS(s)) {
         KiDisplayReleaseProviderMutex(providerHeld);
         const HAL_API* hal = gDisplayHal;
-        if (hal && hal->Screen.PrintChar)
-            hal->Screen.PrintChar(c, fg, bg);
+        if (hal && hal->Screen.PrintChar) {
+            for (size_t i = 0; i < count; i++)
+                hal->Screen.PrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
+        }
         return s;
     }
 
@@ -1875,71 +2035,94 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
 
     (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
 
-    bool drawChar = false;
-    bool needScroll = false;
-    int drawCol = 0;
-    int drawRow = 0;
+    DISPLAY_OUTPUT_DRAW draws[DISPLAY_OUTPUT_DRAIN_COUNT];
+    size_t drawCount = 0;
+    size_t scrollCount = 0;
 
-    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
-    if (c == '\n') {
-        gConsoleX = 0;
-        gConsoleY++;
-    } else if (c == '\r') {
-        gConsoleX = 0;
-    } else if (c == '\b') {
-        if (gConsoleX > 0) {
-            gConsoleX--;
-        } else if (gConsoleY > 0) {
-            gConsoleY--;
-            gConsoleX = cols - 1;
-        }
-    } else {
-        drawChar = true;
-        drawCol = gConsoleX;
-        drawRow = gConsoleY;
-        if (++gConsoleX >= cols) {
+    for (size_t glyphIndex = 0; glyphIndex < count; glyphIndex++) {
+        char c = glyphs[glyphIndex].Character;
+        int drawCol = 0;
+        int drawRow = 0;
+        bool drawChar = false;
+        bool needScroll = false;
+
+        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
+        if (c == '\n') {
             gConsoleX = 0;
             gConsoleY++;
+        } else if (c == '\r') {
+            gConsoleX = 0;
+        } else if (c == '\b') {
+            if (gConsoleX > 0) {
+                gConsoleX--;
+            } else if (gConsoleY > 0) {
+                gConsoleY--;
+                gConsoleX = cols - 1;
+            }
+        } else {
+            drawChar = true;
+            drawCol = gConsoleX;
+            drawRow = gConsoleY;
+            if (++gConsoleX >= cols) {
+                gConsoleX = 0;
+                gConsoleY++;
+            }
+
+            if (drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) {
+                size_t cell = KiDisplayConsoleIndex(drawCol, drawRow);
+                gConsoleCells[cell] = c;
+                gConsoleAttrs[cell] = (uint8_t)gConsoleAttribute;
+            }
+        }
+
+        if (gConsoleY >= rows) {
+            KiDisplayConsoleScrollCellsLocked(cols, rows, 1);
+            gConsoleY = rows - 1;
+            if (gConsoleX < 0) gConsoleX = 0;
+            if (gConsoleX >= cols) gConsoleX = cols - 1;
+            needScroll = true;
+        }
+        KeReleaseSpinlock(&gDisplayLock, flags);
+
+        if (drawChar) {
+            DISPLAY_OUTPUT_DRAW* draw = &draws[drawCount++];
+            draw->Character = c;
+            draw->Foreground = glyphs[glyphIndex].Foreground;
+            draw->Background = glyphs[glyphIndex].Background;
+            draw->Column = drawCol;
+            draw->Row = drawRow;
+            draw->ScrollCount = scrollCount;
+        }
+        if (needScroll)
+            scrollCount++;
+    }
+
+    if (scrollCount != 0) {
+        uint64_t scrollPixels64 = (uint64_t)scrollCount * FONT_H;
+        int scrollPixels = scrollPixels64 >= mode.Height ? (int)mode.Height : (int)scrollPixels64;
+        if ((uint32_t)scrollPixels >= mode.Height) {
+            s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0,
+                                          (int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG);
+        } else {
+            s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, scrollPixels,
+                                          (int)mode.Width, (int)(mode.Height - (uint32_t)scrollPixels));
+            if (KSUCCESS(s)) {
+                s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
+                                              (int)(mode.Height - (uint32_t)scrollPixels),
+                                              (int)mode.Width, scrollPixels, DISPLAY_DEFAULT_BG);
+            }
         }
     }
 
-    if (gConsoleY >= rows)
-        needScroll = true;
-    KeReleaseSpinlock(&gDisplayLock, flags);
-
-    if (drawChar) {
-        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, drawCol * FONT_W, drawRow * FONT_H, fg, bg);
-        if (KSUCCESS(s)) {
-            flags = KeAcquireSpinlock(&gDisplayLock);
-            if (device == gPrimaryDisplay && drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) {
-                size_t i = KiDisplayConsoleIndex(drawCol, drawRow);
-                gConsoleCells[i] = c;
-                gConsoleAttrs[i] = (uint8_t)gConsoleAttribute;
-            }
-            KeReleaseSpinlock(&gDisplayLock, flags);
-        }
-    } else {
-        s = NTLSTATUS_SUCCESS;
-    }
-
-    if (KSUCCESS(s) && needScroll) {
-        s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, FONT_H,
-                                      (int)mode.Width, (int)(mode.Height - FONT_H));
-        if (KSUCCESS(s)) {
-            s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
-                                          (int)(mode.Height - FONT_H),
-                                          (int)mode.Width, FONT_H, DISPLAY_DEFAULT_BG);
-        }
-        if (KSUCCESS(s)) {
-            flags = KeAcquireSpinlock(&gDisplayLock);
-            if (device == gPrimaryDisplay) {
-                KiDisplayConsoleScrollCellsLocked(cols, rows, 1);
-                gConsoleY = rows - 1;
-                if (gConsoleX < 0) gConsoleX = 0;
-                if (gConsoleX >= cols) gConsoleX = cols - 1;
-            }
-            KeReleaseSpinlock(&gDisplayLock, flags);
-        }
+    for (size_t i = 0; i < drawCount && KSUCCESS(s); i++) {
+        DISPLAY_OUTPUT_DRAW* draw = &draws[i];
+        size_t laterScrolls = scrollCount - draw->ScrollCount;
+        int finalRow = draw->Row - (int)laterScrolls;
+        if (finalRow < 0 || finalRow >= rows)
+            continue;
+        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, draw->Character,
+                                       draw->Column * FONT_W, finalRow * FONT_H,
+                                       draw->Foreground, draw->Background);
     }
 
     s = KiDisplayFinishProviderDraw(device, ops, caps, s);
@@ -1947,6 +2130,12 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
     return s;
 }
 
+// Renders one console character through the shared batched console path.
+static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
+    DISPLAY_OUTPUT_GLYPH glyph = { c, fg, bg };
+    return KiDisplayPrintGlyphsInternal(&glyph, 1);
+}
+
 DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
     return KiDisplayPrintCharInternal(c, fg, bg);
 }
@@ -2285,10 +2474,114 @@ static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
     return NTLSTATUS_SUCCESS;
 }
 
-// Implements the boot Fb Blit operation.
-static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
+// Copies one pixel row into the write-combined scanout aperture.
+static void BootFbCopyToScanout(uint32_t* destination,
+                                const uint32_t* source,
+                                size_t pixelCount) {
+    if (!destination || !source || pixelCount == 0)
+        return;
+
+    if (!KeWriteCombiningAvailable()) {
+        memcpy(destination, source, pixelCount * sizeof(uint32_t));
+        return;
+    }
+
+    while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) {
+        *destination++ = *source++;
+        pixelCount--;
+    }
+
+    while (pixelCount >= 16u) {
+        __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
+        __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
+        __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
+        __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
+        _mm_stream_si128((__m128i*)(destination + 0), v0);
+        _mm_stream_si128((__m128i*)(destination + 4), v1);
+        _mm_stream_si128((__m128i*)(destination + 8), v2);
+        _mm_stream_si128((__m128i*)(destination + 12), v3);
+        source += 16;
+        destination += 16;
+        pixelCount -= 16;
+    }
+
+    while (pixelCount >= 4u) {
+        __m128i value = _mm_loadu_si128((const __m128i*)source);
+        _mm_stream_si128((__m128i*)destination, value);
+        source += 4;
+        destination += 4;
+        pixelCount -= 4;
+    }
+
+    while (pixelCount != 0) {
+        *destination++ = *source++;
+        pixelCount--;
+    }
+}
+
+// Copies one source row into the retained shadow and scanout in one read pass.
+static void BootFbCopyToShadowAndScanout(uint32_t* shadow,
+                                         uint32_t* scanout,
+                                         const uint32_t* source,
+                                         size_t pixelCount) {
+    if (!shadow || !scanout || !source || pixelCount == 0)
+        return;
+
+    if (!KeWriteCombiningAvailable()) {
+        memcpy(shadow, source, pixelCount * sizeof(uint32_t));
+        memcpy(scanout, source, pixelCount * sizeof(uint32_t));
+        return;
+    }
+
+    while (pixelCount != 0 && ((uintptr_t)scanout & 15u) != 0) {
+        uint32_t value = *source++;
+        *shadow++ = value;
+        *scanout++ = value;
+        pixelCount--;
+    }
+
+    while (pixelCount >= 16u) {
+        __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
+        __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
+        __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
+        __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
+        _mm_storeu_si128((__m128i*)(shadow + 0), v0);
+        _mm_storeu_si128((__m128i*)(shadow + 4), v1);
+        _mm_storeu_si128((__m128i*)(shadow + 8), v2);
+        _mm_storeu_si128((__m128i*)(shadow + 12), v3);
+        _mm_stream_si128((__m128i*)(scanout + 0), v0);
+        _mm_stream_si128((__m128i*)(scanout + 4), v1);
+        _mm_stream_si128((__m128i*)(scanout + 8), v2);
+        _mm_stream_si128((__m128i*)(scanout + 12), v3);
+        source += 16;
+        shadow += 16;
+        scanout += 16;
+        pixelCount -= 16;
+    }
+
+    while (pixelCount >= 4u) {
+        __m128i value = _mm_loadu_si128((const __m128i*)source);
+        _mm_storeu_si128((__m128i*)shadow, value);
+        _mm_stream_si128((__m128i*)scanout, value);
+        source += 4;
+        shadow += 4;
+        scanout += 4;
+        pixelCount -= 4;
+    }
+
+    while (pixelCount != 0) {
+        uint32_t value = *source++;
+        *shadow++ = value;
+        *scanout++ = value;
+        pixelCount--;
+    }
+}
+
+// Implements the boot framebuffer pitched blit operation.
+static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
     BOOTFB_CONTEXT* fb = BootFbContext(device);
-    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0)
+    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
+        pitchBytes < (uint32_t)w * sizeof(uint32_t))
         return NTLSTATUS_INVALID_PARAMETER;
 
     uint32_t* dstBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer;
@@ -2309,12 +2602,61 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co
 
     for (int row = 0; row < endY - startY; row++) {
         uint32_t* dst = dstBase + ((size_t)(startY + row) * fb->Stride) + startX;
-        const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX;
+        const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
+        const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
         memcpy(dst, src, bytesPerRow);
     }
     return NTLSTATUS_SUCCESS;
 }
 
+// Presents one pitched source rectangle without a shadow-to-scanout second pass.
+static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
+                                    int x,
+                                    int y,
+                                    int w,
+                                    int h,
+                                    const uint32_t* pixels,
+                                    uint32_t pitchBytes) {
+    BOOTFB_CONTEXT* fb = BootFbContext(device);
+    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
+        pitchBytes < (uint32_t)w * sizeof(uint32_t))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    int startX = x < 0 ? 0 : x;
+    int startY = y < 0 ? 0 : y;
+    int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w;
+    int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h;
+    if (startX >= endX || startY >= endY)
+        return NTLSTATUS_SUCCESS;
+
+    int srcOffsetX = startX - x;
+    int srcOffsetY = startY - y;
+    size_t pixelCount = (size_t)(endX - startX);
+    uint64_t bytes = (uint64_t)pixelCount * (uint64_t)(endY - startY) * sizeof(uint32_t);
+
+    fb->Stats.BlitCount++;
+    fb->Stats.PresentCount++;
+    fb->Stats.PresentRectCount++;
+    fb->Stats.DirectPresentCount++;
+    fb->Stats.ScanoutBytes += bytes;
+    fb->Stats.BytesCopied += fb->Backbuffer ? bytes * 2u : bytes;
+
+    for (int row = 0; row < endY - startY; row++) {
+        const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
+        const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
+        uint32_t* scanout = fb->Framebuffer + ((size_t)(startY + row) * fb->Stride) + startX;
+        if (fb->Backbuffer) {
+            uint32_t* shadow = fb->Backbuffer + ((size_t)(startY + row) * fb->Stride) + startX;
+            BootFbCopyToShadowAndScanout(shadow, scanout, source, pixelCount);
+        } else {
+            BootFbCopyToScanout(scanout, source, pixelCount);
+        }
+    }
+
+    _mm_sfence();
+    return NTLSTATUS_SUCCESS;
+}
+
 // Implements the boot Fb Fill Rect operation.
 static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
     BOOTFB_CONTEXT* fb = BootFbContext(device);
@@ -2425,14 +2767,18 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects,
         DISPLAY_RECT r = rects[i];
         if (!KiDisplayClipRect(&mode, &r))
             continue;
-        fb->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
-        size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t);
+        uint64_t bytes = (uint64_t)(uint32_t)r.W *
+                         (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
+        fb->Stats.BytesCopied += bytes;
+        fb->Stats.ScanoutBytes += bytes;
+        size_t pixelCount = (size_t)r.W;
         for (int row = 0; row < r.H; row++) {
             uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
             uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
-            memcpy(dst, src, bytesPerRow);
+            BootFbCopyToScanout(dst, src, pixelCount);
         }
     }
+    _mm_sfence();
     return NTLSTATUS_SUCCESS;
 }
 
@@ -2540,6 +2886,7 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
         fb->Stats.CursorHideCount++;
         BootFbRestoreCursor(fb);
     }
+    _mm_sfence();
     return NTLSTATUS_SUCCESS;
 }
 

code/kernel/ke_dpc.c

@@ -81,7 +81,7 @@ void KiDrainDpcQueue(void) {
     if (oldIrql < DISPATCH_LEVEL)
         KeRaiseIrql(DISPATCH_LEVEL);
 
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
         LIST_ENTRY* node = KeListPopHeadChecked(&gDpcQueue);
         if (!node) {

code/kernel/ke_exception.c

@@ -60,7 +60,7 @@ static _Noreturn void KiTerminateCurrentExceptionProcess(KSTATUS exitStatus) {
     if (thread)
         KeExitThread(thread, exitCode);
 
-    for (;;) {
+    while (true) {
         KeYield();
     }
 }

code/kernel/ke_input.c

@@ -390,7 +390,7 @@ static KSTATUS KiInputWaitKey(KE_INPUT_CONTEXT* context, uint8_t alertable, uint
     if (!context->WaitEvent)
         return NTLSTATUS_FAILURE;
 
-    for (;;) {
+    while (true) {
         EVENT* event = context->WaitEvent;
         uint32_t selected = 0;
         KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected);

code/kernel/ke_irp.c

@@ -5,6 +5,7 @@
 #include <kernel/ke_irp.h>
 #include <kernel/ke_event.h>
 #include <kernel/ke_semaphore.h>
+#include <kernel/ke_sched.h>
 #include <kernel/ke_apc.h>
 #include <kernel/ob_object.h>
 #include <kernel/mm_virt.h>
@@ -38,12 +39,13 @@ __declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
     return CONTAINING_RECORD(node, IRP, Link);
 }
 
-__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
-                                                    KSTATUS status,
-                                                    IRP_QUEUE_CANCEL_CALLBACK callback,
-                                                    void* context) {
+// Cancels every queued IRP and invokes an optional ownership callback.
+__declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q,
+                                                         KSTATUS status,
+                                                         IRP_QUEUE_CANCEL_CALLBACK callback,
+                                                         void* context) {
     if (!q) return;
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&q->Lock);
         LIST_ENTRY* node = ListPopHead(&q->Head);
         KeReleaseSpinlock(&q->Lock, flags);
@@ -60,7 +62,7 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
 
 // Implements the irp Queue Cancel All operation.
 __declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
-    IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
+    IrpQueueCancelAllWithCallback(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
 }
 
 // Remove locks.
@@ -284,6 +286,14 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t
     irp->Status = status;
     irp->Information = information;
 
+    // Completion callbacks may recycle an embedded IRP, while a synchronous
+    // waiter may release stack storage as soon as its event is signaled.
+    // Snapshot every completion target before either ownership handoff.
+    KAPC* completionApc = (KAPC*)irp->CompletionApc;
+    EVENT* completionEvent = (EVENT*)irp->CompletionEvent;
+    IRP_COMPLETION_FN completionFn = irp->CompletionFn;
+    void* completionCtx = irp->CompletionCtx;
+
     IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
     DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
     if (dev) {
@@ -292,12 +302,18 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t
         ios->DeviceObject = (DEVICE_OBJECT*)0;
     }
 
-    if (irp->CompletionApc)
-        KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status);
-    if (irp->CompletionEvent)
-        KeSignalEvent((EVENT*)irp->CompletionEvent);
-    if (irp->CompletionFn)
-        irp->CompletionFn(irp, irp->CompletionCtx);
+    if (completionApc)
+        KeInsertQueueApc(completionApc, irp, (void*)(uintptr_t)status);
+
+    // The callback owns the final callback-based handoff. Do not dereference
+    // the IRP after it returns because embedded requests may be freed there.
+    if (completionFn)
+        completionFn(irp, completionCtx);
+
+    // The event is the final synchronous ownership handoff. A waiter may
+    // invalidate a stack IRP immediately after this signal becomes visible.
+    if (completionEvent)
+        KeSignalEvent(completionEvent);
 
     return NTLSTATUS_SUCCESS;
 }
@@ -314,6 +330,7 @@ __declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
 // Submits irp Sync.
 __declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
     if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER;
+    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
 
     EVENT* ev = (EVENT*)0;
     KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev);
@@ -321,12 +338,22 @@ __declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
 
     irp->CompletionEvent = ev;
 
+    // Drivers may retain this IRP, its transfer buffer, and the caller's
+    // kernel stack until completion. Defer process termination across that
+    // ownership interval rather than letting teardown free live storage.
+    KeEnterTerminationDeferral();
     st = IoSubmitIrp(device, irp);
     if (st == NTLSTATUS_PENDING) {
-        KeWaitForEvent(ev, 0);
+        do {
+            st = KeWaitForEvent(ev, 0);
+        } while (st != NTLSTATUS_SUCCESS);
         st = irp->Status;
     }
 
+    // Completion events are signaled only after IoCompleteRequest has finished
+    // every operation that can dereference a synchronous caller's stack IRP.
+    irp->CompletionEvent = (EVENT*)0;
     ObDereferenceObject(&ev->Header);
+    KeLeaveTerminationDeferral();
     return st;
 }

code/kernel/ke_irq.c

@@ -126,7 +126,9 @@ void KiExitInterrupt(void) {
     // high-IRQL region.
     if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
         THREAD* current = cpu->CurrentThread;
-        if (current && !current->IsIdleThread && current->Owner && KeIsProcessTerminating(current->Owner)) {
+        if (current && !current->IsIdleThread && current->Owner &&
+            KeIsProcessTerminating(current->Owner) &&
+            !KeThreadTerminationDeferred(current)) {
             KeExitThread(current, current->Owner->ExitCode);
             while (true) ;
         }

code/kernel/ke_mutex.c

@@ -1,4 +1,4 @@
-// Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups.
+// Kernel mutex primitive. This file owns exclusive ownership, abandonment, wait queues, and priority inheritance.
 
 #include <stdbool.h>
 #include <kernel/ke_list.h>
@@ -6,76 +6,285 @@
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
 #include <kernel/ke_irql.h>
+#include <kernel/ke_debug.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ob_object.h>
 
-// Implements the mutex Destroy operation.
-static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
+// Reports an invalid mutex ownership transition.
+static void KiMutexBug(const char* where, MUTEX* mutex, THREAD* thread, const char* why) {
+    KePanic(KE_PANIC_CODE_SCHEDULER,
+            "mutex invariant failed: %s: mutex=0x%016llX thread=0x%016llX tid=%u owner=0x%016llX refs=%ld waitEmpty=%u ownerLink=0x%016llX/0x%016llX threadWaiting=0x%016llX threadOwnedEmpty=%u: %s\n",
+            where,
+            (uint64_t)mutex,
+            (uint64_t)thread,
+            thread ? thread->ThreadId : 0,
+            mutex ? (uint64_t)mutex->Owner : 0,
+            mutex ? mutex->Header.RefCount : 0,
+            mutex ? (LIST_IS_EMPTY(&mutex->WaitList) ? 1u : 0u) : 1u,
+            mutex ? (uint64_t)mutex->OwnerLink.Fwd : 0,
+            mutex ? (uint64_t)mutex->OwnerLink.Bwd : 0,
+            thread ? (uint64_t)thread->WaitingMutex : 0,
+            thread ? (LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u) : 1u,
+            why ? why : "?");
+}
 
-// Implements the mutex Create operation.
-__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) {
+// Stabilizes an owner pointer before it crosses the mutex lock boundary.
+static void KiMutexReferenceOwnerLocked(MUTEX* mutex, THREAD* owner) {
+    if (!owner || !ObTryReferenceObject(&owner->Header))
+        KiMutexBug("KiMutexReferenceOwnerLocked", mutex, owner,
+                   "live mutex owner could not be referenced");
+}
+
+// Tests whether one scheduler link is currently attached to a list.
+static int KiMutexWaiterIsLinked(const THREAD* thread) {
+    if (!thread) return 0;
+    return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
+           thread->SchedulerLink.Bwd != &thread->SchedulerLink;
+}
+
+// Propagates one effective-rank change without carrying raw owners across locks.
+static void KiMutexPropagatePriority(THREAD* thread) {
+    int releaseThread = 0;
+
+    for (uint32_t depth = 1; thread && depth <= KE_PRIORITY_INHERITANCE_MAX_DEPTH; depth++) {
+        MUTEX* waitingMutex = (MUTEX*)0;
+        int changed = KiSchedulerRefreshEffectivePriority(thread, &waitingMutex);
+        if (!changed) {
+            if (releaseThread) ObDereferenceObject(&thread->Header);
+            return;
+        }
+
+        KiSchedulerNotePriorityPropagation(depth);
+        if (!waitingMutex) {
+            if (releaseThread) ObDereferenceObject(&thread->Header);
+            return;
+        }
+
+        uintptr_t flags = KeAcquireSpinlock(&waitingMutex->Lock);
+        if (thread->WaitingMutex != waitingMutex ||
+            thread->WaitObject != &waitingMutex->Header ||
+            thread->WaitListHead != &waitingMutex->WaitList ||
+            !KiMutexWaiterIsLinked(thread)) {
+            KeReleaseSpinlock(&waitingMutex->Lock, flags);
+            ObDereferenceObject(&waitingMutex->Header);
+            if (releaseThread) ObDereferenceObject(&thread->Header);
+            return;
+        }
+
+        THREAD* owner = waitingMutex->Owner;
+        if (owner) KiMutexReferenceOwnerLocked(waitingMutex, owner);
+        int donationChanged = KiSchedulerRefreshMutexDonation(waitingMutex);
+        KeReleaseSpinlock(&waitingMutex->Lock, flags);
+        ObDereferenceObject(&waitingMutex->Header);
+
+        if (releaseThread) ObDereferenceObject(&thread->Header);
+        if (!donationChanged || !owner) {
+            if (owner) ObDereferenceObject(&owner->Header);
+            return;
+        }
+
+        thread = owner;
+        releaseThread = 1;
+    }
+
+    if (thread)
+        KiMutexBug("KiMutexPropagatePriority", thread->WaitingMutex, thread,
+                   "priority inheritance chain exceeded the supported depth");
+}
+
+// Initializes one mutex's ownership bookkeeping.
+static void KiMutexInitialize(MUTEX* mutex) {
+    mutex->Lock = (SPINLOCK)SPINLOCK_INIT;
+    mutex->Owner = (THREAD*)0;
+    LIST_INIT(&mutex->WaitList);
+    LIST_INIT(&mutex->OwnerLink);
+    mutex->DonationClass = KeSchedulingNormal;
+    mutex->DonationPriority = PriorityLow;
+    mutex->HasDonation = 0;
+    mutex->Abandoned = 0;
+}
+
+// Destroys one unowned mutex object.
+static void KiMutexDestroy(OBJECT* object) {
+    MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
+    if (mutex->Owner || !LIST_IS_EMPTY(&mutex->WaitList) ||
+        mutex->OwnerLink.Fwd != &mutex->OwnerLink ||
+        mutex->OwnerLink.Bwd != &mutex->OwnerLink) {
+        KiMutexBug("KiMutexDestroy", mutex, mutex->Owner,
+                   "owned or waited mutex reached object destruction");
+    }
+}
+
+// Assigns an unowned mutex to one thread and consumes abandonment state.
+static KSTATUS KiMutexAcquireUnownedLocked(MUTEX* mutex, THREAD* thread) {
+    KSTATUS status = mutex->Abandoned ? NTLSTATUS_ABANDONED : NTLSTATUS_SUCCESS;
+    mutex->Abandoned = 0;
+    ObReferenceObject(&mutex->Header);
+    KiSchedulerAcquireMutexOwnership(thread, mutex);
+    return status;
+}
+
+// Creates a mutex object.
+__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out) {
     if (!out) return NTLSTATUS_INVALID_PARAMETER;
-    MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
-    if (!m)  return NTLSTATUS_NO_MEMORY;
-    m->Header.Destroy = KiMutexDestroy;
-    m->Lock  = (SPINLOCK)SPINLOCK_INIT;
-    m->Owner = (struct THREAD *)0;
-    LIST_INIT(&m->WaitList);
-    *out = m;
+    MUTEX* mutex = (MUTEX*)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
+    if (!mutex) return NTLSTATUS_NO_MEMORY;
+    mutex->Header.Destroy = KiMutexDestroy;
+    KiMutexInitialize(mutex);
+    *out = mutex;
     return NTLSTATUS_SUCCESS;
 }
 
-// Implements the acquire Mutex operation.
-__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) {
+// Acquires a mutex within the requested timeout and donates priority while blocked.
+__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds) {
     if (!mutex) return NTLSTATUS_INVALID_PARAMETER;
-    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
+    if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL;
+
     THREAD* self = KeGetCurrentThread();
+    if (!self) return NTLSTATUS_FAILURE;
 
     uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
     if (!mutex->Owner) {
-        mutex->Owner = self;
+        KSTATUS status = KiMutexAcquireUnownedLocked(mutex, self);
         KeReleaseSpinlock(&mutex->Lock, flags);
-        return NTLSTATUS_SUCCESS;
+        return status;
+    }
+
+    if (mutex->Owner == self) {
+        KeReleaseSpinlock(&mutex->Lock, flags);
+        return NTLSTATUS_DEADLOCK;
+    }
+
+    if (timeoutMilliseconds == 0) {
+        KeReleaseSpinlock(&mutex->Lock, flags);
+        return NTLSTATUS_TIMEOUT;
+    }
+
+    THREAD* owner = mutex->Owner;
+    KiMutexReferenceOwnerLocked(mutex, owner);
+    KSTATUS waitStatus = KiSchedulerPrepareMutexWait(self, mutex, owner);
+    if (!KSUCCESS(waitStatus)) {
+        KeReleaseSpinlock(&mutex->Lock, flags);
+        ObDereferenceObject(&owner->Header);
+        return waitStatus;
     }
 
-    KePrepareBlockThread(self, &mutex->Header, 0);
     KeListInsertTailChecked(&mutex->WaitList, &self->SchedulerLink);
-    KeSetThreadWaitList(self, &mutex->WaitList);
+    if (timeoutMilliseconds != KE_WAIT_INFINITE) {
+        KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT);
+        if (!KSUCCESS(timeoutStatus)) {
+            KeListRemoveChecked(&self->SchedulerLink);
+            KiSchedulerCancelMutexWait(self, mutex);
+            (void)KiSchedulerRefreshMutexDonation(mutex);
+            KeUnblockThreadWithStatus(self, timeoutStatus);
+            KeReleaseSpinlock(&mutex->Lock, flags);
+
+            KiMutexPropagatePriority(owner);
+            ObDereferenceObject(&owner->Header);
+            return timeoutStatus;
+        }
+    }
+    (void)KiSchedulerRefreshMutexDonation(mutex);
     KeReleaseSpinlock(&mutex->Lock, flags);
+
+    KiMutexPropagatePriority(owner);
+    ObDereferenceObject(&owner->Header);
     return KeCommitBlockThread(self, 0);
 }
 
-// Implements the try Acquire Mutex operation.
-__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
-    if (!mutex) return NTLSTATUS_INVALID_PARAMETER;
-    THREAD* self = KeGetCurrentThread();
-    uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
-    if (!mutex->Owner) {
-        mutex->Owner = self;
-        KeReleaseSpinlock(&mutex->Lock, flags);
-        return NTLSTATUS_SUCCESS;
-    }
-    KeReleaseSpinlock(&mutex->Lock, flags);
-    return NTLSTATUS_FAILURE;
-}
-
-// Releases mutex.
+// Releases a mutex and hands ownership to its highest-ranked waiter.
 __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
     if (!mutex) return;
-    uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
 
-    if (!LIST_IS_EMPTY(&mutex->WaitList)) {
-        LIST_ENTRY* node = KeListPopHeadChecked(&mutex->WaitList);
-        THREAD* waiter   = CONTAINING_RECORD(node, THREAD, SchedulerLink);
-        KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
-        // Direct handoff: the selected waiter owns the mutex when its wait
-        // returns. Do not drop Owner to NULL, or another thread can steal it.
-        mutex->Owner = waiter;
-        KeReleaseSpinlock(&mutex->Lock, flags);
+    THREAD* self = KeGetCurrentThread();
+    uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
+    if (!self || mutex->Owner != self)
+        KiMutexBug("KeReleaseMutex", mutex, self, "release by non-owner");
+
+    THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex);
+    if (waiter) {
+        KeListRemoveChecked(&waiter->SchedulerLink);
+        KiSchedulerTransferMutexOwnership(self, waiter, mutex);
+        (void)KiSchedulerRefreshMutexDonation(mutex);
+        (void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0);
         KeUnblockThreadWithStatus(waiter, NTLSTATUS_SUCCESS);
+        KeReleaseSpinlock(&mutex->Lock, flags);
+
+        KiMutexPropagatePriority(self);
         return;
     }
 
-    mutex->Owner = (struct THREAD *)0;
+    KiSchedulerReleaseMutexOwnership(self, mutex);
+    (void)KiSchedulerRefreshMutexDonation(mutex);
     KeReleaseSpinlock(&mutex->Lock, flags);
+
+    KiMutexPropagatePriority(self);
+    ObDereferenceObject(&mutex->Header);
+}
+
+// Removes one blocked mutex waiter during thread-wait rundown.
+int KiMutexAbortWait(MUTEX* mutex, THREAD* thread) {
+    if (!mutex || !thread) return 0;
+
+    uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
+    if (thread->WaitingMutex != mutex ||
+        thread->WaitObject != &mutex->Header ||
+        thread->WaitListHead != &mutex->WaitList ||
+        !KiMutexWaiterIsLinked(thread)) {
+        KeReleaseSpinlock(&mutex->Lock, flags);
+        return 0;
+    }
+
+    KeListRemoveChecked(&thread->SchedulerLink);
+    KiSchedulerCancelMutexWait(thread, mutex);
+    THREAD* owner = mutex->Owner;
+    if (owner) KiMutexReferenceOwnerLocked(mutex, owner);
+    (void)KiSchedulerRefreshMutexDonation(mutex);
+    KeReleaseSpinlock(&mutex->Lock, flags);
+
+    KiMutexPropagatePriority(owner);
+    if (owner) ObDereferenceObject(&owner->Header);
+    KiMutexPropagatePriority(thread);
+    return 1;
+}
+
+// Abandons one mutex owned by a terminating thread.
+static void KiMutexAbandonOwned(MUTEX* mutex, THREAD* owner) {
+    uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
+    if (mutex->Owner != owner)
+        KiMutexBug("KiMutexAbandonOwned", mutex, owner, "ownership list mismatch");
+
+    THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex);
+    if (waiter) {
+        KeListRemoveChecked(&waiter->SchedulerLink);
+        KiSchedulerTransferMutexOwnership(owner, waiter, mutex);
+        mutex->Abandoned = 0;
+        (void)KiSchedulerRefreshMutexDonation(mutex);
+        (void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0);
+        KeUnblockThreadWithStatus(waiter, NTLSTATUS_ABANDONED);
+        KeReleaseSpinlock(&mutex->Lock, flags);
+
+        KiMutexPropagatePriority(owner);
+        return;
+    }
+
+    KiSchedulerReleaseMutexOwnership(owner, mutex);
+    mutex->Abandoned = 1;
+    (void)KiSchedulerRefreshMutexDonation(mutex);
+    KeReleaseSpinlock(&mutex->Lock, flags);
+
+    KiMutexPropagatePriority(owner);
+    ObDereferenceObject(&mutex->Header);
+}
+
+// Releases every mutex still owned by a terminating thread.
+void KiMutexRundownThread(THREAD* thread) {
+    if (!thread) return;
+
+    while (true) {
+        MUTEX* mutex = KiSchedulerFirstOwnedMutex(thread);
+        if (!mutex)
+            return;
+        KiMutexAbandonOwned(mutex, thread);
+    }
 }

code/kernel/ke_process.c

@@ -7,6 +7,7 @@
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
 #include <kernel/ke_event.h>
+#include <kernel/ke_debug.h>
 #include <kernel/ke_system_root.h>
 #include <kernel/ke_userenv.h>
 #include <kernel/io_file.h>
@@ -14,6 +15,7 @@
 #include <kernel/ob_object.h>
 #include <kernel/ke_named_queue.h>
 #include <kernel/ke_cpu.h>
+#include <kernel/ke_audio.h>
 #include <kernel/hal/hal_apic.h>
 #include <ntlite/syscall.h>
 
@@ -217,6 +219,7 @@ static uint32_t KiThreadStateToSys(THREAD_STATE state) {
         case ThreadRunning: return SYS_THREAD_STATE_RUNNING;
         case ThreadWaiting: return SYS_THREAD_STATE_WAITING;
         case ThreadDead:    return SYS_THREAD_STATE_DEAD;
+        case ThreadSuspended: return SYS_THREAD_STATE_SUSPENDED;
         default:            return SYS_THREAD_STATE_DEAD;
     }
 }
@@ -265,25 +268,33 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
             SYS_THREAD_INFO* info = &infoOut[count];
             memset(info, 0, sizeof(*info));
             info->Size = sizeof(*info);
+            KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
+            KiSchedulerSnapshotThread(thread, &snapshot);
+
             info->ProcessId = process->ProcessId;
             info->ThreadId = thread->ThreadId;
-            info->State = KiThreadStateToSys(thread->State);
-            info->Priority = (uint32_t)thread->Priority;
-            info->RunningCpu = thread->RunningCpu;
-            info->LastCpu = thread->LastCpu;
-            info->QuantumRemaining = thread->QuantumRemaining;
-            info->QuantumTicks = thread->QuantumTicks;
-            info->CpuBurstTicks = thread->CpuBurstTicks;
-            info->LastRunTicks = thread->LastRunTicks;
-            info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE;
-            info->WaitStatus = (uint32_t)thread->WaitStatus;
+            info->State = KiThreadStateToSys(snapshot.State);
+            info->BaseSchedulingClass = (uint32_t)snapshot.BaseSchedulingClass;
+            info->BasePriority = snapshot.BasePriority;
+            info->EffectiveSchedulingClass = (uint32_t)snapshot.EffectiveSchedulingClass;
+            info->EffectivePriority = snapshot.EffectivePriority;
+            info->RunningCpu = snapshot.RunningCpu;
+            info->LastCpu = snapshot.LastCpu;
+            info->QuantumRemaining = snapshot.QuantumRemaining;
+            info->QuantumTicks = snapshot.QuantumTicks;
+            info->CpuBurstTicks = snapshot.CpuBurstTicks;
+            info->LastRunTicks = snapshot.LastRunTicks;
+            info->WaitObjectType = snapshot.WaitObjectType != 0
+                ? KiWaitObjectTypeToSys((OBJECT_TYPE)snapshot.WaitObjectType)
+                : SYS_THREAD_WAIT_NONE;
+            info->WaitStatus = (uint32_t)snapshot.WaitStatus;
             info->KernelApcDisable = thread->KernelApcDisable;
 
             if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT;
             if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE;
             if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL;
-            if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED;
-            if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT;
+            if (snapshot.Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED;
+            if (snapshot.AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT;
             if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING;
             if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING;
         }
@@ -296,10 +307,26 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
     return NTLSTATUS_SUCCESS;
 }
 
-// Releases all kernel objects still reachable from a process handle table.
-// This is part of process exit, not merely object destruction: a parent may keep
-// a handle to an exited process for wait/query, but the dead process must not
-// keep its own files, mappings, queues, sockets, or synchronization objects alive.
+// Records type-specific ownership after a handle slot becomes visible.
+static void KiProcessHandleOpened(OBJECT* object) {
+    if (!object)
+        return;
+
+    if (object->Type == ObjectTypeAudioStream)
+        KeAudioStreamHandleOpened((KE_AUDIO_STREAM*)object);
+}
+
+// Releases type-specific ownership before the handle-table reference drops.
+static void KiProcessReleaseHandleObject(OBJECT* object) {
+    if (!object)
+        return;
+
+    if (object->Type == ObjectTypeAudioStream)
+        KeAudioStreamHandleClosed((KE_AUDIO_STREAM*)object);
+    ObDereferenceObject(object);
+}
+
+// Releases every handle-table reference owned by one process.
 static void KiProcessReleaseAllHandles(PROCESS* proc) {
     if (!proc) return;
 
@@ -317,7 +344,7 @@ static void KiProcessReleaseAllHandles(PROCESS* proc) {
 
         if (obj) {
             KeNamedQueueUnpublishHandle(proc, (uint32_t)i, obj);
-            ObDereferenceObject(obj);
+            KiProcessReleaseHandleObject(obj);
         }
     }
 }
@@ -671,14 +698,16 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
         if (t == self)
             continue;
 
-        // A terminating process must not be held open by worker threads that
-        // are blocked in non-alertable waits. User-mode DLLs such as Bink can
-        // leave IO/submit workers parked on semaphores; after the foreground
-        // thread crashes, those waiters would otherwise keep ThreadCount > 0
-        // and prevent the process ExitEvent from waking the shell.
-        if (t->State == ThreadWaiting)
+        // Most user waits must be interrupted immediately. A thread can defer
+        // termination only while synchronous kernel work still owns its stack
+        // or process memory; that thread is allowed to finish the ownership
+        // interval before the normal termination gates reap it.
+        int deferred = KeThreadTerminationDeferred(t);
+        if (t->State == ThreadSuspended && !deferred)
+            (void)KeResumeThread(t, (uint32_t*)0);
+        else if (t->State == ThreadWaiting && !deferred)
             KeAbortThreadWait(t, NTLSTATUS_ALERTED);
-        else
+        else if (!deferred)
             (void)KeAlertThread(t);
 
         if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
@@ -727,6 +756,7 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri
             process->Handles[i].Object = object;
             process->Handles[i].Rights = rights;
             process->Handles[i].Flags = 0;
+            KiProcessHandleOpened(object);
             *handleOut = (uint32_t)i;
             KeReleaseSpinlock(&process->HandleLock, flags);
             return NTLSTATUS_SUCCESS;
@@ -756,7 +786,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) {
         if (obj->Type == ObjectTypeFile)
             (void)IoFlushFile((FILE_OBJECT*)obj);
         KeNamedQueueUnpublishHandle(process, handle, obj);
-        ObDereferenceObject(obj);
+        KiProcessReleaseHandleObject(obj);
         return;
     }
     KeReleaseSpinlock(&process->HandleLock, flags);

code/kernel/ke_queue.c

@@ -133,7 +133,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMi
     if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
         deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
 
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
         if (queue->Count < QUEUE_DEPTH) {
             uint32_t previousCount = queue->Count;
@@ -165,7 +165,7 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilli
     if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
         deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
 
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
         if (queue->Count > 0) {
             uint32_t previousCount = queue->Count;

code/kernel/ke_selftest.c

@@ -11,11 +11,14 @@
 #include <kernel/ke_event.h>
 #include <kernel/ke_irp.h>
 #include <kernel/ke_irql.h>
+#include <kernel/ke_mutex.h>
+#include <kernel/ke_semaphore.h>
 #include <kernel/ke_sched.h>
 #include <ntlite/status.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_process.h>
 #include <kernel/io_file.h>
+#include <kernel/io_dma.h>
 #include <ntlite/ddi/io_driver.h>
 #include <kernel/ke_trap.h>
 #include <kernel/mm_virt.h>
@@ -240,9 +243,10 @@ static void KiRunKernelApcSelfTest(void) {
         s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
         if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: FinishedEvent create failed %d\n", s);
 
+        const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
         s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker,
                                  (uintptr_t)&st,
-                                 PriorityNormal,
+                                 &workerParameters,
                                  &st.WorkerThread);
         if (!KSUCCESS(s))
             KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker create failed %d\n", s);
@@ -310,9 +314,10 @@ static void KiRunKernelApcSelfTest(void) {
         s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
         if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert FinishedEvent create failed %d\n", s);
 
+        const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
         s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker,
                                  (uintptr_t)&st,
-                                 PriorityNormal,
+                                 &workerParameters,
                                  &st.WorkerThread);
         if (!KSUCCESS(s))
             KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable worker create failed %d\n", s);
@@ -584,6 +589,213 @@ static void KiRunPfnDatabaseSelfTest(void) {
     DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
 }
 
+// Verifies common-buffer, direct mapping, bounce mapping, and teardown accounting.
+static void KiRunDmaSelfTest(void) {
+    const NTL_DDK_DMA* dma = IoGetDmaDdk();
+    if (!dma)
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: DDK table missing\n");
+
+    IO_DMA_STATS before;
+    IO_DMA_STATS after;
+    dma->QueryStats(&before);
+
+    DRIVER_OBJECT driver;
+    memset(&driver, 0, sizeof(driver));
+    IoInitializeDriverObject(&driver, SYS_SV_LITERAL("dma-selftest"));
+
+    DEVICE_OBJECT* device = (DEVICE_OBJECT*)0;
+    KSTATUS status = IoCreateDevice(&driver, 0, &device);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: device creation failed status=%u\n", status);
+
+    IO_DMA_REQUIREMENTS requirements;
+    memset(&requirements, 0, sizeof(requirements));
+    requirements.MaximumDeviceAddress = UINT64_MAX;
+    requirements.Alignment = PAGE_SIZE;
+    requirements.MaximumSegmentLength = PAGE_SIZE;
+    requirements.MaximumSegments = 4;
+
+    IO_DMA_ADAPTER* adapter = (IO_DMA_ADAPTER*)0;
+    IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
+    MM_PINNED_MEMORY* pinned = (MM_PINNED_MEMORY*)0;
+    IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)0;
+
+    status = dma->GetAdapter(device, &requirements, &adapter);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: adapter creation failed status=%u\n", status);
+
+    status = dma->AllocateCommonBuffer(adapter, PAGE_SIZE, PAGE_SIZE, &buffer);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: common buffer allocation failed status=%u\n", status);
+
+    uint8_t* bytes = (uint8_t*)dma->GetCommonBufferAddress(buffer);
+    uint64_t deviceAddress = dma->GetCommonBufferDeviceAddress(buffer);
+    if (!bytes || !deviceAddress || (deviceAddress & (PAGE_SIZE - 1)) != 0 ||
+        dma->GetCommonBufferSize(buffer) != PAGE_SIZE)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "DMA selftest: common buffer metadata invalid address=0x%016llX\n",
+                (unsigned long long)deviceAddress);
+
+    status = dma->PinKernelMemory(bytes, PAGE_SIZE, IoDmaBidirectional, &pinned);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: kernel pin failed status=%u\n", status);
+    status = dma->MapPinnedMemory(adapter,
+                                  pinned,
+                                  0,
+                                  PAGE_SIZE,
+                                  IoDmaBidirectional,
+                                  IO_DMA_MAP_NO_BOUNCE,
+                                  &mapping);
+    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
+        dma->GetSegments(mapping)[0].DeviceAddress != deviceAddress)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "DMA selftest: direct mapping failed status=%u segments=%u\n",
+                status,
+                dma->GetSegmentCount(mapping));
+    dma->UnmapMemory(mapping);
+    mapping = (IO_DMA_MAPPING*)0;
+    dma->UnpinMemory(pinned);
+    pinned = (MM_PINNED_MEMORY*)0;
+
+    for (size_t i = 0; i < 128; i++)
+        bytes[i + 1] = (uint8_t)(i ^ 0x5Au);
+    status = dma->PinKernelMemory(bytes + 1, 128, IoDmaToDevice, &pinned);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: unaligned kernel pin failed status=%u\n", status);
+    status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaToDevice, 0, &mapping);
+    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
+        (dma->GetSegments(mapping)[0].DeviceAddress & (PAGE_SIZE - 1)) != 0 ||
+        dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "DMA selftest: bounce mapping failed status=%u segments=%u\n",
+                status,
+                dma->GetSegmentCount(mapping));
+    status = dma->SynchronizeForDevice(mapping);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: bounce synchronization failed status=%u\n", status);
+    dma->UnmapMemory(mapping);
+    mapping = (IO_DMA_MAPPING*)0;
+    dma->UnpinMemory(pinned);
+    pinned = (MM_PINNED_MEMORY*)0;
+
+    memset(bytes + 1, 0, 128);
+    status = dma->PinKernelMemory(bytes + 1, 128, IoDmaFromDevice, &pinned);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive pin failed status=%u\n", status);
+    status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaFromDevice, 0, &mapping);
+    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
+        dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "DMA selftest: receive bounce mapping failed status=%u segments=%u\n",
+                status,
+                dma->GetSegmentCount(mapping));
+    status = dma->SynchronizeForDevice(mapping);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive handoff failed status=%u\n", status);
+
+    uint64_t receiveDeviceAddress = dma->GetSegments(mapping)[0].DeviceAddress;
+    uint8_t* receiveDeviceAlias = (uint8_t*)MmPfnToVirtual(receiveDeviceAddress & PAGE_MASK);
+    if (!receiveDeviceAlias)
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive device alias missing\n");
+    receiveDeviceAlias += receiveDeviceAddress & (PAGE_SIZE - 1);
+    for (size_t i = 0; i < 128; i++)
+        receiveDeviceAlias[i] = (uint8_t)(i ^ 0xA5u);
+
+    status = dma->SynchronizeForCpu(mapping);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive synchronization failed status=%u\n", status);
+    for (size_t i = 0; i < 128; i++) {
+        if (bytes[i + 1] != (uint8_t)(i ^ 0xA5u))
+            KePanic(KE_PANIC_CODE_SELFTEST,
+                    "DMA selftest: receive bounce copy mismatch offset=%llu\n",
+                    (unsigned long long)i);
+    }
+    dma->UnmapMemory(mapping);
+    mapping = (IO_DMA_MAPPING*)0;
+    dma->UnpinMemory(pinned);
+    pinned = (MM_PINNED_MEMORY*)0;
+
+    PROCESS* process = KeGetCurrentProcess();
+    if (!process)
+        process = KeGetSystemProcess();
+    if (!process)
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: no process for user pinning\n");
+
+    void* userBase = (void*)0;
+    status = MmVadCreateCommitted(process,
+                                  PAGE_SIZE,
+                                  PAGE_SIZE,
+                                  PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
+                                  MM_VAD_ZERO,
+                                  MmVadGeneric,
+                                  &userBase);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD creation failed status=%u\n", status);
+    status = MmVadEnsureResident(process, userBase, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD materialization failed status=%u\n", status);
+    ((uint8_t*)userBase)[0] = 0xD5u;
+
+    status = dma->PinUserMemory(process, userBase, PAGE_SIZE, IoDmaToDevice, &pinned);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user pin failed status=%u\n", status);
+    status = MmVadDestroy(process, userBase);
+    if (status != NTLSTATUS_ACCESS_DENIED)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "DMA selftest: pinned user VAD was destroyable status=%u\n",
+                status);
+    status = dma->MapPinnedMemory(adapter,
+                                  pinned,
+                                  0,
+                                  PAGE_SIZE,
+                                  IoDmaToDevice,
+                                  IO_DMA_MAP_NO_BOUNCE,
+                                  &mapping);
+    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "DMA selftest: user mapping failed status=%u segments=%u\n",
+                status,
+                dma->GetSegmentCount(mapping));
+    status = dma->SynchronizeForDevice(mapping);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user synchronization failed status=%u\n", status);
+    dma->UnmapMemory(mapping);
+    mapping = (IO_DMA_MAPPING*)0;
+    dma->UnpinMemory(pinned);
+    pinned = (MM_PINNED_MEMORY*)0;
+    status = MmVadDestroy(process, userBase);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD destroy failed status=%u\n", status);
+
+    dma->FreeCommonBuffer(buffer);
+    dma->PutAdapter(adapter);
+    IoDeleteDevice(device);
+
+    dma->QueryStats(&after);
+    if (after.AdaptersCreated != before.AdaptersCreated + 1 ||
+        after.AdaptersDestroyed != before.AdaptersDestroyed + 1 ||
+        after.AdaptersInUse != before.AdaptersInUse ||
+        after.CommonBuffersAllocated < before.CommonBuffersAllocated + 3 ||
+        after.CommonBuffersFreed < before.CommonBuffersFreed + 3 ||
+        after.CommonBuffersInUse != before.CommonBuffersInUse ||
+        after.CommonBufferBytesInUse != before.CommonBufferBytesInUse ||
+        after.PinnedRangesInUse != before.PinnedRangesInUse ||
+        after.PinnedPagesInUse != before.PinnedPagesInUse ||
+        after.PinnedKernelRanges != before.PinnedKernelRanges + 3 ||
+        after.PinnedUserRanges != before.PinnedUserRanges + 1 ||
+        after.PinnedRangesReleased != before.PinnedRangesReleased + 4 ||
+        after.StreamingMappings != before.StreamingMappings + 4 ||
+        after.StreamingMappingsUnmapped != before.StreamingMappingsUnmapped + 4 ||
+        after.StreamingMappingsInUse != before.StreamingMappingsInUse ||
+        after.DirectMappings != before.DirectMappings + 2 ||
+        after.BounceMappings != before.BounceMappings + 2 ||
+        after.BounceBytesCopiedToDevice < before.BounceBytesCopiedToDevice + 128 ||
+        after.BounceBytesCopiedToCpu < before.BounceBytesCopiedToCpu + 128)
+        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: accounting mismatch\n");
+
+    DbgPrint("DMA selftest: common/kernel-pin/user-pin/direct/send-bounce/receive-bounce/teardown PASS\n");
+}
+
 // Verifies the page-table manager invariants that the higher VMM will depend on.
 static void KiRunMemoryMappingSelfTest(void) {
     PHYSICAL_ADDRESS phys = 0;
@@ -1155,6 +1367,380 @@ static void KiRunAsyncFileIoSelfTest(void) {
     DbgPrint("FILE/ASYNC selftest: PASS\n");
 }
 
+typedef struct KI_SCHEDULER_SELFTEST_STATE {
+    MUTEX* GateMutex;
+    MUTEX* LinkMutex;
+    EVENT* ChainReadyEvent;
+    EVENT* RealtimeReadyEvent;
+    EVENT* ChainFinishedEvent;
+    EVENT* RealtimeFinishedEvent;
+    volatile KSTATUS ChainAcquireStatus;
+    volatile KSTATUS RealtimeAcquireStatus;
+} KI_SCHEDULER_SELFTEST_STATE;
+
+// Waits until one worker has entered the expected mutex wait state.
+static KE_THREAD_SCHEDULER_SNAPSHOT KiSchedulerAwaitMutexWait(THREAD* thread, MUTEX* mutex) {
+    KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
+    memset(&snapshot, 0, sizeof(snapshot));
+
+    for (uint32_t attempt = 0; attempt < 100; attempt++) {
+        KiSchedulerSnapshotThread(thread, &snapshot);
+        if (snapshot.State == ThreadWaiting && snapshot.WaitingMutex == mutex)
+            return snapshot;
+
+        KSTATUS status = KeDelayThread(SCHED_TICK_MS);
+        if (!KSUCCESS(status))
+            KePanic(KE_PANIC_CODE_SELFTEST,
+                    "SCHED selftest: state wait failed status=%u\n", status);
+    }
+
+    KePanic(KE_PANIC_CODE_SELFTEST,
+            "SCHED selftest: thread %u did not block on mutex=0x%016llX state=%u waiting=0x%016llX\n",
+            thread ? thread->ThreadId : 0,
+            (uint64_t)mutex,
+            (unsigned)snapshot.State,
+            (uint64_t)snapshot.WaitingMutex);
+}
+
+typedef struct KI_SYNC_TIMEOUT_SELFTEST_STATE {
+    MUTEX* Mutex;
+    EVENT* ReadyEvent;
+    EVENT* FinishedEvent;
+    volatile KSTATUS MutexStatus;
+} KI_SYNC_TIMEOUT_SELFTEST_STATE;
+
+// Waits on the held mutex long enough for the timeout engine to remove donation state.
+static void KiSynchronizationTimeoutWorker(void* argument) {
+    KI_SYNC_TIMEOUT_SELFTEST_STATE* state = (KI_SYNC_TIMEOUT_SELFTEST_STATE*)argument;
+    KeSignalEvent(state->ReadyEvent);
+
+    state->MutexStatus = KeAcquireMutex(state->Mutex, SCHED_TICK_MS * 20ULL);
+    if (KE_MUTEX_ACQUIRE_SUCCEEDED(state->MutexStatus))
+        KeReleaseMutex(state->Mutex);
+
+    KeSignalEvent(state->FinishedEvent);
+    KeExitThread(KeGetCurrentThread(), 0);
+}
+
+// Verifies finite mutex and semaphore waits consume no object state after timeout.
+static void KiRunSynchronizationTimeoutSelfTest(void) {
+    DbgPrint("SYNC/TIMEOUT selftest: begin\n");
+
+    KI_SYNC_TIMEOUT_SELFTEST_STATE state;
+    memset(&state, 0, sizeof(state));
+    state.MutexStatus = NTLSTATUS_PENDING;
+
+    KSTATUS status = KeMutexCreate(&state.Mutex);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: mutex create failed status=%u\n", status);
+
+    status = KeEventCreate(EventAutoReset, 0, &state.ReadyEvent);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: ready event create failed status=%u\n", status);
+
+    status = KeEventCreate(EventAutoReset, 0, &state.FinishedEvent);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: finished event create failed status=%u\n", status);
+
+    status = KeAcquireMutex(state.Mutex, KE_WAIT_INFINITE);
+    if (status != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: owner acquire failed status=%u\n", status);
+
+    THREAD* worker = (THREAD*)0;
+    const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_REALTIME(12);
+    status = KeCreateKernelThread((uintptr_t)KiSynchronizationTimeoutWorker,
+                                  (uintptr_t)&state,
+                                  &parameters,
+                                  &worker);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: worker create failed status=%u\n", status);
+
+    status = KeWaitForEvent(state.ReadyEvent, 0);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: ready wait failed status=%u\n", status);
+
+    (void)KiSchedulerAwaitMutexWait(worker, state.Mutex);
+
+    THREAD* self = KeGetCurrentThread();
+    KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
+    KiSchedulerSnapshotThread(self, &snapshot);
+    if (snapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
+        snapshot.EffectivePriority != 12) {
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: finite waiter donation missing class=%u priority=%u\n",
+                snapshot.EffectiveSchedulingClass,
+                snapshot.EffectivePriority);
+    }
+
+    status = KeWaitForEvent(state.FinishedEvent, 0);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: finished wait failed status=%u\n", status);
+    if (state.MutexStatus != NTLSTATUS_TIMEOUT)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: mutex wait returned status=%u\n",
+                state.MutexStatus);
+
+    KiSchedulerSnapshotThread(self, &snapshot);
+    if (snapshot.EffectiveSchedulingClass != snapshot.BaseSchedulingClass ||
+        snapshot.EffectivePriority != snapshot.BasePriority) {
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: timed-out donation remained base=%u/%u effective=%u/%u\n",
+                snapshot.BaseSchedulingClass,
+                snapshot.BasePriority,
+                snapshot.EffectiveSchedulingClass,
+                snapshot.EffectivePriority);
+    }
+
+    KeReleaseMutex(state.Mutex);
+    status = KeAcquireMutex(state.Mutex, 0);
+    if (status != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: zero-timeout mutex acquire failed status=%u\n", status);
+    KeReleaseMutex(state.Mutex);
+
+    SEMAPHORE* semaphore = (SEMAPHORE*)0;
+    status = KeSemaphoreCreate(0, 1, &semaphore);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: semaphore create failed status=%u\n", status);
+
+    status = KeWaitSemaphore(semaphore, 0);
+    if (status != NTLSTATUS_TIMEOUT)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: zero-timeout semaphore wait returned status=%u\n", status);
+
+    status = KeWaitSemaphore(semaphore, SCHED_TICK_MS * 3ULL);
+    if (status != NTLSTATUS_TIMEOUT)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: finite semaphore wait returned status=%u\n", status);
+
+    KeSignalSemaphore(semaphore);
+    status = KeWaitSemaphore(semaphore, 0);
+    if (status != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: signaled semaphore probe failed status=%u\n", status);
+    status = KeWaitSemaphore(semaphore, 0);
+    if (status != NTLSTATUS_TIMEOUT)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SYNC/TIMEOUT selftest: semaphore count was consumed twice status=%u\n", status);
+
+    ObDereferenceObject(&worker->Header);
+    ObDereferenceObject(&semaphore->Header);
+    ObDereferenceObject(&state.Mutex->Header);
+    ObDereferenceObject(&state.ReadyEvent->Header);
+    ObDereferenceObject(&state.FinishedEvent->Header);
+
+    DbgPrint("SYNC/TIMEOUT selftest: PASS\n");
+}
+
+// Owns the link mutex and then blocks behind the self-test thread.
+static void KiSchedulerChainWorker(void* argument) {
+    KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;
+
+    KSTATUS status = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
+    if (status != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: chain link acquire failed status=%u\n", status);
+
+    KeSignalEvent(state->ChainReadyEvent);
+    state->ChainAcquireStatus = KeAcquireMutex(state->GateMutex, KE_WAIT_INFINITE);
+    if (state->ChainAcquireStatus != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: chain gate acquire failed status=%u\n",
+                state->ChainAcquireStatus);
+
+    KeReleaseMutex(state->GateMutex);
+    KeReleaseMutex(state->LinkMutex);
+    KeSignalEvent(state->ChainFinishedEvent);
+    KeExitThread(KeGetCurrentThread(), 0);
+}
+
+// Blocks a real-time thread behind the normal chain worker.
+static void KiSchedulerRealtimeWorker(void* argument) {
+    KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;
+    KeSignalEvent(state->RealtimeReadyEvent);
+
+    state->RealtimeAcquireStatus = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
+    if (state->RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: real-time acquire failed status=%u\n",
+                state->RealtimeAcquireStatus);
+
+    KeReleaseMutex(state->LinkMutex);
+    KeSignalEvent(state->RealtimeFinishedEvent);
+    KeExitThread(KeGetCurrentThread(), 0);
+}
+
+// Verifies real-time dispatch and transitive mutex priority inheritance.
+static void KiRunRealtimeSchedulerSelfTest(void) {
+    DbgPrint("SCHED selftest: begin\n");
+
+    THREAD* self = KeGetCurrentThread();
+    if (!self || self->BaseSchedulingClass != KeSchedulingNormal)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: caller is not a normal thread\n");
+
+    KE_SCHED_STATS before;
+    KE_SCHED_STATS after;
+    KeGetSchedulerStats(&before);
+
+    KI_SCHEDULER_SELFTEST_STATE state;
+    memset(&state, 0, sizeof(state));
+    state.ChainAcquireStatus = NTLSTATUS_PENDING;
+    state.RealtimeAcquireStatus = NTLSTATUS_PENDING;
+
+    KSTATUS status = KeMutexCreate(&state.GateMutex);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: gate mutex create failed status=%u\n", status);
+
+    status = KeMutexCreate(&state.LinkMutex);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: link mutex create failed status=%u\n", status);
+
+    status = KeEventCreate(EventAutoReset, 0, &state.ChainReadyEvent);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: chain ready event create failed status=%u\n", status);
+
+    status = KeEventCreate(EventAutoReset, 0, &state.RealtimeReadyEvent);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: real-time ready event create failed status=%u\n", status);
+
+    status = KeEventCreate(EventAutoReset, 0, &state.ChainFinishedEvent);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: chain finished event create failed status=%u\n", status);
+
+    status = KeEventCreate(EventAutoReset, 0, &state.RealtimeFinishedEvent);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: real-time finished event create failed status=%u\n", status);
+
+    status = KeAcquireMutex(state.GateMutex, KE_WAIT_INFINITE);
+    if (status != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: owner acquire failed status=%u\n", status);
+
+    THREAD* chainWorker = (THREAD*)0;
+    const KE_THREAD_CREATE_PARAMETERS chainParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
+    status = KeCreateKernelThread((uintptr_t)KiSchedulerChainWorker,
+                                  (uintptr_t)&state,
+                                  &chainParameters,
+                                  &chainWorker);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: chain worker create failed status=%u\n", status);
+
+    status = KeWaitForEvent(state.ChainReadyEvent, 0);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: chain ready wait failed status=%u\n", status);
+
+    (void)KiSchedulerAwaitMutexWait(chainWorker, state.GateMutex);
+
+    status = KeAcquireMutex(state.LinkMutex, KE_WAIT_INFINITE);
+    if (status != NTLSTATUS_DEADLOCK)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: mutex cycle was not rejected status=%u\n", status);
+
+    THREAD* realtimeWorker = (THREAD*)0;
+    const KE_THREAD_CREATE_PARAMETERS realtimeParameters = KE_THREAD_CREATE_REALTIME(16);
+    status = KeCreateKernelThread((uintptr_t)KiSchedulerRealtimeWorker,
+                                  (uintptr_t)&state,
+                                  &realtimeParameters,
+                                  &realtimeWorker);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: real-time worker create failed status=%u\n", status);
+
+    status = KeWaitForEvent(state.RealtimeReadyEvent, 0);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: real-time ready wait failed status=%u\n", status);
+
+    KE_THREAD_SCHEDULER_SNAPSHOT realtimeSnapshot =
+        KiSchedulerAwaitMutexWait(realtimeWorker, state.LinkMutex);
+    (void)realtimeSnapshot;
+
+    KE_THREAD_SCHEDULER_SNAPSHOT chainSnapshot;
+    KE_THREAD_SCHEDULER_SNAPSHOT selfSnapshot;
+    KiSchedulerSnapshotThread(chainWorker, &chainSnapshot);
+    KiSchedulerSnapshotThread(self, &selfSnapshot);
+
+    if (chainSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
+        chainSnapshot.EffectivePriority != 16)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: intermediate donation missing class=%u priority=%u\n",
+                chainSnapshot.EffectiveSchedulingClass,
+                chainSnapshot.EffectivePriority);
+
+    if (selfSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
+        selfSnapshot.EffectivePriority != 16)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: transitive donation missing class=%u priority=%u\n",
+                selfSnapshot.EffectiveSchedulingClass,
+                selfSnapshot.EffectivePriority);
+
+    KeReleaseMutex(state.GateMutex);
+
+    KiSchedulerSnapshotThread(self, &selfSnapshot);
+    if (selfSnapshot.EffectiveSchedulingClass != selfSnapshot.BaseSchedulingClass ||
+        selfSnapshot.EffectivePriority != selfSnapshot.BasePriority)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: inherited priority was not restored base=%u/%u effective=%u/%u\n",
+                selfSnapshot.BaseSchedulingClass,
+                selfSnapshot.BasePriority,
+                selfSnapshot.EffectiveSchedulingClass,
+                selfSnapshot.EffectivePriority);
+
+    status = KeWaitForEvent(state.ChainFinishedEvent, 0);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: chain finished wait failed status=%u\n", status);
+
+    status = KeWaitForEvent(state.RealtimeFinishedEvent, 0);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: real-time finished wait failed status=%u\n", status);
+
+    if (state.ChainAcquireStatus != NTLSTATUS_SUCCESS ||
+        state.RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: handoff statuses chain=%u realtime=%u\n",
+                state.ChainAcquireStatus,
+                state.RealtimeAcquireStatus);
+
+    KeGetSchedulerStats(&after);
+    if (after.RealtimeDispatches <= before.RealtimeDispatches ||
+        after.PriorityInheritanceBoosts < before.PriorityInheritanceBoosts + 2 ||
+        after.PriorityInheritanceDrops < before.PriorityInheritanceDrops + 2 ||
+        after.PriorityInheritancePropagations < before.PriorityInheritancePropagations + 2 ||
+        after.PriorityInheritanceMaxDepth < 2)
+        KePanic(KE_PANIC_CODE_SELFTEST,
+                "SCHED selftest: scheduler counters did not prove transitive inheritance\n");
+
+    ObDereferenceObject(&chainWorker->Header);
+    ObDereferenceObject(&realtimeWorker->Header);
+    ObDereferenceObject(&state.GateMutex->Header);
+    ObDereferenceObject(&state.LinkMutex->Header);
+    ObDereferenceObject(&state.ChainReadyEvent->Header);
+    ObDereferenceObject(&state.RealtimeReadyEvent->Header);
+    ObDereferenceObject(&state.ChainFinishedEvent->Header);
+    ObDereferenceObject(&state.RealtimeFinishedEvent->Header);
+
+    DbgPrint("SCHED selftest: PASS\n");
+}
+
 // Runs trap Frame Self Test.
 static void KiRunTrapFrameSelfTest(void) {
     DbgPrint("TRAP selftest: begin\n");
@@ -1251,8 +1837,11 @@ void KiRunEarlyKernelSelfTests(void) {
 // Runs kernel Self Tests.
 void KiRunKernelSelfTests(const HAL_API* hal) {
     DbgPrint("Kernel selftests: begin\n");
+    KiRunSynchronizationTimeoutSelfTest();
+    KiRunRealtimeSchedulerSelfTest();
     KiRunKernelApcSelfTest();
     KiRunPfnDatabaseSelfTest();
+    KiRunDmaSelfTest();
     KiRunMemoryMappingSelfTest();
     KiRunVadSparseSelfTest();
     KiRunVadCopyOnWriteSelfTest();

code/kernel/ke_semaphore.c

@@ -26,10 +26,10 @@ __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SE
     return NTLSTATUS_SUCCESS;
 }
 
-// Waits for semaphore.
-__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
+// Waits for one semaphore count within the requested timeout.
+__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem, uint64_t timeoutMilliseconds) {
     if (!sem) return NTLSTATUS_INVALID_PARAMETER;
-    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
+    if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL;
     THREAD* self = KeGetCurrentThread();
 
     uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
@@ -39,24 +39,26 @@ __declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
         return NTLSTATUS_SUCCESS;
     }
 
+    if (timeoutMilliseconds == 0) {
+        KeReleaseSpinlock(&sem->Lock, flags);
+        return NTLSTATUS_TIMEOUT;
+    }
+
     KePrepareBlockThread(self, &sem->Header, 0);
     KeListInsertTailChecked(&sem->WaitList, &self->SchedulerLink);
     KeSetThreadWaitList(self, &sem->WaitList);
-    KeReleaseSpinlock(&sem->Lock, flags);
-    return KeCommitBlockThread(self, 0);
-}
-
-// Implements the try Semaphore operation.
-__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
-    if (!sem) return NTLSTATUS_INVALID_PARAMETER;
-    uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
-    if (sem->Count > 0) {
-        sem->Count--;
-        KeReleaseSpinlock(&sem->Lock, flags);
-        return NTLSTATUS_SUCCESS;
+    if (timeoutMilliseconds != KE_WAIT_INFINITE) {
+        KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT);
+        if (!KSUCCESS(timeoutStatus)) {
+            KeListRemoveChecked(&self->SchedulerLink);
+            KeSetThreadWaitList(self, (LIST_ENTRY*)0);
+            KeUnblockThreadWithStatus(self, timeoutStatus);
+            KeReleaseSpinlock(&sem->Lock, flags);
+            return timeoutStatus;
+        }
     }
     KeReleaseSpinlock(&sem->Lock, flags);
-    return NTLSTATUS_FAILURE;
+    return KeCommitBlockThread(self, 0);
 }
 
 // Signals semaphore.

code/kernel/ke_smp.c

@@ -289,6 +289,8 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
     if ((cr4 & AP_CR4_REQUIRED) != AP_CR4_REQUIRED)
         __writecr4(cr4 | AP_CR4_REQUIRED);
 
+    KeInitializeMemoryTypes();
+
     if (gApBootBlock) {
         gApBootBlock->Stage = AP_STAGE_ENTRY;
         gApBootBlock->State = AP_STATE_KERNEL_ENTRY;
@@ -320,7 +322,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
         gApBootBlock->State = AP_STATE_KERNEL_PARKED;
     }
 
-    for (;;) {
+    while (true) {
         cpu->SchedulerTicks++;
 
         if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
@@ -980,11 +982,12 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
                     cpuIndex, st);
         }
 
-        st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker,
-                                              (uintptr_t)arg,
-                                              PriorityNormal,
-                                              (1ULL << cpuIndex),
-                                              (THREAD**)0);
+        KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
+        workerParameters.AffinityMask = 1ULL << cpuIndex;
+        st = KeCreateKernelThread((uintptr_t)KiSmpApControlledWorker,
+                                  (uintptr_t)arg,
+                                  &workerParameters,
+                                  (THREAD**)0);
         if (!KSUCCESS(st)) {
             KePanic(KE_PANIC_CODE_SELFTEST,
                     "SMP scheduler selftest: AP%u worker create failed status=0x%x\n",

code/kernel/ke_switch.asm

@@ -2,16 +2,16 @@
 ;
 ; Struct offsets:
 ;   sizeof(OBJECT)  = 24  (type:4, refcount:4, state:4, flags:4, destroy:8)
-;   THREAD.SavedContext offset = 96
-;   THREAD.TssRsp0  offset     = 112
-;   THREAD.Teb      offset     = 120
-;   THREAD.FpuContext offset   = 176
+;   THREAD.SavedContext offset = 112
+;   THREAD.TssRsp0  offset     = 128
+;   THREAD.Teb      offset     = 136
+;   THREAD.FpuContext offset   = 192
 ;   TSS64.RSP0      offset     = 4
 ;
-THREAD_SavedContext EQU 96
-THREAD_TssRsp0      EQU 112
-THREAD_Teb          EQU 120
-THREAD_FpuContext   EQU 176
+THREAD_SavedContext EQU 112
+THREAD_TssRsp0      EQU 128
+THREAD_Teb          EQU 136
+THREAD_FpuContext   EQU 192
 TSS_RSP0_OFFSET     EQU 4
 
 EXTERN KeSetCurrentTssRsp0 : PROC
@@ -131,12 +131,14 @@ KeStartInitialThread ENDP
 ; We load RCX from the stack (first arg per MS x64), then IRETQ.
 ; -------------------------------------------------------------------------
 EXTERN KeSchedulerPostSwitch : PROC
+EXTERN KePrepareUserThreadStart : PROC
 EXTERN KeGetCurrentThread        : PROC
 EXTERN KeExitThread           : PROC
 
 KeThreadEntry PROC
     sub     rsp, 40    ; Reserve shadow space and restore MS x64 call alignment.
     call    KeSchedulerPostSwitch
+    call    KePrepareUserThreadStart
     add     rsp, 40
     pop     rcx        ; load user arg
     swapgs             ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU

code/kernel/ke_syscall.c

@@ -71,7 +71,9 @@
 #include <kernel/ke_trap.h>
 #include <kernel/ke_input.h>
 #include <kernel/ke_display.h>
+#include <kernel/ke_audio.h>
 #include <kernel/ke_time.h>
+#include <kernel/mm_pool.h>
 #include <ntlite/syscall.h>
 
 // HAL pointer - set once at boot by KeSyscallSetHal().
@@ -182,7 +184,8 @@ typedef struct SURFACE_OBJECT {
     uint64_t PresentBytesCopied;
     PHYSICAL_ADDRESS* Frames;
     void* UserBase;
-    uint32_t* RowScratch;
+    MM_KERNEL_VIEW* KernelView;
+    void* KernelBase;
     MUTEX* Lock;
 } SURFACE_OBJECT;
 
@@ -274,6 +277,12 @@ static void KiSurfaceDestroy(OBJECT* obj) {
         surface->UserBase = (void*)0;
     }
 
+    if (surface->KernelView) {
+        MmDestroyKernelView(surface->KernelView);
+        surface->KernelView = (MM_KERNEL_VIEW*)0;
+        surface->KernelBase = (void*)0;
+    }
+
     if (surface->Lock) {
         ObDereferenceObject(&surface->Lock->Header);
         surface->Lock = (MUTEX*)0;
@@ -288,52 +297,19 @@ static void KiSurfaceDestroy(OBJECT* obj) {
         surface->Frames = (PHYSICAL_ADDRESS*)0;
     }
 
-    if (surface->RowScratch) {
-        MmFreePool(surface->RowScratch);
-        surface->RowScratch = (uint32_t*)0;
-    }
 }
 
-typedef struct SURFACE_ROW_CONTEXT {
-    SURFACE_OBJECT* Surface;
-    uint32_t SourceX;
-    uint32_t SourceY;
-} SURFACE_ROW_CONTEXT;
-
-static KSTATUS KiSurfaceCopyRow(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount) {
-    SURFACE_ROW_CONTEXT* ctx = (SURFACE_ROW_CONTEXT*)context;
-    if (!ctx || !ctx->Surface || !scratch || rowIndex < 0)
+// Creates the process mapping used for user drawing.
+static KSTATUS KiSurfaceEnsureViewLocked(SURFACE_OBJECT* surface) {
+    if (!surface || !surface->Owner || !surface->Frames || !surface->PageCount)
         return NTLSTATUS_INVALID_PARAMETER;
+    if (surface->UserBase)
+        return NTLSTATUS_SUCCESS;
 
-    SURFACE_OBJECT* surface = ctx->Surface;
-    uint64_t row = (uint64_t)ctx->SourceY + (uint64_t)(uint32_t)rowIndex;
-    uint64_t offset = row * (uint64_t)surface->PitchBytes + (uint64_t)ctx->SourceX * sizeof(uint32_t);
-    uint64_t bytes = (uint64_t)pixelCount * sizeof(uint32_t);
-    if (offset > (uint64_t)surface->PayloadSize || bytes > (uint64_t)surface->PayloadSize - offset)
-        return NTLSTATUS_INVALID_PARAMETER;
-
-    uint8_t* dst = (uint8_t*)scratch;
-    while (bytes) {
-        size_t frameIndex = (size_t)(offset / PAGE_SIZE);
-        size_t pageOffset = (size_t)(offset & (PAGE_SIZE - 1));
-        if (frameIndex >= surface->PageCount || !surface->Frames[frameIndex])
-            return NTLSTATUS_INVALID_PARAMETER;
-
-        size_t chunk = PAGE_SIZE - pageOffset;
-        if (chunk > bytes)
-            chunk = (size_t)bytes;
-
-        const uint8_t* src = (const uint8_t*)MmPfnToVirtual(surface->Frames[frameIndex]);
-        if (!src)
-            return NTLSTATUS_INVALID_PARAMETER;
-        memcpy(dst, src + pageOffset, chunk);
-
-        dst += chunk;
-        offset += chunk;
-        bytes -= chunk;
-    }
-
-    return NTLSTATUS_SUCCESS;
+    return MmVadCreateSharedViewFromFrames(surface->Owner, surface->Frames, surface->PageCount,
+                                           surface->PayloadSize,
+                                           PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
+                                           MM_VAD_ZERO, MmVadShared, &surface->UserBase);
 }
 
 // Implements the syscall Init operation.
@@ -513,8 +489,9 @@ static uint64_t SyscallExitThread(uint64_t a0, uint64_t a1, uint64_t a2,
 }
 
 // SYS_CREATE_THREAD a0=*SYS_CREATE_THREAD_REQUEST.
-// Creates a runnable user thread and optionally returns both its waitable
-// handle and diagnostic TID. The handle is the synchronization identity.
+// Creates a user thread and optionally returns both its waitable handle and
+// diagnostic TID. Suspended creation leaves the thread off every ready queue
+// until SYS_RESUME_THREAD decrements its initial suspend count.
 static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
                                      uint64_t a3, uint64_t a4, uint64_t a5) {
     (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
@@ -527,8 +504,11 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
     if (!proc) return NTLSTATUS_FAILURE;
     if (!req.Entry) return NTLSTATUS_INVALID_PARAMETER;
     if (req.Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
-    if (req.Flags) return NTLSTATUS_INVALID_PARAMETER;
+    if (req.Flags & ~SYS_CREATE_THREAD_FLAG_SUSPENDED)
+        return NTLSTATUS_INVALID_PARAMETER;
     if (!req.ThreadHandleOut && !req.ThreadIdOut) return NTLSTATUS_INVALID_PARAMETER;
+    if ((req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) && !req.ThreadHandleOut)
+        return NTLSTATUS_INVALID_PARAMETER;
 
     s = MmProbeUserRead((const void*)req.Entry, 1);
     if (!KSUCCESS(s)) return s;
@@ -542,16 +522,22 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
     }
 
     THREAD* newt = NULL;
+    KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(req.Priority);
+    if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
+        threadParameters.Flags |= KE_THREAD_CREATE_FLAG_SUSPENDED;
     s = KeCreateThread(proc, req.Entry, req.Argument,
-                       (THREAD_PRIORITY)req.Priority, &newt);
+                       &threadParameters, &newt);
     if (!KSUCCESS(s)) return s;
 
     uint32_t threadHandle = 0;
     if (req.ThreadHandleOut) {
         s = KeAllocateHandleWithRights(proc, &newt->Header,
-                                       HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY,
+                                       HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY |
+                                       HANDLE_RIGHT_RESUME,
                                        &threadHandle);
         if (!KSUCCESS(s)) {
+            if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
+                (void)KeCancelSuspendedThread(newt);
             ObDereferenceObject(&newt->Header);
             return s;
         }
@@ -561,6 +547,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
         s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadHandleOut, threadHandle);
         if (!KSUCCESS(s)) {
             KeCloseHandle(proc, threadHandle);
+            if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
+                (void)KeCancelSuspendedThread(newt);
             ObDereferenceObject(&newt->Header);
             return s;
         }
@@ -570,6 +558,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
         s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadIdOut, newt->ThreadId);
         if (!KSUCCESS(s)) {
             if (threadHandle) KeCloseHandle(proc, threadHandle);
+            if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
+                (void)KeCancelSuspendedThread(newt);
             ObDereferenceObject(&newt->Header);
             return s;
         }
@@ -579,6 +569,35 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
     return NTLSTATUS_SUCCESS;
 }
 
+// SYS_RESUME_THREAD a0=threadHandle, a1=previousSuspendCountOut.
+static uint64_t SyscallResumeThread(uint64_t a0, uint64_t a1, uint64_t a2,
+                                    uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a2; (void)a3; (void)a4; (void)a5;
+    if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER;
+
+    KSTATUS status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(uint32_t));
+    if (!KSUCCESS(status)) return status;
+
+    PROCESS* process = KeGetCurrentProcess();
+    if (!process) return NTLSTATUS_FAILURE;
+
+    OBJECT* object = KeLookupHandleWithRights(process, (uint32_t)a0,
+                                              HANDLE_RIGHT_RESUME);
+    if (!object) return NTLSTATUS_INVALID_HANDLE;
+    if (object->Type != ObjectTypeThread) {
+        ObDereferenceObject(object);
+        return NTLSTATUS_INVALID_HANDLE;
+    }
+
+    THREAD* thread = CONTAINING_RECORD(object, THREAD, Header);
+    uint32_t previousSuspendCount = 0;
+    status = KeResumeThread(thread, &previousSuspendCount);
+    ObDereferenceObject(object);
+    if (!KSUCCESS(status)) return status;
+
+    return KiCopyU32ToUser(a1, previousSuspendCount);
+}
+
 static void KiFinishAlertableUserWait(uint8_t alertable) {
     if (!alertable) return;
 
@@ -619,6 +638,9 @@ static KSTATUS KiHandleToWaitEvent(PROCESS* proc, uint32_t handle, EVENT** event
     case ObjectTypeDisplayClock:
         event = KeDisplayClockWaitEvent((DISPLAY_CLOCK_OBJECT*)obj);
         break;
+    case ObjectTypeAudioStream:
+        event = KeAudioStreamWaitEvent((KE_AUDIO_STREAM*)obj);
+        break;
     default:
         ObDereferenceObject(obj);
         return NTLSTATUS_NOT_SUPPORTED;
@@ -656,28 +678,14 @@ static uint64_t SyscallWaitSingle(uint64_t a0, uint64_t a1, uint64_t a2,
 
     if (obj->Type == ObjectTypeMutex) {
         if (alertable) s = NTLSTATUS_NOT_SUPPORTED;
-        else if (timeoutMs == 0) {
-            s = KeTryAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header));
-            if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT;
-        } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) {
-            s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header));
-        } else {
-            s = NTLSTATUS_NOT_SUPPORTED;
-        }
+        else s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header), timeoutMs);
         ObDereferenceObject(obj);
         return s;
     }
 
     if (obj->Type == ObjectTypeSemaphore) {
         if (alertable) s = NTLSTATUS_NOT_SUPPORTED;
-        else if (timeoutMs == 0) {
-            s = KeTrySemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header));
-            if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT;
-        } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) {
-            s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header));
-        } else {
-            s = NTLSTATUS_NOT_SUPPORTED;
-        }
+        else s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header), timeoutMs);
         ObDereferenceObject(obj);
         return s;
     }
@@ -992,8 +1000,7 @@ static uint64_t SyscallSetThreadPriority(uint64_t a0, uint64_t a1, uint64_t a2,
     if (a0 >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
     THREAD* t = KeGetCurrentThread();
     if (!t) return NTLSTATUS_FAILURE;
-    t->Priority = (THREAD_PRIORITY)a0;
-    return NTLSTATUS_SUCCESS;
+    return KiSchedulerSetNormalPriority(t, (THREAD_PRIORITY)a0);
 }
 
 // SYS_VM_* page-granular user virtual memory.
@@ -1163,7 +1170,8 @@ static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2,
     (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
     MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex, HANDLE_RIGHT_WAIT);
     if (!mx) return NTLSTATUS_NOT_FOUND;
-    KSTATUS s = KeTryAcquireMutex(mx);
+    KSTATUS s = KeAcquireMutex(mx, 0);
+    if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE;
     ObDereferenceObject(&mx->Header);
     return s;
 }
@@ -1204,7 +1212,8 @@ static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2,
     (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
     SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore, HANDLE_RIGHT_WAIT);
     if (!sem) return NTLSTATUS_NOT_FOUND;
-    KSTATUS s = KeTrySemaphore(sem);
+    KSTATUS s = KeWaitSemaphore(sem, 0);
+    if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE;
     ObDereferenceObject(&sem->Header);
     return s;
 }
@@ -1672,7 +1681,7 @@ static void* KiAllocateSyscallIoBuffer(size_t requested, size_t* actualOut) {
     if (!requested) return NULL;
 
     size_t chunk = requested < SYSCALL_FILE_IO_CHUNK_BYTES ? requested : SYSCALL_FILE_IO_CHUNK_BYTES;
-    for (;;) {
+    while (true) {
         void* buf = MmAllocatePoolTag(chunk, MM_POOL_TAG('s', 'i', 'o', ' '));
         if (buf) {
             if (actualOut) *actualOut = chunk;
@@ -2275,9 +2284,8 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2,
     surface->PageCount = pageCount;
 
     surface->Frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS), MM_POOL_TAG('s', 'f', 'r', 'm'));
-    surface->RowScratch = (uint32_t*)MmAllocatePoolTag(rowBytes, MM_POOL_TAG('s', 'r', 'o', 'w'));
     s = KeMutexCreate(&surface->Lock);
-    if (!surface->Frames || !surface->RowScratch || !KSUCCESS(s)) {
+    if (!surface->Frames || !KSUCCESS(s)) {
         ObDereferenceObject(&surface->Header);
         return NTLSTATUS_NO_MEMORY;
     }
@@ -2295,6 +2303,14 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2,
             memset(kva, 0, PAGE_SIZE);
     }
 
+    s = MmCreateKernelViewFromFrames(surface->Frames, surface->PageCount,
+                                     &surface->KernelView);
+    if (!KSUCCESS(s)) {
+        ObDereferenceObject(&surface->Header);
+        return s;
+    }
+    surface->KernelBase = MmKernelViewBase(surface->KernelView);
+
     uint32_t handle = 0;
     s = KiAllocHandleForCurrentWithRights(&surface->Header, HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE | HANDLE_RIGHT_QUERY, &handle);
     if (!KSUCCESS(s))
@@ -2327,18 +2343,13 @@ static uint64_t SyscallMapSurface(uint64_t a0, uint64_t a1, uint64_t a2,
         return NTLSTATUS_ACCESS_DENIED;
     }
 
-    KSTATUS s = KeAcquireMutex(surface->Lock);
-    if (!KSUCCESS(s)) {
+    KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE);
+    if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s)) {
         ObDereferenceObject(&surface->Header);
         return s;
     }
 
-    if (!surface->UserBase) {
-        s = MmVadCreateSharedViewFromFrames(current, surface->Frames, surface->PageCount,
-                                            surface->PayloadSize,
-                                            PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
-                                            MM_VAD_ZERO, MmVadShared, &surface->UserBase);
-    }
+    s = KiSurfaceEnsureViewLocked(surface);
 
     void* userBase = surface->UserBase;
     uint32_t pitchBytes = surface->PitchBytes;
@@ -2374,8 +2385,8 @@ static uint64_t SyscallQuerySurface(uint64_t a0, uint64_t a1, uint64_t a2,
     SYS_SURFACE_DESC desc;
     memset(&desc, 0, sizeof(desc));
 
-    KSTATUS s = KeAcquireMutex(surface->Lock);
-    if (KSUCCESS(s)) {
+    KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE);
+    if (KE_MUTEX_ACQUIRE_SUCCEEDED(s)) {
         desc.Width = surface->Width;
         desc.Height = surface->Height;
         desc.PitchBytes = surface->PitchBytes;
@@ -2475,20 +2486,21 @@ static uint64_t SyscallPresentSurface(uint64_t a0, uint64_t a1, uint64_t a2,
     uint64_t rowsCopied = 0;
     uint64_t bytesCopied = 0;
 
-    KSTATUS s = NTLSTATUS_SUCCESS;
-    for (uint32_t i = 0; i < rectCount; ++i) {
+    KSTATUS s = surface->KernelBase ? NTLSTATUS_SUCCESS : NTLSTATUS_DEVICE_ERROR;
+    const uint8_t* sourceBase = (const uint8_t*)surface->KernelBase;
+
+    for (uint32_t i = 0; i < rectCount && KSUCCESS(s); ++i) {
         DISPLAY_RECT clipped;
         if (!KiClipSurfaceRect(surface, &rects[i], &clipped)) {
             emptyClips++;
             continue;
         }
 
-        SURFACE_ROW_CONTEXT ctx;
-        ctx.Surface = surface;
-        ctx.SourceX = (uint32_t)clipped.X;
-        ctx.SourceY = (uint32_t)clipped.Y;
-        s = KeDisplayBlitRows(clipped.X, clipped.Y, clipped.W, clipped.H,
-                              KiSurfaceCopyRow, &ctx, surface->RowScratch);
+        const uint32_t* source = (const uint32_t*)(sourceBase +
+            (size_t)(uint32_t)clipped.Y * surface->PitchBytes +
+            (size_t)(uint32_t)clipped.X * sizeof(uint32_t));
+        s = KeDisplayPresentPitched(clipped.X, clipped.Y, clipped.W, clipped.H,
+                                    source, surface->PitchBytes);
         if (!KSUCCESS(s))
             break;
 
@@ -2657,7 +2669,7 @@ static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2,
     (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
     if (!a0) return NTLSTATUS_INVALID_PARAMETER;
 
-    for (;;) {
+    while (true) {
         char ch = KeInputReadChar();
         if (ch)
             return CopyToUser((void*)(uintptr_t)a0, &ch, sizeof(ch));
@@ -2674,7 +2686,7 @@ static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2,
     (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
     if (!a0) return NTLSTATUS_INVALID_PARAMETER;
 
-    for (;;) {
+    while (true) {
         INPUT_KEY key = KeInputReadKey();
         if (key != INPUT_KEY_NONE)
             return CopyToUser((void*)(uintptr_t)a0, &key, sizeof(key));
@@ -2722,7 +2734,7 @@ static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
     if (!KSUCCESS(s)) return s;
 
     INPUT_KEY_RECORD record;
-    for (;;) {
+    while (true) {
         bool available = queue ? KeInputQueueReadEvent(queue, &record)
                                : KeInputReadEvent(&record);
         if (available) {
@@ -2759,7 +2771,7 @@ static uint64_t SyscallKbPeekInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
     if (!KSUCCESS(s)) return s;
 
     INPUT_KEY_RECORD record;
-    for (;;) {
+    while (true) {
         bool available = queue ? KeInputQueuePeekEvent(queue, &record)
                                : KeInputPeekEvent(&record);
         if (available) {
@@ -2995,7 +3007,8 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
 
     THREAD* thread;
     uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
-    s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread);
+    const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
+    s = KeCreateThread(proc, entryPoint, 0, &threadParameters, &thread);
     if (!KSUCCESS(s)) {
         ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint);
         MmVadDebugDumpProcess(proc);
@@ -3070,7 +3083,7 @@ static uint64_t SyscallGetProcessExitCode(uint64_t a0, uint64_t a1, uint64_t a2,
 }
 
 // SYS_WRITE_CONSOLE (60) a0=*buf a1=len
-// Writes user console text in white; distinct from green SYS_PRINT_DEBUG.
+// Writes user console text and completes display delivery before returning.
 static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2,
                                      uint64_t a3, uint64_t a4, uint64_t a5) {
     (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
@@ -3092,7 +3105,7 @@ static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2,
         KePrintConsole(chunk, todo);
         offset += todo;
     }
-    return NTLSTATUS_SUCCESS;
+    return KeDisplayFlushConsoleOutput();
 }
 
 // SYS_OPEN_DIR (54) a0=*dirPath a1=*handleOut.
@@ -3309,7 +3322,7 @@ void KeNamedQueueUnpublishHandle(PROCESS* process, uint32_t handle, OBJECT* obje
     if (!process || !object || object->Type != ObjectTypeQueue)
         return;
 
-    for (;;) {
+    while (true) {
         QUEUE* queue = (QUEUE*)0;
         uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
         for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
@@ -4473,7 +4486,7 @@ static uint64_t SyscallPoll(uint64_t a0, uint64_t a1, uint64_t a2,
     uint64_t start = gHal ? gHal->Timer.QueryTicks() : 0;
     size_t ready = 0;
 
-    for (;;) {
+    while (true) {
         ready = 0;
         for (size_t i = 0; i < count; i++) {
             fds[i].ReturnedEvents = 0;
@@ -4982,6 +4995,7 @@ static uint64_t SyscallQueryDisplayInfo(uint64_t a0, uint64_t a1, uint64_t a2,
     out.Mode = ki.Mode;
     out.Caps = ki.Caps;
     out.DeviceCount = ki.DeviceCount;
+    out.Flags = ki.Flags;
     out.ManagerStats = ki.ManagerStats;
     out.DeviceStats = ki.DeviceStats;
 
@@ -5012,6 +5026,12 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
         us.CpuBurstDemotions = ks.CpuBurstDemotions;
         us.SmtSiblingDeferrals = ks.SmtSiblingDeferrals;
         us.SmtPrimaryPokes = ks.SmtPrimaryPokes;
+        us.RealtimeDispatches = ks.RealtimeDispatches;
+        us.RealtimePreemptions = ks.RealtimePreemptions;
+        us.PriorityInheritanceBoosts = ks.PriorityInheritanceBoosts;
+        us.PriorityInheritanceDrops = ks.PriorityInheritanceDrops;
+        us.PriorityInheritancePropagations = ks.PriorityInheritancePropagations;
+        us.PriorityInheritanceMaxDepth = ks.PriorityInheritanceMaxDepth;
         return KiReturnKernelStats(a1, (size_t)a2, a3, &us, sizeof(us));
     }
 
@@ -5199,19 +5219,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
         um.DropFailures = km.DropFailures;
         um.StandbyTransitions = km.StandbyTransitions;
         um.StandbyHits = km.StandbyHits;
-        um.ReadAheadRequests = km.ReadAheadRequests;
-        um.ReadAheadPagesRequested = km.ReadAheadPagesRequested;
-        um.ReadAheadPagesQueued = km.ReadAheadPagesQueued;
-        um.ReadAheadPagesLoaded = km.ReadAheadPagesLoaded;
-        um.ReadAheadPagesSkipped = km.ReadAheadPagesSkipped;
-        um.ReadAheadFailures = km.ReadAheadFailures;
-        um.ReadAheadPagesThrottled = km.ReadAheadPagesThrottled;
-        um.MaxReadAheadPagesPerRequest = km.MaxReadAheadPagesPerRequest;
-        um.BatchReads = km.BatchReads;
-        um.BatchPagesRequested = km.BatchPagesRequested;
-        um.BatchPagesLoaded = km.BatchPagesLoaded;
-        um.MaxBatchPagesRequested = km.MaxBatchPagesRequested;
-        um.MaxBatchPagesLoaded = km.MaxBatchPagesLoaded;
         um.ResidentPages = km.ResidentPages;
         um.ActivePages = km.ActivePages;
         um.StandbyPages = km.StandbyPages;
@@ -5285,17 +5292,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
         um.WriteSectors = km.WriteSectors;
         um.MaxReadSectors = km.MaxReadSectors;
         um.MaxWriteSectors = km.MaxWriteSectors;
-        um.ReadaheadHits = km.ReadaheadHits;
-        um.ReadaheadMisses = km.ReadaheadMisses;
-        um.ReadaheadStores = km.ReadaheadStores;
-        um.ReadaheadInvalidations = km.ReadaheadInvalidations;
-        um.ReadaheadBytesServed = km.ReadaheadBytesServed;
-        um.ReadaheadSectorsLoaded = km.ReadaheadSectorsLoaded;
-        um.ReadaheadSequentialDetections = km.ReadaheadSequentialDetections;
-        um.ReadaheadSuppressed = km.ReadaheadSuppressed;
-        um.ReadaheadAllocFailures = km.ReadaheadAllocFailures;
-        um.ReadaheadReadFailures = km.ReadaheadReadFailures;
-        um.MaxReadaheadSectors = km.MaxReadaheadSectors;
         return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um));
     }
 
@@ -5374,6 +5370,466 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
     }
 }
 
+// Converts one public audio format into the kernel mixer format.
+static KSTATUS KiImportAudioFormat(const SYS_AUDIO_FORMAT* source, IO_AUDIO_FORMAT* destination) {
+    if (!source || !destination)
+        return NTLSTATUS_INVALID_PARAMETER;
+    if (source->SampleFormat != SysAudioSampleS16 ||
+        (source->ChannelCount != 1 && source->ChannelCount != 2) ||
+        (source->SampleRate != 11025u && source->SampleRate != 22050u &&
+         source->SampleRate != 44100u && source->SampleRate != 48000u))
+        return NTLSTATUS_NOT_SUPPORTED;
+
+    destination->SampleRate = source->SampleRate;
+    destination->ChannelCount = source->ChannelCount;
+    destination->SampleFormat = IoAudioSampleS16;
+    return NTLSTATUS_SUCCESS;
+}
+
+// Converts one kernel audio format into the public syscall representation.
+static SYS_AUDIO_FORMAT KiExportAudioFormat(const IO_AUDIO_FORMAT* source) {
+    SYS_AUDIO_FORMAT destination;
+    destination.SampleRate = source->SampleRate;
+    destination.ChannelCount = source->ChannelCount;
+    destination.SampleFormat = source->SampleFormat == IoAudioSampleS16
+                             ? SysAudioSampleS16
+                             : 0;
+    return destination;
+}
+
+// Converts one normalized kernel gain into unsigned Q16.
+static uint32_t KiExportAudioGainQ16(float gain) {
+    if (gain <= 0.0f) return 0;
+    if (gain >= 1.0f) return SYS_AUDIO_GAIN_Q16_UNITY;
+    return (uint32_t)(gain * (float)SYS_AUDIO_GAIN_Q16_UNITY + 0.5f);
+}
+
+// Converts one normalized kernel pan into signed Q16.
+static int32_t KiExportAudioPanQ16(float pan) {
+    if (pan <= -1.0f) return SYS_AUDIO_PAN_Q16_LEFT;
+    if (pan >= 1.0f) return SYS_AUDIO_PAN_Q16_RIGHT;
+    float scaled = pan * (float)SYS_AUDIO_GAIN_Q16_UNITY;
+    return (int32_t)(scaled + (scaled >= 0.0f ? 0.5f : -0.5f));
+}
+
+// Converts one public unsigned Q16 gain into the kernel representation.
+static float KiImportAudioGainQ16(uint32_t gainQ16) {
+    return (float)gainQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY;
+}
+
+// Converts one public signed Q16 pan into the kernel representation.
+static float KiImportAudioPanQ16(int32_t panQ16) {
+    return (float)panQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY;
+}
+
+// Converts one kernel endpoint kind into its public ABI value.
+static uint32_t KiExportAudioEndpointKind(IO_AUDIO_ENDPOINT_KIND kind) {
+    switch (kind) {
+    case IoAudioEndpointLineOut: return SysAudioEndpointLineOut;
+    case IoAudioEndpointSpeakerHeadphone: return SysAudioEndpointSpeakerHeadphone;
+    case IoAudioEndpointOtherAnalog: return SysAudioEndpointOtherAnalog;
+    default: return 0;
+    }
+}
+
+// Converts provider endpoint capabilities into public capability bits.
+static uint32_t KiExportAudioEndpointCapabilities(uint32_t capabilities) {
+    uint32_t result = 0;
+    if (capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE)
+        result |= SYS_AUDIO_ENDPOINT_CAP_JACK_SENSE;
+    if (capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN)
+        result |= SYS_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN;
+    return result;
+}
+
+// Converts one endpoint lifecycle value into its public ABI value.
+static uint32_t KiExportAudioEndpointState(KE_AUDIO_ENDPOINT_STATE state) {
+    switch (state) {
+    case KeAudioEndpointStarting: return SysAudioEndpointStarting;
+    case KeAudioEndpointReady: return SysAudioEndpointReady;
+    case KeAudioEndpointStopping: return SysAudioEndpointStopping;
+    case KeAudioEndpointFailed: return SysAudioEndpointFailed;
+    case KeAudioEndpointRemoved: return SysAudioEndpointRemoved;
+    default: return 0;
+    }
+}
+
+// Converts one stream lifecycle value into its public ABI value.
+static uint32_t KiExportAudioStreamState(KE_AUDIO_STREAM_STATE state) {
+    switch (state) {
+    case KeAudioStreamCreated: return SysAudioStreamCreated;
+    case KeAudioStreamRunning: return SysAudioStreamRunning;
+    case KeAudioStreamPaused: return SysAudioStreamPaused;
+    case KeAudioStreamDraining: return SysAudioStreamDraining;
+    case KeAudioStreamStopped: return SysAudioStreamStopped;
+    case KeAudioStreamFailed: return SysAudioStreamFailed;
+    case KeAudioStreamClosed:
+    default:
+        return 0;
+    }
+}
+
+// Converts public wait flags into the kernel stream condition mask.
+static uint32_t KiImportAudioWaitMask(uint32_t mask) {
+    uint32_t result = 0;
+    if (mask & SYS_AUDIO_WAIT_WRITABLE) result |= KE_AUDIO_WAIT_WRITABLE;
+    if (mask & SYS_AUDIO_WAIT_CONSUMED) result |= KE_AUDIO_WAIT_CONSUMED;
+    if (mask & SYS_AUDIO_WAIT_PRESENTED) result |= KE_AUDIO_WAIT_PRESENTED;
+    if (mask & SYS_AUDIO_WAIT_STOPPED) result |= KE_AUDIO_WAIT_STOPPED;
+    if (mask & SYS_AUDIO_WAIT_DRAINED) result |= KE_AUDIO_WAIT_DRAINED;
+    if (mask & SYS_AUDIO_WAIT_FAILED) result |= KE_AUDIO_WAIT_FAILED;
+    if (mask & SYS_AUDIO_WAIT_DISCONTINUITY) result |= KE_AUDIO_WAIT_DISCONTINUITY;
+    return result;
+}
+
+// Looks up one audio stream handle with the requested operation rights.
+static KE_AUDIO_STREAM* KiLookupAudioStream(uint32_t handle, uint32_t rights) {
+    OBJECT* object = KiHandleToObject(handle, ObjectTypeAudioStream, rights);
+    return object ? (KE_AUDIO_STREAM*)object : (KE_AUDIO_STREAM*)0;
+}
+
+// SYS_AUDIO_ENUMERATE_ENDPOINTS a0=*SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST.
+static uint64_t SyscallAudioEnumerateEndpoints(uint64_t a0, uint64_t a1, uint64_t a2,
+                                               uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request;
+    KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
+    if (!KSUCCESS(status)) return status;
+    if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 ||
+        !request.CountOut || (request.Capacity && !request.Endpoints))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    status = MmProbeUserWrite(request.CountOut, sizeof(*request.CountOut));
+    if (!KSUCCESS(status)) return status;
+    uint32_t copyCapacity = request.Capacity < SYS_AUDIO_MAX_ENDPOINTS
+                          ? request.Capacity
+                          : SYS_AUDIO_MAX_ENDPOINTS;
+    if (copyCapacity) {
+        status = MmProbeUserWrite(request.Endpoints,
+                                  (size_t)copyCapacity * sizeof(*request.Endpoints));
+        if (!KSUCCESS(status)) return status;
+    }
+
+    KE_AUDIO_ENDPOINT_INFO kernelInfo[KE_AUDIO_MAX_ENDPOINTS];
+    uint32_t count = 0;
+    status = KeAudioQueryEndpoints(kernelInfo, KE_AUDIO_MAX_ENDPOINTS, &count);
+    if (KFAILED(status)) return status;
+
+    SYS_AUDIO_ENDPOINT_INFO publicInfo[SYS_AUDIO_MAX_ENDPOINTS];
+    uint32_t copyCount = copyCapacity < count ? copyCapacity : count;
+    for (uint32_t index = 0; index < copyCount; index++) {
+        const KE_AUDIO_ENDPOINT_INFO* source = &kernelInfo[index];
+        SYS_AUDIO_ENDPOINT_INFO* destination = &publicInfo[index];
+        memset(destination, 0, sizeof(*destination));
+        destination->Size = sizeof(*destination);
+        destination->Version = SYS_AUDIO_ABI_VERSION;
+        destination->EndpointId = source->EndpointId;
+        destination->Kind = KiExportAudioEndpointKind(source->Kind);
+        if (source->Available) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE;
+        if (source->IsDefault) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_DEFAULT;
+        destination->Capabilities = KiExportAudioEndpointCapabilities(source->Capabilities);
+        destination->State = KiExportAudioEndpointState(source->State);
+        destination->HardwareFormat = KiExportAudioFormat(&source->HardwareFormat);
+        destination->PeriodCount = source->PeriodCount;
+        destination->FramesPerPeriod = source->FramesPerPeriod;
+        destination->MasterVolumeQ16 = KiExportAudioGainQ16(source->MasterRequestedVolume);
+        destination->MasterMuted = source->MasterMuted ? 1u : 0u;
+        destination->FailureStatus = source->FailureStatus;
+    }
+
+    if (copyCount) {
+        status = CopyToUser(request.Endpoints,
+                            publicInfo,
+                            (size_t)copyCount * sizeof(publicInfo[0]));
+        if (!KSUCCESS(status)) return status;
+    }
+    status = CopyToUser(request.CountOut, &count, sizeof(count));
+    if (!KSUCCESS(status)) return status;
+    return request.Capacity < count ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS;
+}
+
+// SYS_AUDIO_OPEN_STREAM a0=*SYS_AUDIO_OPEN_STREAM_REQUEST.
+static uint64_t SyscallAudioOpenStream(uint64_t a0, uint64_t a1, uint64_t a2,
+                                      uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_AUDIO_OPEN_STREAM_REQUEST request;
+    KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
+    if (!KSUCCESS(status)) return status;
+    if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 ||
+        !request.EndpointId || !request.StreamHandleOut)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    status = MmProbeUserWrite(request.StreamHandleOut, sizeof(*request.StreamHandleOut));
+    if (!KSUCCESS(status)) return status;
+
+    IO_AUDIO_FORMAT format;
+    status = KiImportAudioFormat(&request.Format, &format);
+    if (!KSUCCESS(status)) return status;
+
+    KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)0;
+    status = KeAudioCreateStream(request.EndpointId, &format, &stream);
+    if (!KSUCCESS(status)) return status;
+
+    uint32_t handle = 0;
+    status = KiAllocHandleForCurrentWithRights((OBJECT*)stream,
+                                                HANDLE_RIGHT_WRITE |
+                                                HANDLE_RIGHT_WAIT |
+                                                HANDLE_RIGHT_QUERY,
+                                                &handle);
+    if (!KSUCCESS(status)) return status;
+
+    status = CopyToUser(request.StreamHandleOut, &handle, sizeof(handle));
+    if (!KSUCCESS(status)) {
+        PROCESS* process = KeGetCurrentProcess();
+        if (process) KeCloseHandle(process, handle);
+    }
+    return status;
+}
+
+// SYS_AUDIO_WRITE a0=*SYS_AUDIO_WRITE_REQUEST.
+static uint64_t SyscallAudioWrite(uint64_t a0, uint64_t a1, uint64_t a2,
+                                 uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_AUDIO_WRITE_REQUEST request;
+    KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
+    if (!KSUCCESS(status)) return status;
+    if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle ||
+        !request.WrittenFramesOut || (request.FrameCount && !request.Frames))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    status = MmProbeUserWrite(request.WrittenFramesOut, sizeof(*request.WrittenFramesOut));
+    if (!KSUCCESS(status)) return status;
+    uint32_t written = 0;
+
+    KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE);
+    if (!stream) return NTLSTATUS_INVALID_HANDLE;
+    if (!request.FrameCount) {
+        ObDereferenceObject((OBJECT*)stream);
+        return CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
+    }
+
+    KE_AUDIO_STREAM_INFO info;
+    status = KeAudioQueryStream(stream, &info);
+    if (!KSUCCESS(status)) {
+        ObDereferenceObject((OBJECT*)stream);
+        return status;
+    }
+    if (info.State == KeAudioStreamFailed) {
+        status = KFAILED(info.FailureStatus) ? info.FailureStatus : NTLSTATUS_DEVICE_ERROR;
+        ObDereferenceObject((OBJECT*)stream);
+        (void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
+        return status;
+    }
+    if (info.State == KeAudioStreamDraining) {
+        ObDereferenceObject((OBJECT*)stream);
+        (void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
+        return NTLSTATUS_INVALID_PARAMETER;
+    }
+    if (!info.WritableFrames) {
+        ObDereferenceObject((OBJECT*)stream);
+        status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
+        return KSUCCESS(status) ? NTLSTATUS_WOULD_BLOCK : status;
+    }
+
+    uint32_t frameCount = request.FrameCount < info.WritableFrames
+                        ? request.FrameCount
+                        : info.WritableFrames;
+    uint32_t frameBytes = (uint32_t)info.Format.ChannelCount * sizeof(int16_t);
+    size_t byteCount = (size_t)frameCount * frameBytes;
+    status = MmProbeUserRead(request.Frames, byteCount);
+    if (!KSUCCESS(status)) {
+        ObDereferenceObject((OBJECT*)stream);
+        return status;
+    }
+
+    void* buffer = MmAllocatePoolTag(byteCount, MM_POOL_TAG('a', 'u', 'w', 'r'));
+    if (!buffer) {
+        ObDereferenceObject((OBJECT*)stream);
+        return NTLSTATUS_NO_MEMORY;
+    }
+    status = CopyFromUser(buffer, request.Frames, byteCount);
+    if (KSUCCESS(status))
+        status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
+    if (KSUCCESS(status))
+        status = KeAudioWriteStream(stream, buffer, frameCount, &written);
+    MmFreePool(buffer);
+    ObDereferenceObject((OBJECT*)stream);
+    if (!KSUCCESS(status)) return status;
+    return CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
+}
+
+// SYS_AUDIO_ARM_WAIT a0=*SYS_AUDIO_ARM_WAIT_REQUEST.
+static uint64_t SyscallAudioArmWait(uint64_t a0, uint64_t a1, uint64_t a2,
+                                   uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_AUDIO_ARM_WAIT_REQUEST request;
+    KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
+    if (!KSUCCESS(status)) return status;
+    if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 ||
+        !request.StreamHandle || !request.Mask ||
+        (request.Mask & ~SYS_AUDIO_WAIT_VALID_MASK) != 0)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    KE_AUDIO_STREAM_WAIT_CONDITION condition;
+    condition.Mask = KiImportAudioWaitMask(request.Mask);
+    condition.WritableFrames = request.WritableFrames;
+    condition.ConsumedFrames = request.ConsumedFrames;
+    condition.PresentedFrames = request.PresentedFrames;
+    condition.DrainedGeneration = request.DrainedGeneration;
+    condition.DiscontinuityGeneration = request.DiscontinuityGeneration;
+
+    KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WAIT);
+    if (!stream) return NTLSTATUS_INVALID_HANDLE;
+    status = KeAudioArmStreamWait(stream, &condition);
+    ObDereferenceObject((OBJECT*)stream);
+    return status;
+}
+
+// SYS_AUDIO_QUERY_STREAM a0=streamHandle a1=*SYS_AUDIO_STREAM_INFO.
+static uint64_t SyscallAudioQueryStream(uint64_t a0, uint64_t a1, uint64_t a2,
+                                       uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a2; (void)a3; (void)a4; (void)a5;
+    if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER;
+
+    uint32_t userSize = 0;
+    KSTATUS status = CopyFromUser(&userSize, (const void*)(uintptr_t)a1, sizeof(userSize));
+    if (!KSUCCESS(status)) return status;
+    if (userSize != sizeof(SYS_AUDIO_STREAM_INFO))
+        return NTLSTATUS_INVALID_PARAMETER;
+    status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(SYS_AUDIO_STREAM_INFO));
+    if (!KSUCCESS(status)) return status;
+
+    KE_AUDIO_STREAM* stream = KiLookupAudioStream((uint32_t)a0, HANDLE_RIGHT_QUERY);
+    if (!stream) return NTLSTATUS_INVALID_HANDLE;
+    KE_AUDIO_STREAM_INFO kernelInfo;
+    status = KeAudioQueryStream(stream, &kernelInfo);
+    ObDereferenceObject((OBJECT*)stream);
+    if (!KSUCCESS(status)) return status;
+
+    SYS_AUDIO_STREAM_INFO publicInfo;
+    memset(&publicInfo, 0, sizeof(publicInfo));
+    publicInfo.Size = sizeof(publicInfo);
+    publicInfo.Version = SYS_AUDIO_ABI_VERSION;
+    publicInfo.State = KiExportAudioStreamState(kernelInfo.State);
+    publicInfo.Format = KiExportAudioFormat(&kernelInfo.Format);
+    publicInfo.CapacityFrames = kernelInfo.CapacityFrames;
+    publicInfo.BufferedFrames = kernelInfo.BufferedFrames;
+    publicInfo.WritableFrames = kernelInfo.WritableFrames;
+    publicInfo.FailureStatus = kernelInfo.FailureStatus;
+    publicInfo.FramesSubmitted = kernelInfo.FramesSubmitted;
+    publicInfo.FramesConsumed = kernelInfo.FramesConsumed;
+    publicInfo.FramesPresented = kernelInfo.FramesPresented;
+    publicInfo.FramesDiscarded = kernelInfo.FramesDiscarded;
+    publicInfo.OutputFramesMixed = kernelInfo.OutputFramesMixed;
+    publicInfo.SilenceFramesInserted = kernelInfo.SilenceFramesInserted;
+    publicInfo.Underruns = kernelInfo.Underruns;
+    publicInfo.DrainedGeneration = kernelInfo.DrainedGeneration;
+    publicInfo.DiscontinuityGeneration = kernelInfo.DiscontinuityGeneration;
+    publicInfo.VolumeQ16 = KiExportAudioGainQ16(kernelInfo.Volume);
+    publicInfo.PanQ16 = KiExportAudioPanQ16(kernelInfo.Pan);
+    publicInfo.Muted = kernelInfo.Muted;
+    return CopyToUser((void*)(uintptr_t)a1, &publicInfo, sizeof(publicInfo));
+}
+
+typedef KSTATUS (*KI_AUDIO_STREAM_OPERATION)(KE_AUDIO_STREAM* stream);
+
+// Invokes one existing kernel stream operation through a write-capable handle.
+static KSTATUS KiCallAudioStreamOperation(uint32_t handle, KI_AUDIO_STREAM_OPERATION operation) {
+    if (!handle || !operation) return NTLSTATUS_INVALID_PARAMETER;
+    KE_AUDIO_STREAM* stream = KiLookupAudioStream(handle, HANDLE_RIGHT_WRITE);
+    if (!stream) return NTLSTATUS_INVALID_HANDLE;
+    KSTATUS status = operation(stream);
+    ObDereferenceObject((OBJECT*)stream);
+    return status;
+}
+
+// SYS_AUDIO_START a0=streamHandle.
+static uint64_t SyscallAudioStart(uint64_t a0, uint64_t a1, uint64_t a2,
+                                 uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+    return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStartStream);
+}
+
+// SYS_AUDIO_PAUSE a0=streamHandle.
+static uint64_t SyscallAudioPause(uint64_t a0, uint64_t a1, uint64_t a2,
+                                 uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+    return KiCallAudioStreamOperation((uint32_t)a0, KeAudioPauseStream);
+}
+
+// SYS_AUDIO_RESUME a0=streamHandle.
+static uint64_t SyscallAudioResume(uint64_t a0, uint64_t a1, uint64_t a2,
+                                  uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+    return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResumeStream);
+}
+
+// SYS_AUDIO_DRAIN a0=streamHandle.
+static uint64_t SyscallAudioDrain(uint64_t a0, uint64_t a1, uint64_t a2,
+                                 uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+    return KiCallAudioStreamOperation((uint32_t)a0, KeAudioDrainStream);
+}
+
+// SYS_AUDIO_STOP a0=streamHandle.
+static uint64_t SyscallAudioStop(uint64_t a0, uint64_t a1, uint64_t a2,
+                                uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+    return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStopStream);
+}
+
+// SYS_AUDIO_RESET a0=streamHandle.
+static uint64_t SyscallAudioReset(uint64_t a0, uint64_t a1, uint64_t a2,
+                                 uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+    return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResetStream);
+}
+
+// SYS_AUDIO_SET_VOLUME a0=*SYS_AUDIO_SET_VOLUME_REQUEST.
+static uint64_t SyscallAudioSetVolume(uint64_t a0, uint64_t a1, uint64_t a2,
+                                     uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_AUDIO_SET_VOLUME_REQUEST request;
+    KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
+    if (!KSUCCESS(status)) return status;
+    if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle ||
+        request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY ||
+        request.PanQ16 < SYS_AUDIO_PAN_Q16_LEFT ||
+        request.PanQ16 > SYS_AUDIO_PAN_Q16_RIGHT)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE);
+    if (!stream) return NTLSTATUS_INVALID_HANDLE;
+    status = KeAudioSetStreamVolume(stream,
+                                    KiImportAudioGainQ16(request.VolumeQ16),
+                                    KiImportAudioPanQ16(request.PanQ16),
+                                    request.Muted != 0);
+    ObDereferenceObject((OBJECT*)stream);
+    return status;
+}
+
+// SYS_AUDIO_SET_MASTER_VOLUME a0=*SYS_AUDIO_SET_MASTER_VOLUME_REQUEST.
+static uint64_t SyscallAudioSetMasterVolume(uint64_t a0, uint64_t a1, uint64_t a2,
+                                           uint64_t a3, uint64_t a4, uint64_t a5) {
+    (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
+
+    SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request;
+    KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
+    if (!KSUCCESS(status)) return status;
+    if (request.Size != sizeof(request) || request.Flags != 0 || !request.EndpointId ||
+        request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    return KeAudioSetMasterVolume(request.EndpointId,
+                                  KiImportAudioGainQ16(request.VolumeQ16),
+                                  request.Muted != 0);
+}
+
 // Dispatch table keyed by SYS_* constants.
 typedef uint64_t (*SYSCALL_HANDLER)(uint64_t, uint64_t, uint64_t,
                                     uint64_t, uint64_t, uint64_t);
@@ -5389,6 +5845,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
     [SYS_GET_PROCESS_LIST] = SyscallGetProcessList,
     [SYS_GET_PROCESS_PID] = SyscallGetProcessPid,
     [SYS_CREATE_THREAD] = SyscallCreateThread,
+    [SYS_RESUME_THREAD] = SyscallResumeThread,
     [SYS_GET_THREAD_LIST] = SyscallGetThreadList,
     [SYS_LOAD_MODULE] = SyscallLoadModule,
     [SYS_UNLOAD_MODULE] = SyscallUnloadModule,
@@ -5480,6 +5937,21 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
     [SYS_QUERY_SURFACE] = SyscallQuerySurface,
     [SYS_PRESENT_SURFACE] = SyscallPresentSurface,
 
+    // Native render audio.
+    [SYS_AUDIO_ENUMERATE_ENDPOINTS] = SyscallAudioEnumerateEndpoints,
+    [SYS_AUDIO_OPEN_STREAM] = SyscallAudioOpenStream,
+    [SYS_AUDIO_WRITE] = SyscallAudioWrite,
+    [SYS_AUDIO_ARM_WAIT] = SyscallAudioArmWait,
+    [SYS_AUDIO_QUERY_STREAM] = SyscallAudioQueryStream,
+    [SYS_AUDIO_START] = SyscallAudioStart,
+    [SYS_AUDIO_PAUSE] = SyscallAudioPause,
+    [SYS_AUDIO_RESUME] = SyscallAudioResume,
+    [SYS_AUDIO_DRAIN] = SyscallAudioDrain,
+    [SYS_AUDIO_STOP] = SyscallAudioStop,
+    [SYS_AUDIO_RESET] = SyscallAudioReset,
+    [SYS_AUDIO_SET_VOLUME] = SyscallAudioSetVolume,
+    [SYS_AUDIO_SET_MASTER_VOLUME] = SyscallAudioSetMasterVolume,
+
     // Input.
     [SYS_POLL_KEYBOARD] = SyscallKbPoll,
     [SYS_IS_KEY_DOWN] = SyscallKbIsKeyDown,
@@ -5532,6 +6004,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
     THREAD* self = KeGetCurrentThread();
     PROCESS* owner = self ? self->Owner : (PROCESS*)0;
     if (owner && KeIsProcessTerminating(owner) &&
+        !KeThreadTerminationDeferred(self) &&
         nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) {
         int32_t exitCode = owner->ExitCode;
         KeExitThread(self, exitCode);
@@ -5547,6 +6020,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
     // has been marked terminating.
     owner = self ? self->Owner : (PROCESS*)0;
     if (owner && KeIsProcessTerminating(owner) &&
+        !KeThreadTerminationDeferred(self) &&
         nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) {
         int32_t exitCode = owner->ExitCode;
         KeExitThread(self, exitCode);

code/kernel/ke_system_root.c

@@ -7,10 +7,15 @@
 #include <kernel/ke_system_root.h>
 #include <kernel/ke_debug.h>
 #include <kernel/io_file.h>
+#include <kernel/ke_sched.h>
 
 static uint8_t gSystemRootDrive;
 static bool gSystemRootReady;
 
+#define KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS 30000ull
+#define KE_SYSTEM_ROOT_WAIT_SLICE_MS         1000ull
+#define KE_SYSTEM_ROOT_LOG_INTERVAL_MS       5000ull
+
 // Builds a drive-qualified path without depending on any process CWD.
 static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) {
     if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES)
@@ -50,23 +55,76 @@ static bool KiDriveLooksLikeSystemRoot(uint8_t drive) {
     return true;
 }
 
-// Selects the mounted volume that contains the kernel/system tree.
-KSTATUS KeInitializeSystemRootVolume(void) {
+// Returns the current scheduler time in milliseconds.
+static uint64_t KiSystemRootNowMilliseconds(void) {
+    KE_SCHED_STATS stats;
+    KeGetSchedulerStats(&stats);
+    return stats.Ticks * SCHED_TICK_MS;
+}
+
+// Scans the currently mounted volumes for the system tree.
+static KSTATUS KiTrySelectSystemRootVolume(uint32_t* mountedOut) {
+    uint32_t mounted = 0;
     for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) {
         if (!IoFilesystemPresent(drive))
             continue;
+        mounted++;
         if (!KiDriveLooksLikeSystemRoot(drive))
             continue;
 
         gSystemRootDrive = drive;
         gSystemRootReady = true;
+        if (mountedOut)
+            *mountedOut = mounted;
         DbgPrint("System root: drive=%u\n", drive);
         return NTLSTATUS_SUCCESS;
     }
 
+    if (mountedOut)
+        *mountedOut = mounted;
     return NTLSTATUS_NOT_FOUND;
 }
 
+// Waits for the mounted boot volume instead of racing asynchronous bus discovery.
+KSTATUS KeInitializeSystemRootVolume(void) {
+    uint64_t start = KiSystemRootNowMilliseconds();
+    uint64_t deadline = start + KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS;
+    uint64_t nextLog = start;
+
+    for (;;) {
+        uint64_t generation = IoGetVolumeNamespaceGeneration();
+        uint32_t mounted = 0;
+        KSTATUS status = KiTrySelectSystemRootVolume(&mounted);
+        if (KSUCCESS(status))
+            return status;
+
+        uint64_t now = KiSystemRootNowMilliseconds();
+        if (now >= deadline) {
+            DbgPrint("System root: discovery timed out after %llu ms; mounted=%u generation=%llu\n",
+                     (unsigned long long)KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS,
+                     mounted,
+                     (unsigned long long)generation);
+            return NTLSTATUS_TIMEOUT;
+        }
+
+        if (now >= nextLog) {
+            DbgPrint("System root: waiting for boot volume; mounted=%u generation=%llu remaining=%llu ms\n",
+                     mounted,
+                     (unsigned long long)generation,
+                     (unsigned long long)(deadline - now));
+            nextLog = now + KE_SYSTEM_ROOT_LOG_INTERVAL_MS;
+        }
+
+        uint64_t remaining = deadline - now;
+        uint64_t wait = remaining < KE_SYSTEM_ROOT_WAIT_SLICE_MS
+                      ? remaining
+                      : KE_SYSTEM_ROOT_WAIT_SLICE_MS;
+        KSTATUS waitStatus = IoWaitForVolumeNamespaceChange(generation, wait);
+        if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT)
+            return waitStatus;
+    }
+}
+
 // Returns whether the system root volume has been selected.
 bool KeSystemRootReady(void) {
     return gSystemRootReady;

code/kernel/ke_thread.c

@@ -27,14 +27,105 @@
     _Static_assert(offsetof(type, field) == (expected), \
                    #type "." #field " offset must match assembly")
 
-STATIC_ASSERT_OFFSET(THREAD, SavedContext, 96);
-STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 112);
-STATIC_ASSERT_OFFSET(THREAD, Teb, 120);
-STATIC_ASSERT_OFFSET(THREAD, FpuContext, 176);
+STATIC_ASSERT_OFFSET(THREAD, SavedContext, 112);
+STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 128);
+STATIC_ASSERT_OFFSET(THREAD, Teb, 136);
+STATIC_ASSERT_OFFSET(THREAD, FpuContext, 192);
 
 static uint32_t gNextThreadId = 1;
 extern void KeThreadEntry(void);
 
+// Prevents a newly scheduled user thread from entering a terminating process.
+void KePrepareUserThreadStart(void) {
+    THREAD* thread = KeGetCurrentThread();
+    PROCESS* owner = thread ? thread->Owner : (PROCESS*)0;
+    if (!thread || !owner || !KeIsProcessTerminating(owner))
+        return;
+
+    KeExitThread(thread, owner->ExitCode);
+    while (true) {
+        KeYield();
+    }
+}
+
+// Finalizes one dead thread's completion, user memory, and process membership.
+void KeFinalizeThreadExit(THREAD* thread) {
+    if (!thread)
+        return;
+    if (thread->State != ThreadDead) {
+        KePanic(KE_PANIC_CODE_SCHEDULER,
+                "thread finalized before death: thread=0x%016llX tid=%u state=%u\n",
+                (uint64_t)thread,
+                thread->ThreadId,
+                (unsigned)thread->State);
+    }
+
+    // Thread completion is manual-reset and idempotent. Publish it before
+    // process-owned handles can be released by the final live-thread exit.
+    if (thread->ExitEvent)
+        KeSignalEvent(thread->ExitEvent);
+
+    PROCESS* owner = thread->Owner;
+    if (!owner)
+        return;
+
+    // Every exiting thread releases its own user allocations before leaving
+    // the live-thread set. Therefore the final detach can destroy the remaining
+    // process VADs without racing a later THREAD destructor.
+    if (!owner->ResourcesReleased) {
+        if (thread->Teb && thread->TebSize)
+            MmFreeUserMemoryForProcess(owner, thread->Teb);
+        if (thread->UserStackBase)
+            MmFreeUserMemoryForProcess(owner, thread->UserStackBase);
+    }
+    thread->Teb = (void*)0;
+    thread->TebSize = 0;
+    thread->UserStackBase = (void*)0;
+
+    uint32_t remaining = 0;
+    int detached = 0;
+
+    uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
+    int fwdDetached = thread->ProcessLink.Fwd == &thread->ProcessLink;
+    int bwdDetached = thread->ProcessLink.Bwd == &thread->ProcessLink;
+    if (fwdDetached != bwdDetached) {
+        KeReleaseSpinlock(&owner->HandleLock, flags);
+        KePanic(KE_PANIC_CODE_SCHEDULER,
+                "thread process linkage corrupt: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
+                (uint64_t)thread,
+                thread->ThreadId,
+                (uint64_t)thread->ProcessLink.Fwd,
+                (uint64_t)thread->ProcessLink.Bwd);
+    }
+
+    if (!fwdDetached) {
+        if (owner->ThreadCount == 0) {
+            KeReleaseSpinlock(&owner->HandleLock, flags);
+            KePanic(KE_PANIC_CODE_SCHEDULER,
+                    "thread process count underflow: thread=0x%016llX tid=%u pid=%u\n",
+                    (uint64_t)thread,
+                    thread->ThreadId,
+                    owner->ProcessId);
+        }
+
+        KeListRemoveChecked(&thread->ProcessLink);
+        owner->ThreadCount--;
+        detached = 1;
+    }
+    remaining = owner->ThreadCount;
+    KeReleaseSpinlock(&owner->HandleLock, flags);
+
+    if (!detached)
+        return;
+
+    if (remaining == 0) {
+        // ThreadCount represents live process membership, not retained thread
+        // handles. Dead THREAD objects may remain waitable after process exit.
+        _InterlockedExchange(&owner->Terminating, 1);
+        KeCompleteProcessExitIfLastThread(owner);
+    }
+}
+
 // Implements the thread Destroy operation.
 static void KiThreadDestroy(OBJECT* obj) {
     THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
@@ -44,6 +135,26 @@ static void KiThreadDestroy(OBJECT* obj) {
     // wait syscall to run normal cleanup.
     KeRundownThreadWait(thread, NTLSTATUS_ALERTED);
 
+    if (thread->WaitingMutex || !LIST_IS_EMPTY(&thread->OwnedMutexList)) {
+        KePanic(KE_PANIC_CODE_SCHEDULER,
+                "thread destroyed with mutex state: thread=0x%016llX tid=%u waiting=0x%016llX ownedEmpty=%u\n",
+                (uint64_t)thread,
+                thread->ThreadId,
+                (uint64_t)thread->WaitingMutex,
+                LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u);
+    }
+    if (thread->TimeoutArmed ||
+        thread->TimeoutLink.Fwd != &thread->TimeoutLink ||
+        thread->TimeoutLink.Bwd != &thread->TimeoutLink) {
+        KePanic(KE_PANIC_CODE_SCHEDULER,
+                "thread destroyed with timeout state: thread=0x%016llX tid=%u armed=%u link=0x%016llX/0x%016llX\n",
+                (uint64_t)thread,
+                thread->ThreadId,
+                (unsigned)thread->TimeoutArmed,
+                (uint64_t)thread->TimeoutLink.Fwd,
+                (uint64_t)thread->TimeoutLink.Bwd);
+    }
+
     KiRundownApcState(thread);
 
     if (thread->ExitEvent) {
@@ -51,39 +162,28 @@ static void KiThreadDestroy(OBJECT* obj) {
         thread->ExitEvent = (EVENT*)0;
     }
 
-    PROCESS* ownerForVadList = thread->Owner;
-    if (ownerForVadList) {
-        if (thread->Teb && thread->TebSize) {
-            MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb);
-            thread->Teb = (void*)0;
-            thread->TebSize = 0;
-        }
+    if (thread->Teb || thread->TebSize || thread->UserStackBase) {
+        KePanic(KE_PANIC_CODE_SCHEDULER,
+                "thread destroyed with user memory: thread=0x%016llX tid=%u teb=0x%016llX tebSize=%llu stack=0x%016llX\n",
+                (uint64_t)thread,
+                thread->ThreadId,
+                (uint64_t)thread->Teb,
+                (uint64_t)thread->TebSize,
+                (uint64_t)thread->UserStackBase);
+    }
 
-        if (thread->UserStackBase) {
-            MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase);
-            thread->UserStackBase = (void *)0;
-        }
+    if (thread->ProcessLink.Fwd != &thread->ProcessLink ||
+        thread->ProcessLink.Bwd != &thread->ProcessLink) {
+        KePanic(KE_PANIC_CODE_SCHEDULER,
+                "thread destroyed while still process-linked: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
+                (uint64_t)thread,
+                thread->ThreadId,
+                (uint64_t)thread->ProcessLink.Fwd,
+                (uint64_t)thread->ProcessLink.Bwd);
     }
 
     if (thread->Owner) {
-        PROCESS* owner = thread->Owner;
-        uint32_t remaining = 0;
-        uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
-        KeListRemoveChecked(&thread->ProcessLink);
-        if (owner->ThreadCount) owner->ThreadCount--;
-        remaining = owner->ThreadCount;
-        KeReleaseSpinlock(&owner->HandleLock, flags);
-
-        if (remaining == 0) {
-            // The last thread leaving a process is process exit even if user
-            // code called ExitThread instead of ExitProcess. KeExitProcess /
-            // KeTerminateProcess set Terminating earlier; this covers the last
-            // thread voluntarily exiting without an explicit process-exit call.
-            _InterlockedExchange(&owner->Terminating, 1);
-            KeCompleteProcessExitIfLastThread(owner);
-        }
-
-        ObDereferenceObject(&owner->Header);
+        ObDereferenceObject(&thread->Owner->Header);
         thread->Owner = (PROCESS *)0;
     }
 
@@ -94,35 +194,51 @@ static void KiThreadDestroy(OBJECT* obj) {
 }
 
 __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
-                       THREAD_PRIORITY priority, THREAD** out) {
-    if (!owner || !out) return NTLSTATUS_INVALID_PARAMETER;
-    if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
+                       const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) {
+    if (!owner || !parameters || !out) return NTLSTATUS_INVALID_PARAMETER;
+    if (parameters->SchedulingClass != KeSchedulingNormal ||
+        parameters->Priority >= PriorityCount ||
+        (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    uint8_t initiallySuspended =
+        (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
 
     THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
     if (!thread) return NTLSTATUS_NO_MEMORY;
 
     thread->Header.Destroy   = KiThreadDestroy;
     thread->ThreadId         = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
-    thread->State            = ThreadReady;
-    thread->Priority         = priority;
-    thread->QuantaConsumed   = 0;
+    thread->State                    = initiallySuspended ? ThreadSuspended : ThreadReady;
+    thread->BaseSchedulingClass      = parameters->SchedulingClass;
+    thread->BasePriority             = parameters->Priority;
+    thread->EffectiveSchedulingClass = parameters->SchedulingClass;
+    thread->EffectivePriority        = parameters->Priority;
+    thread->QuantaConsumed           = 0;
     thread->QuantumTicks      = 0;
     thread->QuantumRemaining  = 0;
     thread->CpuBurstTicks     = 0;
     thread->LastRunTicks      = 0;
     thread->RunningCpu        = KE_INVALID_CPU;
+    thread->SuspendCount      = initiallySuspended ? 1u : 0u;
+    LIST_INIT(&thread->OwnedMutexList);
+    thread->WaitingMutex      = (MUTEX*)0;
     thread->Queued            = 0;
     thread->IsIdleThread      = 0;
     thread->SchedulerFlags    = 0;
     thread->LastCpu           = KE_INVALID_CPU;
-    thread->AffinityMask      = KeGetDefaultThreadAffinityMask();
+    thread->AffinityMask      = parameters->AffinityMask
+        ? parameters->AffinityMask
+        : KeGetDefaultThreadAffinityMask();
     LIST_INIT(&thread->ProcessLink);
     LIST_INIT(&thread->SchedulerLink);
     thread->WaitObject       = (OBJECT *)0;
     thread->WaitStatus       = NTLSTATUS_SUCCESS;
     thread->WaitListHead     = (LIST_ENTRY*)0;
     thread->AlertableWait    = 0;
-    thread->SleepUntilTick   = 0;
+    LIST_INIT(&thread->TimeoutLink);
+    thread->TimeoutArmed     = 0;
+    thread->TimeoutTick      = 0;
     thread->TimeoutStatus    = NTLSTATUS_SUCCESS;
     thread->SwitchLockCookie = 0x202ULL;
     KiInitializeApcState(thread);
@@ -131,6 +247,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     thread->UserExceptionNoncontinuableMask = 0;
     thread->UserExceptionFrameUserAddress = 0;
     memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
+    thread->TerminationDeferralCount = 0;
 
     void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
     if (!kstack) {
@@ -245,24 +362,58 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
     ObReferenceObject(&thread->Header);
     *out = thread;
 
-    KeReadyThread(thread);
+    if (!initiallySuspended)
+        KeReadyThread(thread);
     return NTLSTATUS_SUCCESS;
 }
 
+
+// Enters a region that must retain the current thread during process termination.
+void KeEnterTerminationDeferral(void) {
+    KeAssertIrql(PASSIVE_LEVEL);
+    THREAD* thread = KeGetCurrentThread();
+    if (!thread || thread->IsIdleThread)
+        return;
+    _InterlockedIncrement(&thread->TerminationDeferralCount);
+}
+
+// Leaves a process-termination deferral region.
+void KeLeaveTerminationDeferral(void) {
+    KeAssertIrql(PASSIVE_LEVEL);
+    THREAD* thread = KeGetCurrentThread();
+    if (!thread || thread->IsIdleThread)
+        return;
+
+    long remaining = _InterlockedDecrement(&thread->TerminationDeferralCount);
+    if (remaining < 0) {
+        KePanic(KE_PANIC_CODE_SCHEDULER,
+                "thread termination deferral underflow: thread=0x%016llX tid=%u\n",
+                (uint64_t)thread,
+                thread->ThreadId);
+    }
+
+    if (remaining == 0 && thread->Owner && KeIsProcessTerminating(thread->Owner))
+        KiRequestReschedule();
+}
+
+// Returns whether one thread still owns termination-sensitive kernel work.
+int KeThreadTerminationDeferred(const THREAD* thread) {
+    return thread && _InterlockedCompareExchange((volatile long*)&thread->TerminationDeferralCount,
+                                                  0,
+                                                  0) > 0;
+}
+
 // Implements the exit Thread operation.
 __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
     (void)code;
     if (!thread) return;
 
     KiRundownApcState(thread);
-
-    // Wake any pthread_join waiters before the thread is marked dead and
-    // potentially freed. Manual-reset means all concurrent waiters unblock.
-    if (thread->ExitEvent)
-        KeSignalEvent(thread->ExitEvent);
+    KiMutexRundownThread(thread);
 
     thread->State = ThreadDead;
 
+    KeFinalizeThreadExit(thread);
     KeYield();
 }
 
@@ -277,10 +428,9 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
 //   [ksp - 16 - sizeof(KCONTEXT)]  KCONTEXT { .Rip = KeKernelThreadEntry }.
 extern void KeKernelThreadEntry(void);
 
-static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
-                                                THREAD_PRIORITY priority,
-                                                uint64_t affinityMask,
-                                                THREAD** out) {
+static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
+                                      const KE_THREAD_CREATE_PARAMETERS* parameters,
+                                      THREAD** out) {
     // Kernel worker threads are owned by the system process. The scheduler's
     // hand-built CPU idle thread stays attached to PID 0 (idle), while driver,
     // async I/O, and init worker threads are grouped under PID 1 (system).
@@ -289,34 +439,57 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
         THREAD* currentThread = KeGetCurrentThread();
         owner = currentThread ? currentThread->Owner : (PROCESS*)0;
     }
-    if (!owner) return NTLSTATUS_INVALID_PARAMETER;
-    if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
+    if (!owner || !parameters) return NTLSTATUS_INVALID_PARAMETER;
+    if (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED)
+        return NTLSTATUS_INVALID_PARAMETER;
+    if ((parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) && !out)
+        return NTLSTATUS_INVALID_PARAMETER;
+    if (parameters->SchedulingClass == KeSchedulingNormal) {
+        if (parameters->Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
+    } else if (parameters->SchedulingClass == KeSchedulingRealtime) {
+        if (parameters->Priority > KE_REALTIME_PRIORITY_MAX) return NTLSTATUS_INVALID_PARAMETER;
+    } else {
+        return NTLSTATUS_INVALID_PARAMETER;
+    }
+
+    uint8_t initiallySuspended =
+        (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
 
     THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
     if (!thread) return NTLSTATUS_NO_MEMORY;
 
     thread->Header.Destroy  = KiThreadDestroy;
     thread->ThreadId        = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1);
-    thread->State           = ThreadReady;
-    thread->Priority        = priority;
-    thread->QuantaConsumed  = 0;
+    thread->State                    = initiallySuspended ? ThreadSuspended : ThreadReady;
+    thread->BaseSchedulingClass      = parameters->SchedulingClass;
+    thread->BasePriority             = parameters->Priority;
+    thread->EffectiveSchedulingClass = parameters->SchedulingClass;
+    thread->EffectivePriority        = parameters->Priority;
+    thread->QuantaConsumed           = 0;
     thread->QuantumTicks     = 0;
     thread->QuantumRemaining = 0;
     thread->CpuBurstTicks    = 0;
     thread->LastRunTicks     = 0;
     thread->RunningCpu       = KE_INVALID_CPU;
+    thread->SuspendCount     = initiallySuspended ? 1u : 0u;
+    LIST_INIT(&thread->OwnedMutexList);
+    thread->WaitingMutex     = (MUTEX*)0;
     thread->Queued           = 0;
     thread->IsIdleThread     = 0;
     thread->SchedulerFlags   = 0;
     thread->LastCpu          = KE_INVALID_CPU;
-    thread->AffinityMask     = affinityMask ? affinityMask : KeGetDefaultThreadAffinityMask();
+    thread->AffinityMask     = parameters->AffinityMask
+        ? parameters->AffinityMask
+        : KeGetDefaultThreadAffinityMask();
     LIST_INIT(&thread->ProcessLink);
     LIST_INIT(&thread->SchedulerLink);
     thread->WaitObject      = (OBJECT*)0;
     thread->WaitStatus      = NTLSTATUS_SUCCESS;
     thread->WaitListHead    = (LIST_ENTRY*)0;
     thread->AlertableWait   = 0;
-    thread->SleepUntilTick  = 0;
+    LIST_INIT(&thread->TimeoutLink);
+    thread->TimeoutArmed    = 0;
+    thread->TimeoutTick     = 0;
     thread->TimeoutStatus   = NTLSTATUS_SUCCESS;
     thread->SwitchLockCookie = 0x202ULL;
     KiInitializeApcState(thread);
@@ -325,6 +498,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     thread->UserExceptionNoncontinuableMask = 0;
     thread->UserExceptionFrameUserAddress = 0;
     memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
+    thread->TerminationDeferralCount = 0;
     thread->UserStackBase   = (void*)0;   // no user stack
     thread->Teb             = (void*)0;   // no TEB; GS_BASE will be 0 in kernel ctx
     thread->TebSize         = 0;
@@ -371,20 +545,15 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
     }
 
     KeInitThreadFpuContext(thread);
-    KeReadyThread(thread);
+    if (!initiallySuspended)
+        KeReadyThread(thread);
     return NTLSTATUS_SUCCESS;
 }
 
 __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
-                              THREAD_PRIORITY priority, THREAD** out) {
-    return KiCreateKernelThreadWithAffinity(fn, arg, priority,
-                                            KeGetDefaultThreadAffinityMask(), out);
-}
-
-__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
-                              THREAD_PRIORITY priority, uint64_t affinityMask,
+                              const KE_THREAD_CREATE_PARAMETERS* parameters,
                               THREAD** out) {
-    return KiCreateKernelThreadWithAffinity(fn, arg, priority, affinityMask, out);
+    return KiCreateKernelThread(fn, arg, parameters, out);
 }
 
 // Prepares block Thread.
@@ -438,16 +607,6 @@ void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
     routine(thread, context, status);
 }
 
-// Implements the finish Wait operation.
-static void KiFinishWait(THREAD* thread, KSTATUS status) {
-    thread->WaitObject    = (OBJECT*)0;
-    thread->WaitStatus    = status;
-    thread->WaitListHead  = (LIST_ENTRY*)0;
-    thread->AlertableWait = 0;
-    thread->SleepUntilTick = 0;
-    thread->TimeoutStatus = NTLSTATUS_SUCCESS;
-}
-
 // Implements the commit Block Thread operation.
 KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     KeAssertIrql(PASSIVE_LEVEL);
@@ -463,19 +622,14 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     if (thread->State != ThreadWaiting) {
         KSTATUS status = thread->WaitStatus;
         thread->AlertableWait = 0;
-        thread->SleepUntilTick = 0;
-    thread->TimeoutStatus = NTLSTATUS_SUCCESS;
         return status;
     }
 
     // If a user APC was already pending before the thread fully blocked,
     // consume the alertable wait immediately instead of going to sleep.
     if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
-        if (thread->WaitListHead)
-            KeListRemoveChecked(&thread->SchedulerLink);
-        KiFinishWait(thread, NTLSTATUS_USER_APC);
-        thread->State = ThreadRunning;
-        return NTLSTATUS_USER_APC;
+        if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC))
+            return NTLSTATUS_USER_APC;
     }
 
     KiSchedulerNoteVoluntaryBlock(thread);
@@ -486,8 +640,6 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
     KiDeliverKernelApcs();
 
     thread->AlertableWait = 0;
-    thread->SleepUntilTick = 0;
-    thread->TimeoutStatus = NTLSTATUS_SUCCESS;
     return thread->WaitStatus;
 }
 
@@ -525,8 +677,6 @@ static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* wa
 
     KeListRemoveChecked(&thread->SchedulerLink);
     thread->WaitListHead = (LIST_ENTRY*)0;
-    thread->SleepUntilTick = 0;
-    thread->TimeoutStatus = NTLSTATUS_SUCCESS;
     return 1;
 }
 
@@ -546,10 +696,7 @@ static int KiAbortObjectWait(THREAD* thread) {
         }
         case ObjectTypeMutex: {
             MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
-            uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
-            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList);
-            KeReleaseSpinlock(&mutex->Lock, flags);
-            return removed;
+            return KiMutexAbortWait(mutex, thread);
         }
         case ObjectTypeSemaphore: {
             SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
@@ -569,24 +716,29 @@ static int KiAbortObjectWait(THREAD* thread) {
 // unsignaled after the foreground thread has faulted. Object wait-list removal
 // must use the object's own lock; otherwise signal/abort races can leave stale
 // scheduler links in an event, mutex, or semaphore wait queue.
-void KeAbortThreadWait(THREAD* thread, KSTATUS status) {
-    if (!thread) return;
-    if (thread->State != ThreadWaiting) return;
+int KeAbortThreadWait(THREAD* thread, KSTATUS status) {
+    if (!thread) return 0;
+    if (thread->State != ThreadWaiting) return 0;
 
     if (thread->WaitRundownRoutine) {
         KeRundownThreadWait(thread, status);
-        return;
+        return thread->State != ThreadWaiting;
     }
 
     if (KiAbortObjectWait(thread)) {
         KeUnblockThreadWithStatus(thread, status);
-        return;
+        return 1;
     }
 
-    // Sleep/timeouts and already-dequeued handoffs are owned by the scheduler.
-    // This path removes from the scheduler sleep list when present and otherwise
-    // completes the wait in place.
+    // A signaler that already dequeued an object waiter owns completion. Let
+    // that handoff finish rather than returning timeout after ownership or a
+    // semaphore count has already transferred to this thread.
+    if (thread->WaitObject)
+        return 0;
+
+    // Delays and auxiliary scheduler waits have no object-owned queue.
     KeUnblockThreadFromWaitListWithStatus(thread, status);
+    return 1;
 }
 
 // Implements the unblock Thread operation.
@@ -601,11 +753,9 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
     if (thread->State != ThreadWaiting || !thread->AlertableWait)
         return NTLSTATUS_PENDING;
 
-    if (thread->WaitListHead)
-        KeListRemoveChecked(&thread->SchedulerLink);
-
-    KeUnblockThreadWithStatus(thread, status);
-    return NTLSTATUS_SUCCESS;
+    return KeAbortThreadWait(thread, status)
+        ? NTLSTATUS_SUCCESS
+        : NTLSTATUS_PENDING;
 }
 
 // Implements the alert Thread operation.

code/kernel/ke_time.c

@@ -99,7 +99,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out
     uint32_t sod = (uint32_t)(epochSeconds % 86400u);
 
     uint32_t year = 1970u;
-    for (;;) {
+    while (true) {
         uint32_t diy = KiIsLeapYear(year) ? 366u : 365u;
         if (days < diy) break;
         days -= diy;
@@ -107,7 +107,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out
     }
 
     uint32_t month = 1u;
-    for (;;) {
+    while (true) {
         uint32_t dim = KiDaysInMonth(year, month);
         if (days < dim) break;
         days -= dim;
@@ -519,6 +519,13 @@ __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime) {
     return NTLSTATUS_SUCCESS;
 }
 
+// Returns the monotonic platform counter in microseconds.
+uint64_t KeQueryMonotonicTicks(void) {
+    return (gTimeHal && gTimeHal->Timer.QueryTicks)
+         ? gTimeHal->Timer.QueryTicks()
+         : 0;
+}
+
 // Returns unix Time Seconds.
 __declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void) {
     SYS_SYSTEM_TIME t;

code/kernel/ke_work.c

@@ -8,6 +8,7 @@
 
 #include <kernel/ke_work.h>
 #include <kernel/ke_event.h>
+#include <kernel/ke_dpc.h>
 #include <kernel/ke_spinlock.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
@@ -17,17 +18,31 @@
 static SPINLOCK gWorkQueueLock = SPINLOCK_INIT;
 static LIST_ENTRY gWorkQueueList;
 static EVENT* gWorkQueueEvent;
+static KDPC gWorkQueueWakeDpc;
 static volatile long gWorkQueueListReady;
 static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started.
 
+// Signals the passive worker after an enqueue from hardware interrupt context.
+static void KiWorkQueueWakeDpc(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
+    (void)dpc;
+    (void)context;
+    (void)systemArgument1;
+    (void)systemArgument2;
+    KeAssertIrql(DISPATCH_LEVEL);
+
+    EVENT* event = gWorkQueueEvent;
+    if (event)
+        KeSignalEvent(event);
+}
+
 static void KiWorkQueueWorker(void* arg) {
     (void)arg;
     KeAssertIrql(PASSIVE_LEVEL);
 
-    for (;;) {
+    while (true) {
         (void)KeWaitForEvent(gWorkQueueEvent, 0);
 
-        for (;;) {
+        while (true) {
             KWORK_ITEM* item = (KWORK_ITEM*)0;
             uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
             if (!LIST_IS_EMPTY(&gWorkQueueList)) {
@@ -79,8 +94,14 @@ bool KeQueueWorkItem(KWORK_ITEM* item) {
     EVENT* event = gWorkQueueEvent;
     KeReleaseSpinlock(&gWorkQueueLock, flags);
 
-    if (event)
-        KeSignalEvent(event);
+    if (event) {
+        // Event signaling can detach waiters, so callers above passive level
+        // defer the wake until no caller-owned spinlock remains held.
+        if (KeGetCurrentIrql() == PASSIVE_LEVEL)
+            KeSignalEvent(event);
+        else
+            (void)KeInsertQueueDpc(&gWorkQueueWakeDpc, (void*)0, (void*)0);
+    }
     return true;
 }
 
@@ -95,14 +116,16 @@ KSTATUS KeStartWorkQueue(void) {
     }
 
     LIST_INIT(&gWorkQueueList);
+    KeInitializeDpc(&gWorkQueueWakeDpc, KiWorkQueueWakeDpc, (void*)0);
     _InterlockedExchange(&gWorkQueueListReady, 1);
 
     KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
     if (KSUCCESS(s)) {
         THREAD* worker = (THREAD*)0;
+        const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
         s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker,
                                  0,
-                                 PriorityNormal,
+                                 &parameters,
                                  &worker);
         // Persistent system worker; keep the creator reference for kernel life.
         (void)worker;

code/kernel/kmain.c

@@ -29,6 +29,7 @@
 #include <kernel/ke_input.h>
 #include <kernel/ke_time.h>
 #include <kernel/ke_display.h>
+#include <kernel/ke_audio.h>
 #include <kernel/ke_settings.h>
 #include <kernel/ke_work.h>
 #include <kernel/ke_system_root.h>
@@ -422,7 +423,8 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
     uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
 
     // Create the main thread last. KeCreateThread queues it before returning.
-    s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread);
+    const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
+    s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, &threadParameters, &thread);
     if (!KSUCCESS(s)) {
         DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
         proc->ImageBase = 0;
@@ -447,6 +449,9 @@ void SystemInitThread(void* arg) {
     KiRunEarlyKernelSelfTests();
 
     KiInitializeDrivers(args->bootInfo, args->hal);
+
+    if (args->bootInfo->Debug & KE_DEBUG_AUDIO_SELFTEST)
+        KiRunAudioFinalPathSelfTest();
     KeTimeReloadSettings();
 
     KiInitializeSwapFromSettings();
@@ -532,6 +537,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
     gKmainHal = hal;
 
     KeInitDebug(hal, bootInfo->Debug);
+    DbgPrint("CPU: PAT owns WB/WC/UC policy; MTRR default=WB fixed=off variable=off.\n");
     KeTimeInitialize(bootInfo, hal);
     KeDisplayInitialize(hal);
 
@@ -566,6 +572,15 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
 
     KeInitializeScheduler();
 
+    KSTATUS volumeNamespaceStatus = IoInitializeVolumeNamespace();
+    if (!KSUCCESS(volumeNamespaceStatus)) {
+        KePanic(KE_PANIC_CODE_INIT,
+                "I/O: failed to initialize volume namespace notifications, status=0x%x\n",
+                volumeNamespaceStatus);
+    }
+
+    KeAudioInitialize();
+
     KSTATUS workStatus = KeStartWorkQueue();
     if (!KSUCCESS(workStatus)) {
         KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus);
@@ -585,9 +600,10 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
 
     // Spawn the Initialization Thread. No creator reference is needed; the
     // scheduler owns the live thread lifetime.
+    const KE_THREAD_CREATE_PARAMETERS initThreadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
     KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
                                       (uintptr_t)initArgs,
-                                      PriorityNormal, (THREAD**)0);
+                                      &initThreadParameters, (THREAD**)0);
     
     if (!KSUCCESS(st)) {
         KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st);

code/kernel/ldr_section.c

@@ -327,8 +327,18 @@ static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section,
     if (!size)
         return NTLSTATUS_SUCCESS;
 
+    // Image prototype pages are the canonical cache for executable image bytes.
+    // Read the backing filesystem directly so the fault path has one cache owner.
+    if (!section->BackingFile->Fs || !section->BackingFile->Fs->Read)
+        return NTLSTATUS_NOT_SUPPORTED;
+
     size_t got = 0;
-    KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got);
+    KSTATUS s = section->BackingFile->Fs->Read(section->BackingFile->Fs,
+                                               section->BackingFile->FileCtx,
+                                               fileOffset,
+                                               dst,
+                                               size,
+                                               &got);
     if (!KSUCCESS(s))
         return s;
     if (got != size)

code/kernel/mm_file_cache.c

@@ -16,19 +16,12 @@
 #include <kernel/ob_object.h>
 #include <kernel/ke_spinlock.h>
 #include <kernel/ke_irql.h>
-#include <kernel/ke_list.h>
 #include <kernel/ke_semaphore.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
 
 #define MM_FILE_CACHE_BUCKETS 64u
 #define MM_FILE_CACHE_SOFT_MAX_PAGES 512u
-#define MM_FILE_CACHE_READAHEAD_MAX_REQUESTS 64u
-#define MM_FILE_CACHE_READAHEAD_MAX_PAGES 32u
-#define MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET 128u
-#define MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES 16u
-#define MM_FILE_CACHE_BATCH_MAX_PAGES 16u
-#define MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES 64u
 #define MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES 128u
 #define MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES 192u
 #define MM_FILE_CACHE_WRITEBACK_BATCH_PAGES 16u
@@ -48,13 +41,6 @@ typedef struct MM_FILE_CACHE_ENTRY {
     uint64_t          DirtySequence;
 } MM_FILE_CACHE_ENTRY;
 
-typedef struct MM_FILE_CACHE_READAHEAD_REQUEST {
-    LIST_ENTRY Link;
-    FILE_OBJECT* File;
-    uint64_t Offset;
-    uint32_t Pages;
-} MM_FILE_CACHE_READAHEAD_REQUEST;
-
 static SPINLOCK gFileCacheLock = SPINLOCK_INIT;
 static MM_FILE_CACHE_ENTRY* gFileCacheBuckets[MM_FILE_CACHE_BUCKETS];
 static uint64_t gFileCacheResidentPages;
@@ -65,13 +51,6 @@ static uint64_t gFileCacheMaxDirtyPages;
 static uint64_t gFileCacheMaxResidentPages;
 static MM_FILE_CACHE_STATS gFileCacheStats;
 
-static SPINLOCK gReadAheadLock = SPINLOCK_INIT;
-static LIST_ENTRY gReadAheadQueue;
-static SEMAPHORE* gReadAheadSemaphore;
-static uint32_t gReadAheadQueuedRequests;
-static uint32_t gReadAheadQueuedPages;
-static int gReadAheadQueueReady;
-static int gReadAheadStartState;
 
 static SPINLOCK gWritebackLock = SPINLOCK_INIT;
 static SEMAPHORE* gWritebackSemaphore;
@@ -253,7 +232,8 @@ static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOff
     return NTLSTATUS_NOT_FOUND;
 }
 
-static void KiFileCacheAddReadAheadStat(uint64_t* field, uint64_t value) {
+// Adds to one file-cache statistic under the cache lock.
+static void KiFileCacheAddStat(uint64_t* field, uint64_t value) {
     uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
     *field += value;
     KeReleaseSpinlock(&gFileCacheLock, flags);
@@ -265,17 +245,6 @@ static void KiFileCacheSignalWriteback(void) {
         KeSignalSemaphore(gWritebackSemaphore);
 }
 
-// Updates a maximum counter while the file-cache stats lock is held.
-static void KiFileCacheMaxStatLocked(uint64_t* field, uint64_t value) {
-    if (*field < value)
-        *field = value;
-}
-
-static int KiFileCachePageExistsLocked(FILE_OBJECT* file, uint64_t offset) {
-    uint32_t bucket = KiFileCacheHash(file, offset);
-    return KiFileCacheFindLocked(file, offset, bucket) != (MM_FILE_CACHE_ENTRY*)0;
-}
-
 // Finds the lowest dirty page for a file, or globally when file is NULL.
 static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) {
     MM_FILE_CACHE_ENTRY* best = (MM_FILE_CACHE_ENTRY*)0;
@@ -292,115 +261,6 @@ static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file)
     return best;
 }
 
-static MM_FILE_CACHE_READAHEAD_REQUEST* KiFileCacheReadAheadDequeue(void) {
-    uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock);
-    LIST_ENTRY* node = ListPopHead(&gReadAheadQueue);
-    if (node) {
-        MM_FILE_CACHE_READAHEAD_REQUEST* req = CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link);
-        if (gReadAheadQueuedRequests)
-            gReadAheadQueuedRequests--;
-        if (gReadAheadQueuedPages >= req->Pages)
-            gReadAheadQueuedPages -= req->Pages;
-        else
-            gReadAheadQueuedPages = 0;
-    }
-    KeReleaseSpinlock(&gReadAheadLock, flags);
-
-    if (!node)
-        return (MM_FILE_CACHE_READAHEAD_REQUEST*)0;
-    return CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link);
-}
-
-static void KiFileCacheReadAheadWorker(void* arg) {
-    (void)arg;
-    KeAssertIrql(PASSIVE_LEVEL);
-
-    for (;;) {
-        KSTATUS ws = KeWaitSemaphore(gReadAheadSemaphore);
-        if (!KSUCCESS(ws)) {
-            KeYield();
-            continue;
-        }
-
-        MM_FILE_CACHE_READAHEAD_REQUEST* req = KiFileCacheReadAheadDequeue();
-        if (!req)
-            continue;
-
-        for (uint32_t i = 0; i < req->Pages; ++i) {
-            uint64_t offset = req->Offset + ((uint64_t)i * PAGE_SIZE);
-
-            uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
-            if (gFileCacheResidentPages + MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES >=
-                MM_FILE_CACHE_SOFT_MAX_PAGES) {
-                gFileCacheStats.ReadAheadPagesThrottled += (uint64_t)(req->Pages - i);
-                KeReleaseSpinlock(&gFileCacheLock, flags);
-                break;
-            }
-            int exists = KiFileCachePageExistsLocked(req->File, offset);
-            if (exists)
-                gFileCacheStats.ReadAheadPagesSkipped++;
-            KeReleaseSpinlock(&gFileCacheLock, flags);
-            if (exists)
-                continue;
-
-            MM_FILE_CACHE_PAGE page;
-            KSTATUS s = MmFileCacheGetPage(req->File, offset, &page);
-            flags = KeAcquireSpinlock(&gFileCacheLock);
-            if (KSUCCESS(s))
-                gFileCacheStats.ReadAheadPagesLoaded++;
-            else
-                gFileCacheStats.ReadAheadFailures++;
-            KeReleaseSpinlock(&gFileCacheLock, flags);
-
-            if (!KSUCCESS(s))
-                break;
-            MmFileCacheReleasePage(&page);
-        }
-
-        ObDereferenceObject(&req->File->Header);
-        MmFreePool(req);
-    }
-}
-
-static KSTATUS KiFileCacheEnsureReadAheadStarted(void) {
-    uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock);
-    if (gReadAheadStartState == 2) {
-        KeReleaseSpinlock(&gReadAheadLock, flags);
-        return NTLSTATUS_SUCCESS;
-    }
-    if (gReadAheadStartState == 1) {
-        KeReleaseSpinlock(&gReadAheadLock, flags);
-        while (gReadAheadStartState == 1)
-            KeYield();
-        return gReadAheadStartState == 2 ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
-    }
-
-    gReadAheadStartState = 1;
-    KeReleaseSpinlock(&gReadAheadLock, flags);
-
-    LIST_INIT(&gReadAheadQueue);
-    gReadAheadQueuedRequests = 0;
-    gReadAheadQueuedPages = 0;
-    gReadAheadQueueReady = 1;
-
-    KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gReadAheadSemaphore);
-    if (KSUCCESS(s)) {
-        THREAD* worker = (THREAD*)0;
-        s = KeCreateKernelThread((uintptr_t)KiFileCacheReadAheadWorker,
-                                 0,
-                                 PriorityNormal,
-                                 &worker);
-        (void)worker;
-    }
-
-    flags = KeAcquireSpinlock(&gReadAheadLock);
-    gReadAheadStartState = KSUCCESS(s) ? 2 : 0;
-    if (!KSUCCESS(s))
-        gReadAheadQueueReady = 0;
-    KeReleaseSpinlock(&gReadAheadLock, flags);
-    return s;
-}
-
 // Flushes the next dirty cache page for file, or globally when file is NULL.
 static KSTATUS KiFileCacheFlushOneDirtyPage(FILE_OBJECT* file, int* didWork) {
     if (didWork) *didWork = 0;
@@ -462,8 +322,8 @@ static void KiFileCacheWritebackWorker(void* arg) {
     (void)arg;
     KeAssertIrql(PASSIVE_LEVEL);
 
-    for (;;) {
-        KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore);
+    while (true) {
+        KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore, KE_WAIT_INFINITE);
         if (!KSUCCESS(ws)) {
             KeYield();
             continue;
@@ -504,9 +364,10 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) {
     KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gWritebackSemaphore);
     if (KSUCCESS(s)) {
         THREAD* worker = (THREAD*)0;
+        const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
         s = KeCreateKernelThread((uintptr_t)KiFileCacheWritebackWorker,
                                  0,
-                                 PriorityNormal,
+                                 &parameters,
                                  &worker);
         (void)worker;
     }
@@ -517,15 +378,13 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) {
     return s;
 }
 
-static size_t KiMinSize(size_t a, size_t b) { return a < b ? a : b; }
-
 static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_PAGE* outPage, int waitForLoading) {
     if (outPage)
         memset(outPage, 0, sizeof(*outPage));
 
     uint32_t bucket = KiFileCacheHash(file, aligned);
     int counted = 0;
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
         if (!counted) {
             gFileCacheStats.Lookups++;
@@ -589,7 +448,7 @@ static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CA
 }
 
 static void KiFileCacheKeepBounded(FILE_OBJECT* skipFile, uint64_t skipOffset) {
-    for (;;) {
+    while (true) {
         uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
         int over = gFileCacheResidentPages > MM_FILE_CACHE_SOFT_MAX_PAGES;
         KeReleaseSpinlock(&gFileCacheLock, flags);
@@ -718,42 +577,12 @@ static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry,
     return NTLSTATUS_SUCCESS;
 }
 
-static uint32_t KiFileCacheChooseForegroundBatchPages(void) {
-    uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
-    uint64_t resident = gFileCacheResidentPages;
-    KeReleaseSpinlock(&gFileCacheLock, flags);
-
-    if (resident >= MM_FILE_CACHE_SOFT_MAX_PAGES)
-        return 1;
-
-    uint64_t headroom = MM_FILE_CACHE_SOFT_MAX_PAGES - resident;
-    if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES)
-        return 1;
-    if (headroom <= MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES)
-        return 2;
-    if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 2u))
-        return 4;
-    if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 4u))
-        return 8;
-    return MM_FILE_CACHE_BATCH_MAX_PAGES;
-}
-
-static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file,
+// Loads exactly one demanded file page into the coherent cache.
+static KSTATUS KiFileCacheLoadPage(FILE_OBJECT* file,
                                    uint64_t aligned,
-                                   uint32_t maxPages,
-                                   MM_FILE_CACHE_PAGE* requestedPage,
-                                   uint32_t* loadedPagesOut) {
-    if (loadedPagesOut)
-        *loadedPagesOut = 0;
-    if (!file || maxPages == 0)
+                                   MM_FILE_CACHE_PAGE* outPage) {
+    if (!file || !outPage)
         return NTLSTATUS_INVALID_PARAMETER;
-    if (maxPages > MM_FILE_CACHE_BATCH_MAX_PAGES)
-        maxPages = MM_FILE_CACHE_BATCH_MAX_PAGES;
-
-    uintptr_t statFlags = KeAcquireSpinlock(&gFileCacheLock);
-    gFileCacheStats.BatchPagesRequested += maxPages;
-    KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesRequested, maxPages);
-    KeReleaseSpinlock(&gFileCacheLock, statFlags);
 
     uintptr_t fileFlags = KeAcquireSpinlock(&file->Lock);
     uint64_t fileSize = file->Size;
@@ -761,134 +590,55 @@ static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file,
     if (aligned >= fileSize)
         return NTLSTATUS_SUCCESS;
 
-    MM_FILE_CACHE_ENTRY* entries[MM_FILE_CACHE_BATCH_MAX_PAGES];
-    PHYSICAL_ADDRESS frames[MM_FILE_CACHE_BATCH_MAX_PAGES];
-    memset(entries, 0, sizeof(entries));
-    memset(frames, 0, sizeof(frames));
+    MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)0;
+    KSTATUS s = KiFileCacheReserveLoadingPage(file, aligned, &entry);
+    if (s == NTLSTATUS_ALREADY_EXISTS)
+        return KiFileCacheTryHit(file, aligned, outPage, 1);
+    if (!KSUCCESS(s))
+        return s;
 
-    uint32_t pages = 0;
-    KSTATUS reserveStatus = NTLSTATUS_SUCCESS;
-    while (pages < maxPages) {
-        uint64_t pageOffset = aligned + ((uint64_t)pages * PAGE_SIZE);
-        if (pageOffset >= fileSize)
-            break;
-
-        reserveStatus = KiFileCacheReserveLoadingPage(file, pageOffset, &entries[pages]);
-        if (!KSUCCESS(reserveStatus))
-            break;
-        pages++;
-    }
-
-    if (pages == 0) {
-        if (requestedPage)
-            return KiFileCacheTryHit(file, aligned, requestedPage, 1);
-        return reserveStatus == NTLSTATUS_ALREADY_EXISTS ? NTLSTATUS_ALREADY_EXISTS : reserveStatus;
-    }
-
-    uint8_t* readBuffer = (uint8_t*)MmAllocatePoolTag((size_t)pages * PAGE_SIZE, MM_POOL_TAG('f', 'c', 'b', 'l'));
-    if (!readBuffer) {
-        for (uint32_t i = 0; i < pages; ++i)
-            KiFileCacheCancelLoadingPage(entries[i]);
-        KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1);
+    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseCleanFileCache);
+    if (!frame) {
+        KiFileCacheCancelLoadingPage(entry);
+        KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
         return NTLSTATUS_NO_MEMORY;
     }
-    memset(readBuffer, 0, (size_t)pages * PAGE_SIZE);
 
-    for (uint32_t i = 0; i < pages; ++i) {
-        frames[i] = MmPfnAllocate(MmPfnUseCleanFileCache);
-        if (!frames[i]) {
-            for (uint32_t j = 0; j < i; ++j)
-                (void)MmPfnRelease(frames[j]);
-            for (uint32_t j = 0; j < pages; ++j)
-                KiFileCacheCancelLoadingPage(entries[j]);
-            MmFreePool(readBuffer);
-            KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1);
-            return NTLSTATUS_NO_MEMORY;
-        }
+    void* page = MmPfnToVirtual(frame);
+    if (!page) {
+        (void)MmPfnRelease(frame);
+        KiFileCacheCancelLoadingPage(entry);
+        KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
+        return NTLSTATUS_FAILURE;
     }
+    memset(page, 0, PAGE_SIZE);
+
+    size_t toRead = PAGE_SIZE;
+    uint64_t remaining = fileSize - aligned;
+    if (remaining < PAGE_SIZE)
+        toRead = (size_t)remaining;
 
-    size_t toRead = KiMinSize((size_t)pages * PAGE_SIZE, (size_t)(fileSize - aligned));
     size_t got = 0;
-    KSTATUS s = NTLSTATUS_NOT_SUPPORTED;
+    s = NTLSTATUS_NOT_SUPPORTED;
     if (file->Fs && file->Fs->Read)
-        s = file->Fs->Read(file->Fs, file->FileCtx, aligned, readBuffer, toRead, &got);
+        s = file->Fs->Read(file->Fs, file->FileCtx, aligned, page, toRead, &got);
     if (!KSUCCESS(s)) {
-        for (uint32_t i = 0; i < pages; ++i) {
-            if (frames[i])
-                (void)MmPfnRelease(frames[i]);
-            KiFileCacheCancelLoadingPage(entries[i]);
-        }
-        MmFreePool(readBuffer);
-        KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1);
+        (void)MmPfnRelease(frame);
+        KiFileCacheCancelLoadingPage(entry);
+        KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
         return s;
     }
     if (got > toRead)
         got = toRead;
+    if (got < PAGE_SIZE)
+        memset((uint8_t*)page + got, 0, PAGE_SIZE - got);
 
-    uint64_t published = 0;
-    KSTATUS requestedStatus = NTLSTATUS_NOT_FOUND;
-    for (uint32_t i = 0; i < pages; ++i) {
-        uint64_t pageOffset = aligned + ((uint64_t)i * PAGE_SIZE);
-        size_t pageBase = (size_t)i * PAGE_SIZE;
-        if (pageBase >= got && pageOffset >= fileSize) {
-            (void)MmPfnRelease(frames[i]);
-            frames[i] = 0;
-            KiFileCacheCancelLoadingPage(entries[i]);
-            continue;
-        }
-
-        uint32_t validBytes = 0;
-        if (pageBase < got) {
-            size_t avail = got - pageBase;
-            if (avail > PAGE_SIZE) avail = PAGE_SIZE;
-            validBytes = (uint32_t)avail;
-        }
-
-        void* page = MmPfnToVirtual(frames[i]);
-        if (!page) {
-            (void)MmPfnRelease(frames[i]);
-            frames[i] = 0;
-            KiFileCacheCancelLoadingPage(entries[i]);
-            continue;
-        }
-        memcpy(page, readBuffer + pageBase, validBytes);
-        if (validBytes < PAGE_SIZE)
-            memset((uint8_t*)page + validBytes, 0, PAGE_SIZE - validBytes);
-
-        MM_FILE_CACHE_PAGE* out = (pageOffset == aligned) ? requestedPage : (MM_FILE_CACHE_PAGE*)0;
-        KSTATUS ps = KiFileCacheCompleteLoadingPage(entries[i], frames[i], validBytes, out, &published);
-        if (KSUCCESS(ps)) {
-            if (pageOffset == aligned)
-                requestedStatus = NTLSTATUS_SUCCESS;
-            frames[i] = 0;
-        } else {
-            if (pageOffset == aligned)
-                requestedStatus = ps;
-            frames[i] = 0;
-        }
-    }
-
-    MmFreePool(readBuffer);
-
-    uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
-    if (published) {
-        gFileCacheStats.BatchReads++;
-        gFileCacheStats.BatchPagesLoaded += published;
-        KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesLoaded, published);
-    }
-    KeReleaseSpinlock(&gFileCacheLock, flags);
-
-    if (loadedPagesOut)
-        *loadedPagesOut = (uint32_t)published;
+    s = KiFileCacheCompleteLoadingPage(entry, frame, (uint32_t)got, outPage, (uint64_t*)0);
+    if (!KSUCCESS(s))
+        return s;
 
     KiFileCacheKeepBounded(file, aligned);
-
-    if (requestedPage) {
-        if (KSUCCESS(requestedStatus))
-            return NTLSTATUS_SUCCESS;
-        return KiFileCacheTryHit(file, aligned, requestedPage, 1);
-    }
-    return published ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND;
+    return NTLSTATUS_SUCCESS;
 }
 
 // Returns a retained cache page suitable for in-place buffered writes.
@@ -906,7 +656,7 @@ static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file,
         return NTLSTATUS_SUCCESS;
 
     if (needExistingBytes && aligned < oldFileSize) {
-        s = KiFileCacheLoadSpan(file, aligned, 1, outPage, (uint32_t*)0);
+        s = KiFileCacheLoadPage(file, aligned, outPage);
         if (KSUCCESS(s))
             return NTLSTATUS_SUCCESS;
         if (s != NTLSTATUS_NOT_FOUND)
@@ -1063,7 +813,7 @@ KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file) {
     KeReleaseSpinlock(&gFileCacheLock, flags);
 
     KSTATUS firstError = NTLSTATUS_SUCCESS;
-    for (;;) {
+    while (true) {
         int didWork = 0;
         KSTATUS s = KiFileCacheFlushOneDirtyPage(file, &didWork);
         if (!didWork)
@@ -1090,7 +840,7 @@ KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAG
     KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1);
     if (KSUCCESS(s))
         return NTLSTATUS_SUCCESS;
-    return KiFileCacheLoadSpan(file, aligned, KiFileCacheChooseForegroundBatchPages(), outPage, (uint32_t*)0);
+    return KiFileCacheLoadPage(file, aligned, outPage);
 }
 
 void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page) {
@@ -1111,7 +861,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) {
     }
 
     for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
-        for (;;) {
+        while (true) {
             MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
             uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
             MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
@@ -1145,7 +895,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) {
 void MmFileCacheInvalidateAll(FILE_OBJECT* file) {
     if (!file) return;
     for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
-        for (;;) {
+        while (true) {
             MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
             uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
             MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
@@ -1179,83 +929,6 @@ KSTATUS MmFileCacheDropOnePageForPressure(void) {
     return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0);
 }
 
-KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages) {
-    if (!file || pages == 0)
-        return NTLSTATUS_INVALID_PARAMETER;
-    if (!KeCanWait())
-        return NTLSTATUS_INVALID_IRQL;
-
-    uint32_t requestedPages = pages;
-    if (pages > MM_FILE_CACHE_READAHEAD_MAX_PAGES)
-        pages = MM_FILE_CACHE_READAHEAD_MAX_PAGES;
-
-    uintptr_t cacheFlags = KeAcquireSpinlock(&gFileCacheLock);
-    gFileCacheStats.ReadAheadPagesRequested += requestedPages;
-    KiFileCacheMaxStatLocked(&gFileCacheStats.MaxReadAheadPagesPerRequest, requestedPages);
-    uint64_t resident = gFileCacheResidentPages;
-    uint64_t headroom = resident < MM_FILE_CACHE_SOFT_MAX_PAGES
-        ? MM_FILE_CACHE_SOFT_MAX_PAGES - resident
-        : 0;
-    if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) {
-        gFileCacheStats.ReadAheadPagesThrottled += pages;
-        KeReleaseSpinlock(&gFileCacheLock, cacheFlags);
-        return NTLSTATUS_QUEUE_FULL;
-    }
-    headroom -= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES;
-    if ((uint64_t)pages > headroom)
-        pages = (uint32_t)headroom;
-    KeReleaseSpinlock(&gFileCacheLock, cacheFlags);
-    if (pages == 0)
-        return NTLSTATUS_QUEUE_FULL;
-
-    KSTATUS s = KiFileCacheEnsureReadAheadStarted();
-    if (!KSUCCESS(s)) {
-        KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1);
-        return s;
-    }
-
-    MM_FILE_CACHE_READAHEAD_REQUEST* req =
-        (MM_FILE_CACHE_READAHEAD_REQUEST*)MmAllocatePoolTag(sizeof(*req), MM_POOL_TAG('r', 'd', 'a', 'h'));
-    if (!req) {
-        KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1);
-        return NTLSTATUS_NO_MEMORY;
-    }
-
-    LIST_INIT(&req->Link);
-    req->File = file;
-    req->Offset = offset & ~(uint64_t)(PAGE_SIZE - 1);
-    req->Pages = pages;
-    ObReferenceObject(&file->Header);
-
-    uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock);
-    if (!gReadAheadQueueReady ||
-        gReadAheadQueuedRequests >= MM_FILE_CACHE_READAHEAD_MAX_REQUESTS ||
-        gReadAheadQueuedPages >= MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET) {
-        KeReleaseSpinlock(&gReadAheadLock, flags);
-        ObDereferenceObject(&file->Header);
-        MmFreePool(req);
-        KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadPagesThrottled, pages);
-        return NTLSTATUS_QUEUE_FULL;
-    }
-    uint32_t queueHeadroom = MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET - gReadAheadQueuedPages;
-    if (pages > queueHeadroom)
-        pages = queueHeadroom;
-    req->Pages = pages;
-
-    ListInsertTail(&gReadAheadQueue, &req->Link);
-    gReadAheadQueuedRequests++;
-    gReadAheadQueuedPages += pages;
-    KeReleaseSpinlock(&gReadAheadLock, flags);
-
-    flags = KeAcquireSpinlock(&gFileCacheLock);
-    gFileCacheStats.ReadAheadRequests++;
-    gFileCacheStats.ReadAheadPagesQueued += pages;
-    KeReleaseSpinlock(&gFileCacheLock, flags);
-
-    KeSignalSemaphore(gReadAheadSemaphore);
-    return NTLSTATUS_SUCCESS;
-}
-
 void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out) {
     if (!out) return;
     uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);

code/kernel/mm_phys.c

@@ -472,8 +472,13 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min
     return (PHYSICAL_ADDRESS)0;
 }
 
-// Allocates a physically contiguous run for DMA or hardware structures.
-__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) {
+// Allocates a physically contiguous run inside an address and alignment window.
+PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count,
+                                                 PHYSICAL_ADDRESS minInclusive,
+                                                 PHYSICAL_ADDRESS maxExclusive,
+                                                 size_t alignment,
+                                                 size_t boundary,
+                                                 MM_PFN_USAGE usage) {
     if (!count || count > gMaxPfn) {
         uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
         gPfnFailedContiguousAllocations++;
@@ -482,6 +487,50 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_
         return (PHYSICAL_ADDRESS)0;
     }
 
+    if (!alignment)
+        alignment = PAGE_SIZE;
+    if (alignment < PAGE_SIZE)
+        alignment = PAGE_SIZE;
+    if ((alignment & (alignment - 1)) != 0 ||
+        (boundary && (boundary & (boundary - 1)) != 0)) {
+        uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
+        gPfnFailedContiguousAllocations++;
+        gPfnFailedContiguousInvalidRequest++;
+        KeReleaseSpinlock(&gPfnLock, flags);
+        return (PHYSICAL_ADDRESS)0;
+    }
+
+    if (count > ((size_t)-1) / PAGE_SIZE) {
+        uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
+        gPfnFailedContiguousAllocations++;
+        gPfnFailedContiguousInvalidRequest++;
+        KeReleaseSpinlock(&gPfnLock, flags);
+        return (PHYSICAL_ADDRESS)0;
+    }
+
+    size_t byteCount = count * PAGE_SIZE;
+    if (boundary && byteCount > boundary) {
+        uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
+        gPfnFailedContiguousAllocations++;
+        gPfnFailedContiguousInvalidRequest++;
+        KeReleaseSpinlock(&gPfnLock, flags);
+        return (PHYSICAL_ADDRESS)0;
+    }
+
+    minInclusive = KiAlignUp(minInclusive);
+    maxExclusive = KiAlignDown(maxExclusive);
+    PHYSICAL_ADDRESS managedLimit = (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE;
+    if (maxExclusive > managedLimit)
+        maxExclusive = managedLimit;
+    if (maxExclusive <= minInclusive ||
+        (PHYSICAL_ADDRESS)byteCount > maxExclusive - minInclusive) {
+        uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
+        gPfnFailedContiguousAllocations++;
+        gPfnFailedContiguousInvalidRequest++;
+        KeReleaseSpinlock(&gPfnLock, flags);
+        return (PHYSICAL_ADDRESS)0;
+    }
+
     if (!KiPfnUsageMayUseReserve(usage))
         KiPfnReclaimForPressure();
 
@@ -493,23 +542,31 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_
         return (PHYSICAL_ADDRESS)0;
     }
 
-    size_t runStart = 0;
-    size_t runLength = 0;
-
-    for (size_t pfn = 1; pfn < gMaxPfn; pfn++) {
-        if (KiPfnAvailableLocked(pfn)) {
-            if (!runLength)
-                runStart = pfn;
-            runLength++;
-            if (runLength == count) {
-                for (size_t i = 0; i < count; i++)
-                    KiClaimPfnLocked(runStart + i, usage);
-                KeReleaseSpinlock(&gPfnLock, flags);
-                return MmPfnAddress(runStart);
-            }
-        } else {
-            runLength = 0;
+    size_t firstPfn = (size_t)(minInclusive / PAGE_SIZE);
+    size_t limitPfn = (size_t)(maxExclusive / PAGE_SIZE);
+    for (size_t runStart = firstPfn; runStart + count <= limitPfn; ++runStart) {
+        PHYSICAL_ADDRESS runAddress = MmPfnAddress(runStart);
+        if ((runAddress & (alignment - 1)) != 0)
+            continue;
+        if (boundary) {
+            PHYSICAL_ADDRESS runEnd = runAddress + (PHYSICAL_ADDRESS)byteCount - 1;
+            if ((runAddress & ~((PHYSICAL_ADDRESS)boundary - 1)) !=
+                (runEnd & ~((PHYSICAL_ADDRESS)boundary - 1)))
+                continue;
         }
+
+        size_t available = 0;
+        while (available < count && KiPfnAvailableLocked(runStart + available))
+            available++;
+        if (available != count) {
+            runStart += available;
+            continue;
+        }
+
+        for (size_t i = 0; i < count; ++i)
+            KiClaimPfnLocked(runStart + i, usage);
+        KeReleaseSpinlock(&gPfnLock, flags);
+        return runAddress;
     }
 
     gPfnFailedContiguousAllocations++;
@@ -518,6 +575,16 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_
     return (PHYSICAL_ADDRESS)0;
 }
 
+// Allocates a physically contiguous run for DMA or hardware structures.
+__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) {
+    return MmPfnAllocateContiguousInRange(count,
+                                           PAGE_SIZE,
+                                           (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE,
+                                           PAGE_SIZE,
+                                           0,
+                                           usage);
+}
+
 // Increments the reference count on an allocated frame.
 __declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame) {
     size_t pfn = MmPfnIndex(frame);

code/kernel/mm_pte.c

@@ -60,6 +60,14 @@ static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose
     return purpose;
 }
 
+// Returns whether flags select one canonical PAT cache policy.
+static bool KiPteCacheFlagsValid(uint64_t flags) {
+    uint64_t cacheFlags = flags & PAGE_CACHE_MASK;
+    return cacheFlags == PAGE_CACHE_WB ||
+           cacheFlags == PAGE_CACHE_WC ||
+           cacheFlags == PAGE_CACHE_UC;
+}
+
 // Ensures that one parent entry points to a child page table.
 // New child tables are zeroed before publication so no stale mappings become visible.
 static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
@@ -200,6 +208,10 @@ int MmPteHandleShootdownNmi(void) {
 // Maps or replaces one 4 KiB leaf PTE in the shared page tables.
 __declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) {
     KiPteStatInc(&gPteStats.MapCalls);
+    if (!KiPteCacheFlagsValid(flags)) {
+        KiPteStatInc(&gPteStats.MapFailures);
+        return NTLSTATUS_INVALID_PARAMETER;
+    }
     unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
 
     uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
@@ -286,6 +298,10 @@ __declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
 // Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them.
 __declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) {
     KiPteStatInc(&gPteStats.ProtectCalls);
+    if (!KiPteCacheFlagsValid(flags)) {
+        KiPteStatInc(&gPteStats.ProtectFailures);
+        return NTLSTATUS_INVALID_PARAMETER;
+    }
     if (!size) return NTLSTATUS_SUCCESS;
 
     // The protection operation is page-granular even when callers pass an interior address.

code/kernel/mm_virt.c

@@ -7,6 +7,7 @@
 //   FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings.
 //   FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map.
 //   FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys.
+//   FFFFA00000000000-FFFFAFFFFFFFFFFF is the kernel shared-view arena.
 //   FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool.
 //
 // The current VM model uses one shared PML4, so user pointers are globally
@@ -112,6 +113,12 @@ typedef struct MM_VAD_PAGE {
     uint32_t SwapSlot;
 } MM_VAD_PAGE;
 
+struct MM_KERNEL_VIEW {
+    LIST_ENTRY Link;
+    uintptr_t Base;
+    size_t PageCount;
+};
+
 typedef struct MM_VAD {
     LIST_ENTRY ProcessLink;
     LIST_ENTRY GlobalLink;
@@ -143,13 +150,19 @@ static LIST_ENTRY gVadList;
 static SPINLOCK   gVadLock = SPINLOCK_INIT;
 static int        gVadListReady = 0;
 static MM_VAD_STATS gVadStats;
+static LIST_ENTRY gKernelViewList;
+static SPINLOCK gKernelViewLock = SPINLOCK_INIT;
 
-#define MM_VAD_ARENA_BASE 0x0000100000000000ULL
+#define MM_VAD_ARENA_BASE     0x0000100000000000ULL
+#define MM_KERNEL_VIEW_BASE   0xFFFFA00000000000ULL
+#define MM_KERNEL_VIEW_LIMIT  0xFFFFB00000000000ULL
 
 // Initializes the VAD list after the shared page table and kernel pool are ready.
 static void KiVadInitialize(void) {
     LIST_INIT(&gVadList);
+    LIST_INIT(&gKernelViewList);
     gVadLock = (SPINLOCK)SPINLOCK_INIT;
+    gKernelViewLock = (SPINLOCK)SPINLOCK_INIT;
     memset(&gVadStats, 0, sizeof(gVadStats));
     gVadListReady = 1;
 }
@@ -170,6 +183,108 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
     (void)MmPfnPin(gVadZeroPageFrame);
 }
 
+// Inserts a kernel view before one list node while preserving address order.
+static void KiKernelViewInsertBefore(LIST_ENTRY* before, MM_KERNEL_VIEW* view) {
+    view->Link.Fwd = before;
+    view->Link.Bwd = before->Bwd;
+    before->Bwd->Fwd = &view->Link;
+    before->Bwd = &view->Link;
+}
+
+// Maps existing frames into the kernel shared-view arena.
+KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames,
+                                     size_t pageCount,
+                                     MM_KERNEL_VIEW** outView) {
+    if (!outView)
+        return NTLSTATUS_INVALID_PARAMETER;
+    *outView = (MM_KERNEL_VIEW*)0;
+    if (!frames || pageCount == 0 || pageCount > (SIZE_MAX / PAGE_SIZE))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    size_t viewBytes = pageCount * PAGE_SIZE;
+    MM_KERNEL_VIEW* view = (MM_KERNEL_VIEW*)MmAllocatePoolTag(sizeof(MM_KERNEL_VIEW),
+                                                               MM_POOL_TAG('k', 'v', 'i', 'w'));
+    if (!view)
+        return NTLSTATUS_NO_MEMORY;
+    memset(view, 0, sizeof(*view));
+    LIST_INIT(&view->Link);
+
+    uintptr_t candidate = MM_KERNEL_VIEW_BASE;
+    LIST_ENTRY* before = &gKernelViewList;
+    uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock);
+    for (LIST_ENTRY* it = gKernelViewList.Fwd; it != &gKernelViewList; it = it->Fwd) {
+        MM_KERNEL_VIEW* existing = CONTAINING_RECORD(it, MM_KERNEL_VIEW, Link);
+        if (candidate <= existing->Base &&
+            viewBytes <= (size_t)(existing->Base - candidate)) {
+            before = it;
+            break;
+        }
+
+        uintptr_t existingEnd = existing->Base + existing->PageCount * PAGE_SIZE;
+        if (existingEnd > candidate)
+            candidate = existingEnd;
+    }
+
+    if (candidate > MM_KERNEL_VIEW_LIMIT || viewBytes > MM_KERNEL_VIEW_LIMIT - candidate) {
+        KeReleaseSpinlock(&gKernelViewLock, lockFlags);
+        MmFreePool(view);
+        return NTLSTATUS_NO_MEMORY;
+    }
+
+    view->Base = candidate;
+    view->PageCount = pageCount;
+    KiKernelViewInsertBefore(before, view);
+    KeReleaseSpinlock(&gKernelViewLock, lockFlags);
+
+    const uint64_t hardwareFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
+    size_t mappedPages = 0;
+    KSTATUS s = NTLSTATUS_SUCCESS;
+    for (; mappedPages < pageCount; mappedPages++) {
+        if (!frames[mappedPages]) {
+            s = NTLSTATUS_INVALID_PARAMETER;
+            break;
+        }
+        s = MmPteMapPage(candidate + mappedPages * PAGE_SIZE, frames[mappedPages],
+                         hardwareFlags, MmPteMapFailIfPresent);
+        if (!KSUCCESS(s))
+            break;
+    }
+
+    if (!KSUCCESS(s)) {
+        while (mappedPages != 0) {
+            mappedPages--;
+            (void)MmPteUnmapPage(candidate + mappedPages * PAGE_SIZE);
+        }
+        lockFlags = KeAcquireSpinlock(&gKernelViewLock);
+        ListRemove(&view->Link);
+        KeReleaseSpinlock(&gKernelViewLock, lockFlags);
+        MmFreePool(view);
+        return s;
+    }
+
+    *outView = view;
+    return NTLSTATUS_SUCCESS;
+}
+
+// Returns the base address of one kernel shared view.
+void* MmKernelViewBase(const MM_KERNEL_VIEW* view) {
+    return view ? (void*)view->Base : (void*)0;
+}
+
+// Unmaps and releases one kernel shared view without releasing its frames.
+void MmDestroyKernelView(MM_KERNEL_VIEW* view) {
+    if (!view)
+        return;
+
+    for (size_t i = 0; i < view->PageCount; i++)
+        (void)MmPteUnmapPage(view->Base + i * PAGE_SIZE);
+
+    uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock);
+    ListRemove(&view->Link);
+    KeReleaseSpinlock(&gKernelViewLock, lockFlags);
+    MmFreePool(view);
+}
+
 // VAD utility helpers.
 static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
     if (align < PAGE_SIZE) align = PAGE_SIZE;
@@ -199,7 +314,7 @@ static KSTATUS KiFindVadRange(size_t reservationSize, size_t alignment, uintptr_
     uintptr_t candidate = KiAlignUpPtr(MM_VAD_ARENA_BASE, alignment);
     if (!candidate) return NTLSTATUS_INVALID_PARAMETER;
 
-    for (;;) {
+    while (true) {
         if (candidate < MM_VAD_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate))
             return NTLSTATUS_NO_MEMORY;
 
@@ -438,7 +553,7 @@ static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) {
 static int KiVadTryBeginPaging(MM_VAD_PAGE* page) {
     if (!page) return 0;
 
-    for (;;) {
+    while (true) {
         uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0);
         if (oldFlags & MM_VAD_PAGE_PAGING)
             return 0;
@@ -454,7 +569,7 @@ static int KiVadTryBeginPaging(MM_VAD_PAGE* page) {
 static void KiVadEndPaging(MM_VAD_PAGE* page) {
     if (!page) return;
 
-    for (;;) {
+    while (true) {
         uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0);
         uint32_t newFlags = oldFlags & ~MM_VAD_PAGE_PAGING;
         if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags,
@@ -1157,6 +1272,18 @@ __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process,
     return NTLSTATUS_SUCCESS;
 }
 
+// Returns the page count held by a locked user buffer.
+size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer) {
+    return buffer ? buffer->PageCount : 0;
+}
+
+// Returns one pinned frame held by a locked user buffer.
+PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index) {
+    if (!buffer || index >= buffer->PageCount)
+        return (PHYSICAL_ADDRESS)0;
+    return buffer->Pages[index].Frame;
+}
+
 // Unlocks a user buffer locked for asynchronous kernel access.
 __declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer) {
     if (!buffer)
@@ -2256,7 +2383,7 @@ void MmVadDestroyAllForProcess(PROCESS* process) {
     KeAssertIrql(PASSIVE_LEVEL);
     if (!process) return;
 
-    for (;;) {
+    while (true) {
         uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock);
         LIST_ENTRY* node = (LIST_ENTRY*)0;
         int sawLocked = 0;

code/kernel/net_socket.c

@@ -379,8 +379,8 @@ static void NetAsyncSocketExecute(NET_ASYNC_SOCKET_REQUEST_OBJECT* req,
 static void NetAsyncSocketWorker(void* arg) {
     (void)arg;
 
-    for (;;) {
-        KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore);
+    while (true) {
+        KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore, KE_WAIT_INFINITE);
         if (!KSUCCESS(waitStatus)) {
             KeYield();
             continue;
@@ -426,9 +426,10 @@ static KSTATUS NetEnsureAsyncSocketWorkerStarted(void) {
     KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncSocketSemaphore);
     if (KSUCCESS(status)) {
         THREAD* worker = (THREAD*)0;
+        const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
         status = KeCreateKernelThread((uintptr_t)NetAsyncSocketWorker,
                                       0,
-                                      PriorityNormal,
+                                      &parameters,
                                       &worker);
         (void)worker;
     }

code/kernel/ob_object.c

@@ -31,6 +31,8 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
     case ObjectTypeTimer:       return MM_POOL_TAG('t', 'm', 'r', ' ');
     case ObjectTypeModule:      return MM_POOL_TAG('m', 'o', 'd', 'u');
     case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k');
+    case ObjectTypeAudioEndpoint: return MM_POOL_TAG('a', 'e', 'p', 't');
+    case ObjectTypeAudioStream:   return MM_POOL_TAG('a', 's', 't', 'r');
     default:                    return MM_POOL_TAG('o', 'b', 'j', ' ');
     }
 }
@@ -64,7 +66,7 @@ __declspec(dllexport) bool ObTryReferenceObject(OBJECT* object) {
     if (!object)
         return false;
 
-    for (;;) {
+    while (true) {
         if (_InterlockedCompareExchange(&object->LifetimeState, 0, 0) != ObjectAlive)
             return false;
         if ((_InterlockedCompareExchange(&object->Flags, 0, 0) & OBJECT_FLAG_INSERTED) == 0)