add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

code/kernel/ke_display.c

@@ -19,6 +19,7 @@
 #include <kernel/ke_list.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_irql.h>
+#include <kernel/ke_cpu.h>
 
 #define DISPLAY_DEFAULT_FG 0x00CCCCCCu
 #define DISPLAY_DEFAULT_BG 0x00012456u
@@ -44,6 +45,15 @@ typedef struct DISPLAY_OUTPUT_GLYPH {
     uint32_t Background;
 } DISPLAY_OUTPUT_GLYPH;
 
+typedef struct DISPLAY_OUTPUT_DRAW {
+    char Character;
+    uint32_t Foreground;
+    uint32_t Background;
+    int Column;
+    int Row;
+    size_t ScrollCount;
+} DISPLAY_OUTPUT_DRAW;
+
 typedef struct BOOTFB_CONTEXT {
     const HAL_API* Hal;
     uint32_t* Framebuffer;
@@ -59,6 +69,7 @@ static DISPLAY_DEVICE* gDisplayList;
 static DISPLAY_DEVICE* gPrimaryDisplay;
 static SPINLOCK gDisplayLock = SPINLOCK_INIT;
 static MUTEX* gDisplayProviderMutex;
+static MUTEX* gDisplayOutputMutex;
 static KWORK_ITEM gDisplayCursorWorkItem;
 static volatile long gDisplayCursorWorkInitialized;
 static bool gDisplayTransitionInProgress;
@@ -104,8 +115,10 @@ static KWORK_ITEM gDisplayOutputWorkItem;
 static volatile long gDisplayOutputWorkInitialized;
 
 static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg);
+static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count);
 static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
-static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels);
+static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
+static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
 static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color);
 static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h);
 static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride);
@@ -119,6 +132,7 @@ static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats)
 static const DISPLAY_DEVICE_OPS gBootFbOps = {
     .GetMode = BootFbGetMode,
     .Blit = BootFbBlit,
+    .PresentPitched = BootFbPresentPitched,
     .FillRect = BootFbFillRect,
     .CopyRect = BootFbCopyRect,
     .ReadRect = BootFbReadRect,
@@ -192,10 +206,15 @@ static KSTATUS KiDisplayAcquireProviderMutex(bool* acquired) {
     if (KeGetDebugFlags() & KE_DEBUG_DISPLAY_ASSERTS)
         KeAssertIrql(PASSIVE_LEVEL);
 
-    KSTATUS s = KeAcquireMutex(gDisplayProviderMutex);
-    if (KSUCCESS(s) && acquired)
+    KSTATUS s = KeAcquireMutex(gDisplayProviderMutex, KE_WAIT_INFINITE);
+    if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s))
+        return s;
+
+    if (acquired)
         *acquired = true;
-    return s;
+
+    // Provider operations still own the mutex after an abandoned acquisition.
+    return NTLSTATUS_SUCCESS;
 }
 
 static void KiDisplayReleaseProviderMutex(bool acquired) {
@@ -449,6 +468,34 @@ static bool KiDisplayConsoleGridFromMode(const DISPLAY_MODE* mode, int* outCols,
     return true;
 }
 
+// Returns whether two dirty rectangles overlap or share an edge.
+static bool KiDisplayDirtyRectsMergeable(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
+    if (!a || !b)
+        return false;
+
+    int64_t aRight = (int64_t)a->X + a->W;
+    int64_t aBottom = (int64_t)a->Y + a->H;
+    int64_t bRight = (int64_t)b->X + b->W;
+    int64_t bBottom = (int64_t)b->Y + b->H;
+    return (int64_t)a->X <= bRight && (int64_t)b->X <= aRight &&
+           (int64_t)a->Y <= bBottom && (int64_t)b->Y <= aBottom;
+}
+
+// Unions two clipped dirty rectangles.
+static DISPLAY_RECT KiDisplayUnionDirtyRects(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
+    int64_t left = a->X < b->X ? a->X : b->X;
+    int64_t top = a->Y < b->Y ? a->Y : b->Y;
+    int64_t aRight = (int64_t)a->X + a->W;
+    int64_t bRight = (int64_t)b->X + b->W;
+    int64_t aBottom = (int64_t)a->Y + a->H;
+    int64_t bBottom = (int64_t)b->Y + b->H;
+    int64_t right = aRight > bRight ? aRight : bRight;
+    int64_t bottom = aBottom > bBottom ? aBottom : bBottom;
+    DISPLAY_RECT result = { (int32_t)left, (int32_t)top,
+                            (int32_t)(right - left), (int32_t)(bottom - top) };
+    return result;
+}
+
 static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
     if (!mode || gDirtyAll || w <= 0 || h <= 0)
         return;
@@ -457,6 +504,19 @@ static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int
     if (!KiDisplayClipRect(mode, &rect))
         return;
 
+    for (size_t i = 0; i < gDirtyRectCount;) {
+        if (!KiDisplayDirtyRectsMergeable(&rect, &gDirtyRects[i])) {
+            i++;
+            continue;
+        }
+
+        rect = KiDisplayUnionDirtyRects(&rect, &gDirtyRects[i]);
+        gDirtyRectCount--;
+        if (i != gDirtyRectCount)
+            gDirtyRects[i] = gDirtyRects[gDirtyRectCount];
+        i = 0;
+    }
+
     if (rect.X == 0 && rect.Y == 0 &&
         rect.W == (int32_t)mode->Width && rect.H == (int32_t)mode->Height) {
         KiDisplayDirtyAll();
@@ -478,6 +538,24 @@ static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, in
     KiDisplayAddDirtyForModeLocked(mode, x, y, w, h);
 }
 
+static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
+    if (!mode || w <= 0 || h <= 0)
+        return;
+
+    DISPLAY_RECT rect = { x, y, w, h };
+    if (!KiDisplayClipRect(mode, &rect))
+        return;
+
+    uint64_t bytes = (uint64_t)(uint32_t)rect.W *
+                     (uint64_t)(uint32_t)rect.H * sizeof(uint32_t);
+    gDisplayStats.BlitCount++;
+    gDisplayStats.PresentCount++;
+    gDisplayStats.PresentRectCount++;
+    gDisplayStats.DirectPresentCount++;
+    gDisplayStats.ScanoutBytes += bytes;
+    gDisplayStats.BytesCopied += bytes;
+}
+
 static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
     gDisplayStats.FillRectCount++;
     gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
@@ -660,7 +738,7 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
 
     uint32_t glyphPixels[FONT_W * FONT_H];
     KiDisplayBuildGlyphPixels(c, fg, bg, glyphPixels);
-    KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels);
+    KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels, FONT_W * sizeof(uint32_t));
     if (KSUCCESS(s)) {
         uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
         if (device == gPrimaryDisplay)
@@ -670,20 +748,26 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
     return s;
 }
 
-static KSTATUS KiDisplayDrawBlitSnapshot(DISPLAY_DEVICE* device,
-                                         const DISPLAY_DEVICE_OPS* ops,
-                                         const DISPLAY_MODE* mode,
-                                         int x,
-                                         int y,
-                                         int w,
-                                         int h,
-                                         const uint32_t* pixels) {
-    if (!pixels || w <= 0 || h <= 0)
+// Draws a pitched source rectangle and records it as one display operation.
+static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device,
+                                                const DISPLAY_DEVICE_OPS* ops,
+                                                const DISPLAY_MODE* mode,
+                                                int x,
+                                                int y,
+                                                int w,
+                                                int h,
+                                                const uint32_t* pixels,
+                                                uint32_t pitchBytes) {
+    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
         return NTLSTATUS_INVALID_PARAMETER;
-    if (!device || !ops || !ops->Blit || !mode)
+    if (!device || !ops || !mode)
         return NTLSTATUS_NOT_SUPPORTED;
 
-    KSTATUS s = ops->Blit(device, x, y, w, h, pixels);
+    if (!ops->Blit)
+        return NTLSTATUS_NOT_SUPPORTED;
+
+    KSTATUS s = ops->Blit(device, x, y, w, h, pixels, pitchBytes);
+
     if (KSUCCESS(s)) {
         uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
         if (device == gPrimaryDisplay)
@@ -867,7 +951,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3
     if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Blit)
         return NTLSTATUS_NOT_SUPPORTED;
 
-    KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels);
+    KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels, (uint32_t)w * sizeof(uint32_t));
     if (KSUCCESS(s)) {
         gDisplayStats.BlitCount++;
         gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
@@ -1043,11 +1127,18 @@ static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uin
 }
 
 // Drains buffered output through the active display provider.
-static void KiDisplayOutputWorker(void* context) {
-    (void)context;
+static KSTATUS KiDisplayDrainOutput(void) {
     DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT];
+    bool mutexHeld = false;
 
-    for (;;) {
+    if (gDisplayOutputMutex) {
+        KSTATUS acquireStatus = KeAcquireMutex(gDisplayOutputMutex, KE_WAIT_INFINITE);
+        if (!KE_MUTEX_ACQUIRE_SUCCEEDED(acquireStatus))
+            return acquireStatus;
+        mutexHeld = true;
+    }
+
+    while (true) {
         size_t count = 0;
         uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
         while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) {
@@ -1059,12 +1150,28 @@ static void KiDisplayOutputWorker(void* context) {
 
         if (count == 0)
             break;
-        for (size_t i = 0; i < count; i++) {
-            (void)KiDisplayPrintCharInternal(glyphs[i].Character,
-                                             glyphs[i].Foreground,
-                                             glyphs[i].Background);
+        KSTATUS drawStatus = KiDisplayPrintGlyphsInternal(glyphs, count);
+        if (!KSUCCESS(drawStatus)) {
+            if (mutexHeld)
+                KeReleaseMutex(gDisplayOutputMutex);
+            return drawStatus;
         }
     }
+
+    if (mutexHeld)
+        KeReleaseMutex(gDisplayOutputMutex);
+    return NTLSTATUS_SUCCESS;
+}
+
+// Drains buffered output from the passive work queue.
+static void KiDisplayOutputWorker(void* context) {
+    (void)context;
+    (void)KiDisplayDrainOutput();
+}
+
+// Completes buffered console output before a caller overlays console pixels.
+KSTATUS KeDisplayFlushConsoleOutput(void) {
+    return KiDisplayDrainOutput();
 }
 
 // Queues the runtime display output worker when passive work is available.
@@ -1157,6 +1264,8 @@ void KeDisplayInitialize(const HAL_API* hal) {
     _InterlockedExchange(&gDisplayClockInitialized, 1);
     if (!gDisplayProviderMutex)
         (void)KeMutexCreate(&gDisplayProviderMutex);
+    if (!gDisplayOutputMutex)
+        (void)KeMutexCreate(&gDisplayOutputMutex);
     KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
     _InterlockedExchange(&gDisplayCursorWorkInitialized, 1);
     KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0);
@@ -1212,8 +1321,9 @@ void KeDisplayInitialize(const HAL_API* hal) {
         (void)KeDisplaySetPrimary(&gBootFbDevice);
         DISPLAY_MODE mode;
         if (KSUCCESS(KeDisplayGetMode(&mode))) {
-            DbgPrint("Display: bootfb primary %ux%u stride=%u\n",
-                     mode.Width, mode.Height, mode.Stride);
+            DbgPrint("Display: bootfb primary %ux%u stride=%u cache=%s\n",
+                     mode.Width, mode.Height, mode.Stride,
+                     KeWriteCombiningAvailable() ? "WC" : "WT");
         }
 
         // The registration messages above were routed through the HAL boot
@@ -1277,7 +1387,7 @@ DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) {
 
 // Signals every display clock once for a provider-owned frame generation.
 static void KiDisplaySignalFrameGeneration(uint64_t generation) {
-    for (;;) {
+    while (true) {
         EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
         uint32_t count = 0;
 
@@ -1340,7 +1450,7 @@ DISPLAY_API void KeDisplayTick(void) {
         return;
 
     uint64_t now = KiDisplayClockReadSchedulerTick();
-    for (;;) {
+    while (true) {
         EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
         uint32_t count = 0;
 
@@ -1424,7 +1534,8 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE
         s = oldDevice->Ops->ReadRect(oldDevice, 0, (int)y, (int)newMode.Width, 1, row, newMode.Width);
         if (!KSUCCESS(s))
             break;
-        s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row);
+        s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row,
+                                  newMode.Width * sizeof(uint32_t));
         if (!KSUCCESS(s))
             break;
     }
@@ -1610,6 +1721,8 @@ DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) {
     for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next)
         count++;
     info.DeviceCount = count;
+    if (KeWriteCombiningAvailable())
+        info.Flags |= DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE;
     info.ManagerStats = gDisplayStats;
 
     DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
@@ -1660,7 +1773,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi
     s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
     if (KSUCCESS(s)) {
         (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
-        s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y, w, h, pixels);
+        s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels,
+                                               (uint32_t)w * sizeof(uint32_t));
         s = KiDisplayFinishProviderDraw(device, ops, caps, s);
     }
 
@@ -1668,8 +1782,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi
     return s;
 }
 
-DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch) {
-    if (!source || !scratch || w <= 0 || h <= 0)
+DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
+    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
         return NTLSTATUS_INVALID_PARAMETER;
 
     bool providerHeld = false;
@@ -1684,14 +1798,7 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW
     s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
     if (KSUCCESS(s)) {
         (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
-        for (int row = 0; row < h; ++row) {
-            s = source(context, row, scratch, (size_t)w);
-            if (!KSUCCESS(s))
-                break;
-            s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y + row, w, 1, scratch);
-            if (!KSUCCESS(s))
-                break;
-        }
+        s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, pitchBytes);
         s = KiDisplayFinishProviderDraw(device, ops, caps, s);
     }
 
@@ -1699,6 +1806,47 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW
     return s;
 }
 
+DISPLAY_API KSTATUS KeDisplayPresentPitched(int x,
+                                              int y,
+                                              int w,
+                                              int h,
+                                              const uint32_t* pixels,
+                                              uint32_t pitchBytes) {
+    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    bool providerHeld = false;
+    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
+    if (!KSUCCESS(s))
+        return s;
+
+    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
+    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
+    uint32_t caps = 0;
+    DISPLAY_MODE mode;
+    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
+    if (KSUCCESS(s)) {
+        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
+        if (ops->PresentPitched) {
+            s = ops->PresentPitched(device, x, y, w, h, pixels, pitchBytes);
+            if (KSUCCESS(s)) {
+                uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
+                if (device == gPrimaryDisplay)
+                    KiDisplayRecordDirectPresentLocked(&mode, x, y, w, h);
+                KeReleaseSpinlock(&gDisplayLock, flags);
+            }
+            KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps);
+        } else {
+            s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode,
+                                                 x, y, w, h, pixels, pitchBytes);
+            s = KiDisplayFinishProviderDraw(device, ops, caps, s);
+        }
+    }
+
+    KiDisplayReleaseProviderMutex(providerHeld);
+    return s;
+}
+
 DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) {
     bool providerHeld = false;
     KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
@@ -1835,18 +1983,28 @@ DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t
     return s;
 }
 
-static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
+// Renders one or more console glyphs as one provider transaction and one present.
+static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count) {
+    if (!glyphs && count != 0)
+        return NTLSTATUS_INVALID_PARAMETER;
+    if (count == 0)
+        return NTLSTATUS_SUCCESS;
+    if (count > DISPLAY_OUTPUT_DRAIN_COUNT)
+        return NTLSTATUS_INVALID_PARAMETER;
+
     if (KiDisplayOutputSuspended()) {
-        if (!KeCanWait())
-            return KiDisplayFallbackPrintChar(c, fg, bg);
+        if (!KeCanWait()) {
+            for (size_t i = 0; i < count; i++)
+                (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
+        }
         return NTLSTATUS_SUCCESS;
     }
 
-    // Once the scheduler/work-queue era begins, display provider callbacks are
-    // PASSIVE_LEVEL-only. The HAL output endpoint buffers normal text so this
-    // fallback is reserved for explicit display calls made at elevated IRQL.
-    if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait())
-        return KiDisplayFallbackPrintChar(c, fg, bg);
+    if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) {
+        for (size_t i = 0; i < count; i++)
+            (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
+        return NTLSTATUS_SUCCESS;
+    }
 
     bool providerHeld = false;
     KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
@@ -1861,8 +2019,10 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
     if (!KSUCCESS(s)) {
         KiDisplayReleaseProviderMutex(providerHeld);
         const HAL_API* hal = gDisplayHal;
-        if (hal && hal->Screen.PrintChar)
-            hal->Screen.PrintChar(c, fg, bg);
+        if (hal && hal->Screen.PrintChar) {
+            for (size_t i = 0; i < count; i++)
+                hal->Screen.PrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
+        }
         return s;
     }
 
@@ -1875,71 +2035,94 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
 
     (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
 
-    bool drawChar = false;
-    bool needScroll = false;
-    int drawCol = 0;
-    int drawRow = 0;
+    DISPLAY_OUTPUT_DRAW draws[DISPLAY_OUTPUT_DRAIN_COUNT];
+    size_t drawCount = 0;
+    size_t scrollCount = 0;
 
-    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
-    if (c == '\n') {
-        gConsoleX = 0;
-        gConsoleY++;
-    } else if (c == '\r') {
-        gConsoleX = 0;
-    } else if (c == '\b') {
-        if (gConsoleX > 0) {
-            gConsoleX--;
-        } else if (gConsoleY > 0) {
-            gConsoleY--;
-            gConsoleX = cols - 1;
-        }
-    } else {
-        drawChar = true;
-        drawCol = gConsoleX;
-        drawRow = gConsoleY;
-        if (++gConsoleX >= cols) {
+    for (size_t glyphIndex = 0; glyphIndex < count; glyphIndex++) {
+        char c = glyphs[glyphIndex].Character;
+        int drawCol = 0;
+        int drawRow = 0;
+        bool drawChar = false;
+        bool needScroll = false;
+
+        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
+        if (c == '\n') {
             gConsoleX = 0;
             gConsoleY++;
+        } else if (c == '\r') {
+            gConsoleX = 0;
+        } else if (c == '\b') {
+            if (gConsoleX > 0) {
+                gConsoleX--;
+            } else if (gConsoleY > 0) {
+                gConsoleY--;
+                gConsoleX = cols - 1;
+            }
+        } else {
+            drawChar = true;
+            drawCol = gConsoleX;
+            drawRow = gConsoleY;
+            if (++gConsoleX >= cols) {
+                gConsoleX = 0;
+                gConsoleY++;
+            }
+
+            if (drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) {
+                size_t cell = KiDisplayConsoleIndex(drawCol, drawRow);
+                gConsoleCells[cell] = c;
+                gConsoleAttrs[cell] = (uint8_t)gConsoleAttribute;
+            }
+        }
+
+        if (gConsoleY >= rows) {
+            KiDisplayConsoleScrollCellsLocked(cols, rows, 1);
+            gConsoleY = rows - 1;
+            if (gConsoleX < 0) gConsoleX = 0;
+            if (gConsoleX >= cols) gConsoleX = cols - 1;
+            needScroll = true;
+        }
+        KeReleaseSpinlock(&gDisplayLock, flags);
+
+        if (drawChar) {
+            DISPLAY_OUTPUT_DRAW* draw = &draws[drawCount++];
+            draw->Character = c;
+            draw->Foreground = glyphs[glyphIndex].Foreground;
+            draw->Background = glyphs[glyphIndex].Background;
+            draw->Column = drawCol;
+            draw->Row = drawRow;
+            draw->ScrollCount = scrollCount;
+        }
+        if (needScroll)
+            scrollCount++;
+    }
+
+    if (scrollCount != 0) {
+        uint64_t scrollPixels64 = (uint64_t)scrollCount * FONT_H;
+        int scrollPixels = scrollPixels64 >= mode.Height ? (int)mode.Height : (int)scrollPixels64;
+        if ((uint32_t)scrollPixels >= mode.Height) {
+            s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0,
+                                          (int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG);
+        } else {
+            s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, scrollPixels,
+                                          (int)mode.Width, (int)(mode.Height - (uint32_t)scrollPixels));
+            if (KSUCCESS(s)) {
+                s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
+                                              (int)(mode.Height - (uint32_t)scrollPixels),
+                                              (int)mode.Width, scrollPixels, DISPLAY_DEFAULT_BG);
+            }
         }
     }
 
-    if (gConsoleY >= rows)
-        needScroll = true;
-    KeReleaseSpinlock(&gDisplayLock, flags);
-
-    if (drawChar) {
-        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, drawCol * FONT_W, drawRow * FONT_H, fg, bg);
-        if (KSUCCESS(s)) {
-            flags = KeAcquireSpinlock(&gDisplayLock);
-            if (device == gPrimaryDisplay && drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) {
-                size_t i = KiDisplayConsoleIndex(drawCol, drawRow);
-                gConsoleCells[i] = c;
-                gConsoleAttrs[i] = (uint8_t)gConsoleAttribute;
-            }
-            KeReleaseSpinlock(&gDisplayLock, flags);
-        }
-    } else {
-        s = NTLSTATUS_SUCCESS;
-    }
-
-    if (KSUCCESS(s) && needScroll) {
-        s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, FONT_H,
-                                      (int)mode.Width, (int)(mode.Height - FONT_H));
-        if (KSUCCESS(s)) {
-            s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
-                                          (int)(mode.Height - FONT_H),
-                                          (int)mode.Width, FONT_H, DISPLAY_DEFAULT_BG);
-        }
-        if (KSUCCESS(s)) {
-            flags = KeAcquireSpinlock(&gDisplayLock);
-            if (device == gPrimaryDisplay) {
-                KiDisplayConsoleScrollCellsLocked(cols, rows, 1);
-                gConsoleY = rows - 1;
-                if (gConsoleX < 0) gConsoleX = 0;
-                if (gConsoleX >= cols) gConsoleX = cols - 1;
-            }
-            KeReleaseSpinlock(&gDisplayLock, flags);
-        }
+    for (size_t i = 0; i < drawCount && KSUCCESS(s); i++) {
+        DISPLAY_OUTPUT_DRAW* draw = &draws[i];
+        size_t laterScrolls = scrollCount - draw->ScrollCount;
+        int finalRow = draw->Row - (int)laterScrolls;
+        if (finalRow < 0 || finalRow >= rows)
+            continue;
+        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, draw->Character,
+                                       draw->Column * FONT_W, finalRow * FONT_H,
+                                       draw->Foreground, draw->Background);
     }
 
     s = KiDisplayFinishProviderDraw(device, ops, caps, s);
@@ -1947,6 +2130,12 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
     return s;
 }
 
+// Renders one console character through the shared batched console path.
+static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
+    DISPLAY_OUTPUT_GLYPH glyph = { c, fg, bg };
+    return KiDisplayPrintGlyphsInternal(&glyph, 1);
+}
+
 DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
     return KiDisplayPrintCharInternal(c, fg, bg);
 }
@@ -2285,10 +2474,114 @@ static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
     return NTLSTATUS_SUCCESS;
 }
 
-// Implements the boot Fb Blit operation.
-static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
+// Copies one pixel row into the write-combined scanout aperture.
+static void BootFbCopyToScanout(uint32_t* destination,
+                                const uint32_t* source,
+                                size_t pixelCount) {
+    if (!destination || !source || pixelCount == 0)
+        return;
+
+    if (!KeWriteCombiningAvailable()) {
+        memcpy(destination, source, pixelCount * sizeof(uint32_t));
+        return;
+    }
+
+    while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) {
+        *destination++ = *source++;
+        pixelCount--;
+    }
+
+    while (pixelCount >= 16u) {
+        __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
+        __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
+        __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
+        __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
+        _mm_stream_si128((__m128i*)(destination + 0), v0);
+        _mm_stream_si128((__m128i*)(destination + 4), v1);
+        _mm_stream_si128((__m128i*)(destination + 8), v2);
+        _mm_stream_si128((__m128i*)(destination + 12), v3);
+        source += 16;
+        destination += 16;
+        pixelCount -= 16;
+    }
+
+    while (pixelCount >= 4u) {
+        __m128i value = _mm_loadu_si128((const __m128i*)source);
+        _mm_stream_si128((__m128i*)destination, value);
+        source += 4;
+        destination += 4;
+        pixelCount -= 4;
+    }
+
+    while (pixelCount != 0) {
+        *destination++ = *source++;
+        pixelCount--;
+    }
+}
+
+// Copies one source row into the retained shadow and scanout in one read pass.
+static void BootFbCopyToShadowAndScanout(uint32_t* shadow,
+                                         uint32_t* scanout,
+                                         const uint32_t* source,
+                                         size_t pixelCount) {
+    if (!shadow || !scanout || !source || pixelCount == 0)
+        return;
+
+    if (!KeWriteCombiningAvailable()) {
+        memcpy(shadow, source, pixelCount * sizeof(uint32_t));
+        memcpy(scanout, source, pixelCount * sizeof(uint32_t));
+        return;
+    }
+
+    while (pixelCount != 0 && ((uintptr_t)scanout & 15u) != 0) {
+        uint32_t value = *source++;
+        *shadow++ = value;
+        *scanout++ = value;
+        pixelCount--;
+    }
+
+    while (pixelCount >= 16u) {
+        __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
+        __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
+        __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
+        __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
+        _mm_storeu_si128((__m128i*)(shadow + 0), v0);
+        _mm_storeu_si128((__m128i*)(shadow + 4), v1);
+        _mm_storeu_si128((__m128i*)(shadow + 8), v2);
+        _mm_storeu_si128((__m128i*)(shadow + 12), v3);
+        _mm_stream_si128((__m128i*)(scanout + 0), v0);
+        _mm_stream_si128((__m128i*)(scanout + 4), v1);
+        _mm_stream_si128((__m128i*)(scanout + 8), v2);
+        _mm_stream_si128((__m128i*)(scanout + 12), v3);
+        source += 16;
+        shadow += 16;
+        scanout += 16;
+        pixelCount -= 16;
+    }
+
+    while (pixelCount >= 4u) {
+        __m128i value = _mm_loadu_si128((const __m128i*)source);
+        _mm_storeu_si128((__m128i*)shadow, value);
+        _mm_stream_si128((__m128i*)scanout, value);
+        source += 4;
+        shadow += 4;
+        scanout += 4;
+        pixelCount -= 4;
+    }
+
+    while (pixelCount != 0) {
+        uint32_t value = *source++;
+        *shadow++ = value;
+        *scanout++ = value;
+        pixelCount--;
+    }
+}
+
+// Implements the boot framebuffer pitched blit operation.
+static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
     BOOTFB_CONTEXT* fb = BootFbContext(device);
-    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0)
+    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
+        pitchBytes < (uint32_t)w * sizeof(uint32_t))
         return NTLSTATUS_INVALID_PARAMETER;
 
     uint32_t* dstBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer;
@@ -2309,12 +2602,61 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co
 
     for (int row = 0; row < endY - startY; row++) {
         uint32_t* dst = dstBase + ((size_t)(startY + row) * fb->Stride) + startX;
-        const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX;
+        const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
+        const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
         memcpy(dst, src, bytesPerRow);
     }
     return NTLSTATUS_SUCCESS;
 }
 
+// Presents one pitched source rectangle without a shadow-to-scanout second pass.
+static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
+                                    int x,
+                                    int y,
+                                    int w,
+                                    int h,
+                                    const uint32_t* pixels,
+                                    uint32_t pitchBytes) {
+    BOOTFB_CONTEXT* fb = BootFbContext(device);
+    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
+        pitchBytes < (uint32_t)w * sizeof(uint32_t))
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    int startX = x < 0 ? 0 : x;
+    int startY = y < 0 ? 0 : y;
+    int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w;
+    int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h;
+    if (startX >= endX || startY >= endY)
+        return NTLSTATUS_SUCCESS;
+
+    int srcOffsetX = startX - x;
+    int srcOffsetY = startY - y;
+    size_t pixelCount = (size_t)(endX - startX);
+    uint64_t bytes = (uint64_t)pixelCount * (uint64_t)(endY - startY) * sizeof(uint32_t);
+
+    fb->Stats.BlitCount++;
+    fb->Stats.PresentCount++;
+    fb->Stats.PresentRectCount++;
+    fb->Stats.DirectPresentCount++;
+    fb->Stats.ScanoutBytes += bytes;
+    fb->Stats.BytesCopied += fb->Backbuffer ? bytes * 2u : bytes;
+
+    for (int row = 0; row < endY - startY; row++) {
+        const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
+        const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
+        uint32_t* scanout = fb->Framebuffer + ((size_t)(startY + row) * fb->Stride) + startX;
+        if (fb->Backbuffer) {
+            uint32_t* shadow = fb->Backbuffer + ((size_t)(startY + row) * fb->Stride) + startX;
+            BootFbCopyToShadowAndScanout(shadow, scanout, source, pixelCount);
+        } else {
+            BootFbCopyToScanout(scanout, source, pixelCount);
+        }
+    }
+
+    _mm_sfence();
+    return NTLSTATUS_SUCCESS;
+}
+
 // Implements the boot Fb Fill Rect operation.
 static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
     BOOTFB_CONTEXT* fb = BootFbContext(device);
@@ -2425,14 +2767,18 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects,
         DISPLAY_RECT r = rects[i];
         if (!KiDisplayClipRect(&mode, &r))
             continue;
-        fb->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
-        size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t);
+        uint64_t bytes = (uint64_t)(uint32_t)r.W *
+                         (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
+        fb->Stats.BytesCopied += bytes;
+        fb->Stats.ScanoutBytes += bytes;
+        size_t pixelCount = (size_t)r.W;
         for (int row = 0; row < r.H; row++) {
             uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
             uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
-            memcpy(dst, src, bytesPerRow);
+            BootFbCopyToScanout(dst, src, pixelCount);
         }
     }
+    _mm_sfence();
     return NTLSTATUS_SUCCESS;
 }
 
@@ -2540,6 +2886,7 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
         fb->Stats.CursorHideCount++;
         BootFbRestoreCursor(fb);
     }
+    _mm_sfence();
     return NTLSTATUS_SUCCESS;
 }