nt-lite/code/kernel/ke_display.c

// Kernel display manager. This file owns provider registration, framebuffer routing, dirty rectangles, text console state, and cursor delegation.

#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>

#include <kernel/ke_display.h>
#include <kernel/ke_debug.h>
#include <kernel/hal/font8x16.h>
#include <kernel/display_cursor.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_work.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_event.h>
#include <kernel/ke_list.h>
#include <kernel/ob_object.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_cpu.h>

#define DISPLAY_DEFAULT_FG 0x00CCCCCCu
#define DISPLAY_DEFAULT_BG 0x00012456u

#define DISPLAY_MAX_DIRTY_RECTS 64u
#define DISPLAY_CURSOR_W DISPLAY_DEFAULT_CURSOR_W
#define DISPLAY_CURSOR_H DISPLAY_DEFAULT_CURSOR_H
#define FONT_W 8
#define FONT_H 16
#define DISPLAY_CONSOLE_MAX_COLS 256
#define DISPLAY_CONSOLE_MAX_ROWS 128
#define DISPLAY_CONSOLE_MAX_CELLS (DISPLAY_CONSOLE_MAX_COLS * DISPLAY_CONSOLE_MAX_ROWS)
#define DISPLAY_CONSOLE_DEFAULT_ATTR 0x07u
#define DISPLAY_CURSOR_POS_INVALID (-2147483647 - 1)
#define DISPLAY_OUTPUT_BUFFER_CAPACITY 4096u
#define DISPLAY_OUTPUT_DRAIN_COUNT     128u
#define DISPLAY_CLOCK_DEFAULT_PERIOD_MS 1ull
#define DISPLAY_CLOCK_SIGNAL_BATCH      32u

typedef struct DISPLAY_OUTPUT_GLYPH {
    char Character;
    uint32_t Foreground;
    uint32_t Background;
} DISPLAY_OUTPUT_GLYPH;

typedef struct DISPLAY_OUTPUT_DRAW {
    char Character;
    uint32_t Foreground;
    uint32_t Background;
    int Column;
    int Row;
    size_t ScrollCount;
} DISPLAY_OUTPUT_DRAW;

typedef struct BOOTFB_CONTEXT {
    const HAL_API* Hal;
    uint32_t* Framebuffer;
    uint32_t* Backbuffer;
    uint32_t Width;
    uint32_t Height;
    uint32_t Stride;
    DISPLAY_STATS Stats;
} BOOTFB_CONTEXT;

static const HAL_API* gDisplayHal;
static KE_DISPLAY_MANAGER gDisplayManager;
static DISPLAY_DEVICE* gDisplayList;
static DISPLAY_DEVICE* gPrimaryDisplay;
static SPINLOCK gDisplayLock = SPINLOCK_INIT;
static MUTEX* gDisplayProviderMutex;
static MUTEX* gDisplayOutputMutex;
static KWORK_ITEM gDisplayCursorWorkItem;
static volatile long gDisplayCursorWorkInitialized;
static bool gDisplayTransitionInProgress;
static uint64_t gDisplayGeneration;
static LIST_ENTRY gDisplayClockList;
static SPINLOCK gDisplayClockLock = SPINLOCK_INIT;
static volatile long gDisplayClockInitialized;
static uint64_t gDisplayFrameGeneration;

static DISPLAY_DEVICE gBootFbDevice;
static BOOTFB_CONTEXT gBootFb;

static DISPLAY_RECT gDirtyRects[DISPLAY_MAX_DIRTY_RECTS];
static size_t gDirtyRectCount;
static bool gDirtyAll;
static uint32_t gDrawDepth;
static DISPLAY_STATS gDisplayStats;

static bool gCursorVisible;
static bool gCursorDrawn;
static int  gCursorX;
static int  gCursorY;
static int  gCursorDrawX;
static int  gCursorDrawY;
static bool gCursorUpdatePending;
static uint64_t gCursorRequestGeneration;
static uint64_t gCursorAppliedGeneration;
static int gCursorCommittedX;
static int gCursorCommittedY;

static int gConsoleX;
static int gConsoleY;
static uint16_t gConsoleAttribute;
static char gConsoleCells[DISPLAY_CONSOLE_MAX_CELLS];
static uint8_t gConsoleAttrs[DISPLAY_CONSOLE_MAX_CELLS];
static volatile long gDisplayConsoleSuspended;
static SPINLOCK gDisplayOutputLock = SPINLOCK_INIT;
static DISPLAY_OUTPUT_GLYPH* gDisplayOutputBuffer;
static size_t gDisplayOutputRead;
static size_t gDisplayOutputWrite;
static size_t gDisplayOutputUsed;
static KWORK_ITEM gDisplayOutputWorkItem;
static volatile long gDisplayOutputWorkInitialized;

static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg);
static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count);
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color);
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h);
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride);
static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount);
static KSTATUS BootFbSync(DISPLAY_DEVICE* device);
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels);
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y);
static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show);
static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats);

static const DISPLAY_DEVICE_OPS gBootFbOps = {
    .GetMode = BootFbGetMode,
    .Blit = BootFbBlit,
    .PresentPitched = BootFbPresentPitched,
    .FillRect = BootFbFillRect,
    .CopyRect = BootFbCopyRect,
    .ReadRect = BootFbReadRect,
    .Present = BootFbPresent,
    .Sync = BootFbSync,
    .SetCursorImage = BootFbSetCursorImage,
    .SetCursorPosition = BootFbSetCursorPosition,
    .ShowCursor = BootFbShowCursor,
    .QueryStats = BootFbQueryStats,
};

static bool KiDisplayOutputSuspended(void);
static void KiDisplayCursorWorker(void* context);
static void KiDisplayDirtyAll(void);

// Converts a display-clock period to scheduler ticks with minimum one-tick progress.
static uint64_t KiDisplayClockMillisecondsToTicks(uint64_t milliseconds) {
    if (milliseconds == 0)
        milliseconds = DISPLAY_CLOCK_DEFAULT_PERIOD_MS;

    uint64_t ticks = milliseconds / SCHED_TICK_MS;
    if ((milliseconds % SCHED_TICK_MS) != 0)
        ticks++;
    return ticks ? ticks : 1;
}

// Reads the global scheduler tick used by display-clock deadlines.
static uint64_t KiDisplayClockReadSchedulerTick(void) {
    KE_SCHED_STATS stats;
    KeGetSchedulerStats(&stats);
    return stats.Ticks;
}

// Destroys a display clock and removes it from the manager tick list.
static void KiDisplayClockDestroy(OBJECT* object) {
    DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)object;
    if (!clock)
        return;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
    if (clock->Linked) {
        ListRemove(&clock->Link);
        clock->Linked = 0;
    }
    KeReleaseSpinlock(&gDisplayClockLock, flags);

    if (clock->Event) {
        ObDereferenceObject(&clock->Event->Header);
        clock->Event = (EVENT*)0;
    }
}

static bool KiDisplayHasCurrentThread(void) {
    return KeGetCurrentThread() != (THREAD*)0;
}

static bool KiDisplayProviderCallbacksRequirePassive(void) {
    return KiDisplayHasCurrentThread() && KeIsWorkQueueStarted();
}

static KSTATUS KiDisplayAcquireProviderMutex(bool* acquired) {
    if (acquired)
        *acquired = false;

    if (!gDisplayProviderMutex || !KiDisplayProviderCallbacksRequirePassive())
        return NTLSTATUS_SUCCESS;

    if (!KeCanWait())
        return NTLSTATUS_INVALID_IRQL;

    if (KeGetDebugFlags() & KE_DEBUG_DISPLAY_ASSERTS)
        KeAssertIrql(PASSIVE_LEVEL);

    KSTATUS s = KeAcquireMutex(gDisplayProviderMutex, KE_WAIT_INFINITE);
    if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s))
        return s;

    if (acquired)
        *acquired = true;

    // Provider operations still own the mutex after an abandoned acquisition.
    return NTLSTATUS_SUCCESS;
}

static void KiDisplayReleaseProviderMutex(bool acquired) {
    if (acquired && gDisplayProviderMutex)
        KeReleaseMutex(gDisplayProviderMutex);
}

// Writes one character through the boot HAL framebuffer path when the display
// manager cannot safely call provider callbacks, such as during a panic.
static KSTATUS KiDisplayFallbackPrintChar(char c, uint32_t fg, uint32_t bg) {
    const HAL_API* hal = gDisplayHal;
    if (hal && hal->Screen.PrintChar) {
        hal->Screen.PrintChar(c, fg, bg);
        return NTLSTATUS_SUCCESS;
    }
    return NTLSTATUS_INVALID_IRQL;
}

static void KiDisplayQueueCursorWork(void) {
    if (!_InterlockedCompareExchange(&gDisplayCursorWorkInitialized, 1, 1))
        return;
    if (KeIsWorkQueueStarted()) {
        (void)KeQueueWorkItem(&gDisplayCursorWorkItem);
        return;
    }

    // During single-threaded early boot there is no worker thread yet. The
    // boot framebuffer console still needs synchronous cursor setup; once the
    // scheduler work queue exists all cursor commits are asynchronous.
    if (!KiDisplayProviderCallbacksRequirePassive())
        KiDisplayCursorWorker(0);
}

// Checks whether a primary display provider is available.
static bool KiDisplayHasDevice(void) {
    return gPrimaryDisplay && gPrimaryDisplay->Ops;
}

// Checks whether the active provider expects the manager to draw the cursor into the framebuffer.
static bool KiDisplayUsesSoftwareCursor(void) {
    return KiDisplayHasDevice() &&
           (gPrimaryDisplay->Caps & DISPLAY_CAP_SOFTWARE_CURSOR) &&
           gPrimaryDisplay->Ops->ShowCursor;
}

// Checks whether the active provider owns cursor presentation in hardware or device firmware.
static bool KiDisplayUsesHardwareCursor(void) {
    return KiDisplayHasDevice() &&
           (gPrimaryDisplay->Caps & DISPLAY_CAP_HARDWARE_CURSOR) &&
           gPrimaryDisplay->Ops->ShowCursor;
}

// Checks whether the active provider exposes any cursor-control path.
static bool KiDisplayHasCursorOps(void) {
    return KiDisplayHasDevice() &&
           (gPrimaryDisplay->Caps & (DISPLAY_CAP_SOFTWARE_CURSOR | DISPLAY_CAP_HARDWARE_CURSOR)) &&
           gPrimaryDisplay->Ops->ShowCursor;
}

static void KiDisplayCursorWorker(void* context) {
    (void)context;

    bool providerHeld = false;
    if (!KSUCCESS(KiDisplayAcquireProviderMutex(&providerHeld)))
        return;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    int x = 0;
    int y = 0;
    bool visible = false;
    bool drawn = false;
    bool suspended = false;
    uint64_t displayGeneration = 0;
    uint64_t requestGeneration = 0;
    int committedX = 0;
    int committedY = 0;
    bool hadPendingWork = false;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    hadPendingWork = gCursorUpdatePending || (gCursorRequestGeneration != gCursorAppliedGeneration);
    if (!hadPendingWork) {
        KeReleaseSpinlock(&gDisplayLock, flags);
        KiDisplayReleaseProviderMutex(providerHeld);
        return;
    }

    gCursorUpdatePending = false;
    if (KiDisplayHasDevice()) {
        device = gPrimaryDisplay;
        ops = gPrimaryDisplay->Ops;
        caps = gPrimaryDisplay->Caps;
        x = gCursorX;
        y = gCursorY;
        visible = gCursorVisible;
        drawn = gCursorDrawn;
        suspended = KiDisplayOutputSuspended() || gDisplayTransitionInProgress;
        displayGeneration = gDisplayGeneration;
        requestGeneration = gCursorRequestGeneration;
        committedX = gCursorCommittedX;
        committedY = gCursorCommittedY;
    }
    KeReleaseSpinlock(&gDisplayLock, flags);

    bool newDrawn = false;
    bool positionCommitted = false;
    int newCommittedX = committedX;
    int newCommittedY = committedY;
    KSTATUS status = NTLSTATUS_SUCCESS;
    bool hasCursorOps = device && ops && ops->ShowCursor &&
        (caps & (DISPLAY_CAP_SOFTWARE_CURSOR | DISPLAY_CAP_HARDWARE_CURSOR));
    bool softwareCursor = hasCursorOps && (caps & DISPLAY_CAP_SOFTWARE_CURSOR) != 0;
    bool hardwareCursor = hasCursorOps && (caps & DISPLAY_CAP_HARDWARE_CURSOR) != 0;

    if (hasCursorOps) {
        if (!visible || suspended) {
            if (drawn)
                status = ops->ShowCursor(device, false);
            newDrawn = false;
            if (KSUCCESS(status)) {
                positionCommitted = true;
                newCommittedX = DISPLAY_CURSOR_POS_INVALID;
                newCommittedY = DISPLAY_CURSOR_POS_INVALID;
            }
        } else if (hardwareCursor) {
            // Hardware cursors are independent of framebuffer drawing. Commit
            // position only when the logical cursor request changed or the
            // provider has not yet been synchronized. This prevents stale or
            // duplicate work items from hammering device cursor registers.
            bool needPosition = !drawn || x != committedX || y != committedY;
            if (needPosition && ops->SetCursorPosition) {
                status = ops->SetCursorPosition(device, x, y);
                if (KSUCCESS(status)) {
                    positionCommitted = true;
                    newCommittedX = x;
                    newCommittedY = y;
                }
            } else {
                positionCommitted = true;
                newCommittedX = committedX;
                newCommittedY = committedY;
            }

            if (KSUCCESS(status)) {
                if (!drawn)
                    status = ops->ShowCursor(device, true);
                newDrawn = KSUCCESS(status);
            }
        } else if (softwareCursor) {
            if (drawn)
                (void)ops->ShowCursor(device, false);
            if (ops->SetCursorPosition)
                (void)ops->SetCursorPosition(device, x, y);
            status = ops->ShowCursor(device, true);
            newDrawn = KSUCCESS(status);
            if (newDrawn) {
                positionCommitted = true;
                newCommittedX = x;
                newCommittedY = y;
            }
        }
    }

    flags = KeAcquireSpinlock(&gDisplayLock);
    if (displayGeneration == gDisplayGeneration && device == gPrimaryDisplay) {
        gCursorDrawn = newDrawn;
        if (positionCommitted) {
            gCursorCommittedX = newCommittedX;
            gCursorCommittedY = newCommittedY;
        }
        // Do not spin forever on provider failure. A later explicit cursor
        // request, show/hide request, or display-generation change will retry.
        gCursorAppliedGeneration = requestGeneration;
        if (gCursorRequestGeneration != requestGeneration)
            gCursorUpdatePending = true;
    } else {
        gCursorUpdatePending = true;
    }
    bool requeue = gCursorUpdatePending && !gDisplayTransitionInProgress;
    KeReleaseSpinlock(&gDisplayLock, flags);

    KiDisplayReleaseProviderMutex(providerHeld);

    if (requeue)
        KiDisplayQueueCursorWork();
}

// Clips a rectangle to a display mode so providers never see negative or out-of-bounds regions.
static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) {
    if (!mode || !rect || rect->W <= 0 || rect->H <= 0)
        return false;

    int64_t x0 = rect->X;
    int64_t y0 = rect->Y;
    int64_t x1 = x0 + rect->W;
    int64_t y1 = y0 + rect->H;

    if (x0 < 0) x0 = 0;
    if (y0 < 0) y0 = 0;
    if (x1 > (int64_t)mode->Width) x1 = mode->Width;
    if (y1 > (int64_t)mode->Height) y1 = mode->Height;
    if (x0 >= x1 || y0 >= y1)
        return false;

    rect->X = (int32_t)x0;
    rect->Y = (int32_t)y0;
    rect->W = (int32_t)(x1 - x0);
    rect->H = (int32_t)(y1 - y0);
    return true;
}

// Snapshots the current primary under the manager spinlock. The caller must
// hold the provider mutex before invoking callbacks through the returned ops;
// KeDisplaySetPrimary takes the same mutex, so the provider object cannot be
// switched while the callback sequence is in progress.
static bool KiDisplaySnapshotPrimary(DISPLAY_DEVICE** outDevice,
                                     const DISPLAY_DEVICE_OPS** outOps,
                                     uint32_t* outCaps,
                                     uint64_t* outGeneration) {
    if (outDevice) *outDevice = (DISPLAY_DEVICE*)0;
    if (outOps) *outOps = (const DISPLAY_DEVICE_OPS*)0;
    if (outCaps) *outCaps = 0;
    if (outGeneration) *outGeneration = 0;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
    if (device) {
        if (outDevice) *outDevice = device;
        if (outOps) *outOps = device->Ops;
        if (outCaps) *outCaps = device->Caps;
        if (outGeneration) *outGeneration = gDisplayGeneration;
    }
    KeReleaseSpinlock(&gDisplayLock, flags);
    return device != (DISPLAY_DEVICE*)0;
}

static bool KiDisplayConsoleGridFromMode(const DISPLAY_MODE* mode, int* outCols, int* outRows) {
    if (!mode)
        return false;

    int cols = (int)mode->Width / FONT_W;
    int rows = (int)mode->Height / FONT_H;
    if (cols > DISPLAY_CONSOLE_MAX_COLS) cols = DISPLAY_CONSOLE_MAX_COLS;
    if (rows > DISPLAY_CONSOLE_MAX_ROWS) rows = DISPLAY_CONSOLE_MAX_ROWS;
    if (cols <= 0 || rows <= 0)
        return false;

    if (outCols) *outCols = cols;
    if (outRows) *outRows = rows;
    return true;
}

// Returns whether two dirty rectangles overlap or share an edge.
static bool KiDisplayDirtyRectsMergeable(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
    if (!a || !b)
        return false;

    int64_t aRight = (int64_t)a->X + a->W;
    int64_t aBottom = (int64_t)a->Y + a->H;
    int64_t bRight = (int64_t)b->X + b->W;
    int64_t bBottom = (int64_t)b->Y + b->H;
    return (int64_t)a->X <= bRight && (int64_t)b->X <= aRight &&
           (int64_t)a->Y <= bBottom && (int64_t)b->Y <= aBottom;
}

// Unions two clipped dirty rectangles.
static DISPLAY_RECT KiDisplayUnionDirtyRects(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
    int64_t left = a->X < b->X ? a->X : b->X;
    int64_t top = a->Y < b->Y ? a->Y : b->Y;
    int64_t aRight = (int64_t)a->X + a->W;
    int64_t bRight = (int64_t)b->X + b->W;
    int64_t aBottom = (int64_t)a->Y + a->H;
    int64_t bBottom = (int64_t)b->Y + b->H;
    int64_t right = aRight > bRight ? aRight : bRight;
    int64_t bottom = aBottom > bBottom ? aBottom : bBottom;
    DISPLAY_RECT result = { (int32_t)left, (int32_t)top,
                            (int32_t)(right - left), (int32_t)(bottom - top) };
    return result;
}

static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
    if (!mode || gDirtyAll || w <= 0 || h <= 0)
        return;

    DISPLAY_RECT rect = { x, y, w, h };
    if (!KiDisplayClipRect(mode, &rect))
        return;

    for (size_t i = 0; i < gDirtyRectCount;) {
        if (!KiDisplayDirtyRectsMergeable(&rect, &gDirtyRects[i])) {
            i++;
            continue;
        }

        rect = KiDisplayUnionDirtyRects(&rect, &gDirtyRects[i]);
        gDirtyRectCount--;
        if (i != gDirtyRectCount)
            gDirtyRects[i] = gDirtyRects[gDirtyRectCount];
        i = 0;
    }

    if (rect.X == 0 && rect.Y == 0 &&
        rect.W == (int32_t)mode->Width && rect.H == (int32_t)mode->Height) {
        KiDisplayDirtyAll();
        return;
    }

    if (gDirtyRectCount >= DISPLAY_MAX_DIRTY_RECTS) {
        KiDisplayDirtyAll();
        return;
    }

    gDisplayStats.DirtyRectCount++;
    gDirtyRects[gDirtyRectCount++] = rect;
}

static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
    gDisplayStats.BlitCount++;
    gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
    KiDisplayAddDirtyForModeLocked(mode, x, y, w, h);
}

static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
    if (!mode || w <= 0 || h <= 0)
        return;

    DISPLAY_RECT rect = { x, y, w, h };
    if (!KiDisplayClipRect(mode, &rect))
        return;

    uint64_t bytes = (uint64_t)(uint32_t)rect.W *
                     (uint64_t)(uint32_t)rect.H * sizeof(uint32_t);
    gDisplayStats.BlitCount++;
    gDisplayStats.PresentCount++;
    gDisplayStats.PresentRectCount++;
    gDisplayStats.DirectPresentCount++;
    gDisplayStats.ScanoutBytes += bytes;
    gDisplayStats.BytesCopied += bytes;
}

static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
    gDisplayStats.FillRectCount++;
    gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
    KiDisplayAddDirtyForModeLocked(mode, x, y, w, h);
}

static void KiDisplayRecordCopyLocked(const DISPLAY_MODE* mode, int dstX, int dstY, int w, int h) {
    gDisplayStats.CopyRectCount++;
    gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
    KiDisplayAddDirtyForModeLocked(mode, dstX, dstY, w, h);
}

static KSTATUS KiDisplayFlushDirtyWithSnapshot(DISPLAY_DEVICE* device, const DISPLAY_DEVICE_OPS* ops) {
    if (!device || !ops || !ops->Present)
        return NTLSTATUS_SUCCESS;

    DISPLAY_RECT localRects[DISPLAY_MAX_DIRTY_RECTS];
    DISPLAY_RECT fullRect;
    const DISPLAY_RECT* rects = localRects;
    size_t count = 0;
    bool dirtyAll = false;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    if (device != gPrimaryDisplay || (!gDirtyAll && gDirtyRectCount == 0)) {
        KeReleaseSpinlock(&gDisplayLock, flags);
        return NTLSTATUS_SUCCESS;
    }

    dirtyAll = gDirtyAll;
    if (!dirtyAll) {
        count = gDirtyRectCount;
        if (count > DISPLAY_MAX_DIRTY_RECTS)
            count = DISPLAY_MAX_DIRTY_RECTS;
        for (size_t i = 0; i < count; i++)
            localRects[i] = gDirtyRects[i];
    }
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (dirtyAll) {
        DISPLAY_MODE mode;
        if (!ops->GetMode)
            return NTLSTATUS_NOT_SUPPORTED;
        KSTATUS ms = ops->GetMode(device, &mode);
        if (!KSUCCESS(ms))
            return ms;
        fullRect.X = 0;
        fullRect.Y = 0;
        fullRect.W = (int32_t)mode.Width;
        fullRect.H = (int32_t)mode.Height;
        rects = &fullRect;
        count = 1;
    }

    flags = KeAcquireSpinlock(&gDisplayLock);
    if (device == gPrimaryDisplay) {
        gDisplayStats.PresentCount++;
        gDisplayStats.PresentRectCount += count;
    }
    KeReleaseSpinlock(&gDisplayLock, flags);

    KSTATUS s = ops->Present(device, rects, count);
    if (KSUCCESS(s)) {
        flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay) {
            gDirtyAll = false;
            gDirtyRectCount = 0;
        }
        KeReleaseSpinlock(&gDisplayLock, flags);
    }
    return s;
}

static bool KiDisplayHideCursorForDrawSnapshot(DISPLAY_DEVICE* device,
                                               const DISPLAY_DEVICE_OPS* ops,
                                               uint32_t caps) {
    if (!device || !ops || !ops->ShowCursor || !(caps & DISPLAY_CAP_SOFTWARE_CURSOR))
        return false;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    bool hide = (device == gPrimaryDisplay) && gCursorDrawn;
    if (hide)
        gCursorDrawn = false;
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (hide)
        (void)ops->ShowCursor(device, false);
    return hide;
}

static void KiDisplayRestoreCursorAfterDrawSnapshot(DISPLAY_DEVICE* device,
                                                    const DISPLAY_DEVICE_OPS* ops,
                                                    uint32_t caps) {
    // Only software cursors are hidden/restored around framebuffer drawing.
    // Hardware cursors are owned by the provider/device and must not be
    // re-positioned after every blit/present; the coalesced cursor worker is
    // the only path that commits hardware cursor movement.
    if (!device || !ops || !ops->ShowCursor || !(caps & DISPLAY_CAP_SOFTWARE_CURSOR))
        return;

    int x = 0;
    int y = 0;
    bool shouldShow = false;
    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    shouldShow = (device == gPrimaryDisplay) &&
                 gCursorVisible && !gCursorDrawn &&
                 !KiDisplayOutputSuspended() &&
                 !gDisplayTransitionInProgress;
    if (shouldShow) {
        x = gCursorX;
        y = gCursorY;
    }
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (!shouldShow)
        return;

    if (ops->SetCursorPosition)
        (void)ops->SetCursorPosition(device, x, y);
    KSTATUS s = ops->ShowCursor(device, true);

    flags = KeAcquireSpinlock(&gDisplayLock);
    if (device == gPrimaryDisplay && KSUCCESS(s)) {
        gCursorDrawn = true;
        gCursorCommittedX = x;
        gCursorCommittedY = y;
        gCursorAppliedGeneration = gCursorRequestGeneration;
        gCursorUpdatePending = false;
    } else if (device != gPrimaryDisplay) {
        gCursorUpdatePending = true;
    }
    bool requeue = gCursorUpdatePending && !gDisplayTransitionInProgress;
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (requeue)
        KiDisplayQueueCursorWork();
}

static KSTATUS KiDisplayGetProviderSnapshot(DISPLAY_DEVICE** outDevice,
                                            const DISPLAY_DEVICE_OPS** outOps,
                                            uint32_t* outCaps,
                                            DISPLAY_MODE* outMode) {
    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;

    if (!KiDisplaySnapshotPrimary(&device, &ops, &caps, 0) || !device || !ops)
        return NTLSTATUS_NOT_SUPPORTED;
    if (!ops->GetMode)
        return NTLSTATUS_NOT_SUPPORTED;

    KSTATUS s = ops->GetMode(device, outMode);
    if (!KSUCCESS(s))
        return s;

    if (outDevice) *outDevice = device;
    if (outOps) *outOps = ops;
    if (outCaps) *outCaps = caps;
    return NTLSTATUS_SUCCESS;
}

static void KiDisplayBuildGlyphPixels(char c, uint32_t fg, uint32_t bg, uint32_t* outPixels) {
    const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u);
    for (int row = 0; row < FONT_H; row++) {
        uint8_t bits = glyph[row];
        for (int col = 0; col < FONT_W; col++)
            outPixels[row * FONT_W + col] = (bits & (0x80u >> col)) ? fg : bg;
    }
}

static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
                                          const DISPLAY_DEVICE_OPS* ops,
                                          const DISPLAY_MODE* mode,
                                          char c,
                                          int x,
                                          int y,
                                          uint32_t fg,
                                          uint32_t bg) {
    if (!device || !ops || !ops->Blit || !mode)
        return NTLSTATUS_NOT_SUPPORTED;

    uint32_t glyphPixels[FONT_W * FONT_H];
    KiDisplayBuildGlyphPixels(c, fg, bg, glyphPixels);
    KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels, FONT_W * sizeof(uint32_t));
    if (KSUCCESS(s)) {
        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay)
            KiDisplayRecordBlitLocked(mode, x, y, FONT_W, FONT_H);
        KeReleaseSpinlock(&gDisplayLock, flags);
    }
    return s;
}

// Draws a pitched source rectangle and records it as one display operation.
static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device,
                                                const DISPLAY_DEVICE_OPS* ops,
                                                const DISPLAY_MODE* mode,
                                                int x,
                                                int y,
                                                int w,
                                                int h,
                                                const uint32_t* pixels,
                                                uint32_t pitchBytes) {
    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
        return NTLSTATUS_INVALID_PARAMETER;
    if (!device || !ops || !mode)
        return NTLSTATUS_NOT_SUPPORTED;

    if (!ops->Blit)
        return NTLSTATUS_NOT_SUPPORTED;

    KSTATUS s = ops->Blit(device, x, y, w, h, pixels, pitchBytes);

    if (KSUCCESS(s)) {
        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay)
            KiDisplayRecordBlitLocked(mode, x, y, w, h);
        KeReleaseSpinlock(&gDisplayLock, flags);
    }
    return s;
}

static KSTATUS KiDisplayFillRectSnapshot(DISPLAY_DEVICE* device,
                                         const DISPLAY_DEVICE_OPS* ops,
                                         const DISPLAY_MODE* mode,
                                         int x,
                                         int y,
                                         int w,
                                         int h,
                                         uint32_t color) {
    if (w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;
    if (!device || !ops || !ops->FillRect || !mode)
        return NTLSTATUS_NOT_SUPPORTED;

    KSTATUS s = ops->FillRect(device, x, y, w, h, color);
    if (KSUCCESS(s)) {
        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay)
            KiDisplayRecordFillLocked(mode, x, y, w, h);
        KeReleaseSpinlock(&gDisplayLock, flags);
    }
    return s;
}

static KSTATUS KiDisplayCopyRectSnapshot(DISPLAY_DEVICE* device,
                                         const DISPLAY_DEVICE_OPS* ops,
                                         const DISPLAY_MODE* mode,
                                         int dstX,
                                         int dstY,
                                         int srcX,
                                         int srcY,
                                         int w,
                                         int h) {
    if (w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;
    if (!device || !ops || !ops->CopyRect || !mode)
        return NTLSTATUS_NOT_SUPPORTED;

    KSTATUS s = ops->CopyRect(device, dstX, dstY, srcX, srcY, w, h);
    if (KSUCCESS(s)) {
        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay)
            KiDisplayRecordCopyLocked(mode, dstX, dstY, w, h);
        KeReleaseSpinlock(&gDisplayLock, flags);
    }
    return s;
}

static KSTATUS KiDisplayFinishProviderDraw(DISPLAY_DEVICE* device,
                                           const DISPLAY_DEVICE_OPS* ops,
                                           uint32_t caps,
                                           KSTATUS drawStatus) {
    KSTATUS presentStatus = NTLSTATUS_SUCCESS;
    if (KSUCCESS(drawStatus))
        presentStatus = KiDisplayFlushDirtyWithSnapshot(device, ops);
    KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps);
    return KSUCCESS(drawStatus) ? presentStatus : drawStatus;
}

// Implements the display Dirty All operation.
static void KiDisplayDirtyAll(void) {
    gDisplayStats.DirtyAllCount++;
    gDirtyAll = true;
    gDirtyRectCount = 0;
}

// Implements the display Add Dirty operation.
static void KiDisplayAddDirty(int x, int y, int w, int h) {
    if (!KiDisplayHasDevice() || gDirtyAll || w <= 0 || h <= 0)
        return;

    DISPLAY_MODE mode;
    if (!KSUCCESS(gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, &mode)))
        return;

    DISPLAY_RECT rect = { x, y, w, h };
    if (!KiDisplayClipRect(&mode, &rect))
        return;

    if (rect.X == 0 && rect.Y == 0 &&
        rect.W == (int32_t)mode.Width && rect.H == (int32_t)mode.Height) {
        KiDisplayDirtyAll();
        return;
    }

    if (gDirtyRectCount >= DISPLAY_MAX_DIRTY_RECTS) {
        KiDisplayDirtyAll();
        return;
    }

    gDisplayStats.DirtyRectCount++;
    gDirtyRects[gDirtyRectCount++] = rect;
}

// Implements the display Flush Dirty operation.
static KSTATUS KiDisplayFlushDirty(void) {
    if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Present)
        return NTLSTATUS_SUCCESS;
    if (!gDirtyAll && gDirtyRectCount == 0)
        return NTLSTATUS_SUCCESS;

    DISPLAY_RECT full;
    const DISPLAY_RECT* rects = gDirtyRects;
    size_t count = gDirtyRectCount;

    if (gDirtyAll) {
        DISPLAY_MODE mode;
        KSTATUS s = gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, &mode);
        if (!KSUCCESS(s))
            return s;
        full.X = 0;
        full.Y = 0;
        full.W = (int32_t)mode.Width;
        full.H = (int32_t)mode.Height;
        rects = &full;
        count = 1;
    }

    gDisplayStats.PresentCount++;
    gDisplayStats.PresentRectCount += count;
    KSTATUS s = gPrimaryDisplay->Ops->Present(gPrimaryDisplay, rects, count);
    if (KSUCCESS(s)) {
        gDirtyAll = false;
        gDirtyRectCount = 0;
    }
    return s;
}

// Implements the display Hide Cursor For Draw operation.
static void KiDisplayHideCursorForDraw(void) {
    if (!gCursorDrawn || !KiDisplayUsesSoftwareCursor())
        return;

    (void)gPrimaryDisplay->Ops->ShowCursor(gPrimaryDisplay, false);
    gCursorDrawn = false;
}

// Implements the display Restore Cursor After Draw operation.
static void KiDisplayRestoreCursorAfterDraw(void) {
    if (!gCursorVisible || gCursorDrawn || !KiDisplayHasCursorOps())
        return;

    if (gPrimaryDisplay->Ops->SetCursorPosition)
        (void)gPrimaryDisplay->Ops->SetCursorPosition(gPrimaryDisplay, gCursorX, gCursorY);

    if (KSUCCESS(gPrimaryDisplay->Ops->ShowCursor(gPrimaryDisplay, true)))
        gCursorDrawn = true;
}

// Implements the display Begin Draw operation.
static void KiDisplayBeginDraw(void) {
    if (gDrawDepth++ == 0)
        KiDisplayHideCursorForDraw();
}

// Implements the display End Draw operation.
static KSTATUS KiDisplayEndDraw(void) {
    KSTATUS s = NTLSTATUS_SUCCESS;
    if (gDrawDepth > 0)
        gDrawDepth--;

    if (gDrawDepth == 0) {
        s = KiDisplayFlushDirty();
        KiDisplayRestoreCursorAfterDraw();
    }
    return s;
}

// Implements the display Draw Blit No Cursor operation.
static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint32_t* pixels) {
    if (!pixels || w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;
    if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Blit)
        return NTLSTATUS_NOT_SUPPORTED;

    KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels, (uint32_t)w * sizeof(uint32_t));
    if (KSUCCESS(s)) {
        gDisplayStats.BlitCount++;
        gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
        KiDisplayAddDirty(x, y, w, h);
    }
    return s;
}

// Implements the display Fill Rect No Cursor operation.
static KSTATUS KiDisplayFillRectNoCursor(int x, int y, int w, int h, uint32_t color) {
    if (w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;
    if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->FillRect)
        return NTLSTATUS_NOT_SUPPORTED;

    KSTATUS s = gPrimaryDisplay->Ops->FillRect(gPrimaryDisplay, x, y, w, h, color);
    if (KSUCCESS(s)) {
        gDisplayStats.FillRectCount++;
        gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
        KiDisplayAddDirty(x, y, w, h);
    }
    return s;
}

// Implements the display Copy Rect No Cursor operation.
static KSTATUS KiDisplayCopyRectNoCursor(int dstX, int dstY, int srcX, int srcY, int w, int h) {
    if (w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;
    if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->CopyRect)
        return NTLSTATUS_NOT_SUPPORTED;

    KSTATUS s = gPrimaryDisplay->Ops->CopyRect(gPrimaryDisplay, dstX, dstY, srcX, srcY, w, h);
    if (KSUCCESS(s)) {
        gDisplayStats.CopyRectCount++;
        gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
        KiDisplayAddDirty(dstX, dstY, w, h);
    }
    return s;
}

// Implements the display Console Index operation.
static size_t KiDisplayConsoleIndex(int col, int row) {
    return (size_t)row * DISPLAY_CONSOLE_MAX_COLS + (size_t)col;
}

// Implements the display Console Color operation.
static uint32_t KiDisplayConsoleColor(uint8_t index) {
    static const uint32_t colors[16] = {
        0x00000000u, 0x000000AAu, 0x0000AA00u, 0x0000AAAAu,
        0x00AA0000u, 0x00AA00AAu, 0x00AA5500u, 0x00AAAAAAu,
        0x00555555u, 0x005555FFu, 0x0055FF55u, 0x0055FFFFu,
        0x00FF5555u, 0x00FF55FFu, 0x00FFFF55u, 0x00FFFFFFu,
    };
    return colors[index & 0x0Fu];
}

// Implements the display Console Fg operation.
static uint32_t KiDisplayConsoleFg(uint16_t attr) {
    return KiDisplayConsoleColor((uint8_t)(attr & 0x0Fu));
}

// Implements the display Console Bg operation.
static uint32_t KiDisplayConsoleBg(uint16_t attr) {
    return KiDisplayConsoleColor((uint8_t)((attr >> 4) & 0x0Fu));
}

// Implements the display Get Console Grid Locked operation.
static bool KiDisplayGetConsoleGridLocked(int* outCols, int* outRows) {
    DISPLAY_MODE mode;
    if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode)
        return false;
    if (!KSUCCESS(gPrimaryDisplay->Ops->GetMode(gPrimaryDisplay, &mode)))
        return false;

    int cols = (int)mode.Width / FONT_W;
    int rows = (int)mode.Height / FONT_H;
    if (cols > DISPLAY_CONSOLE_MAX_COLS) cols = DISPLAY_CONSOLE_MAX_COLS;
    if (rows > DISPLAY_CONSOLE_MAX_ROWS) rows = DISPLAY_CONSOLE_MAX_ROWS;
    if (cols <= 0 || rows <= 0)
        return false;

    if (outCols) *outCols = cols;
    if (outRows) *outRows = rows;
    return true;
}

// Implements the display Console Reset Cells Locked operation.
static void KiDisplayConsoleResetCellsLocked(void) {
    for (size_t i = 0; i < DISPLAY_CONSOLE_MAX_CELLS; i++) {
        gConsoleCells[i] = ' ';
        gConsoleAttrs[i] = (uint8_t)DISPLAY_CONSOLE_DEFAULT_ATTR;
    }
    gConsoleAttribute = DISPLAY_CONSOLE_DEFAULT_ATTR;
}

// Implements the display Console Scroll Cells Locked operation.
static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines) {
    if (cols <= 0 || rows <= 0 || cellLines <= 0)
        return;

    if (cellLines >= rows) {
        for (int y = 0; y < rows; y++) {
            for (int x = 0; x < cols; x++) {
                size_t i = KiDisplayConsoleIndex(x, y);
                gConsoleCells[i] = ' ';
                gConsoleAttrs[i] = (uint8_t)gConsoleAttribute;
            }
        }
        return;
    }

    for (int y = 0; y < rows - cellLines; y++) {
        for (int x = 0; x < cols; x++) {
            size_t dst = KiDisplayConsoleIndex(x, y);
            size_t src = KiDisplayConsoleIndex(x, y + cellLines);
            gConsoleCells[dst] = gConsoleCells[src];
            gConsoleAttrs[dst] = gConsoleAttrs[src];
        }
    }

    for (int y = rows - cellLines; y < rows; y++) {
        for (int x = 0; x < cols; x++) {
            size_t i = KiDisplayConsoleIndex(x, y);
            gConsoleCells[i] = ' ';
            gConsoleAttrs[i] = (uint8_t)gConsoleAttribute;
        }
    }
}

// Implements the display Draw Char Pixels Locked operation.
static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int x, int y, uint32_t fg, uint32_t bg) {
    uint32_t glyphPixels[FONT_W * FONT_H];
    const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u);

    for (int row = 0; row < FONT_H; row++) {
        uint8_t bits = glyph[row];
        for (int col = 0; col < FONT_W; col++)
            glyphPixels[row * FONT_W + col] = (bits & (0x80u >> col)) ? fg : bg;
    }

    return KiDisplayDrawBlitNoCursor(x, y, FONT_W, FONT_H, glyphPixels);
}

// Implements the display Draw Console Cell Locked operation.
static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t attr, bool updateCell) {
    if (updateCell) {
        size_t i = KiDisplayConsoleIndex(col, row);
        gConsoleCells[i] = c;
        gConsoleAttrs[i] = (uint8_t)attr;
    }

    return KiDisplayDrawCharPixelsLocked(c, col * FONT_W, row * FONT_H,
                                         KiDisplayConsoleFg(attr),
                                         KiDisplayConsoleBg(attr));
}

// Appends one glyph to the runtime display output ring while its lock is held.
static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uint32_t background) {
    if (!gDisplayOutputBuffer)
        return;

    if (gDisplayOutputUsed == DISPLAY_OUTPUT_BUFFER_CAPACITY) {
        gDisplayOutputRead = (gDisplayOutputRead + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
        gDisplayOutputUsed--;
    }

    DISPLAY_OUTPUT_GLYPH* glyph = &gDisplayOutputBuffer[gDisplayOutputWrite];
    glyph->Character = character;
    glyph->Foreground = foreground;
    glyph->Background = background;
    gDisplayOutputWrite = (gDisplayOutputWrite + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
    gDisplayOutputUsed++;
}

// Drains buffered output through the active display provider.
static KSTATUS KiDisplayDrainOutput(void) {
    DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT];
    bool mutexHeld = false;

    if (gDisplayOutputMutex) {
        KSTATUS acquireStatus = KeAcquireMutex(gDisplayOutputMutex, KE_WAIT_INFINITE);
        if (!KE_MUTEX_ACQUIRE_SUCCEEDED(acquireStatus))
            return acquireStatus;
        mutexHeld = true;
    }

    while (true) {
        size_t count = 0;
        uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
        while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) {
            glyphs[count++] = gDisplayOutputBuffer[gDisplayOutputRead];
            gDisplayOutputRead = (gDisplayOutputRead + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
            gDisplayOutputUsed--;
        }
        KeReleaseSpinlock(&gDisplayOutputLock, flags);

        if (count == 0)
            break;
        KSTATUS drawStatus = KiDisplayPrintGlyphsInternal(glyphs, count);
        if (!KSUCCESS(drawStatus)) {
            if (mutexHeld)
                KeReleaseMutex(gDisplayOutputMutex);
            return drawStatus;
        }
    }

    if (mutexHeld)
        KeReleaseMutex(gDisplayOutputMutex);
    return NTLSTATUS_SUCCESS;
}

// Drains buffered output from the passive work queue.
static void KiDisplayOutputWorker(void* context) {
    (void)context;
    (void)KiDisplayDrainOutput();
}

// Completes buffered console output before a caller overlays console pixels.
KSTATUS KeDisplayFlushConsoleOutput(void) {
    return KiDisplayDrainOutput();
}

// Queues the runtime display output worker when passive work is available.
static void KiDisplayQueueOutputWorker(void) {
    if (_InterlockedCompareExchange(&gDisplayOutputWorkInitialized, 1, 1) &&
        KeIsWorkQueueStarted()) {
        (void)KeQueueWorkItem(&gDisplayOutputWorkItem);
    }
}

// Receives one HAL output record for the runtime display endpoint.
static void KiDisplayWriteOutputRecord(void* context, const HAL_OUTPUT_RECORD* record) {
    (void)context;
    if (!record || !record->Buffer || record->Length == 0)
        return;
    if (KiDisplayOutputSuspended())
        return;

    if (KeIsWorkQueueStarted() && gDisplayOutputBuffer) {
        uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
        for (size_t i = 0; i < record->Length; i++) {
            KiDisplayOutputAppendLocked(record->Buffer[i],
                                        record->Foreground,
                                        record->Background);
        }
        KeReleaseSpinlock(&gDisplayOutputLock, flags);
        KiDisplayQueueOutputWorker();
        return;
    }

    for (size_t i = 0; i < record->Length; i++) {
        (void)KiDisplayPrintCharInternal(record->Buffer[i],
                                         record->Foreground,
                                         record->Background);
    }
}

// Registers the runtime display as a HAL output endpoint.
static void KiDisplayRegisterOutputEndpoint(void) {
    if (!gDisplayHal || !gDisplayHal->Output.RegisterEndpoint)
        return;

    HAL_OUTPUT_ENDPOINT endpoint;
    endpoint.Name = "runtime-display";
    endpoint.StreamMask = HAL_OUTPUT_STREAM_CONSOLE | HAL_OUTPUT_STREAM_CRASH;
    if (KeGetDebugFlags() & KE_DEBUG_CONSOLE)
        endpoint.StreamMask |= HAL_OUTPUT_STREAM_DEBUG;
    endpoint.Capabilities = 0;
    endpoint.Write = KiDisplayWriteOutputRecord;
    endpoint.Context = 0;

    uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
    if (!KSUCCESS(gDisplayHal->Output.RegisterEndpoint(&endpoint, &endpointId)))
        return;

    (void)endpointId;
    if (gDisplayHal->Output.SetEndpointStreams &&
        gDisplayHal->Output.BootFramebufferEndpointId != HAL_OUTPUT_ENDPOINT_INVALID) {
        (void)gDisplayHal->Output.SetEndpointStreams(
            gDisplayHal->Output.BootFramebufferEndpointId,
            HAL_OUTPUT_STREAM_PANIC);
    }
}

// References the singleton display-manager authority object used for bootstrap delegation.
KSTATUS KeDisplayReferenceManager(KE_DISPLAY_MANAGER** managerOut) {
    if (!managerOut) return NTLSTATUS_INVALID_PARAMETER;
    *managerOut = &gDisplayManager;
    ObReferenceObject(&gDisplayManager.Header);
    return NTLSTATUS_SUCCESS;
}

// Initializes the kernel display manager and bootstrap framebuffer provider.
void KeDisplayInitialize(const HAL_API* hal) {
    ObInitializeObjectHeader(&gDisplayManager.Header, ObjectTypeDisplayManager);
    gDisplayHal = hal;
    gDisplayList = NULL;
    gPrimaryDisplay = NULL;
    gDirtyRectCount = 0;
    gDirtyAll = false;
    gDrawDepth = 0;
    memset(&gDisplayStats, 0, sizeof(gDisplayStats));
    gCursorVisible = false;
    gCursorDrawn = false;
    gCursorX = 0;
    gCursorY = 0;
    gCursorDrawX = -1;
    gCursorDrawY = -1;
    gCursorUpdatePending = false;
    gCursorRequestGeneration = 0;
    gCursorAppliedGeneration = 0;
    gCursorCommittedX = DISPLAY_CURSOR_POS_INVALID;
    gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID;
    gDisplayTransitionInProgress = false;
    gDisplayGeneration = 1;
    gDisplayClockLock = (SPINLOCK)SPINLOCK_INIT;
    LIST_INIT(&gDisplayClockList);
    gDisplayFrameGeneration = 0;
    _InterlockedExchange(&gDisplayClockInitialized, 1);
    if (!gDisplayProviderMutex)
        (void)KeMutexCreate(&gDisplayProviderMutex);
    if (!gDisplayOutputMutex)
        (void)KeMutexCreate(&gDisplayOutputMutex);
    KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
    _InterlockedExchange(&gDisplayCursorWorkInitialized, 1);
    KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0);
    _InterlockedExchange(&gDisplayOutputWorkInitialized, 1);
    gDisplayOutputRead = 0;
    gDisplayOutputWrite = 0;
    gDisplayOutputUsed = 0;
    if (!gDisplayOutputBuffer) {
        gDisplayOutputBuffer = (DISPLAY_OUTPUT_GLYPH*)MmAllocatePoolTag(
            DISPLAY_OUTPUT_BUFFER_CAPACITY * sizeof(DISPLAY_OUTPUT_GLYPH),
            MM_POOL_TAG('d', 'o', 'u', 't'));
    }
    gConsoleX = 0;
    gConsoleY = 0;
    KiDisplayConsoleResetCellsLocked();
    _InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, 0);
    memset(&gBootFb, 0, sizeof(gBootFb));
    memset(&gBootFbDevice, 0, sizeof(gBootFbDevice));

    if (!hal || !hal->Screen.GetFramebuffer)
        return;

    HAL_FRAMEBUFFER_INFO fb;
    memset(&fb, 0, sizeof(fb));
    if (!KSUCCESS(hal->Screen.GetFramebuffer(&fb)) || !fb.Base || fb.Width == 0 || fb.Height == 0 || fb.Stride == 0)
        return;

    gBootFb.Hal = hal;
    gBootFb.Framebuffer = (uint32_t*)fb.Base;
    gBootFb.Width = fb.Width;
    gBootFb.Height = fb.Height;
    gBootFb.Stride = fb.Stride;

    size_t pixels = (size_t)gBootFb.Stride * (size_t)gBootFb.Height;
    size_t bytes = pixels * sizeof(uint32_t);
    if (gBootFb.Stride > 0 && gBootFb.Height > 0 &&
        pixels / (size_t)gBootFb.Stride == (size_t)gBootFb.Height &&
        bytes / sizeof(uint32_t) == pixels) {
        gBootFb.Backbuffer = (uint32_t*)MmAllocatePool(bytes);
        if (gBootFb.Backbuffer)
            memcpy(gBootFb.Backbuffer, gBootFb.Framebuffer, bytes);
    }

    gBootFbDevice.Name = "bootfb";
    gBootFbDevice.Caps = DISPLAY_CAP_FRAMEBUFFER;
    if (gBootFb.Backbuffer)
        gBootFbDevice.Caps |= DISPLAY_CAP_SOFTWARE_CURSOR;
    gBootFbDevice.DriverContext = &gBootFb;
    gBootFbDevice.Ops = &gBootFbOps;

    KSTATUS s = KeDisplayRegisterDevice(&gBootFbDevice);
    if (KSUCCESS(s)) {
        (void)KeDisplaySetPrimary(&gBootFbDevice);
        DISPLAY_MODE mode;
        if (KSUCCESS(KeDisplayGetMode(&mode))) {
            DbgPrint("Display: bootfb primary %ux%u stride=%u cache=%s\n",
                     mode.Width, mode.Height, mode.Stride,
                     KeWriteCombiningAvailable() ? "WC" : "WT");
        }

        // The registration messages above were routed through the HAL boot
        // framebuffer endpoint. Resync the display-manager backbuffer before
        // the runtime display endpoint becomes the normal screen subscriber.
        if (gBootFb.Backbuffer) {
            size_t pixels2 = (size_t)gBootFb.Stride * (size_t)gBootFb.Height;
            memcpy(gBootFb.Backbuffer, gBootFb.Framebuffer, pixels2 * sizeof(uint32_t));
        }
    }

    KiDisplayRegisterOutputEndpoint();
}

DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock) {
    if (!outClock)
        return NTLSTATUS_INVALID_PARAMETER;
    *outClock = (DISPLAY_CLOCK_OBJECT*)0;

    if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
        return NTLSTATUS_NOT_SUPPORTED;

    uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
    bool hasDevice = KiDisplayHasDevice();
    KeReleaseSpinlock(&gDisplayLock, displayFlags);
    if (!hasDevice)
        return NTLSTATUS_NOT_SUPPORTED;

    DISPLAY_CLOCK_OBJECT* clock = (DISPLAY_CLOCK_OBJECT*)ObAllocateObject(ObjectTypeDisplayClock,
                                                                          sizeof(DISPLAY_CLOCK_OBJECT));
    if (!clock)
        return NTLSTATUS_NO_MEMORY;

    clock->Header.Destroy = KiDisplayClockDestroy;
    LIST_INIT(&clock->Link);
    clock->PeriodTicks = KiDisplayClockMillisecondsToTicks(periodMilliseconds);
    uint64_t now = KiDisplayClockReadSchedulerTick();
    clock->NextTick = (UINT64_MAX - now < clock->PeriodTicks) ? UINT64_MAX : now + clock->PeriodTicks;

    KSTATUS s = KeEventCreate(EventAutoReset, 0, &clock->Event);
    if (!KSUCCESS(s)) {
        ObDereferenceObject(&clock->Header);
        return s;
    }

    uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
    clock->LastFrameGeneration = gDisplayFrameGeneration;
    clock->Linked = 1;
    ListInsertTail(&gDisplayClockList, &clock->Link);
    KeReleaseSpinlock(&gDisplayClockLock, flags);

    *outClock = clock;
    return NTLSTATUS_SUCCESS;
}

DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) {
    if (!clock)
        return (EVENT*)0;
    return clock->Event;
}

// Signals every display clock once for a provider-owned frame generation.
static void KiDisplaySignalFrameGeneration(uint64_t generation) {
    while (true) {
        EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
        uint32_t count = 0;

        uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
        LIST_ENTRY* node = gDisplayClockList.Fwd;
        while (node != &gDisplayClockList && count < DISPLAY_CLOCK_SIGNAL_BATCH) {
            DISPLAY_CLOCK_OBJECT* clock = CONTAINING_RECORD(node, DISPLAY_CLOCK_OBJECT, Link);
            node = node->Fwd;

            if (!clock->Linked || !clock->Event || clock->LastFrameGeneration == generation)
                continue;

            if (!ObTryReferenceObject(&clock->Event->Header))
                continue;
            clock->LastFrameGeneration = generation;
            events[count++] = clock->Event;
        }
        KeReleaseSpinlock(&gDisplayClockLock, flags);

        for (uint32_t i = 0; i < count; i++) {
            KeSignalEvent(events[i]);
            ObDereferenceObject(&events[i]->Header);
        }

        if (count < DISPLAY_CLOCK_SIGNAL_BATCH)
            break;
    }
}

// Publishes a frame-ready event from the current primary display provider.
DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device) {
    if (!device || !_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
        return;

    uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
    bool accepted = device == gPrimaryDisplay &&
                    (device->Caps & DISPLAY_CAP_FRAME_EVENT) != 0;
    KeReleaseSpinlock(&gDisplayLock, displayFlags);
    if (!accepted)
        return;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
    uint64_t generation = ++gDisplayFrameGeneration;
    if (generation == 0)
        generation = ++gDisplayFrameGeneration;
    KeReleaseSpinlock(&gDisplayClockLock, flags);

    KiDisplaySignalFrameGeneration(generation);
}

DISPLAY_API void KeDisplayTick(void) {
    if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
        return;

    uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
    bool providerOwnsFrames = gPrimaryDisplay &&
                              (gPrimaryDisplay->Caps & DISPLAY_CAP_FRAME_EVENT) != 0;
    KeReleaseSpinlock(&gDisplayLock, displayFlags);
    if (providerOwnsFrames)
        return;

    uint64_t now = KiDisplayClockReadSchedulerTick();
    while (true) {
        EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
        uint32_t count = 0;

        uintptr_t flags = KeAcquireSpinlock(&gDisplayClockLock);
        LIST_ENTRY* node = gDisplayClockList.Fwd;
        while (node != &gDisplayClockList && count < DISPLAY_CLOCK_SIGNAL_BATCH) {
            DISPLAY_CLOCK_OBJECT* clock = CONTAINING_RECORD(node, DISPLAY_CLOCK_OBJECT, Link);
            node = node->Fwd;

            if (!clock->Linked || clock->NextTick > now || !clock->Event)
                continue;

            if (!ObTryReferenceObject(&clock->Event->Header))
                continue;
            uint64_t next = now + clock->PeriodTicks;
            clock->NextTick = next < now ? UINT64_MAX : next;
            events[count++] = clock->Event;
        }
        KeReleaseSpinlock(&gDisplayClockLock, flags);

        for (uint32_t i = 0; i < count; i++) {
            KeSignalEvent(events[i]);
            ObDereferenceObject(&events[i]->Header);
        }

        if (count < DISPLAY_CLOCK_SIGNAL_BATCH)
            break;
    }
}

DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
    if (!device || !device->Name || !device->Ops || !device->Ops->GetMode)
        return NTLSTATUS_INVALID_PARAMETER;

    KSTATUS result = NTLSTATUS_SUCCESS;
    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) {
        if (it == device) {
            result = NTLSTATUS_ALREADY_EXISTS;
            break;
        }
    }

    if (KSUCCESS(result)) {
        device->Next = gDisplayList;
        gDisplayList = device;
    }
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (KSUCCESS(result))
        DbgPrint("Display: registered device %s caps=0x%x\n", device->Name, device->Caps);
    return result;
}

// Implements the display Seed New Primary operation.
static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE* newDevice) {
    if (!oldDevice || !newDevice || oldDevice == newDevice)
        return NTLSTATUS_SUCCESS;
    if (!oldDevice->Ops || !oldDevice->Ops->ReadRect ||
        !newDevice->Ops || !newDevice->Ops->GetMode || !newDevice->Ops->Blit)
        return NTLSTATUS_NOT_SUPPORTED;

    DISPLAY_MODE oldMode;
    DISPLAY_MODE newMode;
    KSTATUS s = oldDevice->Ops->GetMode(oldDevice, &oldMode);
    if (!KSUCCESS(s))
        return s;
    s = newDevice->Ops->GetMode(newDevice, &newMode);
    if (!KSUCCESS(s))
        return s;
    if (oldMode.Format != DisplayPixelFormatX8R8G8B8 ||
        newMode.Format != DisplayPixelFormatX8R8G8B8 ||
        oldMode.Width != newMode.Width || oldMode.Height != newMode.Height)
        return NTLSTATUS_NOT_SUPPORTED;

    uint32_t* row = (uint32_t*)MmAllocatePool((size_t)newMode.Width * sizeof(uint32_t));
    if (!row)
        return NTLSTATUS_NO_MEMORY;

    for (uint32_t y = 0; y < newMode.Height; y++) {
        s = oldDevice->Ops->ReadRect(oldDevice, 0, (int)y, (int)newMode.Width, 1, row, newMode.Width);
        if (!KSUCCESS(s))
            break;
        s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row,
                                  newMode.Width * sizeof(uint32_t));
        if (!KSUCCESS(s))
            break;
    }

    MmFreePool(row);
    return s;
}

DISPLAY_API void KeDisplaySuspendConsole(bool suspend) {
    _InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, suspend ? 1 : 0);
}

// Implements the display Output Suspended operation.
static bool KiDisplayOutputSuspended(void) {
    return _InterlockedCompareExchange((volatile long*)&gDisplayConsoleSuspended, 0, 0) != 0;
}

DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) {
    if (!device || !device->Ops || !device->Ops->GetMode)
        return NTLSTATUS_INVALID_PARAMETER;

    const char* name = device->Name ? device->Name : "?";

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    bool found = false;
    for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) {
        if (it == device) {
            found = true;
            break;
        }
    }
    if (!found) {
        KeReleaseSpinlock(&gDisplayLock, flags);
        return NTLSTATUS_NOT_FOUND;
    }

    DISPLAY_DEVICE* oldDevice = gPrimaryDisplay;
    bool oldCursorDrawn = gCursorDrawn;
    gDisplayTransitionInProgress = true;
    gDisplayGeneration++;
    gCursorUpdatePending = gCursorVisible;
    KeReleaseSpinlock(&gDisplayLock, flags);

    bool providerHeld = false;
    KSTATUS result = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(result)) {
        flags = KeAcquireSpinlock(&gDisplayLock);
        gDisplayTransitionInProgress = false;
        gDisplayGeneration++;
        KeReleaseSpinlock(&gDisplayLock, flags);
        KiDisplayQueueCursorWork();
        return result;
    }

    if (oldDevice && oldCursorDrawn && oldDevice->Ops && oldDevice->Ops->ShowCursor)
        (void)oldDevice->Ops->ShowCursor(oldDevice, false);

    // Clean architecture policy: handoff does not read back/copy the old boot
    // framebuffer. The new primary is cleared to the kernel console blue and
    // user/session code redraws state above the provider boundary.
    if (device->Ops->GetMode && device->Ops->FillRect) {
        DISPLAY_MODE mode;
        KSTATUS modeStatus = device->Ops->GetMode(device, &mode);
        if (KSUCCESS(modeStatus)) {
            (void)device->Ops->FillRect(device, 0, 0,
                                        (int)mode.Width,
                                        (int)mode.Height,
                                        DISPLAY_DEFAULT_BG);
            if (device->Ops->Present) {
                DISPLAY_RECT full = { 0, 0, (int32_t)mode.Width, (int32_t)mode.Height };
                (void)device->Ops->Present(device, &full, 1);
            }
        }
    }

    flags = KeAcquireSpinlock(&gDisplayLock);
    gPrimaryDisplay = device;
    gCursorDrawn = false;
    gCursorCommittedX = DISPLAY_CURSOR_POS_INVALID;
    gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID;
    gDirtyAll = false;
    gDirtyRectCount = 0;
    gDrawDepth = 0;

    // The provider handoff clears the framebuffer instead of preserving boot
    // text. Keep the console model coherent with that clear so subsequent
    // kernel/debug output restarts at the top-left of the new primary.
    KiDisplayConsoleResetCellsLocked();
    gConsoleX = 0;
    gConsoleY = 0;

    gDisplayTransitionInProgress = false;
    gDisplayGeneration++;
    bool needCursor = gCursorVisible;
    gCursorUpdatePending = needCursor;
    if (needCursor)
        gCursorRequestGeneration++;
    KeReleaseSpinlock(&gDisplayLock, flags);

    KiDisplayReleaseProviderMutex(providerHeld);

    if (needCursor)
        KiDisplayQueueCursorWork();

    DbgPrint("Display: primary is %s\n", name);
    return result;
}

DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode) {
    if (!outMode)
        return NTLSTATUS_INVALID_PARAMETER;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0;
    KeReleaseSpinlock(&gDisplayLock, flags);

    s = (!device || !ops || !ops->GetMode)
            ? NTLSTATUS_INVALID_PARAMETER
            : ops->GetMode(device, outMode);

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight) {
    DISPLAY_MODE mode;
    KSTATUS s = KeDisplayGetMode(&mode);
    if (!KSUCCESS(s))
        return s;
    if (outWidth) *outWidth = (int)mode.Width;
    if (outHeight) *outHeight = (int)mode.Height;
    return NTLSTATUS_SUCCESS;
}

DISPLAY_API KSTATUS KeDisplaySync(void) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0;
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (!device || !ops || !ops->Sync) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    s = ops->Sync(device);

    flags = KeAcquireSpinlock(&gDisplayLock);
    if (device == gPrimaryDisplay)
        gDisplayStats.SyncCount++;
    KeReleaseSpinlock(&gDisplayLock, flags);

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) {
    if (!outInfo)
        return NTLSTATUS_INVALID_PARAMETER;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    KE_DISPLAY_INFO info;
    memset(&info, 0, sizeof(info));

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);

    uint32_t count = 0;
    for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next)
        count++;
    info.DeviceCount = count;
    if (KeWriteCombiningAvailable())
        info.Flags |= DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE;
    info.ManagerStats = gDisplayStats;

    DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0;
    if (device) {
        const char* name = device->Name ? device->Name : "?";
        size_t len = strlen(name);
        if (len >= sizeof(info.PrimaryName))
            len = sizeof(info.PrimaryName) - 1;
        memcpy(info.PrimaryName, name, len);
        info.PrimaryName[len] = '\0';
        info.Caps = device->Caps;
    }

    KeReleaseSpinlock(&gDisplayLock, flags);

    if (!device || !ops) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_FOUND;
    }

    if (ops->GetMode) {
        s = ops->GetMode(device, &info.Mode);
        if (!KSUCCESS(s)) {
            KiDisplayReleaseProviderMutex(providerHeld);
            return s;
        }
    }

    if (ops->QueryStats)
        (void)ops->QueryStats(device, &info.DeviceStats);

    *outInfo = info;
    KiDisplayReleaseProviderMutex(providerHeld);
    return NTLSTATUS_SUCCESS;
}

DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels,
                                               (uint32_t)w * sizeof(uint32_t));
        s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
        return NTLSTATUS_INVALID_PARAMETER;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, pitchBytes);
        s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayPresentPitched(int x,
                                              int y,
                                              int w,
                                              int h,
                                              const uint32_t* pixels,
                                              uint32_t pitchBytes) {
    if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
        return NTLSTATUS_INVALID_PARAMETER;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        if (ops->PresentPitched) {
            s = ops->PresentPitched(device, x, y, w, h, pixels, pitchBytes);
            if (KSUCCESS(s)) {
                uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
                if (device == gPrimaryDisplay)
                    KiDisplayRecordDirectPresentLocked(&mode, x, y, w, h);
                KeReleaseSpinlock(&gDisplayLock, flags);
            }
            KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps);
        } else {
            s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode,
                                                 x, y, w, h, pixels, pitchBytes);
            s = KiDisplayFinishProviderDraw(device, ops, caps, s);
        }
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        s = KiDisplayFillRectSnapshot(device, ops, &mode, x, y, w, h, color);
        s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayClear(uint32_t color) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0, (int)mode.Width, (int)mode.Height, color);
        if (KSUCCESS(s)) {
            uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
            if (device == gPrimaryDisplay) {
                KiDisplayConsoleResetCellsLocked();
                gConsoleX = 0;
                gConsoleY = 0;
            }
            KeReleaseSpinlock(&gDisplayLock, flags);
        }
        s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        s = KiDisplayCopyRectSnapshot(device, ops, &mode, dstX, dstY, srcX, srcY, w, h);
        s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayScroll(int lines) {
    if (lines <= 0)
        return NTLSTATUS_SUCCESS;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        if ((uint32_t)lines >= mode.Height) {
            s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0, (int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG);
            if (KSUCCESS(s)) {
                uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
                if (device == gPrimaryDisplay) {
                    KiDisplayConsoleResetCellsLocked();
                    gConsoleX = 0;
                    gConsoleY = 0;
                }
                KeReleaseSpinlock(&gDisplayLock, flags);
            }
        } else {
            s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, lines,
                                          (int)mode.Width, (int)(mode.Height - (uint32_t)lines));
            if (KSUCCESS(s)) {
                s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
                                              (int)(mode.Height - (uint32_t)lines),
                                              (int)mode.Width, lines, DISPLAY_DEFAULT_BG);
            }
        }
        s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (KSUCCESS(s)) {
        (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, x, y, fg, bg);
        s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    }

    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

// Renders one or more console glyphs as one provider transaction and one present.
static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count) {
    if (!glyphs && count != 0)
        return NTLSTATUS_INVALID_PARAMETER;
    if (count == 0)
        return NTLSTATUS_SUCCESS;
    if (count > DISPLAY_OUTPUT_DRAIN_COUNT)
        return NTLSTATUS_INVALID_PARAMETER;

    if (KiDisplayOutputSuspended()) {
        if (!KeCanWait()) {
            for (size_t i = 0; i < count; i++)
                (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
        }
        return NTLSTATUS_SUCCESS;
    }

    if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) {
        for (size_t i = 0; i < count; i++)
            (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
        return NTLSTATUS_SUCCESS;
    }

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (!KSUCCESS(s)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        const HAL_API* hal = gDisplayHal;
        if (hal && hal->Screen.PrintChar) {
            for (size_t i = 0; i < count; i++)
                hal->Screen.PrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
        }
        return s;
    }

    int cols = 0;
    int rows = 0;
    if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_DEVICE_ERROR;
    }

    (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);

    DISPLAY_OUTPUT_DRAW draws[DISPLAY_OUTPUT_DRAIN_COUNT];
    size_t drawCount = 0;
    size_t scrollCount = 0;

    for (size_t glyphIndex = 0; glyphIndex < count; glyphIndex++) {
        char c = glyphs[glyphIndex].Character;
        int drawCol = 0;
        int drawRow = 0;
        bool drawChar = false;
        bool needScroll = false;

        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (c == '\n') {
            gConsoleX = 0;
            gConsoleY++;
        } else if (c == '\r') {
            gConsoleX = 0;
        } else if (c == '\b') {
            if (gConsoleX > 0) {
                gConsoleX--;
            } else if (gConsoleY > 0) {
                gConsoleY--;
                gConsoleX = cols - 1;
            }
        } else {
            drawChar = true;
            drawCol = gConsoleX;
            drawRow = gConsoleY;
            if (++gConsoleX >= cols) {
                gConsoleX = 0;
                gConsoleY++;
            }

            if (drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) {
                size_t cell = KiDisplayConsoleIndex(drawCol, drawRow);
                gConsoleCells[cell] = c;
                gConsoleAttrs[cell] = (uint8_t)gConsoleAttribute;
            }
        }

        if (gConsoleY >= rows) {
            KiDisplayConsoleScrollCellsLocked(cols, rows, 1);
            gConsoleY = rows - 1;
            if (gConsoleX < 0) gConsoleX = 0;
            if (gConsoleX >= cols) gConsoleX = cols - 1;
            needScroll = true;
        }
        KeReleaseSpinlock(&gDisplayLock, flags);

        if (drawChar) {
            DISPLAY_OUTPUT_DRAW* draw = &draws[drawCount++];
            draw->Character = c;
            draw->Foreground = glyphs[glyphIndex].Foreground;
            draw->Background = glyphs[glyphIndex].Background;
            draw->Column = drawCol;
            draw->Row = drawRow;
            draw->ScrollCount = scrollCount;
        }
        if (needScroll)
            scrollCount++;
    }

    if (scrollCount != 0) {
        uint64_t scrollPixels64 = (uint64_t)scrollCount * FONT_H;
        int scrollPixels = scrollPixels64 >= mode.Height ? (int)mode.Height : (int)scrollPixels64;
        if ((uint32_t)scrollPixels >= mode.Height) {
            s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0,
                                          (int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG);
        } else {
            s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, scrollPixels,
                                          (int)mode.Width, (int)(mode.Height - (uint32_t)scrollPixels));
            if (KSUCCESS(s)) {
                s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
                                              (int)(mode.Height - (uint32_t)scrollPixels),
                                              (int)mode.Width, scrollPixels, DISPLAY_DEFAULT_BG);
            }
        }
    }

    for (size_t i = 0; i < drawCount && KSUCCESS(s); i++) {
        DISPLAY_OUTPUT_DRAW* draw = &draws[i];
        size_t laterScrolls = scrollCount - draw->ScrollCount;
        int finalRow = draw->Row - (int)laterScrolls;
        if (finalRow < 0 || finalRow >= rows)
            continue;
        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, draw->Character,
                                       draw->Column * FONT_W, finalRow * FONT_H,
                                       draw->Foreground, draw->Background);
    }

    s = KiDisplayFinishProviderDraw(device, ops, caps, s);
    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

// Renders one console character through the shared batched console path.
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
    DISPLAY_OUTPUT_GLYPH glyph = { c, fg, bg };
    return KiDisplayPrintGlyphsInternal(&glyph, 1);
}

DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
    return KiDisplayPrintCharInternal(c, fg, bg);
}

DISPLAY_API KSTATUS KeDisplayGetConsoleInfo(KE_CONSOLE_INFO* outInfo) {
    if (!outInfo)
        return NTLSTATUS_INVALID_PARAMETER;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    (void)caps;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (!KSUCCESS(s)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    int cols = 0;
    int rows = 0;
    if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    outInfo->Columns = (uint32_t)cols;
    outInfo->Rows = (uint32_t)rows;
    outInfo->CursorX = (uint32_t)gConsoleX;
    outInfo->CursorY = (uint32_t)gConsoleY;
    outInfo->Attribute = gConsoleAttribute;
    KeReleaseSpinlock(&gDisplayLock, flags);

    KiDisplayReleaseProviderMutex(providerHeld);
    return NTLSTATUS_SUCCESS;
}

DISPLAY_API KSTATUS KeDisplaySetConsoleCursor(int col, int row) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    (void)caps;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (!KSUCCESS(s)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    int cols = 0;
    int rows = 0;
    if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }
    if (col < 0 || row < 0 || col >= cols || row >= rows) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_INVALID_PARAMETER;
    }

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    gConsoleX = col;
    gConsoleY = row;
    KeReleaseSpinlock(&gDisplayLock, flags);

    KiDisplayReleaseProviderMutex(providerHeld);
    return NTLSTATUS_SUCCESS;
}

DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int col, int row, size_t length, size_t* writtenOut) {
    if (writtenOut) *writtenOut = 0;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (!KSUCCESS(s)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    int cols = 0;
    int rows = 0;
    if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }
    if (col < 0 || row < 0 || col >= cols || row >= rows) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_INVALID_PARAMETER;
    }

    (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);

    size_t written = 0;
    int x = col;
    int y = row;
    while (written < length && y < rows) {
        uint16_t attr = DISPLAY_CONSOLE_DEFAULT_ATTR;
        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay) {
            size_t i = KiDisplayConsoleIndex(x, y);
            attr = gConsoleAttrs[i];
        }
        KeReleaseSpinlock(&gDisplayLock, flags);

        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, x * FONT_W, y * FONT_H,
                                       KiDisplayConsoleFg(attr), KiDisplayConsoleBg(attr));
        if (!KSUCCESS(s))
            break;

        flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay) {
            size_t i = KiDisplayConsoleIndex(x, y);
            gConsoleCells[i] = c;
            gConsoleAttrs[i] = (uint8_t)attr;
        }
        KeReleaseSpinlock(&gDisplayLock, flags);

        written++;
        if (++x >= cols) {
            x = 0;
            y++;
        }
    }

    KSTATUS fs = KiDisplayFinishProviderDraw(device, ops, caps, s);
    KiDisplayReleaseProviderMutex(providerHeld);
    if (writtenOut) *writtenOut = written;
    return KSUCCESS(s) ? fs : s;
}

DISPLAY_API KSTATUS KeDisplayFillConsoleAttrs(uint16_t attr, int col, int row, size_t length, size_t* writtenOut) {
    if (writtenOut) *writtenOut = 0;

    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (!KSUCCESS(s)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    int cols = 0;
    int rows = 0;
    if (!KiDisplayConsoleGridFromMode(&mode, &cols, &rows)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }
    if (col < 0 || row < 0 || col >= cols || row >= rows) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_INVALID_PARAMETER;
    }

    (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);

    size_t written = 0;
    int x = col;
    int y = row;
    while (written < length && y < rows) {
        char ch = ' ';
        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay) {
            size_t i = KiDisplayConsoleIndex(x, y);
            ch = gConsoleCells[i];
        }
        KeReleaseSpinlock(&gDisplayLock, flags);

        s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, ch, x * FONT_W, y * FONT_H,
                                       KiDisplayConsoleFg(attr), KiDisplayConsoleBg(attr));
        if (!KSUCCESS(s))
            break;

        flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay) {
            size_t i = KiDisplayConsoleIndex(x, y);
            gConsoleCells[i] = ch;
            gConsoleAttrs[i] = (uint8_t)attr;
        }
        KeReleaseSpinlock(&gDisplayLock, flags);

        written++;
        if (++x >= cols) {
            x = 0;
            y++;
        }
    }

    KSTATUS fs = KiDisplayFinishProviderDraw(device, ops, caps, s);
    KiDisplayReleaseProviderMutex(providerHeld);
    if (writtenOut) *writtenOut = written;
    return KSUCCESS(s) ? fs : s;
}

DISPLAY_API KSTATUS KeDisplayPresent(const DISPLAY_RECT* rects, size_t rectCount) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
    uint32_t caps = 0;
    DISPLAY_MODE mode;
    s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
    if (!KSUCCESS(s)) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return s;
    }
    if (!ops->Present) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);

    if (!rects || rectCount == 0) {
        s = KiDisplayFlushDirtyWithSnapshot(device, ops);
    } else {
        uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
        if (device == gPrimaryDisplay) {
            gDisplayStats.PresentCount++;
            gDisplayStats.PresentRectCount += rectCount;
        }
        KeReleaseSpinlock(&gDisplayLock, flags);
        s = ops->Present(device, rects, rectCount);
    }

    KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps);
    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplaySetCursorImage(int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
    bool providerHeld = false;
    KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
    if (!KSUCCESS(s))
        return s;

    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
    const DISPLAY_DEVICE_OPS* ops = device ? device->Ops : (const DISPLAY_DEVICE_OPS*)0;
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (!device || !ops || !ops->SetCursorImage) {
        KiDisplayReleaseProviderMutex(providerHeld);
        return NTLSTATUS_NOT_SUPPORTED;
    }

    s = ops->SetCursorImage(device, width, height, hotX, hotY, argbPixels);
    KiDisplayReleaseProviderMutex(providerHeld);
    return s;
}

DISPLAY_API KSTATUS KeDisplaySetCursorPosition(int x, int y) {
    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    gCursorX = x;
    gCursorY = y;
    gDisplayStats.CursorMoveCount++;

    bool shouldQueue = !KiDisplayOutputSuspended() &&
                       gCursorVisible &&
                       KiDisplayHasCursorOps();
    if (shouldQueue) {
        gCursorUpdatePending = true;
        gCursorRequestGeneration++;
    }
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (shouldQueue)
        KiDisplayQueueCursorWork();
    return NTLSTATUS_SUCCESS;
}

DISPLAY_API KSTATUS KeDisplayShowCursor(bool show) {
    uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
    gCursorVisible = show;
    if (show) gDisplayStats.CursorShowCount++; else gDisplayStats.CursorHideCount++;

    bool shouldQueue = !KiDisplayOutputSuspended() && KiDisplayHasCursorOps();
    if (!shouldQueue && !show) {
        gCursorDrawn = false;
        gCursorCommittedX = DISPLAY_CURSOR_POS_INVALID;
        gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID;
    }
    if (shouldQueue) {
        gCursorUpdatePending = true;
        gCursorRequestGeneration++;
    }
    KeReleaseSpinlock(&gDisplayLock, flags);

    if (shouldQueue)
        KiDisplayQueueCursorWork();
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Context operation.
static BOOTFB_CONTEXT* BootFbContext(DISPLAY_DEVICE* device) {
    return device ? (BOOTFB_CONTEXT*)device->DriverContext : NULL;
}

// Implements the boot Fb Get Mode operation.
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer || !outMode)
        return NTLSTATUS_INVALID_PARAMETER;

    outMode->Width = fb->Width;
    outMode->Height = fb->Height;
    outMode->Stride = fb->Stride;
    outMode->Format = DisplayPixelFormatX8R8G8B8;
    return NTLSTATUS_SUCCESS;
}

// Copies one pixel row into the write-combined scanout aperture.
static void BootFbCopyToScanout(uint32_t* destination,
                                const uint32_t* source,
                                size_t pixelCount) {
    if (!destination || !source || pixelCount == 0)
        return;

    if (!KeWriteCombiningAvailable()) {
        memcpy(destination, source, pixelCount * sizeof(uint32_t));
        return;
    }

    while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) {
        *destination++ = *source++;
        pixelCount--;
    }

    while (pixelCount >= 16u) {
        __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
        __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
        __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
        __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
        _mm_stream_si128((__m128i*)(destination + 0), v0);
        _mm_stream_si128((__m128i*)(destination + 4), v1);
        _mm_stream_si128((__m128i*)(destination + 8), v2);
        _mm_stream_si128((__m128i*)(destination + 12), v3);
        source += 16;
        destination += 16;
        pixelCount -= 16;
    }

    while (pixelCount >= 4u) {
        __m128i value = _mm_loadu_si128((const __m128i*)source);
        _mm_stream_si128((__m128i*)destination, value);
        source += 4;
        destination += 4;
        pixelCount -= 4;
    }

    while (pixelCount != 0) {
        *destination++ = *source++;
        pixelCount--;
    }
}

// Copies one source row into the retained shadow and scanout in one read pass.
static void BootFbCopyToShadowAndScanout(uint32_t* shadow,
                                         uint32_t* scanout,
                                         const uint32_t* source,
                                         size_t pixelCount) {
    if (!shadow || !scanout || !source || pixelCount == 0)
        return;

    if (!KeWriteCombiningAvailable()) {
        memcpy(shadow, source, pixelCount * sizeof(uint32_t));
        memcpy(scanout, source, pixelCount * sizeof(uint32_t));
        return;
    }

    while (pixelCount != 0 && ((uintptr_t)scanout & 15u) != 0) {
        uint32_t value = *source++;
        *shadow++ = value;
        *scanout++ = value;
        pixelCount--;
    }

    while (pixelCount >= 16u) {
        __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
        __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
        __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
        __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
        _mm_storeu_si128((__m128i*)(shadow + 0), v0);
        _mm_storeu_si128((__m128i*)(shadow + 4), v1);
        _mm_storeu_si128((__m128i*)(shadow + 8), v2);
        _mm_storeu_si128((__m128i*)(shadow + 12), v3);
        _mm_stream_si128((__m128i*)(scanout + 0), v0);
        _mm_stream_si128((__m128i*)(scanout + 4), v1);
        _mm_stream_si128((__m128i*)(scanout + 8), v2);
        _mm_stream_si128((__m128i*)(scanout + 12), v3);
        source += 16;
        shadow += 16;
        scanout += 16;
        pixelCount -= 16;
    }

    while (pixelCount >= 4u) {
        __m128i value = _mm_loadu_si128((const __m128i*)source);
        _mm_storeu_si128((__m128i*)shadow, value);
        _mm_stream_si128((__m128i*)scanout, value);
        source += 4;
        shadow += 4;
        scanout += 4;
        pixelCount -= 4;
    }

    while (pixelCount != 0) {
        uint32_t value = *source++;
        *shadow++ = value;
        *scanout++ = value;
        pixelCount--;
    }
}

// Implements the boot framebuffer pitched blit operation.
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
        pitchBytes < (uint32_t)w * sizeof(uint32_t))
        return NTLSTATUS_INVALID_PARAMETER;

    uint32_t* dstBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer;

    int startX = x < 0 ? 0 : x;
    int startY = y < 0 ? 0 : y;
    int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w;
    int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h;
    if (startX >= endX || startY >= endY)
        return NTLSTATUS_SUCCESS;

    int srcOffsetX = startX - x;
    int srcOffsetY = startY - y;
    size_t bytesPerRow = (size_t)(endX - startX) * sizeof(uint32_t);

    fb->Stats.BlitCount++;
    fb->Stats.BytesCopied += (uint64_t)(endX - startX) * (uint64_t)(endY - startY) * sizeof(uint32_t);

    for (int row = 0; row < endY - startY; row++) {
        uint32_t* dst = dstBase + ((size_t)(startY + row) * fb->Stride) + startX;
        const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
        const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
        memcpy(dst, src, bytesPerRow);
    }
    return NTLSTATUS_SUCCESS;
}

// Presents one pitched source rectangle without a shadow-to-scanout second pass.
static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
                                    int x,
                                    int y,
                                    int w,
                                    int h,
                                    const uint32_t* pixels,
                                    uint32_t pitchBytes) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
        pitchBytes < (uint32_t)w * sizeof(uint32_t))
        return NTLSTATUS_INVALID_PARAMETER;

    int startX = x < 0 ? 0 : x;
    int startY = y < 0 ? 0 : y;
    int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w;
    int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h;
    if (startX >= endX || startY >= endY)
        return NTLSTATUS_SUCCESS;

    int srcOffsetX = startX - x;
    int srcOffsetY = startY - y;
    size_t pixelCount = (size_t)(endX - startX);
    uint64_t bytes = (uint64_t)pixelCount * (uint64_t)(endY - startY) * sizeof(uint32_t);

    fb->Stats.BlitCount++;
    fb->Stats.PresentCount++;
    fb->Stats.PresentRectCount++;
    fb->Stats.DirectPresentCount++;
    fb->Stats.ScanoutBytes += bytes;
    fb->Stats.BytesCopied += fb->Backbuffer ? bytes * 2u : bytes;

    for (int row = 0; row < endY - startY; row++) {
        const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
        const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
        uint32_t* scanout = fb->Framebuffer + ((size_t)(startY + row) * fb->Stride) + startX;
        if (fb->Backbuffer) {
            uint32_t* shadow = fb->Backbuffer + ((size_t)(startY + row) * fb->Stride) + startX;
            BootFbCopyToShadowAndScanout(shadow, scanout, source, pixelCount);
        } else {
            BootFbCopyToScanout(scanout, source, pixelCount);
        }
    }

    _mm_sfence();
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Fill Rect operation.
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;

    uint32_t* dstBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer;

    int startX = x < 0 ? 0 : x;
    int startY = y < 0 ? 0 : y;
    int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w;
    int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h;
    if (startX >= endX || startY >= endY)
        return NTLSTATUS_SUCCESS;

    fb->Stats.FillRectCount++;
    fb->Stats.BytesCopied += (uint64_t)(endX - startX) * (uint64_t)(endY - startY) * sizeof(uint32_t);

    for (int yy = startY; yy < endY; yy++) {
        uint32_t* dst = dstBase + ((size_t)yy * fb->Stride) + startX;
        for (int xx = startX; xx < endX; xx++)
            *dst++ = color;
    }
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Copy Rect operation.
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;

    uint32_t* base = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer;

    if (srcX < 0) { int d = -srcX; srcX = 0; dstX += d; w -= d; }
    if (srcY < 0) { int d = -srcY; srcY = 0; dstY += d; h -= d; }
    if (dstX < 0) { int d = -dstX; dstX = 0; srcX += d; w -= d; }
    if (dstY < 0) { int d = -dstY; dstY = 0; srcY += d; h -= d; }
    if (srcX + w > (int)fb->Width) w = (int)fb->Width - srcX;
    if (dstX + w > (int)fb->Width) w = (int)fb->Width - dstX;
    if (srcY + h > (int)fb->Height) h = (int)fb->Height - srcY;
    if (dstY + h > (int)fb->Height) h = (int)fb->Height - dstY;
    if (w <= 0 || h <= 0)
        return NTLSTATUS_SUCCESS;

    fb->Stats.CopyRectCount++;
    fb->Stats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);

    size_t rowBytes = (size_t)w * sizeof(uint32_t);
    if (dstY > srcY) {
        for (int row = h - 1; row >= 0; row--) {
            uint32_t* dst = base + ((size_t)(dstY + row) * fb->Stride) + dstX;
            uint32_t* src = base + ((size_t)(srcY + row) * fb->Stride) + srcX;
            memmove(dst, src, rowBytes);
        }
    } else {
        for (int row = 0; row < h; row++) {
            uint32_t* dst = base + ((size_t)(dstY + row) * fb->Stride) + dstX;
            uint32_t* src = base + ((size_t)(srcY + row) * fb->Stride) + srcX;
            memmove(dst, src, rowBytes);
        }
    }
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Read Rect operation.
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
        return NTLSTATUS_INVALID_PARAMETER;

    uint32_t* srcBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer;
    DISPLAY_MODE mode = { fb->Width, fb->Height, fb->Stride, DisplayPixelFormatX8R8G8B8 };
    DISPLAY_RECT r = { x, y, w, h };
    if (!KiDisplayClipRect(&mode, &r))
        return NTLSTATUS_SUCCESS;

    fb->Stats.ReadRectCount++;
    fb->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);

    int dstOffsetX = r.X - x;
    int dstOffsetY = r.Y - y;
    for (int row = 0; row < r.H; row++) {
        const uint32_t* src = srcBase + ((size_t)(r.Y + row) * fb->Stride) + r.X;
        uint32_t* dst = outPixels + ((size_t)(dstOffsetY + row) * outStride) + dstOffsetX;
        memcpy(dst, src, (size_t)r.W * sizeof(uint32_t));
    }
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Present operation.
static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer)
        return NTLSTATUS_INVALID_PARAMETER;
    if (!fb->Backbuffer || !rects || rectCount == 0)
        return NTLSTATUS_SUCCESS;

    DISPLAY_MODE mode;
    KSTATUS s = BootFbGetMode(device, &mode);
    if (!KSUCCESS(s))
        return s;

    fb->Stats.PresentCount++;
    fb->Stats.PresentRectCount += rectCount;

    for (size_t i = 0; i < rectCount; i++) {
        DISPLAY_RECT r = rects[i];
        if (!KiDisplayClipRect(&mode, &r))
            continue;
        uint64_t bytes = (uint64_t)(uint32_t)r.W *
                         (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
        fb->Stats.BytesCopied += bytes;
        fb->Stats.ScanoutBytes += bytes;
        size_t pixelCount = (size_t)r.W;
        for (int row = 0; row < r.H; row++) {
            uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
            uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
            BootFbCopyToScanout(dst, src, pixelCount);
        }
    }
    _mm_sfence();
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Sync operation.
static KSTATUS BootFbSync(DISPLAY_DEVICE* device) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (fb) fb->Stats.SyncCount++;
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Set Cursor Image operation.
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
    (void)device;
    (void)width;
    (void)height;
    (void)hotX;
    (void)hotY;
    (void)argbPixels;
    return NTLSTATUS_NOT_SUPPORTED;
}

// Implements the boot Fb Set Cursor Position operation.
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (fb) fb->Stats.CursorMoveCount++;
    (void)x;
    (void)y;
    return NTLSTATUS_SUCCESS;
}

// Implements the display Blend Pixel operation.
static inline uint32_t KiDisplayBlendPixel(uint32_t dst, uint32_t src) {
    uint8_t a = (uint8_t)(src >> 24);
    if (a == 0) return dst;
    if (a == 255) return src & 0x00FFFFFFu;

    uint8_t sr = (uint8_t)(src >> 16);
    uint8_t sg = (uint8_t)(src >> 8);
    uint8_t sb = (uint8_t)src;
    uint8_t dr = (uint8_t)(dst >> 16);
    uint8_t dg = (uint8_t)(dst >> 8);
    uint8_t db = (uint8_t)dst;
    uint8_t na = (uint8_t)(255 - a);
    uint8_t rr = (uint8_t)((a * sr + na * dr) >> 8);
    uint8_t gg = (uint8_t)((a * sg + na * dg) >> 8);
    uint8_t bb = (uint8_t)((a * sb + na * db) >> 8);
    return ((uint32_t)rr << 16) | ((uint32_t)gg << 8) | bb;
}

// Implements the boot Fb Restore Cursor operation.
static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
    if (!fb || !fb->Framebuffer || !fb->Backbuffer || gCursorDrawX < 0)
        return;

    DISPLAY_RECT r = { gCursorDrawX, gCursorDrawY, DISPLAY_CURSOR_W, DISPLAY_CURSOR_H };
    DISPLAY_MODE mode = { fb->Width, fb->Height, fb->Stride, DisplayPixelFormatX8R8G8B8 };
    if (KiDisplayClipRect(&mode, &r)) {
        for (int row = 0; row < r.H; row++) {
            uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
            uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
            memcpy(dst, src, (size_t)r.W * sizeof(uint32_t));
        }
    }

    gCursorDrawX = -1;
    gCursorDrawY = -1;
    gCursorUpdatePending = false;
}

// Implements the boot Fb Draw Cursor operation.
static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int x, int y) {
    if (!fb || !fb->Framebuffer || !fb->Backbuffer)
        return;

    gCursorDrawX = x;
    gCursorDrawY = y;

    for (int row = 0; row < DISPLAY_CURSOR_H; row++) {
        int fy = y + row;
        if ((unsigned)fy >= fb->Height)
            continue;
        for (int col = 0; col < DISPLAY_CURSOR_W; col++) {
            int fx = x + col;
            if ((unsigned)fx >= fb->Width)
                continue;
            uint32_t src = gDisplayDefaultCursorArgb[row][col];
            if ((src >> 24) == 0)
                continue;
            uint32_t dst = fb->Backbuffer[(size_t)fy * fb->Stride + fx];
            fb->Framebuffer[(size_t)fy * fb->Stride + fx] = KiDisplayBlendPixel(dst, src);
        }
    }
}

// Implements the boot Fb Show Cursor operation.
static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !fb->Framebuffer || !fb->Backbuffer)
        return NTLSTATUS_NOT_SUPPORTED;

    if (show) {
        fb->Stats.CursorShowCount++;
        BootFbDrawCursor(fb, gCursorX, gCursorY);
    } else {
        fb->Stats.CursorHideCount++;
        BootFbRestoreCursor(fb);
    }
    _mm_sfence();
    return NTLSTATUS_SUCCESS;
}

// Implements the boot Fb Query Stats operation.
static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) {
    BOOTFB_CONTEXT* fb = BootFbContext(device);
    if (!fb || !outStats)
        return NTLSTATUS_INVALID_PARAMETER;
    *outStats = fb->Stats;
    return NTLSTATUS_SUCCESS;
}