nt-lite/code/drivers/display/vmwaresvga/vmwaresvga.c

// VMware SVGA-II PCI display provider. This file owns adapter setup, FIFO command submission, framebuffer presentation, and cursor integration.

#include "vmwaresvga.h"

const HAL_API* gVmwSvgaHal;

static DRIVER_OBJECT gVmwSvgaDriver;

static const IO_DEVICE_ID gVmwSvgaIds[] = {
    { IoBusPci, { "PCI\\VEN_15AD&DEV_0405", sizeof("PCI\\VEN_15AD&DEV_0405") - 1 } },
};

static const IO_DRIVER_ID_TABLE gVmwSvgaIdTable = {
    gVmwSvgaIds,
    sizeof(gVmwSvgaIds) / sizeof(gVmwSvgaIds[0]),
};

static KSTATUS VmwSvgaSyncDevice(VMWSVGA_DEVICE* v);
static void VmwSvgaWakeFifo(VMWSVGA_DEVICE* v);

// Reads a 32-bit SVGA register through the indexed I/O port pair.
static uint32_t VmwSvgaReadReg(VMWSVGA_DEVICE* v, uint32_t reg) {
    __outdword((unsigned short)(v->IoBase + SVGA_INDEX_PORT), reg);
    return __indword((unsigned short)(v->IoBase + SVGA_VALUE_PORT));
}

// Writes a 32-bit SVGA register through the indexed I/O port pair.
static void VmwSvgaWriteReg(VMWSVGA_DEVICE* v, uint32_t reg, uint32_t value) {
    __outdword((unsigned short)(v->IoBase + SVGA_INDEX_PORT), reg);
    __outdword((unsigned short)(v->IoBase + SVGA_VALUE_PORT), value);
}

// Negotiates the highest SVGA device ID supported by the adapter.
static uint32_t VmwSvgaProbeId(VMWSVGA_DEVICE* v) {
    for (uint32_t id = SVGA_ID_2; id >= SVGA_ID_0; id--) {
        VmwSvgaWriteReg(v, SVGA_REG_ID, id);
        uint32_t got = VmwSvgaReadReg(v, SVGA_REG_ID);
        if (got == id)
            return id;
        if (id == SVGA_ID_0)
            break;
    }
    return SVGA_ID_INVALID;
}

// Chooses the runtime display mode, preferring the current boot framebuffer size.
static KSTATUS VmwSvgaChooseMode(uint32_t* outWidth, uint32_t* outHeight) {
    if (!outWidth || !outHeight)
        return STATUS_INVALID_PARAMETER;

    DISPLAY_MODE current;
    memset(&current, 0, sizeof(current));
    if (KSUCCESS(KeDisplayGetMode(&current)) &&
        current.Format == DisplayPixelFormatX8R8G8B8 &&
        current.Width > 0 && current.Height > 0) {
        *outWidth = current.Width;
        *outHeight = current.Height;
        return STATUS_SUCCESS;
    }

    *outWidth = 1024;
    *outHeight = 768;
    return STATUS_SUCCESS;
}

// Programs the visible guest display rectangle used by the legacy framebuffer path.
static void VmwSvgaProgramLegacyDisplayTopology(VMWSVGA_DEVICE* v, uint32_t width, uint32_t height, uint32_t caps) {
    if (!v || (caps & SVGA_CAP_DISPLAY_TOPOLOGY) == 0)
        return;

    // Legacy GFB rendering still needs one visible primary display rectangle.
    VmwSvgaWriteReg(v, SVGA_REG_NUM_GUEST_DISPLAYS, 1u);
    VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_ID, 0u);
    VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_IS_PRIMARY, 1u);
    VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_POSITION_X, 0u);
    VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_POSITION_Y, 0u);
    VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_WIDTH, width);
    VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_HEIGHT, height);
}

// Programs the SVGA device mode before FIFO and framebuffer use.
static KSTATUS VmwSvgaSetMode(VMWSVGA_DEVICE* v) {
    if (!v)
        return STATUS_INVALID_PARAMETER;

    uint32_t desiredWidth = 0;
    uint32_t desiredHeight = 0;
    KSTATUS st = VmwSvgaChooseMode(&desiredWidth, &desiredHeight);
    if (!KSUCCESS(st))
        return st;

    uint32_t maxWidth = VmwSvgaReadReg(v, SVGA_REG_MAX_WIDTH);
    uint32_t maxHeight = VmwSvgaReadReg(v, SVGA_REG_MAX_HEIGHT);
    uint32_t caps = VmwSvgaReadReg(v, SVGA_REG_CAPABILITIES);
    if (maxWidth && desiredWidth > maxWidth)
        desiredWidth = maxWidth;
    if (maxHeight && desiredHeight > maxHeight)
        desiredHeight = maxHeight;
    if (desiredWidth == 0 || desiredHeight == 0)
        return STATUS_DEVICE_ERROR;

    VmwSvgaWriteReg(v, SVGA_REG_CONFIG_DONE, 0u);
    VmwSvgaWriteReg(v, SVGA_REG_ENABLE, 0u);

    VmwSvgaWriteReg(v, SVGA_REG_WIDTH, desiredWidth);
    VmwSvgaWriteReg(v, SVGA_REG_HEIGHT, desiredHeight);
    VmwSvgaWriteReg(v, SVGA_REG_BITS_PER_PIXEL, 32u);
    if (caps & SVGA_CAP_PITCHLOCK)
        VmwSvgaWriteReg(v, SVGA_REG_PITCHLOCK, desiredWidth * sizeof(uint32_t));

#if !VMWSVGA_ENABLE_SCREEN_OBJECTS
    VmwSvgaProgramLegacyDisplayTopology(v, desiredWidth, desiredHeight, caps);
#endif

    VmwSvgaWriteReg(v, SVGA_REG_ENABLE, 1u);

    DbgPrint("VMWSVGA: requested mode %ux%u max=%ux%u caps=0x%x\n",
             desiredWidth, desiredHeight, maxWidth, maxHeight, caps);
    return STATUS_SUCCESS;
}

// Clips a display rectangle to the current framebuffer bounds.
static bool VmwSvgaClipRect(VMWSVGA_DEVICE* v, DISPLAY_RECT* r) {
    if (!v || !r || r->W <= 0 || r->H <= 0)
        return false;

    int64_t x0 = r->X;
    int64_t y0 = r->Y;
    int64_t x1 = x0 + r->W;
    int64_t y1 = y0 + r->H;

    if (x0 < 0) x0 = 0;
    if (y0 < 0) y0 = 0;
    if (x1 > (int64_t)v->Width) x1 = v->Width;
    if (y1 > (int64_t)v->Height) y1 = v->Height;
    if (x0 >= x1 || y0 >= y1)
        return false;

    r->X = (int32_t)x0;
    r->Y = (int32_t)y0;
    r->W = (int32_t)(x1 - x0);
    r->H = (int32_t)(y1 - y0);
    return true;
}

// Returns the next FIFO dword offset, wrapping at FIFO_MAX.
static uint32_t VmwSvgaFifoNext(VMWSVGA_DEVICE* v, uint32_t offset) {
    uint32_t next = offset + sizeof(uint32_t);
    uint32_t min = v->Fifo[SVGA_FIFO_MIN];
    uint32_t max = v->Fifo[SVGA_FIFO_MAX];
    if (next >= max)
        next = min;
    return next;
}

// Writes one command dword into the SVGA FIFO and waits for space if needed.
static KSTATUS VmwSvgaFifoWrite(VMWSVGA_DEVICE* v, uint32_t value) {
    if (!v || !v->FifoReady || !v->Fifo)
        return STATUS_NOT_SUPPORTED;

    uint32_t min = v->Fifo[SVGA_FIFO_MIN];
    uint32_t max = v->Fifo[SVGA_FIFO_MAX];
    if (min < SVGA_FIFO_NUM_LEGACY_REGS * sizeof(uint32_t) || max > v->FifoBytes || min >= max)
        return STATUS_DEVICE_ERROR;

    for (uint32_t guard = 0; guard < VMWSVGA_MAX_WAIT_ITERS; guard++) {
        uint32_t next = v->Fifo[SVGA_FIFO_NEXT_CMD];
        uint32_t stop = v->Fifo[SVGA_FIFO_STOP];
        if (next < min || next >= max || stop < min || stop >= max)
            return STATUS_DEVICE_ERROR;

        uint32_t nextAfter = VmwSvgaFifoNext(v, next);
        if (nextAfter != stop) {
            v->Fifo[next / sizeof(uint32_t)] = value;
            _ReadWriteBarrier();
            v->Fifo[SVGA_FIFO_NEXT_CMD] = nextAfter;
            _ReadWriteBarrier();
            return STATUS_SUCCESS;
        }

        KSTATUS syncSt = VmwSvgaSyncDevice(v);
        if (!KSUCCESS(syncSt))
            return syncSt;
    }

    return STATUS_TIMEOUT;
}

// Checks whether an extended FIFO register is valid for this device.
static bool VmwSvgaHasFifoRegister(VMWSVGA_DEVICE* v, uint32_t index) {
    if (!v || !v->Fifo || !v->FifoReady)
        return false;
    return v->FifoRegCount > index &&
           v->Fifo[SVGA_FIFO_MIN] >= (index + 1u) * sizeof(uint32_t);
}

// Wakes the host side of the SVGA FIFO after new commands are queued.
static void VmwSvgaWakeFifo(VMWSVGA_DEVICE* v) {
    if (!v || !v->FifoReady)
        return;

    _ReadWriteBarrier();

    if (VmwSvgaHasFifoRegister(v, SVGA_FIFO_BUSY)) {
        if (v->Fifo[SVGA_FIFO_BUSY])
            return;
        v->Fifo[SVGA_FIFO_BUSY] = 1u;
        _ReadWriteBarrier();
    }

    VmwSvgaWriteReg(v, SVGA_REG_SYNC, 1u);
}

static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels);

// Implements the premultiply Argb For Bgra operation.
static uint32_t VmwSvgaPremultiplyArgbForBgra(uint32_t argb) {
    uint32_t a = argb >> 24;
    if (a == 0)
        return 0u;
    if (a == 255u)
        return argb;

    uint32_t r = (argb >> 16) & 0xFFu;
    uint32_t g = (argb >> 8) & 0xFFu;
    uint32_t b = argb & 0xFFu;

    r = (r * a + 127u) / 255u;
    g = (g * a + 127u) / 255u;
    b = (b * a + 127u) / 255u;
    return (a << 24) | (r << 16) | (g << 8) | b;
}

// Implements the cursor Commit operation.
static void VmwSvgaCursorCommit(VMWSVGA_DEVICE* v) {
    if (!v || !v->HardwareCursor || !v->CursorImageDefined)
        return;

    if (v->CursorBypass3 &&
        VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_ON) &&
        VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_X) &&
        // Checks whether an extended FIFO register is valid for this device.
        VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_Y)) {
        v->Fifo[SVGA_FIFO_CURSOR_X] = (uint32_t)v->CursorX;
        v->Fifo[SVGA_FIFO_CURSOR_Y] = (uint32_t)v->CursorY;
        if (VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_SCREEN_ID))
            v->Fifo[SVGA_FIFO_CURSOR_SCREEN_ID] = SVGA_ID_INVALID;
        v->Fifo[SVGA_FIFO_CURSOR_ON] = v->CursorVisible ? SVGA_CURSOR_ON_SHOW : SVGA_CURSOR_ON_HIDE;
        _ReadWriteBarrier();
        if (VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_COUNT))
            v->Fifo[SVGA_FIFO_CURSOR_COUNT]++;
        _ReadWriteBarrier();
        VmwSvgaWakeFifo(v);
        return;
    }

    // Legacy QEMU SVGA cursors are frontend/host cursors.  Re-defining the
    // cursor image from the input path can stall the FIFO/frontend after a
    // GTK focus/grab transition, so cursor activity must only update the
    // legacy position/visibility registers.  The image is defined once when
    // SetCursorImage runs.

    VmwSvgaWriteReg(v, SVGA_REG_CURSOR_ID, 0u);
    VmwSvgaWriteReg(v, SVGA_REG_CURSOR_X, (uint32_t)v->CursorX);
    VmwSvgaWriteReg(v, SVGA_REG_CURSOR_Y, (uint32_t)v->CursorY);
    VmwSvgaWriteReg(v, SVGA_REG_CURSOR_ON, v->CursorVisible ? SVGA_CURSOR_ON_SHOW : SVGA_CURSOR_ON_HIDE);
}

// Implements the cursor Set Visible operation.
static void VmwSvgaCursorSetVisible(VMWSVGA_DEVICE* v, bool visible) {
    if (!v || !v->HardwareCursor)
        return;
    v->CursorVisible = visible;
    VmwSvgaCursorCommit(v);
}

// Implements the cursor Set Position operation.
static void VmwSvgaCursorSetPosition(VMWSVGA_DEVICE* v, int x, int y) {
    if (!v || !v->HardwareCursor)
        return;
    v->CursorX = x;
    v->CursorY = y;
    VmwSvgaCursorCommit(v);
}

// Implements the define Alpha Cursor operation.
static KSTATUS VmwSvgaDefineAlphaCursor(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
    KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_ALPHA_CURSOR);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, 0u); // cursor id, reserved
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)hotX);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)hotY);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)width);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)height);
    if (!KSUCCESS(st)) return st;

    for (int y = 0; y < height; y++) {
        for (int x = 0; x < width; x++) {
            uint32_t p = argbPixels[(size_t)y * (size_t)width + (size_t)x];
            st = VmwSvgaFifoWrite(v, VmwSvgaPremultiplyArgbForBgra(p));
            if (!KSUCCESS(st)) return st;
        }
    }

    return VmwSvgaSyncDevice(v);
}

static uint32_t VmwSvgaPackMonoCursorDword(const uint8_t* bits,
                                           int width,
                                           int height,
                                           int y,
                                           uint32_t dwordIndex) {

    // SVGA classic cursor scanlines are padded to 32 bits.  QEMU stores
    // FIFO dwords in guest-endian order and then interprets the result as
    // bytes via cursor_set_mono(), whose first pixel is bit 0x80 of byte 0.
    // Therefore byte 0 of the numeric dword must contain pixels 0..7,
    // byte 1 pixels 8..15, and so on.

    uint8_t bytes[4] = { 0, 0, 0, 0 };
    uint32_t baseX = dwordIndex * 32u;

    for (uint32_t bit = 0; bit < 32u; bit++) {
        uint32_t x = baseX + bit;
        if (y >= 0 && y < height && x < (uint32_t)width) {
            uint8_t v = bits[(size_t)y * (size_t)width + (size_t)x];
            if (v) {
                uint32_t byteIndex = bit >> 3;
                uint8_t bitMask = (uint8_t)(0x80u >> (bit & 7u));
                bytes[byteIndex] |= bitMask;
            }
        }
    }

    return ((uint32_t)bytes[0]) |
           ((uint32_t)bytes[1] << 8) |
           ((uint32_t)bytes[2] << 16) |
           ((uint32_t)bytes[3] << 24);
}

// Implements the classic Cursor Width operation.
static int VmwSvgaClassicCursorWidth(int width) {

    // QEMU's VMware SVGA implementation reads classic cursor masks as
    // SVGA dword-padded scanlines, then passes the raw byte stream to
    // cursor_set_mono(), which advances rows by ceil(width / 8) bytes.
    // For widths that are not a multiple of 32 pixels, those two row
    // strides differ and every other row is decoded from padding bytes.
    // Define the classic cursor with a 32-pixel-aligned visible width and
    // leave the extra pixels transparent.  The real cursor image remains
    // at the left edge and the hotspot is unchanged.

    return (width + 31) & ~31;
}

static void VmwSvgaBuildMonoCursorBits(const uint32_t* argbPixels,
                                       int sourceWidth,
                                       int encodedWidth,
                                       int height,
                                       uint8_t* andBits,
                                       uint8_t* xorBits) {

    // Classic SVGA cursors are AND/XOR masks.  The most compatible path is
    // a 1bpp AND mask plus a 1bpp XOR mask:
    // transparent: AND=1 XOR=0
    // black:       AND=0 XOR=0
    // white:       AND=0 XOR=1
    // This intentionally discards antialiasing, but avoids QEMU/GTK's
    // fragile 32bpp classic host-cursor path.

    for (int y = 0; y < height; y++) {
        for (int x = 0; x < encodedWidth; x++) {
            size_t out = (size_t)y * (size_t)encodedWidth + (size_t)x;
            if (x >= sourceWidth) {
                andBits[out] = 1u;
                xorBits[out] = 0u;
                continue;
            }

            size_t in = (size_t)y * (size_t)sourceWidth + (size_t)x;
            uint32_t p = argbPixels[in];
            uint32_t a = p >> 24;
            uint32_t r = (p >> 16) & 0xFFu;
            uint32_t g = (p >> 8) & 0xFFu;
            uint32_t b = p & 0xFFu;
            if (a < 128u) {
                andBits[out] = 1u;
                xorBits[out] = 0u;
            } else {
                uint32_t luma = r * 30u + g * 59u + b * 11u;
                andBits[out] = 0u;
                xorBits[out] = (luma >= 12800u) ? 1u : 0u;
            }
        }
    }
}

// Implements the define Classic Cursor operation.
static KSTATUS VmwSvgaDefineClassicCursor(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
    int encodedWidth = VmwSvgaClassicCursorWidth(width);
    uint32_t dwordsPerRow = ((uint32_t)encodedWidth + 31u) / 32u;
    size_t pixelCount = (size_t)encodedWidth * (size_t)height;

    if (encodedWidth > 256 || pixelCount > 256u * 256u)
        return STATUS_INVALID_PARAMETER;

    uint8_t* andBits = (uint8_t*)MmAllocatePool(pixelCount);
    uint8_t* xorBits = (uint8_t*)MmAllocatePool(pixelCount);
    if (!andBits || !xorBits) {
        if (andBits) MmFreePool(andBits);
        if (xorBits) MmFreePool(xorBits);
        return STATUS_NO_MEMORY;
    }

    VmwSvgaBuildMonoCursorBits(argbPixels, width, encodedWidth, height, andBits, xorBits);

    DbgPrint("VMWSVGA: classic cursor define source=%dx%d encoded=%dx%d hot=%d,%d\n",
             width, height, encodedWidth, height, hotX, hotY);

    KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_CURSOR);
    if (!KSUCCESS(st)) goto done;
    st = VmwSvgaFifoWrite(v, 0u); // cursor id, reserved
    if (!KSUCCESS(st)) goto done;
    st = VmwSvgaFifoWrite(v, (uint32_t)hotX);
    if (!KSUCCESS(st)) goto done;
    st = VmwSvgaFifoWrite(v, (uint32_t)hotY);
    if (!KSUCCESS(st)) goto done;
    st = VmwSvgaFifoWrite(v, (uint32_t)encodedWidth);
    if (!KSUCCESS(st)) goto done;
    st = VmwSvgaFifoWrite(v, (uint32_t)height);
    if (!KSUCCESS(st)) goto done;
    st = VmwSvgaFifoWrite(v, 1u); // AND mask depth
    if (!KSUCCESS(st)) goto done;
    st = VmwSvgaFifoWrite(v, 1u); // XOR mask depth
    if (!KSUCCESS(st)) goto done;

    for (int y = 0; y < height; y++) {
        for (uint32_t d = 0; d < dwordsPerRow; d++) {
            st = VmwSvgaFifoWrite(v, VmwSvgaPackMonoCursorDword(andBits, encodedWidth, height, y, d));
            if (!KSUCCESS(st)) goto done;
        }
    }

    for (int y = 0; y < height; y++) {
        for (uint32_t d = 0; d < dwordsPerRow; d++) {
            st = VmwSvgaFifoWrite(v, VmwSvgaPackMonoCursorDword(xorBits, encodedWidth, height, y, d));
            if (!KSUCCESS(st)) goto done;
        }
    }

    st = VmwSvgaSyncDevice(v);

done:
    MmFreePool(andBits);
    MmFreePool(xorBits);
    return st;
}

// Implements the upload Cursor Image operation.
static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
    if (!v || !v->HardwareCursor || !argbPixels || width <= 0 || height <= 0)
        return STATUS_INVALID_PARAMETER;
    if (width > 256 || height > 256 || hotX < 0 || hotY < 0 || hotX >= width || hotY >= height)
        return STATUS_INVALID_PARAMETER;

    KSTATUS st;
    if (v->AlphaCursor)
        st = VmwSvgaDefineAlphaCursor(v, width, height, hotX, hotY, argbPixels);
    else
        st = VmwSvgaDefineClassicCursor(v, width, height, hotX, hotY, argbPixels);

    if (KSUCCESS(st))
        VmwSvgaWriteReg(v, SVGA_REG_CURSOR_ID, 0u);
    return st;
}

// Implements the define Cursor Image operation.
static KSTATUS VmwSvgaDefineCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
    if (!v || !v->HardwareCursor || !argbPixels || width <= 0 || height <= 0)
        return STATUS_INVALID_PARAMETER;
    if (width > 256 || height > 256 || hotX < 0 || hotY < 0 || hotX >= width || hotY >= height)
        return STATUS_INVALID_PARAMETER;

    size_t bytes = (size_t)width * (size_t)height * sizeof(uint32_t);
    uint32_t* copy = (uint32_t*)MmAllocatePool(bytes);
    if (!copy)
        return STATUS_NO_MEMORY;
    memcpy(copy, argbPixels, bytes);

    KSTATUS st = VmwSvgaUploadCursorImage(v, width, height, hotX, hotY, copy);
    if (!KSUCCESS(st)) {
        MmFreePool(copy);
        return st;
    }

    if (v->CursorImageArgb)
        MmFreePool(v->CursorImageArgb);
    v->CursorImageArgb = copy;
    v->CursorImageBytes = bytes;
    v->CursorImageWidth = width;
    v->CursorImageHeight = height;
    v->CursorHotX = hotX;
    v->CursorHotY = hotY;
    v->CursorImageDefined = true;
    return STATUS_SUCCESS;
}


// Implements the align Up32 operation.
static uint32_t VmwSvgaAlignUp32(uint32_t value, uint32_t alignment) {
    return (value + alignment - 1u) & ~(alignment - 1u);
}

// Implements the define Screen Object operation.
static KSTATUS VmwSvgaDefineScreenObject(VMWSVGA_DEVICE* v) {
    if (!v || !v->FifoReady || !v->UseScreenObject)
        return STATUS_INVALID_PARAMETER;

    KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_SCREEN);
    if (!KSUCCESS(st)) return st;

    // SVGAScreenObject, structSize = 11 dwords with backingStore+cloneCount.
    st = VmwSvgaFifoWrite(v, 44u);  // structSize
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->ScreenId);  // id
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, SVGA_SCREEN_MUST_BE_SET | SVGA_SCREEN_IS_PRIMARY);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->Width);  // size.width
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->Height);  // size.height
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, 0u);  // root.x
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, 0u);  // root.y
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, SVGA_GMR_FRAMEBUFFER);  // backingStore.ptr.gmrId
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->ScreenBackingOffset);  // backingStore.ptr.offset
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->BytesPerLine);  // backingStore.pitch
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, 0u);  // cloneCount
    if (!KSUCCESS(st)) return st;

    return VmwSvgaSyncDevice(v);
}

// Implements the define Gmrfb operation.
static KSTATUS VmwSvgaDefineGmrfb(VMWSVGA_DEVICE* v) {
    if (!v || !v->FifoReady || !v->UseScreenObject)
        return STATUS_INVALID_PARAMETER;

    KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_GMRFB);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, SVGA_GMR_FRAMEBUFFER);  // ptr.gmrId
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->ScreenSourceOffset);  // ptr.offset
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->BytesPerLine);  // bytesPerLine
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, SVGA_GMRFB_FORMAT_32BPP_24DEPTH);
    if (!KSUCCESS(st)) return st;
    return STATUS_SUCCESS;
}

// Blits gmrfb To Screen.
static KSTATUS VmwSvgaBlitGmrfbToScreen(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
    DISPLAY_RECT r = { x, y, w, h };
    if (!VmwSvgaClipRect(v, &r))
        return STATUS_SUCCESS;

    KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_BLIT_GMRFB_TO_SCREEN);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.X);  // srcOrigin.x
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.Y);  // srcOrigin.y
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.X);  // destRect.left
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.Y);  // destRect.top
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)(r.X + r.W));  // destRect.right
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)(r.Y + r.H));  // destRect.bottom
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, v->ScreenId);  // destScreenId
    if (!KSUCCESS(st)) return st;

    VmwSvgaWakeFifo(v);
    return STATUS_SUCCESS;
}

// Initializes screen Object.
static KSTATUS VmwSvgaInitializeScreenObject(VMWSVGA_DEVICE* v) {
    if (!v || !v->FifoReady)
        return STATUS_INVALID_PARAMETER;

    DbgPrint("VMWSVGA: screen object probe caps=0x%x fifoCaps=0x%x\n", v->Capabilities, v->FifoCaps);

    bool hasScreenObject1 = (v->FifoCaps & SVGA_FIFO_CAP_SCREEN_OBJECT) != 0;
    bool hasScreenObject2 = ((v->FifoCaps & SVGA_FIFO_CAP_SCREEN_OBJECT_2) != 0) &&
                            ((v->Capabilities & SVGA_CAP_SCREEN_OBJECT_2) != 0);

    // Real VMware can advertise SCREEN_OBJECT_2 without also setting the
    // original FIFO_CAP_SCREEN_OBJECT bit.  The command set is still the
    // Screen Object command set: DEFINE_SCREEN, DEFINE_GMRFB, and
    // BLIT_GMRFB_TO_SCREEN.  Treat either capability generation as usable.

    if (!hasScreenObject1 && !hasScreenObject2) {
        DbgPrint("VMWSVGA: screen object unavailable: screenObject1=%u screenObject2=%u caps=0x%x fifoCaps=0x%x\n",
                 hasScreenObject1 ? 1u : 0u,
                 hasScreenObject2 ? 1u : 0u,
                 v->Capabilities,
                 v->FifoCaps);
        return STATUS_NOT_SUPPORTED;
    }

    DbgPrint("VMWSVGA: using screen object generation %u\n", hasScreenObject2 ? 2u : 1u);

    uint32_t sourceBytes = (uint32_t)VmwSvgaAlignUp32((uint32_t)v->BackbufferBytes, 4096u);
    uint32_t sourceOffset = VmwSvgaAlignUp32(v->FramebufferOffset, 4096u);
    uint32_t backingOffset = VmwSvgaAlignUp32(sourceOffset + sourceBytes, 4096u);
    uint32_t backingBytes = sourceBytes;

    if ((uint64_t)sourceOffset + sourceBytes > v->FramebufferBytes ||
        (uint64_t)backingOffset + backingBytes > v->FramebufferBytes) {
        DbgPrint("VMWSVGA: screen object disabled; insufficient GFB source=0x%x backing=0x%x fbLen=0x%llx\n",
                 sourceBytes, backingBytes, (unsigned long long)v->FramebufferBytes);
        return STATUS_NOT_SUPPORTED;
    }

    v->ScreenId = SVGA_SCREEN_OBJECT_ID_PRIMARY;
    v->ScreenSourceOffset = sourceOffset;
    v->ScreenSourceBytes = sourceBytes;
    v->ScreenBackingOffset = backingOffset;
    v->ScreenBackingBytes = backingBytes;
    v->FramebufferOffset = sourceOffset;
    v->Framebuffer = v->FramebufferMapping + sourceOffset;
    v->UseScreenObject = true;

    KSTATUS st = VmwSvgaDefineGmrfb(v);
    if (!KSUCCESS(st)) {
        v->UseScreenObject = false;
        return st;
    }
    st = VmwSvgaDefineScreenObject(v);
    if (!KSUCCESS(st)) {
        v->UseScreenObject = false;
        return st;
    }

    DbgPrint("VMWSVGA: screen object enabled id=%u sourceOff=0x%x backingOff=0x%x bytes=0x%x\n",
             v->ScreenId, v->ScreenSourceOffset, v->ScreenBackingOffset, v->ScreenSourceBytes);
    return STATUS_SUCCESS;
}

// Implements the legacy Update operation.
static KSTATUS VmwSvgaLegacyUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
    DISPLAY_RECT r = { x, y, w, h };
    if (!VmwSvgaClipRect(v, &r))
        return STATUS_SUCCESS;

    KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_UPDATE);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.X);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.Y);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.W);
    if (!KSUCCESS(st)) return st;
    st = VmwSvgaFifoWrite(v, (uint32_t)r.H);
    if (!KSUCCESS(st)) return st;

    VmwSvgaWakeFifo(v);
    return STATUS_SUCCESS;
}

// Updates the requested value.
static KSTATUS VmwSvgaUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
    DISPLAY_RECT r = { x, y, w, h };
    if (!VmwSvgaClipRect(v, &r))
        return STATUS_SUCCESS;

    v->Stats.DirtyRectCount++;

    if (v->UseScreenObject) {
        KSTATUS st = VmwSvgaBlitGmrfbToScreen(v, r.X, r.Y, r.W, r.H);
        KSTATUS up = VmwSvgaLegacyUpdate(v, r.X, r.Y, r.W, r.H);
        return KSUCCESS(st) ? up : st;
    }

    return VmwSvgaLegacyUpdate(v, r.X, r.Y, r.W, r.H);
}

// Waits until the SVGA host has consumed the queued FIFO work.
static KSTATUS VmwSvgaSyncDevice(VMWSVGA_DEVICE* v) {
    if (!v)
        return STATUS_INVALID_PARAMETER;
    v->Stats.SyncCount++;
    VmwSvgaWriteReg(v, SVGA_REG_SYNC, 1u);
    for (uint32_t guard = 0; guard < VMWSVGA_MAX_WAIT_ITERS; guard++) {
        if (VmwSvgaReadReg(v, SVGA_REG_BUSY) == 0) {
            if (VmwSvgaHasFifoRegister(v, SVGA_FIFO_BUSY))
                v->Fifo[SVGA_FIFO_BUSY] = 0u;
            return STATUS_SUCCESS;
        }
    }
    return STATUS_TIMEOUT;
}

// Implements the blend Pixel operation.
static uint32_t VmwSvgaBlendPixel(uint32_t dst, uint32_t src) {
    uint8_t a = (uint8_t)(src >> 24);
    if (a == 0) return dst;
    if (a == 255) return src & 0x00FFFFFFu;

    uint8_t sr = (uint8_t)(src >> 16);
    uint8_t sg = (uint8_t)(src >> 8);
    uint8_t sb = (uint8_t)src;
    uint8_t dr = (uint8_t)(dst >> 16);
    uint8_t dg = (uint8_t)(dst >> 8);
    uint8_t db = (uint8_t)dst;
    uint8_t na = (uint8_t)(255 - a);
    uint8_t rr = (uint8_t)((a * sr + na * dr) >> 8);
    uint8_t gg = (uint8_t)((a * sg + na * dg) >> 8);
    uint8_t bb = (uint8_t)((a * sb + na * db) >> 8);
    return ((uint32_t)rr << 16) | ((uint32_t)gg << 8) | bb;
}

// Implements the from Display operation.
static VMWSVGA_DEVICE* VmwSvgaFromDisplay(DISPLAY_DEVICE* device) {
    return device ? (VMWSVGA_DEVICE*)device->DriverContext : NULL;
}

// Implements the display Get Mode operation.
static KSTATUS VmwSvgaDisplayGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !outMode || !v->Framebuffer)
        return STATUS_INVALID_PARAMETER;
    outMode->Width = v->Width;
    outMode->Height = v->Height;
    outMode->Stride = v->Stride;
    outMode->Format = DisplayPixelFormatX8R8G8B8;
    return STATUS_SUCCESS;
}

// Implements the display Blit operation.
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0)
        return STATUS_INVALID_PARAMETER;

    DISPLAY_RECT r = { x, y, w, h };
    if (!VmwSvgaClipRect(v, &r))
        return STATUS_SUCCESS;

    v->Stats.BlitCount++;
    v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);

    int srcOffsetX = r.X - x;
    int srcOffsetY = r.Y - y;
    size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t);
    for (int row = 0; row < r.H; row++) {
        uint32_t* dst = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
        const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX;
        memcpy(dst, src, bytesPerRow);
    }
    return STATUS_SUCCESS;
}

// Implements the display Fill Rect operation.
static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !v->Backbuffer || w <= 0 || h <= 0)
        return STATUS_INVALID_PARAMETER;

    DISPLAY_RECT r = { x, y, w, h };
    if (!VmwSvgaClipRect(v, &r))
        return STATUS_SUCCESS;

    v->Stats.FillRectCount++;
    v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);

    for (int yy = r.Y; yy < r.Y + r.H; yy++) {
        uint32_t* dst = v->Backbuffer + ((size_t)yy * v->Stride) + r.X;
        for (int xx = 0; xx < r.W; xx++)
            *dst++ = color;
    }
    return STATUS_SUCCESS;
}

// Implements the display Copy Rect operation.
static KSTATUS VmwSvgaDisplayCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !v->Backbuffer || w <= 0 || h <= 0)
        return STATUS_INVALID_PARAMETER;

    if (srcX < 0) { int d = -srcX; srcX = 0; dstX += d; w -= d; }
    if (srcY < 0) { int d = -srcY; srcY = 0; dstY += d; h -= d; }
    if (dstX < 0) { int d = -dstX; dstX = 0; srcX += d; w -= d; }
    if (dstY < 0) { int d = -dstY; dstY = 0; srcY += d; h -= d; }
    if (srcX + w > (int)v->Width) w = (int)v->Width - srcX;
    if (dstX + w > (int)v->Width) w = (int)v->Width - dstX;
    if (srcY + h > (int)v->Height) h = (int)v->Height - srcY;
    if (dstY + h > (int)v->Height) h = (int)v->Height - dstY;
    if (w <= 0 || h <= 0)
        return STATUS_SUCCESS;

    v->Stats.CopyRectCount++;
    v->Stats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);

    size_t rowBytes = (size_t)w * sizeof(uint32_t);
    if (dstY > srcY) {
        for (int row = h - 1; row >= 0; row--) {
            uint32_t* dst = v->Backbuffer + ((size_t)(dstY + row) * v->Stride) + dstX;
            uint32_t* src = v->Backbuffer + ((size_t)(srcY + row) * v->Stride) + srcX;
            memmove(dst, src, rowBytes);
        }
    } else {
        for (int row = 0; row < h; row++) {
            uint32_t* dst = v->Backbuffer + ((size_t)(dstY + row) * v->Stride) + dstX;
            uint32_t* src = v->Backbuffer + ((size_t)(srcY + row) * v->Stride) + srcX;
            memmove(dst, src, rowBytes);
        }
    }
    return STATUS_SUCCESS;
}

// Implements the display Read Rect operation.
static KSTATUS VmwSvgaDisplayReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !v->Backbuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
        return STATUS_INVALID_PARAMETER;

    DISPLAY_RECT r = { x, y, w, h };
    if (!VmwSvgaClipRect(v, &r))
        return STATUS_SUCCESS;

    v->Stats.ReadRectCount++;
    v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);

    int dstOffsetX = r.X - x;
    int dstOffsetY = r.Y - y;
    for (int row = 0; row < r.H; row++) {
        const uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
        uint32_t* dst = outPixels + ((size_t)(dstOffsetY + row) * outStride) + dstOffsetX;
        memcpy(dst, src, (size_t)r.W * sizeof(uint32_t));
    }
    return STATUS_SUCCESS;
}

// Implements the display Present operation.
static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !v->Framebuffer || !v->Backbuffer)
        return STATUS_INVALID_PARAMETER;
    if (!rects || rectCount == 0)
        return STATUS_SUCCESS;

    v->Stats.PresentCount++;
    v->Stats.PresentRectCount += rectCount;

    KSTATUS result = STATUS_SUCCESS;
    for (size_t i = 0; i < rectCount; i++) {
        DISPLAY_RECT r = rects[i];
        if (!VmwSvgaClipRect(v, &r))
            continue;
        v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
        size_t rowBytes = (size_t)r.W * sizeof(uint32_t);
        for (int row = 0; row < r.H; row++) {
            volatile uint8_t* dstBytes = v->Framebuffer + ((size_t)(r.Y + row) * v->BytesPerLine) + ((size_t)r.X * sizeof(uint32_t));
            uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
            memcpy((void*)dstBytes, src, rowBytes);
        }
        _ReadWriteBarrier();

        KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H);
        if (!KSUCCESS(st))
            result = st;
    }

    if (KSUCCESS(result)) {
        KSTATUS syncSt = VmwSvgaSyncDevice(v);
        if (!KSUCCESS(syncSt))
            result = syncSt;
    }

    return result;
}

// Implements the display Sync operation.
static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    return VmwSvgaSyncDevice(v);
}

// Implements the display Set Cursor Image operation.
static KSTATUS VmwSvgaDisplaySetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !v->HardwareCursor)
        return STATUS_NOT_SUPPORTED;
    return VmwSvgaDefineCursorImage(v, width, height, hotX, hotY, argbPixels);
}

// Implements the display Set Cursor Position operation.
static KSTATUS VmwSvgaDisplaySetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v)
        return STATUS_INVALID_PARAMETER;
    v->CursorX = x;
    v->CursorY = y;
    v->Stats.CursorMoveCount++;
    if (v->HardwareCursor && v->CursorImageDefined)
        VmwSvgaCursorSetPosition(v, x, y);
    return STATUS_SUCCESS;
}

// Implements the restore Cursor operation.
static void VmwSvgaRestoreCursor(VMWSVGA_DEVICE* v) {
    if (!v || !v->CursorDrawn || !v->Backbuffer || !v->Framebuffer)
        return;

    DISPLAY_RECT r = { v->CursorDrawX, v->CursorDrawY, VMWSVGA_CURSOR_W, VMWSVGA_CURSOR_H };
    if (VmwSvgaClipRect(v, &r)) {
        for (int row = 0; row < r.H; row++) {
            volatile uint8_t* dstBytes = v->Framebuffer + ((size_t)(r.Y + row) * v->BytesPerLine) + ((size_t)r.X * sizeof(uint32_t));
            uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
            memcpy((void*)dstBytes, src, (size_t)r.W * sizeof(uint32_t));
        }
        (void)VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H);
    }

    v->CursorDrawn = false;
    v->CursorDrawX = -1;
    v->CursorDrawY = -1;
}

// Draws cursor.
static void VmwSvgaDrawCursor(VMWSVGA_DEVICE* v) {
    if (!v || !v->Backbuffer || !v->Framebuffer)
        return;

    int x = v->CursorX;
    int y = v->CursorY;
    v->CursorDrawX = x;
    v->CursorDrawY = y;

    DISPLAY_RECT dirty = { x, y, VMWSVGA_CURSOR_W, VMWSVGA_CURSOR_H };
    for (int row = 0; row < VMWSVGA_CURSOR_H; row++) {
        int fy = y + row;
        if ((unsigned)fy >= v->Height)
            continue;
        for (int col = 0; col < VMWSVGA_CURSOR_W; col++) {
            int fx = x + col;
            if ((unsigned)fx >= v->Width)
                continue;
            uint32_t src = gDisplayDefaultCursorArgb[row][col];
            if ((src >> 24) == 0)
                continue;
            uint32_t dst = v->Backbuffer[(size_t)fy * v->Stride + fx];
            uint32_t pixel = VmwSvgaBlendPixel(dst, src);
            volatile uint32_t* fb = (volatile uint32_t*)(v->Framebuffer + ((size_t)fy * v->BytesPerLine) + ((size_t)fx * sizeof(uint32_t)));
            *fb = pixel;
        }
    }
    v->CursorDrawn = true;
    if (VmwSvgaClipRect(v, &dirty))
        (void)VmwSvgaUpdate(v, dirty.X, dirty.Y, dirty.W, dirty.H);
}

// Implements the display Show Cursor operation.
static KSTATUS VmwSvgaDisplayShowCursor(DISPLAY_DEVICE* device, bool show) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v)
        return STATUS_INVALID_PARAMETER;

    if (v->HardwareCursor) {
        if (!v->CursorImageDefined)
            return STATUS_NOT_SUPPORTED;
        if (show) {
            v->Stats.CursorShowCount++;
            VmwSvgaCursorSetPosition(v, v->CursorX, v->CursorY);
            VmwSvgaCursorSetVisible(v, true);
        } else {
            v->Stats.CursorHideCount++;
            VmwSvgaCursorSetVisible(v, false);
        }
        return STATUS_SUCCESS;
    }

    if (!v->Framebuffer || !v->Backbuffer)
        return STATUS_NOT_SUPPORTED;

    if (show) {
        v->Stats.CursorShowCount++;
        if (v->CursorDrawn)
            VmwSvgaRestoreCursor(v);
        VmwSvgaDrawCursor(v);
    } else {
        v->Stats.CursorHideCount++;
        VmwSvgaRestoreCursor(v);
    }
    return STATUS_SUCCESS;
}

// Implements the display Query Stats operation.
static KSTATUS VmwSvgaDisplayQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) {
    VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
    if (!v || !outStats)
        return STATUS_INVALID_PARAMETER;
    *outStats = v->Stats;
    return STATUS_SUCCESS;
}

static const DISPLAY_DEVICE_OPS gVmwSvgaDisplayOps = {
    .GetMode = VmwSvgaDisplayGetMode,
    .Blit = VmwSvgaDisplayBlit,
    .FillRect = VmwSvgaDisplayFillRect,
    .CopyRect = VmwSvgaDisplayCopyRect,
    .ReadRect = VmwSvgaDisplayReadRect,
    .Present = VmwSvgaDisplayPresent,
    .Sync = VmwSvgaDisplaySync,
    .SetCursorImage = VmwSvgaDisplaySetCursorImage,
    .SetCursorPosition = VmwSvgaDisplaySetCursorPosition,
    .ShowCursor = VmwSvgaDisplayShowCursor,
    .QueryStats = VmwSvgaDisplayQueryStats,
};

// Implements the select Bars operation.
static KSTATUS VmwSvgaSelectBars(VMWSVGA_DEVICE* v) {
    PCI_BAR bar;
    memset(&bar, 0, sizeof(bar));

    KSTATUS st = PciGetBar(v->Pdo, 0, &bar);
    if (!KSUCCESS(st) || bar.Type != PciBarIo || bar.Base > 0xFFFFu) {
        DbgPrint("VMWSVGA: BAR0 is not a usable I/O register BAR st=0x%x type=%u base=0x%llx\n",
                 st, bar.Type, (unsigned long long)bar.Base);
        return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
    }
    v->IoBar = bar;
    v->IoBase = (uint16_t)bar.Base;

    st = PciGetBar(v->Pdo, 1, &bar);
    if (!KSUCCESS(st) || (bar.Type != PciBarMemory32 && bar.Type != PciBarMemory64)) {
        DbgPrint("VMWSVGA: BAR1 is not a usable framebuffer BAR st=0x%x type=%u\n", st, bar.Type);
        return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
    }
    v->FramebufferBar = bar;

    st = PciGetBar(v->Pdo, 2, &bar);
    if (!KSUCCESS(st) || (bar.Type != PciBarMemory32 && bar.Type != PciBarMemory64)) {
        DbgPrint("VMWSVGA: BAR2 is not a usable FIFO BAR st=0x%x type=%u\n", st, bar.Type);
        return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
    }
    v->FifoBar = bar;
    return STATUS_SUCCESS;
}

// Implements the mmio Page Flags operation.
static uint64_t VmwSvgaMmioPageFlags(void) {
    return PAGE_PRESENT | PAGE_WRITE | PAGE_WRITETHROUGH | PAGE_NOCACHE | PAGE_NX;
}

// Maps bars.
static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) {
    uint64_t mmioFlags = VmwSvgaMmioPageFlags();
    KSTATUS st = PciMapBar(v->Pdo, 1, mmioFlags,
                           (void**)&v->FramebufferMapping, &v->FramebufferBytes);
    if (!KSUCCESS(st) || !v->FramebufferMapping) {
        DbgPrint("VMWSVGA: PciMapBar framebuffer failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n",
                 st,
                 (unsigned long long)(uintptr_t)v->FramebufferMapping,
                 (unsigned long long)v->FramebufferBar.Base,
                 (unsigned long long)v->FramebufferBar.Length);
        return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
    }

    st = PciMapBar(v->Pdo, 2, mmioFlags,
                   (void**)&v->Fifo, &v->FifoBytes);
    if (!KSUCCESS(st) || !v->Fifo) {
        DbgPrint("VMWSVGA: PciMapBar FIFO failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n",
                 st,
                 (unsigned long long)(uintptr_t)v->Fifo,
                 (unsigned long long)v->FifoBar.Base,
                 (unsigned long long)v->FifoBar.Length);
        return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
    }

    DbgPrint("VMWSVGA: BARs io=0x%04x fbBase=0x%llx fbLen=0x%llx fbVirt=0x%llx fifoBase=0x%llx fifoLen=0x%llx fifoVirt=0x%llx\n",
             v->IoBase,
             (unsigned long long)v->FramebufferBar.Base,
             (unsigned long long)v->FramebufferBar.Length,
             (unsigned long long)(uintptr_t)v->FramebufferMapping,
             (unsigned long long)v->FifoBar.Base,
             (unsigned long long)v->FifoBar.Length,
             (unsigned long long)(uintptr_t)v->Fifo);
    return STATUS_SUCCESS;
}

// Returns mode.
static KSTATUS VmwSvgaReadMode(VMWSVGA_DEVICE* v) {
    v->Width = VmwSvgaReadReg(v, SVGA_REG_WIDTH);
    v->Height = VmwSvgaReadReg(v, SVGA_REG_HEIGHT);
    v->BytesPerLine = VmwSvgaReadReg(v, SVGA_REG_BYTES_PER_LINE);
    v->BitsPerPixel = VmwSvgaReadReg(v, SVGA_REG_BITS_PER_PIXEL);
    if (v->BitsPerPixel == 0)
        v->BitsPerPixel = VmwSvgaReadReg(v, SVGA_REG_HOST_BITS_PER_PIXEL);

    uint32_t fbSize = VmwSvgaReadReg(v, SVGA_REG_FB_SIZE);
    uint32_t fifoSize = VmwSvgaReadReg(v, SVGA_REG_MEM_SIZE);
    uint32_t fbOffset = VmwSvgaReadReg(v, SVGA_REG_FB_OFFSET);
    uint32_t fbStart = VmwSvgaReadReg(v, SVGA_REG_FB_START);

    v->FifoDeviceBytes = fifoSize;
    v->Capabilities = VmwSvgaReadReg(v, SVGA_REG_CAPABILITIES);

    if (v->Width == 0 || v->Height == 0 || v->BytesPerLine == 0) {
        DbgPrint("VMWSVGA: invalid mode %ux%u pitch=%u\n", v->Width, v->Height, v->BytesPerLine);
        return STATUS_DEVICE_ERROR;
    }
    if (v->BitsPerPixel != 32) {
        DbgPrint("VMWSVGA: unsupported bpp=%u\n", v->BitsPerPixel);
        return STATUS_NOT_SUPPORTED;
    }
    if ((v->BytesPerLine & 3u) != 0) {
        DbgPrint("VMWSVGA: unsupported non-dword pitch=%u\n", v->BytesPerLine);
        return STATUS_NOT_SUPPORTED;
    }

    v->Stride = v->BytesPerLine / sizeof(uint32_t);
    if (v->Stride < v->Width)
        return STATUS_DEVICE_ERROR;

    if (!v->FramebufferMapping)
        return STATUS_DEVICE_ERROR;

    uint64_t needed = (uint64_t)v->BytesPerLine * (uint64_t)v->Height;
    if (fbSize && needed > fbSize)
        return STATUS_DEVICE_ERROR;
    if ((uint64_t)fbOffset > v->FramebufferBytes || needed > v->FramebufferBytes - (uint64_t)fbOffset)
        return STATUS_DEVICE_ERROR;

    v->FramebufferOffset = fbOffset;
    v->Framebuffer = v->FramebufferMapping + fbOffset;

    v->BackbufferBytes = (size_t)needed;
    v->Backbuffer = (uint32_t*)MmAllocatePool(v->BackbufferBytes);
    if (!v->Backbuffer)
        return STATUS_NO_MEMORY;

    memset(v->Backbuffer, 0, v->BackbufferBytes);

    DbgPrint("VMWSVGA: mode %ux%u stride=%u bpp=%u fbSize=0x%x fbOff=0x%x fbStart=0x%x fifoSize=0x%x caps=0x%x\n",
             v->Width,
             v->Height,
             v->Stride,
             v->BitsPerPixel,
             fbSize,
             fbOffset,
             fbStart,
             fifoSize,
             v->Capabilities);
    return STATUS_SUCCESS;
}

// Returns the active FIFO size reported by the device, not the larger PCI BAR aperture.
static uint32_t VmwSvgaGetFifoMaxBytes(VMWSVGA_DEVICE* v) {
    uint64_t limit = v->FifoBytes;
    if (v->FifoDeviceBytes != 0 && (uint64_t)v->FifoDeviceBytes < limit)
        limit = v->FifoDeviceBytes;
    if (limit > 0xFFFFFFFFu)
        limit = 0xFFFFFFFFu;
    return (uint32_t)limit & ~3u;
}

// Initializes fifo.
static KSTATUS VmwSvgaInitializeFifo(VMWSVGA_DEVICE* v) {
    if (!v->Fifo || v->FifoBytes < 4096u)
        return STATUS_DEVICE_ERROR;

    VmwSvgaWriteReg(v, SVGA_REG_CONFIG_DONE, 0u);

    uint32_t regs = VmwSvgaReadReg(v, SVGA_REG_MEM_REGS);
    if (regs < SVGA_FIFO_NUM_LEGACY_REGS || regs > 4096u)
        regs = SVGA_FIFO_NUM_REGS_FALLBACK;

    uint32_t min = regs * sizeof(uint32_t);
    uint32_t max = VmwSvgaGetFifoMaxBytes(v);

    if (max < (SVGA_FIFO_NUM_REGS_FALLBACK * sizeof(uint32_t)) + (10u * 1024u))
        min = SVGA_FIFO_NUM_LEGACY_REGS * sizeof(uint32_t);

    if (max <= min + 32u)
        return STATUS_DEVICE_ERROR;

    v->FifoRegCount = min / sizeof(uint32_t);

    memset((void*)v->Fifo, 0, min);
    v->Fifo[SVGA_FIFO_MIN] = min;
    v->Fifo[SVGA_FIFO_MAX] = max;
    v->Fifo[SVGA_FIFO_NEXT_CMD] = min;
    v->Fifo[SVGA_FIFO_STOP] = min;
    _ReadWriteBarrier();

    VmwSvgaWriteReg(v, SVGA_REG_CONFIG_DONE, 1u);
    v->FifoReady = true;

    if (v->FifoRegCount > SVGA_FIFO_CAPABILITIES)
        v->FifoCaps = v->Fifo[SVGA_FIFO_CAPABILITIES];
    if (VmwSvgaHasFifoRegister(v, SVGA_FIFO_BUSY))
        v->Fifo[SVGA_FIFO_BUSY] = 0u;

    DbgPrint("VMWSVGA: FIFO init regs=%u min=0x%x max=0x%x fifoCaps=0x%x deviceBytes=0x%x barBytes=0x%llx\n",
             v->FifoRegCount,
             min,
             max,
             v->FifoCaps,
             v->FifoDeviceBytes,
             (unsigned long long)v->FifoBytes);
    return STATUS_SUCCESS;
}

// Starts device.
static KSTATUS VmwSvgaStartDevice(VMWSVGA_DEVICE* v) {
    if (!v || v->Magic != VMWSVGA_DEVICE_MAGIC || !v->Fdo || !v->Pdo)
        return STATUS_INVALID_PARAMETER;
    if (v->Started)
        return STATUS_SUCCESS;

    KSTATUS st = PciGetDeviceInfo(v->Pdo, &v->PciInfo);
    if (!KSUCCESS(st))
        return st;

    DbgPrint("VMWSVGA: controller at PCI %02x:%02x.%x vendor=%04x device=%04x\n",
             v->PciInfo.Address.Bus,
             v->PciInfo.Address.Device,
             v->PciInfo.Address.Function,
             v->PciInfo.VendorId,
             v->PciInfo.DeviceId);

    st = VmwSvgaSelectBars(v);
    if (!KSUCCESS(st))
        return st;

    st = PciEnableDevice(v->Pdo, PCI_ENABLE_IO | PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER | PCI_ENABLE_INTX);
    if (!KSUCCESS(st))
        return st;

    v->DeviceId = VmwSvgaProbeId(v);
    if (v->DeviceId == SVGA_ID_INVALID) {
        DbgPrint("VMWSVGA: SVGA ID probe failed\n");
        return STATUS_NOT_SUPPORTED;
    }
    VmwSvgaWriteReg(v, SVGA_REG_GUEST_ID, 0u);

    st = VmwSvgaMapBars(v);
    if (!KSUCCESS(st))
        return st;

    KeDisplaySuspendConsole(true);

    st = VmwSvgaSetMode(v);
    if (!KSUCCESS(st)) {
        KeDisplaySuspendConsole(false);
        return st;
    }

    st = VmwSvgaReadMode(v);
    if (!KSUCCESS(st)) {
        KeDisplaySuspendConsole(false);
        return st;
    }

    st = VmwSvgaInitializeFifo(v);
    if (!KSUCCESS(st)) {
        KeDisplaySuspendConsole(false);
        DbgPrint("VMWSVGA: FIFO init failed st=0x%x\n", st);
        return st;
    }

#if VMWSVGA_ENABLE_SCREEN_OBJECTS
    KSTATUS screenSt = VmwSvgaInitializeScreenObject(v);
    if (!KSUCCESS(screenSt) && screenSt != STATUS_NOT_SUPPORTED)
        DbgPrint("VMWSVGA: screen object init failed st=0x%x; using legacy GFB UPDATE\n", screenSt);
#else
    v->UseScreenObject = false;
    DbgPrint("VMWSVGA: screen objects disabled; using legacy GFB UPDATE path\n");
#endif

    v->CursorDrawX = -1;
    v->CursorDrawY = -1;

    bool cursorCap = (v->Capabilities & SVGA_CAP_CURSOR) != 0;
    v->AlphaCursor = (v->Capabilities & SVGA_CAP_ALPHA_CURSOR) != 0;

    v->HardwareCursor = cursorCap;

    v->CursorBypass3 = v->HardwareCursor &&
                       (v->FifoCaps & SVGA_FIFO_CAP_CURSOR_BYPASS_3) &&
                       VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_ON) &&
                       VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_X) &&
                       VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_Y);

    memset(&v->DisplayDevice, 0, sizeof(v->DisplayDevice));
    v->DisplayDevice.Name = "vmwaresvga";
    v->DisplayDevice.Caps = v->HardwareCursor ? VMWSVGA_CAPS_HW_CURSOR : VMWSVGA_CAPS_MINIMAL;
    v->DisplayDevice.DriverContext = v;
    v->DisplayDevice.Ops = &gVmwSvgaDisplayOps;

    DbgPrint("VMWSVGA: cursor caps cursor=%u alpha=%u bypass3=%u active=%u\n",
             cursorCap ? 1u : 0u,
             v->AlphaCursor ? 1u : 0u,
             v->CursorBypass3 ? 1u : 0u,
             v->HardwareCursor ? 1u : 0u);

    if (v->HardwareCursor) {
        st = VmwSvgaDefineCursorImage(v,
                                      DISPLAY_DEFAULT_CURSOR_W,
                                      DISPLAY_DEFAULT_CURSOR_H,
                                      0,
                                      0,
                                      &gDisplayDefaultCursorArgb[0][0]);
        if (!KSUCCESS(st)) {
            DbgPrint("VMWSVGA: hardware cursor init failed st=0x%x; using software cursor fallback\n", st);
            v->HardwareCursor = false;
            v->AlphaCursor = false;
            v->CursorBypass3 = false;
            v->DisplayDevice.Caps = VMWSVGA_CAPS_MINIMAL;
        } else {
            VmwSvgaCursorSetPosition(v, v->CursorX, v->CursorY);
            VmwSvgaCursorSetVisible(v, false);
            DbgPrint("VMWSVGA: hardware cursor ready alpha=%u bypass3=%u\n",
                     v->AlphaCursor ? 1u : 0u,
                     v->CursorBypass3 ? 1u : 0u);
        }
    }

    st = KeDisplayRegisterDevice(&v->DisplayDevice);
    if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) {
        KeDisplaySuspendConsole(false);
        return st;
    }

    st = KeDisplaySetPrimary(&v->DisplayDevice);
    KeDisplaySuspendConsole(false);
    if (!KSUCCESS(st))
        return st;

    v->Started = true;
    v->Fdo->PnpState = DevicePnpStarted;

    (void)VmwSvgaDisplaySync(&v->DisplayDevice);

    DbgPrint("VMWSVGA: initialized as primary display id=0x%x io=0x%04x\n", v->DeviceId, v->IoBase);
    return STATUS_SUCCESS;
}

// Dispatches pnp.
KSTATUS VmwSvgaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
    if (!device || !irp || !device->DeviceExtension)
        return STATUS_INVALID_PARAMETER;

    VMWSVGA_DEVICE* v = (VMWSVGA_DEVICE*)device->DeviceExtension;
    if (v->Magic != VMWSVGA_DEVICE_MAGIC)
        return STATUS_INVALID_PARAMETER;

    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
    if (!ios || ios->MajorFunction != IRP_MJ_PNP)
        return STATUS_INVALID_PARAMETER;

    switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
        case IRP_MN_START_DEVICE:
            return VmwSvgaStartDevice(v);

        case IRP_MN_QUERY_REMOVE_DEVICE:
        case IRP_MN_CANCEL_REMOVE_DEVICE:
            return STATUS_SUCCESS;

        case IRP_MN_SURPRISE_REMOVAL:
            v->SurpriseRemoved = true;
            device->PnpState = DevicePnpSurpriseRemoved;
            return STATUS_SUCCESS;

        case IRP_MN_REMOVE_DEVICE:
            v->Removed = true;
            device->PnpState = DevicePnpDeleted;
            if (v->Backbuffer) {
                MmFreePool(v->Backbuffer);
                v->Backbuffer = NULL;
            }
            if (v->CursorImageArgb) {
                MmFreePool(v->CursorImageArgb);
                v->CursorImageArgb = NULL;
                v->CursorImageBytes = 0;
            }
            IoDetachDevice(device);
            IoDeleteDevice(device);
            return STATUS_SUCCESS;

        default:
            return IoForwardIrpToLower(device, irp);
    }
}

// Implements the add Device operation.
KSTATUS VmwSvgaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
    if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
        return STATUS_INVALID_PARAMETER;

    DEVICE_OBJECT* fdo = NULL;
    KSTATUS st = IoCreateDevice(driver, sizeof(VMWSVGA_DEVICE), &fdo);
    if (!KSUCCESS(st))
        return st;
    (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("vmwaresvga"));

    VMWSVGA_DEVICE* v = (VMWSVGA_DEVICE*)fdo->DeviceExtension;
    memset(v, 0, sizeof(*v));
    v->Magic = VMWSVGA_DEVICE_MAGIC;
    v->Fdo = fdo;
    v->Pdo = physicalDeviceObject;
    v->CursorDrawX = -1;
    v->CursorDrawY = -1;

    v->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
    if (!v->LowerDevice) {
        IoDeleteDevice(fdo);
        return STATUS_FAILURE;
    }

    DbgPrint("VMWSVGA: FDO attached to %v\n", physicalDeviceObject->Name);
    return STATUS_SUCCESS;
}

// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
    if (!hal)
        return STATUS_INVALID_PARAMETER;

    DbgPrint("VMWSVGA: driver loading...\n");
    gVmwSvgaHal = hal;

    memset(&gVmwSvgaDriver, 0, sizeof(gVmwSvgaDriver));
    IoInitializeDriverObject(&gVmwSvgaDriver, SYS_SV_LITERAL("vmwaresvga"));
    gVmwSvgaDriver.IdTable = &gVmwSvgaIdTable;
    gVmwSvgaDriver.DriverAddDevice = VmwSvgaAddDevice;
    gVmwSvgaDriver.MajorFunction[IRP_MJ_PNP] = VmwSvgaDispatchPnp;

    KSTATUS st = IoRegisterDriverObject(&gVmwSvgaDriver);
    if (!KSUCCESS(st))
        DbgPrint("VMWSVGA: IoRegisterDriverObject failed: 0x%x\n", st);
    return st;
}