nt-lite/code/kernel/io_device.c

// I/O manager. This file owns driver objects, device stacks, PnP dispatch, and the disk namespace.

#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_irp.h>
#include <kernel/io_dma.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
#include <kernel/ke_sched.h>

#define MAX_DRIVER_OBJECTS    64
#define MAX_PDO_OBJECTS       128

// Implements the sv Or Unnamed operation.
static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) {
    return SysSvIsEmpty(name) ? SYS_SV_LITERAL("<unnamed>") : name;
}

// Implements the bus Type Name operation.
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
    switch (busType) {
        case IoBusPci: return "pci";
        case IoBusUsb: return "usb";
        default: return "unknown";
    }
}

// Driver object registry / PDO AddDevice binding.

static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS];
static uint32_t gDriverObjectCount;
static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS];
static uint32_t gPhysicalDeviceCount;

// Implements the driver Matches Pdo operation.
static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
    if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical)
        return false;

    for (size_t d = 0; d < driver->IdTable->Count; d++) {
        const IO_DEVICE_ID* want = &driver->IdTable->Ids[d];
        if (want->BusType != IoBusUnknown && want->BusType != pdo->u.Pdo.BusType)
            continue;
        for (uint8_t p = 0; p < pdo->u.Pdo.IdCount; p++) {
            if (SysSvEqual(want->Id, pdo->u.Pdo.Ids[p].Id))
                return true;
        }
    }
    return false;
}

// Implements the try Add Device For Driver operation.
static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) {
    if (!driver || !pdo || !driver->DriverAddDevice)
        return false;
    if (!IoDriverMatchesPdo(driver, pdo))
        return false;

    KSTATUS st = driver->DriverAddDevice(driver, pdo);
    if (KSUCCESS(st)) {
        DbgPrint("io: AddDevice driver=%v pdo=%v bus=%s\n",
                 IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), IoBusTypeName(pdo->u.Pdo.BusType));

        KSTATUS start = IoSendPnpIrpSync(pdo, IRP_MN_START_DEVICE);
        if (!KSUCCESS(start) && start != NTLSTATUS_NOT_SUPPORTED) {
            DbgPrint("io: START_DEVICE failed driver=%v pdo=%v st=0x%x\n",
                     IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), start);
        }
        return true;
    } else {
        DbgPrint("io: AddDevice failed driver=%v pdo=%v st=0x%x\n",
                 IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), st);
        return true;
    }
}

__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
                                                    SYS_STRING_VIEW name) {
    if (!driver) return;
    memset(driver, 0, sizeof(*driver));
    ObInitializeObjectHeader(&driver->Header, ObjectTypeDriver);
    driver->Name = name;
}

// Registers driver Object.
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
    if (!driver) return NTLSTATUS_INVALID_PARAMETER;
    if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
        return NTLSTATUS_NO_MEMORY;

    if (driver->Header.Type != ObjectTypeDriver ||
        driver->Header.LifetimeState != ObjectAlive) {
        ObInitializeObjectHeader(&driver->Header, ObjectTypeDriver);
    }

    gDriverObjects[gDriverObjectCount++] = driver;
    DbgPrint("io: registered driver object %v\n", IoSvOrUnnamed(driver->Name));

    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++)
        (void)IoTryAddDeviceForDriver(driver, gPhysicalDevices[i]);

    return NTLSTATUS_SUCCESS;
}

// Device object lifetime.

static void IoDeviceDestroy(OBJECT* object) {
    DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
    IoDmaDetachDevice(dev);

    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
        if (gPhysicalDevices[i] == dev) {
            gPhysicalDevices[i] = gPhysicalDevices[gPhysicalDeviceCount - 1u];
            gPhysicalDevices[gPhysicalDeviceCount - 1u] = (DEVICE_OBJECT*)0;
            gPhysicalDeviceCount--;
            break;
        }
    }

    if (dev->LowerDevice)
        IoDetachDevice(dev);

    if (dev->Parent) {
        DEVICE_OBJECT* parent = dev->Parent;
        uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
        DEVICE_OBJECT** link = &parent->FirstChild;
        while (*link) {
            if (*link == dev) {
                *link = dev->NextSibling;
                break;
            }
            link = &(*link)->NextSibling;
        }
        KeReleaseSpinlock(&parent->Lock, flags);
        dev->Parent = (DEVICE_OBJECT*)0;
        dev->NextSibling = (DEVICE_OBJECT*)0;
        IoDereferenceDeviceObject(parent);
    }
}

static KSTATUS IoCreateDeviceInternal(DRIVER_OBJECT* driver,
                                      DEVICE_OBJECT_TYPE type,
                                      size_t extensionSize,
                                      DEVICE_OBJECT** outDevice) {
    if (!driver || !outDevice) return NTLSTATUS_INVALID_PARAMETER;

    size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
    DEVICE_OBJECT* dev = (DEVICE_OBJECT*)ObAllocateObject(ObjectTypeDevice, total);
    if (!dev) return NTLSTATUS_NO_MEMORY;

    dev->Header.Destroy = IoDeviceDestroy;
    dev->DriverObject = driver;
    dev->Type = type;
    dev->PnpState = DevicePnpNotStarted;
    dev->AlignmentRequirement = 1;
    dev->Lock = (SPINLOCK)SPINLOCK_INIT;

    // The per-device union is interpreted differently for PDOs and disk FDOs.
    // Do not write disk defaults into a PDO: u.Disk.DiskIndex aliases the low
    // byte of u.Pdo.BusType, which prevents ID-table AddDevice matching.
    if (type != DeviceObjectPhysical) {
        dev->u.Disk.DiskIndex = 0xFF;
        dev->u.Disk.SectorSize = 512;
    }

    if (extensionSize)
        dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));

    *outDevice = dev;
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
                                             size_t extensionSize,
                                             DEVICE_OBJECT** outDevice) {
    return IoCreateDeviceInternal(driver, DeviceObjectFunction, extensionSize, outDevice);
}

__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
                                                     DEVICE_OBJECT* parent,
                                                     const IO_DEVICE_ID* ids,
                                                     size_t idCount,
                                                     size_t extensionSize,
                                                     DEVICE_OBJECT** outPdo) {
    if (!busDriver || !outPdo) return NTLSTATUS_INVALID_PARAMETER;
    if (idCount > IO_MAX_DEVICE_IDS) return NTLSTATUS_INVALID_PARAMETER;

    DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
    KSTATUS st = IoCreateDeviceInternal(busDriver, DeviceObjectPhysical, extensionSize, &pdo);
    if (!KSUCCESS(st)) return st;

    if (idCount && ids) {
        for (size_t i = 0; i < idCount; i++) {
            if (!SysSvIsValid(ids[i].Id) || ids[i].Id.Length >= IO_MAX_DEVICE_ID_CHARS) {
                IoDeleteDevice(pdo);
                return NTLSTATUS_INVALID_PARAMETER;
            }

            char* dst = pdo->u.Pdo.IdStorage[i];
            if (ids[i].Id.Length)
                memcpy(dst, ids[i].Id.Data, ids[i].Id.Length);
            dst[ids[i].Id.Length] = '\0';

            pdo->u.Pdo.Ids[i].BusType = ids[i].BusType;
            pdo->u.Pdo.Ids[i].Id = SysSvFromParts(dst, ids[i].Id.Length);
            if (pdo->u.Pdo.BusType == IoBusUnknown)
                pdo->u.Pdo.BusType = ids[i].BusType;
        }
        pdo->u.Pdo.IdCount = (uint8_t)idCount;
    }

    if (parent) {
        IoReferenceDeviceObject(parent);
        pdo->Parent = parent;
        uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
        pdo->NextSibling = parent->FirstChild;
        parent->FirstChild = pdo;
        KeReleaseSpinlock(&parent->Lock, flags);
    }

    *outPdo = pdo;
    return NTLSTATUS_SUCCESS;
}

// Registers physical Device.
__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
    if (!pdo || pdo->Type != DeviceObjectPhysical)
        return NTLSTATUS_INVALID_PARAMETER;

    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
        if (gPhysicalDevices[i] == pdo)
            return NTLSTATUS_ALREADY_EXISTS;
    }

    if (gPhysicalDeviceCount >= MAX_PDO_OBJECTS)
        return NTLSTATUS_NO_MEMORY;

    gPhysicalDevices[gPhysicalDeviceCount++] = pdo;

    bool anyMatched = false;
    for (uint32_t i = 0; i < gDriverObjectCount; i++) {
        if (IoTryAddDeviceForDriver(gDriverObjects[i], pdo))
            anyMatched = true;
    }

    if (!anyMatched && pdo->u.Pdo.BusType == IoBusUsb) {
        SYS_STRING_VIEW id = pdo->u.Pdo.IdCount ? pdo->u.Pdo.Ids[0].Id : SYS_SV_LITERAL("<no-id>");
        DbgPrint("io: no function driver for pdo=%v id=%v bus=%s\n",
                 IoSvOrUnnamed(pdo->Name), id, IoBusTypeName(pdo->u.Pdo.BusType));
    }
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
                                               SYS_STRING_VIEW name) {
    if (!device || !SysSvIsValid(name))
        return NTLSTATUS_INVALID_PARAMETER;
    if (name.Length >= IO_MAX_DEVICE_NAME_CHARS)
        return NTLSTATUS_BUFFER_TOO_SMALL;

    if (name.Length)
        memcpy(device->NameStorage, name.Data, name.Length);
    device->NameStorage[name.Length] = '\0';
    device->Name = SysSvFromParts(device->NameStorage, name.Length);
    return NTLSTATUS_SUCCESS;
}

// Implements the reference Device Object operation.
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) {
    if (device) ObReferenceObject(&device->Header);
}

// Implements the dereference Device Object operation.
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
    if (device) ObDereferenceObject(&device->Header);
}

// Releases physical Device Registry Entry.
static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
    if (!device || device->Type != DeviceObjectPhysical)
        return;

    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
        if (gPhysicalDevices[i] == device) {
            for (uint32_t j = i + 1; j < gPhysicalDeviceCount; j++)
                gPhysicalDevices[j - 1] = gPhysicalDevices[j];
            gPhysicalDeviceCount--;
            if (gPhysicalDeviceCount < MAX_PDO_OBJECTS)
                gPhysicalDevices[gPhysicalDeviceCount] = (DEVICE_OBJECT*)0;
            return;
        }
    }
}

// Implements the unlink Device From Parent operation.
static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
    if (!device || !device->Parent)
        return;

    DEVICE_OBJECT* parent = device->Parent;
    uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
    DEVICE_OBJECT** link = &parent->FirstChild;
    while (*link) {
        if (*link == device) {
            *link = device->NextSibling;
            break;
        }
        link = &(*link)->NextSibling;
    }
    device->Parent = (DEVICE_OBJECT*)0;
    device->NextSibling = (DEVICE_OBJECT*)0;
    KeReleaseSpinlock(&parent->Lock, flags);

    IoDereferenceDeviceObject(parent);
}

// Releases device.
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
    if (!device)
        return;

    uintptr_t flags = KeAcquireSpinlock(&device->Lock);
    if (device->DeletePending) {
        KeReleaseSpinlock(&device->Lock, flags);
        return;
    }
    device->DeletePending = true;
    device->PnpState = DevicePnpDeleted;
    KeReleaseSpinlock(&device->Lock, flags);

    IoDmaDetachDevice(device);
    IoUnlinkDeviceFromParent(device);
    IoRemovePhysicalDeviceRegistryEntry(device);
    IoDereferenceDeviceObject(device);
}

// Device stack management.

__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
                                                                 DEVICE_OBJECT* target) {
    if (!source || !target) return (DEVICE_OBJECT*)0;

    DEVICE_OBJECT* top = target;
    while (top->AttachedDevice)
        top = top->AttachedDevice;

    top->AttachedDevice = source;
    source->LowerDevice = top;
    IoReferenceDeviceObject(top);
    return top;
}

// Implements the detach Device operation.
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
    if (!source || !source->LowerDevice) return;

    DEVICE_OBJECT* lower = source->LowerDevice;
    if (lower->AttachedDevice == source)
        lower->AttachedDevice = (DEVICE_OBJECT*)0;
    source->LowerDevice = (DEVICE_OBJECT*)0;
    IoDereferenceDeviceObject(lower);
}

// Returns attached Device Reference.
__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device) {
    if (!device) return (DEVICE_OBJECT*)0;
    DEVICE_OBJECT* top = device;
    while (top->AttachedDevice)
        top = top->AttachedDevice;
    IoReferenceDeviceObject(top);
    return top;
}

// Device-stack helpers / PnP dispatch.

__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
    uint8_t count = 0;
    for (DEVICE_OBJECT* cur = device; cur && count < 0xFEu; cur = cur->LowerDevice)
        count++;
    return count ? count : 1;
}

// Implements the forward Irp To Lower operation.
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) {
    if (!device || !irp || !device->LowerDevice)
        return NTLSTATUS_NOT_SUPPORTED;

    IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
    IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
    if (!cur || !next)
        return NTLSTATUS_INVALID_PARAMETER;

    DEVICE_OBJECT* currentDevice = cur->DeviceObject;
    memcpy(next, cur, sizeof(*next));

    if (currentDevice == device) {
        cur->DeviceObject = (DEVICE_OBJECT*)0;
        _InterlockedDecrement(&device->OutstandingIrps);
        IoDereferenceDeviceObject(device);
    }

    irp->CurrentLocation++;
    return IoCallDriver(device->LowerDevice, irp);
}

__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
                                                IO_PNP_MINOR_FUNCTION minor) {
    if (!device) return NTLSTATUS_INVALID_PARAMETER;

    DEVICE_OBJECT* top = IoGetAttachedDeviceReference(device);
    if (!top) return NTLSTATUS_INVALID_PARAMETER;

    uint8_t stackCount = IoGetDeviceStackSize(top);
    IRP* irp = (IRP*)0;
    KSTATUS st = IoAllocateIrp(stackCount, &irp);
    if (!KSUCCESS(st)) {
        IoDereferenceDeviceObject(top);
        return st;
    }

    IoBuildPnpIrp(irp, minor);
    st = IoSubmitIrpSync(top, irp);
    IoFreeIrp(irp);
    IoDereferenceDeviceObject(top);
    return st;
}

// Disk namespace.

#define MAX_DISK_DEVICES 8
#define MAX_DISK_CALLBACKS 16

typedef struct IO_DISK_CHANGE_CALLBACK_ENTRY {
    bool Used;
    IO_DISK_CHANGE_CALLBACK Callback;
    void* Context;
} IO_DISK_CHANGE_CALLBACK_ENTRY;

typedef struct IO_DISK_ARRIVAL_CALLBACK_ENTRY {
    bool Used;
    IO_DISK_ARRIVAL_CALLBACK Callback;
    void* Context;
} IO_DISK_ARRIVAL_CALLBACK_ENTRY;

static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES];
static IO_DISK_CHANGE_CALLBACK_ENTRY gDiskChangeCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;

#define IO_DISK_SECTOR_BYTES 512u

typedef struct IO_DISK_STATS_STATE {
    SPINLOCK Lock;
    IO_DISK_STATS Stats;
} IO_DISK_STATS_STATE;

static IO_DISK_STATS_STATE gDiskStats[MAX_DISK_DEVICES];

// Initializes per-disk I/O statistics.
static void IoDiskStatsPrepare(uint8_t driveIndex) {
    if (driveIndex >= MAX_DISK_DEVICES)
        return;

    IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
    state->Lock = (SPINLOCK)SPINLOCK_INIT;
    memset(&state->Stats, 0, sizeof(state->Stats));
}

// Records one disk read request.
static void IoDiskStatsNoteRead(uint8_t driveIndex, uint32_t count) {
    if (driveIndex >= MAX_DISK_DEVICES)
        return;

    IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
    state->Stats.ReadRequests++;
    state->Stats.ReadSectors += count;
    if (state->Stats.MaxReadSectors < count)
        state->Stats.MaxReadSectors = count;
    KeReleaseSpinlock(&state->Lock, flags);
}

// Records one successful disk write request.
static void IoDiskStatsNoteWrite(uint8_t driveIndex, uint32_t count) {
    if (driveIndex >= MAX_DISK_DEVICES)
        return;

    IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
    uintptr_t flags = KeAcquireSpinlock(&state->Lock);
    state->Stats.WriteRequests++;
    state->Stats.WriteSectors += count;
    if (state->Stats.MaxWriteSectors < count)
        state->Stats.MaxWriteSectors = count;
    KeReleaseSpinlock(&state->Lock, flags);
}

// Implements the notify Disk Change operation.
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
    for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
        IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i];
        IO_DISK_CHANGE_CALLBACK callback = entry->Callback;
        void* context = entry->Context;
        if (entry->Used && callback) {
            callback(driveIndex, device, change, context);
        } else if (entry->Used) {
            KePanic(KE_PANIC_CODE_DRIVER,
                    "io: disk change callback entry has null callback index=%u ctx=0x%016llX change=%u drive=%u dev=0x%016llX\n",
                    i, context, (uint32_t)change, driveIndex, device);
        }
    }
    if (change == IoDeviceArrived) {
        for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
            IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
            IO_DISK_ARRIVAL_CALLBACK callback = entry->Callback;
            void* context = entry->Context;
            if (entry->Used && callback) {
                callback(driveIndex, device, context);
            } else if (entry->Used) {
                KePanic(KE_PANIC_CODE_DRIVER,
                        "io: disk arrival callback entry has null callback index=%u ctx=0x%016llX drive=%u dev=0x%016llX\n",
                        i, context, driveIndex, device);
            }
        }
    }
}

__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
                                                   DEVICE_OBJECT* device) {
    if (driveIndex >= MAX_DISK_DEVICES || !device || !device->DriverObject)
        return NTLSTATUS_INVALID_PARAMETER;
    if (gDiskDevices[driveIndex])
        return NTLSTATUS_ALREADY_EXISTS;

    device->Flags |= DEVICE_OBJECT_DISK;
    device->u.Disk.DiskIndex = driveIndex;
    if (device->u.Disk.SectorSize == 0)
        device->u.Disk.SectorSize = 512;

    KSTATUS pnpStatus = IoSendPnpIrpSync(device, IRP_MN_START_DEVICE);
    if (!KSUCCESS(pnpStatus) && pnpStatus != NTLSTATUS_NOT_SUPPORTED)
        return pnpStatus;
    if (pnpStatus == NTLSTATUS_NOT_SUPPORTED)
        device->PnpState = DevicePnpStarted;

    IoDiskStatsPrepare(driveIndex);

    IoReferenceDeviceObject(device);
    gDiskDevices[driveIndex] = device;
    IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
    return NTLSTATUS_SUCCESS;
}

// Implements the notify Disk Surprise Removed operation.
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
    if (driveIndex >= MAX_DISK_DEVICES)
        return NTLSTATUS_INVALID_PARAMETER;
    DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
    if (!dev) return NTLSTATUS_NOT_FOUND;

    dev->PnpState = DevicePnpSurpriseRemoved;
    IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
    (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
    return NTLSTATUS_SUCCESS;
}

// Implements the unregister Disk Device operation.
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
    if (driveIndex >= MAX_DISK_DEVICES)
        return NTLSTATUS_INVALID_PARAMETER;
    DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
    if (!dev) return NTLSTATUS_NOT_FOUND;

    if (dev->PnpState != DevicePnpSurpriseRemoved) {
        IoNotifyDiskChange(driveIndex, dev, IoDeviceQueryRemove);
        KSTATUS queryStatus = IoSendPnpIrpSync(dev, IRP_MN_QUERY_REMOVE_DEVICE);
        if (!KSUCCESS(queryStatus) && queryStatus != NTLSTATUS_NOT_SUPPORTED)
            return queryStatus;

        dev->PnpState = DevicePnpSurpriseRemoved;
        IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
        (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
    }

    dev->PnpState = DevicePnpRemovePending;
    (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);

    gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
    IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
    IoDereferenceDeviceObject(dev);
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback,
                                                            void* context) {
    if (!callback)
        return NTLSTATUS_INVALID_PARAMETER;
    if (gDiskArrivalCallbackCount >= MAX_DISK_CALLBACKS)
        return NTLSTATUS_NO_MEMORY;

    IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
    memset(entry, 0, sizeof(*entry));
    entry->Used = true;
    entry->Callback = callback;
    entry->Context = context;
    DbgPrint("io: registered disk arrival callback index=%u cb=0x%016llX ctx=0x%016llX\n",
             gDiskArrivalCallbackCount - 1u, callback, context);

    for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
        if (gDiskDevices[i])
            callback(i, gDiskDevices[i], context);
    }
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback,
                                                           void* context) {
    if (!callback)
        return NTLSTATUS_INVALID_PARAMETER;
    if (gDiskChangeCallbackCount >= MAX_DISK_CALLBACKS)
        return NTLSTATUS_NO_MEMORY;

    IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[gDiskChangeCallbackCount++];
    memset(entry, 0, sizeof(*entry));
    entry->Used = true;
    entry->Callback = callback;
    entry->Context = context;
    DbgPrint("io: registered disk change callback index=%u cb=0x%016llX ctx=0x%016llX\n",
             gDiskChangeCallbackCount - 1u, callback, context);
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback,
                                                     void* context) {
    if (!callback)
        return NTLSTATUS_INVALID_PARAMETER;

    for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
        if (gDiskDevices[i])
            callback(i, gDiskDevices[i], context);
    }
    return NTLSTATUS_SUCCESS;
}

// Returns disk Device.
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
    if (driveIndex >= MAX_DISK_DEVICES) return (DEVICE_OBJECT*)0;
    return gDiskDevices[driveIndex];
}

// Implements the reference Disk Device operation.
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) {
    DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
    if (dev) IoReferenceDeviceObject(dev);
    return dev;
}

// Implements the disk Device Present operation.
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
    return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
}

// Implements the disk Sector Count operation.
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
    DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
    return dev ? dev->u.Disk.SectorCount : 0;
}

// Dispatch.

__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
    if (!device || !device->DriverObject || !irp)
        return NTLSTATUS_INVALID_PARAMETER;

    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
    if (!ios)
        return NTLSTATUS_INVALID_PARAMETER;

    IO_MAJOR_FUNCTION major = ios->MajorFunction;
    if (major >= IRP_MJ_MAXIMUM_FUNCTION)
        return NTLSTATUS_INVALID_PARAMETER;

    if (major != IRP_MJ_PNP &&
        (device->DeletePending || device->PnpState == DevicePnpSurpriseRemoved ||
         device->PnpState == DevicePnpRemovePending || device->PnpState == DevicePnpDeleted))
        return NTLSTATUS_OBJECT_DEAD;

    PDRIVER_DISPATCH dispatch = device->DriverObject->MajorFunction[major];
    if (!dispatch)
        return NTLSTATUS_NOT_SUPPORTED;

    ios->DeviceObject = device;
    IoReferenceDeviceObject(device);
    _InterlockedIncrement(&device->OutstandingIrps);

    KSTATUS st = dispatch(device, irp);
    if (st != NTLSTATUS_PENDING) {
        if (ios->DeviceObject == device) {
            ios->DeviceObject = (DEVICE_OBJECT*)0;
            _InterlockedDecrement(&device->OutstandingIrps);
            IoDereferenceDeviceObject(device);
        }
    }
    return st;
}

// Validates that a sector range fits the disk and IRP byte length.
static KSTATUS IoValidateDiskSectorRange(DEVICE_OBJECT* dev, uint64_t lba, uint32_t count) {
    if (!dev || count == 0) return NTLSTATUS_INVALID_PARAMETER;
    if (count > (0xffffffffu / IO_DISK_SECTOR_BYTES)) return NTLSTATUS_INVALID_PARAMETER;
    if (lba > dev->u.Disk.SectorCount) return NTLSTATUS_INVALID_PARAMETER;
    if ((uint64_t)count > dev->u.Disk.SectorCount - lba) return NTLSTATUS_INVALID_PARAMETER;
    return NTLSTATUS_SUCCESS;
}

// Submits one raw disk read IRP.
static KSTATUS IoReadDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
                                    uint32_t count, void* buffer) {
    if (!dev || !buffer || count == 0) return NTLSTATUS_INVALID_PARAMETER;

    IRP irp;
    IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
    return IoSubmitIrpSync(dev, &irp);
}

// Submits one raw disk write IRP.
static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
                                     uint32_t count, void* buffer) {
    if (!dev || !buffer || count == 0) return NTLSTATUS_INVALID_PARAMETER;

    IRP irp;
    IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, buffer);
    return IoSubmitIrpSync(dev, &irp);
}

__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
                                                uint64_t lba,
                                                uint32_t count,
                                                void* buffer) {
    if (!buffer || count == 0) return NTLSTATUS_INVALID_PARAMETER;

    DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
    if (!dev) return NTLSTATUS_NOT_FOUND;

    KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
    if (!KSUCCESS(rangeStatus)) {
        IoDereferenceDeviceObject(dev);
        return rangeStatus;
    }

    KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
    IoDiskStatsNoteRead(driveIndex, count);

    IoDereferenceDeviceObject(dev);
    return st;
}

__declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
                                                 uint64_t lba,
                                                 uint32_t count,
                                                 const void* buffer) {
    if (!buffer || count == 0) return NTLSTATUS_INVALID_PARAMETER;

    DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
    if (!dev) return NTLSTATUS_NOT_FOUND;

    KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
    if (!KSUCCESS(rangeStatus)) {
        IoDereferenceDeviceObject(dev);
        return rangeStatus;
    }

    KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
    if (KSUCCESS(st))
        IoDiskStatsNoteWrite(driveIndex, count);

    IoDereferenceDeviceObject(dev);
    return st;
}

// Flushes volatile write caches for one disk device.
__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex) {
    DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
    if (!dev) return NTLSTATUS_NOT_FOUND;

    IRP irp;
    IoInitializeIrp(&irp, 1);
    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(&irp);
    if (!ios) {
        IoDereferenceDeviceObject(dev);
        return NTLSTATUS_FAILURE;
    }
    ios->MajorFunction = IRP_MJ_FLUSH_BUFFERS;
    ios->MinorFunction = 0;

    KSTATUS st = IoSubmitIrpSync(dev, &irp);
    IoDereferenceDeviceObject(dev);
    return st;
}

// Queries aggregate disk I/O statistics.
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
    if (!out) return;
    memset(out, 0, sizeof(*out));

    for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
        IO_DISK_STATS_STATE* state = &gDiskStats[i];
        uintptr_t flags = KeAcquireSpinlock(&state->Lock);
        if (gDiskDevices[i])
            out->RegisteredDisks++;
        out->ReadRequests += state->Stats.ReadRequests;
        out->ReadSectors += state->Stats.ReadSectors;
        out->WriteRequests += state->Stats.WriteRequests;
        out->WriteSectors += state->Stats.WriteSectors;
        if (out->MaxReadSectors < state->Stats.MaxReadSectors)
            out->MaxReadSectors = state->Stats.MaxReadSectors;
        if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
            out->MaxWriteSectors = state->Stats.MaxWriteSectors;
        KeReleaseSpinlock(&state->Lock, flags);
    }
}