usb hubs, hotplug support, unplug support, saving log files to disk, smt

code/boot.ini

@@ -1,5 +1,5 @@
 [kernel]
-; bitmask: 1 = debug prints, 2 = self check
+; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file
-debug = 1
+debug = 4
 [init]
 commandline = apps/shell_native.exe

code/boot/uefi/main.c

@@ -264,12 +264,35 @@ typedef struct {
     char       commandLine[64];
 } PARSE_STATE;
 
+
+static int ParseBootIniInteger(const char* val) {
+    int out = 0;
+    if (!val) return 0;
+    while (*val == ' ' || *val == '\t') val++;
+    int base = 10;
+    if (val[0] == '0' && (val[1] == 'x' || val[1] == 'X')) {
+        base = 16;
+        val += 2;
+    }
+    while (*val) {
+        char c = *val++;
+        int digit;
+        if (c >= '0' && c <= '9') digit = c - '0';
+        else if (c >= 'a' && c <= 'f') digit = 10 + c - 'a';
+        else if (c >= 'A' && c <= 'F') digit = 10 + c - 'A';
+        else break;
+        if (digit >= base) break;
+        out = out * base + digit;
+    }
+    return out;
+}
+
 static void BootIniCb(const char* section, const char* key,
                       const char* val, void* ud) {
     PARSE_STATE* s = (PARSE_STATE *)ud;
-    // [kernel] debug = 1
+    // [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log
     if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug"))
-        s->debug = (int)(val[0] - '0');
+        s->debug = ParseBootIniInteger(val);
     // [init] commandline = hello.exe
      else if (!AsciiStrCmp(section, "init") && !AsciiStrCmp(key, "commandline"))
         AsciiStrCpyS(s->commandLine, 64, val);

code/build/build.c

@@ -28,7 +28,7 @@ static const Driver_Target STANDALONE_DRIVERS[] = {
     DRIVER_TARGET("ide",    "obj\\drivers\\ide",    "code\\drivers\\storage\\ide\\ide.c code\\drivers\\storage\\ide\\ide_bus.c"),
     DRIVER_TARGET("ahci",   "obj\\drivers\\ahci",   "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c"),
     DRIVER_TARGET("fat32",  "obj\\drivers\\fat32",  "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
-    DRIVER_TARGET("usb",    "obj\\drivers\\usb",    "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c"),
+    DRIVER_TARGET("usb",    "obj\\drivers\\usb",    "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_bus.c code\\drivers\\bus\\usb\\usb_composite.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c code\\drivers\\bus\\usb\\usb_hub.c"),
     DRIVER_TARGET("rtl816x", "obj\\drivers\\rtl816x", "code\\drivers\\net\\rtl816x\\rtl816x.c code\\drivers\\net\\rtl816x\\rtl816x_pci.c"),
     DRIVER_TARGET("rtl8139", "obj\\drivers\\rtl8139", "code\\drivers\\net\\rtl8139\\rtl8139.c code\\drivers\\net\\rtl8139\\rtl8139_pci.c"),
 };

code/drivers/bus/usb/usb.c

@@ -75,9 +75,33 @@ KSTATUS DriverEntry(const HAL_API* hal) {
     if (!hal) return STATUS_INVALID_PARAMETER;
     DbgPrint("usb: discovery starting\n");
 
-    KSTATUS st = UsbRegisterClassMatchers();
+    KSTATUS st = UsbInitializeBusModel();
     if (!KSUCCESS(st)) {
-        DbgPrint("usb: class matcher registration failed st=0x%x\n", st);
+        DbgPrint("usb: bus model initialisation failed st=0x%x\n", st);
+        return st;
+    }
+
+    st = UsbInitializeCompositeDriver();
+    if (!KSUCCESS(st)) {
+        DbgPrint("usb: composite driver initialisation failed st=0x%x\n", st);
+        return st;
+    }
+
+    st = UsbInitializeHidDriver();
+    if (!KSUCCESS(st)) {
+        DbgPrint("usb: HID driver initialisation failed st=0x%x\n", st);
+        return st;
+    }
+
+    st = UsbInitializeStorageDriver();
+    if (!KSUCCESS(st)) {
+        DbgPrint("usb: storage driver initialisation failed st=0x%x\n", st);
+        return st;
+    }
+
+    st = UsbInitializeHubDriver();
+    if (!KSUCCESS(st)) {
+        DbgPrint("usb: hub driver initialisation failed st=0x%x\n", st);
         return st;
     }
 

code/drivers/bus/usb/usb.h

@@ -16,6 +16,8 @@
 #include <kernel/ke_dpc.h>
 #include <kernel/ke_irp.h>
 #include <kernel/ke_spinlock.h>
+#include <kernel/ke_semaphore.h>
+#include <kernel/ke_thread.h>
 #include <kernel/mm_phys.h>
 #include <kernel/mm_virt.h>
 
@@ -114,6 +116,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
 #define XHCI_TRB_TYPE_DATA       3u
 #define XHCI_TRB_TYPE_STATUS     4u
 #define XHCI_TRB_TYPE_ENABLE_SLOT 9u
+#define XHCI_TRB_TYPE_DISABLE_SLOT 10u
 #define XHCI_TRB_TYPE_ADDRESS_DEVICE 11u
 #define XHCI_TRB_TYPE_CONFIGURE_ENDPOINT 12u
 
@@ -144,10 +147,15 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
 #define USB_REQ_SET_CONFIGURATION 0x09u
 #define USB_REQ_SET_IDLE         0x0Au
 #define USB_REQ_SET_PROTOCOL     0x0Bu
+#define USB_REQ_GET_STATUS       0x00u
+#define USB_REQ_CLEAR_FEATURE    0x01u
+#define USB_REQ_SET_FEATURE      0x03u
 #define USB_DESC_DEVICE          0x01u
 #define USB_DESC_CONFIGURATION   0x02u
 #define USB_DESC_INTERFACE       0x04u
 #define USB_DESC_ENDPOINT        0x05u
+#define USB_DESC_HUB             0x29u
+#define USB_DESC_SUPERSPEED_HUB  0x2Au
 #define USB_DESC_HID             0x21u
 #define USB_DESC_CS_INTERFACE    0x24u
 #define USB_DIR_OUT              0x00u
@@ -155,6 +163,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
 #define USB_TYPE_STANDARD        0x00u
 #define USB_TYPE_CLASS           0x20u
 #define USB_RECIP_DEVICE         0x00u
+#define USB_RECIP_OTHER          0x03u
 #define USB_RECIP_INTERFACE      0x01u
 #define USB_ENDPOINT_DIR_IN      0x80u
 #define USB_ENDPOINT_NUMBER_MASK 0x0Fu
@@ -165,6 +174,23 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
 #define USB_ENDPOINT_TYPE_INTERRUPT 0x03u
 #define USB_CLASS_HID            0x03u
 #define USB_CLASS_MASS_STORAGE   0x08u
+#define USB_CLASS_HUB            0x09u
+#define USB_HUB_PORT_FEAT_CONNECTION   0u
+#define USB_HUB_PORT_FEAT_ENABLE       1u
+#define USB_HUB_PORT_FEAT_SUSPEND      2u
+#define USB_HUB_PORT_FEAT_OVER_CURRENT 3u
+#define USB_HUB_PORT_FEAT_RESET        4u
+#define USB_HUB_PORT_FEAT_POWER        8u
+#define USB_HUB_PORT_FEAT_C_CONNECTION 16u
+#define USB_HUB_PORT_FEAT_C_ENABLE     17u
+#define USB_HUB_PORT_FEAT_C_RESET      20u
+#define USB_HUB_PORT_STATUS_CONNECTION (1u << 0)
+#define USB_HUB_PORT_STATUS_ENABLE     (1u << 1)
+#define USB_HUB_PORT_STATUS_LOW_SPEED  (1u << 9)
+#define USB_HUB_PORT_STATUS_HIGH_SPEED (1u << 10)
+#define USB_HUB_PORT_CHANGE_CONNECTION (1u << 0)
+#define USB_HUB_PORT_CHANGE_ENABLE     (1u << 1)
+#define USB_HUB_PORT_CHANGE_RESET      (1u << 4)
 #define USB_MASS_SUBCLASS_SCSI   0x06u
 #define USB_MASS_PROTOCOL_BOT    0x50u
 #define USB_HID_SUBCLASS_BOOT    0x01u
@@ -193,12 +219,22 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
 #define XHCI_EP0_RING_TRBS       256u
 #define XHCI_HID_RING_TRBS       256u
 #define XHCI_BULK_RING_TRBS      256u
+#define XHCI_PIPE_RING_TRBS      256u
 #define XHCI_MAX_HID_REPORT      16u
 #define USB_STORAGE_DMA_PAGES    1u
 #define USB_STORAGE_MAX_BYTES    (USB_STORAGE_DMA_PAGES * PAGE_SIZE)
 #define USB_STORAGE_SECTOR_SIZE  512u
 #define USB_MAX_TRACKED_INTERFACES 8u
 #define USB_MAX_TRACKED_ENDPOINTS  8u
+#define USB_MAX_DEVICE_ID_TEXT      96u
+#define USB_MAX_PIPES              32u
+#define USB_IOCTL_SUBMIT_URB       0x55534201u  /* USB1 */
+#define USB_ROOT_HUB_MAX_PORTS      XHCI_MAX_ROOT_PORTS
+#define USB_EXTERNAL_HUB_MAX_PORTS  31u
+#define USB_ROOT_PORT_ENUM_RETRIES  3u
+#define USB_XHCI_CONTROL_TIMEOUT_US       1000000u
+#define USB_XHCI_ENUM_CONTROL_TIMEOUT_US   120000u
+#define USB_HUB_CHANGE_BITMAP_BYTES 8u
 
 #pragma pack(push, 1)
 typedef struct XHCI_TRB {
@@ -271,6 +307,21 @@ typedef struct USB_ENDPOINT_DESCRIPTOR {
     uint8_t  bInterval;
 } USB_ENDPOINT_DESCRIPTOR;
 
+typedef struct USB_HUB_DESCRIPTOR {
+    uint8_t  bLength;
+    uint8_t  bDescriptorType;
+    uint8_t  bNbrPorts;
+    uint16_t wHubCharacteristics;
+    uint8_t  bPwrOn2PwrGood;
+    uint8_t  bHubContrCurrent;
+    uint8_t  DeviceRemovable[1];
+} USB_HUB_DESCRIPTOR;
+
+typedef struct USB_HUB_PORT_STATUS {
+    uint16_t Status;
+    uint16_t Change;
+} USB_HUB_PORT_STATUS;
+
 typedef struct USB_MASS_CBW {
     uint32_t Signature;
     uint32_t Tag;
@@ -308,6 +359,309 @@ typedef struct USB_BAR {
     uint8_t      Index;
 } USB_BAR;
 
+
+typedef enum USB_DEVICE_STATE {
+    UsbDeviceAttached = 0,
+    UsbDevicePowered,
+    UsbDeviceDefault,
+    UsbDeviceAddressed,
+    UsbDeviceConfigured,
+    UsbDeviceSuspended,
+    UsbDeviceSurpriseRemoved,
+    UsbDeviceRemoved,
+} USB_DEVICE_STATE;
+
+typedef struct USB_PORT_PATH {
+    uint8_t RootPort;
+    uint8_t Depth;
+    uint32_t RouteString;
+    uint8_t ParentHubSlot;
+    uint8_t ParentHubPort;
+    bool BehindHub;
+} USB_PORT_PATH;
+
+typedef enum USB_PIPE_TYPE {
+    UsbPipeControl = 0,
+    UsbPipeBulk,
+    UsbPipeInterrupt,
+    UsbPipeIsochronous,
+} USB_PIPE_TYPE;
+
+typedef enum USB_TRANSFER_DIRECTION {
+    UsbTransferNoData = 0,
+    UsbTransferOut,
+    UsbTransferIn,
+} USB_TRANSFER_DIRECTION;
+
+typedef enum USB_URB_FUNCTION {
+    UrbFunctionControlTransfer = 0,
+    UrbFunctionBulkTransfer,
+    UrbFunctionInterruptTransfer,
+    UrbFunctionSelectConfiguration,
+    UrbFunctionSelectInterface,
+    UrbFunctionResetPipe,
+    UrbFunctionAbortPipe,
+} USB_URB_FUNCTION;
+
+typedef struct USB_HCD USB_HCD;
+typedef struct USB_DEVICE USB_DEVICE;
+typedef struct USB_PIPE USB_PIPE;
+typedef struct USB_URB USB_URB;
+typedef struct XHCI_DEVICE XHCI_DEVICE;
+typedef void (*USB_URB_COMPLETION)(USB_URB* urb, void* context);
+
+typedef struct USB_HCD_OPERATIONS {
+    KSTATUS (*SubmitUrb)(USB_HCD* hcd, USB_URB* urb);
+    KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb);
+    KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed);
+    KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port);
+    void*   (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys);
+} USB_HCD_OPERATIONS;
+
+struct USB_HCD {
+    const USB_HCD_OPERATIONS* Operations;
+    void* ControllerContext;
+    DRIVER_OBJECT* DriverObject;
+    DEVICE_OBJECT* ControllerPdo;
+    DEVICE_OBJECT* RootHubPdo;
+    void* RootHubContext;
+    uint8_t ControllerIndex;
+    uint8_t MaxRootPorts;
+};
+
+struct USB_DEVICE {
+    OBJECT Header;
+    USB_HCD* Hcd;
+    DEVICE_OBJECT* Pdo;
+    void* HcdDeviceContext;
+    USB_DEVICE_STATE State;
+    USB_PORT_PATH Path;
+
+    uint8_t Address;
+    uint8_t SlotId;
+    uint8_t Speed;
+    uint16_t MaxPacketSize0;
+
+    uint16_t VendorId;
+    uint16_t ProductId;
+    uint16_t BcdUsb;
+    uint16_t BcdDevice;
+    uint8_t DeviceClass;
+    uint8_t DeviceSubclass;
+    uint8_t DeviceProtocol;
+    uint8_t ConfigurationValue;
+
+    IO_REMOVE_LOCK RemoveLock;
+    bool RemoveLockReady;
+};
+
+struct USB_PIPE {
+    OBJECT Header;
+    USB_DEVICE* Device;
+    USB_PIPE_TYPE Type;
+    uint8_t EndpointAddress;
+    uint8_t EndpointId;
+    uint16_t MaxPacketSize;
+    uint8_t Interval;
+    bool Open;
+    bool Halted;
+    void* HcdPipeContext;
+};
+
+struct USB_URB {
+    LIST_ENTRY Link;
+    USB_URB_FUNCTION Function;
+    USB_TRANSFER_DIRECTION Direction;
+    USB_DEVICE* Device;
+    USB_PIPE* Pipe;
+    USB_SETUP_PACKET Setup;
+    void* Buffer;
+    PHYSICAL_ADDRESS BufferPhys;
+    uint32_t Length;
+    uint32_t ActualLength;
+    KSTATUS Status;
+    IRP* Irp;
+    USB_URB_COMPLETION Complete;
+    void* CompleteContext;
+    bool Pending;
+    bool Cancelled;
+};
+
+typedef enum USB_PDO_KIND {
+    UsbPdoController = 1,
+    UsbPdoDevice = 2,
+    UsbPdoInterface = 3,
+    UsbPdoRootHub = 4,
+} USB_PDO_KIND;
+
+typedef struct USB_PDO_EXTENSION {
+    USB_PDO_KIND Kind;
+    USB_HCD* Hcd;
+    USB_DEVICE* Device;
+    uint8_t InterfaceIndex;
+    USB_INTERFACE_DESCRIPTOR InterfaceDescriptor;
+    uint8_t EndpointCount;
+    USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS];
+} USB_PDO_EXTENSION;
+
+
+
+typedef enum USB_HUB_KIND {
+    UsbHubKindRoot = 1,
+    UsbHubKindExternal = 2,
+} USB_HUB_KIND;
+
+typedef enum USB_HUB_PORT_STATE {
+    UsbHubPortDisconnected = 0,
+    UsbHubPortPowered,
+    UsbHubPortConnectedDebounce,
+    UsbHubPortResetting,
+    UsbHubPortEnabled,
+    UsbHubPortEnumerating,
+    UsbHubPortChildPresent,
+    UsbHubPortFailedConnected,
+    UsbHubPortRemoving,
+} USB_HUB_PORT_STATE;
+
+typedef struct USB_HUB_EXTENSION {
+    DEVICE_OBJECT* Self;
+    DEVICE_OBJECT* LowerDevice;
+    DEVICE_OBJECT* HubPdo;
+    USB_HUB_KIND Kind;
+    USB_HCD* Hcd;
+    USB_DEVICE* Device;
+    IO_REMOVE_LOCK RemoveLock;
+    bool RemoveLockReady;
+    bool Started;
+    bool SurpriseRemoved;
+    bool Removed;
+    uint8_t PortCount;
+    uint8_t RootPortSlot[USB_ROOT_HUB_MAX_PORTS + 1u];
+    uint8_t RootPortFailCount[USB_ROOT_HUB_MAX_PORTS + 1u];
+    USB_HUB_PORT_STATE RootPortState[USB_ROOT_HUB_MAX_PORTS + 1u];
+    uint8_t ChildSlot[USB_EXTERNAL_HUB_MAX_PORTS + 1u];
+    USB_HUB_PORT_STATE ChildPortState[USB_EXTERNAL_HUB_MAX_PORTS + 1u];
+    uint32_t PendingRootPorts[8];
+    uint32_t PendingHubPorts[1];
+    SPINLOCK PendingLock;
+    SEMAPHORE* WorkerSemaphore;
+    THREAD* WorkerThread;
+    bool WorkerRunning;
+    bool InitialScanPending;
+    bool InitialScanComplete;
+    bool InitialScanInProgress;
+
+    uint8_t InterfaceNumber;
+    uint8_t InterruptEndpointAddress;
+    uint16_t InterruptMaxPacket;
+    uint8_t InterruptInterval;
+    USB_PIPE* InterruptPipe;
+    uint8_t* ChangeBitmap;
+    PHYSICAL_ADDRESS ChangeBitmapPhys;
+    USB_URB ChangeUrb;
+    bool ChangeUrbPending;
+    uint8_t PowerOnToPowerGoodMs;
+} USB_HUB_EXTENSION;
+
+typedef struct USB_COMPOSITE_EXTENSION {
+    DEVICE_OBJECT* Self;
+    DEVICE_OBJECT* LowerDevice;
+    DEVICE_OBJECT* DevicePdo;
+    USB_DEVICE* Device;
+    bool Started;
+    bool InterfacesCreated;
+} USB_COMPOSITE_EXTENSION;
+
+typedef struct USB_HID_EXTENSION {
+    DEVICE_OBJECT* Self;
+    DEVICE_OBJECT* LowerDevice;
+    DEVICE_OBJECT* InterfacePdo;
+    USB_DEVICE* Device;
+    IO_REMOVE_LOCK RemoveLock;
+    bool RemoveLockReady;
+    bool Started;
+    bool SurpriseRemoved;
+    bool Removed;
+    bool Keyboard;
+    bool Mouse;
+    uint8_t InterfaceNumber;
+    uint8_t Protocol;
+    uint8_t EndpointAddress;
+    uint16_t MaxPacketSize;
+    uint8_t Interval;
+
+    USB_PIPE* InterruptPipe;
+    uint8_t* ReportBuffer;
+    PHYSICAL_ADDRESS ReportBufferPhys;
+    uint16_t ReportLength;
+    USB_URB InterruptUrb;
+    bool InterruptPending;
+
+    uint8_t LastModifiers;
+    uint8_t LastKeys[6];
+    uint8_t LastButtons;
+    uint32_t InputDeviceId;
+} USB_HID_EXTENSION;
+
+
+
+typedef enum USB_STORAGE_EXTENSION_KIND {
+    UsbStorageExtensionInterface = 0x55534946u,
+    UsbStorageExtensionDisk      = 0x5553444Bu,
+} USB_STORAGE_EXTENSION_KIND;
+
+typedef struct USB_STORAGE_EXTENSION USB_STORAGE_EXTENSION;
+typedef struct USB_STORAGE_DISK_EXTENSION USB_STORAGE_DISK_EXTENSION;
+
+struct USB_STORAGE_EXTENSION {
+    USB_STORAGE_EXTENSION_KIND Kind;
+    DEVICE_OBJECT* Self;
+    DEVICE_OBJECT* LowerDevice;
+    DEVICE_OBJECT* InterfacePdo;
+    DEVICE_OBJECT* DiskDevice;
+
+    USB_DEVICE* Device;
+    XHCI_DEVICE* XhciDevice;
+
+    USB_PIPE* BulkInPipe;
+    USB_PIPE* BulkOutPipe;
+
+    IO_REMOVE_LOCK RemoveLock;
+    bool RemoveLockReady;
+    bool Started;
+    bool SurpriseRemoved;
+    bool Removed;
+
+    uint8_t InterfaceNumber;
+    uint8_t BulkInAddress;
+    uint8_t BulkOutAddress;
+    uint16_t BulkInMaxPacket;
+    uint16_t BulkOutMaxPacket;
+
+    uint32_t Tag;
+    uint32_t BlockSize;
+    uint64_t BlockCount;
+    volatile long IoBusy;
+};
+
+struct USB_STORAGE_DISK_EXTENSION {
+    USB_STORAGE_EXTENSION_KIND Kind;
+    USB_STORAGE_EXTENSION* Storage;
+};
+
+typedef struct XHCI_PIPE_CONTEXT {
+    XHCI_TRB* Ring;
+    PHYSICAL_ADDRESS RingPhys;
+    uint32_t Enqueue;
+    uint8_t Cycle;
+    uint8_t EndpointId;
+    USB_PIPE_TYPE Type;
+    bool Configured;
+    bool ActiveTransfer;
+    uint64_t ActiveTrbPhys;
+    USB_URB* ActiveUrb;
+} XHCI_PIPE_CONTEXT;
+
 typedef struct XHCI_DEVICE {
     bool Allocated;
     bool Addressed;
@@ -324,6 +678,11 @@ typedef struct XHCI_DEVICE {
     uint8_t DeviceProtocol;
     uint8_t ConfigurationValue;
     bool Configured;
+    USB_DEVICE* UsbDevice;
+    DEVICE_OBJECT* UsbPdo;
+    DEVICE_OBJECT* ParentDeviceObject;
+    USB_PORT_PATH Path;
+    USB_PIPE* EndpointPipes[32];
 
     uint8_t InterfaceCount;
     USB_TRACKED_INTERFACE Interfaces[USB_MAX_TRACKED_INTERFACES];
@@ -364,6 +723,10 @@ typedef struct XHCI_DEVICE {
     uint32_t MassBlockSize;
     uint64_t MassBlockCount;
     DEVICE_OBJECT* MassDeviceObject;
+    IO_REMOVE_LOCK MassRemoveLock;
+    bool MassRemoveLockReady;
+    bool MassSurpriseRemoved;
+    bool MassRemoved;
     volatile long MassIoBusy;
 
     PHYSICAL_ADDRESS InputContextPhys;
@@ -403,12 +766,14 @@ typedef struct XHCI_TRANSFER_RESULT {
 } XHCI_TRANSFER_RESULT;
 
 typedef struct XHCI_WAIT_TARGET {
+    struct XHCI_WAIT_TARGET* Next;
     uint64_t CommandPhys;
     uint64_t TransferPhys;
     uint8_t TransferSlotId;
     uint8_t TransferEndpointId;
-    bool CommandMatched;
+    volatile bool CommandMatched;
-    bool TransferMatched;
+    volatile bool TransferMatched;
+    volatile bool Registered;
     XHCI_COMMAND_RESULT Command;
     XHCI_TRANSFER_RESULT Transfer;
 } XHCI_WAIT_TARGET;
@@ -470,15 +835,58 @@ typedef struct XHCI_CONTROLLER {
     volatile uint32_t InterruptCount;
     volatile uint32_t EventDpcCount;
     SPINLOCK EventLock;
+    SPINLOCK WaitLock;
+    SPINLOCK CommandLock;
+    XHCI_WAIT_TARGET* WaiterList;
 
-    volatile XHCI_WAIT_TARGET* ActiveWait;
     volatile bool Running;
+
+    USB_HCD Hcd;
 } XHCI_CONTROLLER;
 
 
 extern XHCI_CONTROLLER gXhci;
 extern bool gHaveXhci;
 extern DRIVER_OBJECT gUsbStorageDriver;
+extern DRIVER_OBJECT gUsbBusDriver;
+extern DRIVER_OBJECT gUsbCompositeDriver;
+extern DRIVER_OBJECT gUsbHidDriver;
+extern DRIVER_OBJECT gUsbHubDriver;
+
+KSTATUS UsbInitializeBusModel(void);
+KSTATUS UsbInitializeCompositeDriver(void);
+KSTATUS UsbInitializeHidDriver(void);
+KSTATUS UsbInitializeStorageDriver(void);
+KSTATUS UsbInitializeHubDriver(void);
+KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
+KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
+void UsbInitializeHcd(USB_HCD* hcd,
+                      const USB_HCD_OPERATIONS* ops,
+                      void* controllerContext,
+                      uint8_t controllerIndex,
+                      uint8_t maxRootPorts);
+KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device);
+KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd);
+KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise);
+KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise);
+KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent);
+KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent);
+KSTATUS UsbCreatePipe(USB_DEVICE* dev,
+                      USB_PIPE_TYPE type,
+                      uint8_t endpointAddress,
+                      uint16_t maxPacketSize,
+                      uint8_t interval,
+                      USB_PIPE** outPipe);
+void UsbReferenceDevice(USB_DEVICE* dev);
+void UsbDereferenceDevice(USB_DEVICE* dev);
+void UsbReferencePipe(USB_PIPE* pipe);
+void UsbDereferencePipe(USB_PIPE* pipe);
+KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb);
+KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb);
+KSTATUS UsbCancelUrb(USB_URB* urb);
+void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys);
+KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp);
+KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp);
 
 uint32_t UsbAlignUp32(uint32_t value, uint32_t align);
 void XhciBarrier(void);
@@ -512,11 +920,15 @@ const char* UsbEndpointTypeName(uint8_t type);
 const char* UsbDirectionName(uint8_t endpointAddress);
 const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto);
 void XhciLogPortStatus(XHCI_CONTROLLER* xhci);
+void XhciPowerRootPorts(XHCI_CONTROLLER* xhci);
 void XhciAckInterrupter0(XHCI_CONTROLLER* xhci);
 void XhciUpdateErdp(XHCI_CONTROLLER* xhci);
 bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
 bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
 bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target);
+void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
+void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
+KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs);
 KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs);
 void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2);
 void XhciIsr(void);
@@ -535,12 +947,15 @@ void XhciEnableInterrupts(XHCI_CONTROLLER* xhci);
 uint32_t XhciPortReg(uint8_t port);
 void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port);
 KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed);
+KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId);
 KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
 void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
 uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index);
 KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
+KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs);
 KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length);
 KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup);
+KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs);
 KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc);
 KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength);
 uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress);
@@ -561,33 +976,42 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
 const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub, uint8_t endpointType, bool wantIn);
 bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
 KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context);
+KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
+KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
 void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle);
+KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe);
 KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual);
-KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
+KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
-KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
+KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
-KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
+KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
 uint32_t UsbBe32(const uint8_t* p);
 void UsbStoreBe32(uint8_t* p, uint32_t v);
 void UsbStoreBe16(uint8_t* p, uint16_t v);
-KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
+KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage);
-KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
+KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage);
-KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
+KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage);
-KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
+KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage);
-KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, bool write, uint64_t lba, uint32_t sectors, void* buffer);
+KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer);
-void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
+void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage);
-void UsbStorageReleaseIo(XHCI_DEVICE* dev);
+void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage);
 KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp);
+KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
+KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
 KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
 KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
 uint8_t UsbFindFreeDiskIndex(void);
-KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev);
+KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage);
 bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
 KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context);
-KSTATUS UsbRegisterClassMatchers(void);
 KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationValue);
 void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_CONFIGURATION_DESCRIPTOR* config, uint16_t totalLength);
-void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
 KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_DEVICE_DESCRIPTOR* devDesc);
+KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId);
+KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci, USB_HUB_EXTENSION* hub, uint8_t hubPort, uint8_t speed, uint8_t* outSlotId);
 KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port);
 void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci);
+KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd);
+void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port);
+void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port);
+extern const USB_HCD_OPERATIONS gXhciHcdOperations;
 KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device);

code/drivers/bus/usb/usb_bus.c

@@ -0,0 +1,656 @@
+// usb_bus.c - USB bus/HCD/PDO/URB core model.
+//
+// This is the structural USB bus layer used by the upcoming composite, hub,
+// and hotplug stages.  The legacy direct xHCI class path is still present for
+// boot HID/storage during migration, but newly enumerated xHCI devices now also
+// get first-class USB PDOs and can accept internal URB IRPs.
+#include "usb.h"
+
+DRIVER_OBJECT gUsbBusDriver;
+
+static bool gUsbBusModelReady;
+static uint8_t gUsbNextControllerIndex;
+
+static void UsbDeviceDestroy(OBJECT* object) {
+    USB_DEVICE* dev = (USB_DEVICE*)object;
+    if (dev->RemoveLockReady) {
+        IoDestroyRemoveLock(&dev->RemoveLock);
+        dev->RemoveLockReady = false;
+    }
+    if (dev->Pdo) {
+        IoDereferenceDeviceObject(dev->Pdo);
+        dev->Pdo = (DEVICE_OBJECT*)0;
+    }
+}
+
+static void UsbPipeDestroy(OBJECT* object) {
+    USB_PIPE* pipe = (USB_PIPE*)object;
+    if (pipe->Device) {
+        UsbDereferenceDevice(pipe->Device);
+        pipe->Device = (USB_DEVICE*)0;
+    }
+}
+
+void UsbReferenceDevice(USB_DEVICE* dev) {
+    if (dev) ObReferenceObject(&dev->Header);
+}
+
+void UsbDereferenceDevice(USB_DEVICE* dev) {
+    if (dev) ObDereferenceObject(&dev->Header);
+}
+
+void UsbReferencePipe(USB_PIPE* pipe) {
+    if (pipe) ObReferenceObject(&pipe->Header);
+}
+
+void UsbDereferencePipe(USB_PIPE* pipe) {
+    if (pipe) ObDereferenceObject(&pipe->Header);
+}
+
+void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) {
+    if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma)
+        return (void*)0;
+    return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys);
+}
+
+static USB_DEVICE* UsbAllocateDevice(void) {
+    USB_DEVICE* dev = (USB_DEVICE*)ObAllocateObject(ObjectTypeUsbDevice, sizeof(USB_DEVICE));
+    if (!dev) return (USB_DEVICE*)0;
+    dev->Header.Destroy = UsbDeviceDestroy;
+    KSTATUS st = IoInitializeRemoveLock(&dev->RemoveLock);
+    if (!KSUCCESS(st)) {
+        ObDereferenceObject(&dev->Header);
+        return (USB_DEVICE*)0;
+    }
+    dev->RemoveLockReady = true;
+    dev->State = UsbDeviceAttached;
+    return dev;
+}
+
+KSTATUS UsbCreatePipe(USB_DEVICE* dev,
+                      USB_PIPE_TYPE type,
+                      uint8_t endpointAddress,
+                      uint16_t maxPacketSize,
+                      uint8_t interval,
+                      USB_PIPE** outPipe) {
+    if (!dev || !outPipe) return STATUS_INVALID_PARAMETER;
+
+    USB_PIPE* pipe = (USB_PIPE*)ObAllocateObject(ObjectTypeUsbPipe, sizeof(USB_PIPE));
+    if (!pipe) return STATUS_NO_MEMORY;
+    pipe->Header.Destroy = UsbPipeDestroy;
+    pipe->Device = dev;
+    UsbReferenceDevice(dev);
+    pipe->Type = type;
+    pipe->EndpointAddress = endpointAddress;
+    pipe->EndpointId = XhciEndpointIdFromAddress(endpointAddress);
+    pipe->MaxPacketSize = maxPacketSize;
+    pipe->Interval = interval;
+    pipe->Open = true;
+    *outPipe = pipe;
+    return STATUS_SUCCESS;
+}
+
+static SYS_STRING_VIEW UsbIdFromBuffer(char* buf) {
+    return SysSvFromCstr(buf);
+}
+
+static void UsbIdPutChar(char* buf, size_t cap, size_t* pos, char ch) {
+    if (!buf || !pos || cap == 0)
+        return;
+    if (*pos + 1 < cap)
+        buf[*pos] = ch;
+    (*pos)++;
+    if (*pos < cap)
+        buf[*pos] = '\0';
+    else
+        buf[cap - 1] = '\0';
+}
+
+static void UsbIdAppendLiteral(char* buf, size_t cap, size_t* pos, const char* text) {
+    if (!text)
+        return;
+    while (*text)
+        UsbIdPutChar(buf, cap, pos, *text++);
+}
+
+static void UsbIdAppendHex(char* buf, size_t cap, size_t* pos, uint32_t value, uint8_t digits) {
+    static const char kHex[] = "0123456789ABCDEF";
+    while (digits) {
+        digits--;
+        UsbIdPutChar(buf, cap, pos, kHex[(value >> (digits * 4u)) & 0xFu]);
+    }
+}
+
+static void UsbFormatVidPidRev(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint16_t rev) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
+    UsbIdAppendHex(buf, cap, &pos, vid, 4);
+    UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
+    UsbIdAppendHex(buf, cap, &pos, pid, 4);
+    UsbIdAppendLiteral(buf, cap, &pos, "&REV_");
+    UsbIdAppendHex(buf, cap, &pos, rev, 4);
+}
+
+static void UsbFormatVidPid(char* buf, size_t cap, uint16_t vid, uint16_t pid) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
+    UsbIdAppendHex(buf, cap, &pos, vid, 4);
+    UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
+    UsbIdAppendHex(buf, cap, &pos, pid, 4);
+}
+
+static void UsbFormatVidPidMi(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint8_t mi) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
+    UsbIdAppendHex(buf, cap, &pos, vid, 4);
+    UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
+    UsbIdAppendHex(buf, cap, &pos, pid, 4);
+    UsbIdAppendLiteral(buf, cap, &pos, "&MI_");
+    UsbIdAppendHex(buf, cap, &pos, mi, 2);
+}
+
+static void UsbFormatClassSubProt(char* buf, size_t cap, uint8_t cls, uint8_t sub, uint8_t proto) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_");
+    UsbIdAppendHex(buf, cap, &pos, cls, 2);
+    UsbIdAppendLiteral(buf, cap, &pos, "&SUBCLASS_");
+    UsbIdAppendHex(buf, cap, &pos, sub, 2);
+    UsbIdAppendLiteral(buf, cap, &pos, "&PROT_");
+    UsbIdAppendHex(buf, cap, &pos, proto, 2);
+}
+
+static void UsbFormatClass(char* buf, size_t cap, uint8_t cls) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_");
+    UsbIdAppendHex(buf, cap, &pos, cls, 2);
+}
+
+
+static void UsbFormatComposite(char* buf, size_t cap) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "USB\\COMPOSITE");
+}
+
+static void UsbFormatPciXhci(char* buf, size_t cap, uint16_t vendor, uint16_t device) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "PCI\\VEN_");
+    UsbIdAppendHex(buf, cap, &pos, vendor, 4);
+    UsbIdAppendLiteral(buf, cap, &pos, "&DEV_");
+    UsbIdAppendHex(buf, cap, &pos, device, 4);
+    UsbIdAppendLiteral(buf, cap, &pos, "&CC_0C0330");
+}
+
+static void UsbFormatRootHub(char* buf, size_t cap) {
+    size_t pos = 0;
+    if (cap) buf[0] = '\0';
+    UsbIdAppendLiteral(buf, cap, &pos, "USB\\ROOT_HUB");
+}
+
+static void UsbMakeDeviceIds(const XHCI_DEVICE* xdev, IO_DEVICE_ID* ids, char idbuf[][USB_MAX_DEVICE_ID_TEXT], size_t* outCount) {
+    size_t n = 0;
+    UsbFormatVidPidRev(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, xdev->BcdDevice);
+    ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+    n++;
+
+    UsbFormatVidPid(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId);
+    ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+    n++;
+
+    UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass, xdev->DeviceSubclass, xdev->DeviceProtocol);
+    ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+    n++;
+
+    UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass);
+    ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+    n++;
+
+    if (xdev->DeviceClass == 0 || xdev->InterfaceCount > 1) {
+        UsbFormatComposite(idbuf[n], USB_MAX_DEVICE_ID_TEXT);
+        ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+        n++;
+    }
+
+    *outCount = n;
+}
+
+static void UsbMakeInterfaceIds(const XHCI_DEVICE* xdev,
+                                const USB_TRACKED_INTERFACE* iface,
+                                IO_DEVICE_ID* ids,
+                                char idbuf[][USB_MAX_DEVICE_ID_TEXT],
+                                size_t* outCount) {
+    size_t n = 0;
+    const USB_INTERFACE_DESCRIPTOR* d = &iface->Descriptor;
+
+    UsbFormatVidPidMi(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, d->bInterfaceNumber);
+    ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+    n++;
+
+    UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass, d->bInterfaceSubClass, d->bInterfaceProtocol);
+    ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+    n++;
+
+    UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass);
+    ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
+    n++;
+
+    *outCount = n;
+}
+
+KSTATUS UsbInitializeBusModel(void) {
+    if (gUsbBusModelReady)
+        return STATUS_SUCCESS;
+
+    memset(&gUsbBusDriver, 0, sizeof(gUsbBusDriver));
+    IoInitializeDriverObject(&gUsbBusDriver, SYS_SV_LITERAL("usb-bus"));
+    gUsbBusDriver.MajorFunction[IRP_MJ_PNP] = UsbDispatchPdoPnp;
+    gUsbBusDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl;
+
+    KSTATUS st = IoRegisterDriverObject(&gUsbBusDriver);
+    if (!KSUCCESS(st)) return st;
+
+    gUsbBusModelReady = true;
+    return STATUS_SUCCESS;
+}
+
+void UsbInitializeHcd(USB_HCD* hcd,
+                      const USB_HCD_OPERATIONS* ops,
+                      void* controllerContext,
+                      uint8_t controllerIndex,
+                      uint8_t maxRootPorts) {
+    if (!hcd) return;
+    memset(hcd, 0, sizeof(*hcd));
+    hcd->Operations = ops;
+    hcd->ControllerContext = controllerContext;
+    hcd->DriverObject = &gUsbBusDriver;
+    hcd->ControllerIndex = controllerIndex ? controllerIndex : ++gUsbNextControllerIndex;
+    hcd->MaxRootPorts = maxRootPorts;
+}
+
+KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device) {
+    if (!hcd) return STATUS_INVALID_PARAMETER;
+    if (hcd->ControllerPdo) return STATUS_ALREADY_EXISTS;
+
+    IO_DEVICE_ID ids[2];
+    char idbuf[2][USB_MAX_DEVICE_ID_TEXT];
+    UsbFormatPciXhci(idbuf[0], USB_MAX_DEVICE_ID_TEXT, vendor, device);
+    {
+        size_t pos = 0;
+        idbuf[1][0] = '\0';
+        UsbIdAppendLiteral(idbuf[1], USB_MAX_DEVICE_ID_TEXT, &pos, "PCI\\CC_0C0330");
+    }
+    ids[0] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[0]) };
+    ids[1] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[1]) };
+
+    DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
+    KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver, (DEVICE_OBJECT*)0, ids, 2, sizeof(USB_PDO_EXTENSION), &pdo);
+    if (!KSUCCESS(st)) return st;
+    (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("xhci-controller"));
+
+    USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
+    ext->Kind = UsbPdoController;
+    ext->Hcd = hcd;
+    hcd->ControllerPdo = pdo;
+
+    st = IoRegisterPhysicalDevice(pdo);
+    if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
+        return st;
+    return STATUS_SUCCESS;
+}
+
+
+KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) {
+    if (!hcd) return STATUS_INVALID_PARAMETER;
+    if (hcd->RootHubPdo) return STATUS_ALREADY_EXISTS;
+
+    IO_DEVICE_ID ids[2];
+    char idbuf[2][USB_MAX_DEVICE_ID_TEXT];
+    UsbFormatRootHub(idbuf[0], USB_MAX_DEVICE_ID_TEXT);
+    UsbFormatClass(idbuf[1], USB_MAX_DEVICE_ID_TEXT, USB_CLASS_HUB);
+    ids[0] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[0]) };
+    ids[1] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[1]) };
+
+    DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
+    KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
+                                         hcd->ControllerPdo,
+                                         ids,
+                                         2,
+                                         sizeof(USB_PDO_EXTENSION),
+                                         &pdo);
+    if (!KSUCCESS(st)) return st;
+    (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-root-hub"));
+
+    USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
+    ext->Kind = UsbPdoRootHub;
+    ext->Hcd = hcd;
+    ext->Device = (USB_DEVICE*)0;
+    hcd->RootHubPdo = pdo;
+
+    st = IoRegisterPhysicalDevice(pdo);
+    if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
+        return st;
+
+    DbgPrint("usb: created root hub PDO ports=%u\n", hcd->MaxRootPorts);
+    return STATUS_SUCCESS;
+}
+
+KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
+    if (!hcd || !xdev) return STATUS_INVALID_PARAMETER;
+    if (xdev->UsbPdo) return STATUS_ALREADY_EXISTS;
+
+    USB_DEVICE* udev = UsbAllocateDevice();
+    if (!udev) return STATUS_NO_MEMORY;
+
+    udev->Hcd = hcd;
+    udev->HcdDeviceContext = xdev;
+    udev->State = xdev->Configured ? UsbDeviceConfigured : (xdev->Addressed ? UsbDeviceAddressed : UsbDeviceAttached);
+    udev->SlotId = xdev->SlotId;
+    udev->Speed = xdev->Speed;
+    udev->Address = xdev->SlotId;
+    udev->MaxPacketSize0 = xdev->MaxPacketSize0;
+    udev->VendorId = xdev->VendorId;
+    udev->ProductId = xdev->ProductId;
+    udev->BcdUsb = xdev->BcdUsb;
+    udev->BcdDevice = xdev->BcdDevice;
+    udev->DeviceClass = xdev->DeviceClass;
+    udev->DeviceSubclass = xdev->DeviceSubclass;
+    udev->DeviceProtocol = xdev->DeviceProtocol;
+    udev->ConfigurationValue = xdev->ConfigurationValue;
+    udev->Path = xdev->Path;
+    if (udev->Path.RootPort == 0) {
+        udev->Path.RootPort = xdev->Port;
+        udev->Path.Depth = 0;
+        udev->Path.RouteString = 0;
+    }
+
+    IO_DEVICE_ID ids[5];
+    char idbuf[5][USB_MAX_DEVICE_ID_TEXT];
+    size_t idCount = 0;
+    UsbMakeDeviceIds(xdev, ids, idbuf, &idCount);
+
+    DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
+    KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
+                                         parent ? parent : (hcd->RootHubPdo ? hcd->RootHubPdo : hcd->ControllerPdo),
+                                         ids,
+                                         idCount,
+                                         sizeof(USB_PDO_EXTENSION),
+                                         &pdo);
+    if (!KSUCCESS(st)) {
+        UsbDereferenceDevice(udev);
+        return st;
+    }
+    (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-device"));
+
+    udev->Pdo = pdo;
+    IoReferenceDeviceObject(pdo);
+
+    USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
+    ext->Kind = UsbPdoDevice;
+    ext->Hcd = hcd;
+    ext->Device = udev;
+
+    xdev->UsbDevice = udev;
+    xdev->UsbPdo = pdo;
+
+    st = IoRegisterPhysicalDevice(pdo);
+    if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
+        return st;
+
+    DbgPrint("usb: created device PDO slot=%u vid=%04x pid=%04x ids=%llu\n",
+             xdev->SlotId, xdev->VendorId, xdev->ProductId, (unsigned long long)idCount);
+    return STATUS_SUCCESS;
+}
+
+KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
+    if (!hcd || !xdev || !xdev->UsbDevice) return STATUS_INVALID_PARAMETER;
+    DEVICE_OBJECT* deviceParent = parent ? parent : xdev->UsbPdo;
+    if (!deviceParent) return STATUS_INVALID_PARAMETER;
+
+    for (uint8_t i = 0; i < xdev->InterfaceCount; i++) {
+        USB_TRACKED_INTERFACE* iface = &xdev->Interfaces[i];
+        if (!iface->Present)
+            continue;
+
+        IO_DEVICE_ID ids[3];
+        char idbuf[3][USB_MAX_DEVICE_ID_TEXT];
+        size_t idCount = 0;
+        UsbMakeInterfaceIds(xdev, iface, ids, idbuf, &idCount);
+
+        DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
+        KSTATUS st = IoCreatePhysicalDevice(&gUsbCompositeDriver,
+                                             deviceParent,
+                                             ids,
+                                             idCount,
+                                             sizeof(USB_PDO_EXTENSION),
+                                             &pdo);
+        if (!KSUCCESS(st)) {
+            DbgPrint("usb: create interface PDO slot=%u if=%u failed st=0x%x\n",
+                     xdev->SlotId, iface->Descriptor.bInterfaceNumber, st);
+            return st;
+        }
+        (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-interface"));
+
+        USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
+        ext->Kind = UsbPdoInterface;
+        ext->Hcd = hcd;
+        ext->Device = xdev->UsbDevice;
+        ext->InterfaceIndex = i;
+        memcpy(&ext->InterfaceDescriptor, &iface->Descriptor, sizeof(iface->Descriptor));
+        ext->EndpointCount = iface->EndpointCount;
+        for (uint8_t ep = 0; ep < iface->EndpointCount && ep < USB_MAX_TRACKED_ENDPOINTS; ep++)
+            memcpy(&ext->Endpoints[ep], &iface->Endpoints[ep], sizeof(USB_ENDPOINT_DESCRIPTOR));
+
+        st = IoRegisterPhysicalDevice(pdo);
+        if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
+            return st;
+
+        DbgPrint("usb: created interface PDO slot=%u if=%u class=%02x subclass=%02x proto=%02x\n",
+                 xdev->SlotId,
+                 iface->Descriptor.bInterfaceNumber,
+                 iface->Descriptor.bInterfaceClass,
+                 iface->Descriptor.bInterfaceSubClass,
+                 iface->Descriptor.bInterfaceProtocol);
+    }
+    return STATUS_SUCCESS;
+}
+
+
+static DEVICE_OBJECT* UsbReferenceFirstChild(DEVICE_OBJECT* parent) {
+    if (!parent) return (DEVICE_OBJECT*)0;
+    uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
+    DEVICE_OBJECT* child = parent->FirstChild;
+    if (child)
+        IoReferenceDeviceObject(child);
+    KeReleaseSpinlock(&parent->Lock, flags);
+    return child;
+}
+
+static void UsbSendPnpIfAlive(DEVICE_OBJECT* pdo, IO_PNP_MINOR_FUNCTION minor) {
+    if (!pdo || pdo->DeletePending)
+        return;
+    (void)IoSendPnpIrpSync(pdo, minor);
+}
+
+KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) {
+    if (!parent)
+        return STATUS_INVALID_PARAMETER;
+
+    for (;;) {
+        DEVICE_OBJECT* child = UsbReferenceFirstChild(parent);
+        if (!child)
+            break;
+
+        if (child->Type == DeviceObjectPhysical)
+            (void)UsbRemovePdoSubtree(child, surprise);
+        else
+            (void)UsbRemoveDeviceObjectChildren(child, surprise);
+
+        IoDereferenceDeviceObject(child);
+    }
+    return STATUS_SUCCESS;
+}
+
+KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise) {
+    if (!pdo || pdo->Type != DeviceObjectPhysical)
+        return STATUS_INVALID_PARAMETER;
+
+    // Child PDOs may be owned either directly by the PDO or by an attached FDO
+    // such as usb-composite or usb-hub.  The attached driver's PnP handler is
+    // therefore responsible for removing its own child PDOs before forwarding
+    // REMOVE_DEVICE to the lower PDO.
+    (void)UsbRemoveDeviceObjectChildren(pdo, surprise);
+
+    if (surprise)
+        UsbSendPnpIfAlive(pdo, IRP_MN_SURPRISE_REMOVAL);
+
+    UsbSendPnpIfAlive(pdo, IRP_MN_REMOVE_DEVICE);
+    return STATUS_SUCCESS;
+}
+
+static KSTATUS UsbSubmitUrbToHcd(USB_PDO_EXTENSION* ext, USB_URB* urb) {
+    if (!ext || !urb) return STATUS_INVALID_PARAMETER;
+    USB_HCD* hcd = ext->Hcd;
+    if (!hcd || !hcd->Operations || !hcd->Operations->SubmitUrb)
+        return STATUS_NOT_SUPPORTED;
+
+    if (!urb->Device)
+        urb->Device = ext->Device;
+    if (!urb->Device)
+        return STATUS_INVALID_PARAMETER;
+
+    return hcd->Operations->SubmitUrb(hcd, urb);
+}
+
+KSTATUS UsbCancelUrb(USB_URB* urb) {
+    if (!urb || !urb->Device || !urb->Device->Hcd)
+        return STATUS_INVALID_PARAMETER;
+    USB_HCD* hcd = urb->Device->Hcd;
+    if (!hcd->Operations || !hcd->Operations->CancelUrb)
+        return STATUS_NOT_SUPPORTED;
+    return hcd->Operations->CancelUrb(hcd, urb);
+}
+
+KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb) {
+    if (!target || !urb) return STATUS_INVALID_PARAMETER;
+
+    IRP irp;
+    IoInitializeIrp(&irp, 1);
+    IoBuildDeviceIoControlIrp(&irp,
+                              IRP_MJ_INTERNAL_DEVICE_CONTROL,
+                              USB_IOCTL_SUBMIT_URB,
+                              urb,
+                              sizeof(*urb),
+                              urb,
+                              sizeof(*urb));
+    KSTATUS st = IoSubmitIrpSync(target, &irp);
+    return KSUCCESS(st) ? urb->Status : st;
+}
+
+static void UsbAsyncUrbIrpComplete(IRP* irp, void* context) {
+    (void)context;
+    IoFreeIrp(irp);
+}
+
+KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb) {
+    if (!target || !urb) return STATUS_INVALID_PARAMETER;
+
+    IRP* irp = (IRP*)0;
+    KSTATUS st = IoAllocateIrp(1, &irp);
+    if (!KSUCCESS(st)) return st;
+
+    IoBuildDeviceIoControlIrp(irp,
+                              IRP_MJ_INTERNAL_DEVICE_CONTROL,
+                              USB_IOCTL_SUBMIT_URB,
+                              urb,
+                              sizeof(*urb),
+                              urb,
+                              sizeof(*urb));
+    IoSetCompletionRoutine(irp, UsbAsyncUrbIrpComplete, (void*)0);
+
+    st = IoSubmitIrp(target, irp);
+    if (st != STATUS_PENDING && !(irp->Flags & IRP_FLAG_COMPLETED)) {
+        IoFreeIrp(irp);
+        return st;
+    }
+    return st;
+}
+
+KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp) return STATUS_INVALID_PARAMETER;
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios) return STATUS_INVALID_PARAMETER;
+
+    if (ios->Parameters.DeviceIoControl.IoControlCode != USB_IOCTL_SUBMIT_URB) {
+        IoCompleteRequest(irp, STATUS_NOT_SUPPORTED, 0);
+        return STATUS_PENDING;
+    }
+    if (ios->Parameters.DeviceIoControl.InputLength < sizeof(USB_URB) ||
+        !ios->Parameters.DeviceIoControl.InputBuffer) {
+        IoCompleteRequest(irp, STATUS_INVALID_PARAMETER, 0);
+        return STATUS_PENDING;
+    }
+
+    USB_URB* urb = (USB_URB*)ios->Parameters.DeviceIoControl.InputBuffer;
+    USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension;
+    urb->Irp = irp;
+    KSTATUS st = UsbSubmitUrbToHcd(ext, urb);
+    urb->Status = st;
+
+    if (st == STATUS_PENDING) {
+        urb->Pending = true;
+        return STATUS_PENDING;
+    }
+
+    urb->Pending = false;
+    IoCompleteRequest(irp, st, KSUCCESS(st) ? urb->ActualLength : 0);
+    return STATUS_PENDING;
+}
+
+KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp) return STATUS_INVALID_PARAMETER;
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios || ios->MajorFunction != IRP_MJ_PNP)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension;
+    KSTATUS st = STATUS_SUCCESS;
+
+    switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
+        case IRP_MN_START_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            if (ext && ext->Device && ext->Device->State < UsbDeviceConfigured)
+                ext->Device->State = UsbDeviceConfigured;
+            break;
+        case IRP_MN_QUERY_REMOVE_DEVICE:
+            device->PnpState = DevicePnpRemovePending;
+            break;
+        case IRP_MN_CANCEL_REMOVE_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            break;
+        case IRP_MN_SURPRISE_REMOVAL:
+            device->PnpState = DevicePnpSurpriseRemoved;
+            if (ext && ext->Device)
+                ext->Device->State = UsbDeviceSurpriseRemoved;
+            break;
+        case IRP_MN_REMOVE_DEVICE:
+            device->PnpState = DevicePnpDeleted;
+            if (ext && ext->Device)
+                ext->Device->State = UsbDeviceRemoved;
+            IoDeleteDevice(device);
+            break;
+        default:
+            st = STATUS_NOT_SUPPORTED;
+            break;
+    }
+
+    if (st == STATUS_NOT_SUPPORTED)
+        return st;
+    IoCompleteRequest(irp, st, 0);
+    return STATUS_PENDING;
+}

code/drivers/bus/usb/usb_composite.c

@@ -0,0 +1,150 @@
+// usb_composite.c - USB composite function driver and interface-PDO bus.
+#include "usb.h"
+
+DRIVER_OBJECT gUsbCompositeDriver;
+
+static const IO_DEVICE_ID gUsbCompositeIds[] = {
+    { IoBusUsb, { "USB\\COMPOSITE", sizeof("USB\\COMPOSITE") - 1 } },
+};
+
+static const IO_DRIVER_ID_TABLE gUsbCompositeIdTable = {
+    gUsbCompositeIds,
+    sizeof(gUsbCompositeIds) / sizeof(gUsbCompositeIds[0]),
+};
+
+static KSTATUS UsbCompositeCreateInterfaces(USB_COMPOSITE_EXTENSION* ext) {
+    if (!ext || !ext->Device || ext->InterfacesCreated)
+        return STATUS_SUCCESS;
+
+    XHCI_DEVICE* xdev = (XHCI_DEVICE*)ext->Device->HcdDeviceContext;
+    if (!xdev)
+        return STATUS_INVALID_PARAMETER;
+
+    KSTATUS st = UsbCreateInterfacePdos(ext->Device->Hcd, xdev, ext->Self);
+    if (KSUCCESS(st))
+        ext->InterfacesCreated = true;
+    return st;
+}
+
+static KSTATUS UsbCompositeForwardPnp(DEVICE_OBJECT* device, IRP* irp, KSTATUS localStatus) {
+    if (!KSUCCESS(localStatus)) {
+        IoCompleteRequest(irp, localStatus, 0);
+        return STATUS_PENDING;
+    }
+    return IoForwardIrpToLower(device, irp);
+}
+
+KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp)
+        return STATUS_INVALID_PARAMETER;
+
+    // The composite driver owns two different device-object roles:
+    //   * an FDO attached above a USB device PDO, and
+    //   * interface PDOs that it creates for class drivers.
+    // Forwarded PnP IRPs from HID/storage land on the interface PDO, so the
+    // PDO must use the generic USB PDO PnP handler.  Treating its
+    // USB_PDO_EXTENSION as USB_COMPOSITE_EXTENSION corrupts the dispatch path.
+    if (device->Type == DeviceObjectPhysical)
+        return UsbDispatchPdoPnp(device, irp);
+
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios || ios->MajorFunction != IRP_MJ_PNP)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)device->DeviceExtension;
+    KSTATUS st = STATUS_SUCCESS;
+
+    switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
+        case IRP_MN_START_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            if (ext) {
+                if (ext->DevicePdo)
+                    ext->DevicePdo->PnpState = DevicePnpStarted;
+                ext->Started = true;
+                st = UsbCompositeCreateInterfaces(ext);
+            }
+            return UsbCompositeForwardPnp(device, irp, st);
+
+        case IRP_MN_QUERY_REMOVE_DEVICE:
+            device->PnpState = DevicePnpRemovePending;
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_CANCEL_REMOVE_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_SURPRISE_REMOVAL:
+            device->PnpState = DevicePnpSurpriseRemoved;
+            if (ext)
+                ext->Started = false;
+            (void)UsbRemoveDeviceObjectChildren(device, true);
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_REMOVE_DEVICE:
+            device->PnpState = DevicePnpDeleted;
+            if (ext) {
+                ext->Started = false;
+                (void)UsbRemoveDeviceObjectChildren(device, false);
+                if (ext->LowerDevice) {
+                    KSTATUS lower = IoForwardIrpToLower(device, irp);
+                    IoDetachDevice(device);
+                    ext->LowerDevice = (DEVICE_OBJECT*)0;
+                    IoDeleteDevice(device);
+                    if (lower == STATUS_NOT_SUPPORTED) {
+                        IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+                        return STATUS_PENDING;
+                    }
+                    if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
+                        IoCompleteRequest(irp, lower, 0);
+                        return STATUS_PENDING;
+                    }
+                    return lower;
+                }
+            }
+            IoDeleteDevice(device);
+            IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+            return STATUS_PENDING;
+
+        default:
+            return IoForwardIrpToLower(device, irp);
+    }
+}
+
+KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
+    if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
+    if (!pdoExt || pdoExt->Kind != UsbPdoDevice || !pdoExt->Device)
+        return STATUS_INVALID_PARAMETER;
+
+    DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
+    KSTATUS st = IoCreateDevice(driver, sizeof(USB_COMPOSITE_EXTENSION), &fdo);
+    if (!KSUCCESS(st))
+        return st;
+
+    (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-composite"));
+
+    USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)fdo->DeviceExtension;
+    ext->Self = fdo;
+    ext->DevicePdo = physicalDeviceObject;
+    ext->Device = pdoExt->Device;
+    ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
+    if (!ext->LowerDevice) {
+        IoDeleteDevice(fdo);
+        return STATUS_FAILURE;
+    }
+
+    DbgPrint("usb: composite attached slot=%u\n", ext->Device->SlotId);
+    return STATUS_SUCCESS;
+}
+
+KSTATUS UsbInitializeCompositeDriver(void) {
+    memset(&gUsbCompositeDriver, 0, sizeof(gUsbCompositeDriver));
+    IoInitializeDriverObject(&gUsbCompositeDriver, SYS_SV_LITERAL("usb-composite"));
+    gUsbCompositeDriver.IdTable = &gUsbCompositeIdTable;
+    gUsbCompositeDriver.DriverAddDevice = UsbCompositeAddDevice;
+    gUsbCompositeDriver.MajorFunction[IRP_MJ_PNP] = UsbCompositeDispatchPnp;
+    gUsbCompositeDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl;
+    return IoRegisterDriverObject(&gUsbCompositeDriver);
+}

code/drivers/bus/usb/usb_core.c

@@ -25,6 +25,16 @@ KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
+static USB_TRACKED_INTERFACE* UsbFindTrackedInterfaceByNumber(XHCI_DEVICE* dev, uint8_t interfaceNumber) {
+    if (!dev) return (USB_TRACKED_INTERFACE*)0;
+    for (uint8_t i = 0; i < dev->InterfaceCount && i < USB_MAX_TRACKED_INTERFACES; i++) {
+        if (dev->Interfaces[i].Present &&
+            dev->Interfaces[i].Descriptor.bInterfaceNumber == interfaceNumber)
+            return &dev->Interfaces[i];
+    }
+    return (USB_TRACKED_INTERFACE*)0;
+}
+
 void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
                                      const USB_CONFIGURATION_DESCRIPTOR* config,
                                      uint16_t totalLength) {
@@ -79,17 +89,38 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
             massBulkOut = 0;
             tracked = (USB_TRACKED_INTERFACE*)0;
 
-            if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) {
+            if (iface->bAlternateSetting == 0) {
-                tracked = &dev->Interfaces[dev->InterfaceCount++];
+                USB_TRACKED_INTERFACE* existing = UsbFindTrackedInterfaceByNumber(dev, iface->bInterfaceNumber);
-                memset(tracked, 0, sizeof(*tracked));
+                if (existing) {
-                tracked->Present = true;
+                    tracked = existing;
-                memcpy(&tracked->Descriptor, iface, sizeof(*iface));
+                    memset(tracked->Endpoints, 0, sizeof(tracked->Endpoints));
+                    tracked->EndpointCount = 0;
+                    memcpy(&tracked->Descriptor, iface, sizeof(*iface));
+                    UsbTrace("usb: slot=%u duplicate alt0 interface %u replaced existing PDO candidate\n",
+                             dev->SlotId, iface->bInterfaceNumber);
+                } else if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) {
+                    tracked = &dev->Interfaces[dev->InterfaceCount++];
+                    memset(tracked, 0, sizeof(*tracked));
+                    tracked->Present = true;
+                    memcpy(&tracked->Descriptor, iface, sizeof(*iface));
+                } else {
+                    DbgPrint("usb: slot=%u too many interfaces, interface %u alt=0 not published\n",
+                             dev->SlotId, iface->bInterfaceNumber);
+                }
             } else {
-                DbgPrint("usb: slot=%u too many interfaces, interface %u not published\n",
+                /*
-                         dev->SlotId, iface->bInterfaceNumber);
+                 * Alternate settings are not independent USB interfaces.  Real
+                 * UVC cameras often expose one VideoStreaming interface with
+                 * many alternate settings, and publishing each alternate setting
+                 * as a separate PDO causes duplicate interface devices and can
+                 * exhaust USB_MAX_TRACKED_INTERFACES.  Track only alternate
+                 * setting zero until a function driver explicitly selects a
+                 * different setting.
+                 */
+                tracked = (USB_TRACKED_INTERFACE*)0;
             }
 
-            UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)\n",
+            UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)%s\n",
                      dev->SlotId,
                      iface->bInterfaceNumber,
                      iface->bAlternateSetting,
@@ -97,7 +128,8 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
                      currentClass,
                      currentSubClass,
                      currentProtocol,
-                     UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol));
+                     UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol),
+                     iface->bAlternateSetting ? " alt-not-published" : "");
         } else if (type == USB_DESC_ENDPOINT && len >= sizeof(USB_ENDPOINT_DESCRIPTOR)) {
             const USB_ENDPOINT_DESCRIPTOR* ep = (const USB_ENDPOINT_DESCRIPTOR*)desc;
             uint8_t epType = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
@@ -178,54 +210,6 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
-void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
-    for (uint8_t i = 0; i < dev->InterfaceCount; i++) {
-        USB_TRACKED_INTERFACE* iface = &dev->Interfaces[i];
-        if (!iface->Present)
-            continue;
-
-        IO_PUBLISHED_DEVICE pub;
-        memset(&pub, 0, sizeof(pub));
-        pub.BusType = IoBusUsb;
-        pub.Name = "usb-interface";
-        pub.BusContext = xhci;
-        pub.DeviceContext = dev;
-
-        pub.u.Usb.VendorId = dev->VendorId;
-        pub.u.Usb.ProductId = dev->ProductId;
-        pub.u.Usb.BcdUsb = dev->BcdUsb;
-        pub.u.Usb.BcdDevice = dev->BcdDevice;
-        pub.u.Usb.DeviceClass = dev->DeviceClass;
-        pub.u.Usb.DeviceSubclass = dev->DeviceSubclass;
-        pub.u.Usb.DeviceProtocol = dev->DeviceProtocol;
-        pub.u.Usb.SlotId = dev->SlotId;
-        pub.u.Usb.Port = dev->Port;
-        pub.u.Usb.Speed = dev->Speed;
-        pub.u.Usb.ConfigurationValue = dev->ConfigurationValue;
-        pub.u.Usb.InterfaceNumber = iface->Descriptor.bInterfaceNumber;
-        pub.u.Usb.AlternateSetting = iface->Descriptor.bAlternateSetting;
-        pub.u.Usb.InterfaceClass = iface->Descriptor.bInterfaceClass;
-        pub.u.Usb.InterfaceSubclass = iface->Descriptor.bInterfaceSubClass;
-        pub.u.Usb.InterfaceProtocol = iface->Descriptor.bInterfaceProtocol;
-        pub.u.Usb.EndpointCount = iface->EndpointCount;
-        if (pub.u.Usb.EndpointCount > IO_PUBLISHED_USB_MAX_ENDPOINTS)
-            pub.u.Usb.EndpointCount = IO_PUBLISHED_USB_MAX_ENDPOINTS;
-
-        for (uint8_t ep = 0; ep < pub.u.Usb.EndpointCount; ep++) {
-            pub.u.Usb.Endpoints[ep].Address = iface->Endpoints[ep].bEndpointAddress;
-            pub.u.Usb.Endpoints[ep].Attributes = iface->Endpoints[ep].bmAttributes;
-            pub.u.Usb.Endpoints[ep].MaxPacketSize = iface->Endpoints[ep].wMaxPacketSize;
-            pub.u.Usb.Endpoints[ep].Interval = iface->Endpoints[ep].bInterval;
-        }
-
-        KSTATUS st = IoPublishDevice(&pub);
-        if (!KSUCCESS(st)) {
-            DbgPrint("usb: publish interface slot=%u if=%u failed st=0x%x\n",
-                     dev->SlotId, iface->Descriptor.bInterfaceNumber, st);
-        }
-    }
-}
-
 
 KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
                                                const USB_DEVICE_DESCRIPTOR* devDesc) {
@@ -255,43 +239,89 @@ KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     st = XhciSetConfiguration(xhci, dev, config.bConfigurationValue);
     if (!KSUCCESS(st)) return st;
 
-    XhciPublishUsbInterfaces(xhci, dev);
+    if (!dev->UsbPdo) {
+        st = UsbCreateDevicePdo(&xhci->Hcd, dev,
+                                 dev->ParentDeviceObject ? dev->ParentDeviceObject :
+                                     (xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : xhci->Hcd.ControllerPdo));
+        if (!KSUCCESS(st)) {
+            DbgPrint("usb: create device PDO slot=%u failed st=0x%x\n", dev->SlotId, st);
+            return st;
+        }
+    }
 
     return STATUS_SUCCESS;
 }
 
 
-KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
+static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
-    uint8_t speed = 0;
+    if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS)
-    KSTATUS st = XhciResetPort(xhci, port, &speed);
+        return;
-    if (!KSUCCESS(st)) return st;
+
+    XHCI_DEVICE* dev = &xhci->Devices[slotId];
+
+    if (dev->UsbPdo) {
+        (void)UsbRemovePdoSubtree(dev->UsbPdo, true);
+        dev->UsbPdo = (DEVICE_OBJECT*)0;
+    }
+
+    if (dev->UsbDevice) {
+        dev->UsbDevice->State = UsbDeviceRemoved;
+        UsbDereferenceDevice(dev->UsbDevice);
+        dev->UsbDevice = (USB_DEVICE*)0;
+    }
+
+    if (dev->Allocated)
+        (void)XhciDisableSlot(xhci, slotId);
+
+    memset(dev, 0, sizeof(*dev));
+}
+
+
+static KSTATUS XhciEnumeratePathEx(XHCI_CONTROLLER* xhci,
+                                    uint8_t logicalPort,
+                                    uint8_t speed,
+                                    const USB_PORT_PATH* path,
+                                    DEVICE_OBJECT* parentDevice,
+                                    uint8_t* outSlotId) {
+    if (outSlotId) *outSlotId = 0;
+    if (!xhci || !path || path->RootPort == 0)
+        return STATUS_INVALID_PARAMETER;
 
     uint8_t slotId = 0;
-    st = XhciEnableSlot(xhci, &slotId);
+    KSTATUS st = XhciEnableSlot(xhci, &slotId);
     if (!KSUCCESS(st)) return st;
 
     XHCI_DEVICE* dev = &xhci->Devices[slotId];
     memset(dev, 0, sizeof(*dev));
     dev->Allocated = true;
     dev->SlotId = slotId;
-    dev->Port = port;
+    dev->Port = path->RootPort;
     dev->Speed = speed;
+    dev->Path = *path;
+    dev->ParentDeviceObject = parentDevice;
 
     st = XhciAllocateDeviceMemory(xhci, dev);
-    if (!KSUCCESS(st)) return st;
+    if (!KSUCCESS(st)) {
+        XhciRollbackEnumeratedSlot(xhci, slotId);
+        return st;
+    }
 
     XhciBuildDefaultControlContext(xhci, dev);
 
     st = XhciAddressDevice(xhci, dev);
-    if (!KSUCCESS(st)) return st;
+    if (!KSUCCESS(st)) {
+        XhciRollbackEnumeratedSlot(xhci, slotId);
+        return st;
+    }
     dev->Addressed = true;
 
     USB_DEVICE_DESCRIPTOR desc;
     memset(&desc, 0, sizeof(desc));
-    st = XhciGetDeviceDescriptor(xhci, dev, &desc);
+    st = XhciGetDeviceDescriptorTimeout(xhci, dev, &desc, USB_XHCI_ENUM_CONTROL_TIMEOUT_US);
     if (!KSUCCESS(st)) {
         DbgPrint("usb: port %u slot %u descriptor read failed st=0x%x\n",
-                 port, slotId, st);
+                 logicalPort, slotId, st);
+        XhciRollbackEnumeratedSlot(xhci, slotId);
         return st;
     }
 
@@ -304,24 +334,100 @@ KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
     dev->DeviceSubclass = desc.bDeviceSubClass;
     dev->DeviceProtocol = desc.bDeviceProtocol;
 
-    DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n",
+    if (path->BehindHub) {
-             port,
+        DbgPrint("usb: device root-port=%u hub-slot=%u hub-port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x route=%05x\n",
-             slotId,
+                 path->RootPort,
-             XhciPortSpeedName(speed),
+                 path->ParentHubSlot,
-             speed,
+                 path->ParentHubPort,
-             desc.idVendor,
+                 slotId,
-             desc.idProduct,
+                 XhciPortSpeedName(speed),
-             desc.bDeviceClass,
+                 speed,
-             desc.bDeviceSubClass,
+                 desc.idVendor,
-             desc.bDeviceProtocol,
+                 desc.idProduct,
-             desc.bMaxPacketSize0,
+                 desc.bDeviceClass,
-             desc.bNumConfigurations,
+                 desc.bDeviceSubClass,
-             desc.bcdUSB);
+                 desc.bDeviceProtocol,
+                 desc.bMaxPacketSize0,
+                 desc.bNumConfigurations,
+                 desc.bcdUSB,
+                 path->RouteString);
+    } else {
+        DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n",
+                 logicalPort,
+                 slotId,
+                 XhciPortSpeedName(speed),
+                 speed,
+                 desc.idVendor,
+                 desc.idProduct,
+                 desc.bDeviceClass,
+                 desc.bDeviceSubClass,
+                 desc.bDeviceProtocol,
+                 desc.bMaxPacketSize0,
+                 desc.bNumConfigurations,
+                 desc.bcdUSB);
+    }
 
     st = XhciDiscoverAndSetConfiguration(xhci, dev, &desc);
+    if (!KSUCCESS(st)) {
+        XhciRollbackEnumeratedSlot(xhci, slotId);
+        return st;
+    }
+
+    if (outSlotId) *outSlotId = slotId;
+    return STATUS_SUCCESS;
+}
+
+KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId) {
+    if (outSlotId) *outSlotId = 0;
+
+    uint8_t speed = 0;
+    KSTATUS st = XhciResetPort(xhci, port, &speed);
     if (!KSUCCESS(st)) return st;
 
-    return STATUS_SUCCESS;
+    USB_PORT_PATH path;
+    memset(&path, 0, sizeof(path));
+    path.RootPort = port;
+    path.Depth = 0;
+    path.RouteString = 0;
+    path.BehindHub = false;
+
+    DEVICE_OBJECT* parent = xhci && xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : (DEVICE_OBJECT*)0;
+    return XhciEnumeratePathEx(xhci, port, speed, &path, parent, outSlotId);
+}
+
+KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci,
+                               USB_HUB_EXTENSION* hub,
+                               uint8_t hubPort,
+                               uint8_t speed,
+                               uint8_t* outSlotId) {
+    if (outSlotId) *outSlotId = 0;
+    if (!xhci || !hub || !hub->Device || !hub->Self || hubPort == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_DEVICE* parent = hub->Device;
+    if (parent->Path.Depth >= 5u) {
+        DbgPrint("usb-hub: hub slot=%u port=%u exceeds xHCI route depth\n",
+                 parent->SlotId, hubPort);
+        return STATUS_NOT_SUPPORTED;
+    }
+    if (hubPort > 15u) {
+        DbgPrint("usb-hub: hub slot=%u port=%u cannot be represented in xHCI route string yet\n",
+                 parent->SlotId, hubPort);
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    USB_PORT_PATH path = parent->Path;
+    path.BehindHub = true;
+    path.ParentHubSlot = parent->SlotId;
+    path.ParentHubPort = hubPort;
+    path.RouteString |= ((uint32_t)hubPort << (path.Depth * 4u));
+    path.Depth++;
+
+    return XhciEnumeratePathEx(xhci, hubPort, speed, &path, hub->Self, outSlotId);
+}
+
+KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
+    return XhciEnumeratePortEx(xhci, port, (uint8_t*)0);
 }
 
 
@@ -335,7 +441,8 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
             continue;
 
         attempted++;
-        KSTATUS st = XhciEnumeratePort(xhci, port);
+        uint8_t slot = 0;
+        KSTATUS st = XhciEnumeratePortEx(xhci, port, &slot);
         if (KSUCCESS(st)) configured++;
         else DbgPrint("usb: root-port %u enumeration failed st=0x%x\n", port, st);
     }
@@ -343,19 +450,3 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
     DbgPrint("usb: xHCI root-port enumeration complete; attempted=%u configured=%u\n",
              attempted, configured);
 }
-
-
-KSTATUS UsbRegisterClassMatchers(void) {
-    memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
-    gUsbStorageDriver.Name = "usbstor";
-    gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
-    gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
-
-    KSTATUS st = IoRegisterDriverMatcher("usbhid", IoBusUsb, UsbHidMatch, UsbHidBind, NULL);
-    if (!KSUCCESS(st)) return st;
-
-    st = IoRegisterDriverMatcher("usbstor", IoBusUsb, UsbStorageMatch, UsbStorageBind, NULL);
-    if (!KSUCCESS(st)) return st;
-
-    return STATUS_SUCCESS;
-}

code/drivers/bus/usb/usb_hid.c

@@ -125,7 +125,7 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
 }
 
 
-void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
+static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
                                          uint32_t completionCode, uint8_t slotId,
                                          uint8_t endpointId, uint32_t residue) {
     if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return;
@@ -158,11 +158,10 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
 
 
 KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
-    if (xhci->HidDeviceCount == 0)
+    if (!xhci || xhci->HidDeviceCount == 0)
         return STATUS_SUCCESS;
 
-    XhciArmAllHid(xhci);
+    DbgPrint("usb: HID interrupt URBs active devices=%u\n", xhci->HidDeviceCount);
-    DbgPrint("usb: HID interrupt input armed devices=%u\n", xhci->HidDeviceCount);
     return STATUS_SUCCESS;
 }
 
@@ -365,3 +364,412 @@ KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context) {
     }
     return st;
 }
+
+// -------------------------------------------------------------------------
+// PDO/FDO boot HID function driver path.
+// -------------------------------------------------------------------------
+
+DRIVER_OBJECT gUsbHidDriver;
+
+static const IO_DEVICE_ID gUsbHidIds[] = {
+    { IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_01", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_01") - 1 } },
+    { IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_02", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_02") - 1 } },
+};
+
+static const IO_DRIVER_ID_TABLE gUsbHidIdTable = {
+    gUsbHidIds,
+    sizeof(gUsbHidIds) / sizeof(gUsbHidIds[0]),
+};
+
+static const USB_ENDPOINT_DESCRIPTOR* UsbHidFindInterruptInPdo(const USB_PDO_EXTENSION* ext) {
+    if (!ext) return (const USB_ENDPOINT_DESCRIPTOR*)0;
+
+    for (uint8_t i = 0; i < ext->EndpointCount; i++) {
+        const USB_ENDPOINT_DESCRIPTOR* ep = &ext->Endpoints[i];
+        uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
+        bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0;
+        if (type == USB_ENDPOINT_TYPE_INTERRUPT && in)
+            return ep;
+    }
+    return (const USB_ENDPOINT_DESCRIPTOR*)0;
+}
+
+
+static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
+    if (!ext || len < 8) return;
+
+    uint8_t modifiers = report[0];
+    for (uint8_t bit = 0; bit < 8; bit++) {
+        uint8_t mask = (uint8_t)(1u << bit);
+        bool wasDown = (ext->LastModifiers & mask) != 0;
+        bool isDown = (modifiers & mask) != 0;
+        if (wasDown != isDown)
+            XhciSubmitHidKey(XhciHidModifierKey(bit), isDown);
+    }
+
+    const uint8_t* keys = &report[2];
+    for (uint8_t i = 0; i < 6; i++) {
+        uint8_t oldKey = ext->LastKeys[i];
+        if (oldKey > 1 && !XhciHidKeyInReport(keys, oldKey))
+            XhciSubmitHidKey((INPUT_KEY)oldKey, false);
+    }
+    for (uint8_t i = 0; i < 6; i++) {
+        uint8_t newKey = keys[i];
+        if (newKey > 1 && !XhciHidKeyInReport(ext->LastKeys, newKey))
+            XhciSubmitHidKey((INPUT_KEY)newKey, true);
+    }
+
+    ext->LastModifiers = modifiers;
+    memcpy(ext->LastKeys, keys, sizeof(ext->LastKeys));
+}
+
+static void UsbHidProcessBootMouseReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
+    if (!ext || len < 3) return;
+
+    int dx = (int)(int8_t)report[1];
+    int dy = (int)(int8_t)report[2];
+    int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0;
+    uint8_t buttons = report[0] & 0x07u;
+
+    if (dx || dy || wheel || buttons != ext->LastButtons) {
+        INPUT_MOUSE_EVENT ev;
+        ev.dx = dx;
+        ev.dy = dy;
+        ev.wheel = wheel;
+        ev.buttons = buttons;
+        KeInputSubmitMouse(&ev);
+    }
+
+    ext->LastButtons = buttons;
+}
+
+static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext);
+
+static void UsbHidInterruptComplete(USB_URB* urb, void* context) {
+    USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)context;
+    if (!ext || urb != &ext->InterruptUrb)
+        return;
+
+    ext->InterruptPending = false;
+    if (ext->SurpriseRemoved || ext->Removed || !ext->Started)
+        return;
+
+    if (KSUCCESS(urb->Status)) {
+        uint16_t length = (uint16_t)urb->ActualLength;
+        if (length > ext->ReportLength) length = ext->ReportLength;
+
+        if (ext->Keyboard)
+            UsbHidProcessBootKeyboardReport(ext, ext->ReportBuffer, length);
+        else if (ext->Mouse)
+            UsbHidProcessBootMouseReport(ext, ext->ReportBuffer, length);
+    } else if (urb->Status != STATUS_CANCELLED) {
+        DbgPrint("usb: HID interrupt URB failed slot=%u if=%u ep=%02x st=0x%x\n",
+                 ext->Device ? ext->Device->SlotId : 0,
+                 ext->InterfaceNumber,
+                 ext->EndpointAddress,
+                 urb->Status);
+    }
+
+    if (!ext->SurpriseRemoved && !ext->Removed)
+        (void)UsbHidArmInterrupt(ext);
+}
+
+static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) {
+    if (!ext || !ext->LowerDevice || !ext->Device)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_URB urb;
+    memset(&urb, 0, sizeof(urb));
+    urb.Function = UrbFunctionControlTransfer;
+    urb.Direction = UsbTransferNoData;
+    urb.Device = ext->Device;
+    urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
+    urb.Setup.bRequest = USB_REQ_SET_PROTOCOL;
+    urb.Setup.wValue = 0;
+    urb.Setup.wIndex = ext->InterfaceNumber;
+    urb.Setup.wLength = 0;
+
+    KSTATUS st = UsbSubmitUrbSync(ext->LowerDevice, &urb);
+    if (!KSUCCESS(st)) {
+        DbgPrint("usb: HID SET_PROTOCOL boot failed slot=%u if=%u st=0x%x\n",
+                 ext->Device->SlotId, ext->InterfaceNumber, st);
+        return st;
+    }
+
+    memset(&urb, 0, sizeof(urb));
+    urb.Function = UrbFunctionControlTransfer;
+    urb.Direction = UsbTransferNoData;
+    urb.Device = ext->Device;
+    urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
+    urb.Setup.bRequest = USB_REQ_SET_IDLE;
+    urb.Setup.wValue = 0;
+    urb.Setup.wIndex = ext->InterfaceNumber;
+    urb.Setup.wLength = 0;
+
+    st = UsbSubmitUrbSync(ext->LowerDevice, &urb);
+    if (!KSUCCESS(st)) {
+        DbgPrint("usb: HID SET_IDLE failed slot=%u if=%u st=0x%x; continuing\n",
+                 ext->Device->SlotId, ext->InterfaceNumber, st);
+    }
+
+    DbgPrint("usb: HID boot protocol selected slot=%u if=%u proto=%u\n",
+             ext->Device->SlotId, ext->InterfaceNumber, ext->Protocol);
+    return STATUS_SUCCESS;
+}
+
+static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
+    if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed)
+        return STATUS_OBJECT_DEAD;
+    if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys)
+        return STATUS_INVALID_PARAMETER;
+    if (ext->InterruptPending)
+        return STATUS_SUCCESS;
+
+    memset(ext->ReportBuffer, 0, ext->ReportLength);
+    memset(&ext->InterruptUrb, 0, sizeof(ext->InterruptUrb));
+    ext->InterruptUrb.Function = UrbFunctionInterruptTransfer;
+    ext->InterruptUrb.Direction = UsbTransferIn;
+    ext->InterruptUrb.Device = ext->Device;
+    ext->InterruptUrb.Pipe = ext->InterruptPipe;
+    ext->InterruptUrb.Buffer = ext->ReportBuffer;
+    ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys;
+    ext->InterruptUrb.Length = ext->ReportLength;
+    ext->InterruptUrb.Complete = UsbHidInterruptComplete;
+    ext->InterruptUrb.CompleteContext = ext;
+
+    KSTATUS st = UsbSubmitUrbAsync(ext->LowerDevice, &ext->InterruptUrb);
+    if (st != STATUS_PENDING && !KSUCCESS(st))
+        return st;
+    if (ext->InterruptUrb.Status != STATUS_PENDING && ext->InterruptUrb.Status != STATUS_SUCCESS)
+        return ext->InterruptUrb.Status;
+
+    ext->InterruptPending = true;
+    return STATUS_SUCCESS;
+}
+
+static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) {
+    if (!device || !ext || !ext->Device || !ext->InterfacePdo)
+        return STATUS_INVALID_PARAMETER;
+    if (ext->Started)
+        return STATUS_SUCCESS;
+    if (ext->SurpriseRemoved || ext->Removed)
+        return STATUS_OBJECT_DEAD;
+
+    USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)ext->InterfacePdo->DeviceExtension;
+    if (!pdoExt || pdoExt->Kind != UsbPdoInterface)
+        return STATUS_INVALID_PARAMETER;
+
+    const USB_ENDPOINT_DESCRIPTOR* ep = UsbHidFindInterruptInPdo(pdoExt);
+    if (!ep)
+        return STATUS_NOT_FOUND;
+
+    ext->Keyboard = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD);
+    ext->Mouse = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_MOUSE);
+    ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
+    ext->Protocol = pdoExt->InterfaceDescriptor.bInterfaceProtocol;
+    ext->EndpointAddress = ep->bEndpointAddress;
+    ext->MaxPacketSize = ep->wMaxPacketSize;
+    ext->Interval = ep->bInterval;
+
+    uint16_t reportLength = ext->MaxPacketSize & 0x07FFu;
+    if (reportLength == 0) reportLength = ext->Keyboard ? 8u : 4u;
+    if (ext->Keyboard && reportLength < 8) reportLength = 8;
+    if (ext->Mouse && reportLength < 3) reportLength = 3;
+    if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT;
+    ext->ReportLength = reportLength;
+
+    ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys);
+    if (!ext->ReportBuffer)
+        return STATUS_NO_MEMORY;
+
+    KSTATUS st = UsbCreatePipe(ext->Device,
+                               UsbPipeInterrupt,
+                               ext->EndpointAddress,
+                               ext->MaxPacketSize,
+                               ext->Interval,
+                               &ext->InterruptPipe);
+    if (!KSUCCESS(st))
+        return st;
+
+    st = UsbHidSendBootProtocol(ext);
+    if (!KSUCCESS(st))
+        return st;
+
+    memset(ext->LastKeys, 0, sizeof(ext->LastKeys));
+    ext->LastModifiers = 0;
+    ext->LastButtons = 0;
+
+    KeInputRegisterDevice(ext->Keyboard ? "usb-hid-keyboard" : "usb-hid-mouse", 0, &ext->InputDeviceId);
+
+    ext->Started = true;
+    device->PnpState = DevicePnpStarted;
+    if (ext->InterfacePdo)
+        ext->InterfacePdo->PnpState = DevicePnpStarted;
+
+    XHCI_CONTROLLER* xhci = ext->Device && ext->Device->Hcd ?
+        (XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
+
+    st = UsbHidArmInterrupt(ext);
+    if (!KSUCCESS(st))
+        return st;
+
+    if (xhci)
+        xhci->HidDeviceCount++;
+
+    DbgPrint("usb: HID boot %s bound slot=%u if=%u ep=%02x maxpkt=%u interval=%u input=%u urb\n",
+             ext->Keyboard ? "keyboard" : "mouse",
+             ext->Device->SlotId,
+             ext->InterfaceNumber,
+             ext->EndpointAddress,
+             ext->MaxPacketSize,
+             ext->Interval,
+             ext->InputDeviceId);
+    return STATUS_SUCCESS;
+}
+
+
+KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp)
+        return STATUS_INVALID_PARAMETER;
+
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios || ios->MajorFunction != IRP_MJ_PNP)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)device->DeviceExtension;
+    KSTATUS st = STATUS_SUCCESS;
+
+    switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
+        case IRP_MN_START_DEVICE:
+            st = UsbHidStartDevice(device, ext);
+            if (!KSUCCESS(st)) {
+                IoCompleteRequest(irp, st, 0);
+                return STATUS_PENDING;
+            }
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_QUERY_REMOVE_DEVICE:
+            device->PnpState = DevicePnpRemovePending;
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_CANCEL_REMOVE_DEVICE:
+            if (!ext || !ext->SurpriseRemoved)
+                device->PnpState = DevicePnpStarted;
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_SURPRISE_REMOVAL:
+            device->PnpState = DevicePnpSurpriseRemoved;
+            if (ext) {
+                ext->SurpriseRemoved = true;
+                ext->Started = false;
+                if (ext->InterruptPending)
+                    (void)UsbCancelUrb(&ext->InterruptUrb);
+            }
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_REMOVE_DEVICE: {
+            device->PnpState = DevicePnpDeleted;
+            if (ext) {
+                ext->Removed = true;
+                ext->Started = false;
+                if (ext->InterruptPending)
+                    (void)UsbCancelUrb(&ext->InterruptUrb);
+                if (ext->InterruptPipe) {
+                    ext->InterruptPipe->Open = false;
+                    UsbDereferencePipe(ext->InterruptPipe);
+                    ext->InterruptPipe = (USB_PIPE*)0;
+                }
+                if (ext->Device && ext->Device->Hcd) {
+                    XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext;
+                    if (xhci && xhci->HidDeviceCount)
+                        xhci->HidDeviceCount--;
+                }
+                if (ext->RemoveLockReady) {
+                    IoReleaseRemoveLockAndWait(&ext->RemoveLock, irp);
+                    IoDestroyRemoveLock(&ext->RemoveLock);
+                    ext->RemoveLockReady = false;
+                }
+            }
+
+            KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS;
+            if (ext && ext->LowerDevice) {
+                IoDetachDevice(device);
+                ext->LowerDevice = (DEVICE_OBJECT*)0;
+            }
+            IoDeleteDevice(device);
+            if (lower == STATUS_NOT_SUPPORTED) {
+                IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+                return STATUS_PENDING;
+            }
+            if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
+                IoCompleteRequest(irp, lower, 0);
+                return STATUS_PENDING;
+            }
+            return lower;
+        }
+
+        default:
+            return IoForwardIrpToLower(device, irp);
+    }
+}
+
+KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
+    if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
+    if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device)
+        return STATUS_INVALID_PARAMETER;
+    if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_HID ||
+        pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_HID_SUBCLASS_BOOT)
+        return STATUS_INVALID_PARAMETER;
+    if (pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_KEYBOARD &&
+        pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_MOUSE)
+        return STATUS_INVALID_PARAMETER;
+    if (!UsbHidFindInterruptInPdo(pdoExt))
+        return STATUS_NOT_FOUND;
+
+    DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
+    KSTATUS st = IoCreateDevice(driver, sizeof(USB_HID_EXTENSION), &fdo);
+    if (!KSUCCESS(st))
+        return st;
+
+    (void)IoSetDeviceName(fdo,
+                          pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD
+                              ? SYS_SV_LITERAL("usb-hid-keyboard")
+                              : SYS_SV_LITERAL("usb-hid-mouse"));
+
+    USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)fdo->DeviceExtension;
+    ext->Self = fdo;
+    ext->InterfacePdo = physicalDeviceObject;
+    ext->Device = pdoExt->Device;
+    st = IoInitializeRemoveLock(&ext->RemoveLock);
+    if (!KSUCCESS(st)) {
+        IoDeleteDevice(fdo);
+        return st;
+    }
+    ext->RemoveLockReady = true;
+
+    ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
+    if (!ext->LowerDevice) {
+        IoDestroyRemoveLock(&ext->RemoveLock);
+        ext->RemoveLockReady = false;
+        IoDeleteDevice(fdo);
+        return STATUS_FAILURE;
+    }
+
+    DbgPrint("usb: HID FDO attached slot=%u if=%u proto=%u\n",
+             ext->Device->SlotId,
+             pdoExt->InterfaceDescriptor.bInterfaceNumber,
+             pdoExt->InterfaceDescriptor.bInterfaceProtocol);
+    return STATUS_SUCCESS;
+}
+
+KSTATUS UsbInitializeHidDriver(void) {
+    memset(&gUsbHidDriver, 0, sizeof(gUsbHidDriver));
+    IoInitializeDriverObject(&gUsbHidDriver, SYS_SV_LITERAL("usb-hid"));
+    gUsbHidDriver.IdTable = &gUsbHidIdTable;
+    gUsbHidDriver.DriverAddDevice = UsbHidAddDevice;
+    gUsbHidDriver.MajorFunction[IRP_MJ_PNP] = UsbHidDispatchPnp;
+    return IoRegisterDriverObject(&gUsbHidDriver);
+}

code/drivers/bus/usb/usb_storage.c

@@ -1,53 +1,115 @@
-// usb_storage.c - USB mass-storage BOT/SCSI class binding and disk bridge.
+// usb_storage.c - USB mass-storage BOT/SCSI class driver.
+//
+// The storage driver now binds as a USB function driver to an interface PDO
+// (USB\CLASS_08&SUBCLASS_06&PROT_50).  It creates a disk FDO above that
+// interface stack and issues BOT transfers through USB URBs.  The xHCI HCD
+// still backs the bulk URBs with the existing per-device rings; later stages
+// will make endpoint rings fully pipe-owned and asynchronous.
 #include "usb.h"
 
 DRIVER_OBJECT gUsbStorageDriver;
 
-KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+static const IO_DEVICE_ID gUsbStorageIds[] = {
-                                PHYSICAL_ADDRESS phys, uint32_t length,
+    { IoBusUsb, { "USB\\CLASS_08&SUBCLASS_06&PROT_50", sizeof("USB\\CLASS_08&SUBCLASS_06&PROT_50") - 1 } },
-                                uint32_t* outActual) {
+};
-    return XhciBulkTransfer(xhci,
+
-                            dev,
+static const IO_DRIVER_ID_TABLE gUsbStorageIdTable = {
-                            dev->MassBulkInEndpointId,
+    gUsbStorageIds,
-                            dev->MassInRing,
+    sizeof(gUsbStorageIds) / sizeof(gUsbStorageIds[0]),
-                            dev->MassInRingPhys,
+};
-                            &dev->MassInEnqueue,
+
-                            &dev->MassInCycle,
+static USB_STORAGE_EXTENSION* UsbStorageFromDiskDevice(DEVICE_OBJECT* device) {
-                            phys,
+    if (!device || !device->DeviceExtension)
-                            length,
+        return (USB_STORAGE_EXTENSION*)0;
-                            outActual);
+
+    USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)device->DeviceExtension;
+    if (diskExt->Kind != UsbStorageExtensionDisk)
+        return (USB_STORAGE_EXTENSION*)0;
+    return diskExt->Storage;
 }
 
+static USB_STORAGE_EXTENSION* UsbStorageFromInterfaceDevice(DEVICE_OBJECT* device) {
+    if (!device || !device->DeviceExtension)
+        return (USB_STORAGE_EXTENSION*)0;
 
-KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+    USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)device->DeviceExtension;
-                                 PHYSICAL_ADDRESS phys, uint32_t length,
+    if (ext->Kind != UsbStorageExtensionInterface)
-                                 uint32_t* outActual) {
+        return (USB_STORAGE_EXTENSION*)0;
-    return XhciBulkTransfer(xhci,
+    return ext;
-                            dev,
-                            dev->MassBulkOutEndpointId,
-                            dev->MassOutRing,
-                            dev->MassOutRingPhys,
-                            &dev->MassOutEnqueue,
-                            &dev->MassOutCycle,
-                            phys,
-                            length,
-                            outActual);
 }
 
+static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_EXTENSION* pdoExt, bool wantIn) {
+    if (!pdoExt) return (const USB_ENDPOINT_DESCRIPTOR*)0;
 
-KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+    for (uint8_t i = 0; i < pdoExt->EndpointCount; i++) {
-                                    const uint8_t* cdb, uint8_t cdbLen,
+        const USB_ENDPOINT_DESCRIPTOR* ep = &pdoExt->Endpoints[i];
-                                    void* data, PHYSICAL_ADDRESS dataPhys,
+        uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
-                                    uint32_t dataLength, bool dataIn) {
+        bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0;
-    if (!xhci || !dev || !cdb || cdbLen == 0 || cdbLen > 16)
+        if (type == USB_ENDPOINT_TYPE_BULK && in == wantIn)
+            return ep;
+    }
+    return (const USB_ENDPOINT_DESCRIPTOR*)0;
+}
+
+static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
+                                    bool dataIn,
+                                    PHYSICAL_ADDRESS phys,
+                                    uint32_t length,
+                                    uint32_t* outActual) {
+    if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe;
+    if (!pipe || !pipe->Open)
+        return STATUS_OBJECT_DEAD;
+
+    USB_URB urb;
+    memset(&urb, 0, sizeof(urb));
+    urb.Function = UrbFunctionBulkTransfer;
+    urb.Direction = dataIn ? UsbTransferIn : UsbTransferOut;
+    urb.Device = storage->Device;
+    urb.Pipe = pipe;
+    urb.Buffer = (void*)0;
+    urb.BufferPhys = phys;
+    urb.Length = length;
+
+    KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb);
+    if (outActual)
+        *outActual = KSUCCESS(st) ? urb.ActualLength : 0;
+    return st;
+}
+
+KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage,
+                         PHYSICAL_ADDRESS phys,
+                         uint32_t length,
+                         uint32_t* outActual) {
+    return UsbStorageSubmitBulk(storage, true, phys, length, outActual);
+}
+
+KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage,
+                          PHYSICAL_ADDRESS phys,
+                          uint32_t length,
+                          uint32_t* outActual) {
+    return UsbStorageSubmitBulk(storage, false, phys, length, outActual);
+}
+
+KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
+                             const uint8_t* cdb,
+                             uint8_t cdbLen,
+                             void* data,
+                             PHYSICAL_ADDRESS dataPhys,
+                             uint32_t dataLength,
+                             bool dataIn) {
+    if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16)
         return STATUS_INVALID_PARAMETER;
     if (dataLength && (!data || !dataPhys))
         return STATUS_INVALID_PARAMETER;
 
+    XHCI_DEVICE* dev = storage->XhciDevice;
     USB_MASS_CBW* cbw = (USB_MASS_CBW*)dev->DescriptorBuffer;
     memset(cbw, 0, sizeof(*cbw));
     cbw->Signature = USB_MASS_BOT_CBW_SIGNATURE;
-    cbw->Tag = ++dev->MassTag;
+    cbw->Tag = ++storage->Tag;
     cbw->DataTransferLength = dataLength;
     cbw->Flags = dataIn ? USB_DIR_IN : USB_DIR_OUT;
     cbw->Lun = 0;
@@ -55,16 +117,14 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     memcpy(cbw->CommandBlock, cdb, cdbLen);
 
     uint32_t actual = 0;
-    KSTATUS st = UsbStorageBulkOut(xhci, dev, dev->DescriptorPhys, sizeof(*cbw), &actual);
+    KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual);
     if (!KSUCCESS(st) || actual != sizeof(*cbw))
         return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
 
     if (dataLength) {
         actual = 0;
-        if (dataIn)
+        st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual)
-            st = UsbStorageBulkIn(xhci, dev, dataPhys, dataLength, &actual);
+                    : UsbStorageBulkOut(storage, dataPhys, dataLength, &actual);
-        else
-            st = UsbStorageBulkOut(xhci, dev, dataPhys, dataLength, &actual);
         if (!KSUCCESS(st))
             return st;
     }
@@ -72,13 +132,13 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer;
     memset(csw, 0, sizeof(*csw));
     actual = 0;
-    st = UsbStorageBulkIn(xhci, dev, dev->DescriptorPhys, sizeof(*csw), &actual);
+    st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual);
     if (!KSUCCESS(st) || actual != sizeof(*csw))
         return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
 
-    if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != dev->MassTag) {
+    if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != storage->Tag) {
         DbgPrint("usb-storage: invalid CSW slot=%u sig=%08x tag=%08x expected=%08x\n",
-                 dev->SlotId, csw->Signature, csw->Tag, dev->MassTag);
+                 dev->SlotId, csw->Signature, csw->Tag, storage->Tag);
         return STATUS_DEVICE_ERROR;
     }
     if (csw->Status != USB_MASS_BOT_CSW_PASSED) {
@@ -90,13 +150,11 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     return STATUS_SUCCESS;
 }
 
-
 uint32_t UsbBe32(const uint8_t* p) {
     return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
            ((uint32_t)p[2] << 8) | (uint32_t)p[3];
 }
 
-
 void UsbStoreBe32(uint8_t* p, uint32_t v) {
     p[0] = (uint8_t)(v >> 24);
     p[1] = (uint8_t)(v >> 16);
@@ -104,17 +162,19 @@ void UsbStoreBe32(uint8_t* p, uint32_t v) {
     p[3] = (uint8_t)v;
 }
 
-
 void UsbStoreBe16(uint8_t* p, uint16_t v) {
     p[0] = (uint8_t)(v >> 8);
     p[1] = (uint8_t)v;
 }
 
+KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
+    if (!storage || !storage->XhciDevice)
+        return STATUS_INVALID_PARAMETER;
 
-KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
+    XHCI_DEVICE* dev = storage->XhciDevice;
     uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 };
     memset(dev->MassBuffer, 0, 36);
-    KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
+    KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
                                       dev->MassBuffer, dev->MassBufferPhys,
                                       36, true);
     if (!KSUCCESS(st)) return st;
@@ -128,37 +188,46 @@ KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     return STATUS_SUCCESS;
 }
 
+KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) {
+    if (!storage || !storage->XhciDevice)
+        return STATUS_INVALID_PARAMETER;
 
-KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
+    XHCI_DEVICE* dev = storage->XhciDevice;
     uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
     memset(dev->MassBuffer, 0, 18);
-    return UsbStorageBotCommand(xhci, dev, rs, sizeof(rs),
+    return UsbStorageBotCommand(storage, rs, sizeof(rs),
                                 dev->MassBuffer, dev->MassBufferPhys, 18, true);
 }
 
+KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
+    if (!storage || !storage->XhciDevice)
+        return STATUS_INVALID_PARAMETER;
 
-KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     uint8_t cdb[6] = { SCSI_TEST_UNIT_READY, 0, 0, 0, 0, 0 };
-
     KSTATUS last = STATUS_DEVICE_ERROR;
+    XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext;
+
     for (uint32_t attempt = 0; attempt < 10; attempt++) {
-        KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
+        KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
                                           (void*)0, 0, 0, false);
         if (KSUCCESS(st)) return STATUS_SUCCESS;
 
         last = st;
-        (void)UsbStorageRequestSense(xhci, dev);
+        (void)UsbStorageRequestSense(storage);
         UsbDelayUs(xhci->Hal, 10000u);
     }
 
     return last;
 }
 
+KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) {
+    if (!storage || !storage->XhciDevice)
+        return STATUS_INVALID_PARAMETER;
 
-KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
+    XHCI_DEVICE* dev = storage->XhciDevice;
     uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
     memset(dev->MassBuffer, 0, 8);
-    KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
+    KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
                                       dev->MassBuffer, dev->MassBufferPhys,
                                       8, true);
     if (!KSUCCESS(st)) return st;
@@ -168,27 +237,33 @@ KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     if (blockSize == 0)
         return STATUS_DEVICE_ERROR;
 
+    storage->BlockSize = blockSize;
+    storage->BlockCount = (uint64_t)lastLba + 1ull;
+
     dev->MassBlockSize = blockSize;
-    dev->MassBlockCount = (uint64_t)lastLba + 1ull;
+    dev->MassBlockCount = storage->BlockCount;
+
     DbgPrint("usb-storage: capacity slot=%u blocks=%llu block_size=%u (%llu MB)\n",
              dev->SlotId,
-             (unsigned long long)dev->MassBlockCount,
+             (unsigned long long)storage->BlockCount,
-             dev->MassBlockSize,
+             storage->BlockSize,
-             (unsigned long long)((dev->MassBlockCount * (uint64_t)dev->MassBlockSize) / (1024ull * 1024ull)));
+             (unsigned long long)((storage->BlockCount * (uint64_t)storage->BlockSize) / (1024ull * 1024ull)));
     return STATUS_SUCCESS;
 }
 
-
+KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
-KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+                                   bool write,
-                                          bool write, uint64_t lba,
+                                   uint64_t lba,
-                                          uint32_t sectors, void* buffer) {
+                                   uint32_t sectors,
-    if (!xhci || !dev || !buffer || sectors == 0)
+                                   void* buffer) {
+    if (!storage || !storage->XhciDevice || !buffer || sectors == 0)
         return STATUS_INVALID_PARAMETER;
-    if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE)
+    if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE)
         return STATUS_NOT_SUPPORTED;
     if (lba > 0xFFFFFFFFull || sectors > 0xFFFFu)
         return STATUS_INVALID_PARAMETER;
 
+    XHCI_DEVICE* dev = storage->XhciDevice;
     uint32_t bytes = sectors * USB_STORAGE_SECTOR_SIZE;
     if (bytes > USB_STORAGE_MAX_BYTES)
         return STATUS_INVALID_PARAMETER;
@@ -202,7 +277,7 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     if (write)
         memcpy(dev->MassBuffer, buffer, bytes);
 
-    KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
+    KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
                                       dev->MassBuffer, dev->MassBufferPhys,
                                       bytes, !write);
     if (!KSUCCESS(st)) return st;
@@ -212,42 +287,67 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     return STATUS_SUCCESS;
 }
 
-
+void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
-void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
+    if (!storage) return;
-    while (_InterlockedCompareExchange(&dev->MassIoBusy, 1, 0) != 0) {
+    XHCI_CONTROLLER* xhci = storage->Device ? (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
-        UsbDelayUs(xhci->Hal, 10);
+    while (_InterlockedCompareExchange(&storage->IoBusy, 1, 0) != 0) {
+        if (xhci) UsbDelayUs(xhci->Hal, 10);
     }
 }
 
-
+void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage) {
-void UsbStorageReleaseIo(XHCI_DEVICE* dev) {
+    if (storage) _InterlockedExchange(&storage->IoBusy, 0);
-    _InterlockedExchange(&dev->MassIoBusy, 0);
 }
 
-
 KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
-    if (!device || !irp || !device->DeviceExtension)
+    if (!device || !irp)
         return STATUS_INVALID_PARAMETER;
 
-    XHCI_DEVICE* dev = (XHCI_DEVICE*)device->DeviceExtension;
+    USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device);
-    if (!dev->MassActive || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
+    if (!storage || !storage->RemoveLockReady)
-        IoCompleteRequest(irp, STATUS_DEVICE_ERROR, 0);
+        return STATUS_OBJECT_DEAD;
+
+    KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp);
+    if (!KSUCCESS(st))
+        return st;
+
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios) {
+        IoReleaseRemoveLock(&storage->RemoveLock, irp);
+        return STATUS_INVALID_PARAMETER;
+    }
+    if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE) {
+        IoReleaseRemoveLock(&storage->RemoveLock, irp);
+        return STATUS_INVALID_PARAMETER;
+    }
+    if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
+        (ios->Parameters.ReadWrite.Length & 511u) ||
+        ios->Parameters.ReadWrite.Length == 0 ||
+        !irp->UserBuffer) {
+        IoReleaseRemoveLock(&storage->RemoveLock, irp);
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    if (!storage->Started || storage->SurpriseRemoved || storage->Removed ||
+        storage->BlockSize != USB_STORAGE_SECTOR_SIZE) {
+        IoReleaseRemoveLock(&storage->RemoveLock, irp);
+        IoCompleteRequest(irp, STATUS_OBJECT_DEAD, 0);
         return STATUS_PENDING;
     }
 
-    uint8_t* buffer = (uint8_t*)irp->Buffer;
+    uint8_t* buffer = (uint8_t*)irp->UserBuffer;
-    uint64_t lba = irp->Lba;
+    uint64_t lba = ios->Parameters.ReadWrite.ByteOffset >> 9;
-    uint32_t remaining = irp->SectorCount;
+    uint32_t remaining = ios->Parameters.ReadWrite.Length >> 9;
     uint32_t done = 0;
-    bool write = (irp->Flags & IRP_FLAG_WRITE) != 0;
+    bool write = (ios->MajorFunction == IRP_MJ_WRITE);
     uint32_t maxSectors = USB_STORAGE_MAX_BYTES / USB_STORAGE_SECTOR_SIZE;
 
-    KSTATUS st = STATUS_SUCCESS;
+    st = STATUS_SUCCESS;
-    UsbStorageAcquireIo(&gXhci, dev);
+    UsbStorageAcquireIo(storage);
     while (remaining) {
         uint32_t chunk = remaining;
         if (chunk > maxSectors) chunk = maxSectors;
-        st = UsbStorageReadWriteSectors(&gXhci, dev, write, lba, chunk,
+        st = UsbStorageReadWriteSectors(storage, write, lba, chunk,
                                         buffer + (done * USB_STORAGE_SECTOR_SIZE));
         if (!KSUCCESS(st))
             break;
@@ -255,13 +355,13 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
         done += chunk;
         remaining -= chunk;
     }
-    UsbStorageReleaseIo(dev);
+    UsbStorageReleaseIo(storage);
 
+    IoReleaseRemoveLock(&storage->RemoveLock, irp);
     IoCompleteRequest(irp, st, KSUCCESS(st) ? (size_t)done * USB_STORAGE_SECTOR_SIZE : 0);
     return STATUS_PENDING;
 }
 
-
 KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer)
         return STATUS_SUCCESS;
@@ -282,7 +382,6 @@ KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     return STATUS_SUCCESS;
 }
 
-
 KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     if (!dev->MassCandidate)
         return STATUS_SUCCESS;
@@ -352,7 +451,6 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de
     return STATUS_SUCCESS;
 }
 
-
 uint8_t UsbFindFreeDiskIndex(void) {
     for (uint8_t i = 0; i < 8; i++) {
         if (!IoDiskDevicePresent(i))
@@ -361,17 +459,23 @@ uint8_t UsbFindFreeDiskIndex(void) {
     return 0xFFu;
 }
 
-
+KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage) {
-KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) {
+    if (!storage || storage->BlockSize != USB_STORAGE_SECTOR_SIZE || storage->BlockCount == 0)
-    if (!dev || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE || dev->MassBlockCount == 0)
         return STATUS_NOT_SUPPORTED;
+    if (storage->DiskDevice)
+        return STATUS_ALREADY_EXISTS;
 
     DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0;
-    KSTATUS st = IoCreateDevice(&gUsbStorageDriver, 0, &disk);
+    KSTATUS st = IoCreateDevice(&gUsbStorageDriver, sizeof(USB_STORAGE_DISK_EXTENSION), &disk);
     if (!KSUCCESS(st)) return st;
+    (void)IoSetDeviceName(disk, SYS_SV_LITERAL("usb-storage-disk"));
 
-    disk->DeviceExtension = dev;
+    USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)disk->DeviceExtension;
-    disk->SectorCount = dev->MassBlockCount;
+    diskExt->Kind = UsbStorageExtensionDisk;
+    diskExt->Storage = storage;
+
+    disk->u.Disk.SectorCount = storage->BlockCount;
+    disk->u.Disk.SectorSize = USB_STORAGE_SECTOR_SIZE;
     disk->AlignmentRequirement = USB_STORAGE_SECTOR_SIZE;
     disk->Flags |= DEVICE_OBJECT_REMOVABLE;
 
@@ -387,12 +491,278 @@ KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) {
         return st;
     }
 
-    dev->MassDeviceObject = disk;
+    storage->DiskDevice = disk;
+    if (storage->XhciDevice)
+        storage->XhciDevice->MassDeviceObject = disk;
+
     DbgPrint("usb-storage: registered disk %u slot=%u sectors=%llu\n",
-             index, dev->SlotId, (unsigned long long)dev->MassBlockCount);
+             index,
+             storage->Device ? storage->Device->SlotId : 0,
+             (unsigned long long)storage->BlockCount);
     return STATUS_SUCCESS;
 }
 
+static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSION* storage) {
+    if (!device || !storage || !storage->InterfacePdo || !storage->Device)
+        return STATUS_INVALID_PARAMETER;
+    if (storage->Started)
+        return STATUS_SUCCESS;
+    if (storage->SurpriseRemoved || storage->Removed)
+        return STATUS_OBJECT_DEAD;
+
+    USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)storage->InterfacePdo->DeviceExtension;
+    if (!pdoExt || pdoExt->Kind != UsbPdoInterface)
+        return STATUS_INVALID_PARAMETER;
+
+    const USB_ENDPOINT_DESCRIPTOR* bulkIn = UsbStorageFindBulkEndpoint(pdoExt, true);
+    const USB_ENDPOINT_DESCRIPTOR* bulkOut = UsbStorageFindBulkEndpoint(pdoExt, false);
+    if (!bulkIn || !bulkOut)
+        return STATUS_NOT_FOUND;
+
+    XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext;
+    XHCI_DEVICE* xdev = (XHCI_DEVICE*)storage->Device->HcdDeviceContext;
+    if (!xhci || !xdev)
+        return STATUS_INVALID_PARAMETER;
+
+    storage->XhciDevice = xdev;
+    storage->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
+    storage->BulkInAddress = bulkIn->bEndpointAddress;
+    storage->BulkOutAddress = bulkOut->bEndpointAddress;
+    storage->BulkInMaxPacket = bulkIn->wMaxPacketSize;
+    storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize;
+    storage->Tag = 0x1000u + storage->Device->SlotId;
+
+    xdev->MassCandidate = true;
+    xdev->MassInterface = storage->InterfaceNumber;
+    xdev->MassBulkInAddress = storage->BulkInAddress;
+    xdev->MassBulkOutAddress = storage->BulkOutAddress;
+    xdev->MassBulkInEndpointId = XhciEndpointIdFromAddress(storage->BulkInAddress);
+    xdev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(storage->BulkOutAddress);
+    xdev->MassBulkInMaxPacket = storage->BulkInMaxPacket;
+    xdev->MassBulkOutMaxPacket = storage->BulkOutMaxPacket;
+
+    KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, xdev);
+    if (!KSUCCESS(st)) goto fail;
+
+    st = UsbCreatePipe(storage->Device,
+                       UsbPipeBulk,
+                       storage->BulkInAddress,
+                       storage->BulkInMaxPacket,
+                       bulkIn->bInterval,
+                       &storage->BulkInPipe);
+    if (!KSUCCESS(st)) goto fail;
+
+    st = UsbCreatePipe(storage->Device,
+                       UsbPipeBulk,
+                       storage->BulkOutAddress,
+                       storage->BulkOutMaxPacket,
+                       bulkOut->bInterval,
+                       &storage->BulkOutPipe);
+    if (!KSUCCESS(st)) goto fail;
+
+    st = UsbStorageInquiry(storage);
+    if (!KSUCCESS(st)) goto fail;
+
+    st = UsbStorageTestUnitReady(storage);
+    if (!KSUCCESS(st)) {
+        DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", storage->Device->SlotId, st);
+        goto fail;
+    }
+
+    st = UsbStorageReadCapacity(storage);
+    if (!KSUCCESS(st)) goto fail;
+    if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE) {
+        DbgPrint("usb-storage: unsupported block size slot=%u size=%u\n",
+                 storage->Device->SlotId, storage->BlockSize);
+        st = STATUS_NOT_SUPPORTED;
+        goto fail;
+    }
+
+    // The disk namespace publishes an arrival notification synchronously from
+    // IoRegisterDiskDevice().  Mark the storage FDO usable before publishing the
+    // disk so filesystem hotplug callbacks can read sector 0 immediately.
+    xdev->MassActive = true;
+    storage->Started = true;
+    device->PnpState = DevicePnpStarted;
+    if (storage->InterfacePdo)
+        storage->InterfacePdo->PnpState = DevicePnpStarted;
+
+    st = UsbStorageRegisterDisk(storage);
+    if (!KSUCCESS(st)) {
+        storage->Started = false;
+        device->PnpState = DevicePnpNotStarted;
+        if (storage->InterfacePdo)
+            storage->InterfacePdo->PnpState = DevicePnpNotStarted;
+        goto fail;
+    }
+
+    DbgPrint("usb-storage: FDO bound slot=%u if=%u disk-ready\n",
+             storage->Device->SlotId, storage->InterfaceNumber);
+    return STATUS_SUCCESS;
+
+fail:
+    xdev->MassCandidate = false;
+    xdev->MassActive = false;
+    if (storage->BulkInPipe) {
+        UsbDereferencePipe(storage->BulkInPipe);
+        storage->BulkInPipe = (USB_PIPE*)0;
+    }
+    if (storage->BulkOutPipe) {
+        UsbDereferencePipe(storage->BulkOutPipe);
+        storage->BulkOutPipe = (USB_PIPE*)0;
+    }
+    DbgPrint("usb-storage: start failed slot=%u if=%u st=0x%x\n",
+             storage->Device ? storage->Device->SlotId : 0,
+             storage->InterfaceNumber,
+             st);
+    return st;
+}
+
+static KSTATUS UsbStorageDiskPnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
+    if (!device || !irp || !storage)
+        return STATUS_INVALID_PARAMETER;
+
+    switch (minor) {
+        case IRP_MN_START_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+            return STATUS_PENDING;
+
+        case IRP_MN_QUERY_REMOVE_DEVICE:
+            device->PnpState = DevicePnpRemovePending;
+            IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+            return STATUS_PENDING;
+
+        case IRP_MN_CANCEL_REMOVE_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+            return STATUS_PENDING;
+
+        case IRP_MN_SURPRISE_REMOVAL:
+            storage->SurpriseRemoved = true;
+            if (storage->XhciDevice) {
+                storage->XhciDevice->MassSurpriseRemoved = true;
+                storage->XhciDevice->MassActive = false;
+            }
+            device->PnpState = DevicePnpSurpriseRemoved;
+            IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+            return STATUS_PENDING;
+
+        case IRP_MN_REMOVE_DEVICE:
+            storage->Removed = true;
+            if (storage->XhciDevice) {
+                storage->XhciDevice->MassRemoved = true;
+                storage->XhciDevice->MassActive = false;
+                if (storage->XhciDevice->MassDeviceObject == device)
+                    storage->XhciDevice->MassDeviceObject = (DEVICE_OBJECT*)0;
+            }
+            device->PnpState = DevicePnpDeleted;
+            if (storage->DiskDevice == device)
+                storage->DiskDevice = (DEVICE_OBJECT*)0;
+            IoDeleteDevice(device);
+            IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+            return STATUS_PENDING;
+
+        default:
+            return STATUS_NOT_SUPPORTED;
+    }
+}
+
+static KSTATUS UsbStorageInterfacePnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
+    if (!device || !irp || !storage)
+        return STATUS_INVALID_PARAMETER;
+
+    KSTATUS st = STATUS_SUCCESS;
+    switch (minor) {
+        case IRP_MN_START_DEVICE:
+            st = UsbStorageStartDevice(device, storage);
+            if (!KSUCCESS(st)) {
+                IoCompleteRequest(irp, st, 0);
+                return STATUS_PENDING;
+            }
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_QUERY_REMOVE_DEVICE:
+            device->PnpState = DevicePnpRemovePending;
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_CANCEL_REMOVE_DEVICE:
+            if (!storage->SurpriseRemoved)
+                device->PnpState = DevicePnpStarted;
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_SURPRISE_REMOVAL:
+            storage->SurpriseRemoved = true;
+            storage->Started = false;
+            if (storage->XhciDevice) {
+                storage->XhciDevice->MassSurpriseRemoved = true;
+                storage->XhciDevice->MassActive = false;
+            }
+            if (storage->DiskDevice)
+                (void)IoNotifyDiskSurpriseRemoved(storage->DiskDevice->u.Disk.DiskIndex);
+            device->PnpState = DevicePnpSurpriseRemoved;
+            return IoForwardIrpToLower(device, irp);
+
+        case IRP_MN_REMOVE_DEVICE: {
+            storage->Removed = true;
+            storage->Started = false;
+            if (storage->DiskDevice)
+                (void)IoUnregisterDiskDevice(storage->DiskDevice->u.Disk.DiskIndex);
+            if (storage->BulkInPipe) {
+                UsbDereferencePipe(storage->BulkInPipe);
+                storage->BulkInPipe = (USB_PIPE*)0;
+            }
+            if (storage->BulkOutPipe) {
+                UsbDereferencePipe(storage->BulkOutPipe);
+                storage->BulkOutPipe = (USB_PIPE*)0;
+            }
+            if (storage->RemoveLockReady) {
+                IoReleaseRemoveLockAndWait(&storage->RemoveLock, irp);
+                IoDestroyRemoveLock(&storage->RemoveLock);
+                storage->RemoveLockReady = false;
+            }
+            device->PnpState = DevicePnpDeleted;
+
+            KSTATUS lower = storage->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS;
+            if (storage->LowerDevice) {
+                IoDetachDevice(device);
+                storage->LowerDevice = (DEVICE_OBJECT*)0;
+            }
+            IoDeleteDevice(device);
+            if (lower == STATUS_NOT_SUPPORTED) {
+                IoCompleteRequest(irp, STATUS_SUCCESS, 0);
+                return STATUS_PENDING;
+            }
+            if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
+                IoCompleteRequest(irp, lower, 0);
+                return STATUS_PENDING;
+            }
+            return lower;
+        }
+
+        default:
+            return IoForwardIrpToLower(device, irp);
+    }
+}
+
+KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp)
+        return STATUS_INVALID_PARAMETER;
+
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios || ios->MajorFunction != IRP_MJ_PNP)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_STORAGE_EXTENSION* iface = UsbStorageFromInterfaceDevice(device);
+    if (iface)
+        return UsbStorageInterfacePnp(device, irp, iface, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction);
+
+    USB_STORAGE_EXTENSION* disk = UsbStorageFromDiskDevice(device);
+    if (disk)
+        return UsbStorageDiskPnp(device, irp, disk, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction);
+
+    return STATUS_INVALID_PARAMETER;
+}
 
 bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
     (void)context;
@@ -409,64 +779,69 @@ bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
            UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
 }
 
-
 KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context) {
+    (void)pub;
     (void)context;
-    if (!pub || !pub->BusContext || !pub->DeviceContext)
+    return STATUS_NOT_SUPPORTED;
-        return STATUS_INVALID_PARAMETER;
-
-    XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext;
-    XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext;
-    if (dev->MassActive || dev->MassCandidate)
-        return STATUS_ALREADY_EXISTS;
-
-    const IO_PUBLISHED_USB_ENDPOINT* bulkIn = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true);
-    const IO_PUBLISHED_USB_ENDPOINT* bulkOut = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
-    if (!bulkIn || !bulkOut)
-        return STATUS_NOT_FOUND;
-
-    dev->MassCandidate = true;
-    dev->MassInterface = pub->u.Usb.InterfaceNumber;
-    dev->MassBulkInAddress = bulkIn->Address;
-    dev->MassBulkOutAddress = bulkOut->Address;
-    dev->MassBulkInEndpointId = XhciEndpointIdFromAddress(bulkIn->Address);
-    dev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(bulkOut->Address);
-    dev->MassBulkInMaxPacket = bulkIn->MaxPacketSize;
-    dev->MassBulkOutMaxPacket = bulkOut->MaxPacketSize;
-
-    KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, dev);
-    if (!KSUCCESS(st)) goto fail;
-
-    st = UsbStorageInquiry(xhci, dev);
-    if (!KSUCCESS(st)) goto fail;
-
-    st = UsbStorageTestUnitReady(xhci, dev);
-    if (!KSUCCESS(st)) {
-        DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", dev->SlotId, st);
-        goto fail;
-    }
-
-    st = UsbStorageReadCapacity(xhci, dev);
-    if (!KSUCCESS(st)) goto fail;
-    if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
-        DbgPrint("usb-storage: unsupported block size slot=%u block_size=%u\n",
-                 dev->SlotId, dev->MassBlockSize);
-        st = STATUS_NOT_SUPPORTED;
-        goto fail;
-    }
-
-    dev->MassActive = true;
-    st = UsbStorageRegisterDisk(dev);
-    if (!KSUCCESS(st)) goto fail;
-
-    DbgPrint("usb-storage: bound slot=%u if=%u\n", dev->SlotId, dev->MassInterface);
-    return STATUS_SUCCESS;
-
-fail:
-    dev->MassCandidate = false;
-    dev->MassActive = false;
-    DbgPrint("usb-storage: bind failed slot=%u if=%u st=0x%x\n",
-             dev->SlotId, pub->u.Usb.InterfaceNumber, st);
-    return st;
 }
 
+KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
+    if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
+        return STATUS_INVALID_PARAMETER;
+
+    USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
+    if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device)
+        return STATUS_INVALID_PARAMETER;
+    if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_MASS_STORAGE ||
+        pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_MASS_SUBCLASS_SCSI ||
+        pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_MASS_PROTOCOL_BOT)
+        return STATUS_INVALID_PARAMETER;
+    if (!UsbStorageFindBulkEndpoint(pdoExt, true) || !UsbStorageFindBulkEndpoint(pdoExt, false))
+        return STATUS_NOT_FOUND;
+
+    DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
+    KSTATUS st = IoCreateDevice(driver, sizeof(USB_STORAGE_EXTENSION), &fdo);
+    if (!KSUCCESS(st))
+        return st;
+    (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-storage"));
+
+    USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)fdo->DeviceExtension;
+    ext->Kind = UsbStorageExtensionInterface;
+    ext->Self = fdo;
+    ext->InterfacePdo = physicalDeviceObject;
+    ext->Device = pdoExt->Device;
+    ext->XhciDevice = (XHCI_DEVICE*)pdoExt->Device->HcdDeviceContext;
+    ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
+    ext->IoBusy = 0;
+
+    st = IoInitializeRemoveLock(&ext->RemoveLock);
+    if (!KSUCCESS(st)) {
+        IoDeleteDevice(fdo);
+        return st;
+    }
+    ext->RemoveLockReady = true;
+
+    ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
+    if (!ext->LowerDevice) {
+        IoDestroyRemoveLock(&ext->RemoveLock);
+        ext->RemoveLockReady = false;
+        IoDeleteDevice(fdo);
+        return STATUS_FAILURE;
+    }
+
+    DbgPrint("usb-storage: FDO attached slot=%u if=%u\n",
+             ext->Device->SlotId,
+             pdoExt->InterfaceDescriptor.bInterfaceNumber);
+    return STATUS_SUCCESS;
+}
+
+KSTATUS UsbInitializeStorageDriver(void) {
+    memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
+    IoInitializeDriverObject(&gUsbStorageDriver, SYS_SV_LITERAL("usb-storage"));
+    gUsbStorageDriver.IdTable = &gUsbStorageIdTable;
+    gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice;
+    gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
+    gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
+    gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp;
+    return IoRegisterDriverObject(&gUsbStorageDriver);
+}

code/drivers/bus/usb/usb_xhci.c

@@ -353,6 +353,31 @@ void XhciLogPortStatus(XHCI_CONTROLLER* xhci) {
     }
 }
 
+void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
+    if (!xhci || !xhci->Op)
+        return;
+
+    uint32_t powered = 0;
+    for (uint8_t port = 1; port <= xhci->MaxPorts; port++) {
+        uint32_t off = XhciPortReg(port);
+        uint32_t portsc = XhciRead32(xhci->Op, off);
+        uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK;
+
+        if (!(portsc & XHCI_PORTSC_PP)) {
+            XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
+            powered++;
+            UsbTrace("usb: xHCI powered root port %u old-portsc=%08x\n", port, portsc);
+        } else if (changes) {
+            XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
+        }
+    }
+
+    if (powered) {
+        UsbDelayUs(xhci->Hal, 20000u);
+        DbgPrint("usb: xHCI powered %u previously-off root port(s)\n", powered);
+    }
+}
+
 
 void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
     if (!xhci->Runtime) return;
@@ -369,7 +394,71 @@ void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
 }
 
 
+static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
+    for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
+        if (w->CommandPhys && w->CommandPhys == commandPhys)
+            return w;
+    }
+    return (XHCI_WAIT_TARGET*)0;
+}
+
+
+static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
+                                                      uint64_t transferPhys,
+                                                      uint8_t slotId,
+                                                      uint8_t endpointId) {
+    for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
+        if (w->TransferPhys &&
+            w->TransferPhys == transferPhys &&
+            w->TransferSlotId == slotId &&
+            w->TransferEndpointId == endpointId) {
+            return w;
+        }
+    }
+    return (XHCI_WAIT_TARGET*)0;
+}
+
+
+static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
+                                                uint64_t commandPhys,
+                                                uint32_t completionCode,
+                                                uint8_t slotId) {
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys);
+    if (w) {
+        w->Command.CompletionCode = completionCode;
+        w->Command.SlotId = slotId;
+        XhciBarrier();
+        w->CommandMatched = true;
+    }
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+    return w;
+}
+
+
+static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
+                                                 uint64_t transferPhys,
+                                                 uint8_t slotId,
+                                                 uint8_t endpointId,
+                                                 uint32_t completionCode,
+                                                 uint32_t residue) {
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId);
+    if (w) {
+        w->Transfer.CompletionCode = completionCode;
+        w->Transfer.SlotId = slotId;
+        w->Transfer.EndpointId = endpointId;
+        w->Transfer.Residue = residue;
+        XhciBarrier();
+        w->TransferMatched = true;
+    }
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+    return w;
+}
+
+
 bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
+    (void)target;
     bool processed = false;
 
     for (;;) {
@@ -384,41 +473,28 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
         uint32_t cc = XhciCompletionCode(ev);
         uint64_t param = ((uint64_t)ev->ParameterHi << 32) | ev->ParameterLo;
 
-        XHCI_WAIT_TARGET* waiter = target ? target : (XHCI_WAIT_TARGET*)xhci->ActiveWait;
-
         if (type == XHCI_TRB_TYPE_CMD_DONE) {
             uint8_t slotId = XhciEventSlotId(ev);
             UsbTrace("usb: xHCI command completion ptr=0x%016llX cc=%u slot=%u\n",
                      (unsigned long long)param, cc, slotId);
-            if (waiter && waiter->CommandPhys && param == waiter->CommandPhys) {
+            (void)XhciClaimCommandWaiter(xhci, param, cc, slotId);
-                waiter->CommandMatched = true;
-                waiter->Command.CompletionCode = cc;
-                waiter->Command.SlotId = slotId;
-            }
         } else if (type == XHCI_TRB_TYPE_TRANSFER) {
             uint8_t slotId = XhciEventSlotId(ev);
             uint8_t epId = XhciEventEndpointId(ev);
             uint32_t residue = XhciTransferResidue(ev);
-            bool claimedByWaiter = false;
-            bool targetMatches = waiter && waiter->TransferPhys && param == waiter->TransferPhys &&
-                                 slotId == waiter->TransferSlotId && epId == waiter->TransferEndpointId;
             if (cc != XHCI_CC_SUCCESS) {
                 UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n",
                          (unsigned long long)param, cc, slotId, epId, residue);
             }
-            if (targetMatches) {
+
-                waiter->TransferMatched = true;
+            XHCI_WAIT_TARGET* waiter = XhciClaimTransferWaiter(xhci, param, slotId, epId, cc, residue);
-                waiter->Transfer.CompletionCode = cc;
+            if (!waiter)
-                waiter->Transfer.SlotId = slotId;
-                waiter->Transfer.EndpointId = epId;
-                waiter->Transfer.Residue = residue;
-                claimedByWaiter = true;
-            }
-            if (!claimedByWaiter)
                 XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue);
         } else if (type == XHCI_TRB_TYPE_PORT_SC) {
             uint32_t portId = ev->ParameterLo >> 24;
             UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc);
+            if (portId && portId <= xhci->MaxPorts)
+                UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)portId);
         } else {
             DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n",
                      type, cc, (unsigned long long)param, ev->Status, ev->Control);
@@ -454,22 +530,53 @@ bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
 }
 
 
-KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
+void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
-                                 uint32_t timeoutUs) {
+    if (!xhci || !target || target->Registered)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    target->Next = xhci->WaiterList;
+    target->Registered = true;
+    xhci->WaiterList = target;
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+}
+
+
+void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
+    if (!xhci || !target || !target->Registered)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    XHCI_WAIT_TARGET** link = &xhci->WaiterList;
+    while (*link) {
+        if (*link == target) {
+            *link = target->Next;
+            target->Next = (XHCI_WAIT_TARGET*)0;
+            target->Registered = false;
+            break;
+        }
+        link = &(*link)->Next;
+    }
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+}
+
+
+KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
+                               uint32_t timeoutUs) {
     if (!xhci || !target)
         return STATUS_INVALID_PARAMETER;
 
-    xhci->ActiveWait = target;
     for (uint32_t i = 0; i < timeoutUs; i++) {
         if (XhciWaitTargetComplete(target)) {
-            xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
+            XhciUnregisterWaitTarget(xhci, target);
             return STATUS_SUCCESS;
         }
 
         // The IRQ/DPC path normally owns the event ring after enumeration, but
         // synchronous storage/control waits must not depend exclusively on an
-        // interrupt being delivered.  Opportunistically drain the ring from the
+        // interrupt being delivered. Opportunistically drain the ring too;
-        // waiter too; EventLock serializes this with the DPC.
+        // EventLock serializes this with the DPC while WaitLock protects waiter
+        // registration/matching.
         if (!xhci->InterruptsEnabled || (i & 0x7u) == 0) {
             XhciDrainEventRingLocked(xhci, target);
         } else if ((XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_EINT) && (i & 0x3Fu) == 0) {
@@ -479,11 +586,20 @@ KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
         UsbDelayUs(xhci->Hal, 1);
     }
 
-    xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
+    XhciUnregisterWaitTarget(xhci, target);
     return STATUS_TIMEOUT;
 }
 
 
+KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
+                          uint32_t timeoutUs) {
+    if (!xhci || !target)
+        return STATUS_INVALID_PARAMETER;
+    XhciRegisterWaitTarget(xhci, target);
+    return XhciWaitForArmedTarget(xhci, target, timeoutUs);
+}
+
+
 void XhciEventDpcRoutine(KDPC* dpc, void* context,
                                 void* systemArgument1, void* systemArgument2) {
     (void)dpc;
@@ -695,13 +811,26 @@ uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
 }
 
 
-KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
+static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
-                                  uint64_t parameter,
+                                      uint64_t parameter,
-                                  uint32_t status,
+                                      uint32_t status,
-                                  uint32_t controlWithoutCycle,
+                                      uint32_t controlWithoutCycle,
-                                  XHCI_COMMAND_RESULT* outResult) {
+                                      uint64_t* outPhys) {
-    if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u)
+    if (!xhci || !xhci->CommandRing)
-        return STATUS_QUEUE_FULL;
+        return STATUS_INVALID_PARAMETER;
+
+    if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) {
+        XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
+        link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
+        link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
+        link->Status = 0;
+        XhciBarrier();
+        link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
+                        XHCI_TRB_LINK_TC |
+                        xhci->CommandCycle;
+        xhci->CommandEnqueue = 0;
+        xhci->CommandCycle ^= 1u;
+    }
 
     uint32_t index = xhci->CommandEnqueue++;
     XHCI_TRB* trb = &xhci->CommandRing[index];
@@ -713,15 +842,38 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
     XhciBarrier();
     trb->Control = controlWithoutCycle | xhci->CommandCycle;
 
+    if (outPhys) *outPhys = trbPhys;
+    return STATUS_SUCCESS;
+}
+
+
+KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
+                                  uint64_t parameter,
+                                  uint32_t status,
+                                  uint32_t controlWithoutCycle,
+                                  XHCI_COMMAND_RESULT* outResult) {
+    if (!xhci)
+        return STATUS_INVALID_PARAMETER;
+
     XHCI_WAIT_TARGET target;
     memset(&target, 0, sizeof(target));
+
+    uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock);
+    uint64_t trbPhys = 0;
+    KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys);
+    if (!KSUCCESS(st)) {
+        KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
+        return st;
+    }
+
     target.CommandPhys = trbPhys;
-    xhci->ActiveWait = &target;
+    XhciRegisterWaitTarget(xhci, &target);
 
     XhciBarrier();
     XhciWrite32(xhci->Doorbells, 0, 0); // host-controller doorbell, command ring
+    KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
 
-    KSTATUS waitStatus = XhciWaitForTarget(xhci, &target, 500000u);
+    KSTATUS waitStatus = XhciWaitForArmedTarget(xhci, &target, 500000u);
     if (KSUCCESS(waitStatus) && target.CommandMatched) {
         if (outResult) *outResult = target.Command;
         return (target.Command.CompletionCode == XHCI_CC_SUCCESS)
@@ -781,6 +933,28 @@ KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
 }
 
 
+
+KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
+    if (!xhci || slotId == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    XHCI_COMMAND_RESULT res;
+    memset(&res, 0, sizeof(res));
+    KSTATUS st = XhciSubmitCommand(xhci,
+                                   0,
+                                   0,
+                                   (XHCI_TRB_TYPE_DISABLE_SLOT << XHCI_TRB_TYPE_SHIFT) |
+                                       ((uint32_t)slotId << 24),
+                                   &res);
+    if (!KSUCCESS(st))
+        return st;
+    if (res.CompletionCode != XHCI_CC_SUCCESS) {
+        DbgPrint("usb: Disable Slot slot=%u failed cc=%u\n", slotId, res.CompletionCode);
+        return STATUS_DEVICE_ERROR;
+    }
+    return STATUS_SUCCESS;
+}
+
 KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     XHCI_COMMAND_RESULT result;
     memset(&result, 0, sizeof(result));
@@ -970,12 +1144,22 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     uint32_t maxPacket = XhciEpMaxPacket(dev->Speed);
     dev->MaxPacketSize0 = (uint16_t)maxPacket;
 
-    slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed);
+    uint8_t rootPort = dev->Path.RootPort ? dev->Path.RootPort : dev->Port;
+    uint32_t routeString = dev->Path.RouteString & 0x000fffffu;
+
+    slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed) | routeString;
     // Slot context dword 1 owns Root Hub Port Number bits 23:16.
     // Putting the port number in dword 2 makes Address Device fail with a
     // context/TRB error on QEMU xHCI.
-    slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
+    slot[1] = XHCI_SLOT_ROOT_PORT(rootPort);
     slot[2] = 0;
+    if (dev->Path.BehindHub && dev->Speed <= 2u && dev->Path.ParentHubSlot) {
+        // TT Hub Slot ID bits 7:0 and TT Port Number bits 15:8.
+        // This is required for low/full-speed devices behind high-speed hubs;
+        // harmless for older full-speed hub paths on the controllers we target.
+        slot[2] = (uint32_t)dev->Path.ParentHubSlot |
+                  ((uint32_t)dev->Path.ParentHubPort << 8);
+    }
     slot[3] = 0;
 
     ep0[0] = 0;
@@ -999,8 +1183,21 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
                                 uint32_t status,
                                 uint32_t controlWithoutCycle,
                                 uint64_t* outPhys) {
-    if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u)
+    if (!dev || !dev->Ep0Ring)
-        return STATUS_QUEUE_FULL;
+        return STATUS_INVALID_PARAMETER;
+
+    if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) {
+        XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
+        link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
+        link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
+        link->Status = 0;
+        XhciBarrier();
+        link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
+                        XHCI_TRB_LINK_TC |
+                        dev->Ep0Cycle;
+        dev->Ep0Enqueue = 0;
+        dev->Ep0Cycle ^= 1u;
+    }
 
     uint32_t index = dev->Ep0Enqueue++;
     XHCI_TRB* trb = &dev->Ep0Ring[index];
@@ -1017,11 +1214,12 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
 }
 
 
-KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
-                             const USB_SETUP_PACKET* setup,
+                                    const USB_SETUP_PACKET* setup,
-                             void* buffer,
+                                    void* buffer,
-                             PHYSICAL_ADDRESS bufferPhys,
+                                    PHYSICAL_ADDRESS bufferPhys,
-                             uint16_t length) {
+                                    uint16_t length,
+                                    uint32_t timeoutUs) {
     if (!buffer || length == 0 || length > PAGE_SIZE)
         return STATUS_INVALID_PARAMETER;
 
@@ -1057,18 +1255,21 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
         &statusTrbPhys);
     if (!KSUCCESS(st)) return st;
 
-    XhciBarrier();
-    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
-
     XHCI_WAIT_TARGET target;
     memset(&target, 0, sizeof(target));
     target.TransferPhys = statusTrbPhys;
     target.TransferSlotId = dev->SlotId;
     target.TransferEndpointId = 1;
+    XhciRegisterWaitTarget(xhci, &target);
 
-    st = XhciWaitForTarget(xhci, &target, 1000000u);
+    XhciBarrier();
+    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
+
+    if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
+    st = XhciWaitForArmedTarget(xhci, &target, timeoutUs);
     if (!KSUCCESS(st)) {
-        DbgPrint("usb: xHCI control IN timeout slot=%u len=%u\n", dev->SlotId, length);
+        DbgPrint("usb: xHCI control IN timeout slot=%u len=%u timeout=%uus\n",
+                 dev->SlotId, length, timeoutUs);
         return st;
     }
     if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS)
@@ -1082,6 +1283,16 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
+KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+                             const USB_SETUP_PACKET* setup,
+                             void* buffer,
+                             PHYSICAL_ADDRESS bufferPhys,
+                             uint16_t length) {
+    return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length,
+                                USB_XHCI_CONTROL_TIMEOUT_US);
+}
+
+
 KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
                                       const USB_SETUP_PACKET* setup) {
     uint64_t setupValue = UsbSetupPacketToU64(setup);
@@ -1106,16 +1317,17 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
         &statusTrbPhys);
     if (!KSUCCESS(st)) return st;
 
-    XhciBarrier();
-    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
-
     XHCI_WAIT_TARGET target;
     memset(&target, 0, sizeof(target));
     target.TransferPhys = statusTrbPhys;
     target.TransferSlotId = dev->SlotId;
     target.TransferEndpointId = 1;
+    XhciRegisterWaitTarget(xhci, &target);
 
-    st = XhciWaitForTarget(xhci, &target, 1000000u);
+    XhciBarrier();
+    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
+
+    st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
     if (!KSUCCESS(st)) {
         DbgPrint("usb: xHCI control no-data timeout slot=%u req=%u value=%04x\n",
                  dev->SlotId, setup->bRequest, setup->wValue);
@@ -1132,8 +1344,9 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
-KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
-                                       USB_DEVICE_DESCRIPTOR* outDesc) {
+                                              USB_DEVICE_DESCRIPTOR* outDesc,
+                                              uint32_t timeoutUs) {
     USB_SETUP_PACKET setup;
     setup.bmRequestType = USB_DIR_IN;
     setup.bRequest = USB_REQ_GET_DESCRIPTOR;
@@ -1141,10 +1354,12 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     setup.wIndex = 0;
     setup.wLength = sizeof(USB_DEVICE_DESCRIPTOR);
 
-    KSTATUS st = XhciControlIn(xhci, dev, &setup,
+    if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
-                               dev->DescriptorBuffer,
+    KSTATUS st = XhciControlInTimeout(xhci, dev, &setup,
-                               dev->DescriptorPhys,
+                                      dev->DescriptorBuffer,
-                               sizeof(USB_DEVICE_DESCRIPTOR));
+                                      dev->DescriptorPhys,
+                                      sizeof(USB_DEVICE_DESCRIPTOR),
+                                      timeoutUs);
     if (!KSUCCESS(st)) return st;
 
     memcpy(outDesc, dev->DescriptorBuffer, sizeof(*outDesc));
@@ -1159,6 +1374,13 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
+KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+                                       USB_DEVICE_DESCRIPTOR* outDesc) {
+    return XhciGetDeviceDescriptorTimeout(xhci, dev, outDesc,
+                                          USB_XHCI_CONTROL_TIMEOUT_US);
+}
+
+
 KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
                                             uint8_t configurationIndex,
                                             USB_CONFIGURATION_DESCRIPTOR* outConfig,
@@ -1331,11 +1553,11 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     target.TransferSlotId = dev->SlotId;
     target.TransferEndpointId = endpointId;
 
-    xhci->ActiveWait = &target;
+    XhciRegisterWaitTarget(xhci, &target);
     XhciBarrier();
     XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, endpointId);
 
-    st = XhciWaitForTarget(xhci, &target, 1000000u);
+    st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
     if (!KSUCCESS(st)) {
         DbgPrint("usb-storage: bulk transfer timeout slot=%u ep=%u len=%u\n",
                  dev->SlotId, endpointId, length);
@@ -1356,6 +1578,300 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
+
+
+KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) {
+    if (!xhci || !dev || !pipe || !pipe->Open)
+        return STATUS_INVALID_PARAMETER;
+
+    uint8_t dci = pipe->EndpointId;
+    if (dci < 2 || dci > 31)
+        return STATUS_INVALID_PARAMETER;
+
+    if (pipe->HcdPipeContext) {
+        XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
+        return ctx->Configured ? STATUS_SUCCESS : STATUS_DEVICE_ERROR;
+    }
+
+    XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)MmAllocatePool(sizeof(XHCI_PIPE_CONTEXT));
+    if (!ctx) return STATUS_NO_MEMORY;
+    memset(ctx, 0, sizeof(*ctx));
+
+    ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys);
+    if (!ctx->Ring) {
+        MmFreePool(ctx);
+        return STATUS_NO_MEMORY;
+    }
+    ctx->Enqueue = 0;
+    ctx->Cycle = 1;
+    ctx->EndpointId = dci;
+    ctx->Type = pipe->Type;
+    XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle);
+
+    memset(dev->InputContext, 0, PAGE_SIZE);
+    uint32_t* icc = XhciInputControlContext(xhci, dev);
+    uint32_t* slot = XhciInputSlotContext(xhci, dev);
+    uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci);
+
+    icc[0] = 0;
+    icc[1] = (1u << 0) | (1u << dci);
+
+    slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed);
+    slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
+    slot[2] = 0;
+    slot[3] = 0;
+
+    uint16_t maxPacket = pipe->MaxPacketSize & 0x07FFu;
+    if (maxPacket == 0) maxPacket = 8u;
+
+    uint32_t endpointType = 0;
+    uint8_t interval = 0;
+    if (pipe->Type == UsbPipeInterrupt && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
+        endpointType = XHCI_EP_TYPE_INT_IN;
+        interval = XhciInterruptInterval(dev->Speed, pipe->Interval);
+    } else if (pipe->Type == UsbPipeBulk && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
+        endpointType = XHCI_EP_TYPE_BULK_IN;
+    } else if (pipe->Type == UsbPipeBulk) {
+        endpointType = XHCI_EP_TYPE_BULK_OUT;
+    } else {
+        MmFreePool(ctx);
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    ep[0] = (uint32_t)interval << 16;
+    ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16);
+    ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS);
+    ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32);
+    ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
+
+    XHCI_COMMAND_RESULT result;
+    memset(&result, 0, sizeof(result));
+    KSTATUS st = XhciSubmitCommand(
+        xhci,
+        dev->InputContextPhys,
+        0,
+        (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
+            ((uint32_t)dev->SlotId << 24),
+        &result);
+    if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
+        DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
+                 dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
+        MmFreePool(ctx);
+        return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
+    }
+
+    ctx->Configured = true;
+    pipe->HcdPipeContext = ctx;
+    dev->EndpointPipes[dci] = pipe;
+    UsbReferencePipe(pipe);
+    return STATUS_SUCCESS;
+}
+
+static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
+    if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
+        return STATUS_INVALID_PARAMETER;
+    if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN))
+        return STATUS_NOT_SUPPORTED;
+
+    KSTATUS st = XhciConfigurePipeEndpoint(xhci, dev, urb->Pipe);
+    if (!KSUCCESS(st)) return st;
+
+    XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext;
+    if (!ctx || !ctx->Configured)
+        return STATUS_DEVICE_ERROR;
+    if (ctx->ActiveTransfer)
+        return STATUS_QUEUE_FULL;
+
+    uint64_t trbPhys = 0;
+    st = XhciRingAppend(ctx->Ring,
+                        ctx->RingPhys,
+                        XHCI_PIPE_RING_TRBS,
+                        &ctx->Enqueue,
+                        &ctx->Cycle,
+                        urb->BufferPhys,
+                        urb->Length,
+                        (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
+                            XHCI_TRB_ISP |
+                            XHCI_TRB_IOC,
+                        &trbPhys);
+    if (!KSUCCESS(st)) return st;
+
+    ctx->ActiveTransfer = true;
+    ctx->ActiveTrbPhys = trbPhys;
+    ctx->ActiveUrb = urb;
+    urb->Pending = true;
+    urb->Cancelled = false;
+    urb->Status = STATUS_PENDING;
+    urb->ActualLength = 0;
+
+    XhciBarrier();
+    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, ctx->EndpointId);
+    return STATUS_PENDING;
+}
+
+void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
+                                  uint32_t completionCode, uint8_t slotId,
+                                  uint8_t endpointId, uint32_t residue) {
+    if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
+        return;
+
+    XHCI_DEVICE* dev = &xhci->Devices[slotId];
+    USB_PIPE* pipe = dev->EndpointPipes[endpointId];
+    if (!pipe || !pipe->HcdPipeContext)
+        return;
+
+    XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
+    if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys)
+        return;
+
+    USB_URB* urb = ctx->ActiveUrb;
+    ctx->ActiveTransfer = false;
+    ctx->ActiveTrbPhys = 0;
+    ctx->ActiveUrb = (USB_URB*)0;
+
+    if (!urb)
+        return;
+
+    KSTATUS st = STATUS_SUCCESS;
+    if (urb->Cancelled) {
+        st = STATUS_CANCELLED;
+    } else if (completionCode != XHCI_CC_SUCCESS) {
+        st = STATUS_DEVICE_ERROR;
+    }
+
+    uint32_t actual = 0;
+    if (KSUCCESS(st)) {
+        actual = (residue <= urb->Length) ? (urb->Length - residue) : 0;
+    }
+
+    urb->Pending = false;
+    urb->Status = st;
+    urb->ActualLength = actual;
+
+    IRP* completeIrp = urb->Irp;
+    urb->Irp = (IRP*)0;
+
+    if (urb->Complete)
+        urb->Complete(urb, urb->CompleteContext);
+
+    if (completeIrp)
+        IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0);
+}
+
+static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
+    if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    uint8_t endpointId = urb->Pipe->EndpointId;
+    XHCI_TRB* ring = (XHCI_TRB*)0;
+    PHYSICAL_ADDRESS ringPhys = 0;
+    uint32_t* enqueue = (uint32_t*)0;
+    uint8_t* cycle = (uint8_t*)0;
+
+    if (endpointId == dev->MassBulkInEndpointId) {
+        ring = dev->MassInRing;
+        ringPhys = dev->MassInRingPhys;
+        enqueue = &dev->MassInEnqueue;
+        cycle = &dev->MassInCycle;
+    } else if (endpointId == dev->MassBulkOutEndpointId) {
+        ring = dev->MassOutRing;
+        ringPhys = dev->MassOutRingPhys;
+        enqueue = &dev->MassOutEnqueue;
+        cycle = &dev->MassOutCycle;
+    } else {
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    uint32_t actual = 0;
+    KSTATUS st = XhciBulkTransfer(xhci,
+                                  dev,
+                                  endpointId,
+                                  ring,
+                                  ringPhys,
+                                  enqueue,
+                                  cycle,
+                                  urb->BufferPhys,
+                                  urb->Length,
+                                  &actual);
+    urb->ActualLength = KSUCCESS(st) ? actual : 0;
+    return st;
+}
+
+static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
+    if (!hcd || !urb || !urb->Device)
+        return STATUS_INVALID_PARAMETER;
+
+    XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext;
+    XHCI_DEVICE* dev = (XHCI_DEVICE*)urb->Device->HcdDeviceContext;
+    if (!xhci || !dev || !dev->Allocated)
+        return STATUS_INVALID_PARAMETER;
+
+    KSTATUS st;
+    switch (urb->Function) {
+        case UrbFunctionControlTransfer:
+            if (urb->Length == 0) {
+                st = XhciControlNoData(xhci, dev, &urb->Setup);
+                urb->ActualLength = 0;
+            } else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) {
+                st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length);
+                urb->ActualLength = KSUCCESS(st) ? urb->Length : 0;
+            } else {
+                st = STATUS_NOT_SUPPORTED;
+                urb->ActualLength = 0;
+            }
+            break;
+
+        case UrbFunctionBulkTransfer:
+            st = XhciHcdSubmitBulkUrb(xhci, dev, urb);
+            break;
+
+        case UrbFunctionInterruptTransfer:
+            st = XhciHcdSubmitInterruptUrb(xhci, dev, urb);
+            break;
+
+        default:
+            st = STATUS_NOT_SUPPORTED;
+            urb->ActualLength = 0;
+            break;
+    }
+
+    urb->Status = st;
+    return st;
+}
+
+static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
+    (void)hcd;
+    if (!urb) return STATUS_INVALID_PARAMETER;
+    urb->Cancelled = true;
+    if (!urb->Pending)
+        urb->Status = STATUS_CANCELLED;
+    return STATUS_SUCCESS;
+}
+
+static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
+    if (!hcd) return (void*)0;
+    return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
+}
+
+static KSTATUS XhciHcdResetRootPort(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed) {
+    if (!hcd) return STATUS_INVALID_PARAMETER;
+    return XhciResetPort((XHCI_CONTROLLER*)hcd->ControllerContext, port, outSpeed);
+}
+
+static KSTATUS XhciHcdClearRootPortChanges(USB_HCD* hcd, uint8_t port) {
+    if (!hcd) return STATUS_INVALID_PARAMETER;
+    XhciClearPortChanges((XHCI_CONTROLLER*)hcd->ControllerContext, port);
+    return STATUS_SUCCESS;
+}
+
+const USB_HCD_OPERATIONS gXhciHcdOperations = {
+    XhciHcdSubmitUrb,
+    XhciHcdCancelUrb,
+    XhciHcdResetRootPort,
+    XhciHcdClearRootPortChanges,
+    XhciHcdAllocateDma,
+};
+
 KSTATUS XhciInitialise(const HAL_API* hal,
                               uint8_t bus, uint8_t dev, uint8_t fn,
                               uint16_t vendor, uint16_t device) {
@@ -1438,6 +1954,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
         return STATUS_DEVICE_ERROR;
     }
 
+    UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts);
+    KSTATUS hcdPdoStatus = UsbCreateControllerPdo(&xhci->Hcd, vendor, device);
+    if (!KSUCCESS(hcdPdoStatus)) {
+        DbgPrint("usb: xHCI controller PDO creation failed st=0x%x\n", hcdPdoStatus);
+        return hcdPdoStatus;
+    }
+
     uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE +
                        (uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE;
     uint32_t needBytes = portEnd;
@@ -1489,12 +2012,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
     st = XhciSubmitNoopCommand(xhci);
     if (!KSUCCESS(st)) return st;
 
+    XhciPowerRootPorts(xhci);
     XhciLogPortStatus(xhci);
-    XhciEnumerateRootPorts(xhci);
 
-    if (xhci->HidDeviceCount > 0) {
+    st = UsbCreateRootHubPdo(&xhci->Hcd);
-        st = XhciStartHidInterrupts(xhci);
+    if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) {
-        if (!KSUCCESS(st)) return st;
+        DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st);
+        return st;
     }
 
     KeIrqRegister(xhci->Irq, XhciIsr);
@@ -1504,8 +2028,14 @@ KSTATUS XhciInitialise(const HAL_API* hal,
     DbgPrint("usb: xHCI ISR registered on IRQ %u%s; event ring handled by DPC\n",
              xhci->Irq, xhci->UsingMsi ? "(msi)" : "");
 
+    st = UsbHubScheduleRootScan(&xhci->Hcd);
+    if (!KSUCCESS(st))
+        DbgPrint("usb: root hub async scan scheduling failed st=0x%x\n", st);
+    else
+        DbgPrint("usb: root hub initial scan queued asynchronously\n");
+
     gHaveXhci = true;
-    DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage is next\n",
+    DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
              xhci->HidDeviceCount);
     return STATUS_SUCCESS;
 }

code/drivers/fs/fat32/fat32.c

@@ -1612,7 +1612,10 @@ static void Fat32AddCandidate(FAT32_VOL* probe,
 
 static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
                                           FAT32_PARTITION_CANDIDATE* out,
-                                          uint32_t maxCount) {
+                                          uint32_t maxCount,
+                                          bool* outMediaReadable) {
+    if (outMediaReadable)
+        *outMediaReadable = false;
     if (!out || maxCount == 0) return 0;
 
     FAT32_VOL probe;
@@ -1623,6 +1626,8 @@ static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
     uint8_t mbr[512];
     if (!KSUCCESS(ReadSectors(&probe, 0, 1, mbr)))
         return 0;
+    if (outMediaReadable)
+        *outMediaReadable = true;
 
     // A disk image may be a raw FAT32 volume with no partition table.
     Fat32AddCandidate(&probe, out, &count, maxCount, 0, IoDiskSectorCount(diskIndex));
@@ -1734,6 +1739,7 @@ typedef struct FAT32_ASYNC_RW {
     size_t CurrentCopy;
 
     IRP Irp;
+    DEVICE_OBJECT* CurrentDevice;
     IO_FS_ASYNC_COMPLETE Complete;
     void* CompleteContext;
 } FAT32_ASYNC_RW;
@@ -1811,6 +1817,10 @@ static void Fat32AsyncQueueContinuation(FAT32_ASYNC_RW* ctx) {
 
 static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t information) {
     if (!ctx) return;
+    if (ctx->CurrentDevice) {
+        IoDereferenceDeviceObject(ctx->CurrentDevice);
+        ctx->CurrentDevice = (DEVICE_OBJECT*)0;
+    }
     if (ctx->RunBuf) MmFreePool(ctx->RunBuf);
     IO_FS_ASYNC_COMPLETE complete = ctx->Complete;
     void* completeContext = ctx->CompleteContext;
@@ -1821,6 +1831,10 @@ static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t informa
 static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
     FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
     if (!ctx) return;
+    if (ctx->CurrentDevice) {
+        IoDereferenceDeviceObject(ctx->CurrentDevice);
+        ctx->CurrentDevice = (DEVICE_OBJECT*)0;
+    }
 
     if (!KSUCCESS(irp->Status)) {
         Fat32AsyncFinish(ctx, irp->Status, ctx->Done);
@@ -1872,17 +1886,25 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
     ctx->CurrentOffset = ctx->OffsetInCluster;
     ctx->CurrentCopy = copy;
 
-    memset(&ctx->Irp, 0, sizeof(ctx->Irp));
+    ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex);
-    ctx->Irp.DriveIndex = v->DriveIndex;
+    if (!ctx->CurrentDevice) {
-    ctx->Irp.Lba = ClusterToLba(v, runStart);
+        Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done);
-    ctx->Irp.SectorCount = runSectors;
+        return;
-    ctx->Irp.Buffer = ctx->RunBuf;
+    }
+
+    IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_READ,
+                        ClusterToLba(v, runStart) * 512ull,
+                        runSectors * 512u,
+                        ctx->RunBuf);
     ctx->Irp.CompletionFn = Fat32AsyncReadIrpComplete;
     ctx->Irp.CompletionCtx = ctx;
 
-    KSTATUS s = IoSubmitIrp(&ctx->Irp);
+    KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp);
-    if (s != STATUS_PENDING)
+    if (s != STATUS_PENDING) {
+        IoDereferenceDeviceObject(ctx->CurrentDevice);
+        ctx->CurrentDevice = (DEVICE_OBJECT*)0;
         Fat32AsyncFinish(ctx, s, ctx->Done);
+    }
 }
 
 KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
@@ -1949,6 +1971,10 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
 static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) {
     FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
     if (!ctx) return;
+    if (ctx->CurrentDevice) {
+        IoDereferenceDeviceObject(ctx->CurrentDevice);
+        ctx->CurrentDevice = (DEVICE_OBJECT*)0;
+    }
 
     if (!KSUCCESS(irp->Status)) {
         Fat32AsyncFinish(ctx, irp->Status, ctx->Done);
@@ -2024,18 +2050,25 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
     ctx->CurrentOffset = ctx->OffsetInCluster;
     ctx->CurrentCopy = copy;
 
-    memset(&ctx->Irp, 0, sizeof(ctx->Irp));
+    ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex);
-    ctx->Irp.DriveIndex = v->DriveIndex;
+    if (!ctx->CurrentDevice) {
-    ctx->Irp.Flags = IRP_FLAG_WRITE;
+        Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done);
-    ctx->Irp.Lba = ClusterToLba(v, runStart);
+        return;
-    ctx->Irp.SectorCount = runSectors;
+    }
-    ctx->Irp.Buffer = ctx->RunBuf;
+
+    IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_WRITE,
+                        ClusterToLba(v, runStart) * 512ull,
+                        runSectors * 512u,
+                        ctx->RunBuf);
     ctx->Irp.CompletionFn = Fat32AsyncWriteIrpComplete;
     ctx->Irp.CompletionCtx = ctx;
 
-    KSTATUS s = IoSubmitIrp(&ctx->Irp);
+    KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp);
-    if (s != STATUS_PENDING)
+    if (s != STATUS_PENDING) {
+        IoDereferenceDeviceObject(ctx->CurrentDevice);
+        ctx->CurrentDevice = (DEVICE_OBJECT*)0;
         Fat32AsyncFinish(ctx, s, ctx->Done);
+    }
 }
 
 KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
@@ -2124,31 +2157,55 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
 
 static FAT32_VOL     gVols[FAT32_MAX_VOLUMES];
 static IO_FILESYSTEM gFs[FAT32_MAX_VOLUMES];
-
+static bool          gVolumeMounted[FAT32_MAX_VOLUMES];
 
 static uint32_t gMountedCount;
 static bool     gDiskScanned[FAT32_MAX_VOLUMES];
 
+static int Fat32FindFreeVolumeIndex(void) {
+    for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) {
+        if (!gVolumeMounted[i] && !IoFilesystemPresent(i))
+            return i;
+    }
+    return -1;
+}
+
 static uint32_t Fat32TryMountDisk(uint8_t disk) {
     if (disk >= FAT32_MAX_VOLUMES)
         return 0;
     if (gDiskScanned[disk])
         return 0;
-    gDiskScanned[disk] = true;
 
     if (!IoDiskDevicePresent(disk))
         return 0;
 
     uint32_t mountedHere = 0;
     FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK];
-    uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK);
+    bool mediaReadable = false;
+    uint32_t partCount = Fat32EnumerateDiskVolumes(disk,
+                                                   parts,
+                                                   FAT32_MAX_PARTITIONS_PER_DISK,
+                                                   &mediaReadable);
+    if (!mediaReadable) {
+        DbgPrint("FAT32: disk %u: media probe deferred; sector 0 unreadable\n", disk);
+        return 0;
+    }
+
+    // Only remember a completed probe.  A transient hotplug/readiness failure
+    // must not permanently suppress a later mount attempt for the same disk
+    // index after a device is unplugged and replugged.
+    gDiskScanned[disk] = true;
+
     if (!partCount) {
         DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk);
         return 0;
     }
 
-    for (uint32_t part = 0; part < partCount && gMountedCount < FAT32_MAX_VOLUMES; part++) {
+    for (uint32_t part = 0; part < partCount; part++) {
-        uint8_t volumeIndex = (uint8_t)gMountedCount;
+        int freeVolume = Fat32FindFreeVolumeIndex();
+        if (freeVolume < 0)
+            break;
+        uint8_t volumeIndex = (uint8_t)freeVolume;
         FAT32_VOL* v = &gVols[volumeIndex];
         memset(v, 0, sizeof(*v));
         v->DriveIndex = disk;
@@ -2192,6 +2249,7 @@ static uint32_t Fat32TryMountDisk(uint8_t disk) {
 
         KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams);
         if (KSUCCESS(status)) {
+            gVolumeMounted[volumeIndex] = true;
             gMountedCount++;
             mountedHere++;
             DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n",
@@ -2222,6 +2280,60 @@ static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* co
     }
 }
 
+static void Fat32UnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) {
+    uint32_t removed = 0;
+
+    for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) {
+        if (!gVolumeMounted[i])
+            continue;
+        if (gVols[i].DriveIndex != diskIndex)
+            continue;
+
+        // Surprise removal means the backing device may already be gone; do not
+        // attempt writeback here.  Drop cached FAT sectors instead.
+        FatCacheInvalidate();
+        (void)IoUnregisterFilesystem(i);
+        memset(&gVols[i], 0, sizeof(gVols[i]));
+        memset(&gFs[i], 0, sizeof(gFs[i]));
+        gVolumeMounted[i] = false;
+        if (gMountedCount)
+            gMountedCount--;
+        removed++;
+    }
+
+    if (diskIndex < FAT32_MAX_VOLUMES)
+        gDiskScanned[diskIndex] = false;
+
+    if (removed) {
+        DbgPrint("FAT32: disk %u %s; unmounted %u volume(s); total=%u\n",
+                 diskIndex,
+                 change == IoDeviceSurpriseRemoved ? "surprise removed" : "removed",
+                 removed,
+                 gMountedCount);
+    }
+}
+
+static void Fat32DiskChange(uint8_t driveIndex,
+                            DEVICE_OBJECT* device,
+                            IO_DEVICE_CHANGE change,
+                            void* context) {
+    (void)device;
+    (void)context;
+
+    switch (change) {
+        case IoDeviceArrived:
+            Fat32DiskArrival(driveIndex, device, context);
+            break;
+        case IoDeviceSurpriseRemoved:
+        case IoDeviceRemoved:
+            Fat32UnmountDisk(driveIndex, change);
+            break;
+        case IoDeviceQueryRemove:
+        default:
+            break;
+    }
+}
+
 KSTATUS DriverEntry(const HAL_API *hal) {
     DbgPrint("FAT32: Init\n");
     if (!hal) return STATUS_INVALID_PARAMETER;
@@ -2231,9 +2343,14 @@ KSTATUS DriverEntry(const HAL_API *hal) {
     gMountedCount = 0;
     memset(gDiskScanned, 0, sizeof(gDiskScanned));
 
-    KSTATUS cbStatus = IoRegisterDiskArrivalCallback(Fat32DiskArrival, (void*)0);
+    KSTATUS cbStatus = IoRegisterDiskChangeCallback(Fat32DiskChange, (void*)0);
     if (!KSUCCESS(cbStatus)) {
-        DbgPrint("FAT32: disk-arrival callback registration failed st=0x%x\n", cbStatus);
+        DbgPrint("FAT32: disk-change callback registration failed st=0x%x\n", cbStatus);
+    }
+
+    cbStatus = IoEnumerateDiskDevices(Fat32DiskArrival, (void*)0);
+    if (!KSUCCESS(cbStatus)) {
+        DbgPrint("FAT32: initial disk enumeration failed st=0x%x\n", cbStatus);
     }
 
     if (!gMountedCount) {

code/drivers/storage/ahci/ahci.c

@@ -382,6 +382,28 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
 // Async IRP dispatch
 // ---------------------------------------------------------------------------
 
+
+static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
+                                   uint32_t* outSectors, void** outBuffer) {
+    if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
+        return STATUS_INVALID_PARAMETER;
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios) return STATUS_INVALID_PARAMETER;
+    if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
+        return STATUS_INVALID_PARAMETER;
+    if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
+        (ios->Parameters.ReadWrite.Length & 511u))
+        return STATUS_INVALID_PARAMETER;
+    if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    *outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
+    *outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
+    *outSectors = ios->Parameters.ReadWrite.Length >> 9;
+    *outBuffer = irp->UserBuffer;
+    return STATUS_SUCCESS;
+}
+
 // AhciDispatchReadWrite - DEVICE_OBJECT dispatch entry for disk IRPs.
 // Accepted IRPs are queued to the per-drive worker and completed later with
 // IoCompleteRequest().  Rejected IRPs are returned synchronously and are not
@@ -390,8 +412,22 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
     if (!device || !irp) return STATUS_INVALID_PARAMETER;
     AHCI_DRIVE *drv = (AHCI_DRIVE *)device->DeviceExtension;
     if (!drv) return STATUS_INVALID_PARAMETER;
-    if (irp->SectorCount == 0 || !irp->Buffer) return STATUS_INVALID_PARAMETER;
+
-    if (irp->Lba + irp->SectorCount > drv->sectorCount) return STATUS_INVALID_PARAMETER;
+    KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp);
+    if (!KSUCCESS(st))
+        return st;
+
+    bool write = false;
+    uint64_t lba = 0;
+    uint32_t sectors = 0;
+    void* buffer = (void*)0;
+    st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
+    (void)write;
+    (void)buffer;
+    if (!KSUCCESS(st) || lba + sectors > drv->sectorCount) {
+        IoReleaseRemoveLock(&drv->removeLock, irp);
+        return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
+    }
 
     irp->Status = STATUS_FAILURE;
     irp->Information = 0;
@@ -399,6 +435,49 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
     return STATUS_PENDING;
 }
 
+static void AhciCancelReleaseRemoveLock(IRP* irp, void* context) {
+    AHCI_DRIVE* drv = (AHCI_DRIVE*)context;
+    if (drv)
+        IoReleaseRemoveLock(&drv->removeLock, irp);
+}
+
+static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp) return STATUS_INVALID_PARAMETER;
+    AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
+    if (!drv) return STATUS_INVALID_PARAMETER;
+
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios || ios->MajorFunction != IRP_MJ_PNP)
+        return STATUS_INVALID_PARAMETER;
+
+    switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
+        case IRP_MN_START_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            return STATUS_SUCCESS;
+        case IRP_MN_QUERY_REMOVE_DEVICE:
+            device->PnpState = DevicePnpRemovePending;
+            return STATUS_SUCCESS;
+        case IRP_MN_CANCEL_REMOVE_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            return STATUS_SUCCESS;
+        case IRP_MN_SURPRISE_REMOVAL:
+            drv->surpriseRemoved = true;
+            device->PnpState = DevicePnpSurpriseRemoved;
+            IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
+            return STATUS_SUCCESS;
+        case IRP_MN_REMOVE_DEVICE:
+            drv->removed = true;
+            device->PnpState = DevicePnpDeleted;
+            IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
+            IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
+            IoDeleteDevice(device);
+            drv->deviceObject = (DEVICE_OBJECT*)0;
+            return STATUS_SUCCESS;
+        default:
+            return STATUS_NOT_SUPPORTED;
+    }
+}
+
 // AhciWorker - kernel thread, one per drive.
 // Dequeues IRPs and executes them one at a time using the existing
 // single-slot interrupt-driven path. Completion fires the IRP's event
@@ -420,17 +499,24 @@ static void AhciWorker(uintptr_t arg) {
         KeWaitSemaphore(drv->irpQueue.Semaphore);
 
         IRP *irp = IrpQueueDequeue(&drv->irpQueue);
-        if (!irp) KePanic(KE_PANIC_CODE_DRIVER, "AhciWorker: semaphore fired but queue is empty - IRP accounting bug\n");
+        if (!irp)
+            continue;
 
-        bool write = !!(irp->Flags & IRP_FLAG_WRITE);
+        bool write = false;
-        KSTATUS st;
+        uint64_t lba = 0;
-        if (write)
+        uint32_t sectors = 0;
-            st = do_write(drv, irp->Lba, irp->SectorCount, irp->Buffer);
+        void* buffer = (void*)0;
-        else
+        KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
-            st = do_read (drv, irp->Lba, irp->SectorCount, irp->Buffer);
+        if (KSUCCESS(st)) {
+            if (write)
+                st = do_write(drv, lba, sectors, buffer);
+            else
+                st = do_read (drv, lba, sectors, buffer);
+        }
 
         // Publish result then fire completion hooks through the common I/O path.
-        IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
+        IoReleaseRemoveLock(&drv->removeLock, irp);
+        IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
     }
 }
 
@@ -467,9 +553,10 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
     g_driveCount = 0;
 
     memset(&g_driver, 0, sizeof(g_driver));
-    g_driver.Name = "ahci";
+    IoInitializeDriverObject(&g_driver, SYS_SV_LITERAL("ahci"));
     g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
     g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
+    g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
 
     // -----------------------------------------------------------------------
     // Locate AHCI controller via PCI class scan
@@ -588,6 +675,11 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
             continue;
         }
         IrpQueueInit(&drv->irpQueue, irpSem);
+        st = IoInitializeRemoveLock(&drv->removeLock);
+        if (!KSUCCESS(st)) {
+            DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st);
+            continue;
+        }
 
         // Phase 6: IDENTIFY (polled - scheduler not yet running)
         st = port_identify(drv);
@@ -613,7 +705,8 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
             continue;
         }
         drv->deviceObject->DeviceExtension = drv;
-        drv->deviceObject->SectorCount = drv->sectorCount;
+        drv->deviceObject->u.Disk.SectorCount = drv->sectorCount;
+        drv->deviceObject->u.Disk.SectorSize = 512;
         drv->deviceObject->AlignmentRequirement = 512;
 
         st = IoRegisterDiskDevice((uint8_t)driveIdx, drv->deviceObject);

code/drivers/storage/ahci/ahci.h

@@ -344,6 +344,9 @@ typedef struct {
     // AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP
     // and signals the semaphore to wake the worker.
     IRP_QUEUE       irpQueue;
+    IO_REMOVE_LOCK  removeLock;
+    bool            surpriseRemoved;
+    bool            removed;
     THREAD*         workerThread;
     DEVICE_OBJECT*  deviceObject;
 

code/drivers/storage/ide/ide.c

@@ -21,6 +21,9 @@ IRP_QUEUE  gIrpQueue;
 SEMAPHORE *gIrpSemaphore = NULL;
 EVENT     *gDmaEvent     = NULL;  // auto-reset; DPC signals after each xfer
 KDPC       gDmaDpc;               // bottom half for DMA completion
+IO_REMOVE_LOCK gRemoveLock;
+bool       gSurpriseRemoved = false;
+bool       gRemoved = false;
 
 // ---------------------------------------------------------------------------
 // Bus Master I/O helpers
@@ -307,6 +310,28 @@ static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) {
     return STATUS_SUCCESS;
 }
 
+
+static KSTATUS IdeGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
+                                  uint32_t* outSectors, void** outBuffer) {
+    if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
+        return STATUS_INVALID_PARAMETER;
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios) return STATUS_INVALID_PARAMETER;
+    if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
+        return STATUS_INVALID_PARAMETER;
+    if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
+        (ios->Parameters.ReadWrite.Length & 511u))
+        return STATUS_INVALID_PARAMETER;
+    if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    *outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
+    *outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
+    *outSectors = ios->Parameters.ReadWrite.Length >> 9;
+    *outBuffer = irp->UserBuffer;
+    return STATUS_SUCCESS;
+}
+
 // ---------------------------------------------------------------------------
 // IdeWorker — kernel thread, one per IDE controller.
 //
@@ -320,17 +345,22 @@ void IdeWorker(uintptr_t arg) {
 
         IRP *irp = IrpQueueDequeue(&gIrpQueue);
         if (!irp)
-            KePanic(KE_PANIC_CODE_DRIVER, "IdeWorker: semaphore fired but IRP queue is empty — accounting bug\n");
+            continue;
 
-        KSTATUS st;
+        bool write = false;
-        if (irp->Lba + irp->SectorCount > gSectorCount) {
+        uint64_t lba = 0;
-            st = STATUS_INVALID_PARAMETER;
+        uint32_t sectors = 0;
-        } else {
+        void* buffer = (void*)0;
-            bool write = !!(irp->Flags & IRP_FLAG_WRITE);
+        KSTATUS st = IdeGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
-            st = IdeDmaRw(irp->Lba, irp->SectorCount, irp->Buffer, write);
+        if (KSUCCESS(st)) {
+            if (lba + sectors > gSectorCount)
+                st = STATUS_INVALID_PARAMETER;
+            else
+                st = IdeDmaRw(lba, sectors, buffer, write);
         }
 
-        IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
+        IoReleaseRemoveLock(&gRemoveLock, irp);
+        IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
     }
 }
 
@@ -344,8 +374,20 @@ void IdeWorker(uintptr_t arg) {
 // ---------------------------------------------------------------------------
 static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
     (void)device;
-    if (!irp || !irp->Buffer || irp->SectorCount == 0) return STATUS_INVALID_PARAMETER;
+
-    if (irp->Lba + irp->SectorCount > gSectorCount) return STATUS_INVALID_PARAMETER;
+    KSTATUS st = IoAcquireRemoveLock(&gRemoveLock, irp);
+    if (!KSUCCESS(st))
+        return st;
+
+    bool write = false;
+    uint64_t lba = 0;
+    uint32_t sectors = 0;
+    void* buffer = (void*)0;
+    st = IdeGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
+    if (!KSUCCESS(st) || lba + sectors > gSectorCount) {
+        IoReleaseRemoveLock(&gRemoveLock, irp);
+        return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
+    }
 
     irp->Status = STATUS_FAILURE;
     irp->Information = 0;
@@ -355,13 +397,53 @@ static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
         return STATUS_PENDING;
     }
 
-    bool write = !!(irp->Flags & IRP_FLAG_WRITE);
+    st = write ? IdeWrite(NULL, lba, sectors, buffer)
-    KSTATUS st = write ? IdeWrite(NULL, irp->Lba, irp->SectorCount, irp->Buffer)
+               : IdeRead (NULL, lba, sectors, buffer);
-                       : IdeRead (NULL, irp->Lba, irp->SectorCount, irp->Buffer);
+    IoReleaseRemoveLock(&gRemoveLock, irp);
-    IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
+    IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
     return STATUS_PENDING;
 }
 
+static void IdeCancelReleaseRemoveLock(IRP* irp, void* context) {
+    (void)context;
+    IoReleaseRemoveLock(&gRemoveLock, irp);
+}
+
+static KSTATUS IdeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp) return STATUS_INVALID_PARAMETER;
+
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios || ios->MajorFunction != IRP_MJ_PNP)
+        return STATUS_INVALID_PARAMETER;
+
+    switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
+        case IRP_MN_START_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            return STATUS_SUCCESS;
+        case IRP_MN_QUERY_REMOVE_DEVICE:
+            device->PnpState = DevicePnpRemovePending;
+            return STATUS_SUCCESS;
+        case IRP_MN_CANCEL_REMOVE_DEVICE:
+            device->PnpState = DevicePnpStarted;
+            return STATUS_SUCCESS;
+        case IRP_MN_SURPRISE_REMOVAL:
+            gSurpriseRemoved = true;
+            device->PnpState = DevicePnpSurpriseRemoved;
+            IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
+            return STATUS_SUCCESS;
+        case IRP_MN_REMOVE_DEVICE:
+            gRemoved = true;
+            device->PnpState = DevicePnpDeleted;
+            IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
+            IoReleaseRemoveLockAndWait(&gRemoveLock, irp);
+            IoDeleteDevice(device);
+            gDevice = (DEVICE_OBJECT*)0;
+            return STATUS_SUCCESS;
+        default:
+            return STATUS_NOT_SUPPORTED;
+    }
+}
+
 // ---------------------------------------------------------------------------
 // IdeDmaInit — locate the PCI IDE controller, grab BAR4, allocate the PRDT,
 // create sync objects, and register IRQ 14.
@@ -427,13 +509,18 @@ KSTATUS DriverEntry(const HAL_API *hal) {
 
     // ---- Register definitive DEVICE_OBJECT --------------------------------
     memset(&gDriver, 0, sizeof(gDriver));
-    gDriver.Name = "ide";
+    IoInitializeDriverObject(&gDriver, SYS_SV_LITERAL("ide"));
     gDriver.MajorFunction[IRP_MJ_READ] = IdeDispatchReadWrite;
     gDriver.MajorFunction[IRP_MJ_WRITE] = IdeDispatchReadWrite;
+    gDriver.MajorFunction[IRP_MJ_PNP] = IdeDispatchPnp;
 
-    KSTATUS st = IoCreateDevice(&gDriver, 0, &gDevice);
+    KSTATUS st = IoInitializeRemoveLock(&gRemoveLock);
     if (!KSUCCESS(st)) return st;
-    gDevice->SectorCount = gSectorCount;
+
+    st = IoCreateDevice(&gDriver, 0, &gDevice);
+    if (!KSUCCESS(st)) return st;
+    gDevice->u.Disk.SectorCount = gSectorCount;
+    gDevice->u.Disk.SectorSize = 512;
     gDevice->AlignmentRequirement = 512;
 
     // ---- Try to initialise Bus Master DMA ---------------------------------

code/kernel/hal/hal_main.c

@@ -82,11 +82,13 @@ static uintptr_t gIoapicPhys = 0xFEC00000;
 static uintptr_t gHpetPhys   = 0xFED00000;
 static HAL_CPU_TOPOLOGY gCpuTopology;
 
-// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5)
+// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5).
+// Do not guess SLP_TYP on real hardware: these values are firmware-specific.
 static uint16_t gPm1aCntBlk = 0;
 static uint16_t gPm1bCntBlk = 0;
-static uint8_t  gSlpTypA    = 5;   // S5 SLP_TYP for PM1a; 5 is correct for QEMU/most VMs
+static uint8_t  gSlpTypA    = 0;
-static uint8_t  gSlpTypB    = 5;   // S5 SLP_TYP for PM1b
+static uint8_t  gSlpTypB    = 0;
+static int      gS5Found    = 0;
 
 int debugEnabled = 0;
 int _fltused; // MSVC floating-point helper anchor
@@ -106,73 +108,183 @@ static void *HalPhysToVirt(uint64_t phys) {
     return (void *)(phys + gPhysBase);
 }
 
-// FADT (Fixed ACPI Description Table) - partial layout up to PM1b_CNT_BLK.
+// FADT (Fixed ACPI Description Table) offsets used here.  ACPI 2.0+
-// We only need offsets 40 (DSDT ptr), 64 (PM1a_CNT_BLK), 68 (PM1b_CNT_BLK).
+// systems may put the real DSDT and PM1 control registers in the extended
-// The full table is ~276 bytes (ACPI 2.0+); we never access beyond offset 68.
+// X_* fields, so do not rely only on the legacy 32-bit fields.
+#define FADT_DSDT_OFFSET           40u
+#define FADT_PM1A_CNT_BLK_OFFSET   64u
+#define FADT_PM1B_CNT_BLK_OFFSET   68u
+#define FADT_X_DSDT_OFFSET         140u
+#define FADT_X_PM1A_CNT_BLK_OFFSET 172u
+#define FADT_X_PM1B_CNT_BLK_OFFSET 184u
+
 #pragma pack(push, 1)
 typedef struct {
-    ACPISDTHeader Header;         // 0  (36 bytes)
+    uint8_t  AddressSpaceId;
-    uint32_t      FirmwareCtrl;   // 36
+    uint8_t  RegisterBitWidth;
-    uint32_t      Dsdt;           // 40 - physical addr of DSDT
+    uint8_t  RegisterBitOffset;
-    uint8_t       Ignored44;      // 44
+    uint8_t  AccessSize;
-    uint8_t       PreferredPmProfile; // 45
+    uint64_t Address;
-    uint16_t      SciInt;         // 46
+} ACPI_GAS;
-    uint32_t      SmiCmd;         // 48
-    uint8_t       AcpiEnable;     // 52
-    uint8_t       AcpiDisable;    // 53
-    uint8_t       S4BiosReq;      // 54
-    uint8_t       PstateCnt;      // 55
-    uint32_t      Pm1aEvtBlk;     // 56
-    uint32_t      Pm1bEvtBlk;     // 60
-    uint32_t      Pm1aCntBlk;     // 64 - I/O port for PM1a control register
-    uint32_t      Pm1bCntBlk;     // 68 - I/O port for PM1b control register
-} FADT;
 #pragma pack(pop)
 
-// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values.
+static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) {
-// The encoding looks like:
+    uint32_t value = 0;
-//   0x08 '_' 'S' '5' '_'   ; NameOp + "_S5_"
+    if (off + sizeof(value) <= tableLen)
-//   0x12 <PkgLen>           ; PackageOp + encoded length
+        memcpy(&value, (const uint8_t*)table + off, sizeof(value));
-//   <NumElements>           ; usually 0x04
+    return value;
-//   0x0A <slp_typ_a>        ; BytePrefix + value  (or 0x00 for zero)
+}
-//   0x0A <slp_typ_b>
-static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
-    for (uint32_t i = 0; i + 9 < len; i++) {
-        // Match NameOp + "_S5_" + PackageOp
-        if (aml[i]   != 0x08) continue;
-        if (aml[i+1] != 0x5F || aml[i+2] != 0x53 ||
-            aml[i+3] != 0x35 || aml[i+4] != 0x5F) continue;
-        if (aml[i+5] != 0x12) continue;
 
-        // PkgLength encoding: low 6 bits = byte count (0 → only this byte);
+static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off) {
-        // bits [7:6] = number of following bytes in the length field.
+    uint64_t value = 0;
-        uint8_t  pkgByte  = aml[i+6];
+    if (off + sizeof(value) <= tableLen)
-        uint32_t pkgExtra = pkgByte >> 6;           // 0..3 extra bytes
+        memcpy(&value, (const uint8_t*)table + off, sizeof(value));
-        uint32_t j        = i + 7 + pkgExtra;       // skip PkgLen field
+    return value;
-        if (j + 2 >= len) break;
+}
-        j++;                                         // skip NumElements byte
 
-        // SLP_TYP_A
+static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t off) {
-        if (aml[j] == 0x0A && j + 1 < len) {
+    ACPI_GAS gas;
-            gSlpTypA = aml[j + 1]; j += 2;
+    if (off + sizeof(gas) > tableLen) return 0;
-        } else if (aml[j] == 0x00) {
+
-            gSlpTypA = 0; j++;
+    memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
-        } else {
+
-            break;  // unexpected encoding - keep defaults
+    // 0x01 == System I/O.  HalShutdown currently writes PM1_CNT with outw(),
+    // so ignore MMIO GAS entries until the HAL grows MMIO register access here.
+    if (gas.AddressSpaceId != 0x01 || gas.Address == 0 || gas.Address > 0xFFFF)
+        return 0;
+
+    return (uint16_t)gas.Address;
+}
+
+static int AmlParsePkgLength(const uint8_t* p, const uint8_t* end,
+                             const uint8_t** body, const uint8_t** pkgEnd) {
+    if (p >= end) return 0;
+
+    const uint8_t* start = p;
+    uint8_t first = *p++;
+    uint32_t follow = first >> 6;
+    uint32_t length;
+
+    if (follow == 0) {
+        length = first & 0x3F;
+    } else {
+        length = first & 0x0F;
+        for (uint32_t i = 0; i < follow; i++) {
+            if (p >= end) return 0;
+            length |= ((uint32_t)*p++) << (4 + i * 8);
         }
+    }
 
-        // SLP_TYP_B
+    if (start + length < p || start + length > end) return 0;
-        if (j < len && aml[j] == 0x0A && j + 1 < len)
+    *body = p;
-            gSlpTypB = aml[j + 1];
+    *pkgEnd = start + length;
-        else if (j < len && aml[j] == 0x00)
+    return 1;
-            gSlpTypB = 0;
+}
+
+static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
+                                         int* lastSegmentIsS5) {
+    uint32_t segments = 1;
+    *lastSegmentIsS5 = 0;
+
+    // Optional root or parent prefixes: '\\', '^', '^^', ...
+    while (p < end && (*p == 0x5C || *p == 0x5E)) p++;
+    if (p >= end) return NULL;
+
+    if (*p == 0x00) {          // NullName
+        return p + 1;
+    } else if (*p == 0x2E) {   // DualNamePrefix
+        p++;
+        segments = 2;
+    } else if (*p == 0x2F) {   // MultiNamePrefix
+        p++;
+        if (p >= end) return NULL;
+        segments = *p++;
+    }
+
+    for (uint32_t s = 0; s < segments; s++) {
+        if (p + 4 > end) return NULL;
+        if (s == segments - 1 && memcmp(p, "_S5_", 4) == 0)
+            *lastSegmentIsS5 = 1;
+        p += 4;
+    }
+
+    return p;
+}
+
+static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
+    const uint8_t* p = *pp;
+    if (p >= end) return 0;
+
+    switch (*p++) {
+    case 0x00: // ZeroOp
+        *out = 0;
+        break;
+    case 0x01: // OneOp
+        *out = 1;
+        break;
+    case 0x0A: // BytePrefix
+        if (p >= end) return 0;
+        *out = *p++;
+        break;
+    case 0x0B: // WordPrefix
+        if (p + 2 > end) return 0;
+        *out = p[0];
+        p += 2;
+        break;
+    case 0x0C: // DWordPrefix
+        if (p + 4 > end) return 0;
+        *out = p[0];
+        p += 4;
+        break;
+    case 0x0E: // QWordPrefix
+        if (p + 8 > end) return 0;
+        *out = p[0];
+        p += 8;
+        break;
+    default:
+        return 0;
+    }
+
+    *pp = p;
+    return 1;
+}
+
+// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values.  Real
+// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and
+// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants.
+static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
+    const uint8_t* end = aml + len;
+
+    for (const uint8_t* cur = aml; cur < end; cur++) {
+        if (*cur != 0x08) continue; // NameOp
+
+        const uint8_t* p = cur + 1;
+        int isS5 = 0;
+        p = AmlParseNameString(p, end, &isS5);
+        if (!p || !isS5 || p >= end || *p != 0x12) continue; // PackageOp
+
+        const uint8_t* body;
+        const uint8_t* pkgEnd;
+        if (!AmlParsePkgLength(p + 1, end, &body, &pkgEnd)) continue;
+        if (body >= pkgEnd) continue;
+
+        uint8_t elements = *body++;
+        if (elements < 2) continue;
+
+        uint8_t typA, typB;
+        if (!AmlReadInteger(&body, pkgEnd, &typA)) continue;
+        if (!AmlReadInteger(&body, pkgEnd, &typB)) continue;
+
+        gSlpTypA = typA & 0x07;
+        gSlpTypB = typB & 0x07;
+        gS5Found = 1;
 
         HalDbgPrint("HAL: _S5_ SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n",
                     gSlpTypA, gSlpTypB);
         return;
     }
-    HalDbgPrint("HAL: _S5_ not found in DSDT, using defaults (A=0x%02x B=0x%02x)\n",
+
-                gSlpTypA, gSlpTypB);
+    HalDbgPrint("HAL: _S5_ not found in DSDT; ACPI shutdown disabled\n");
 }
 
 static int validate_checksum(void* table, uint32_t length) {
@@ -184,19 +296,33 @@ static int validate_checksum(void* table, uint32_t length) {
     return sum == 0;
 }
 
-static void ParseFadt(const FADT* fadt) {
+static void ParseFadt(const ACPISDTHeader* header) {
-    gPm1aCntBlk = (uint16_t)fadt->Pm1aCntBlk;
+    const void* fadt = header;
-    gPm1bCntBlk = (uint16_t)fadt->Pm1bCntBlk;
+    uint32_t len = header->Length;
-    HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%08x\n",
-                gPm1aCntBlk, gPm1bCntBlk, fadt->Dsdt);
 
-    if (fadt->Dsdt) {
+    gPm1aCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1A_CNT_BLK_OFFSET);
-        const ACPISDTHeader* dsdt =
+    gPm1bCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1B_CNT_BLK_OFFSET);
-            (const ACPISDTHeader*)((uintptr_t)fadt->Dsdt + gPhysBase);
+
+    if (!gPm1aCntBlk)
+        gPm1aCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1A_CNT_BLK_OFFSET);
+    if (!gPm1bCntBlk)
+        gPm1bCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1B_CNT_BLK_OFFSET);
+
+    uint64_t dsdtPhys = AcpiReadU64(fadt, len, FADT_X_DSDT_OFFSET);
+    if (!dsdtPhys)
+        dsdtPhys = AcpiReadU32(fadt, len, FADT_DSDT_OFFSET);
+
+    HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%016llx\n",
+                gPm1aCntBlk, gPm1bCntBlk, dsdtPhys);
+
+    if (dsdtPhys) {
+        const ACPISDTHeader* dsdt = (const ACPISDTHeader*)(dsdtPhys + gPhysBase);
         if (memcmp(dsdt->Signature, "DSDT", 4) == 0 &&
             validate_checksum((void*)dsdt, dsdt->Length)) {
             const uint8_t* aml = (const uint8_t*)(dsdt + 1);
             ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader));
+        } else {
+            HalDbgPrint("HAL: DSDT invalid or checksum failed\n");
         }
     }
 }
@@ -319,7 +445,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
         }
         // Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer
         else if (memcmp(header->Signature, "FACP", 4) == 0) {
-            ParseFadt((const FADT*)header);
+            ParseFadt(header);
         }
         // Parse HPET
         else if (memcmp(header->Signature, "HPET", 4) == 0) {
@@ -337,11 +463,17 @@ static void HalShutdown(void) {
     // Silence! Stop the world!
     _disable();
 
-    // SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
+    if (!gS5Found || !gPm1aCntBlk) {
-    uint16_t valA = (uint16_t)((gSlpTypA << 10) | (1 << 13));
+        // Refuse to guess.  Guessing SLP_TYP is exactly how real hardware gets
-    uint16_t valB = (uint16_t)((gSlpTypB << 10) | (1 << 13));
+        // put into S3/S4 sleep instead of S5 soft-off.
+        while (true) __halt();
+    }
 
-    if (gPm1aCntBlk) __outword(gPm1aCntBlk, valA);
+    // SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
+    uint16_t valA = (uint16_t)(((uint16_t)gSlpTypA << 10) | (1 << 13));
+    uint16_t valB = (uint16_t)(((uint16_t)gSlpTypB << 10) | (1 << 13));
+
+    __outword(gPm1aCntBlk, valA);
     if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB);
 
     // Die!

code/kernel/include/kernel/io_file.h

@@ -4,6 +4,7 @@
 // Kernel and user code open files through IoOpenFile.
 #include <stdint.h>
 #include <stddef.h>
+#include <stdbool.h>
 #include <ntlite/status.h>
 #include <kernel/ob_object.h>
 #include <kernel/ke_spinlock.h>
@@ -143,6 +144,9 @@ typedef struct IO_VOLUME_PARAMS {
 // ---------------------------------------------------------------------------
 // driveIndex matches the index passed to IoRegisterDiskDevice.
 __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol);
+__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
+__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
+__declspec(dllexport) bool    IoFilesystemPresent(uint8_t driveIndex);
 
 // ---------------------------------------------------------------------------
 // Kernel-side API

code/kernel/include/kernel/ke_cpu.h

@@ -30,6 +30,24 @@ typedef struct KCPU {
     uint8_t  Online;
     uint8_t  IsBsp;
 
+    // CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded
+    // from the APIC ID using CPUID leaf 0x1F/0x0B when available, with a
+    // conservative CPUID(1)/CPUID(4) fallback. PhysicalCoreIndex is a compact
+    // kernel-local core number used by the scheduler. CoreSiblingMask contains
+    // all logical CPUs that share this physical core. PackageMask contains all
+    // logical CPUs in the same package. IsPrimaryThread is the scheduler's
+    // preferred logical CPU for a physical core, currently the lowest CpuIndex
+    // in that core.
+    uint32_t PackageId;
+    uint32_t CoreId;
+    uint32_t ThreadId;
+    uint32_t PhysicalCoreIndex;
+    uint64_t CoreSiblingMask;
+    uint64_t PackageMask;
+    uint8_t  IsPrimaryThread;
+    uint8_t  SmtThreadCount;
+    uint16_t TopologyFlags;
+
     struct THREAD* CurrentThread;
     struct THREAD* IdleThread;
     struct THREAD* OutgoingThread;

code/kernel/include/kernel/ke_debug.h

@@ -3,10 +3,18 @@
 
 #include <stdint.h>
 #include <stddef.h>
+#include <ntlite/status.h>
+#include <ntlite/string_view.h>
 
-void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v);
+#define KE_DEBUG_CONSOLE   0x00000001u
+#define KE_DEBUG_SELFTEST  0x00000002u
+#define KE_DEBUG_FILE_LOG  0x00000004u
+
+void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
+uint32_t KeGetDebugFlags(void);
 __declspec(dllexport) void DbgPrint(const char* fmt, ...);
 __declspec(dllexport) void KePrintConsole(const char* buf, size_t len);
+__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path);
 
 typedef enum KE_PANIC_CODE {
     KE_PANIC_CODE_GENERIC  = 0,

code/kernel/include/kernel/ke_irp.h

@@ -1,83 +1,157 @@
 #pragma once
-// ke_irp.h  -  I/O Request Packet
+// ke_irp.h  -  I/O Request Packet and remove-lock primitives
 //
-// This header intentionally includes only ke_list.h, ke_spinlock.h, and
+// IRPs are stack-based. A caller builds an IRP with one or more
-// ke_status.h - all of which are pure-inline / macro headers with no
+// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O
-// external symbol references. ke_event.h and ke_semaphore.h are forward-
+// manager/driver stack advances CurrentLocation as the IRP is forwarded.
-// declared here so the HAL layer (hal_disk.c, io_driver.h) can include this
-// header without dragging in kernel sync-object link dependencies.
-//
-// Files that actually call KeEventCreate / KeSignalEvent / KeWaitForEvent
-// (ke_irp.c, ahci.c) include ke_event.h and ke_semaphore.h themselves.
 
 #include <stdint.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <kernel/ke_list.h>
 #include <kernel/ke_spinlock.h>
+#include <kernel/ke_irql.h>
 #include <ntlite/status.h>
 
-// Forward declarations - full types live in ke_event.h / ke_semaphore.h.
-// We only ever hold pointers in IRP and IRP_QUEUE, so a forward decl suffices.
 struct EVENT;
 struct SEMAPHORE;
 struct KAPC;
+typedef struct DEVICE_OBJECT DEVICE_OBJECT;
+
+// -------------------------------------------------------------------------
+// IRP major/minor functions
+// -------------------------------------------------------------------------
+
+typedef enum IO_MAJOR_FUNCTION {
+    IRP_MJ_CREATE                  = 0,
+    IRP_MJ_CLOSE                   = 1,
+    IRP_MJ_READ                    = 2,
+    IRP_MJ_WRITE                   = 3,
+    IRP_MJ_DEVICE_CONTROL          = 4,
+    IRP_MJ_INTERNAL_DEVICE_CONTROL = 5,
+    IRP_MJ_PNP                     = 6,
+    IRP_MJ_POWER                   = 7,
+    IRP_MJ_MAXIMUM_FUNCTION        = 8,
+} IO_MAJOR_FUNCTION;
+
+typedef enum IO_PNP_MINOR_FUNCTION {
+    IRP_MN_START_DEVICE           = 0,
+    IRP_MN_QUERY_REMOVE_DEVICE    = 1,
+    IRP_MN_CANCEL_REMOVE_DEVICE   = 2,
+    IRP_MN_SURPRISE_REMOVAL       = 3,
+    IRP_MN_REMOVE_DEVICE          = 4,
+    IRP_MN_QUERY_DEVICE_RELATIONS = 5,
+    IRP_MN_QUERY_ID               = 6,
+    IRP_MN_QUERY_CAPABILITIES     = 7,
+} IO_PNP_MINOR_FUNCTION;
+
+typedef enum IO_DEVICE_RELATION_TYPE {
+    IoBusRelations    = 0,
+    IoRemovalRelations = 1,
+} IO_DEVICE_RELATION_TYPE;
 
 // -------------------------------------------------------------------------
 // IRP flags
 // -------------------------------------------------------------------------
 
-#define IRP_FLAG_WRITE      (1u << 0)   // direction: write to device
+#define IRP_FLAG_PRIORITY   (1u << 0)   // insert at head of a driver queue
-#define IRP_FLAG_PRIORITY   (1u << 1)   // insert at head of queue (elevated)
+#define IRP_FLAG_CANCELLED  (1u << 1)
+#define IRP_FLAG_COMPLETED  (1u << 2)
 
 // -------------------------------------------------------------------------
-// Forward declaration (IRP is self-referential via CompletionFn)
+// IRP stack parameters
+// -------------------------------------------------------------------------
+
+typedef struct IO_STACK_LOCATION {
+    IO_MAJOR_FUNCTION MajorFunction;
+    uint8_t           MinorFunction;
+    DEVICE_OBJECT*    DeviceObject;
+
+    union {
+        struct {
+            uint64_t ByteOffset;
+            uint32_t Length;
+        } ReadWrite;
+
+        struct {
+            uint32_t IoControlCode;
+            void*    InputBuffer;
+            uint32_t InputLength;
+            void*    OutputBuffer;
+            uint32_t OutputLength;
+        } DeviceIoControl;
+
+        struct {
+            IO_DEVICE_RELATION_TYPE RelationType;
+        } QueryDeviceRelations;
+
+        struct {
+            uint32_t IdType;
+        } QueryId;
+    } Parameters;
+} IO_STACK_LOCATION;
+
+// -------------------------------------------------------------------------
+// Forward declaration / callbacks
 // -------------------------------------------------------------------------
 
 typedef struct IRP IRP;
 
-// Completion callback: invoked by IoCompleteRequest() in the completing
+typedef void (*IRP_COMPLETION_FN)(IRP* irp, void* ctx);
-// context. Driver worker, synchronous fallback, DPC, and APC completion paths
+typedef void (*IRP_CANCEL_ROUTINE)(DEVICE_OBJECT* device, IRP* irp);
-// all centralize policy here.
-typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx);
 
 // -------------------------------------------------------------------------
 // I/O Request Packet
 // -------------------------------------------------------------------------
 
 typedef struct IRP {
-    // Embedded list node - driver-internal; do not touch after submit.
     LIST_ENTRY              Link;
 
-    // ---- Request parameters (caller fills before IoSubmitIrp) ------------
-
-    uint8_t                 DriveIndex;
-    uint32_t                Flags;          // IRP_FLAG_*
-    uint64_t                Lba;
-    uint32_t                SectorCount;
-    void*                   Buffer;         // kernel VA; valid until completion
-
-    // ---- Completion (caller fills before submit; driver writes Status) ----
-
     volatile KSTATUS        Status;
-    volatile size_t         Information;    // bytes transferred on success
+    volatile size_t         Information;
 
-    // Forward-declared pointer: callers that create events include ke_event.h.
+    uint32_t                Flags;
-    struct EVENT           *CompletionEvent;
+    volatile bool           Cancel;
-
+    SPINLOCK                CancelLock;
-    // Optional kernel APC. If set, IoCompleteRequest queues it to its
+    KIRQL                   CancelIrql;
-    // target thread with arg1=irp and arg2=(void*)(uintptr_t)status.
+    IRP_CANCEL_ROUTINE      CancelRoutine;
-    struct KAPC            *CompletionApc;
 
+    struct EVENT*           CompletionEvent;
+    struct KAPC*            CompletionApc;
     IRP_COMPLETION_FN       CompletionFn;
     void*                   CompletionCtx;
 
-    // ---- Driver-internal --------------------------------------------------
+    void*                   UserBuffer;
+    void*                   SystemBuffer;
 
+    uint8_t                 StackCount;
+    uint8_t                 CurrentLocation;
+
+    // Driver-owned scratch fields. They are deliberately not interpreted by
+    // the I/O manager and are preserved across queueing/completion.
     uint8_t                 HwSlot;
     void*                   DriverCtx;
+
+    IO_STACK_LOCATION       Stack[1];
 } IRP;
 
+// -------------------------------------------------------------------------
+// Remove lock
+// -------------------------------------------------------------------------
+
+typedef struct IO_REMOVE_LOCK {
+    SPINLOCK        Lock;
+    volatile long   IoCount;
+    bool            Removing;
+    struct EVENT*   RemovedEvent;
+} IO_REMOVE_LOCK;
+
+__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock);
+__declspec(dllexport) void    IoDestroyRemoveLock(IO_REMOVE_LOCK* lock);
+__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
+__declspec(dllexport) void    IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
+__declspec(dllexport) void    IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp);
+
 // -------------------------------------------------------------------------
 // IRP_QUEUE  (per-device, driver-internal)
 // -------------------------------------------------------------------------
@@ -85,24 +159,56 @@ typedef struct IRP {
 typedef struct IRP_QUEUE {
     SPINLOCK            Lock;
     LIST_ENTRY          Head;
-    struct SEMAPHORE   *Semaphore;  // forward-declared pointer
+    struct SEMAPHORE*   Semaphore;
 } IRP_QUEUE;
 
-// Non-inline: defined in ke_irp.c, which can pull in ke_semaphore.h safely.
+__declspec(dllexport) void    IrpQueueInit    (IRP_QUEUE* q, struct SEMAPHORE* sem);
-__declspec(dllexport) void    IrpQueueInit    (IRP_QUEUE *q, struct SEMAPHORE *sem);
+__declspec(dllexport) void    IrpQueueEnqueue (IRP_QUEUE* q, IRP* irp);
-__declspec(dllexport) void    IrpQueueEnqueue (IRP_QUEUE *q, IRP *irp);
+__declspec(dllexport) IRP*    IrpQueueDequeue (IRP_QUEUE* q);
-__declspec(dllexport) IRP    *IrpQueueDequeue (IRP_QUEUE *q);
+typedef void (*IRP_QUEUE_CANCEL_CALLBACK)(IRP* irp, void* context);
+__declspec(dllexport) void    IrpQueueCancelAllEx(IRP_QUEUE* q,
+                                                  KSTATUS status,
+                                                  IRP_QUEUE_CANCEL_CALLBACK callback,
+                                                  void* context);
+__declspec(dllexport) void    IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status);
 
 // -------------------------------------------------------------------------
-// Public dispatch API - implemented in kernel/ke_irp.c
+// IRP allocation / stack helpers
 // -------------------------------------------------------------------------
 
+__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp);
+__declspec(dllexport) void    IoFreeIrp(IRP* irp);
+__declspec(dllexport) void    IoInitializeIrp(IRP* irp, uint8_t stackCount);
+
+__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp);
+__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp);
+__declspec(dllexport) void    IoSkipCurrentIrpStackLocation(IRP* irp);
+__declspec(dllexport) void    IoCopyCurrentIrpStackLocationToNext(IRP* irp);
+__declspec(dllexport) void    IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx);
+__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine);
+__declspec(dllexport) void    IoCancelIrp(IRP* irp);
+
+__declspec(dllexport) void    IoBuildReadWriteIrp(IRP* irp,
+                                                  IO_MAJOR_FUNCTION major,
+                                                  uint64_t byteOffset,
+                                                  uint32_t length,
+                                                  void* buffer);
+__declspec(dllexport) void    IoBuildPnpIrp(IRP* irp,
+                                            IO_PNP_MINOR_FUNCTION minor);
+__declspec(dllexport) void    IoBuildDeviceIoControlIrp(IRP* irp,
+                                                        IO_MAJOR_FUNCTION major,
+                                                        uint32_t ioControlCode,
+                                                        void* inputBuffer,
+                                                        uint32_t inputLength,
+                                                        void* outputBuffer,
+                                                        uint32_t outputLength);
+
 // Complete an IRP exactly once: publish status/information, signal the
-// optional completion event, then invoke the optional completion callback.
+// optional completion event, queue the optional APC, then invoke the optional
-__declspec(dllexport) void    IoCompleteRequest(IRP *irp, KSTATUS status, size_t information);
+// completion callback.
+__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information);
 
-// Submit asynchronously. Returns STATUS_PENDING on success.
+// Submit to a specific top-of-stack device object. The current stack location
-__declspec(dllexport) KSTATUS IoSubmitIrp    (IRP *irp);
+// is stamped with the target device before dispatch.
-
+__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp);
-// Submit and block until complete. Returns irp->Status.
+__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp);
-__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp);

code/kernel/include/kernel/ke_sched.h

@@ -23,6 +23,8 @@ typedef struct KE_SCHED_STATS {
     uint64_t PriorityPreemptions;
     uint64_t VoluntaryBlocks;
     uint64_t CpuBurstDemotions;
+    uint64_t SmtSiblingDeferrals;
+    uint64_t SmtPrimaryPokes;
 } KE_SCHED_STATS;
 
 // Called once by KernelMain before any threads are created.

code/kernel/include/kernel/ke_smp.h

@@ -13,6 +13,12 @@ KSTATUS KeStartApplicationProcessors(void);
 // park loop into their idle threads.
 KSTATUS KeAdmitApplicationProcessorsToScheduler(void);
 
+
+// Decode CPUID topology and build per-KCPU package/core/SMT sibling masks.
+// KeStartApplicationProcessors calls this after MADT enumeration; it is also
+// safe to call again after failed AP startup disables a CPU record.
+void KeBuildProcessorTopology(void);
+
 // Run explicit AP-affine kernel workers that exercise the global scheduler,
 // reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver
 // threads remain BSP-affine unless their affinity is explicitly widened.

code/kernel/include/kernel/ob_object.h

@@ -17,6 +17,10 @@ typedef enum OBJECT_TYPE {
     ObjectTypeDir       = 8,
     ObjectTypeAsyncIo   = 9,
     ObjectTypeSocket    = 10,
+    ObjectTypeDevice    = 11,
+    ObjectTypeDriver    = 12,
+    ObjectTypeUsbDevice = 13,
+    ObjectTypeUsbPipe   = 14,
 } OBJECT_TYPE;
 
 typedef struct OBJECT {

code/kernel/io_device.c

@@ -1,13 +1,423 @@
-// io_device.c - kernel driver/device dispatch and disk registry
+// io_device.c - I/O manager: driver/device objects, stacks, PnP skeleton, disk namespace
 #include <string.h>
+#include <intrin.h>
 #include <ntlite/ddi/io_driver.h>
 #include <kernel/ke_irp.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ke_debug.h>
-
+#include <kernel/ob_object.h>
 
 #define MAX_PUBLISHED_DEVICES 64
-#define MAX_DRIVER_MATCHERS 32
+#define MAX_DRIVER_MATCHERS   32
+#define MAX_DRIVER_OBJECTS    64
+#define MAX_PDO_OBJECTS       128
+
+static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) {
+    return SysSvIsEmpty(name) ? SYS_SV_LITERAL("<unnamed>") : name;
+}
+
+static const char* IoBusTypeName(IO_BUS_TYPE busType) {
+    switch (busType) {
+        case IoBusPci: return "pci";
+        case IoBusUsb: return "usb";
+        default: return "unknown";
+    }
+}
+
+// -------------------------------------------------------------------------
+// Driver object registry / PDO AddDevice binding
+// -------------------------------------------------------------------------
+
+static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS];
+static uint32_t gDriverObjectCount;
+static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS];
+static uint32_t gPhysicalDeviceCount;
+
+static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
+    if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical)
+        return false;
+
+    for (size_t d = 0; d < driver->IdTable->Count; d++) {
+        const IO_DEVICE_ID* want = &driver->IdTable->Ids[d];
+        if (want->BusType != IoBusUnknown && want->BusType != pdo->u.Pdo.BusType)
+            continue;
+        for (uint8_t p = 0; p < pdo->u.Pdo.IdCount; p++) {
+            if (SysSvEqual(want->Id, pdo->u.Pdo.Ids[p].Id))
+                return true;
+        }
+    }
+    return false;
+}
+
+static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) {
+    if (!driver || !pdo || !driver->DriverAddDevice)
+        return false;
+    if (!IoDriverMatchesPdo(driver, pdo))
+        return false;
+
+    KSTATUS st = driver->DriverAddDevice(driver, pdo);
+    if (KSUCCESS(st)) {
+        DbgPrint("io: AddDevice driver=%v pdo=%v bus=%s\n",
+                 IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), IoBusTypeName(pdo->u.Pdo.BusType));
+
+        KSTATUS start = IoSendPnpIrpSync(pdo, IRP_MN_START_DEVICE);
+        if (!KSUCCESS(start) && start != STATUS_NOT_SUPPORTED) {
+            DbgPrint("io: START_DEVICE failed driver=%v pdo=%v st=0x%x\n",
+                     IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), start);
+        }
+        return true;
+    } else {
+        DbgPrint("io: AddDevice failed driver=%v pdo=%v st=0x%x\n",
+                 IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), st);
+        return true;
+    }
+}
+
+__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
+                                                    SYS_STRING_VIEW name) {
+    if (!driver) return;
+    memset(driver, 0, sizeof(*driver));
+    driver->Header.Type = ObjectTypeDriver;
+    driver->Header.RefCount = 1;
+    driver->Header.Destroy = (void*)0;
+    driver->Name = name;
+}
+
+__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
+    if (!driver) return STATUS_INVALID_PARAMETER;
+    if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
+        return STATUS_NO_MEMORY;
+
+    if (driver->Header.Type != ObjectTypeDriver) {
+        driver->Header.Type = ObjectTypeDriver;
+        driver->Header.RefCount = 1;
+    }
+
+    gDriverObjects[gDriverObjectCount++] = driver;
+    DbgPrint("io: registered driver object %v\n", IoSvOrUnnamed(driver->Name));
+
+    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++)
+        (void)IoTryAddDeviceForDriver(driver, gPhysicalDevices[i]);
+
+    return STATUS_SUCCESS;
+}
+
+// -------------------------------------------------------------------------
+// Device object lifetime
+// -------------------------------------------------------------------------
+
+static void IoDeviceDestroy(OBJECT* object) {
+    DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
+
+    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
+        if (gPhysicalDevices[i] == dev) {
+            gPhysicalDevices[i] = gPhysicalDevices[gPhysicalDeviceCount - 1u];
+            gPhysicalDevices[gPhysicalDeviceCount - 1u] = (DEVICE_OBJECT*)0;
+            gPhysicalDeviceCount--;
+            break;
+        }
+    }
+
+    if (dev->LowerDevice)
+        IoDetachDevice(dev);
+
+    if (dev->Parent) {
+        DEVICE_OBJECT* parent = dev->Parent;
+        uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
+        DEVICE_OBJECT** link = &parent->FirstChild;
+        while (*link) {
+            if (*link == dev) {
+                *link = dev->NextSibling;
+                break;
+            }
+            link = &(*link)->NextSibling;
+        }
+        KeReleaseSpinlock(&parent->Lock, flags);
+        dev->Parent = (DEVICE_OBJECT*)0;
+        dev->NextSibling = (DEVICE_OBJECT*)0;
+        IoDereferenceDeviceObject(parent);
+    }
+}
+
+static KSTATUS IoCreateDeviceInternal(DRIVER_OBJECT* driver,
+                                      DEVICE_OBJECT_TYPE type,
+                                      size_t extensionSize,
+                                      DEVICE_OBJECT** outDevice) {
+    if (!driver || !outDevice) return STATUS_INVALID_PARAMETER;
+
+    size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
+    DEVICE_OBJECT* dev = (DEVICE_OBJECT*)ObAllocateObject(ObjectTypeDevice, total);
+    if (!dev) return STATUS_NO_MEMORY;
+
+    dev->Header.Destroy = IoDeviceDestroy;
+    dev->DriverObject = driver;
+    dev->Type = type;
+    dev->PnpState = DevicePnpNotStarted;
+    dev->AlignmentRequirement = 1;
+    dev->Lock = (SPINLOCK)SPINLOCK_INIT;
+
+    // The per-device union is interpreted differently for PDOs and disk FDOs.
+    // Do not write disk defaults into a PDO: u.Disk.DiskIndex aliases the low
+    // byte of u.Pdo.BusType, which prevents ID-table AddDevice matching.
+    if (type != DeviceObjectPhysical) {
+        dev->u.Disk.DiskIndex = 0xFF;
+        dev->u.Disk.SectorSize = 512;
+    }
+
+    if (extensionSize)
+        dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));
+
+    *outDevice = dev;
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
+                                             size_t extensionSize,
+                                             DEVICE_OBJECT** outDevice) {
+    return IoCreateDeviceInternal(driver, DeviceObjectFunction, extensionSize, outDevice);
+}
+
+__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
+                                                     DEVICE_OBJECT* parent,
+                                                     const IO_DEVICE_ID* ids,
+                                                     size_t idCount,
+                                                     size_t extensionSize,
+                                                     DEVICE_OBJECT** outPdo) {
+    if (!busDriver || !outPdo) return STATUS_INVALID_PARAMETER;
+    if (idCount > IO_MAX_DEVICE_IDS) return STATUS_INVALID_PARAMETER;
+
+    DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
+    KSTATUS st = IoCreateDeviceInternal(busDriver, DeviceObjectPhysical, extensionSize, &pdo);
+    if (!KSUCCESS(st)) return st;
+
+    if (idCount && ids) {
+        for (size_t i = 0; i < idCount; i++) {
+            if (!SysSvIsValid(ids[i].Id) || ids[i].Id.Length >= IO_MAX_DEVICE_ID_CHARS) {
+                IoDeleteDevice(pdo);
+                return STATUS_INVALID_PARAMETER;
+            }
+
+            char* dst = pdo->u.Pdo.IdStorage[i];
+            if (ids[i].Id.Length)
+                memcpy(dst, ids[i].Id.Data, ids[i].Id.Length);
+            dst[ids[i].Id.Length] = '\0';
+
+            pdo->u.Pdo.Ids[i].BusType = ids[i].BusType;
+            pdo->u.Pdo.Ids[i].Id = SysSvFromParts(dst, ids[i].Id.Length);
+            if (pdo->u.Pdo.BusType == IoBusUnknown)
+                pdo->u.Pdo.BusType = ids[i].BusType;
+        }
+        pdo->u.Pdo.IdCount = (uint8_t)idCount;
+    }
+
+    if (parent) {
+        IoReferenceDeviceObject(parent);
+        pdo->Parent = parent;
+        uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
+        pdo->NextSibling = parent->FirstChild;
+        parent->FirstChild = pdo;
+        KeReleaseSpinlock(&parent->Lock, flags);
+    }
+
+    *outPdo = pdo;
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
+    if (!pdo || pdo->Type != DeviceObjectPhysical)
+        return STATUS_INVALID_PARAMETER;
+
+    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
+        if (gPhysicalDevices[i] == pdo)
+            return STATUS_ALREADY_EXISTS;
+    }
+
+    if (gPhysicalDeviceCount >= MAX_PDO_OBJECTS)
+        return STATUS_NO_MEMORY;
+
+    gPhysicalDevices[gPhysicalDeviceCount++] = pdo;
+
+    bool anyMatched = false;
+    for (uint32_t i = 0; i < gDriverObjectCount; i++) {
+        if (IoTryAddDeviceForDriver(gDriverObjects[i], pdo))
+            anyMatched = true;
+    }
+
+    if (!anyMatched && pdo->u.Pdo.BusType == IoBusUsb) {
+        SYS_STRING_VIEW id = pdo->u.Pdo.IdCount ? pdo->u.Pdo.Ids[0].Id : SYS_SV_LITERAL("<no-id>");
+        DbgPrint("io: no function driver for pdo=%v id=%v bus=%s\n",
+                 IoSvOrUnnamed(pdo->Name), id, IoBusTypeName(pdo->u.Pdo.BusType));
+    }
+    return STATUS_SUCCESS;
+}
+
+
+__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
+                                               SYS_STRING_VIEW name) {
+    if (!device || !SysSvIsValid(name))
+        return STATUS_INVALID_PARAMETER;
+    if (name.Length >= IO_MAX_DEVICE_NAME_CHARS)
+        return STATUS_BUFFER_TOO_SMALL;
+
+    if (name.Length)
+        memcpy(device->NameStorage, name.Data, name.Length);
+    device->NameStorage[name.Length] = '\0';
+    device->Name = SysSvFromParts(device->NameStorage, name.Length);
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) {
+    if (device) ObReferenceObject(&device->Header);
+}
+
+__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
+    if (device) ObDereferenceObject(&device->Header);
+}
+
+static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
+    if (!device || device->Type != DeviceObjectPhysical)
+        return;
+
+    for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
+        if (gPhysicalDevices[i] == device) {
+            for (uint32_t j = i + 1; j < gPhysicalDeviceCount; j++)
+                gPhysicalDevices[j - 1] = gPhysicalDevices[j];
+            gPhysicalDeviceCount--;
+            if (gPhysicalDeviceCount < MAX_PDO_OBJECTS)
+                gPhysicalDevices[gPhysicalDeviceCount] = (DEVICE_OBJECT*)0;
+            return;
+        }
+    }
+}
+
+static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
+    if (!device || !device->Parent)
+        return;
+
+    DEVICE_OBJECT* parent = device->Parent;
+    uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
+    DEVICE_OBJECT** link = &parent->FirstChild;
+    while (*link) {
+        if (*link == device) {
+            *link = device->NextSibling;
+            break;
+        }
+        link = &(*link)->NextSibling;
+    }
+    device->Parent = (DEVICE_OBJECT*)0;
+    device->NextSibling = (DEVICE_OBJECT*)0;
+    KeReleaseSpinlock(&parent->Lock, flags);
+
+    IoDereferenceDeviceObject(parent);
+}
+
+__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
+    if (!device) return;
+    if (device->DeletePending)
+        return;
+
+    device->DeletePending = true;
+    device->PnpState = DevicePnpDeleted;
+    IoUnlinkDeviceFromParent(device);
+    IoRemovePhysicalDeviceRegistryEntry(device);
+    IoDereferenceDeviceObject(device);
+}
+
+// -------------------------------------------------------------------------
+// Device stack management
+// -------------------------------------------------------------------------
+
+__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
+                                                                 DEVICE_OBJECT* target) {
+    if (!source || !target) return (DEVICE_OBJECT*)0;
+
+    DEVICE_OBJECT* top = target;
+    while (top->AttachedDevice)
+        top = top->AttachedDevice;
+
+    top->AttachedDevice = source;
+    source->LowerDevice = top;
+    IoReferenceDeviceObject(top);
+    return top;
+}
+
+__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
+    if (!source || !source->LowerDevice) return;
+
+    DEVICE_OBJECT* lower = source->LowerDevice;
+    if (lower->AttachedDevice == source)
+        lower->AttachedDevice = (DEVICE_OBJECT*)0;
+    source->LowerDevice = (DEVICE_OBJECT*)0;
+    IoDereferenceDeviceObject(lower);
+}
+
+__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device) {
+    if (!device) return (DEVICE_OBJECT*)0;
+    DEVICE_OBJECT* top = device;
+    while (top->AttachedDevice)
+        top = top->AttachedDevice;
+    IoReferenceDeviceObject(top);
+    return top;
+}
+
+// -------------------------------------------------------------------------
+// Device-stack helpers / PnP dispatch
+// -------------------------------------------------------------------------
+
+__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
+    uint8_t count = 0;
+    for (DEVICE_OBJECT* cur = device; cur && count < 0xFEu; cur = cur->LowerDevice)
+        count++;
+    return count ? count : 1;
+}
+
+__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) {
+    if (!device || !irp || !device->LowerDevice)
+        return STATUS_NOT_SUPPORTED;
+
+    IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
+    IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
+    if (!cur || !next)
+        return STATUS_INVALID_PARAMETER;
+
+    DEVICE_OBJECT* currentDevice = cur->DeviceObject;
+    memcpy(next, cur, sizeof(*next));
+
+    if (currentDevice == device) {
+        cur->DeviceObject = (DEVICE_OBJECT*)0;
+        _InterlockedDecrement(&device->OutstandingIrps);
+        IoDereferenceDeviceObject(device);
+    }
+
+    irp->CurrentLocation++;
+    return IoCallDriver(device->LowerDevice, irp);
+}
+
+__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
+                                                IO_PNP_MINOR_FUNCTION minor) {
+    if (!device) return STATUS_INVALID_PARAMETER;
+
+    DEVICE_OBJECT* top = IoGetAttachedDeviceReference(device);
+    if (!top) return STATUS_INVALID_PARAMETER;
+
+    uint8_t stackCount = IoGetDeviceStackSize(top);
+    IRP* irp = (IRP*)0;
+    KSTATUS st = IoAllocateIrp(stackCount, &irp);
+    if (!KSUCCESS(st)) {
+        IoDereferenceDeviceObject(top);
+        return st;
+    }
+
+    IoBuildPnpIrp(irp, minor);
+    st = IoSubmitIrpSync(top, irp);
+    IoFreeIrp(irp);
+    IoDereferenceDeviceObject(top);
+    return st;
+}
+
+// -------------------------------------------------------------------------
+// Legacy publish/matcher path
+// -------------------------------------------------------------------------
 
 typedef struct IO_DRIVER_MATCHER_ENTRY {
     bool Used;
@@ -23,14 +433,6 @@ static uint32_t gPublishedDeviceCount;
 static IO_DRIVER_MATCHER_ENTRY gDriverMatchers[MAX_DRIVER_MATCHERS];
 static uint32_t gDriverMatcherCount;
 
-static const char* IoBusTypeName(IO_BUS_TYPE busType) {
-    switch (busType) {
-        case IoBusPci: return "pci";
-        case IoBusUsb: return "usb";
-        default: return "unknown";
-    }
-}
-
 static bool IoTryBindDevice(IO_PUBLISHED_DEVICE* device,
                             IO_DRIVER_MATCHER_ENTRY* matcher) {
     if (!device || !matcher || !matcher->Used || device->Bound)
@@ -80,12 +482,10 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
     entry->Bind = bind;
     entry->Context = context;
 
-    DbgPrint("io: registered %s matcher %s\n",
+    DbgPrint("io: registered %s matcher %s\n", IoBusTypeName(busType), driverName);
-             IoBusTypeName(busType), driverName);
 
-    for (uint32_t i = 0; i < gPublishedDeviceCount; i++) {
+    for (uint32_t i = 0; i < gPublishedDeviceCount; i++)
         IoTryBindDevice(&gPublishedDevices[i], entry);
-    }
 
     return STATUS_SUCCESS;
 }
@@ -101,7 +501,6 @@ __declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device)
     stored->Bound = false;
     stored->BoundDriverName = (const char*)0;
 
-
     for (uint32_t i = 0; i < gDriverMatcherCount; i++) {
         if (IoTryBindDevice(stored, &gDriverMatchers[i]))
             break;
@@ -114,50 +513,44 @@ __declspec(dllexport) uint32_t IoPublishedDeviceCount(void) {
     return gPublishedDeviceCount;
 }
 
-#define MAX_DISK_DEVICES 8
+// -------------------------------------------------------------------------
-#define MAX_DISK_ARRIVAL_CALLBACKS 16
+// Disk namespace
+// -------------------------------------------------------------------------
 
-typedef struct IO_DISK_CALLBACK_ENTRY {
+#define MAX_DISK_DEVICES 8
+#define MAX_DISK_CALLBACKS 16
+
+typedef struct IO_DISK_CHANGE_CALLBACK_ENTRY {
+    bool Used;
+    IO_DISK_CHANGE_CALLBACK Callback;
+    void* Context;
+} IO_DISK_CHANGE_CALLBACK_ENTRY;
+
+typedef struct IO_DISK_ARRIVAL_CALLBACK_ENTRY {
     bool Used;
     IO_DISK_ARRIVAL_CALLBACK Callback;
     void* Context;
-} IO_DISK_CALLBACK_ENTRY;
+} IO_DISK_ARRIVAL_CALLBACK_ENTRY;
 
 static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES];
-static IO_DISK_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_ARRIVAL_CALLBACKS];
+static IO_DISK_CHANGE_CALLBACK_ENTRY gDiskChangeCallbacks[MAX_DISK_CALLBACKS];
+static uint32_t gDiskChangeCallbackCount;
+static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
 static uint32_t gDiskArrivalCallbackCount;
 
-static void IoNotifyDiskArrived(uint8_t driveIndex, DEVICE_OBJECT* device) {
+static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
-    for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
+    for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
-        IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
+        IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i];
         if (entry->Used && entry->Callback)
-            entry->Callback(driveIndex, device, entry->Context);
+            entry->Callback(driveIndex, device, change, entry->Context);
+    }
+    if (change == IoDeviceArrived) {
+        for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
+            IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
+            if (entry->Used && entry->Callback)
+                entry->Callback(driveIndex, device, entry->Context);
+        }
     }
-}
-
-__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
-                                             size_t extensionSize,
-                                             DEVICE_OBJECT** outDevice) {
-    if (!driver || !outDevice) return STATUS_INVALID_PARAMETER;
-
-    size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
-    DEVICE_OBJECT* dev = (DEVICE_OBJECT*)MmAllocatePool(total);
-    if (!dev) return STATUS_NO_MEMORY;
-
-    memset(dev, 0, total);
-    dev->DriverObject = driver;
-    if (extensionSize)
-        dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));
-    dev->AlignmentRequirement = 1;
-    dev->DiskIndex = 0xFF;
-
-    *outDevice = dev;
-    return STATUS_SUCCESS;
-}
-
-__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
-    if (!device) return;
-    MmFreePool(device);
 }
 
 __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
@@ -168,9 +561,57 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
         return STATUS_ALREADY_EXISTS;
 
     device->Flags |= DEVICE_OBJECT_DISK;
-    device->DiskIndex = driveIndex;
+    device->u.Disk.DiskIndex = driveIndex;
+    if (device->u.Disk.SectorSize == 0)
+        device->u.Disk.SectorSize = 512;
+
+    KSTATUS pnpStatus = IoSendPnpIrpSync(device, IRP_MN_START_DEVICE);
+    if (!KSUCCESS(pnpStatus) && pnpStatus != STATUS_NOT_SUPPORTED)
+        return pnpStatus;
+    if (pnpStatus == STATUS_NOT_SUPPORTED)
+        device->PnpState = DevicePnpStarted;
+
+    IoReferenceDeviceObject(device);
     gDiskDevices[driveIndex] = device;
-    IoNotifyDiskArrived(driveIndex, device);
+    IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
+    if (driveIndex >= MAX_DISK_DEVICES)
+        return STATUS_INVALID_PARAMETER;
+    DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
+    if (!dev) return STATUS_NOT_FOUND;
+
+    dev->PnpState = DevicePnpSurpriseRemoved;
+    IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
+    (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
+    if (driveIndex >= MAX_DISK_DEVICES)
+        return STATUS_INVALID_PARAMETER;
+    DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
+    if (!dev) return STATUS_NOT_FOUND;
+
+    if (dev->PnpState != DevicePnpSurpriseRemoved) {
+        IoNotifyDiskChange(driveIndex, dev, IoDeviceQueryRemove);
+        KSTATUS queryStatus = IoSendPnpIrpSync(dev, IRP_MN_QUERY_REMOVE_DEVICE);
+        if (!KSUCCESS(queryStatus) && queryStatus != STATUS_NOT_SUPPORTED)
+            return queryStatus;
+
+        dev->PnpState = DevicePnpSurpriseRemoved;
+        IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
+        (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
+    }
+
+    dev->PnpState = DevicePnpRemovePending;
+    (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
+
+    gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
+    IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
+    IoDereferenceDeviceObject(dev);
     return STATUS_SUCCESS;
 }
 
@@ -178,10 +619,10 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL
                                                             void* context) {
     if (!callback)
         return STATUS_INVALID_PARAMETER;
-    if (gDiskArrivalCallbackCount >= MAX_DISK_ARRIVAL_CALLBACKS)
+    if (gDiskArrivalCallbackCount >= MAX_DISK_CALLBACKS)
         return STATUS_NO_MEMORY;
 
-    IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
+    IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
     memset(entry, 0, sizeof(*entry));
     entry->Used = true;
     entry->Callback = callback;
@@ -191,7 +632,21 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL
         if (gDiskDevices[i])
             callback(i, gDiskDevices[i], context);
     }
+    return STATUS_SUCCESS;
+}
 
+__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback,
+                                                           void* context) {
+    if (!callback)
+        return STATUS_INVALID_PARAMETER;
+    if (gDiskChangeCallbackCount >= MAX_DISK_CALLBACKS)
+        return STATUS_NO_MEMORY;
+
+    IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[gDiskChangeCallbackCount++];
+    memset(entry, 0, sizeof(*entry));
+    entry->Used = true;
+    entry->Callback = callback;
+    entry->Context = context;
     return STATUS_SUCCESS;
 }
 
@@ -204,7 +659,6 @@ __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK ca
         if (gDiskDevices[i])
             callback(i, gDiskDevices[i], context);
     }
-
     return STATUS_SUCCESS;
 }
 
@@ -213,28 +667,59 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
     return gDiskDevices[driveIndex];
 }
 
+__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) {
+    DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
+    if (dev) IoReferenceDeviceObject(dev);
+    return dev;
+}
+
 __declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
     return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
 }
 
 __declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
     DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
-    return dev ? dev->SectorCount : 0;
+    return dev ? dev->u.Disk.SectorCount : 0;
 }
 
+// -------------------------------------------------------------------------
+// Dispatch
+// -------------------------------------------------------------------------
+
 __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
     if (!device || !device->DriverObject || !irp)
         return STATUS_INVALID_PARAMETER;
 
-    IO_MAJOR_FUNCTION major = (irp->Flags & IRP_FLAG_WRITE) ? IRP_MJ_WRITE : IRP_MJ_READ;
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios)
+        return STATUS_INVALID_PARAMETER;
+
+    IO_MAJOR_FUNCTION major = ios->MajorFunction;
     if (major >= IRP_MJ_MAXIMUM_FUNCTION)
         return STATUS_INVALID_PARAMETER;
 
+    if (major != IRP_MJ_PNP &&
+        (device->DeletePending || device->PnpState == DevicePnpSurpriseRemoved ||
+         device->PnpState == DevicePnpRemovePending || device->PnpState == DevicePnpDeleted))
+        return STATUS_OBJECT_DEAD;
+
     PDRIVER_DISPATCH dispatch = device->DriverObject->MajorFunction[major];
     if (!dispatch)
         return STATUS_NOT_SUPPORTED;
 
-    return dispatch(device, irp);
+    ios->DeviceObject = device;
+    IoReferenceDeviceObject(device);
+    _InterlockedIncrement(&device->OutstandingIrps);
+
+    KSTATUS st = dispatch(device, irp);
+    if (st != STATUS_PENDING) {
+        if (ios->DeviceObject == device) {
+            ios->DeviceObject = (DEVICE_OBJECT*)0;
+            _InterlockedDecrement(&device->OutstandingIrps);
+            IoDereferenceDeviceObject(device);
+        }
+    }
+    return st;
 }
 
 __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
@@ -243,17 +728,15 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
                                                 void* buffer) {
     if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
 
-    IRP irp;
+    DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
-    memset(&irp, 0, sizeof(irp));
+    if (!dev) return STATUS_NOT_FOUND;
-    irp.DriveIndex = driveIndex;
-    irp.Flags = 0;
-    irp.Lba = lba;
-    irp.SectorCount = count;
-    irp.Buffer = buffer;
-    irp.Status = STATUS_FAILURE;
-    irp.Information = 0;
 
-    return IoSubmitIrpSync(&irp);
+    IRP irp;
+    IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
+    KSTATUS st = IoSubmitIrpSync(dev, &irp);
+
+    IoDereferenceDeviceObject(dev);
+    return st;
 }
 
 __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
@@ -262,15 +745,13 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
                                                  const void* buffer) {
     if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
 
-    IRP irp;
+    DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
-    memset(&irp, 0, sizeof(irp));
+    if (!dev) return STATUS_NOT_FOUND;
-    irp.DriveIndex = driveIndex;
-    irp.Flags = IRP_FLAG_WRITE;
-    irp.Lba = lba;
-    irp.SectorCount = count;
-    irp.Buffer = (void*)buffer;
-    irp.Status = STATUS_FAILURE;
-    irp.Information = 0;
 
-    return IoSubmitIrpSync(&irp);
+    IRP irp;
+    IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer);
+    KSTATUS st = IoSubmitIrpSync(dev, &irp);
+
+    IoDereferenceDeviceObject(dev);
+    return st;
 }

code/kernel/io_file.c

@@ -586,6 +586,33 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
     return STATUS_SUCCESS;
 }
 
+__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
+    if (volumeIndex >= MAX_DRIVES)
+        return STATUS_INVALID_PARAMETER;
+    if (!gVolumes[volumeIndex].Fs)
+        return STATUS_NOT_FOUND;
+
+    memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex]));
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
+    KSTATUS last = STATUS_NOT_FOUND;
+    for (uint8_t i = 0; i < MAX_DRIVES; i++) {
+        if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) {
+            memset(&gVolumes[i], 0, sizeof(gVolumes[i]));
+            last = STATUS_SUCCESS;
+        }
+    }
+    return last;
+}
+
+__declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
+    if (volumeIndex >= MAX_DRIVES)
+        return false;
+    return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
+}
+
 // ---------------------------------------------------------------------------
 // Destructor
 // ---------------------------------------------------------------------------

code/kernel/ke_cpu.c

@@ -48,6 +48,15 @@ void KeInitializeBootstrapCpu(void) {
     cpu->Parked = 0;
     cpu->Online = 1;
     cpu->IsBsp = 1;
+    cpu->PackageId = 0;
+    cpu->CoreId = 0;
+    cpu->ThreadId = 0;
+    cpu->PhysicalCoreIndex = 0;
+    cpu->CoreSiblingMask = 1ULL;
+    cpu->PackageMask = 1ULL;
+    cpu->IsPrimaryThread = 1;
+    cpu->SmtThreadCount = 1;
+    cpu->TopologyFlags = 0;
     cpu->CurrentIrql = PASSIVE_LEVEL;
     cpu->InterruptDepth = 0;
     cpu->ReschedulePending = 0;

code/kernel/ke_debug.c

@@ -13,12 +13,53 @@
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
 #include <kernel/mm_virt.h>
+#include <kernel/io_file.h>
 #include <common/printf.h>
 int _fltused; // MSVC floating-point helper anchor
 
 int gDebugEnabled = 0;
+static uint32_t gDebugFlags = 0;
 void (*PrintChar)(char, uint32_t, uint32_t);
 
+#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
+#define KE_DEBUG_LOG_LINE_SIZE   1024u
+
+static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
+static char*    gDebugLogBuffer;
+static size_t   gDebugLogSize;
+static size_t   gDebugLogWrite;
+static size_t   gDebugLogUsed;
+static volatile long gDebugLogFlushing;
+
+static void KiDebugLogAppendLocked(const char* text, size_t len) {
+    if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
+        return;
+
+    if (len >= gDebugLogSize) {
+        text += len - (gDebugLogSize - 1u);
+        len = gDebugLogSize - 1u;
+    }
+
+    size_t first = gDebugLogSize - gDebugLogWrite;
+    if (first > len) first = len;
+    memcpy(gDebugLogBuffer + gDebugLogWrite, text, first);
+    if (len > first)
+        memcpy(gDebugLogBuffer, text + first, len - first);
+
+    gDebugLogWrite = (gDebugLogWrite + len) % gDebugLogSize;
+    if (gDebugLogUsed + len >= gDebugLogSize)
+        gDebugLogUsed = gDebugLogSize;
+    else
+        gDebugLogUsed += len;
+}
+
+static void KiDebugConsoleWrite(const char* text, size_t len) {
+    if (!PrintChar || !text)
+        return;
+    for (size_t i = 0; i < len; i++)
+        PrintChar(text[i], 0x00FF00, 0x000000);
+}
+
 static volatile long gPanicInProgress = 0;
 static volatile long gPanicStoppedCpuCount = 0;
 
@@ -50,22 +91,98 @@ void KiKernelPutChar(char c) {
     PrintChar(c, 0x00FF00, 0x000000);
 }
 
-void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) {
+void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
-    gDebugEnabled = v;
+    gDebugFlags = flags;
+    gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
     PrintChar = PrintPtr;
     printf_set_putchar(KiKernelPutChar);
+
+    if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
+        gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE);
+        if (gDebugLogBuffer) {
+            gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
+            gDebugLogWrite = 0;
+            gDebugLogUsed = 0;
+            memset(gDebugLogBuffer, 0, gDebugLogSize);
+        }
+    }
+}
+
+uint32_t KeGetDebugFlags(void) {
+    return gDebugFlags;
 }
 
 __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
-    if (!gDebugEnabled) return;
+    if (!gDebugEnabled || !fmt) return;
 
-    static SPINLOCK sl = {0};
+    char line[KE_DEBUG_LOG_LINE_SIZE];
-    uintptr_t flags = KeAcquireSpinlock(&sl);
     va_list va;
     va_start(va, fmt);
-    vprintf(fmt, va);
+    int n = vsnprintf_(line, sizeof(line), fmt, va);
     va_end(va);
-    KeReleaseSpinlock(&sl, flags);
+
+    size_t len = 0;
+    if (n > 0) {
+        len = (size_t)n;
+        if (len >= sizeof(line))
+            len = sizeof(line) - 1u;
+    }
+
+    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
+        KiDebugLogAppendLocked(line, len);
+    if (gDebugFlags & KE_DEBUG_CONSOLE)
+        KiDebugConsoleWrite(line, len);
+    KeReleaseSpinlock(&gDebugPrintLock, flags);
+}
+
+__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
+    if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
+        return STATUS_INVALID_PARAMETER;
+    if (!KeCanWait())
+        return STATUS_INVALID_IRQL;
+
+    if (_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 1, 0) != 0)
+        return STATUS_PENDING;
+
+    size_t used;
+    size_t start;
+    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    used = gDebugLogUsed;
+    start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
+    KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+    if (used == 0) {
+        _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
+        return STATUS_SUCCESS;
+    }
+
+    (void)IoDeleteFile(path);
+
+    FILE_OBJECT* file = (FILE_OBJECT*)0;
+    KSTATUS st = IoCreateFile(path, &file);
+    if (!KSUCCESS(st)) {
+        _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
+        return st;
+    }
+
+    size_t written = 0;
+    size_t totalWritten = 0;
+    size_t first = gDebugLogSize - start;
+    if (first > used) first = used;
+    st = IoWriteFile(file, gDebugLogBuffer + start, first, &written);
+    totalWritten += written;
+
+    if (KSUCCESS(st) && used > first) {
+        written = 0;
+        st = IoWriteFile(file, gDebugLogBuffer, used - first, &written);
+        totalWritten += written;
+    }
+
+    IoCloseFile(file);
+
+    _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
+    return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
 }
 
 // KePanic

code/kernel/ke_irp.c

@@ -1,130 +1,321 @@
-// ke_irp.c  -  I/O Request Packet dispatch and queue helpers
+// ke_irp.c  -  I/O Request Packet dispatch, queue, cancellation, remove-locks
-//
-// This file lives in the kernel layer and is the ONLY place that bridges the
-// IRP abstraction to kernel sync objects. All calls to KeEventCreate,
-// KeSignalEvent, KeWaitForEvent, KeSignalSemaphore, and ObDereferenceObject
-// are confined here; hal_disk.c and io_driver.h remain free of those symbols.
-//
-// Dependency graph (→ = includes / calls):
-//   ke_irp.c  →  ke_irp.h          (IRP type, IRP_QUEUE, forward decls)
-//             →  ke_event.h        (KeEventCreate, KeSignalEvent, KeWaitForEvent)
-//             →  ke_semaphore.h    (KeSignalSemaphore)
-//             →  ob_object.h       (ObDereferenceObject)
-//             →  io_driver.h       (DEVICE_OBJECT / IoCallDriver)
 
+#include <string.h>
+#include <intrin.h>
 #include <kernel/ke_irp.h>
 #include <kernel/ke_event.h>
 #include <kernel/ke_semaphore.h>
 #include <kernel/ke_apc.h>
 #include <kernel/ob_object.h>
+#include <kernel/mm_virt.h>
 #include <ntlite/ddi/io_driver.h>
 
 // -------------------------------------------------------------------------
 // IRP_QUEUE operations
 // -------------------------------------------------------------------------
-// These are defined here (not as static inlines in ke_irp.h) so that
-// ke_irp.h can avoid including ke_semaphore.h, keeping the header safe
-// for inclusion from the HAL layer.
 
-__declspec(dllexport) void IrpQueueInit(IRP_QUEUE *q, struct SEMAPHORE *sem) {
+__declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
     q->Lock      = (SPINLOCK)SPINLOCK_INIT;
     q->Semaphore = sem;
     LIST_INIT(&q->Head);
 }
 
-__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE *q, IRP *irp) {
+__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
     uintptr_t flags = KeAcquireSpinlock(&q->Lock);
     if (irp->Flags & IRP_FLAG_PRIORITY)
         ListInsertHead(&q->Head, &irp->Link);
     else
         ListInsertTail(&q->Head, &irp->Link);
     KeReleaseSpinlock(&q->Lock, flags);
-    // Signal outside the spinlock - safe, and avoids priority-inversion risk.
+    KeSignalSemaphore((SEMAPHORE*)q->Semaphore);
-    KeSignalSemaphore((SEMAPHORE *)q->Semaphore);
 }
 
-__declspec(dllexport) IRP *IrpQueueDequeue(IRP_QUEUE *q) {
+__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
     uintptr_t flags = KeAcquireSpinlock(&q->Lock);
-    LIST_ENTRY *node = ListPopHead(&q->Head);
+    LIST_ENTRY* node = ListPopHead(&q->Head);
     KeReleaseSpinlock(&q->Lock, flags);
-    if (!node) return (IRP *)0;
+    if (!node) return (IRP*)0;
     return CONTAINING_RECORD(node, IRP, Link);
 }
 
-// -------------------------------------------------------------------------
+__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
-// IoCompleteRequest
+                                                    KSTATUS status,
-// -------------------------------------------------------------------------
+                                                    IRP_QUEUE_CANCEL_CALLBACK callback,
+                                                    void* context) {
+    if (!q) return;
+    for (;;) {
+        uintptr_t flags = KeAcquireSpinlock(&q->Lock);
+        LIST_ENTRY* node = ListPopHead(&q->Head);
+        KeReleaseSpinlock(&q->Lock, flags);
+        if (!node) break;
 
-static size_t IoIrpTransferBytes(const IRP *irp, KSTATUS status) {
+        IRP* irp = CONTAINING_RECORD(node, IRP, Link);
-    if (!irp || !KSUCCESS(status)) return 0;
+        irp->Cancel = true;
-    return (size_t)irp->SectorCount * 512u;
+        irp->Flags |= IRP_FLAG_CANCELLED;
+        if (callback)
+            callback(irp, context);
+        IoCompleteRequest(irp, status, 0);
+    }
 }
 
-__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information) {
+__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
+    IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
+}
+
+// -------------------------------------------------------------------------
+// Remove locks
+// -------------------------------------------------------------------------
+
+__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
+    if (!lock) return STATUS_INVALID_PARAMETER;
+    memset(lock, 0, sizeof(*lock));
+    lock->Lock = (SPINLOCK)SPINLOCK_INIT;
+    lock->IoCount = 1; // sentinel reference released by IoReleaseRemoveLockAndWait
+    KSTATUS st = KeEventCreate(EventManualReset, 0, &lock->RemovedEvent);
+    if (!KSUCCESS(st)) return st;
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
+    if (!lock) return;
+    if (lock->RemovedEvent) {
+        ObDereferenceObject(&lock->RemovedEvent->Header);
+        lock->RemovedEvent = (EVENT*)0;
+    }
+}
+
+__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
+    (void)irp;
+    if (!lock) return STATUS_INVALID_PARAMETER;
+    uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
+    if (lock->Removing) {
+        KeReleaseSpinlock(&lock->Lock, flags);
+        return STATUS_OBJECT_DEAD;
+    }
+    _InterlockedIncrement(&lock->IoCount);
+    KeReleaseSpinlock(&lock->Lock, flags);
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
+    (void)irp;
+    if (!lock) return;
+    long left = _InterlockedDecrement(&lock->IoCount);
+    if (left == 0 && lock->Removing && lock->RemovedEvent)
+        KeSignalEvent(lock->RemovedEvent);
+}
+
+__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp) {
+    (void)removeIrp;
+    if (!lock) return;
+
+    uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
+    lock->Removing = true;
+    long left = _InterlockedDecrement(&lock->IoCount); // release sentinel
+    bool done = (left == 0);
+    KeReleaseSpinlock(&lock->Lock, flags);
+
+    if (done) {
+        if (lock->RemovedEvent) KeSignalEvent(lock->RemovedEvent);
+        return;
+    }
+
+    if (lock->RemovedEvent)
+        KeWaitForEvent(lock->RemovedEvent, 0);
+}
+
+// -------------------------------------------------------------------------
+// IRP helpers
+// -------------------------------------------------------------------------
+
+static size_t IoIrpSize(uint8_t stackCount) {
+    if (stackCount == 0) stackCount = 1;
+    return sizeof(IRP) + ((size_t)stackCount - 1u) * sizeof(IO_STACK_LOCATION);
+}
+
+__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
+    if (!outIrp) return STATUS_INVALID_PARAMETER;
+    if (stackCount == 0) stackCount = 1;
+    IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount));
+    if (!irp) return STATUS_NO_MEMORY;
+    IoInitializeIrp(irp, stackCount);
+    *outIrp = irp;
+    return STATUS_SUCCESS;
+}
+
+__declspec(dllexport) void IoFreeIrp(IRP* irp) {
+    if (irp) MmFreePool(irp);
+}
+
+__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
+    if (!irp) return;
+    if (stackCount == 0) stackCount = 1;
+    memset(irp, 0, IoIrpSize(stackCount));
+    LIST_INIT(&irp->Link);
+    irp->Status = STATUS_FAILURE;
+    irp->Information = 0;
+    irp->CancelLock = (SPINLOCK)SPINLOCK_INIT;
+    irp->StackCount = stackCount;
+    irp->CurrentLocation = 0;
+}
+
+__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp) {
+    if (!irp || irp->CurrentLocation >= irp->StackCount) return (IO_STACK_LOCATION*)0;
+    return &irp->Stack[irp->CurrentLocation];
+}
+
+__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) {
+    if (!irp || irp->CurrentLocation + 1u >= irp->StackCount) return (IO_STACK_LOCATION*)0;
+    return &irp->Stack[irp->CurrentLocation + 1u];
+}
+
+__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) {
+    if (!irp) return;
+    if (irp->CurrentLocation + 1u < irp->StackCount)
+        irp->CurrentLocation++;
+}
+
+__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
+    IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
+    IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
+    if (!cur || !next) return;
+    memcpy(next, cur, sizeof(*next));
+    irp->CurrentLocation++;
+}
+
+__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx) {
+    if (!irp) return;
+    irp->CompletionFn = fn;
+    irp->CompletionCtx = ctx;
+}
+
+__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine) {
+    if (!irp) return (IRP_CANCEL_ROUTINE)0;
+    uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
+    IRP_CANCEL_ROUTINE old = irp->CancelRoutine;
+    irp->CancelRoutine = routine;
+    KeReleaseSpinlock(&irp->CancelLock, flags);
+    return old;
+}
+
+__declspec(dllexport) void IoCancelIrp(IRP* irp) {
     if (!irp) return;
 
+    uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
+    irp->Cancel = true;
+    irp->Flags |= IRP_FLAG_CANCELLED;
+    IRP_CANCEL_ROUTINE routine = irp->CancelRoutine;
+    irp->CancelRoutine = (IRP_CANCEL_ROUTINE)0;
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
+    KeReleaseSpinlock(&irp->CancelLock, flags);
+
+    if (routine)
+        routine(dev, irp);
+}
+
+__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp,
+                                               IO_MAJOR_FUNCTION major,
+                                               uint64_t byteOffset,
+                                               uint32_t length,
+                                               void* buffer) {
+    if (!irp) return;
+    IoInitializeIrp(irp, 1);
+    irp->UserBuffer = buffer;
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios) return;
+    ios->MajorFunction = major;
+    ios->MinorFunction = 0;
+    ios->Parameters.ReadWrite.ByteOffset = byteOffset;
+    ios->Parameters.ReadWrite.Length = length;
+}
+
+__declspec(dllexport) void IoBuildPnpIrp(IRP* irp,
+                                         IO_PNP_MINOR_FUNCTION minor) {
+    if (!irp) return;
+    uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
+    IoInitializeIrp(irp, stackCount);
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios) return;
+    ios->MajorFunction = IRP_MJ_PNP;
+    ios->MinorFunction = (uint8_t)minor;
+}
+
+__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
+                                                     IO_MAJOR_FUNCTION major,
+                                                     uint32_t ioControlCode,
+                                                     void* inputBuffer,
+                                                     uint32_t inputLength,
+                                                     void* outputBuffer,
+                                                     uint32_t outputLength) {
+    if (!irp) return;
+    uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
+    IoInitializeIrp(irp, stackCount);
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
+    if (!ios) return;
+    ios->MajorFunction = major;
+    ios->MinorFunction = 0;
+    ios->Parameters.DeviceIoControl.IoControlCode = ioControlCode;
+    ios->Parameters.DeviceIoControl.InputBuffer = inputBuffer;
+    ios->Parameters.DeviceIoControl.InputLength = inputLength;
+    ios->Parameters.DeviceIoControl.OutputBuffer = outputBuffer;
+    ios->Parameters.DeviceIoControl.OutputLength = outputLength;
+}
+
+// -------------------------------------------------------------------------
+// Completion/submission
+// -------------------------------------------------------------------------
+
+__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information) {
+    if (!irp) return STATUS_INVALID_PARAMETER;
+    if (irp->Flags & IRP_FLAG_COMPLETED)
+        return STATUS_ALREADY_EXISTS;
+
+    irp->Flags |= IRP_FLAG_COMPLETED;
     irp->Status = status;
     irp->Information = information;
 
-    // Queue the APC before waking event waiters so a thread that waits on the
+    IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
-    // completion event will see and deliver the APC on its wait-return path.
+    DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
+    if (dev) {
+        _InterlockedDecrement(&dev->OutstandingIrps);
+        IoDereferenceDeviceObject(dev);
+        ios->DeviceObject = (DEVICE_OBJECT*)0;
+    }
+
     if (irp->CompletionApc)
-        KeInsertQueueApc((KAPC *)irp->CompletionApc, irp, (void *)(uintptr_t)status);
+        KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status);
     if (irp->CompletionEvent)
-        KeSignalEvent((EVENT *)irp->CompletionEvent);
+        KeSignalEvent((EVENT*)irp->CompletionEvent);
     if (irp->CompletionFn)
         irp->CompletionFn(irp, irp->CompletionCtx);
+
+    return STATUS_SUCCESS;
 }
 
-// -------------------------------------------------------------------------
+__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
-// IoSubmitIrp
+    if (!device || !irp) return STATUS_INVALID_PARAMETER;
-// -------------------------------------------------------------------------
-// Dispatches through the registered DEVICE_OBJECT.
-// Accepted IRPs are always completed by the target driver via IoCompleteRequest().
-
-__declspec(dllexport) KSTATUS IoSubmitIrp(IRP *irp) {
-    if (!irp) return STATUS_INVALID_PARAMETER;
-
     irp->Status = STATUS_FAILURE;
     irp->Information = 0;
-
+    irp->Flags &= ~IRP_FLAG_COMPLETED;
-    DEVICE_OBJECT *device = IoGetDiskDevice(irp->DriveIndex);
+    return IoCallDriver(device, irp);
-    if (!device) return STATUS_NOT_FOUND;
-
-    KSTATUS st = IoCallDriver(device, irp);
-
-    // Definitive driver contract:
-    //   STATUS_PENDING means the IRP was accepted and will be completed via
-    //   IoCompleteRequest(), possibly inline before IoCallDriver returns.
-    //   Any other status means the IRP was rejected and no completion occurs.
-    return st;
 }
 
-// -------------------------------------------------------------------------
+__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
-// IoSubmitIrpSync
+    if (!device || !irp) return STATUS_INVALID_PARAMETER;
-// -------------------------------------------------------------------------
-// Creates a stack-allocated auto-reset event, attaches it to the IRP,
-// submits, and blocks. Callers that already provided a CompletionEvent
-// should use IoSubmitIrp() directly and wait on their own event instead.
 
-__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp) {
+    EVENT* ev = (EVENT*)0;
-    if (!irp) return STATUS_INVALID_PARAMETER;
-
-    EVENT *ev = (EVENT *)0;
     KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev);
     if (!KSUCCESS(st)) return st;
 
-    // Attach our event; preserve any APC/callback the caller may have set.
     irp->CompletionEvent = ev;
 
-    st = IoSubmitIrp(irp);
+    st = IoSubmitIrp(device, irp);
     if (st == STATUS_PENDING) {
         KeWaitForEvent(ev, 0);
         st = irp->Status;
     }
 
-    // Release our reference to the temporary event object.
     ObDereferenceObject(&ev->Header);
-
     return st;
 }

code/kernel/ke_sched.c

@@ -36,6 +36,44 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
     return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
 }
 
+static int KiCpuIsIdleLocked(const KCPU* cpu) {
+    if (!cpu || !cpu->Online)
+        return 0;
+    THREAD* cur = cpu->CurrentThread;
+    return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
+}
+
+static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
+    if (!thread || thread->IsIdleThread)
+        return 0;
+
+    for (uint32_t i = 0; i < gCpuCount; i++) {
+        KCPU* cpu = &gCpus[i];
+        if (!cpu->Present || !cpu->Started || !cpu->Online)
+            continue;
+        if (!cpu->IsPrimaryThread)
+            continue;
+        if (!KiCpuIsIdleLocked(cpu))
+            continue;
+        if (KiThreadCanRunOnCpuLocked(thread, cpu))
+            return 1;
+    }
+    return 0;
+}
+
+static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
+    if (!cpu || !thread || thread->IsIdleThread)
+        return 0;
+    if (cpu->IsPrimaryThread || cpu->SmtThreadCount <= 1)
+        return 0;
+
+    // Do not fill a secondary SMT sibling while any primary hardware thread
+    // that can run this thread is idle. KiPokeRunnableCpusLocked targets those
+    // primary CPUs first, so this CPU can safely remain idle and wait for more
+    // work or for physical cores to fill.
+    return KiAnyIdlePrimaryForThreadLocked(thread);
+}
+
 static void KiRequestRescheduleCpuLocked(KCPU* target) {
     if (!target || !target->Online)
         return;
@@ -52,6 +90,9 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
     if (!thread || thread->IsIdleThread)
         return;
 
+    int haveIdlePrimary = KiAnyIdlePrimaryForThreadLocked(thread);
+    int requestedIdle = 0;
+
     for (uint32_t i = 0; i < gCpuCount; i++) {
         KCPU* cpu = &gCpus[i];
         if (!cpu->Present || !cpu->Started || !cpu->Online)
@@ -61,12 +102,35 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
 
         THREAD* cur = cpu->CurrentThread;
         if (!cur || cur == cpu->IdleThread) {
+            if (haveIdlePrimary && !cpu->IsPrimaryThread)
+                continue;
+            if (cpu->IsPrimaryThread)
+                gSchedStats.SmtPrimaryPokes++;
             KiRequestRescheduleCpuLocked(cpu);
+            requestedIdle = 1;
         } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) {
             gSchedStats.PriorityPreemptions++;
             KiRequestRescheduleCpuLocked(cpu);
         }
     }
+
+    // If no preferred idle primary was actually targetable, allow an idle SMT
+    // sibling to take the work. This covers explicit affinity to a secondary
+    // logical CPU and partially-online systems.
+    if (!requestedIdle && haveIdlePrimary == 0) {
+        for (uint32_t i = 0; i < gCpuCount; i++) {
+            KCPU* cpu = &gCpus[i];
+            if (!cpu->Present || !cpu->Started || !cpu->Online)
+                continue;
+            if (!KiThreadCanRunOnCpuLocked(thread, cpu))
+                continue;
+            THREAD* cur = cpu->CurrentThread;
+            if (!cur || cur == cpu->IdleThread) {
+                KiRequestRescheduleCpuLocked(cpu);
+                break;
+            }
+        }
+    }
 }
 
 static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
@@ -369,8 +433,11 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
         LIST_ENTRY* head = &gRunQueue[p];
         for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
             THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
-            if (KiThreadCanRunOnCpuLocked(thread, cpu))
+            if (!KiThreadCanRunOnCpuLocked(thread, cpu))
-                return 1;
+                continue;
+            if (KiShouldDeferToIdlePrimaryLocked(cpu, thread))
+                continue;
+            return 1;
         }
     }
     return 0;
@@ -385,6 +452,10 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
             THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
             if (!KiThreadCanRunOnCpuLocked(thread, cpu))
                 continue;
+            if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) {
+                gSchedStats.SmtSiblingDeferrals++;
+                continue;
+            }
 
             ListRemove(node);
             thread->Queued = 0;

code/kernel/ke_selftest.c

@@ -352,6 +352,7 @@ static void KiRunKernelApcSelfTest(void) {
 
 typedef struct KI_IRP_APC_SELFTEST_STATE {
     EVENT* CompletionEvent;
+    DEVICE_OBJECT* DiskDevice;
     IRP Irp;
     KAPC Apc;
     uint8_t* Buffer;
@@ -430,23 +431,18 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
                     KiIrpApcSelfTestNormalRoutine,
                     (void*)&st);
 
-    LIST_INIT(&st.Irp.Link);
+    st.DiskDevice = IoReferenceDiskDevice(0);
-    st.Irp.DriveIndex = 0;
+    if (!st.DiskDevice)
-    st.Irp.Flags = 0;
+        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: drive 0 disappeared before submit\n");
-    st.Irp.Lba = 0;
+
-    st.Irp.SectorCount = 1;
+    IoBuildReadWriteIrp(&st.Irp, IRP_MJ_READ, 0, 512, st.Buffer);
-    st.Irp.Buffer = st.Buffer;
-    st.Irp.Status = STATUS_FAILURE;
-    st.Irp.Information = 0;
     st.Irp.CompletionEvent = st.CompletionEvent;
     st.Irp.CompletionApc = &st.Apc;
     st.Irp.CompletionFn = (IRP_COMPLETION_FN)0;
     st.Irp.CompletionCtx = (void*)0;
-    st.Irp.HwSlot = 0;
-    st.Irp.DriverCtx = (void*)0;
 
     DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n");
-    s = IoSubmitIrp(&st.Irp);
+    s = IoSubmitIrp(st.DiskDevice, &st.Irp);
     if (s != STATUS_PENDING)
         KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: IoSubmitIrp returned %u\n", s);
 
@@ -472,6 +468,7 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
              st.Buffer[510], st.Buffer[511]);
 
     MmFreePool(st.Buffer);
+    if (st.DiskDevice) IoDereferenceDeviceObject(st.DiskDevice);
     ObDereferenceObject(&st.CompletionEvent->Header);
 
     DbgPrint("IRP/APC selftest: PASS\n");

code/kernel/ke_smp.c

@@ -321,6 +321,15 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
     cpu->Parked = 0;
     cpu->Online = 0;
     cpu->IsBsp = info->IsBsp ? 1 : 0;
+    cpu->PackageId = 0;
+    cpu->CoreId = cpuIndex;
+    cpu->ThreadId = 0;
+    cpu->PhysicalCoreIndex = cpuIndex;
+    cpu->CoreSiblingMask = (cpuIndex < 64) ? (1ULL << cpuIndex) : 0ULL;
+    cpu->PackageMask = cpu->CoreSiblingMask;
+    cpu->IsPrimaryThread = 1;
+    cpu->SmtThreadCount = 1;
+    cpu->TopologyFlags = 0;
     cpu->CurrentIrql = PASSIVE_LEVEL;
     cpu->InterruptDepth = 0;
     cpu->ReschedulePending = 0;
@@ -328,6 +337,223 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
     cpu->SchedulerActive = 0;
 }
 
+
+static uint32_t KiCeilLog2U32(uint32_t value) {
+    if (value <= 1u)
+        return 0u;
+    uint32_t shift = 0;
+    uint32_t n = value - 1u;
+    while (n) {
+        shift++;
+        n >>= 1;
+    }
+    return shift;
+}
+
+static uint32_t KiMaskWidth(uint32_t shift) {
+    if (shift >= 32u)
+        return 0xFFFFFFFFu;
+    if (shift == 0u)
+        return 0u;
+    return (1u << shift) - 1u;
+}
+
+typedef struct KI_CPUID_TOPOLOGY_PARAMS {
+    uint32_t SmtShift;
+    uint32_t CoreShift;
+    uint32_t ThreadsPerCore;
+    uint32_t LogicalPerPackage;
+    const char* Source;
+} KI_CPUID_TOPOLOGY_PARAMS;
+
+static void KiReadCpuidTopologyLeaf(uint32_t leaf, KI_CPUID_TOPOLOGY_PARAMS* out) {
+    if (!out)
+        return;
+
+    for (uint32_t sub = 0; sub < 32u; sub++) {
+        int r[4] = {0, 0, 0, 0};
+        __cpuidex(r, (int)leaf, (int)sub);
+
+        uint32_t levelShift = (uint32_t)r[0] & 0x1Fu;
+        uint32_t logicals   = (uint32_t)r[1] & 0xFFFFu;
+        uint32_t levelType  = ((uint32_t)r[2] >> 8) & 0xFFu;
+
+        if (levelType == 0u || logicals == 0u)
+            break;
+
+        if (levelType == 1u) { // SMT level
+            out->SmtShift = levelShift;
+            out->ThreadsPerCore = logicals ? logicals : 1u;
+        } else if (levelType == 2u) { // Core/package level
+            out->CoreShift = levelShift;
+            out->LogicalPerPackage = logicals ? logicals : 1u;
+            break;
+        }
+    }
+}
+
+static KI_CPUID_TOPOLOGY_PARAMS KiDetectCpuidTopologyParams(void) {
+    KI_CPUID_TOPOLOGY_PARAMS topo;
+    memset(&topo, 0, sizeof(topo));
+    topo.ThreadsPerCore = 1;
+    topo.LogicalPerPackage = 1;
+    topo.Source = "flat";
+
+    int maxRegs[4] = {0, 0, 0, 0};
+    __cpuid(maxRegs, 0);
+    uint32_t maxLeaf = (uint32_t)maxRegs[0];
+
+    if (maxLeaf >= 0x1Fu) {
+        KiReadCpuidTopologyLeaf(0x1Fu, &topo);
+        if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) {
+            topo.Source = "cpuid.1f";
+            return topo;
+        }
+        memset(&topo, 0, sizeof(topo));
+        topo.ThreadsPerCore = 1;
+        topo.LogicalPerPackage = 1;
+        topo.Source = "flat";
+    }
+
+    if (maxLeaf >= 0x0Bu) {
+        KiReadCpuidTopologyLeaf(0x0Bu, &topo);
+        if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) {
+            topo.Source = "cpuid.0b";
+            return topo;
+        }
+        memset(&topo, 0, sizeof(topo));
+        topo.ThreadsPerCore = 1;
+        topo.LogicalPerPackage = 1;
+        topo.Source = "flat";
+    }
+
+    int r1[4] = {0, 0, 0, 0};
+    __cpuid(r1, 1);
+    uint32_t logicalPerPackage = ((uint32_t)r1[1] >> 16) & 0xFFu;
+    if (logicalPerPackage == 0u)
+        logicalPerPackage = 1u;
+
+    uint32_t coresPerPackage = 1u;
+    if (maxLeaf >= 4u) {
+        int r4[4] = {0, 0, 0, 0};
+        __cpuidex(r4, 4, 0);
+        coresPerPackage = (((uint32_t)r4[0] >> 26) & 0x3Fu) + 1u;
+        if (coresPerPackage == 0u)
+            coresPerPackage = 1u;
+    }
+
+    uint32_t threadsPerCore = logicalPerPackage / coresPerPackage;
+    if (threadsPerCore == 0u)
+        threadsPerCore = 1u;
+
+    topo.ThreadsPerCore = threadsPerCore;
+    topo.LogicalPerPackage = logicalPerPackage;
+    topo.SmtShift = KiCeilLog2U32(threadsPerCore);
+    topo.CoreShift = KiCeilLog2U32(logicalPerPackage);
+    if (topo.CoreShift < topo.SmtShift)
+        topo.CoreShift = topo.SmtShift;
+    topo.Source = "cpuid.1/4";
+    return topo;
+}
+
+void KeBuildProcessorTopology(void) {
+    KI_CPUID_TOPOLOGY_PARAMS topo = KiDetectCpuidTopologyParams();
+
+    uint32_t smtMask = KiMaskWidth(topo.SmtShift);
+    uint32_t coreBits = (topo.CoreShift > topo.SmtShift) ?
+                        (topo.CoreShift - topo.SmtShift) : 0u;
+    uint32_t coreMask = KiMaskWidth(coreBits);
+
+    for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
+        KCPU* cpu = &gCpus[i];
+        if (!cpu->Present)
+            continue;
+
+        uint32_t apic = cpu->LapicId;
+        cpu->ThreadId = smtMask ? (apic & smtMask) : 0u;
+        cpu->CoreId = coreBits ? ((apic >> topo.SmtShift) & coreMask) :
+                                  (apic >> topo.SmtShift);
+        cpu->PackageId = (topo.CoreShift < 32u) ? (apic >> topo.CoreShift) : 0u;
+        cpu->PhysicalCoreIndex = KE_INVALID_CPU;
+        cpu->CoreSiblingMask = 0;
+        cpu->PackageMask = 0;
+        cpu->IsPrimaryThread = 0;
+        cpu->SmtThreadCount = 1;
+        cpu->TopologyFlags = (topo.ThreadsPerCore > 1u) ? 1u : 0u;
+    }
+
+    uint32_t nextCoreIndex = 0;
+    for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
+        KCPU* cpu = &gCpus[i];
+        if (!cpu->Present)
+            continue;
+        if (cpu->PhysicalCoreIndex != KE_INVALID_CPU)
+            continue;
+
+        uint32_t coreIndex = nextCoreIndex++;
+        uint64_t coreMaskBits = 0;
+        uint64_t packageMaskBits = 0;
+        uint32_t primaryCpuIndex = i;
+        uint32_t siblingCount = 0;
+
+        for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
+            KCPU* peer = &gCpus[j];
+            if (!peer->Present)
+                continue;
+            if (peer->PackageId == cpu->PackageId) {
+                if (j < 64)
+                    packageMaskBits |= (1ULL << j);
+            }
+            if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
+                peer->PhysicalCoreIndex = coreIndex;
+                if (j < 64)
+                    coreMaskBits |= (1ULL << j);
+                if (j < primaryCpuIndex)
+                    primaryCpuIndex = j;
+                siblingCount++;
+            }
+        }
+
+        if (siblingCount == 0)
+            siblingCount = 1;
+
+        for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
+            KCPU* peer = &gCpus[j];
+            if (!peer->Present)
+                continue;
+            if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
+                peer->CoreSiblingMask = coreMaskBits;
+                peer->PackageMask = packageMaskBits;
+                peer->SmtThreadCount = (uint8_t)((siblingCount > 255u) ? 255u : siblingCount);
+                peer->IsPrimaryThread = (j == primaryCpuIndex) ? 1u : 0u;
+            }
+        }
+    }
+
+    DbgPrint("SMP: SMT topology source=%s smtShift=%u coreShift=%u threads/core=%u logical/package=%u physicalCores=%u\n",
+             topo.Source ? topo.Source : "?",
+             topo.SmtShift,
+             topo.CoreShift,
+             topo.ThreadsPerCore,
+             topo.LogicalPerPackage,
+             nextCoreIndex);
+
+    for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
+        KCPU* cpu = &gCpus[i];
+        if (!cpu->Present)
+            continue;
+        DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
+                 i,
+                 cpu->LapicId,
+                 cpu->PackageId,
+                 cpu->CoreId,
+                 cpu->ThreadId,
+                 cpu->IsPrimaryThread,
+                 (unsigned long long)cpu->CoreSiblingMask,
+                 (unsigned long long)cpu->PackageMask);
+    }
+}
+
 static KSTATUS KiAllocateApStack(KCPU* cpu) {
     // AP kernel stacks do not need physically contiguous memory.  Requiring
     // MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
@@ -510,6 +736,7 @@ KSTATUS KeStartApplicationProcessors(void) {
     if (claimed == 0 && gCpuCount > 1)
         DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n");
 
+    KeBuildProcessorTopology();
     DbgPrint("SMP: kernel claimed %u APs\n", claimed);
     return STATUS_SUCCESS;
 }

code/kernel/kmain.c

@@ -341,7 +341,7 @@ void SystemInitThread(void* arg) {
     KiResolveSystemImports(args->bootInfo);
     KiReclaimBootloaderMemory(args->bootInfo);
 
-    if (args->bootInfo->Debug & 2)
+    if (args->bootInfo->Debug & KE_DEBUG_SELFTEST)
         KiRunKernelSelfTests(args->hal);
 
     // Launch User-Mode init.exe and restart it until it exits cleanly.
@@ -363,6 +363,11 @@ void SystemInitThread(void* arg) {
     }
 
     DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n");
+    if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
+        KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log"));
+        if (!KSUCCESS(logStatus))
+            DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus);
+    }
     gKmainHal->Power.Shutdown();
     MmFreePool(args);
 }
@@ -416,7 +421,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
     const HAL_API* hal = HalGetApi();
     gKmainHal = hal;
 
-    KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1);
+    KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug);
 
     // CPU startup intentionally lives after HAL initialization but before IRQs
     // and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI;

code/public/ntlite/ddi/io_driver.h

@@ -1,39 +1,35 @@
 #pragma once
-// io_driver.h - definitive kernel driver/device I/O model
+// io_driver.h - kernel driver/device I/O model
 //
-// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through
+// Device lifetime is owned by the I/O manager. Bus drivers create PDOs,
-// IoCallDriver()/IoSubmitIrp(); drivers complete accepted IRPs by calling
+// function/filter drivers attach FDOs, and PnP/remove IRPs tear stacks down.
-// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table.
+// Disk indexes are only a compatibility namespace above DEVICE_OBJECTs.
 
 #include <stdint.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <ntlite/status.h>
+#include <ntlite/string_view.h>
+#include <kernel/ob_object.h>
+#include <kernel/ke_list.h>
+#include <kernel/ke_spinlock.h>
+#include <kernel/ke_irp.h>
 
 typedef struct HAL_API HAL_API;
 typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API* hal);
 typedef void    (*DRIVER_UNLOAD_FN)(void);
 
-struct IRP;
 typedef struct DRIVER_OBJECT DRIVER_OBJECT;
 typedef struct DEVICE_OBJECT DEVICE_OBJECT;
+typedef struct IRP IRP;
 
-typedef enum IO_MAJOR_FUNCTION {
+typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, IRP* irp);
-    IRP_MJ_READ  = 0,
+typedef KSTATUS (*PDRIVER_ADD_DEVICE)(DRIVER_OBJECT* driver,
-    IRP_MJ_WRITE = 1,
+                                      DEVICE_OBJECT* physicalDeviceObject);
-    IRP_MJ_DEVICE_CONTROL = 2,
-    IRP_MJ_MAXIMUM_FUNCTION = 3,
-} IO_MAJOR_FUNCTION;
-
-typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, struct IRP* irp);
-
-struct DRIVER_OBJECT {
-    const char*       Name;
-    PDRIVER_DISPATCH  MajorFunction[IRP_MJ_MAXIMUM_FUNCTION];
-    DRIVER_UNLOAD_FN  DriverUnload;
-};
-
 
+// -------------------------------------------------------------------------
+// Bus / identity
+// -------------------------------------------------------------------------
 
 typedef enum IO_BUS_TYPE {
     IoBusUnknown = 0,
@@ -41,6 +37,139 @@ typedef enum IO_BUS_TYPE {
     IoBusUsb     = 2,
 } IO_BUS_TYPE;
 
+typedef struct IO_DEVICE_ID {
+    IO_BUS_TYPE     BusType;
+    SYS_STRING_VIEW Id;
+} IO_DEVICE_ID;
+
+typedef struct IO_DRIVER_ID_TABLE {
+    const IO_DEVICE_ID* Ids;
+    size_t              Count;
+} IO_DRIVER_ID_TABLE;
+
+#define IO_MAX_DEVICE_IDS 8u
+#define IO_MAX_DEVICE_ID_CHARS 96u
+#define IO_MAX_DEVICE_NAME_CHARS 64u
+
+// -------------------------------------------------------------------------
+// Driver/device objects
+// -------------------------------------------------------------------------
+
+typedef enum DEVICE_OBJECT_TYPE {
+    DeviceObjectPhysical = 1,
+    DeviceObjectFunction = 2,
+    DeviceObjectFilter   = 3,
+} DEVICE_OBJECT_TYPE;
+
+typedef enum DEVICE_PNP_STATE {
+    DevicePnpNotStarted = 0,
+    DevicePnpStarted,
+    DevicePnpStopPending,
+    DevicePnpStopped,
+    DevicePnpRemovePending,
+    DevicePnpSurpriseRemoved,
+    DevicePnpDeleted,
+} DEVICE_PNP_STATE;
+
+struct DRIVER_OBJECT {
+    OBJECT              Header;
+    SYS_STRING_VIEW     Name;
+    PDRIVER_DISPATCH    MajorFunction[IRP_MJ_MAXIMUM_FUNCTION];
+    PDRIVER_ADD_DEVICE  DriverAddDevice;
+    DRIVER_UNLOAD_FN    DriverUnload;
+    const IO_DRIVER_ID_TABLE* IdTable;
+};
+
+struct DEVICE_OBJECT {
+    OBJECT              Header;
+
+    DRIVER_OBJECT*      DriverObject;
+    DEVICE_OBJECT_TYPE  Type;
+    DEVICE_PNP_STATE    PnpState;
+
+    DEVICE_OBJECT*      Parent;
+    DEVICE_OBJECT*      FirstChild;
+    DEVICE_OBJECT*      NextSibling;
+
+    DEVICE_OBJECT*      AttachedDevice; // device directly above this object
+    DEVICE_OBJECT*      LowerDevice;    // device directly below this FDO/filter
+
+    void*               DeviceExtension;
+
+    uint32_t            Flags;
+    uint32_t            Characteristics;
+    uint32_t            AlignmentRequirement;
+
+    SPINLOCK            Lock;
+    volatile long       OutstandingIrps;
+    bool                DeletePending;
+
+    SYS_STRING_VIEW     Name;
+    char                NameStorage[IO_MAX_DEVICE_NAME_CHARS];
+
+    union {
+        struct {
+            IO_BUS_TYPE     BusType;
+            void*           BusContext;
+            void*           DeviceContext;
+            IO_DEVICE_ID    Ids[IO_MAX_DEVICE_IDS];
+            char            IdStorage[IO_MAX_DEVICE_IDS][IO_MAX_DEVICE_ID_CHARS];
+            uint8_t         IdCount;
+        } Pdo;
+
+        struct {
+            uint8_t         DiskIndex;
+            uint64_t        SectorCount;
+            uint32_t        SectorSize;
+        } Disk;
+    } u;
+};
+
+#define DEVICE_OBJECT_DISK       0x00000001u
+#define DEVICE_OBJECT_REMOVABLE  0x00000002u
+
+// -------------------------------------------------------------------------
+// Object/device creation and stack management
+// -------------------------------------------------------------------------
+
+__declspec(dllexport) void    IoInitializeDriverObject(DRIVER_OBJECT* driver,
+                                                       SYS_STRING_VIEW name);
+__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver);
+
+__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
+                                             size_t extensionSize,
+                                             DEVICE_OBJECT** outDevice);
+
+__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
+                                                     DEVICE_OBJECT* parent,
+                                                     const IO_DEVICE_ID* ids,
+                                                     size_t idCount,
+                                                     size_t extensionSize,
+                                                     DEVICE_OBJECT** outPdo);
+__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo);
+
+__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
+                                                 SYS_STRING_VIEW name);
+__declspec(dllexport) void    IoReferenceDeviceObject(DEVICE_OBJECT* device);
+__declspec(dllexport) void    IoDereferenceDeviceObject(DEVICE_OBJECT* device);
+__declspec(dllexport) void    IoDeleteDevice(DEVICE_OBJECT* device);
+
+__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
+                                                                 DEVICE_OBJECT* target);
+__declspec(dllexport) void    IoDetachDevice(DEVICE_OBJECT* source);
+__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device);
+
+__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp);
+__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp);
+__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device);
+__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
+                                                IO_PNP_MINOR_FUNCTION minor);
+
+// -------------------------------------------------------------------------
+// Legacy USB publish/matcher path. This remains only until USB is migrated to
+// PDO/interface-PDO enumeration in the later USB stages.
+// -------------------------------------------------------------------------
+
 #define IO_PUBLISHED_USB_MAX_ENDPOINTS 8u
 
 typedef struct IO_PUBLISHED_USB_ENDPOINT {
@@ -102,41 +231,42 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
 __declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device);
 __declspec(dllexport) uint32_t IoPublishedDeviceCount(void);
 
-struct DEVICE_OBJECT {
+// -------------------------------------------------------------------------
-    DRIVER_OBJECT* DriverObject;
+// Disk namespace above storage DEVICE_OBJECTs
-    void*          DeviceExtension;
+// -------------------------------------------------------------------------
-    uint32_t       Flags;
-    uint32_t       AlignmentRequirement;
-    uint64_t       SectorCount;
-    uint8_t        DiskIndex;
-};
 
-#define DEVICE_OBJECT_DISK       0x00000001u
+typedef enum IO_DEVICE_CHANGE {
-#define DEVICE_OBJECT_REMOVABLE  0x00000002u
+    IoDeviceArrived = 0,
+    IoDeviceQueryRemove,
+    IoDeviceSurpriseRemoved,
+    IoDeviceRemoved,
+} IO_DEVICE_CHANGE;
 
-__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
+typedef void (*IO_DISK_CHANGE_CALLBACK)(uint8_t driveIndex,
-                                             size_t extensionSize,
+                                        DEVICE_OBJECT* device,
-                                             DEVICE_OBJECT** outDevice);
+                                        IO_DEVICE_CHANGE change,
-__declspec(dllexport) void    IoDeleteDevice(DEVICE_OBJECT* device);
+                                        void* context);
-
-__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
-                                                   DEVICE_OBJECT* device);
-__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex);
-__declspec(dllexport) bool    IoDiskDevicePresent(uint8_t driveIndex);
-__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex);
 
 typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex,
                                           DEVICE_OBJECT* device,
                                           void* context);
 
+__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
+                                                   DEVICE_OBJECT* device);
+__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex);
+__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex);
+__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex);
+__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex);
+__declspec(dllexport) bool    IoDiskDevicePresent(uint8_t driveIndex);
+__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex);
+
 __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback,
                                                             void* context);
+__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback,
+                                                           void* context);
 __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback,
                                                      void* context);
 
-__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device,
-                                           struct IRP* irp);
-
 __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
                                                 uint64_t lba,
                                                 uint32_t count,

code/public/ntlite/syscall.h

@@ -106,7 +106,6 @@ typedef struct SYS_USER_APC_FRAME {
 
 void SysUserApcDispatcher(void);
 
-
 // ---------------------------------------------------------------------------
 // Trap-frame query ABI
 // ---------------------------------------------------------------------------

readme.md

@@ -10,7 +10,7 @@ NT-Lite is a 64-bit operating system kernel written from scratch. It has a UEFI
 
 It boots into a shell, launches user processes, survives application crashes, and can run things it probably should not be able to run yet.
 
-The codebase is still small enough to understand, but large enough to hurt your feelings. The entire project is about 40k lines.
+The codebase is still small enough to understand, but large enough to hurt your feelings.
 
 ## features
 

runa.bat

@@ -1 +1,18 @@
-@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=vpc,file=disk.vhd,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -serial stdio -M q35 -m 256M -smp cpus=4,cores=4,threads=1,sockets=1 -netdev user,id=net0,hostfwd=tcp::8080-:8080 -device rtl8139,netdev=net0 -device qemu-xhci,id=usb -device usb-kbd,bus=usb.0 -device usb-mouse,bus=usb.0 -drive if=none,id=usbflash,format=vpc,file=usb.vhd -device usb-storage,drive=usbflash,bus=usb.0
+@start qemu-system-x86_64 ^
+-drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd ^
+-drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd ^
+-device ahci,id=ahci ^
+-drive format=vpc,file=disk.vhd,if=none,id=disk0 ^
+-device ide-hd,bus=ahci.0,drive=disk0 ^
+-serial stdio ^
+-M q35 ^
+-m 256M ^
+-smp cpus=4,cores=4,threads=1,sockets=1 ^
+-netdev user,id=net0,hostfwd=tcp::8080-:8080 ^
+-device rtl8139,netdev=net0 ^
+-device qemu-xhci,id=usb ^
+-device usb-kbd,bus=usb.0 ^
+-device usb-mouse,bus=usb.0 ^
+-device usb-hub,id=myhub,bus=usb.0,port=3 ^
+-drive if=none,id=usbflash,format=vpc,file=usb.vhd ^
+-device usb-storage,drive=usbflash,bus=usb.0,port=3.1,id=myusbflash