usb hubs, hotplug support, unplug support, saving log files to disk, smt

code/drivers/bus/usb/usb_xhci.c

@@ -353,6 +353,31 @@ void XhciLogPortStatus(XHCI_CONTROLLER* xhci) {
     }
 }
 
+void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
+    if (!xhci || !xhci->Op)
+        return;
+
+    uint32_t powered = 0;
+    for (uint8_t port = 1; port <= xhci->MaxPorts; port++) {
+        uint32_t off = XhciPortReg(port);
+        uint32_t portsc = XhciRead32(xhci->Op, off);
+        uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK;
+
+        if (!(portsc & XHCI_PORTSC_PP)) {
+            XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
+            powered++;
+            UsbTrace("usb: xHCI powered root port %u old-portsc=%08x\n", port, portsc);
+        } else if (changes) {
+            XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
+        }
+    }
+
+    if (powered) {
+        UsbDelayUs(xhci->Hal, 20000u);
+        DbgPrint("usb: xHCI powered %u previously-off root port(s)\n", powered);
+    }
+}
+
 
 void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
     if (!xhci->Runtime) return;
@@ -369,7 +394,71 @@ void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
 }
 
 
+static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
+    for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
+        if (w->CommandPhys && w->CommandPhys == commandPhys)
+            return w;
+    }
+    return (XHCI_WAIT_TARGET*)0;
+}
+
+
+static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
+                                                      uint64_t transferPhys,
+                                                      uint8_t slotId,
+                                                      uint8_t endpointId) {
+    for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
+        if (w->TransferPhys &&
+            w->TransferPhys == transferPhys &&
+            w->TransferSlotId == slotId &&
+            w->TransferEndpointId == endpointId) {
+            return w;
+        }
+    }
+    return (XHCI_WAIT_TARGET*)0;
+}
+
+
+static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
+                                                uint64_t commandPhys,
+                                                uint32_t completionCode,
+                                                uint8_t slotId) {
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys);
+    if (w) {
+        w->Command.CompletionCode = completionCode;
+        w->Command.SlotId = slotId;
+        XhciBarrier();
+        w->CommandMatched = true;
+    }
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+    return w;
+}
+
+
+static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
+                                                 uint64_t transferPhys,
+                                                 uint8_t slotId,
+                                                 uint8_t endpointId,
+                                                 uint32_t completionCode,
+                                                 uint32_t residue) {
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId);
+    if (w) {
+        w->Transfer.CompletionCode = completionCode;
+        w->Transfer.SlotId = slotId;
+        w->Transfer.EndpointId = endpointId;
+        w->Transfer.Residue = residue;
+        XhciBarrier();
+        w->TransferMatched = true;
+    }
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+    return w;
+}
+
+
 bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
+    (void)target;
     bool processed = false;
 
     for (;;) {
@@ -384,41 +473,28 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
         uint32_t cc = XhciCompletionCode(ev);
         uint64_t param = ((uint64_t)ev->ParameterHi << 32) | ev->ParameterLo;
 
-        XHCI_WAIT_TARGET* waiter = target ? target : (XHCI_WAIT_TARGET*)xhci->ActiveWait;
-
         if (type == XHCI_TRB_TYPE_CMD_DONE) {
             uint8_t slotId = XhciEventSlotId(ev);
             UsbTrace("usb: xHCI command completion ptr=0x%016llX cc=%u slot=%u\n",
                      (unsigned long long)param, cc, slotId);
-            if (waiter && waiter->CommandPhys && param == waiter->CommandPhys) {
-                waiter->CommandMatched = true;
-                waiter->Command.CompletionCode = cc;
-                waiter->Command.SlotId = slotId;
-            }
+            (void)XhciClaimCommandWaiter(xhci, param, cc, slotId);
         } else if (type == XHCI_TRB_TYPE_TRANSFER) {
             uint8_t slotId = XhciEventSlotId(ev);
             uint8_t epId = XhciEventEndpointId(ev);
             uint32_t residue = XhciTransferResidue(ev);
-            bool claimedByWaiter = false;
-            bool targetMatches = waiter && waiter->TransferPhys && param == waiter->TransferPhys &&
-                                 slotId == waiter->TransferSlotId && epId == waiter->TransferEndpointId;
             if (cc != XHCI_CC_SUCCESS) {
                 UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n",
                          (unsigned long long)param, cc, slotId, epId, residue);
             }
-            if (targetMatches) {
-                waiter->TransferMatched = true;
-                waiter->Transfer.CompletionCode = cc;
-                waiter->Transfer.SlotId = slotId;
-                waiter->Transfer.EndpointId = epId;
-                waiter->Transfer.Residue = residue;
-                claimedByWaiter = true;
-            }
-            if (!claimedByWaiter)
+
+            XHCI_WAIT_TARGET* waiter = XhciClaimTransferWaiter(xhci, param, slotId, epId, cc, residue);
+            if (!waiter)
                 XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue);
         } else if (type == XHCI_TRB_TYPE_PORT_SC) {
             uint32_t portId = ev->ParameterLo >> 24;
             UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc);
+            if (portId && portId <= xhci->MaxPorts)
+                UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)portId);
         } else {
             DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n",
                      type, cc, (unsigned long long)param, ev->Status, ev->Control);
@@ -454,22 +530,53 @@ bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
 }
 
 
-KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
-                                 uint32_t timeoutUs) {
+void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
+    if (!xhci || !target || target->Registered)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    target->Next = xhci->WaiterList;
+    target->Registered = true;
+    xhci->WaiterList = target;
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+}
+
+
+void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
+    if (!xhci || !target || !target->Registered)
+        return;
+
+    uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
+    XHCI_WAIT_TARGET** link = &xhci->WaiterList;
+    while (*link) {
+        if (*link == target) {
+            *link = target->Next;
+            target->Next = (XHCI_WAIT_TARGET*)0;
+            target->Registered = false;
+            break;
+        }
+        link = &(*link)->Next;
+    }
+    KeReleaseSpinlock(&xhci->WaitLock, flags);
+}
+
+
+KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
+                               uint32_t timeoutUs) {
     if (!xhci || !target)
         return STATUS_INVALID_PARAMETER;
 
-    xhci->ActiveWait = target;
     for (uint32_t i = 0; i < timeoutUs; i++) {
         if (XhciWaitTargetComplete(target)) {
-            xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
+            XhciUnregisterWaitTarget(xhci, target);
             return STATUS_SUCCESS;
         }
 
         // The IRQ/DPC path normally owns the event ring after enumeration, but
         // synchronous storage/control waits must not depend exclusively on an
-        // interrupt being delivered.  Opportunistically drain the ring from the
-        // waiter too; EventLock serializes this with the DPC.
+        // interrupt being delivered. Opportunistically drain the ring too;
+        // EventLock serializes this with the DPC while WaitLock protects waiter
+        // registration/matching.
         if (!xhci->InterruptsEnabled || (i & 0x7u) == 0) {
             XhciDrainEventRingLocked(xhci, target);
         } else if ((XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_EINT) && (i & 0x3Fu) == 0) {
@@ -479,11 +586,20 @@ KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
         UsbDelayUs(xhci->Hal, 1);
     }
 
-    xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
+    XhciUnregisterWaitTarget(xhci, target);
     return STATUS_TIMEOUT;
 }
 
 
+KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
+                          uint32_t timeoutUs) {
+    if (!xhci || !target)
+        return STATUS_INVALID_PARAMETER;
+    XhciRegisterWaitTarget(xhci, target);
+    return XhciWaitForArmedTarget(xhci, target, timeoutUs);
+}
+
+
 void XhciEventDpcRoutine(KDPC* dpc, void* context,
                                 void* systemArgument1, void* systemArgument2) {
     (void)dpc;
@@ -695,13 +811,26 @@ uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
 }
 
 
-KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
-                                  uint64_t parameter,
-                                  uint32_t status,
-                                  uint32_t controlWithoutCycle,
-                                  XHCI_COMMAND_RESULT* outResult) {
-    if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u)
-        return STATUS_QUEUE_FULL;
+static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
+                                      uint64_t parameter,
+                                      uint32_t status,
+                                      uint32_t controlWithoutCycle,
+                                      uint64_t* outPhys) {
+    if (!xhci || !xhci->CommandRing)
+        return STATUS_INVALID_PARAMETER;
+
+    if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) {
+        XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
+        link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
+        link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
+        link->Status = 0;
+        XhciBarrier();
+        link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
+                        XHCI_TRB_LINK_TC |
+                        xhci->CommandCycle;
+        xhci->CommandEnqueue = 0;
+        xhci->CommandCycle ^= 1u;
+    }
 
     uint32_t index = xhci->CommandEnqueue++;
     XHCI_TRB* trb = &xhci->CommandRing[index];
@@ -713,15 +842,38 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
     XhciBarrier();
     trb->Control = controlWithoutCycle | xhci->CommandCycle;
 
+    if (outPhys) *outPhys = trbPhys;
+    return STATUS_SUCCESS;
+}
+
+
+KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
+                                  uint64_t parameter,
+                                  uint32_t status,
+                                  uint32_t controlWithoutCycle,
+                                  XHCI_COMMAND_RESULT* outResult) {
+    if (!xhci)
+        return STATUS_INVALID_PARAMETER;
+
     XHCI_WAIT_TARGET target;
     memset(&target, 0, sizeof(target));
+
+    uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock);
+    uint64_t trbPhys = 0;
+    KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys);
+    if (!KSUCCESS(st)) {
+        KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
+        return st;
+    }
+
     target.CommandPhys = trbPhys;
-    xhci->ActiveWait = &target;
+    XhciRegisterWaitTarget(xhci, &target);
 
     XhciBarrier();
     XhciWrite32(xhci->Doorbells, 0, 0); // host-controller doorbell, command ring
+    KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
 
-    KSTATUS waitStatus = XhciWaitForTarget(xhci, &target, 500000u);
+    KSTATUS waitStatus = XhciWaitForArmedTarget(xhci, &target, 500000u);
     if (KSUCCESS(waitStatus) && target.CommandMatched) {
         if (outResult) *outResult = target.Command;
         return (target.Command.CompletionCode == XHCI_CC_SUCCESS)
@@ -781,6 +933,28 @@ KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
 }
 
 
+
+KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
+    if (!xhci || slotId == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    XHCI_COMMAND_RESULT res;
+    memset(&res, 0, sizeof(res));
+    KSTATUS st = XhciSubmitCommand(xhci,
+                                   0,
+                                   0,
+                                   (XHCI_TRB_TYPE_DISABLE_SLOT << XHCI_TRB_TYPE_SHIFT) |
+                                       ((uint32_t)slotId << 24),
+                                   &res);
+    if (!KSUCCESS(st))
+        return st;
+    if (res.CompletionCode != XHCI_CC_SUCCESS) {
+        DbgPrint("usb: Disable Slot slot=%u failed cc=%u\n", slotId, res.CompletionCode);
+        return STATUS_DEVICE_ERROR;
+    }
+    return STATUS_SUCCESS;
+}
+
 KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     XHCI_COMMAND_RESULT result;
     memset(&result, 0, sizeof(result));
@@ -970,12 +1144,22 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
     uint32_t maxPacket = XhciEpMaxPacket(dev->Speed);
     dev->MaxPacketSize0 = (uint16_t)maxPacket;
 
-    slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed);
+    uint8_t rootPort = dev->Path.RootPort ? dev->Path.RootPort : dev->Port;
+    uint32_t routeString = dev->Path.RouteString & 0x000fffffu;
+
+    slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed) | routeString;
     // Slot context dword 1 owns Root Hub Port Number bits 23:16.
     // Putting the port number in dword 2 makes Address Device fail with a
     // context/TRB error on QEMU xHCI.
-    slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
+    slot[1] = XHCI_SLOT_ROOT_PORT(rootPort);
     slot[2] = 0;
+    if (dev->Path.BehindHub && dev->Speed <= 2u && dev->Path.ParentHubSlot) {
+        // TT Hub Slot ID bits 7:0 and TT Port Number bits 15:8.
+        // This is required for low/full-speed devices behind high-speed hubs;
+        // harmless for older full-speed hub paths on the controllers we target.
+        slot[2] = (uint32_t)dev->Path.ParentHubSlot |
+                  ((uint32_t)dev->Path.ParentHubPort << 8);
+    }
     slot[3] = 0;
 
     ep0[0] = 0;
@@ -999,8 +1183,21 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
                                 uint32_t status,
                                 uint32_t controlWithoutCycle,
                                 uint64_t* outPhys) {
-    if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u)
-        return STATUS_QUEUE_FULL;
+    if (!dev || !dev->Ep0Ring)
+        return STATUS_INVALID_PARAMETER;
+
+    if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) {
+        XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
+        link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
+        link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
+        link->Status = 0;
+        XhciBarrier();
+        link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
+                        XHCI_TRB_LINK_TC |
+                        dev->Ep0Cycle;
+        dev->Ep0Enqueue = 0;
+        dev->Ep0Cycle ^= 1u;
+    }
 
     uint32_t index = dev->Ep0Enqueue++;
     XHCI_TRB* trb = &dev->Ep0Ring[index];
@@ -1017,11 +1214,12 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
 }
 
 
-KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
-                             const USB_SETUP_PACKET* setup,
-                             void* buffer,
-                             PHYSICAL_ADDRESS bufferPhys,
-                             uint16_t length) {
+KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+                                    const USB_SETUP_PACKET* setup,
+                                    void* buffer,
+                                    PHYSICAL_ADDRESS bufferPhys,
+                                    uint16_t length,
+                                    uint32_t timeoutUs) {
     if (!buffer || length == 0 || length > PAGE_SIZE)
         return STATUS_INVALID_PARAMETER;
 
@@ -1057,18 +1255,21 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
         &statusTrbPhys);
     if (!KSUCCESS(st)) return st;
 
-    XhciBarrier();
-    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
-
     XHCI_WAIT_TARGET target;
     memset(&target, 0, sizeof(target));
     target.TransferPhys = statusTrbPhys;
     target.TransferSlotId = dev->SlotId;
     target.TransferEndpointId = 1;
+    XhciRegisterWaitTarget(xhci, &target);
 
-    st = XhciWaitForTarget(xhci, &target, 1000000u);
+    XhciBarrier();
+    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
+
+    if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
+    st = XhciWaitForArmedTarget(xhci, &target, timeoutUs);
     if (!KSUCCESS(st)) {
-        DbgPrint("usb: xHCI control IN timeout slot=%u len=%u\n", dev->SlotId, length);
+        DbgPrint("usb: xHCI control IN timeout slot=%u len=%u timeout=%uus\n",
+                 dev->SlotId, length, timeoutUs);
         return st;
     }
     if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS)
@@ -1082,6 +1283,16 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
+KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+                             const USB_SETUP_PACKET* setup,
+                             void* buffer,
+                             PHYSICAL_ADDRESS bufferPhys,
+                             uint16_t length) {
+    return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length,
+                                USB_XHCI_CONTROL_TIMEOUT_US);
+}
+
+
 KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
                                       const USB_SETUP_PACKET* setup) {
     uint64_t setupValue = UsbSetupPacketToU64(setup);
@@ -1106,16 +1317,17 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
         &statusTrbPhys);
     if (!KSUCCESS(st)) return st;
 
-    XhciBarrier();
-    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
-
     XHCI_WAIT_TARGET target;
     memset(&target, 0, sizeof(target));
     target.TransferPhys = statusTrbPhys;
     target.TransferSlotId = dev->SlotId;
     target.TransferEndpointId = 1;
+    XhciRegisterWaitTarget(xhci, &target);
 
-    st = XhciWaitForTarget(xhci, &target, 1000000u);
+    XhciBarrier();
+    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
+
+    st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
     if (!KSUCCESS(st)) {
         DbgPrint("usb: xHCI control no-data timeout slot=%u req=%u value=%04x\n",
                  dev->SlotId, setup->bRequest, setup->wValue);
@@ -1132,8 +1344,9 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
-KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
-                                       USB_DEVICE_DESCRIPTOR* outDesc) {
+KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+                                              USB_DEVICE_DESCRIPTOR* outDesc,
+                                              uint32_t timeoutUs) {
     USB_SETUP_PACKET setup;
     setup.bmRequestType = USB_DIR_IN;
     setup.bRequest = USB_REQ_GET_DESCRIPTOR;
@@ -1141,10 +1354,12 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     setup.wIndex = 0;
     setup.wLength = sizeof(USB_DEVICE_DESCRIPTOR);
 
-    KSTATUS st = XhciControlIn(xhci, dev, &setup,
-                               dev->DescriptorBuffer,
-                               dev->DescriptorPhys,
-                               sizeof(USB_DEVICE_DESCRIPTOR));
+    if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
+    KSTATUS st = XhciControlInTimeout(xhci, dev, &setup,
+                                      dev->DescriptorBuffer,
+                                      dev->DescriptorPhys,
+                                      sizeof(USB_DEVICE_DESCRIPTOR),
+                                      timeoutUs);
     if (!KSUCCESS(st)) return st;
 
     memcpy(outDesc, dev->DescriptorBuffer, sizeof(*outDesc));
@@ -1159,6 +1374,13 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
+KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
+                                       USB_DEVICE_DESCRIPTOR* outDesc) {
+    return XhciGetDeviceDescriptorTimeout(xhci, dev, outDesc,
+                                          USB_XHCI_CONTROL_TIMEOUT_US);
+}
+
+
 KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
                                             uint8_t configurationIndex,
                                             USB_CONFIGURATION_DESCRIPTOR* outConfig,
@@ -1331,11 +1553,11 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
     target.TransferSlotId = dev->SlotId;
     target.TransferEndpointId = endpointId;
 
-    xhci->ActiveWait = &target;
+    XhciRegisterWaitTarget(xhci, &target);
     XhciBarrier();
     XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, endpointId);
 
-    st = XhciWaitForTarget(xhci, &target, 1000000u);
+    st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
     if (!KSUCCESS(st)) {
         DbgPrint("usb-storage: bulk transfer timeout slot=%u ep=%u len=%u\n",
                  dev->SlotId, endpointId, length);
@@ -1356,6 +1578,300 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
 }
 
 
+
+
+KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) {
+    if (!xhci || !dev || !pipe || !pipe->Open)
+        return STATUS_INVALID_PARAMETER;
+
+    uint8_t dci = pipe->EndpointId;
+    if (dci < 2 || dci > 31)
+        return STATUS_INVALID_PARAMETER;
+
+    if (pipe->HcdPipeContext) {
+        XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
+        return ctx->Configured ? STATUS_SUCCESS : STATUS_DEVICE_ERROR;
+    }
+
+    XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)MmAllocatePool(sizeof(XHCI_PIPE_CONTEXT));
+    if (!ctx) return STATUS_NO_MEMORY;
+    memset(ctx, 0, sizeof(*ctx));
+
+    ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys);
+    if (!ctx->Ring) {
+        MmFreePool(ctx);
+        return STATUS_NO_MEMORY;
+    }
+    ctx->Enqueue = 0;
+    ctx->Cycle = 1;
+    ctx->EndpointId = dci;
+    ctx->Type = pipe->Type;
+    XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle);
+
+    memset(dev->InputContext, 0, PAGE_SIZE);
+    uint32_t* icc = XhciInputControlContext(xhci, dev);
+    uint32_t* slot = XhciInputSlotContext(xhci, dev);
+    uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci);
+
+    icc[0] = 0;
+    icc[1] = (1u << 0) | (1u << dci);
+
+    slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed);
+    slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
+    slot[2] = 0;
+    slot[3] = 0;
+
+    uint16_t maxPacket = pipe->MaxPacketSize & 0x07FFu;
+    if (maxPacket == 0) maxPacket = 8u;
+
+    uint32_t endpointType = 0;
+    uint8_t interval = 0;
+    if (pipe->Type == UsbPipeInterrupt && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
+        endpointType = XHCI_EP_TYPE_INT_IN;
+        interval = XhciInterruptInterval(dev->Speed, pipe->Interval);
+    } else if (pipe->Type == UsbPipeBulk && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
+        endpointType = XHCI_EP_TYPE_BULK_IN;
+    } else if (pipe->Type == UsbPipeBulk) {
+        endpointType = XHCI_EP_TYPE_BULK_OUT;
+    } else {
+        MmFreePool(ctx);
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    ep[0] = (uint32_t)interval << 16;
+    ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16);
+    ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS);
+    ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32);
+    ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
+
+    XHCI_COMMAND_RESULT result;
+    memset(&result, 0, sizeof(result));
+    KSTATUS st = XhciSubmitCommand(
+        xhci,
+        dev->InputContextPhys,
+        0,
+        (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
+            ((uint32_t)dev->SlotId << 24),
+        &result);
+    if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
+        DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
+                 dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
+        MmFreePool(ctx);
+        return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
+    }
+
+    ctx->Configured = true;
+    pipe->HcdPipeContext = ctx;
+    dev->EndpointPipes[dci] = pipe;
+    UsbReferencePipe(pipe);
+    return STATUS_SUCCESS;
+}
+
+static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
+    if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
+        return STATUS_INVALID_PARAMETER;
+    if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN))
+        return STATUS_NOT_SUPPORTED;
+
+    KSTATUS st = XhciConfigurePipeEndpoint(xhci, dev, urb->Pipe);
+    if (!KSUCCESS(st)) return st;
+
+    XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext;
+    if (!ctx || !ctx->Configured)
+        return STATUS_DEVICE_ERROR;
+    if (ctx->ActiveTransfer)
+        return STATUS_QUEUE_FULL;
+
+    uint64_t trbPhys = 0;
+    st = XhciRingAppend(ctx->Ring,
+                        ctx->RingPhys,
+                        XHCI_PIPE_RING_TRBS,
+                        &ctx->Enqueue,
+                        &ctx->Cycle,
+                        urb->BufferPhys,
+                        urb->Length,
+                        (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
+                            XHCI_TRB_ISP |
+                            XHCI_TRB_IOC,
+                        &trbPhys);
+    if (!KSUCCESS(st)) return st;
+
+    ctx->ActiveTransfer = true;
+    ctx->ActiveTrbPhys = trbPhys;
+    ctx->ActiveUrb = urb;
+    urb->Pending = true;
+    urb->Cancelled = false;
+    urb->Status = STATUS_PENDING;
+    urb->ActualLength = 0;
+
+    XhciBarrier();
+    XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, ctx->EndpointId);
+    return STATUS_PENDING;
+}
+
+void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
+                                  uint32_t completionCode, uint8_t slotId,
+                                  uint8_t endpointId, uint32_t residue) {
+    if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
+        return;
+
+    XHCI_DEVICE* dev = &xhci->Devices[slotId];
+    USB_PIPE* pipe = dev->EndpointPipes[endpointId];
+    if (!pipe || !pipe->HcdPipeContext)
+        return;
+
+    XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
+    if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys)
+        return;
+
+    USB_URB* urb = ctx->ActiveUrb;
+    ctx->ActiveTransfer = false;
+    ctx->ActiveTrbPhys = 0;
+    ctx->ActiveUrb = (USB_URB*)0;
+
+    if (!urb)
+        return;
+
+    KSTATUS st = STATUS_SUCCESS;
+    if (urb->Cancelled) {
+        st = STATUS_CANCELLED;
+    } else if (completionCode != XHCI_CC_SUCCESS) {
+        st = STATUS_DEVICE_ERROR;
+    }
+
+    uint32_t actual = 0;
+    if (KSUCCESS(st)) {
+        actual = (residue <= urb->Length) ? (urb->Length - residue) : 0;
+    }
+
+    urb->Pending = false;
+    urb->Status = st;
+    urb->ActualLength = actual;
+
+    IRP* completeIrp = urb->Irp;
+    urb->Irp = (IRP*)0;
+
+    if (urb->Complete)
+        urb->Complete(urb, urb->CompleteContext);
+
+    if (completeIrp)
+        IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0);
+}
+
+static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
+    if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
+        return STATUS_INVALID_PARAMETER;
+
+    uint8_t endpointId = urb->Pipe->EndpointId;
+    XHCI_TRB* ring = (XHCI_TRB*)0;
+    PHYSICAL_ADDRESS ringPhys = 0;
+    uint32_t* enqueue = (uint32_t*)0;
+    uint8_t* cycle = (uint8_t*)0;
+
+    if (endpointId == dev->MassBulkInEndpointId) {
+        ring = dev->MassInRing;
+        ringPhys = dev->MassInRingPhys;
+        enqueue = &dev->MassInEnqueue;
+        cycle = &dev->MassInCycle;
+    } else if (endpointId == dev->MassBulkOutEndpointId) {
+        ring = dev->MassOutRing;
+        ringPhys = dev->MassOutRingPhys;
+        enqueue = &dev->MassOutEnqueue;
+        cycle = &dev->MassOutCycle;
+    } else {
+        return STATUS_NOT_SUPPORTED;
+    }
+
+    uint32_t actual = 0;
+    KSTATUS st = XhciBulkTransfer(xhci,
+                                  dev,
+                                  endpointId,
+                                  ring,
+                                  ringPhys,
+                                  enqueue,
+                                  cycle,
+                                  urb->BufferPhys,
+                                  urb->Length,
+                                  &actual);
+    urb->ActualLength = KSUCCESS(st) ? actual : 0;
+    return st;
+}
+
+static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
+    if (!hcd || !urb || !urb->Device)
+        return STATUS_INVALID_PARAMETER;
+
+    XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext;
+    XHCI_DEVICE* dev = (XHCI_DEVICE*)urb->Device->HcdDeviceContext;
+    if (!xhci || !dev || !dev->Allocated)
+        return STATUS_INVALID_PARAMETER;
+
+    KSTATUS st;
+    switch (urb->Function) {
+        case UrbFunctionControlTransfer:
+            if (urb->Length == 0) {
+                st = XhciControlNoData(xhci, dev, &urb->Setup);
+                urb->ActualLength = 0;
+            } else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) {
+                st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length);
+                urb->ActualLength = KSUCCESS(st) ? urb->Length : 0;
+            } else {
+                st = STATUS_NOT_SUPPORTED;
+                urb->ActualLength = 0;
+            }
+            break;
+
+        case UrbFunctionBulkTransfer:
+            st = XhciHcdSubmitBulkUrb(xhci, dev, urb);
+            break;
+
+        case UrbFunctionInterruptTransfer:
+            st = XhciHcdSubmitInterruptUrb(xhci, dev, urb);
+            break;
+
+        default:
+            st = STATUS_NOT_SUPPORTED;
+            urb->ActualLength = 0;
+            break;
+    }
+
+    urb->Status = st;
+    return st;
+}
+
+static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
+    (void)hcd;
+    if (!urb) return STATUS_INVALID_PARAMETER;
+    urb->Cancelled = true;
+    if (!urb->Pending)
+        urb->Status = STATUS_CANCELLED;
+    return STATUS_SUCCESS;
+}
+
+static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
+    if (!hcd) return (void*)0;
+    return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
+}
+
+static KSTATUS XhciHcdResetRootPort(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed) {
+    if (!hcd) return STATUS_INVALID_PARAMETER;
+    return XhciResetPort((XHCI_CONTROLLER*)hcd->ControllerContext, port, outSpeed);
+}
+
+static KSTATUS XhciHcdClearRootPortChanges(USB_HCD* hcd, uint8_t port) {
+    if (!hcd) return STATUS_INVALID_PARAMETER;
+    XhciClearPortChanges((XHCI_CONTROLLER*)hcd->ControllerContext, port);
+    return STATUS_SUCCESS;
+}
+
+const USB_HCD_OPERATIONS gXhciHcdOperations = {
+    XhciHcdSubmitUrb,
+    XhciHcdCancelUrb,
+    XhciHcdResetRootPort,
+    XhciHcdClearRootPortChanges,
+    XhciHcdAllocateDma,
+};
+
 KSTATUS XhciInitialise(const HAL_API* hal,
                               uint8_t bus, uint8_t dev, uint8_t fn,
                               uint16_t vendor, uint16_t device) {
@@ -1438,6 +1954,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
         return STATUS_DEVICE_ERROR;
     }
 
+    UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts);
+    KSTATUS hcdPdoStatus = UsbCreateControllerPdo(&xhci->Hcd, vendor, device);
+    if (!KSUCCESS(hcdPdoStatus)) {
+        DbgPrint("usb: xHCI controller PDO creation failed st=0x%x\n", hcdPdoStatus);
+        return hcdPdoStatus;
+    }
+
     uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE +
                        (uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE;
     uint32_t needBytes = portEnd;
@@ -1489,12 +2012,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
     st = XhciSubmitNoopCommand(xhci);
     if (!KSUCCESS(st)) return st;
 
+    XhciPowerRootPorts(xhci);
     XhciLogPortStatus(xhci);
-    XhciEnumerateRootPorts(xhci);
 
-    if (xhci->HidDeviceCount > 0) {
-        st = XhciStartHidInterrupts(xhci);
-        if (!KSUCCESS(st)) return st;
+    st = UsbCreateRootHubPdo(&xhci->Hcd);
+    if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) {
+        DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st);
+        return st;
     }
 
     KeIrqRegister(xhci->Irq, XhciIsr);
@@ -1504,8 +2028,14 @@ KSTATUS XhciInitialise(const HAL_API* hal,
     DbgPrint("usb: xHCI ISR registered on IRQ %u%s; event ring handled by DPC\n",
              xhci->Irq, xhci->UsingMsi ? "(msi)" : "");
 
+    st = UsbHubScheduleRootScan(&xhci->Hcd);
+    if (!KSUCCESS(st))
+        DbgPrint("usb: root hub async scan scheduling failed st=0x%x\n", st);
+    else
+        DbgPrint("usb: root hub initial scan queued asynchronously\n");
+
     gHaveXhci = true;
-    DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage is next\n",
+    DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
              xhci->HidDeviceCount);
     return STATUS_SUCCESS;
 }