// xHCI USB host-controller driver. This file owns MMIO setup, rings, slots, endpoints, and transfer completion. #include "usb.h" // Implements the align Up32 operation. uint32_t UsbAlignUp32(uint32_t value, uint32_t align) { return (value + align - 1u) & ~(align - 1u); } // Implements the barrier operation. void XhciBarrier(void) { _ReadWriteBarrier(); _mm_mfence(); _ReadWriteBarrier(); } // Implements the read8 operation. uint8_t XhciRead8(volatile uint8_t* base, uint32_t reg) { return *(volatile uint8_t*)(base + reg); } // Implements the read16 operation. uint16_t XhciRead16(volatile uint8_t* base, uint32_t reg) { return *(volatile uint16_t*)(base + reg); } // Implements the read32 operation. uint32_t XhciRead32(volatile uint8_t* base, uint32_t reg) { return *(volatile uint32_t*)(base + reg); } // Implements the write32 operation. void XhciWrite32(volatile uint8_t* base, uint32_t reg, uint32_t value) { *(volatile uint32_t*)(base + reg) = value; } // Implements the read64 operation. uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg) { uint32_t lo = XhciRead32(base, reg); uint32_t hi = XhciRead32(base, reg + 4u); return ((uint64_t)hi << 32) | lo; } // Implements the write64 operation. void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value) { XhciWrite32(base, reg, (uint32_t)value); XhciWrite32(base, reg + 4u, (uint32_t)(value >> 32)); } // Implements the delay Us operation. void UsbDelayUs(const HAL_API* hal, uint32_t us) { if (hal && hal->Timer.Stall) hal->Timer.Stall(us); else { for (uint32_t i = 0; i < us * 32u; i++) _mm_pause(); } } bool UsbWaitSet32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs) { for (uint32_t i = 0; i < timeoutUs; i++) { if ((XhciRead32(base, reg) & mask) == mask) return true; UsbDelayUs(hal, 1); } return false; } bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs) { for (uint32_t i = 0; i < timeoutUs; i++) { if ((XhciRead32(base, reg) & mask) == 0) return true; UsbDelayUs(hal, 1); } return false; } void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys) { if (!outPhys || pages == 0) return NULL; // Most xHCI runtime objects are one page. Do not route those through the // physically-contiguous allocator: on real machines that allocator is a // scarce DMA primitive, while a single frame is already contiguous. QEMU // hid this because it had simple low memory; laptop firmware does not. PHYSICAL_ADDRESS phys = 0; if (pages == 1) { phys = MmPfnAllocate(MmPfnUseDma); } else { phys = MmPfnAllocateContiguous(pages, MmPfnUseDma); } if (!phys) { DbgPrint("usb: DMA allocation failed pages=%llu\n", (unsigned long long)pages); return NULL; } void* virt = xhci->Hal->PhysToVirt(phys); memset(virt, 0, pages * (size_t)PAGE_SIZE); *outPhys = phys; return virt; } // Implements the scratchpad Count operation. uint16_t XhciScratchpadCount(uint32_t hcs2) { return (uint16_t)((((hcs2 >> 16) & 0x3E0u) | ((hcs2 >> 27) & 0x1Fu)) & 0x3FFu); } // Implements the trb Type operation. uint32_t XhciTrbType(const XHCI_TRB* trb) { return (trb->Control >> XHCI_TRB_TYPE_SHIFT) & XHCI_TRB_TYPE_MASK; } // Implements the completion Code operation. uint32_t XhciCompletionCode(const XHCI_TRB* trb) { return (trb->Status >> 24) & 0xFFu; } // Implements the event Slot Id operation. uint8_t XhciEventSlotId(const XHCI_TRB* trb) { return (uint8_t)(trb->Control >> 24); } // Implements the event Endpoint Id operation. uint8_t XhciEventEndpointId(const XHCI_TRB* trb) { return (uint8_t)((trb->Control >> 16) & 0x1Fu); } // Implements the transfer Residue operation. uint32_t XhciTransferResidue(const XHCI_TRB* trb) { return trb->Status & 0xFFFFFFu; } // Implements the input Control Context operation. uint32_t* XhciInputControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { (void)xhci; return (uint32_t*)dev->InputContext; } // Implements the input Slot Context operation. uint32_t* XhciInputSlotContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { return (uint32_t*)(dev->InputContext + xhci->ContextBytes); } uint32_t* XhciInputEndpointContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId) { return (uint32_t*)(dev->InputContext + (uint32_t)(endpointId + 1u) * xhci->ContextBytes); } // Implements the ep Max Packet operation. uint32_t XhciEpMaxPacket(uint8_t speed) { switch (speed) { case 2: return 8; // low-speed case 3: return 64; // high-speed case 4: return 512; // SuperSpeed case 5: return 512; // SuperSpeed+ case 1: default: return 8; // full-speed before descriptor refinement } } // Implements the setup Packet To U64 operation. uint64_t UsbSetupPacketToU64(const USB_SETUP_PACKET* setup) { uint64_t v = 0; v |= (uint64_t)setup->bmRequestType; v |= (uint64_t)setup->bRequest << 8; v |= (uint64_t)setup->wValue << 16; v |= (uint64_t)setup->wIndex << 32; v |= (uint64_t)setup->wLength << 48; return v; } const char* XhciPortSpeedName(uint32_t speed) { switch (speed) { case 1: return "full"; case 2: return "low"; case 3: return "high"; case 4: return "super"; case 5: return "super+"; default: return "unknown"; } } // Parses supported Protocols. void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) { memset(xhci->PortProtocolMajor, 0, sizeof(xhci->PortProtocolMajor)); memset(xhci->PortProtocolMinor, 0, sizeof(xhci->PortProtocolMinor)); uint32_t xecp = (xhci->Hcc1 >> 16) & 0xFFFFu; for (uint32_t guard = 0; xecp && guard < 64; guard++) { uint32_t off = xecp << 2; if (off + 12u > xhci->MmioBytes) break; uint32_t cap = XhciRead32(xhci->Mmio, off); uint8_t id = (uint8_t)(cap & 0xFFu); uint8_t next = (uint8_t)((cap >> 8) & 0xFFu); if (id == XHCI_EXT_CAP_PROTOCOL) { uint8_t minor = (uint8_t)((cap >> 16) & 0xFFu); uint8_t major = (uint8_t)((cap >> 24) & 0xFFu); uint32_t name = XhciRead32(xhci->Mmio, off + 4u); uint32_t ports = XhciRead32(xhci->Mmio, off + 8u); uint8_t portOffset = (uint8_t)(ports & 0xFFu); uint8_t portCount = (uint8_t)((ports >> 8) & 0xFFu); UsbTrace("usb: xHCI protocol cap +0x%x name='%c%c%c%c' usb=%u.%u ports=%u..%u\n", off, (char)(name & 0xFFu), (char)((name >> 8) & 0xFFu), (char)((name >> 16) & 0xFFu), (char)((name >> 24) & 0xFFu), major, minor, portOffset, portCount ? (uint8_t)(portOffset + portCount - 1u) : portOffset); if (portOffset != 0 && portCount != 0) { for (uint32_t i = 0; i < portCount; i++) { uint32_t port = (uint32_t)portOffset + i; if (port <= XHCI_MAX_ROOT_PORTS) { xhci->PortProtocolMajor[port] = major; xhci->PortProtocolMinor[port] = minor; } } } } xecp = next; } } // Implements the port Protocol Major operation. uint8_t XhciPortProtocolMajor(XHCI_CONTROLLER* xhci, uint8_t port, uint32_t portsc) { if (port <= XHCI_MAX_ROOT_PORTS && xhci->PortProtocolMajor[port]) return xhci->PortProtocolMajor[port]; uint32_t speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; if (speed >= 4u) return 3u; if (speed >= 1u && speed <= 3u) return 2u; return 0u; } const char* UsbEndpointTypeName(uint8_t type) { switch (type & USB_ENDPOINT_ATTR_TYPE_MASK) { case USB_ENDPOINT_TYPE_CONTROL: return "control"; case USB_ENDPOINT_TYPE_ISOCH: return "isochronous"; case USB_ENDPOINT_TYPE_BULK: return "bulk"; case USB_ENDPOINT_TYPE_INTERRUPT: return "interrupt"; default: return "unknown"; } } const char* UsbDirectionName(uint8_t endpointAddress) { return (endpointAddress & USB_ENDPOINT_DIR_IN) ? "in" : "out"; } const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto) { if (klass == USB_CLASS_HID) { if (subclass == 1 && proto == 1) return "hid-boot-keyboard"; if (subclass == 1 && proto == 2) return "hid-boot-mouse"; return "hid"; } if (klass == USB_CLASS_MASS_STORAGE) { if (subclass == USB_MASS_SUBCLASS_SCSI && proto == USB_MASS_PROTOCOL_BOT) return "mass-storage-bot"; return "mass-storage"; } return "class"; } // Implements the log Port Status operation. void XhciLogPortStatus(XHCI_CONTROLLER* xhci) { for (uint8_t port = 1; port <= xhci->MaxPorts; port++) { uint32_t off = XHCI_OP_PORT_BASE + ((uint32_t)port - 1u) * XHCI_OP_PORT_STRIDE; uint32_t portsc = XhciRead32(xhci->Op, off); uint32_t speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; if ((portsc & (XHCI_PORTSC_CCS | XHCI_PORTSC_CHANGE_MASK)) || port <= 8u) { uint8_t proto = XhciPortProtocolMajor(xhci, port, portsc); UsbTrace("usb: xHCI port %u: portsc=%08x ccs=%u ped=%u pp=%u proto=usb%u speed=%s(%u)%s%s\n", port, portsc, (portsc & XHCI_PORTSC_CCS) ? 1u : 0u, (portsc & XHCI_PORTSC_PED) ? 1u : 0u, (portsc & XHCI_PORTSC_PP) ? 1u : 0u, proto, XhciPortSpeedName(speed), speed, (portsc & XHCI_PORTSC_OCA) ? " over-current" : "", (portsc & XHCI_PORTSC_PR) ? " resetting" : ""); } } } // Implements the power Root Ports operation. void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) { if (!xhci || !xhci->Op) return; uint32_t powered = 0; for (uint8_t port = 1; port <= xhci->MaxPorts; port++) { uint32_t off = XhciPortReg(port); uint32_t portsc = XhciRead32(xhci->Op, off); uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK; if (!(portsc & XHCI_PORTSC_PP)) { XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes); powered++; UsbTrace("usb: xHCI powered root port %u old-portsc=%08x\n", port, portsc); } else if (changes) { XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes); } } if (powered) { UsbDelayUs(xhci->Hal, 20000u); DbgPrint("usb: xHCI powered %u previously-off root port(s)\n", powered); } } // Implements the ack Interrupter0 operation. void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) { if (!xhci->Runtime) return; volatile uint8_t* ir0 = xhci->Runtime + 0x20u; uint32_t iman = XhciRead32(ir0, 0x00u); XhciWrite32(ir0, 0x00u, (iman & XHCI_IMAN_IE) | XHCI_IMAN_IP); } // Updates erdp. void XhciUpdateErdp(XHCI_CONTROLLER* xhci) { volatile uint8_t* ir0 = xhci->Runtime + 0x20u; uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB); XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear } // Returns command Waiter Locked. static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) { for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { if (w->CommandPhys && w->CommandPhys == commandPhys) return w; } return (XHCI_WAIT_TARGET*)0; } static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t transferPhys, uint8_t slotId, uint8_t endpointId) { for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { if (w->TransferPhys && w->TransferPhys == transferPhys && w->TransferSlotId == slotId && w->TransferEndpointId == endpointId) { return w; } } return (XHCI_WAIT_TARGET*)0; } static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci, uint64_t commandPhys, uint32_t completionCode, uint8_t slotId) { uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys); if (w) { w->Command.CompletionCode = completionCode; w->Command.SlotId = slotId; XhciBarrier(); w->CommandMatched = true; } KeReleaseSpinlock(&xhci->WaitLock, flags); return w; } static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci, uint64_t transferPhys, uint8_t slotId, uint8_t endpointId, uint32_t completionCode, uint32_t residue) { uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId); if (w) { w->Transfer.CompletionCode = completionCode; w->Transfer.SlotId = slotId; w->Transfer.EndpointId = endpointId; w->Transfer.Residue = residue; XhciBarrier(); w->TransferMatched = true; } KeReleaseSpinlock(&xhci->WaitLock, flags); return w; } // Implements the drain Event Ring operation. bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { (void)target; bool processed = false; for (;;) { XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue]; uint32_t control = ev->Control; uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE); if (cycle != xhci->EventCycle) break; processed = true; uint32_t type = XhciTrbType(ev); uint32_t cc = XhciCompletionCode(ev); uint64_t param = ((uint64_t)ev->ParameterHi << 32) | ev->ParameterLo; if (type == XHCI_TRB_TYPE_CMD_DONE) { uint8_t slotId = XhciEventSlotId(ev); UsbTrace("usb: xHCI command completion ptr=0x%016llX cc=%u slot=%u\n", (unsigned long long)param, cc, slotId); (void)XhciClaimCommandWaiter(xhci, param, cc, slotId); } else if (type == XHCI_TRB_TYPE_TRANSFER) { uint8_t slotId = XhciEventSlotId(ev); uint8_t epId = XhciEventEndpointId(ev); uint32_t residue = XhciTransferResidue(ev); if (cc != XHCI_CC_SUCCESS) { UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n", (unsigned long long)param, cc, slotId, epId, residue); } XHCI_WAIT_TARGET* waiter = XhciClaimTransferWaiter(xhci, param, slotId, epId, cc, residue); if (!waiter) XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue); } else if (type == XHCI_TRB_TYPE_PORT_SC) { uint32_t portId = ev->ParameterLo >> 24; UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc); if (portId && portId <= xhci->MaxPorts) UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)portId); } else { DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n", type, cc, (unsigned long long)param, ev->Status, ev->Control); } xhci->EventDequeue++; if (xhci->EventDequeue == XHCI_EVENT_TRBS) { xhci->EventDequeue = 0; xhci->EventCycle ^= 1u; } } if (processed) XhciUpdateErdp(xhci); return processed; } // Implements the drain Event Ring Locked operation. bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { if (!xhci) return false; uintptr_t flags = KeAcquireSpinlock(&xhci->EventLock); bool processed = XhciDrainEventRing(xhci, target); KeReleaseSpinlock(&xhci->EventLock, flags); return processed; } // Waits for target Complete. bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) { if (!target) return false; if (target->CommandPhys && !target->CommandMatched) return false; if (target->TransferPhys && !target->TransferMatched) return false; return true; } // Registers wait Target. void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { if (!xhci || !target || target->Registered) return; uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); target->Next = xhci->WaiterList; target->Registered = true; xhci->WaiterList = target; KeReleaseSpinlock(&xhci->WaitLock, flags); } // Implements the unregister Wait Target operation. void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { if (!xhci || !target || !target->Registered) return; uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); XHCI_WAIT_TARGET** link = &xhci->WaiterList; while (*link) { if (*link == target) { *link = target->Next; target->Next = (XHCI_WAIT_TARGET*)0; target->Registered = false; break; } link = &(*link)->Next; } KeReleaseSpinlock(&xhci->WaitLock, flags); } KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs) { if (!xhci || !target) return STATUS_INVALID_PARAMETER; for (uint32_t i = 0; i < timeoutUs; i++) { if (XhciWaitTargetComplete(target)) { XhciUnregisterWaitTarget(xhci, target); return STATUS_SUCCESS; } // The IRQ/DPC path normally owns the event ring after enumeration, but // synchronous storage/control waits must not depend exclusively on an // interrupt being delivered. Opportunistically drain the ring too; // EventLock serializes this with the DPC while WaitLock protects waiter // registration/matching. if (!xhci->InterruptsEnabled || (i & 0x7u) == 0) { XhciDrainEventRingLocked(xhci, target); } else if ((XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_EINT) && (i & 0x3Fu) == 0) { KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); } UsbDelayUs(xhci->Hal, 1); } XhciUnregisterWaitTarget(xhci, target); return STATUS_TIMEOUT; } KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs) { if (!xhci || !target) return STATUS_INVALID_PARAMETER; XhciRegisterWaitTarget(xhci, target); return XhciWaitForArmedTarget(xhci, target, timeoutUs); } void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { (void)dpc; (void)systemArgument1; (void)systemArgument2; XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)context; if (!xhci || !xhci->Running || !xhci->Runtime) return; xhci->EventDpcCount++; while (XhciDrainEventRingLocked(xhci, NULL)) { // Keep draining until the software dequeue catches up to the producer. } } #define XHCI_CONTROLLER_MAGIC 0x58484349u // XHCI #define XHCI_MAX_CONTROLLERS 8u static XHCI_CONTROLLER* gXhciControllers[XHCI_MAX_CONTROLLERS]; static uint32_t gXhciControllerCount; // Registers controller. static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) { if (!xhci) return STATUS_INVALID_PARAMETER; for (uint32_t i = 0; i < gXhciControllerCount; i++) { if (gXhciControllers[i] == xhci) return STATUS_SUCCESS; } if (gXhciControllerCount >= XHCI_MAX_CONTROLLERS) return STATUS_NO_MEMORY; gXhciControllers[gXhciControllerCount++] = xhci; return STATUS_SUCCESS; } // Implements the isr operation. void XhciIsr(void) { for (uint32_t i = 0; i < gXhciControllerCount; i++) { XHCI_CONTROLLER* xhci = gXhciControllers[i]; if (!xhci || !xhci->Running || !xhci->Op || !xhci->Runtime) continue; uint32_t status = XhciRead32(xhci->Op, XHCI_OP_USBSTS); if (!(status & (XHCI_USBSTS_EINT | XHCI_USBSTS_PCD | XHCI_USBSTS_HSE | XHCI_USBSTS_HCE))) continue; xhci->InterruptCount++; XhciWrite32(xhci->Op, XHCI_OP_USBSTS, status & XHCI_USBSTS_CLEAR_MASK); XhciAckInterrupter0(xhci); KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); } } // Implements the bios Handoff operation. KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) { uint32_t xecp = (xhci->Hcc1 >> 16) & 0xFFFFu; if (!xecp) return STATUS_SUCCESS; for (uint32_t guard = 0; xecp && guard < 64; guard++) { uint32_t off = xecp << 2; if (off + 8u > xhci->MmioBytes) break; uint32_t cap = XhciRead32(xhci->Mmio, off); uint8_t id = (uint8_t)(cap & 0xFFu); uint8_t next = (uint8_t)((cap >> 8) & 0xFFu); if (id == XHCI_EXT_CAP_LEGACY) { UsbTrace("usb: xHCI legacy-support cap at +0x%x legsup=%08x\n", off, cap); if (cap & XHCI_LEGSUP_BIOS_OWNED) { XhciWrite32(xhci->Mmio, off, cap | XHCI_LEGSUP_OS_OWNED); bool ok = false; for (uint32_t i = 0; i < 1000000u; i++) { uint32_t cur = XhciRead32(xhci->Mmio, off); if ((cur & XHCI_LEGSUP_BIOS_OWNED) == 0 && (cur & XHCI_LEGSUP_OS_OWNED)) { ok = true; break; } UsbDelayUs(xhci->Hal, 1); } if (!ok) { DbgPrint("usb: xHCI BIOS handoff timed out; continuing anyway\n"); } } // Clear legacy SMI enables/status. RW1C status bits tolerate this. XhciWrite32(xhci->Mmio, off + 4u, 0xFFFFFFFFu); return STATUS_SUCCESS; } xecp = next; } return STATUS_SUCCESS; } // Implements the stop And Reset operation. KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) { uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); if (cmd & XHCI_USBCMD_RS) { XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS); if (!UsbWaitSet32(xhci->Hal, xhci->Op, XHCI_OP_USBSTS, XHCI_USBSTS_HCH, 1000000u)) { DbgPrint("usb: xHCI stop timeout usbsts=%08x\n", XhciRead32(xhci->Op, XHCI_OP_USBSTS)); return STATUS_TIMEOUT; } } XhciWrite32(xhci->Op, XHCI_OP_USBCMD, XHCI_USBCMD_HCRST); if (!UsbWaitClear32(xhci->Hal, xhci->Op, XHCI_OP_USBCMD, XHCI_USBCMD_HCRST, 1000000u)) { DbgPrint("usb: xHCI reset timeout usbcmd=%08x\n", XhciRead32(xhci->Op, XHCI_OP_USBCMD)); return STATUS_TIMEOUT; } if (!UsbWaitClear32(xhci->Hal, xhci->Op, XHCI_OP_USBSTS, XHCI_USBSTS_CNR, 1000000u)) { DbgPrint("usb: xHCI controller-not-ready timeout usbsts=%08x\n", XhciRead32(xhci->Op, XHCI_OP_USBSTS)); return STATUS_TIMEOUT; } XhciWrite32(xhci->Op, XHCI_OP_USBSTS, XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_CLEAR_MASK); return STATUS_SUCCESS; } // Creates scratchpads. KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { if (xhci->ScratchpadCount == 0) return STATUS_SUCCESS; if (xhci->ScratchpadCount > XHCI_MAX_SCRATCHPADS) { DbgPrint("usb: xHCI wants %u scratchpads, capped driver supports %u\n", xhci->ScratchpadCount, XHCI_MAX_SCRATCHPADS); return STATUS_NOT_SUPPORTED; } xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayPhys); if (!xhci->ScratchpadArray) { DbgPrint("usb: xHCI scratchpad pointer-array allocation failed count=%u\n", xhci->ScratchpadCount); return STATUS_NO_MEMORY; } for (uint16_t i = 0; i < xhci->ScratchpadCount; i++) { void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadPhys[i]); if (!page) { DbgPrint("usb: xHCI scratchpad buffer allocation failed index=%u/%u\n", i, xhci->ScratchpadCount); return STATUS_NO_MEMORY; } (void)page; xhci->ScratchpadArray[i] = xhci->ScratchpadPhys[i]; } xhci->Dcbaa[0] = xhci->ScratchpadArrayPhys; UsbTrace("usb: xHCI allocated %u scratchpad buffer(s) array=0x%016llX\n", xhci->ScratchpadCount, (unsigned long long)xhci->ScratchpadArrayPhys); return STATUS_SUCCESS; } // Creates runtime Memory. KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys); xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys); xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingPhys); xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstPhys); if (!xhci->Dcbaa || !xhci->CommandRing || !xhci->EventRing || !xhci->Erst) { DbgPrint("usb: xHCI runtime DMA allocation failed dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", xhci->Dcbaa, xhci->CommandRing, xhci->EventRing, xhci->Erst); return STATUS_NO_MEMORY; } xhci->CommandEnqueue = 0; xhci->CommandCycle = 1; xhci->EventDequeue = 0; xhci->EventCycle = 1; XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; link->ParameterLo = (uint32_t)xhci->CommandRingPhys; link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | xhci->CommandCycle; xhci->Erst[0].RingSegmentBase = xhci->EventRingPhys; xhci->Erst[0].RingSegmentSize = XHCI_EVENT_TRBS; xhci->Erst[0].Reserved = 0; KSTATUS st = XhciAllocateScratchpads(xhci); if (!KSUCCESS(st)) return st; UsbTrace("usb: xHCI dma dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", (unsigned long long)xhci->DcbaaPhys, (unsigned long long)xhci->CommandRingPhys, (unsigned long long)xhci->EventRingPhys, (unsigned long long)xhci->ErstPhys); return STATUS_SUCCESS; } // Implements the program Runtime operation. KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) { uint8_t slots = xhci->MaxSlots; if (slots == 0) return STATUS_DEVICE_ERROR; XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaPhys); XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingPhys | xhci->CommandCycle); XhciWrite32(xhci->Op, XHCI_OP_DNCTRL, 0); XhciWrite32(xhci->Op, XHCI_OP_CONFIG, slots); volatile uint8_t* ir0 = xhci->Runtime + 0x20u; XhciWrite32(ir0, 0x00u, XHCI_IMAN_IP); // clear pending, leave disabled until smoke test passes XhciWrite32(ir0, 0x04u, 0); // no interrupt moderation yet XhciWrite32(ir0, 0x08u, 1); // one ERST entry XhciWrite64(ir0, 0x10u, xhci->ErstPhys); XhciWrite64(ir0, 0x18u, xhci->EventRingPhys); XhciBarrier(); return STATUS_SUCCESS; } // Implements the command Trb Phys operation. uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) { return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB); } static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys) { if (!xhci || !xhci->CommandRing) return STATUS_INVALID_PARAMETER; if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) { XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; link->ParameterLo = (uint32_t)xhci->CommandRingPhys; link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | xhci->CommandCycle; xhci->CommandEnqueue = 0; xhci->CommandCycle ^= 1u; } uint32_t index = xhci->CommandEnqueue++; XHCI_TRB* trb = &xhci->CommandRing[index]; uint64_t trbPhys = XhciCommandTrbPhys(xhci, index); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); trb->Status = status; XhciBarrier(); trb->Control = controlWithoutCycle | xhci->CommandCycle; if (outPhys) *outPhys = trbPhys; return STATUS_SUCCESS; } KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, XHCI_COMMAND_RESULT* outResult) { if (!xhci) return STATUS_INVALID_PARAMETER; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock); uint64_t trbPhys = 0; KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys); if (!KSUCCESS(st)) { KeReleaseSpinlock(&xhci->CommandLock, cmdFlags); return st; } target.CommandPhys = trbPhys; XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); XhciWrite32(xhci->Doorbells, 0, 0); // host-controller doorbell, command ring KeReleaseSpinlock(&xhci->CommandLock, cmdFlags); KSTATUS waitStatus = XhciWaitForArmedTarget(xhci, &target, 500000u); if (KSUCCESS(waitStatus) && target.CommandMatched) { if (outResult) *outResult = target.Command; return (target.Command.CompletionCode == XHCI_CC_SUCCESS) ? STATUS_SUCCESS : STATUS_DEVICE_ERROR; } DbgPrint("usb: xHCI command timeout type=%u usbsts=%08x crcr=0x%016llX\n", (controlWithoutCycle >> XHCI_TRB_TYPE_SHIFT) & XHCI_TRB_TYPE_MASK, XhciRead32(xhci->Op, XHCI_OP_USBSTS), (unsigned long long)XhciRead64(xhci->Op, XHCI_OP_CRCR)); return waitStatus; } // Submits noop Command. KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci) { XHCI_COMMAND_RESULT result; KSTATUS st = XhciSubmitCommand( xhci, 0, 0, (XHCI_TRB_TYPE_NOOP_CMD << XHCI_TRB_TYPE_SHIFT), &result); if (KSUCCESS(st)) { UsbTrace("usb: xHCI No-Op command completed successfully\n"); return STATUS_SUCCESS; } DbgPrint("usb: xHCI No-Op command failed st=0x%x cc=%u\n", st, result.CompletionCode); return st; } // Enables slot. KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) { XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); KSTATUS st = XhciSubmitCommand( xhci, 0, 0, (XHCI_TRB_TYPE_ENABLE_SLOT << XHCI_TRB_TYPE_SHIFT), &result); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI Enable Slot failed st=0x%x cc=%u\n", st, result.CompletionCode); return st; } if (result.SlotId == 0 || result.SlotId > XHCI_MAX_TRACKED_SLOTS) { DbgPrint("usb: xHCI Enable Slot returned unsupported slot=%u\n", result.SlotId); return STATUS_DEVICE_ERROR; } *outSlotId = result.SlotId; UsbTrace("usb: xHCI Enable Slot -> slot %u\n", result.SlotId); return STATUS_SUCCESS; } // Disables slot. KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) { if (!xhci || slotId == 0) return STATUS_INVALID_PARAMETER; XHCI_COMMAND_RESULT res; memset(&res, 0, sizeof(res)); KSTATUS st = XhciSubmitCommand(xhci, 0, 0, (XHCI_TRB_TYPE_DISABLE_SLOT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)slotId << 24), &res); if (!KSUCCESS(st)) return st; if (res.CompletionCode != XHCI_CC_SUCCESS) { DbgPrint("usb: Disable Slot slot=%u failed cc=%u\n", slotId, res.CompletionCode); return STATUS_DEVICE_ERROR; } return STATUS_SUCCESS; } // Implements the address Device operation. KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); KSTATUS st = XhciSubmitCommand( xhci, dev->InputContextPhys, 0, (XHCI_TRB_TYPE_ADDRESS_DEVICE << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), &result); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI Address Device slot=%u failed st=0x%x cc=%u\n", dev->SlotId, st, result.CompletionCode); return st; } UsbTrace("usb: xHCI Address Device slot=%u complete\n", dev->SlotId); return STATUS_SUCCESS; } // Starts the requested operation. KSTATUS XhciStart(XHCI_CONTROLLER* xhci) { uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); cmd |= XHCI_USBCMD_RS; XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd); if (!UsbWaitClear32(xhci->Hal, xhci->Op, XHCI_OP_USBSTS, XHCI_USBSTS_HCH, 1000000u)) { DbgPrint("usb: xHCI start timeout usbsts=%08x\n", XhciRead32(xhci->Op, XHCI_OP_USBSTS)); return STATUS_TIMEOUT; } xhci->Running = true; return STATUS_SUCCESS; } // Enables interrupts. void XhciEnableInterrupts(XHCI_CONTROLLER* xhci) { volatile uint8_t* ir0 = xhci->Runtime + 0x20u; XhciWrite32(xhci->Op, XHCI_OP_USBSTS, XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_CLEAR_MASK); XhciWrite32(ir0, 0x00u, XHCI_IMAN_IE | XHCI_IMAN_IP); uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd | XHCI_USBCMD_INTE | XHCI_USBCMD_HSEE); } // Implements the port Reg operation. uint32_t XhciPortReg(uint8_t port) { return XHCI_OP_PORT_BASE + ((uint32_t)port - 1u) * XHCI_OP_PORT_STRIDE; } // Clears port Changes. void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port) { uint32_t off = XhciPortReg(port); uint32_t portsc = XhciRead32(xhci->Op, off); uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK; if (changes) { // PORTSC is a mixed register. The change bits are RW1C, but many // other bits are either RO or have side effects when written as one // (PED disables the port; PR starts a reset). Preserve only Port // Power and write one to the RW1C change bits. XhciWrite32(xhci->Op, off, (portsc & XHCI_PORTSC_PP) | changes); } } // Implements the reset Port operation. KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) { uint32_t off = XhciPortReg(port); uint32_t portsc = XhciRead32(xhci->Op, off); if (!(portsc & XHCI_PORTSC_CCS)) return STATUS_NOT_FOUND; uint32_t speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; uint8_t protocol = XhciPortProtocolMajor(xhci, port, portsc); XhciClearPortChanges(xhci, port); portsc = XhciRead32(xhci->Op, off); if (protocol >= 3u || speed >= 4u) { // SuperSpeed ports are enabled by link training. Do not issue a // USB2-style PORTSC.PR reset; on real Intel controllers that can make // the link drop during early enumeration. If firmware/controller // already has PR set, just wait for it to settle. bool enabled = false; for (uint32_t i = 0; i < 1000000u; i++) { portsc = XhciRead32(xhci->Op, off); if (!(portsc & XHCI_PORTSC_CCS)) break; if (!(portsc & XHCI_PORTSC_PR) && (portsc & XHCI_PORTSC_PED)) { enabled = true; break; } XhciDrainEventRingLocked(xhci, NULL); UsbDelayUs(xhci->Hal, 1); } XhciClearPortChanges(xhci, port); portsc = XhciRead32(xhci->Op, off); speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; if (outSpeed) *outSpeed = (uint8_t)speed; UsbTrace("usb: xHCI port %u link ready portsc=%08x ped=%u proto=usb%u speed=%s(%u)\n", port, portsc, (portsc & XHCI_PORTSC_PED) ? 1u : 0u, protocol, XhciPortSpeedName(speed), speed); if (!enabled || !(portsc & XHCI_PORTSC_CCS) || !(portsc & XHCI_PORTSC_PED)) return STATUS_DEVICE_ERROR; return STATUS_SUCCESS; } uint32_t reset = (portsc & XHCI_PORTSC_PP) | XHCI_PORTSC_PR; XhciWrite32(xhci->Op, off, reset); bool resetDone = false; for (uint32_t i = 0; i < 500000u; i++) { portsc = XhciRead32(xhci->Op, off); if (!(portsc & XHCI_PORTSC_PR) && (portsc & XHCI_PORTSC_PRC)) { resetDone = true; break; } XhciDrainEventRingLocked(xhci, NULL); UsbDelayUs(xhci->Hal, 1); } if (!resetDone) { DbgPrint("usb: xHCI port %u reset timeout portsc=%08x\n", port, portsc); return STATUS_TIMEOUT; } XhciClearPortChanges(xhci, port); portsc = XhciRead32(xhci->Op, off); speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; if (outSpeed) *outSpeed = (uint8_t)speed; UsbTrace("usb: xHCI port %u reset complete portsc=%08x ped=%u proto=usb%u speed=%s(%u)\n", port, portsc, (portsc & XHCI_PORTSC_PED) ? 1u : 0u, protocol, XhciPortSpeedName(speed), speed); if (!(portsc & XHCI_PORTSC_CCS) || !(portsc & XHCI_PORTSC_PED)) return STATUS_DEVICE_ERROR; return STATUS_SUCCESS; } // Creates device Memory. KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys); dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys); dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingPhys); dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorPhys); if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) return STATUS_NO_MEMORY; dev->Ep0Enqueue = 0; dev->Ep0Cycle = 1; XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; link->ParameterLo = (uint32_t)dev->Ep0RingPhys; link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | dev->Ep0Cycle; return STATUS_SUCCESS; } // Builds default Control Context. void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { memset(dev->InputContext, 0, PAGE_SIZE); memset(dev->DeviceContext, 0, PAGE_SIZE); uint32_t* icc = XhciInputControlContext(xhci, dev); uint32_t* slot = XhciInputSlotContext(xhci, dev); uint32_t* ep0 = XhciInputEndpointContext(xhci, dev, 1); icc[0] = 0; // drop flags icc[1] = 0x3u; // add slot context and endpoint 0 context uint32_t maxPacket = XhciEpMaxPacket(dev->Speed); dev->MaxPacketSize0 = (uint16_t)maxPacket; uint8_t rootPort = dev->Path.RootPort ? dev->Path.RootPort : dev->Port; uint32_t routeString = dev->Path.RouteString & 0x000fffffu; slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed) | routeString; // Slot context dword 1 owns Root Hub Port Number bits 23:16. // Putting the port number in dword 2 makes Address Device fail with a // context/TRB error on QEMU xHCI. slot[1] = XHCI_SLOT_ROOT_PORT(rootPort); slot[2] = 0; if (dev->Path.BehindHub && dev->Speed <= 2u && dev->Path.ParentHubSlot) { // TT Hub Slot ID bits 7:0 and TT Port Number bits 15:8. // This is required for low/full-speed devices behind high-speed hubs; // harmless for older full-speed hub paths on the controllers we target. slot[2] = (uint32_t)dev->Path.ParentHubSlot | ((uint32_t)dev->Path.ParentHubPort << 8); } slot[3] = 0; ep0[0] = 0; ep0[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_CONTROL | (maxPacket << 16); ep0[2] = (uint32_t)(dev->Ep0RingPhys | XHCI_EP_DCS); ep0[3] = (uint32_t)((dev->Ep0RingPhys | XHCI_EP_DCS) >> 32); ep0[4] = 8; xhci->Dcbaa[dev->SlotId] = dev->DeviceContextPhys; XhciBarrier(); } // Implements the ep0 Trb Phys operation. uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) { return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB); } KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys) { if (!dev || !dev->Ep0Ring) return STATUS_INVALID_PARAMETER; if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) { XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; link->ParameterLo = (uint32_t)dev->Ep0RingPhys; link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | dev->Ep0Cycle; dev->Ep0Enqueue = 0; dev->Ep0Cycle ^= 1u; } uint32_t index = dev->Ep0Enqueue++; XHCI_TRB* trb = &dev->Ep0Ring[index]; uint64_t phys = XhciEp0TrbPhys(dev, index); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); trb->Status = status; XhciBarrier(); trb->Control = controlWithoutCycle | dev->Ep0Cycle; if (outPhys) *outPhys = phys; return STATUS_SUCCESS; } KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs) { if (!buffer || length == 0 || length > PAGE_SIZE) return STATUS_INVALID_PARAMETER; memset(buffer, 0, length); uint64_t setupValue = UsbSetupPacketToU64(setup); KSTATUS st = XhciEp0AppendTrb( dev, setupValue, 8, (XHCI_TRB_TYPE_SETUP << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_IDT | XHCI_TRB_SETUP_TRT_IN, NULL); if (!KSUCCESS(st)) return st; st = XhciEp0AppendTrb( dev, bufferPhys, length, (XHCI_TRB_TYPE_DATA << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_DATA_DIR_IN, NULL); if (!KSUCCESS(st)) return st; uint64_t statusTrbPhys = 0; st = XhciEp0AppendTrb( dev, 0, 0, (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_IOC, &statusTrbPhys); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); target.TransferPhys = statusTrbPhys; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0 if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US; st = XhciWaitForArmedTarget(xhci, &target, timeoutUs); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI control IN timeout slot=%u len=%u timeout=%uus\n", dev->SlotId, length, timeoutUs); return st; } if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS) return STATUS_SUCCESS; DbgPrint("usb: xHCI control IN failed slot=%u cc=%u residue=%u\n", dev->SlotId, target.Transfer.CompletionCode, target.Transfer.Residue); return STATUS_DEVICE_ERROR; } KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length) { return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length, USB_XHCI_CONTROL_TIMEOUT_US); } KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup) { uint64_t setupValue = UsbSetupPacketToU64(setup); KSTATUS st = XhciEp0AppendTrb( dev, setupValue, 8, (XHCI_TRB_TYPE_SETUP << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_IDT | XHCI_TRB_SETUP_TRT_NONE, NULL); if (!KSUCCESS(st)) return st; uint64_t statusTrbPhys = 0; st = XhciEp0AppendTrb( dev, 0, 0, (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_STATUS_DIR_IN | XHCI_TRB_IOC, &statusTrbPhys); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); target.TransferPhys = statusTrbPhys; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); st = XhciWaitForArmedTarget(xhci, &target, 1000000u); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI control no-data timeout slot=%u req=%u value=%04x\n", dev->SlotId, setup->bRequest, setup->wValue); return st; } if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS) return STATUS_SUCCESS; DbgPrint("usb: xHCI control no-data failed slot=%u cc=%u residue=%u\n", dev->SlotId, target.Transfer.CompletionCode, target.Transfer.Residue); return STATUS_DEVICE_ERROR; } KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs) { USB_SETUP_PACKET setup; setup.bmRequestType = USB_DIR_IN; setup.bRequest = USB_REQ_GET_DESCRIPTOR; setup.wValue = (uint16_t)(USB_DESC_DEVICE << 8); setup.wIndex = 0; setup.wLength = sizeof(USB_DEVICE_DESCRIPTOR); if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US; KSTATUS st = XhciControlInTimeout(xhci, dev, &setup, dev->DescriptorBuffer, dev->DescriptorPhys, sizeof(USB_DEVICE_DESCRIPTOR), timeoutUs); if (!KSUCCESS(st)) return st; memcpy(outDesc, dev->DescriptorBuffer, sizeof(*outDesc)); if (outDesc->bLength < sizeof(USB_DEVICE_DESCRIPTOR) || outDesc->bDescriptorType != USB_DESC_DEVICE) { DbgPrint("usb: invalid device descriptor slot=%u len=%u type=%u\n", dev->SlotId, outDesc->bLength, outDesc->bDescriptorType); return STATUS_DEVICE_ERROR; } return STATUS_SUCCESS; } KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc) { return XhciGetDeviceDescriptorTimeout(xhci, dev, outDesc, USB_XHCI_CONTROL_TIMEOUT_US); } KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength) { USB_SETUP_PACKET setup; setup.bmRequestType = USB_DIR_IN; setup.bRequest = USB_REQ_GET_DESCRIPTOR; setup.wValue = (uint16_t)((USB_DESC_CONFIGURATION << 8) | configurationIndex); setup.wIndex = 0; setup.wLength = sizeof(USB_CONFIGURATION_DESCRIPTOR); KSTATUS st = XhciControlIn(xhci, dev, &setup, dev->DescriptorBuffer, dev->DescriptorPhys, sizeof(USB_CONFIGURATION_DESCRIPTOR)); if (!KSUCCESS(st)) return st; memcpy(outConfig, dev->DescriptorBuffer, sizeof(*outConfig)); if (outConfig->bLength < sizeof(USB_CONFIGURATION_DESCRIPTOR) || outConfig->bDescriptorType != USB_DESC_CONFIGURATION || outConfig->wTotalLength < sizeof(USB_CONFIGURATION_DESCRIPTOR)) { DbgPrint("usb: invalid config descriptor slot=%u index=%u len=%u type=%u total=%u\n", dev->SlotId, configurationIndex, outConfig->bLength, outConfig->bDescriptorType, outConfig->wTotalLength); return STATUS_DEVICE_ERROR; } uint16_t total = outConfig->wTotalLength; if (total > PAGE_SIZE) { DbgPrint("usb: config descriptor slot=%u index=%u total=%u too large, capped at %u\n", dev->SlotId, configurationIndex, total, (uint32_t)PAGE_SIZE); total = (uint16_t)PAGE_SIZE; } setup.wLength = total; st = XhciControlIn(xhci, dev, &setup, dev->DescriptorBuffer, dev->DescriptorPhys, total); if (!KSUCCESS(st)) return st; memcpy(outConfig, dev->DescriptorBuffer, sizeof(*outConfig)); if (outConfig->bLength < sizeof(USB_CONFIGURATION_DESCRIPTOR) || outConfig->bDescriptorType != USB_DESC_CONFIGURATION) { DbgPrint("usb: invalid full config descriptor slot=%u index=%u len=%u type=%u\n", dev->SlotId, configurationIndex, outConfig->bLength, outConfig->bDescriptorType); return STATUS_DEVICE_ERROR; } if (outTotalLength) *outTotalLength = total; return STATUS_SUCCESS; } // Implements the endpoint Id From Address operation. uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress) { uint8_t number = endpointAddress & USB_ENDPOINT_NUMBER_MASK; if (number == 0) return 1; return (uint8_t)(number * 2u + ((endpointAddress & USB_ENDPOINT_DIR_IN) ? 1u : 0u)); } // Implements the interrupt Interval operation. uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) { if (usbInterval == 0) usbInterval = 1; if (speed == 3 || speed == 4 || speed == 5) { // High-speed and newer interrupt endpoints already encode bInterval as // an exponent in USB descriptors. xHCI stores the zero-based exponent. if (usbInterval > 16) usbInterval = 16; return (uint8_t)(usbInterval - 1u); } // Full/low-speed interrupt endpoints express bInterval in milliseconds. // xHCI wants the nearest exponent in 125us units. uint32_t units125us = (uint32_t)usbInterval * 8u; uint8_t interval = 0; uint32_t v = 1; while (v < units125us && interval < 15u) { v <<= 1; interval++; } return interval; } KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys) { if (!ring || !enqueue || !cycle || trbCount < 2) return STATUS_INVALID_PARAMETER; if (*enqueue >= trbCount - 1u) { XHCI_TRB* link = &ring[trbCount - 1u]; link->ParameterLo = (uint32_t)ringPhys; link->ParameterHi = (uint32_t)(ringPhys >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | *cycle; *enqueue = 0; *cycle ^= 1u; } uint32_t index = (*enqueue)++; XHCI_TRB* trb = &ring[index]; uint64_t phys = ringPhys + (uint64_t)index * sizeof(XHCI_TRB); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); trb->Status = status; XhciBarrier(); trb->Control = controlWithoutCycle | *cycle; if (outPhys) *outPhys = phys; return STATUS_SUCCESS; } void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle) { memset(ring, 0, trbCount * sizeof(XHCI_TRB)); XHCI_TRB* link = &ring[trbCount - 1u]; link->ParameterLo = (uint32_t)ringPhys; link->ParameterHi = (uint32_t)(ringPhys >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | cycle; } KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual) { if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferPhys || length == 0) return STATUS_INVALID_PARAMETER; if (length > USB_STORAGE_MAX_BYTES) return STATUS_INVALID_PARAMETER; uint64_t trbPhys = 0; KSTATUS st = XhciRingAppend(ring, ringPhys, XHCI_BULK_RING_TRBS, enqueue, cycle, bufferPhys, length, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, &trbPhys); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); target.TransferPhys = trbPhys; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = endpointId; XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, endpointId); st = XhciWaitForArmedTarget(xhci, &target, 1000000u); if (!KSUCCESS(st)) { DbgPrint("usb-storage: bulk transfer timeout slot=%u ep=%u len=%u\n", dev->SlotId, endpointId, length); return st; } if (target.Transfer.CompletionCode != XHCI_CC_SUCCESS) { DbgPrint("usb-storage: bulk transfer failed slot=%u ep=%u cc=%u residue=%u\n", dev->SlotId, endpointId, target.Transfer.CompletionCode, target.Transfer.Residue); return STATUS_DEVICE_ERROR; } if (outActual) *outActual = length - target.Transfer.Residue; return STATUS_SUCCESS; } // Implements the configure Pipe Endpoint operation. KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) { if (!xhci || !dev || !pipe || !pipe->Open) return STATUS_INVALID_PARAMETER; uint8_t dci = pipe->EndpointId; if (dci < 2 || dci > 31) return STATUS_INVALID_PARAMETER; if (pipe->HcdPipeContext) { XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; return ctx->Configured ? STATUS_SUCCESS : STATUS_DEVICE_ERROR; } XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)MmAllocatePool(sizeof(XHCI_PIPE_CONTEXT)); if (!ctx) return STATUS_NO_MEMORY; memset(ctx, 0, sizeof(*ctx)); ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys); if (!ctx->Ring) { MmFreePool(ctx); return STATUS_NO_MEMORY; } ctx->Enqueue = 0; ctx->Cycle = 1; ctx->EndpointId = dci; ctx->Type = pipe->Type; XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle); memset(dev->InputContext, 0, PAGE_SIZE); uint32_t* icc = XhciInputControlContext(xhci, dev); uint32_t* slot = XhciInputSlotContext(xhci, dev); uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci); icc[0] = 0; icc[1] = (1u << 0) | (1u << dci); slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed); slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port); slot[2] = 0; slot[3] = 0; uint16_t maxPacket = pipe->MaxPacketSize & 0x07FFu; if (maxPacket == 0) maxPacket = 8u; uint32_t endpointType = 0; uint8_t interval = 0; if (pipe->Type == UsbPipeInterrupt && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) { endpointType = XHCI_EP_TYPE_INT_IN; interval = XhciInterruptInterval(dev->Speed, pipe->Interval); } else if (pipe->Type == UsbPipeBulk && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) { endpointType = XHCI_EP_TYPE_BULK_IN; } else if (pipe->Type == UsbPipeBulk) { endpointType = XHCI_EP_TYPE_BULK_OUT; } else { MmFreePool(ctx); return STATUS_NOT_SUPPORTED; } ep[0] = (uint32_t)interval << 16; ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16); ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS); ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32); ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16); XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); KSTATUS st = XhciSubmitCommand( xhci, dev->InputContextPhys, 0, (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), &result); if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) { DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n", dev->SlotId, dci, pipe->Type, st, result.CompletionCode); MmFreePool(ctx); return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st; } ctx->Configured = true; pipe->HcdPipeContext = ctx; dev->EndpointPipes[dci] = pipe; UsbReferencePipe(pipe); return STATUS_SUCCESS; } // Implements the hcd Submit Interrupt Urb operation. static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) return STATUS_INVALID_PARAMETER; if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) return STATUS_NOT_SUPPORTED; KSTATUS st = XhciConfigurePipeEndpoint(xhci, dev, urb->Pipe); if (!KSUCCESS(st)) return st; XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext; if (!ctx || !ctx->Configured) return STATUS_DEVICE_ERROR; if (ctx->ActiveTransfer) return STATUS_QUEUE_FULL; uint64_t trbPhys = 0; st = XhciRingAppend(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, &ctx->Enqueue, &ctx->Cycle, urb->BufferPhys, urb->Length, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, &trbPhys); if (!KSUCCESS(st)) return st; ctx->ActiveTransfer = true; ctx->ActiveTrbPhys = trbPhys; ctx->ActiveUrb = urb; urb->Pending = true; urb->Cancelled = false; urb->Status = STATUS_PENDING; urb->ActualLength = 0; urb->HcdCompletionCode = 0; XhciBarrier(); XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, ctx->EndpointId); return STATUS_PENDING; } void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue) { if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32) return; XHCI_DEVICE* dev = &xhci->Devices[slotId]; USB_PIPE* pipe = dev->EndpointPipes[endpointId]; if (!pipe || !pipe->HcdPipeContext) return; XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys) return; USB_URB* urb = ctx->ActiveUrb; ctx->ActiveTransfer = false; ctx->ActiveTrbPhys = 0; ctx->ActiveUrb = (USB_URB*)0; if (!urb) return; KSTATUS st = STATUS_SUCCESS; if (urb->Cancelled) { st = STATUS_CANCELLED; } else if (completionCode != XHCI_CC_SUCCESS && completionCode != XHCI_CC_SHORT_PACKET) { st = STATUS_DEVICE_ERROR; } uint32_t actual = 0; if (KSUCCESS(st)) { actual = (residue <= urb->Length) ? (urb->Length - residue) : 0; } urb->Pending = false; urb->Status = st; urb->ActualLength = actual; urb->HcdCompletionCode = completionCode; IRP* completeIrp = urb->Irp; urb->Irp = (IRP*)0; if (urb->Complete) urb->Complete(urb, urb->CompleteContext); if (completeIrp) IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0); } // Implements the hcd Submit Bulk Urb operation. static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) return STATUS_INVALID_PARAMETER; uint8_t endpointId = urb->Pipe->EndpointId; XHCI_TRB* ring = (XHCI_TRB*)0; PHYSICAL_ADDRESS ringPhys = 0; uint32_t* enqueue = (uint32_t*)0; uint8_t* cycle = (uint8_t*)0; if (endpointId == dev->MassBulkInEndpointId) { ring = dev->MassInRing; ringPhys = dev->MassInRingPhys; enqueue = &dev->MassInEnqueue; cycle = &dev->MassInCycle; } else if (endpointId == dev->MassBulkOutEndpointId) { ring = dev->MassOutRing; ringPhys = dev->MassOutRingPhys; enqueue = &dev->MassOutEnqueue; cycle = &dev->MassOutCycle; } else { return STATUS_NOT_SUPPORTED; } uint32_t actual = 0; KSTATUS st = XhciBulkTransfer(xhci, dev, endpointId, ring, ringPhys, enqueue, cycle, urb->BufferPhys, urb->Length, &actual); urb->ActualLength = KSUCCESS(st) ? actual : 0; return st; } // Implements the hcd Submit Urb operation. static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) { if (!hcd || !urb || !urb->Device) return STATUS_INVALID_PARAMETER; XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext; XHCI_DEVICE* dev = (XHCI_DEVICE*)urb->Device->HcdDeviceContext; if (!xhci || !dev || !dev->Allocated) return STATUS_INVALID_PARAMETER; KSTATUS st; switch (urb->Function) { case UrbFunctionControlTransfer: if (urb->Length == 0) { st = XhciControlNoData(xhci, dev, &urb->Setup); urb->ActualLength = 0; } else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) { st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length); urb->ActualLength = KSUCCESS(st) ? urb->Length : 0; } else { st = STATUS_NOT_SUPPORTED; urb->ActualLength = 0; } break; case UrbFunctionBulkTransfer: st = XhciHcdSubmitBulkUrb(xhci, dev, urb); break; case UrbFunctionInterruptTransfer: st = XhciHcdSubmitInterruptUrb(xhci, dev, urb); break; default: st = STATUS_NOT_SUPPORTED; urb->ActualLength = 0; break; } urb->Status = st; return st; } // Implements the hcd Cancel Urb operation. static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) { (void)hcd; if (!urb) return STATUS_INVALID_PARAMETER; urb->Cancelled = true; if (!urb->Pending) urb->Status = STATUS_CANCELLED; return STATUS_SUCCESS; } // Implements the hcd Allocate Dma operation. static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) { if (!hcd) return (void*)0; return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys); } // Implements the hcd Reset Root Port operation. static KSTATUS XhciHcdResetRootPort(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed) { if (!hcd) return STATUS_INVALID_PARAMETER; return XhciResetPort((XHCI_CONTROLLER*)hcd->ControllerContext, port, outSpeed); } // Implements the hcd Clear Root Port Changes operation. static KSTATUS XhciHcdClearRootPortChanges(USB_HCD* hcd, uint8_t port) { if (!hcd) return STATUS_INVALID_PARAMETER; XhciClearPortChanges((XHCI_CONTROLLER*)hcd->ControllerContext, port); return STATUS_SUCCESS; } const USB_HCD_OPERATIONS gXhciHcdOperations = { XhciHcdSubmitUrb, XhciHcdCancelUrb, XhciHcdResetRootPort, XhciHcdClearRootPortChanges, XhciHcdAllocateDma, }; // Starts controller. KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { if (!xhci || xhci->Magic != XHCI_CONTROLLER_MAGIC || !xhci->Pdo || !xhci->Fdo) return STATUS_INVALID_PARAMETER; if (xhci->Running) return STATUS_SUCCESS; KSTATUS st = PciGetDeviceInfo(xhci->Pdo, &xhci->PciInfo); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI PciGetDeviceInfo failed st=0x%x\n", st); return st; } xhci->Bus = xhci->PciInfo.Address.Bus; xhci->Dev = xhci->PciInfo.Address.Device; xhci->Fn = xhci->PciInfo.Address.Function; st = PciGetBar(xhci->Pdo, 0, &xhci->MmioBar); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI PCI %02x:%02x.%x BAR0 missing st=0x%x\n", xhci->Bus, xhci->Dev, xhci->Fn, st); return st; } if (xhci->MmioBar.Type != PciBarMemory32 && xhci->MmioBar.Type != PciBarMemory64) { DbgPrint("usb: xHCI PCI %02x:%02x.%x BAR0 is not MMIO type=%u\n", xhci->Bus, xhci->Dev, xhci->Fn, xhci->MmioBar.Type); return STATUS_DEVICE_ERROR; } xhci->Irq = 0xFFu; xhci->UsingMsi = false; st = PciEnableMsiAuto(xhci->Pdo, &xhci->Irq); if (KSUCCESS(st)) { xhci->UsingMsi = true; DbgPrint("usb: xHCI using MSI IRQ slot %u\n", xhci->Irq); } else { KSTATUS intxStatus = PciGetInterrupt(xhci->Pdo, &xhci->Irq); if (KSUCCESS(intxStatus) && xhci->Irq != 0xFFu) { DbgPrint("usb: xHCI using legacy INTx IRQ %u\n", xhci->Irq); } else { xhci->Irq = 0xFFu; DbgPrint("usb: xHCI no MSI/INTx interrupt available; using polled event-ring mode (msi=0x%x intx=0x%x)\n", st, intxStatus); } } uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER; if (xhci->UsingMsi || xhci->Irq == 0xFFu) enableFlags |= PCI_DISABLE_INTX; else enableFlags |= PCI_ENABLE_INTX; st = PciEnableDevice(xhci->Pdo, enableFlags); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI PciEnableDevice failed st=0x%x\n", st); return st; } xhci->MmioPhys = xhci->MmioBar.Base; uint64_t mappedLength = 0; st = PciMapBar(xhci->Pdo, 0, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, (void**)&xhci->Mmio, &mappedLength); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI PciMapBar(BAR0) failed st=0x%x\n", st); return st; } xhci->MmioBytes = (mappedLength > 0xFFFFFFFFull) ? 0xFFFFFFFFu : (uint32_t)mappedLength; xhci->Cap = xhci->Mmio; uint32_t cap0 = XhciRead32(xhci->Cap, XHCI_CAP_CAPLENGTH); xhci->CapLength = (uint8_t)(cap0 & 0xFFu); xhci->Version = (uint16_t)(cap0 >> 16); xhci->Hcs1 = XhciRead32(xhci->Cap, XHCI_CAP_HCSPARAMS1); xhci->Hcs2 = XhciRead32(xhci->Cap, XHCI_CAP_HCSPARAMS2); xhci->Hcc1 = XhciRead32(xhci->Cap, XHCI_CAP_HCCPARAMS1); xhci->DbOff = XhciRead32(xhci->Cap, XHCI_CAP_DBOFF) & ~0x3u; xhci->RtsOff = XhciRead32(xhci->Cap, XHCI_CAP_RTSOFF) & ~0x1Fu; xhci->MaxSlots = (uint8_t)(xhci->Hcs1 & 0xFFu); xhci->MaxIntrs = (uint16_t)((xhci->Hcs1 >> 8) & 0x7FFu); xhci->MaxPorts = (uint8_t)((xhci->Hcs1 >> 24) & 0xFFu); xhci->ScratchpadCount = XhciScratchpadCount(xhci->Hcs2); xhci->ContextBytes = (xhci->Hcc1 & XHCI_HCC1_CSZ) ? 64u : 32u; if (xhci->CapLength < 0x20 || xhci->DbOff == 0 || xhci->RtsOff == 0 || xhci->MaxSlots == 0 || xhci->MaxPorts == 0) { DbgPrint("usb: xHCI invalid caps caplen=%u dboff=0x%x rtsoff=0x%x hcs1=%08x\n", xhci->CapLength, xhci->DbOff, xhci->RtsOff, xhci->Hcs1); return STATUS_DEVICE_ERROR; } uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE + (uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE; uint32_t needBytes = portEnd; if (xhci->DbOff + 0x100u > needBytes) needBytes = xhci->DbOff + 0x100u; if (xhci->RtsOff + 0x1000u > needBytes) needBytes = xhci->RtsOff + 0x1000u; needBytes = UsbAlignUp32(needBytes, (uint32_t)PAGE_SIZE); if (xhci->MmioBytes < needBytes) { DbgPrint("usb: xHCI BAR0 too small length=0x%x need=0x%x dboff=0x%x rtsoff=0x%x ports=%u\n", xhci->MmioBytes, needBytes, xhci->DbOff, xhci->RtsOff, xhci->MaxPorts); return STATUS_DEVICE_ERROR; } xhci->Op = xhci->Mmio + xhci->CapLength; xhci->Runtime = xhci->Mmio + xhci->RtsOff; xhci->Doorbells = xhci->Mmio + xhci->DbOff; UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts); xhci->Hcd.ControllerDevice = xhci->Fdo; DbgPrint("usb: xHCI init %02x:%02x.%x vendor=%04x device=%04x irq=%u mmio=0x%016llX caplen=%u version=%04x\n", xhci->Bus, xhci->Dev, xhci->Fn, xhci->PciInfo.VendorId, xhci->PciInfo.DeviceId, xhci->Irq, (unsigned long long)xhci->MmioPhys, xhci->CapLength, xhci->Version); DbgPrint("usb: xHCI caps slots=%u intrs=%u ports=%u ctx=%u scratchpads=%u hcs2=%08x hcc1=%08x dboff=0x%x rtsoff=0x%x pagesize=%08x\n", xhci->MaxSlots, xhci->MaxIntrs, xhci->MaxPorts, xhci->ContextBytes, xhci->ScratchpadCount, xhci->Hcs2, xhci->Hcc1, xhci->DbOff, xhci->RtsOff, XhciRead32(xhci->Op, XHCI_OP_PAGESIZE)); XhciParseSupportedProtocols(xhci); st = XhciBiosHandoff(xhci); if (!KSUCCESS(st)) return st; st = XhciStopAndReset(xhci); if (!KSUCCESS(st)) return st; uint32_t pageSize = XhciRead32(xhci->Op, XHCI_OP_PAGESIZE); if ((pageSize & 1u) == 0) { DbgPrint("usb: xHCI does not advertise 4K pages pagesize=%08x\n", pageSize); return STATUS_NOT_SUPPORTED; } st = XhciAllocateRuntimeMemory(xhci); if (!KSUCCESS(st)) return st; st = XhciProgramRuntime(xhci); if (!KSUCCESS(st)) return st; st = XhciStart(xhci); if (!KSUCCESS(st)) return st; st = XhciSubmitNoopCommand(xhci); if (!KSUCCESS(st)) return st; XhciPowerRootPorts(xhci); XhciLogPortStatus(xhci); st = UsbCreateRootHubPdo(&xhci->Hcd); if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) { DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st); return st; } st = XhciRegisterController(xhci); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI controller registry full st=0x%x\n", st); return st; } if (xhci->Irq != 0xFFu) { KeIrqRegister(xhci->Irq, XhciIsr); XhciEnableInterrupts(xhci); xhci->InterruptsEnabled = true; KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); DbgPrint("usb: xHCI ISR registered on IRQ %u (%s); event ring handled by DPC\n", xhci->Irq, xhci->UsingMsi ? "MSI" : "INTx"); } else { xhci->InterruptsEnabled = false; DbgPrint("usb: xHCI started in polling mode; synchronous waits drain event ring\n"); } xhci->Fdo->PnpState = DevicePnpStarted; st = UsbHubScheduleRootScan(&xhci->Hcd); if (!KSUCCESS(st)) DbgPrint("usb: root hub async scan scheduling failed st=0x%x\n", st); else DbgPrint("usb: root hub initial scan queued asynchronously\n"); DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n", xhci->HidDeviceCount); return STATUS_SUCCESS; } // Dispatches pnp. KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { if (!device || !irp || !device->DeviceExtension) return STATUS_INVALID_PARAMETER; XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)device->DeviceExtension; if (xhci->Magic != XHCI_CONTROLLER_MAGIC) return STATUS_INVALID_PARAMETER; IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); if (!ios) return STATUS_INVALID_PARAMETER; switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { case IRP_MN_START_DEVICE: return XhciStartController(xhci); case IRP_MN_QUERY_REMOVE_DEVICE: device->PnpState = DevicePnpRemovePending; return STATUS_SUCCESS; case IRP_MN_CANCEL_REMOVE_DEVICE: device->PnpState = xhci->Running ? DevicePnpStarted : DevicePnpNotStarted; return STATUS_SUCCESS; case IRP_MN_SURPRISE_REMOVAL: device->PnpState = DevicePnpSurpriseRemoved; xhci->Running = false; xhci->InterruptsEnabled = false; if (xhci->Op) { uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS); } if (xhci->Hcd.RootHubPdo) (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); return STATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: device->PnpState = DevicePnpDeleted; xhci->Running = false; xhci->InterruptsEnabled = false; if (xhci->Hcd.RootHubPdo) (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); IoDetachDevice(device); IoDeleteDevice(device); return STATUS_SUCCESS; default: return IoForwardIrpToLower(device, irp); } } // Implements the add Device operation. KSTATUS XhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) { if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical) return STATUS_INVALID_PARAMETER; DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0; KSTATUS st = IoCreateDevice(driver, sizeof(XHCI_CONTROLLER), &fdo); if (!KSUCCESS(st)) return st; (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("xhci-controller")); XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)fdo->DeviceExtension; memset(xhci, 0, sizeof(*xhci)); xhci->Magic = XHCI_CONTROLLER_MAGIC; xhci->Hal = gUsbHal; xhci->Fdo = fdo; xhci->Pdo = physicalDeviceObject; xhci->Irq = 0xFFu; KeInitializeDpc(&xhci->EventDpc, XhciEventDpcRoutine, xhci); xhci->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject); if (!xhci->LowerDevice) { IoDeleteDevice(fdo); return STATUS_FAILURE; } DbgPrint("usb: xHCI FDO attached to %v\n", physicalDeviceObject->Name); return STATUS_SUCCESS; }