maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace ad hoc driver loading with a proper driver manager, new DDK, new PnP manager with device matching, asynchronous startup, resource ownership and removal, add shared PCI capability helpers, replace typed, rigid services with a proper service manager, implement proper system shutdown and restart, shutting down unloads drivers and shuts down services, add simple power button, remove heavyweight boot self-tests and SMP stress runs, add monotonic timestamps to kernel debug output, fruitless USB and PS/2 fixes, make test builds opt in.

Browse Source
This commit is contained in:
maksw committed 2026-10-26 10:08:03 +01:00
1 parent c0ad63c27f
commit d36f19509c
160 files changed
+6563 -7366
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/drivers/bus/usb/usb_xhci.c
+293 -104
View File
@@ -48,27 +48,22 @@ void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value) {
}
// Implements the delay Us operation.
void UsbDelayUs(const NTL_DDK* ddk, uint32_t us) {
if (ddk && ddk->Timer && ddk->Timer->Stall) ddk->Timer->Stall(us);
else {
for (uint32_t i = 0; i < us * 32u; i++) _mm_pause();
}
void UsbDelayUs(uint32_t us) {
KeStallExecution(us);
}
bool UsbWaitSet32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg,
uint32_t mask, uint32_t timeoutUs) {
bool UsbWaitSet32(volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs) {
for (uint32_t i = 0; i < timeoutUs; i++) {
if ((XhciRead32(base, reg) & mask) == mask) return true;
UsbDelayUs(ddk, 1);
UsbDelayUs(1);
}
return false;
}
bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg,
uint32_t mask, uint32_t timeoutUs) {
bool UsbWaitClear32(volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs) {
for (uint32_t i = 0; i < timeoutUs; i++) {
if ((XhciRead32(base, reg) & mask) == 0) return true;
UsbDelayUs(ddk, 1);
UsbDelayUs(1);
}
return false;
}
@@ -83,7 +78,7 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci,
return (void*)0;
IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
KSTATUS status = xhci->Ddk->Dma->AllocateCommonBuffer(xhci->DmaAdapter,
KSTATUS status = IoDmaAllocateCommonBuffer(xhci->DmaAdapter,
pages * (size_t)PAGE_SIZE,
PAGE_SIZE,
&buffer);
@@ -93,14 +88,14 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci,
return (void*)0;
}
*outDeviceAddress = xhci->Ddk->Dma->GetCommonBufferDeviceAddress(buffer);
*outDeviceAddress = IoDmaGetCommonBufferDeviceAddress(buffer);
*outBuffer = buffer;
return xhci->Ddk->Dma->GetCommonBufferAddress(buffer);
return IoDmaGetCommonBufferAddress(buffer);
}
// Releases all DMA allocations owned by one xHCI slot.
void XhciReleaseDeviceDma(XHCI_DEVICE* dev) {
if (!dev || !gUsbDdk || !gUsbDdk->Dma)
if (!dev)
return;
for (uint32_t endpoint = 1; endpoint < 32; endpoint++) {
@@ -110,15 +105,15 @@ void XhciReleaseDeviceDma(XHCI_DEVICE* dev) {
UsbDereferencePipe(pipe);
}
gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DescriptorDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->Ep0RingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DeviceContextDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->InputContextDma);
IoDmaFreeCommonBuffer(dev->MassBufferDma);
IoDmaFreeCommonBuffer(dev->HidBufferDma);
IoDmaFreeCommonBuffer(dev->DescriptorDma);
IoDmaFreeCommonBuffer(dev->MassOutRingDma);
IoDmaFreeCommonBuffer(dev->MassInRingDma);
IoDmaFreeCommonBuffer(dev->HidRingDma);
IoDmaFreeCommonBuffer(dev->Ep0RingDma);
IoDmaFreeCommonBuffer(dev->DeviceContextDma);
IoDmaFreeCommonBuffer(dev->InputContextDma);
dev->MassBufferDma = (IO_DMA_BUFFER*)0;
dev->HidBufferDma = (IO_DMA_BUFFER*)0;
@@ -143,22 +138,22 @@ void XhciReleaseDeviceDma(XHCI_DEVICE* dev) {
// Releases controller-wide xHCI DMA allocations.
static void XhciReleaseControllerDma(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->Ddk || !xhci->Ddk->Dma)
if (!xhci)
return;
for (uint32_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++)
XhciReleaseDeviceDma(&xhci->Devices[slot]);
for (uint16_t i = 0; i < XHCI_MAX_SCRATCHPADS; i++) {
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadDma[i]);
IoDmaFreeCommonBuffer(xhci->ScratchpadDma[i]);
xhci->ScratchpadDma[i] = (IO_DMA_BUFFER*)0;
xhci->ScratchpadDeviceAddress[i] = 0;
}
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadArrayDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ErstDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->EventRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->CommandRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->DcbaaDma);
IoDmaFreeCommonBuffer(xhci->ScratchpadArrayDma);
IoDmaFreeCommonBuffer(xhci->ErstDma);
IoDmaFreeCommonBuffer(xhci->EventRingDma);
IoDmaFreeCommonBuffer(xhci->CommandRingDma);
IoDmaFreeCommonBuffer(xhci->DcbaaDma);
xhci->ScratchpadArrayDma = (IO_DMA_BUFFER*)0;
xhci->ErstDma = (IO_DMA_BUFFER*)0;
xhci->EventRingDma = (IO_DMA_BUFFER*)0;
@@ -170,7 +165,7 @@ static void XhciReleaseControllerDma(XHCI_CONTROLLER* xhci) {
xhci->CommandRing = (XHCI_TRB*)0;
xhci->Dcbaa = (uint64_t*)0;
xhci->Ddk->Dma->PutAdapter(xhci->DmaAdapter);
IoDmaPutAdapter(xhci->DmaAdapter);
xhci->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
@@ -220,6 +215,76 @@ uint32_t* XhciInputEndpointContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return (uint32_t*)(dev->InputContext + (uint32_t)(endpointId + 1u) * xhci->ContextBytes);
}
// Reconstructs the addressed device topology when configuring an endpoint or hub.
// Configure Endpoint must not discard route, TT, or hub attributes on the slot.
void XhciBuildConfigureSlotContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint32_t* slot, uint8_t endpointId) {
if (!xhci || !dev || !slot)
return;
memset(slot, 0, xhci->ContextBytes);
uint8_t lastContext = endpointId;
if (dev->DeviceContext) {
uint32_t previous = ((const uint32_t*)dev->DeviceContext)[0];
uint8_t previousLast = (uint8_t)((previous >> 27) & 31u);
if (previousLast > lastContext)
lastContext = previousLast;
}
uint8_t rootPort = dev->Path.RootPort ? dev->Path.RootPort : dev->Port;
slot[0] = XHCI_CONTEXT_ENTRIES(lastContext) | XHCI_SLOT_SPEED(dev->Speed) |
(dev->Path.RouteString & 0xfffffu);
slot[1] = XHCI_SLOT_ROOT_PORT(rootPort);
if (dev->Path.BehindHub && dev->Speed <= 2u && dev->Path.ParentHubSlot) {
slot[2] = (uint32_t)dev->Path.ParentHubSlot |
((uint32_t)dev->Path.ParentHubPort << 8);
}
if (dev->HubPortCount) {
slot[0] |= XHCI_SLOT_HUB;
slot[1] |= XHCI_SLOT_PORT_COUNT(dev->HubPortCount);
if (dev->Speed == 3u)
slot[2] |= XHCI_SLOT_TT_THINK(dev->HubCharacteristics >> 5);
}
}
// Publishes an external hub's topology to xHCI before its child ports are used.
KSTATUS XhciConfigureHubContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t portCount, uint16_t characteristics) {
if (!xhci || !dev || !dev->Addressed || !dev->InputContext ||
!dev->DeviceContext || !portCount)
return NTLSTATUS_INVALID_PARAMETER;
uint8_t oldPortCount = dev->HubPortCount;
uint16_t oldCharacteristics = dev->HubCharacteristics;
dev->HubPortCount = portCount;
dev->HubCharacteristics = characteristics;
memset(dev->InputContext, 0, PAGE_SIZE);
uint32_t* icc = XhciInputControlContext(xhci, dev);
uint32_t* slot = XhciInputSlotContext(xhci, dev);
// Update only the slot context; existing endpoint contexts remain active.
icc[1] = 1u;
XhciBuildConfigureSlotContext(xhci, dev, slot, 1u);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
DbgPrint("usb-stage: hcd=%02x:%02x.%u hub-slot=%u configure-context begin ports=%u tt=%u speed=%u\n",
xhci->Bus, xhci->Dev, xhci->Fn, dev->SlotId,
portCount, (characteristics >> 5) & 3u, dev->Speed);
KSTATUS st = XhciSubmitCommand(xhci, dev->InputContextDeviceAddress, 0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
&result);
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb-stage: hub-slot=%u configure-context failed st=0x%x cc=%u\n",
dev->SlotId, st, result.CompletionCode);
dev->HubPortCount = oldPortCount;
dev->HubCharacteristics = oldCharacteristics;
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
DbgPrint("usb-stage: hub-slot=%u configure-context done\n", dev->SlotId);
return NTLSTATUS_SUCCESS;
}
// Implements the ep Max Packet operation.
uint32_t XhciEpMaxPacket(uint8_t speed) {
switch (speed) {
@@ -254,6 +319,15 @@ const char* XhciPortSpeedName(uint32_t speed) {
}
}
// Advances an xHCI extended-capability offset using the relative DWORD link.
static uint32_t XhciNextExtendedCapabilityDword(uint32_t current, uint8_t next,
uint32_t mappedBytes) {
if (!next) return 0;
uint64_t following = (uint64_t)current + next;
if (following * 4u + sizeof(uint32_t) > mappedBytes) return 0;
return (uint32_t)following;
}
// Parses supported Protocols.
void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) {
memset(xhci->PortProtocolMajor, 0, sizeof(xhci->PortProtocolMajor));
@@ -297,8 +371,18 @@ void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) {
}
}
xecp = next;
xecp = XhciNextExtendedCapabilityDword(xecp, next, xhci->MmioBytes);
}
uint32_t usb2Ports = 0;
uint32_t usb3Ports = 0;
for (uint32_t port = 1; port <= xhci->MaxPorts; port++) {
if (xhci->PortProtocolMajor[port] == 2u) usb2Ports++;
else if (xhci->PortProtocolMajor[port] >= 3u) usb3Ports++;
}
DbgPrint("usb: xHCI hcd=%02x:%02x.%u protocol ports usb2=%u usb3=%u unknown=%u\n",
xhci->Bus, xhci->Dev, xhci->Fn, usb2Ports, usb3Ports,
(uint32_t)xhci->MaxPorts - usb2Ports - usb3Ports);
}
// Implements the port Protocol Major operation.
@@ -385,7 +469,7 @@ void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
}
if (powered) {
UsbDelayUs(xhci->Ddk, 20000u);
UsbDelayUs(20000u);
DbgPrint("usb: xHCI powered %u previously-off root port(s)\n", powered);
}
}
@@ -465,12 +549,43 @@ static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
return w;
}
// Implements the drain Event Ring operation.
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
bool processed = false;
// A completion leaves EventLock with its class-device rundown reference held.
// Client callbacks can rearm transfers or acquire unrelated PnP locks.
#define XHCI_COMPLETION_BATCH 16u
while (true) {
typedef struct XHCI_ASYNC_COMPLETION {
USB_URB* Urb;
IRP* Irp;
KSTATUS Status;
uint32_t ActualLength;
} XHCI_ASYNC_COMPLETION;
// Claims an asynchronous TD exactly once while the event lock is owned.
static bool XhciClaimAsyncCompletion(XHCI_CONTROLLER* xhci, uint64_t trbAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue,
XHCI_ASYNC_COMPLETION* completion);
// Executes callbacks outside the event lock; class remove locks own their lifetime.
static void XhciDispatchAsyncCompletion(const XHCI_ASYNC_COMPLETION* completed) {
USB_URB* urb = completed->Urb;
if (urb->Complete)
urb->Complete(urb, urb->CompleteContext);
if (completed->Irp)
IoCompleteRequest(completed->Irp, completed->Status,
KSUCCESS(completed->Status) ? completed->ActualLength : 0);
}
// Drains a bounded batch while the event lock is held.
static bool XhciDrainEventRing(XHCI_CONTROLLER* xhci,
XHCI_ASYNC_COMPLETION* completions, uint32_t* completionCount,
uint32_t* changedPorts) {
bool processed = false;
*completionCount = 0;
for (uint32_t drained = 0; drained < XHCI_EVENT_TRBS; drained++) {
if (*completionCount == XHCI_COMPLETION_BATCH)
break;
XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue];
uint32_t control = ev->Control;
uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE);
@@ -495,15 +610,15 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n",
(unsigned long long)param, cc, slotId, epId, residue);
}
XHCI_WAIT_TARGET* waiter = XhciClaimTransferWaiter(xhci, param, slotId, epId, cc, residue);
if (!waiter)
XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue);
if (!waiter && XhciClaimAsyncCompletion(xhci, param, cc, slotId, epId, residue,
&completions[*completionCount]))
(*completionCount)++;
} else if (type == XHCI_TRB_TYPE_PORT_SC) {
uint32_t portId = ev->ParameterLo >> 24;
UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc);
if (portId && portId <= xhci->MaxPorts)
UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)portId);
if (portId && portId <= xhci->MaxPorts && portId <= USB_ROOT_HUB_MAX_PORTS)
changedPorts[portId / 32u] |= 1u << (portId & 31u);
} else {
DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n",
type, cc, (unsigned long long)param, ev->Status, ev->Control);
@@ -521,12 +636,27 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
return processed;
}
// Implements the drain Event Ring Locked operation.
// Serializes event-ring ownership without calling into class drivers under the lock.
bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
if (!xhci) return false;
XHCI_ASYNC_COMPLETION completions[XHCI_COMPLETION_BATCH];
uint32_t changedPorts[8] = { 0 };
uint32_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&xhci->EventLock);
bool processed = XhciDrainEventRing(xhci, target);
bool processed = XhciDrainEventRing(xhci, completions, &count, changedPorts);
KeReleaseSpinlock(&xhci->EventLock, flags);
// Hub wakeups can take scheduler locks; keep them outside EventLock too.
for (uint32_t word = 0; word < 8; word++) {
uint32_t bits = changedPorts[word];
for (uint32_t bit = 0; bit < 32; bit++) {
if (bits & (1u << bit))
UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)(word * 32u + bit));
}
}
for (uint32_t i = 0; i < count; i++)
XhciDispatchAsyncCompletion(&completions[i]);
return processed;
}
@@ -591,7 +721,7 @@ KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
}
UsbDelayUs(xhci->Ddk, 1);
UsbDelayUs(1);
}
XhciUnregisterWaitTarget(xhci, target);
@@ -617,9 +747,17 @@ void XhciEventDpcRoutine(KDPC* dpc, void* context,
return;
xhci->EventDpcCount++;
while (XhciDrainEventRingLocked(xhci, NULL)) {
// Keep draining until the software dequeue catches up to the producer.
// Processing is bounded even if connected hubs continuously report changes.
// Updating ERDP clears EHB; remaining events cause the interrupter to
// assert again, allowing other DPCs and scheduled threads to make progress.
for (uint32_t pass = 0; pass < 4u; pass++) {
if (!XhciDrainEventRingLocked(xhci, NULL))
return;
}
uint32_t hits = ++xhci->EventDpcBudgetHits;
if ((hits & (hits - 1u)) == 0u)
DbgPrint("usb: xHCI hcd=%02x:%02x.%u event DPC budget reached count=%u\n",
xhci->Bus, xhci->Dev, xhci->Fn, hits);
}
#define XHCI_CONTROLLER_MAGIC 0x58484349u // XHCI
@@ -700,7 +838,7 @@ KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) {
ok = true;
break;
}
UsbDelayUs(xhci->Ddk, 1);
UsbDelayUs(1);
}
if (!ok) {
DbgPrint("usb: xHCI BIOS handoff timed out; continuing anyway\n");
@@ -712,7 +850,7 @@ KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_SUCCESS;
}
xecp = next;
xecp = XhciNextExtendedCapabilityDword(xecp, next, xhci->MmioBytes);
}
return NTLSTATUS_SUCCESS;
@@ -723,7 +861,7 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
if (cmd & XHCI_USBCMD_RS) {
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS);
if (!UsbWaitSet32(xhci->Ddk, xhci->Op, XHCI_OP_USBSTS,
if (!UsbWaitSet32(xhci->Op, XHCI_OP_USBSTS,
XHCI_USBSTS_HCH, 1000000u)) {
DbgPrint("usb: xHCI stop timeout usbsts=%08x\n",
XhciRead32(xhci->Op, XHCI_OP_USBSTS));
@@ -732,14 +870,14 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
}
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, XHCI_USBCMD_HCRST);
if (!UsbWaitClear32(xhci->Ddk, xhci->Op, XHCI_OP_USBCMD,
if (!UsbWaitClear32(xhci->Op, XHCI_OP_USBCMD,
XHCI_USBCMD_HCRST, 1000000u)) {
DbgPrint("usb: xHCI reset timeout usbcmd=%08x\n",
XhciRead32(xhci->Op, XHCI_OP_USBCMD));
return NTLSTATUS_TIMEOUT;
}
if (!UsbWaitClear32(xhci->Ddk, xhci->Op, XHCI_OP_USBSTS,
if (!UsbWaitClear32(xhci->Op, XHCI_OP_USBSTS,
XHCI_USBSTS_CNR, 1000000u)) {
DbgPrint("usb: xHCI controller-not-ready timeout usbsts=%08x\n",
XhciRead32(xhci->Op, XHCI_OP_USBSTS));
@@ -769,8 +907,7 @@ static void XhciHaltForDmaRelease(XHCI_CONTROLLER* xhci) {
uint32_t command = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
command &= ~(XHCI_USBCMD_RS | XHCI_USBCMD_INTE);
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, command);
if (!UsbWaitSet32(xhci->Ddk,
xhci->Op,
if (!UsbWaitSet32(xhci->Op,
XHCI_OP_USBSTS,
XHCI_USBSTS_HCH,
1000000u))
@@ -1048,7 +1185,7 @@ KSTATUS XhciStart(XHCI_CONTROLLER* xhci) {
cmd |= XHCI_USBCMD_RS;
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd);
if (!UsbWaitClear32(xhci->Ddk, xhci->Op, XHCI_OP_USBSTS,
if (!UsbWaitClear32(xhci->Op, XHCI_OP_USBSTS,
XHCI_USBSTS_HCH, 1000000u)) {
DbgPrint("usb: xHCI start timeout usbsts=%08x\n",
XhciRead32(xhci->Op, XHCI_OP_USBSTS));
@@ -1099,6 +1236,8 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
uint32_t speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT;
uint8_t protocol = XhciPortProtocolMajor(xhci, port, portsc);
DbgPrint("usb-stage: hcd=%02x:%02x.%u root-port=%u reset begin proto=%u speed=%u portsc=%08x\n",
xhci->Bus, xhci->Dev, xhci->Fn, port, protocol, speed, portsc);
XhciClearPortChanges(xhci, port);
portsc = XhciRead32(xhci->Op, off);
@@ -1120,7 +1259,7 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
break;
}
XhciDrainEventRingLocked(xhci, NULL);
UsbDelayUs(xhci->Ddk, 1);
UsbDelayUs(1);
}
XhciClearPortChanges(xhci, port);
@@ -1133,6 +1272,8 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
(portsc & XHCI_PORTSC_PED) ? 1u : 0u,
protocol, XhciPortSpeedName(speed), speed);
DbgPrint("usb-stage: hcd=%02x:%02x.%u root-port=%u superspeed ready=%u portsc=%08x\n",
xhci->Bus, xhci->Dev, xhci->Fn, port, enabled ? 1u : 0u, portsc);
if (!enabled || !(portsc & XHCI_PORTSC_CCS) || !(portsc & XHCI_PORTSC_PED))
return NTLSTATUS_DEVICE_ERROR;
return NTLSTATUS_SUCCESS;
@@ -1149,11 +1290,12 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
break;
}
XhciDrainEventRingLocked(xhci, NULL);
UsbDelayUs(xhci->Ddk, 1);
UsbDelayUs(1);
}
if (!resetDone) {
DbgPrint("usb: xHCI port %u reset timeout portsc=%08x\n", port, portsc);
DbgPrint("usb: xHCI hcd=%02x:%02x.%u port %u reset timeout portsc=%08x\n",
xhci->Bus, xhci->Dev, xhci->Fn, port, portsc);
return NTLSTATUS_TIMEOUT;
}
@@ -1167,6 +1309,8 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
(portsc & XHCI_PORTSC_PED) ? 1u : 0u,
protocol, XhciPortSpeedName(speed), speed);
DbgPrint("usb-stage: hcd=%02x:%02x.%u root-port=%u reset done portsc=%08x speed=%u\n",
xhci->Bus, xhci->Dev, xhci->Fn, port, portsc, speed);
if (!(portsc & XHCI_PORTSC_CCS) || !(portsc & XHCI_PORTSC_PED))
return NTLSTATUS_DEVICE_ERROR;
@@ -1674,10 +1818,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
icc[0] = 0;
icc[1] = (1u << 0) | (1u << dci);
slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed);
slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
slot[2] = 0;
slot[3] = 0;
XhciBuildConfigureSlotContext(xhci, dev, slot, dci);
uint16_t maxPacket = pipe->MaxPacketSize & 0x07FFu;
if (maxPacket == 0) maxPacket = 8u;
@@ -1692,7 +1833,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
} else if (pipe->Type == UsbPipeBulk) {
endpointType = XHCI_EP_TYPE_BULK_OUT;
} else {
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
IoDmaFreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return NTLSTATUS_NOT_SUPPORTED;
}
@@ -1715,7 +1856,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
IoDmaFreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
@@ -1740,8 +1881,15 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext;
if (!ctx || !ctx->Configured)
return NTLSTATUS_DEVICE_ERROR;
if (ctx->ActiveTransfer)
uintptr_t flags = KeAcquireSpinlock(&xhci->EventLock);
if (ctx->ActiveTransfer) {
KeReleaseSpinlock(&xhci->EventLock, flags);
return NTLSTATUS_QUEUE_FULL;
}
if (urb->Pipe->Halted) {
KeReleaseSpinlock(&xhci->EventLock, flags);
return NTLSTATUS_OBJECT_DEAD;
}
uint64_t trbDeviceAddress = 0;
st = XhciRingAppend(ctx->Ring,
@@ -1755,7 +1903,10 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
KeReleaseSpinlock(&xhci->EventLock, flags);
return st;
}
ctx->ActiveTransfer = true;
ctx->ActiveTrbDeviceAddress = trbDeviceAddress;
@@ -1768,58 +1919,51 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, ctx->EndpointId);
KeReleaseSpinlock(&xhci->EventLock, flags);
return NTLSTATUS_PENDING;
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
return;
// Completes ownership of an interrupt URB before its callback is dispatched.
static bool XhciClaimAsyncCompletion(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue,
XHCI_ASYNC_COMPLETION* completed) {
if (!xhci || !completed || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
return false;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
USB_PIPE* pipe = dev->EndpointPipes[endpointId];
if (!pipe || !pipe->HcdPipeContext)
return;
return false;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (!ctx->ActiveTransfer || ctx->ActiveTrbDeviceAddress != trbDeviceAddress)
return;
return false;
USB_URB* urb = ctx->ActiveUrb;
ctx->ActiveTransfer = false;
ctx->ActiveTrbDeviceAddress = 0;
ctx->ActiveUrb = (USB_URB*)0;
if (!urb)
return;
return false;
KSTATUS st = NTLSTATUS_SUCCESS;
if (urb->Cancelled) {
if (urb->Cancelled)
st = NTLSTATUS_CANCELLED;
} else if (completionCode != XHCI_CC_SUCCESS &&
completionCode != XHCI_CC_SHORT_PACKET) {
else if (completionCode != XHCI_CC_SUCCESS && completionCode != XHCI_CC_SHORT_PACKET)
st = NTLSTATUS_DEVICE_ERROR;
}
uint32_t actual = 0;
if (KSUCCESS(st)) {
actual = (residue <= urb->Length) ? (urb->Length - residue) : 0;
}
uint32_t actual = KSUCCESS(st) && residue <= urb->Length ? urb->Length - residue : 0;
urb->Pending = false;
urb->Status = st;
urb->ActualLength = actual;
urb->HcdCompletionCode = completionCode;
IRP* completeIrp = urb->Irp;
completed->Urb = urb;
completed->Irp = urb->Irp;
completed->Status = st;
completed->ActualLength = actual;
urb->Irp = (IRP*)0;
if (urb->Complete)
urb->Complete(urb, urb->CompleteContext);
if (completeIrp)
IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0);
return true;
}
// Implements the hcd Submit Bulk Urb operation.
@@ -1905,13 +2049,59 @@ static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
return st;
}
// Implements the hcd Cancel Urb operation.
static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
(void)hcd;
if (!urb) return NTLSTATUS_INVALID_PARAMETER;
if (!hcd || !urb || !urb->Device || !urb->Pipe)
return NTLSTATUS_INVALID_PARAMETER;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext;
XHCI_DEVICE* dev = (XHCI_DEVICE*)urb->Device->HcdDeviceContext;
if (!xhci || !dev || dev->SlotId == 0 || !urb->Pipe->HcdPipeContext)
return NTLSTATUS_INVALID_PARAMETER;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext;
uintptr_t flags = KeAcquireSpinlock(&xhci->EventLock);
urb->Cancelled = true;
if (!urb->Pending)
urb->Status = NTLSTATUS_CANCELLED;
bool active = ctx->ActiveTransfer && ctx->ActiveUrb == urb;
KeReleaseSpinlock(&xhci->EventLock, flags);
if (!active)
return NTLSTATUS_SUCCESS;
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(xhci, 0, 0,
(XHCI_TRB_TYPE_STOP_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)ctx->EndpointId << 16) |
((uint32_t)dev->SlotId << 24),
&result);
flags = KeAcquireSpinlock(&xhci->EventLock);
if (!ctx->ActiveTransfer || ctx->ActiveUrb != urb) {
KeReleaseSpinlock(&xhci->EventLock, flags);
return NTLSTATUS_SUCCESS;
}
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
KeReleaseSpinlock(&xhci->EventLock, flags);
DbgPrint("usb: xHCI STOP_ENDPOINT failed slot=%u ep=%u st=0x%x cc=%u\n",
dev->SlotId, ctx->EndpointId, st, result.CompletionCode);
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
ctx->ActiveTransfer = false;
ctx->ActiveTrbDeviceAddress = 0;
ctx->ActiveUrb = (USB_URB*)0;
urb->Pipe->Halted = true;
urb->Pending = false;
urb->Status = NTLSTATUS_CANCELLED;
urb->ActualLength = 0;
urb->HcdCompletionCode = result.CompletionCode;
IRP* completeIrp = urb->Irp;
urb->Irp = (IRP*)0;
KeReleaseSpinlock(&xhci->EventLock, flags);
if (urb->Complete)
urb->Complete(urb, urb->CompleteContext);
if (completeIrp)
IoCompleteRequest(completeIrp, NTLSTATUS_CANCELLED, 0);
return NTLSTATUS_SUCCESS;
}
@@ -1933,8 +2123,7 @@ static void XhciHcdDestroyPipe(USB_HCD* hcd, USB_PIPE* pipe) {
if (!pipe || !pipe->HcdPipeContext)
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (gUsbDdk && gUsbDdk->Dma)
gUsbDdk->Dma->FreeCommonBuffer(ctx->RingDma);
IoDmaFreeCommonBuffer(ctx->RingDma);
ctx->RingDma = (IO_DMA_BUFFER*)0;
ctx->Ring = (XHCI_TRB*)0;
pipe->HcdPipeContext = (void*)0;
@@ -2109,7 +2298,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
dmaRequirements.Boundary = 0x10000u;
dmaRequirements.MaximumSegmentLength = 0x10000u;
dmaRequirements.MaximumSegments = XHCI_PIPE_RING_TRBS - 1u;
st = xhci->Ddk->Dma->GetAdapter(xhci->Fdo,
st = IoDmaGetAdapter(xhci->Fdo,
&dmaRequirements,
&xhci->DmaAdapter);
if (!KSUCCESS(st)) {
@@ -2165,7 +2354,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
if (!KSUCCESS(st))
DbgPrint("usb: root hub async scan scheduling failed st=0x%x\n", st);
else
DbgPrint("usb: root hub initial scan queued asynchronously\n");
DbgPrint("usb: root hub initial scan requested\n");
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
xhci->HidDeviceCount);
@@ -2247,7 +2436,7 @@ KSTATUS XhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)fdo->DeviceExtension;
memset(xhci, 0, sizeof(*xhci));
xhci->Magic = XHCI_CONTROLLER_MAGIC;
xhci->Ddk = gUsbDdk;
xhci->Fdo = fdo;
xhci->Pdo = physicalDeviceObject;
xhci->Irq = 0xFFu;