maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

Browse Source
This commit is contained in:
maksw committed 2026-08-11 18:38:36 +02:00
1 parent 8b254f84f4
commit 4add6789ba
177 files changed
+22565 -3070
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/drivers/bus/usb/usb_xhci.c
+322 -151
View File
@@ -73,32 +73,105 @@ bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg,
return false;
}
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
PHYSICAL_ADDRESS* outPhys) {
if (!outPhys || pages == 0)
return NULL;
// Allocates page-aligned common-buffer memory from the controller adapter.
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!xhci || !xhci->DmaAdapter || !outDeviceAddress || !outBuffer || pages == 0 ||
pages > ((size_t)-1) / PAGE_SIZE)
return (void*)0;
// Most xHCI runtime objects are one page. Do not route those through the
// physically-contiguous allocator: on real machines that allocator is a
// scarce DMA primitive, while a single frame is already contiguous. QEMU
// hid this because it had simple low memory; laptop firmware does not.
PHYSICAL_ADDRESS phys = 0;
if (pages == 1) {
phys = MmPfnAllocate(MmPfnUseDma);
} else {
phys = MmPfnAllocateContiguous(pages, MmPfnUseDma);
IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
KSTATUS status = xhci->Ddk->Dma->AllocateCommonBuffer(xhci->DmaAdapter,
pages * (size_t)PAGE_SIZE,
PAGE_SIZE,
&buffer);
if (!KSUCCESS(status)) {
DbgPrint("usb: DMA allocation failed pages=%llu st=0x%x\n",
(unsigned long long)pages, status);
return (void*)0;
}
if (!phys) {
DbgPrint("usb: DMA allocation failed pages=%llu\n",
(unsigned long long)pages);
return NULL;
*outDeviceAddress = xhci->Ddk->Dma->GetCommonBufferDeviceAddress(buffer);
*outBuffer = buffer;
return xhci->Ddk->Dma->GetCommonBufferAddress(buffer);
}
// Releases all DMA allocations owned by one xHCI slot.
void XhciReleaseDeviceDma(XHCI_DEVICE* dev) {
if (!dev || !gUsbDdk || !gUsbDdk->Dma)
return;
for (uint32_t endpoint = 1; endpoint < 32; endpoint++) {
USB_PIPE* pipe = dev->EndpointPipes[endpoint];
dev->EndpointPipes[endpoint] = (USB_PIPE*)0;
if (pipe)
UsbDereferencePipe(pipe);
}
void* virt = xhci->Ddk->Memory->PhysToVirt(phys);
memset(virt, 0, pages * (size_t)PAGE_SIZE);
*outPhys = phys;
return virt;
gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DescriptorDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->Ep0RingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DeviceContextDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->InputContextDma);
dev->MassBufferDma = (IO_DMA_BUFFER*)0;
dev->HidBufferDma = (IO_DMA_BUFFER*)0;
dev->DescriptorDma = (IO_DMA_BUFFER*)0;
dev->MassOutRingDma = (IO_DMA_BUFFER*)0;
dev->MassInRingDma = (IO_DMA_BUFFER*)0;
dev->HidRingDma = (IO_DMA_BUFFER*)0;
dev->Ep0RingDma = (IO_DMA_BUFFER*)0;
dev->DeviceContextDma = (IO_DMA_BUFFER*)0;
dev->InputContextDma = (IO_DMA_BUFFER*)0;
dev->MassBuffer = (uint8_t*)0;
dev->HidBuffer = (uint8_t*)0;
dev->DescriptorBuffer = (uint8_t*)0;
dev->MassOutRing = (XHCI_TRB*)0;
dev->MassInRing = (XHCI_TRB*)0;
dev->HidRing = (XHCI_TRB*)0;
dev->Ep0Ring = (XHCI_TRB*)0;
dev->DeviceContext = (uint8_t*)0;
dev->InputContext = (uint8_t*)0;
}
// Releases controller-wide xHCI DMA allocations.
static void XhciReleaseControllerDma(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->Ddk || !xhci->Ddk->Dma)
return;
for (uint32_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++)
XhciReleaseDeviceDma(&xhci->Devices[slot]);
for (uint16_t i = 0; i < XHCI_MAX_SCRATCHPADS; i++) {
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadDma[i]);
xhci->ScratchpadDma[i] = (IO_DMA_BUFFER*)0;
xhci->ScratchpadDeviceAddress[i] = 0;
}
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadArrayDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ErstDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->EventRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->CommandRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->DcbaaDma);
xhci->ScratchpadArrayDma = (IO_DMA_BUFFER*)0;
xhci->ErstDma = (IO_DMA_BUFFER*)0;
xhci->EventRingDma = (IO_DMA_BUFFER*)0;
xhci->CommandRingDma = (IO_DMA_BUFFER*)0;
xhci->DcbaaDma = (IO_DMA_BUFFER*)0;
xhci->ScratchpadArray = (uint64_t*)0;
xhci->Erst = (XHCI_ERST_ENTRY*)0;
xhci->EventRing = (XHCI_TRB*)0;
xhci->CommandRing = (XHCI_TRB*)0;
xhci->Dcbaa = (uint64_t*)0;
xhci->Ddk->Dma->PutAdapter(xhci->DmaAdapter);
xhci->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
// Implements the scratchpad Count operation.
@@ -328,26 +401,26 @@ void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
// Updates erdp.
void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB);
uint64_t erdp = xhci->EventRingDeviceAddress + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB);
XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear
}
// Returns command Waiter Locked.
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandDeviceAddress) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->CommandPhys && w->CommandPhys == commandPhys)
if (w->CommandDeviceAddress && w->CommandDeviceAddress == commandDeviceAddress)
return w;
}
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint64_t transferDeviceAddress,
uint8_t slotId,
uint8_t endpointId) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->TransferPhys &&
w->TransferPhys == transferPhys &&
if (w->TransferDeviceAddress &&
w->TransferDeviceAddress == transferDeviceAddress &&
w->TransferSlotId == slotId &&
w->TransferEndpointId == endpointId) {
return w;
@@ -357,11 +430,11 @@ static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
}
static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
uint64_t commandPhys,
uint64_t commandDeviceAddress,
uint32_t completionCode,
uint8_t slotId) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandDeviceAddress);
if (w) {
w->Command.CompletionCode = completionCode;
w->Command.SlotId = slotId;
@@ -373,13 +446,13 @@ static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
}
static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint64_t transferDeviceAddress,
uint8_t slotId,
uint8_t endpointId,
uint32_t completionCode,
uint32_t residue) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferDeviceAddress, slotId, endpointId);
if (w) {
w->Transfer.CompletionCode = completionCode;
w->Transfer.SlotId = slotId;
@@ -397,7 +470,7 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
bool processed = false;
for (;;) {
while (true) {
XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue];
uint32_t control = ev->Control;
uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE);
@@ -460,8 +533,8 @@ bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
// Waits for target Complete.
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
if (!target) return false;
if (target->CommandPhys && !target->CommandMatched) return false;
if (target->TransferPhys && !target->TransferMatched) return false;
if (target->CommandDeviceAddress && !target->CommandMatched) return false;
if (target->TransferDeviceAddress && !target->TransferMatched) return false;
return true;
}
@@ -568,6 +641,21 @@ static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the interrupt-dispatch registry.
static void XhciUnregisterController(XHCI_CONTROLLER* xhci) {
if (!xhci)
return;
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
if (gXhciControllers[i] != xhci)
continue;
for (uint32_t next = i + 1; next < gXhciControllerCount; next++)
gXhciControllers[next - 1] = gXhciControllers[next];
gXhciControllerCount--;
gXhciControllers[gXhciControllerCount] = (XHCI_CONTROLLER*)0;
return;
}
}
// Implements the isr operation.
void XhciIsr(void) {
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
@@ -663,6 +751,34 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_SUCCESS;
}
// Halts the controller before any device-visible memory can be released.
static void XhciHaltForDmaRelease(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->DmaAdapter)
return;
xhci->InterruptsEnabled = false;
xhci->Running = false;
if (xhci->SurpriseRemoved || !xhci->Op)
return;
if (xhci->Runtime) {
volatile uint8_t* interrupter = xhci->Runtime + 0x20u;
XhciWrite32(interrupter, 0x00u, XHCI_IMAN_IP);
}
uint32_t command = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
command &= ~(XHCI_USBCMD_RS | XHCI_USBCMD_INTE);
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, command);
if (!UsbWaitSet32(xhci->Ddk,
xhci->Op,
XHCI_OP_USBSTS,
XHCI_USBSTS_HCH,
1000000u))
KePanic(KE_PANIC_CODE_DRIVER,
"usb: xHCI failed to halt before DMA release usbsts=%08x\n",
XhciRead32(xhci->Op, XHCI_OP_USBSTS));
}
// Creates scratchpads.
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
if (xhci->ScratchpadCount == 0) return NTLSTATUS_SUCCESS;
@@ -672,7 +788,7 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_NOT_SUPPORTED;
}
xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayPhys);
xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayDeviceAddress, &xhci->ScratchpadArrayDma);
if (!xhci->ScratchpadArray) {
DbgPrint("usb: xHCI scratchpad pointer-array allocation failed count=%u\n",
xhci->ScratchpadCount);
@@ -680,29 +796,29 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
}
for (uint16_t i = 0; i < xhci->ScratchpadCount; i++) {
void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadPhys[i]);
void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadDeviceAddress[i], &xhci->ScratchpadDma[i]);
if (!page) {
DbgPrint("usb: xHCI scratchpad buffer allocation failed index=%u/%u\n",
i, xhci->ScratchpadCount);
return NTLSTATUS_NO_MEMORY;
}
(void)page;
xhci->ScratchpadArray[i] = xhci->ScratchpadPhys[i];
xhci->ScratchpadArray[i] = xhci->ScratchpadDeviceAddress[i];
}
xhci->Dcbaa[0] = xhci->ScratchpadArrayPhys;
xhci->Dcbaa[0] = xhci->ScratchpadArrayDeviceAddress;
UsbTrace("usb: xHCI allocated %u scratchpad buffer(s) array=0x%016llX\n",
xhci->ScratchpadCount,
(unsigned long long)xhci->ScratchpadArrayPhys);
(unsigned long long)xhci->ScratchpadArrayDeviceAddress);
return NTLSTATUS_SUCCESS;
}
// Creates runtime Memory.
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys);
xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys);
xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingPhys);
xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstPhys);
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaDeviceAddress, &xhci->DcbaaDma);
xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingDeviceAddress, &xhci->CommandRingDma);
xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingDeviceAddress, &xhci->EventRingDma);
xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstDeviceAddress, &xhci->ErstDma);
if (!xhci->Dcbaa || !xhci->CommandRing || !xhci->EventRing || !xhci->Erst) {
DbgPrint("usb: xHCI runtime DMA allocation failed dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n",
@@ -716,14 +832,14 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->EventCycle = 1;
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress;
link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
xhci->CommandCycle;
xhci->Erst[0].RingSegmentBase = xhci->EventRingPhys;
xhci->Erst[0].RingSegmentBase = xhci->EventRingDeviceAddress;
xhci->Erst[0].RingSegmentSize = XHCI_EVENT_TRBS;
xhci->Erst[0].Reserved = 0;
@@ -731,10 +847,10 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
if (!KSUCCESS(st)) return st;
UsbTrace("usb: xHCI dma dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n",
(unsigned long long)xhci->DcbaaPhys,
(unsigned long long)xhci->CommandRingPhys,
(unsigned long long)xhci->EventRingPhys,
(unsigned long long)xhci->ErstPhys);
(unsigned long long)xhci->DcbaaDeviceAddress,
(unsigned long long)xhci->CommandRingDeviceAddress,
(unsigned long long)xhci->EventRingDeviceAddress,
(unsigned long long)xhci->ErstDeviceAddress);
return NTLSTATUS_SUCCESS;
}
@@ -743,8 +859,8 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
uint8_t slots = xhci->MaxSlots;
if (slots == 0) return NTLSTATUS_DEVICE_ERROR;
XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaPhys);
XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingPhys | xhci->CommandCycle);
XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaDeviceAddress);
XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingDeviceAddress | xhci->CommandCycle);
XhciWrite32(xhci->Op, XHCI_OP_DNCTRL, 0);
XhciWrite32(xhci->Op, XHCI_OP_CONFIG, slots);
@@ -752,30 +868,30 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
XhciWrite32(ir0, 0x00u, XHCI_IMAN_IP); // clear pending, leave disabled until smoke test passes
XhciWrite32(ir0, 0x04u, 0); // no interrupt moderation yet
XhciWrite32(ir0, 0x08u, 1); // one ERST entry
XhciWrite64(ir0, 0x10u, xhci->ErstPhys);
XhciWrite64(ir0, 0x18u, xhci->EventRingPhys);
XhciWrite64(ir0, 0x10u, xhci->ErstDeviceAddress);
XhciWrite64(ir0, 0x18u, xhci->EventRingDeviceAddress);
XhciBarrier();
return NTLSTATUS_SUCCESS;
}
// Implements the command Trb Phys operation.
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB);
// Returns the device address of one command TRB.
uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
}
static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
uint64_t* outDeviceAddress) {
if (!xhci || !xhci->CommandRing)
return NTLSTATUS_INVALID_PARAMETER;
if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) {
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress;
link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -787,7 +903,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint32_t index = xhci->CommandEnqueue++;
XHCI_TRB* trb = &xhci->CommandRing[index];
uint64_t trbPhys = XhciCommandTrbPhys(xhci, index);
uint64_t trbDeviceAddress = XhciCommandTrbDeviceAddress(xhci, index);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -795,7 +911,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
XhciBarrier();
trb->Control = controlWithoutCycle | xhci->CommandCycle;
if (outPhys) *outPhys = trbPhys;
if (outDeviceAddress) *outDeviceAddress = trbDeviceAddress;
return NTLSTATUS_SUCCESS;
}
@@ -811,14 +927,14 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
memset(&target, 0, sizeof(target));
uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock);
uint64_t trbPhys = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys);
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbDeviceAddress);
if (!KSUCCESS(st)) {
KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
return st;
}
target.CommandPhys = trbPhys;
target.CommandDeviceAddress = trbDeviceAddress;
XhciRegisterWaitTarget(xhci, &target);
XhciBarrier();
@@ -912,7 +1028,7 @@ KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_ADDRESS_DEVICE << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -1059,20 +1175,22 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
// Creates device Memory.
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys);
dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys);
dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingPhys);
dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorPhys);
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextDeviceAddress, &dev->InputContextDma);
dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextDeviceAddress, &dev->DeviceContextDma);
dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingDeviceAddress, &dev->Ep0RingDma);
dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorDeviceAddress, &dev->DescriptorDma);
if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer)
if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) {
XhciReleaseDeviceDma(dev);
return NTLSTATUS_NO_MEMORY;
}
dev->Ep0Enqueue = 0;
dev->Ep0Cycle = 1;
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -1116,31 +1234,31 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
ep0[0] = 0;
ep0[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_CONTROL | (maxPacket << 16);
ep0[2] = (uint32_t)(dev->Ep0RingPhys | XHCI_EP_DCS);
ep0[3] = (uint32_t)((dev->Ep0RingPhys | XHCI_EP_DCS) >> 32);
ep0[2] = (uint32_t)(dev->Ep0RingDeviceAddress | XHCI_EP_DCS);
ep0[3] = (uint32_t)((dev->Ep0RingDeviceAddress | XHCI_EP_DCS) >> 32);
ep0[4] = 8;
xhci->Dcbaa[dev->SlotId] = dev->DeviceContextPhys;
xhci->Dcbaa[dev->SlotId] = dev->DeviceContextDeviceAddress;
XhciBarrier();
}
// Implements the ep0 Trb Phys operation.
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB);
// Returns the device address of one endpoint-zero TRB.
uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
}
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
uint64_t* outDeviceAddress) {
if (!dev || !dev->Ep0Ring)
return NTLSTATUS_INVALID_PARAMETER;
if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) {
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -1152,7 +1270,7 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint32_t index = dev->Ep0Enqueue++;
XHCI_TRB* trb = &dev->Ep0Ring[index];
uint64_t phys = XhciEp0TrbPhys(dev, index);
uint64_t deviceAddress = XhciEp0TrbDeviceAddress(dev, index);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -1160,14 +1278,14 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
XhciBarrier();
trb->Control = controlWithoutCycle | dev->Ep0Cycle;
if (outPhys) *outPhys = phys;
if (outDeviceAddress) *outDeviceAddress = deviceAddress;
return NTLSTATUS_SUCCESS;
}
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length,
uint32_t timeoutUs) {
if (!buffer || length == 0 || length > PAGE_SIZE)
@@ -1188,26 +1306,26 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
st = XhciEp0AppendTrb(
dev,
bufferPhys,
bufferDeviceAddress,
length,
(XHCI_TRB_TYPE_DATA << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_DATA_DIR_IN,
NULL);
if (!KSUCCESS(st)) return st;
uint64_t statusTrbPhys = 0;
uint64_t statusTrbDeviceAddress = 0;
st = XhciEp0AppendTrb(
dev,
0,
0,
(XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_IOC,
&statusTrbPhys);
&statusTrbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferDeviceAddress = statusTrbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
@@ -1235,9 +1353,9 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length) {
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length,
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferDeviceAddress, length,
USB_XHCI_CONTROL_TIMEOUT_US);
}
@@ -1254,7 +1372,7 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
NULL);
if (!KSUCCESS(st)) return st;
uint64_t statusTrbPhys = 0;
uint64_t statusTrbDeviceAddress = 0;
st = XhciEp0AppendTrb(
dev,
0,
@@ -1262,12 +1380,12 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
(XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_STATUS_DIR_IN |
XHCI_TRB_IOC,
&statusTrbPhys);
&statusTrbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferDeviceAddress = statusTrbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
@@ -1304,7 +1422,7 @@ KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
KSTATUS st = XhciControlInTimeout(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
sizeof(USB_DEVICE_DESCRIPTOR),
timeoutUs);
if (!KSUCCESS(st)) return st;
@@ -1339,7 +1457,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
KSTATUS st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
sizeof(USB_CONFIGURATION_DESCRIPTOR));
if (!KSUCCESS(st)) return st;
@@ -1366,7 +1484,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
setup.wLength = total;
st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
total);
if (!KSUCCESS(st)) return st;
@@ -1415,17 +1533,17 @@ uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) {
return interval;
}
KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress,
uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle,
uint64_t parameter, uint32_t status,
uint32_t controlWithoutCycle, uint64_t* outPhys) {
uint32_t controlWithoutCycle, uint64_t* outDeviceAddress) {
if (!ring || !enqueue || !cycle || trbCount < 2)
return NTLSTATUS_INVALID_PARAMETER;
if (*enqueue >= trbCount - 1u) {
XHCI_TRB* link = &ring[trbCount - 1u];
link->ParameterLo = (uint32_t)ringPhys;
link->ParameterHi = (uint32_t)(ringPhys >> 32);
link->ParameterLo = (uint32_t)ringDeviceAddress;
link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -1437,7 +1555,7 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
uint32_t index = (*enqueue)++;
XHCI_TRB* trb = &ring[index];
uint64_t phys = ringPhys + (uint64_t)index * sizeof(XHCI_TRB);
uint64_t deviceAddress = ringDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -1445,16 +1563,16 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
XhciBarrier();
trb->Control = controlWithoutCycle | *cycle;
if (outPhys) *outPhys = phys;
if (outDeviceAddress) *outDeviceAddress = deviceAddress;
return NTLSTATUS_SUCCESS;
}
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress,
uint32_t trbCount, uint8_t cycle) {
memset(ring, 0, trbCount * sizeof(XHCI_TRB));
XHCI_TRB* link = &ring[trbCount - 1u];
link->ParameterLo = (uint32_t)ringPhys;
link->ParameterHi = (uint32_t)(ringPhys >> 32);
link->ParameterLo = (uint32_t)ringDeviceAddress;
link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -1464,34 +1582,34 @@ void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t endpointId,
XHCI_TRB* ring,
PHYSICAL_ADDRESS ringPhys,
uint64_t ringDeviceAddress,
uint32_t* enqueue,
uint8_t* cycle,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint32_t length,
uint32_t* outActual) {
if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferPhys || length == 0)
if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferDeviceAddress || length == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (length > USB_STORAGE_MAX_BYTES)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciRingAppend(ring,
ringPhys,
ringDeviceAddress,
XHCI_BULK_RING_TRBS,
enqueue,
cycle,
bufferPhys,
bufferDeviceAddress,
length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = trbPhys;
target.TransferDeviceAddress = trbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = endpointId;
@@ -1537,7 +1655,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
if (!ctx) return NTLSTATUS_NO_MEMORY;
memset(ctx, 0, sizeof(*ctx));
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys);
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingDeviceAddress, &ctx->RingDma);
if (!ctx->Ring) {
MmFreePool(ctx);
return NTLSTATUS_NO_MEMORY;
@@ -1546,7 +1664,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
ctx->Cycle = 1;
ctx->EndpointId = dci;
ctx->Type = pipe->Type;
XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle);
XhciInitialiseTransferRing(ctx->Ring, ctx->RingDeviceAddress, XHCI_PIPE_RING_TRBS, ctx->Cycle);
memset(dev->InputContext, 0, PAGE_SIZE);
uint32_t* icc = XhciInputControlContext(xhci, dev);
@@ -1574,21 +1692,22 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
} else if (pipe->Type == UsbPipeBulk) {
endpointType = XHCI_EP_TYPE_BULK_OUT;
} else {
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return NTLSTATUS_NOT_SUPPORTED;
}
ep[0] = (uint32_t)interval << 16;
ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16);
ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32);
ep[2] = (uint32_t)(ctx->RingDeviceAddress | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingDeviceAddress | XHCI_EP_DCS) >> 32);
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -1596,6 +1715,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
@@ -1609,7 +1729,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
// Implements the hcd Submit Interrupt Urb operation.
static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN))
return NTLSTATUS_NOT_SUPPORTED;
@@ -1623,22 +1743,22 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
if (ctx->ActiveTransfer)
return NTLSTATUS_QUEUE_FULL;
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
st = XhciRingAppend(ctx->Ring,
ctx->RingPhys,
ctx->RingDeviceAddress,
XHCI_PIPE_RING_TRBS,
&ctx->Enqueue,
&ctx->Cycle,
urb->BufferPhys,
urb->BufferDeviceAddress,
urb->Length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
ctx->ActiveTransfer = true;
ctx->ActiveTrbPhys = trbPhys;
ctx->ActiveTrbDeviceAddress = trbDeviceAddress;
ctx->ActiveUrb = urb;
urb->Pending = true;
urb->Cancelled = false;
@@ -1651,7 +1771,7 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
return NTLSTATUS_PENDING;
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
@@ -1663,12 +1783,12 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys)
if (!ctx->ActiveTransfer || ctx->ActiveTrbDeviceAddress != trbDeviceAddress)
return;
USB_URB* urb = ctx->ActiveUrb;
ctx->ActiveTransfer = false;
ctx->ActiveTrbPhys = 0;
ctx->ActiveTrbDeviceAddress = 0;
ctx->ActiveUrb = (USB_URB*)0;
if (!urb)
@@ -1704,23 +1824,23 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
// Implements the hcd Submit Bulk Urb operation.
static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0)
return NTLSTATUS_INVALID_PARAMETER;
uint8_t endpointId = urb->Pipe->EndpointId;
XHCI_TRB* ring = (XHCI_TRB*)0;
PHYSICAL_ADDRESS ringPhys = 0;
uint64_t ringDeviceAddress = 0;
uint32_t* enqueue = (uint32_t*)0;
uint8_t* cycle = (uint8_t*)0;
if (endpointId == dev->MassBulkInEndpointId) {
ring = dev->MassInRing;
ringPhys = dev->MassInRingPhys;
ringDeviceAddress = dev->MassInRingDeviceAddress;
enqueue = &dev->MassInEnqueue;
cycle = &dev->MassInCycle;
} else if (endpointId == dev->MassBulkOutEndpointId) {
ring = dev->MassOutRing;
ringPhys = dev->MassOutRingPhys;
ringDeviceAddress = dev->MassOutRingDeviceAddress;
enqueue = &dev->MassOutEnqueue;
cycle = &dev->MassOutCycle;
} else {
@@ -1732,10 +1852,10 @@ static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB
dev,
endpointId,
ring,
ringPhys,
ringDeviceAddress,
enqueue,
cycle,
urb->BufferPhys,
urb->BufferDeviceAddress,
urb->Length,
&actual);
urb->ActualLength = KSUCCESS(st) ? actual : 0;
@@ -1759,7 +1879,7 @@ static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
st = XhciControlNoData(xhci, dev, &urb->Setup);
urb->ActualLength = 0;
} else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) {
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length);
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferDeviceAddress, (uint16_t)urb->Length);
urb->ActualLength = KSUCCESS(st) ? urb->Length : 0;
} else {
st = NTLSTATUS_NOT_SUPPORTED;
@@ -1796,9 +1916,29 @@ static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
}
// Implements the hcd Allocate Dma operation.
static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
static void* XhciHcdAllocateDma(USB_HCD* hcd,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!hcd) return (void*)0;
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext,
pages,
outDeviceAddress,
outBuffer);
}
// Releases the host-controller context attached to a USB pipe.
static void XhciHcdDestroyPipe(USB_HCD* hcd, USB_PIPE* pipe) {
(void)hcd;
if (!pipe || !pipe->HcdPipeContext)
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (gUsbDdk && gUsbDdk->Dma)
gUsbDdk->Dma->FreeCommonBuffer(ctx->RingDma);
ctx->RingDma = (IO_DMA_BUFFER*)0;
ctx->Ring = (XHCI_TRB*)0;
pipe->HcdPipeContext = (void*)0;
MmFreePool(ctx);
}
// Implements the hcd Reset Root Port operation.
@@ -1819,6 +1959,7 @@ const USB_HCD_OPERATIONS gXhciHcdOperations = {
XhciHcdCancelUrb,
XhciHcdResetRootPort,
XhciHcdClearRootPortChanges,
XhciHcdDestroyPipe,
XhciHcdAllocateDma,
};
@@ -1886,7 +2027,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
uint64_t mappedLength = 0;
st = PciMapBar(xhci->Pdo,
0,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&xhci->Mmio,
&mappedLength);
if (!KSUCCESS(st)) {
@@ -1960,17 +2101,37 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_NOT_SUPPORTED;
}
IO_DMA_REQUIREMENTS dmaRequirements;
memset(&dmaRequirements, 0, sizeof(dmaRequirements));
dmaRequirements.MaximumDeviceAddress =
(xhci->Hcc1 & XHCI_HCC1_AC64) ? UINT64_MAX : UINT32_MAX;
dmaRequirements.Alignment = 1;
dmaRequirements.Boundary = 0x10000u;
dmaRequirements.MaximumSegmentLength = 0x10000u;
dmaRequirements.MaximumSegments = XHCI_PIPE_RING_TRBS - 1u;
st = xhci->Ddk->Dma->GetAdapter(xhci->Fdo,
&dmaRequirements,
&xhci->DmaAdapter);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI DMA adapter creation failed st=0x%x\n", st);
return st;
}
st = XhciAllocateRuntimeMemory(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciProgramRuntime(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciStart(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciSubmitNoopCommand(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
XhciPowerRootPorts(xhci);
XhciLogPortStatus(xhci);
@@ -1978,13 +2139,13 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
st = UsbCreateRootHubPdo(&xhci->Hcd);
if (!KSUCCESS(st) && st != NTLSTATUS_ALREADY_EXISTS) {
DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st);
return st;
goto failDma;
}
st = XhciRegisterController(xhci);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI controller registry full st=0x%x\n", st);
return st;
goto failDma;
}
if (xhci->Irq != 0xFFu) {
@@ -2009,6 +2170,16 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
xhci->HidDeviceCount);
return NTLSTATUS_SUCCESS;
failDma:
XhciHaltForDmaRelease(xhci);
XhciUnregisterController(xhci);
if (xhci->Hcd.RootHubPdo) {
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0;
}
XhciReleaseControllerDma(xhci);
return st;
}
// Dispatches pnp.
@@ -2037,22 +2208,22 @@ KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
xhci->SurpriseRemoved = true;
xhci->Running = false;
xhci->InterruptsEnabled = false;
if (xhci->Op) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS);
}
if (xhci->Hcd.RootHubPdo)
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
xhci->Running = false;
xhci->InterruptsEnabled = false;
if (xhci->Hcd.RootHubPdo)
XhciHaltForDmaRelease(xhci);
if (xhci->Hcd.RootHubPdo) {
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0;
}
XhciUnregisterController(xhci);
XhciReleaseControllerDma(xhci);
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;