maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

Browse Source
This commit is contained in:
maksw committed 2026-08-11 18:38:36 +02:00
1 parent 8b254f84f4
commit 4add6789ba
177 files changed
+22565 -3070
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/drivers/audio/hda/hda.c
+403
View File
@@ -0,0 +1,403 @@
// Intel High Definition Audio PCI attachment and controller lifetime management.
#include "hda.h"
#include <string.h>
const NTL_DDK* gHdaDdk;
static DRIVER_OBJECT gHdaDriver;
static HDA_CONTROLLER* gHdaControllers[HDA_MAX_CONTROLLERS];
static uint32_t gHdaControllerCount;
static SPINLOCK gHdaRegistryLock = SPINLOCK_INIT;
static const IO_DEVICE_ID gHdaIds[] = {
{ IoBusPci, { "PCI\\VEN_8086&DEV_2668", sizeof("PCI\\VEN_8086&DEV_2668") - 1u } },
{ IoBusPci, { "PCI\\VEN_15AD&DEV_1977", sizeof("PCI\\VEN_15AD&DEV_1977") - 1u } },
{ IoBusPci, { "PCI\\CC_040300", sizeof("PCI\\CC_040300") - 1u } },
};
static const IO_DRIVER_ID_TABLE gHdaIdTable = {
gHdaIds,
sizeof(gHdaIds) / sizeof(gHdaIds[0]),
};
// Registers a controller in the shared interrupt scan set.
static KSTATUS HdaRegisterController(HDA_CONTROLLER* controller) {
if (!controller)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock);
for (uint32_t index = 0; index < gHdaControllerCount; index++) {
if (gHdaControllers[index] == controller) {
KeReleaseSpinlock(&gHdaRegistryLock, flags);
return NTLSTATUS_SUCCESS;
}
}
if (gHdaControllerCount == HDA_MAX_CONTROLLERS) {
KeReleaseSpinlock(&gHdaRegistryLock, flags);
return NTLSTATUS_NO_MEMORY;
}
gHdaControllers[gHdaControllerCount++] = controller;
KeReleaseSpinlock(&gHdaRegistryLock, flags);
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the shared interrupt scan set.
static void HdaUnregisterController(HDA_CONTROLLER* controller) {
if (!controller)
return;
uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock);
for (uint32_t index = 0; index < gHdaControllerCount; index++) {
if (gHdaControllers[index] != controller)
continue;
gHdaControllers[index] = gHdaControllers[gHdaControllerCount - 1u];
gHdaControllers[gHdaControllerCount - 1u] = (HDA_CONTROLLER*)0;
gHdaControllerCount--;
break;
}
KeReleaseSpinlock(&gHdaRegistryLock, flags);
}
// Dispatches a hardware interrupt to every live HDA controller.
static void HdaIsr(void) {
DEVICE_OBJECT* devices[HDA_MAX_CONTROLLERS];
uint32_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock);
count = gHdaControllerCount;
if (count > HDA_MAX_CONTROLLERS)
count = HDA_MAX_CONTROLLERS;
for (uint32_t index = 0; index < count; index++) {
HDA_CONTROLLER* controller = gHdaControllers[index];
devices[index] = controller ? controller->Fdo : (DEVICE_OBJECT*)0;
if (devices[index])
IoReferenceDeviceObject(devices[index]);
}
KeReleaseSpinlock(&gHdaRegistryLock, flags);
for (uint32_t index = 0; index < count; index++) {
DEVICE_OBJECT* device = devices[index];
if (!device)
continue;
HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension;
if (controller && controller->Magic == HDA_CONTROLLER_MAGIC)
HdaHandleInterrupt(controller);
IoDereferenceDeviceObject(device);
}
}
// Registers the global ISR once for each kernel IRQ slot used by HDA.
static KSTATUS HdaRegisterIrq(uint8_t irq) {
if (irq >= 24u)
return NTLSTATUS_NOT_SUPPORTED;
KeIrqRegister(irq, HdaIsr);
return NTLSTATUS_SUCCESS;
}
// Maps BAR0 and validates the stream-descriptor aperture described by GCAP.
static KSTATUS HdaMapRegisters(HDA_CONTROLLER* controller) {
PCI_BAR bar;
KSTATUS status = PciGetBar(controller->Pdo, 0, &bar);
if (!KSUCCESS(status))
return status;
if (bar.Type != PciBarMemory32 && bar.Type != PciBarMemory64)
return NTLSTATUS_NOT_SUPPORTED;
status = PciMapBar(controller->Pdo,
0,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&controller->Registers,
&controller->RegisterBytes);
if (!KSUCCESS(status))
return status;
if (controller->RegisterBytes < HDA_REG_STREAM_BASE)
return NTLSTATUS_BUFFER_TOO_SMALL;
controller->Gcap = HdaRead16(controller, HDA_REG_GCAP);
controller->InputStreamCount =
(uint8_t)((controller->Gcap & HDA_GCAP_ISS_MASK) >> HDA_GCAP_ISS_SHIFT);
controller->OutputStreamCount =
(uint8_t)((controller->Gcap & HDA_GCAP_OSS_MASK) >> HDA_GCAP_OSS_SHIFT);
controller->BidirectionalStreamCount =
(uint8_t)((controller->Gcap & HDA_GCAP_BSS_MASK) >> HDA_GCAP_BSS_SHIFT);
controller->TotalStreamCount =
(uint8_t)(controller->InputStreamCount +
controller->OutputStreamCount +
controller->BidirectionalStreamCount);
if ((controller->OutputStreamCount == 0 &&
controller->BidirectionalStreamCount == 0) ||
controller->TotalStreamCount == 0 ||
controller->TotalStreamCount > 30u)
return NTLSTATUS_NOT_SUPPORTED;
uint64_t requiredBytes = HDA_REG_STREAM_BASE +
(uint64_t)controller->TotalStreamCount *
HDA_REG_STREAM_STRIDE;
if (controller->RegisterBytes < requiredBytes)
return NTLSTATUS_BUFFER_TOO_SMALL;
// Dedicated output descriptors precede bidirectional descriptors. Use the
// first bidirectional descriptor when no output-only descriptor exists.
uint8_t outputDescriptorBase = controller->InputStreamCount;
uint8_t bidirectionalDescriptorBase =
(uint8_t)(controller->InputStreamCount + controller->OutputStreamCount);
controller->Output.DescriptorIndex = controller->OutputStreamCount
? outputDescriptorBase
: bidirectionalDescriptorBase;
controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG;
return NTLSTATUS_SUCCESS;
}
// Configures MSI first and enables INTx only after MSI setup reports failure.
static KSTATUS HdaConfigureInterrupt(HDA_CONTROLLER* controller) {
KSTATUS status = PciEnableMsiAuto(controller->Pdo, &controller->Irq);
if (KSUCCESS(status)) {
controller->UsingMsi = true;
return NTLSTATUS_SUCCESS;
}
gHdaDdk->Debug->Print(
"HDA: MSI setup failed status=0x%x; attempting explicit INTx fallback\n",
status);
controller->UsingMsi = false;
status = PciGetInterrupt(controller->Pdo, &controller->Irq);
if (!KSUCCESS(status))
return status;
if (controller->Irq == 0xFFu || controller->Irq >= 24u)
return NTLSTATUS_NOT_SUPPORTED;
status = PciEnableDevice(controller->Pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX);
return status;
}
// Waits until queued or active interrupt work no longer references the controller.
static KSTATUS HdaWaitForControllerInterruptWorkIdle(HDA_CONTROLLER* controller) {
if (!controller)
return NTLSTATUS_INVALID_PARAMETER;
for (uint32_t elapsed = 0; elapsed < 100000u; elapsed += 10u) {
if (controller->IsrActive == 0 &&
controller->Dpc.Inserted == 0 && controller->DpcActive == 0)
return NTLSTATUS_SUCCESS;
gHdaDdk->Timer->Stall(10);
}
return NTLSTATUS_TIMEOUT;
}
// Starts one PCI HDA controller and leaves endpoint registration to codec discovery.
static KSTATUS HdaStartDevice(HDA_CONTROLLER* controller) {
if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC ||
!controller->Pdo || !controller->Fdo)
return NTLSTATUS_INVALID_PARAMETER;
if (controller->Started)
return NTLSTATUS_SUCCESS;
KSTATUS status = PciGetDeviceInfo(controller->Pdo, &controller->PciInfo);
if (!KSUCCESS(status))
return status;
if (controller->PciInfo.ClassCode != 0x04u ||
controller->PciInfo.Subclass != 0x03u ||
controller->PciInfo.ProgIf != 0x00u)
return NTLSTATUS_NOT_SUPPORTED;
status = PciEnableDevice(controller->Pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_DISABLE_INTX);
if (!KSUCCESS(status))
return status;
status = HdaMapRegisters(controller);
if (!KSUCCESS(status))
return status;
status = HdaConfigureInterrupt(controller);
if (!KSUCCESS(status))
return status;
status = HdaAllocateDma(controller);
if (!KSUCCESS(status))
return status;
KeInitializeDpc(&controller->Dpc, HdaDpcRoutine, controller);
status = HdaStartRecoveryWorker(controller);
if (!KSUCCESS(status)) {
HdaReleaseDma(controller);
return status;
}
status = HdaInitializeController(controller);
if (!KSUCCESS(status)) {
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
status = HdaRegisterController(controller);
if (!KSUCCESS(status)) {
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
status = HdaRegisterIrq(controller->Irq);
if (!KSUCCESS(status)) {
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
controller->Started = true;
controller->Fdo->PnpState = DevicePnpStarted;
status = HdaCodecInitialize(controller);
if (!KSUCCESS(status) && status != NTLSTATUS_NOT_SUPPORTED &&
status != NTLSTATUS_NOT_FOUND) {
controller->Started = false;
controller->Fdo->PnpState = DevicePnpNotStarted;
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
if (!KSUCCESS(status)) {
gHdaDdk->Debug->Print(
"HDA: no analog endpoint registered status=0x%x\n",
status);
}
HdaArmInterrupts(controller);
HdaHandleInterrupt(controller);
return NTLSTATUS_SUCCESS;
}
// Handles PnP lifetime transitions for one HDA function device object.
static KSTATUS HdaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return NTLSTATUS_INVALID_PARAMETER;
HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension;
if (controller->Magic != HDA_CONTROLLER_MAGIC)
return NTLSTATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* stack = IoGetCurrentIrpStackLocation(irp);
if (!stack || stack->MajorFunction != IRP_MJ_PNP)
return NTLSTATUS_INVALID_PARAMETER;
switch ((IO_PNP_MINOR_FUNCTION)stack->MinorFunction) {
case IRP_MN_START_DEVICE:
return HdaStartDevice(controller);
case IRP_MN_QUERY_REMOVE_DEVICE:
case IRP_MN_CANCEL_REMOVE_DEVICE:
return NTLSTATUS_SUCCESS;
case IRP_MN_SURPRISE_REMOVAL:
controller->Started = false;
controller->SurpriseRemoved = true;
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
KSTATUS surpriseDpcStatus =
HdaWaitForControllerInterruptWorkIdle(controller);
if (!KSUCCESS(surpriseDpcStatus)) {
KePanic(KE_PANIC_CODE_DRIVER,
"HDA: interrupt work did not quiesce during surprise removal status=0x%x\n",
surpriseDpcStatus);
}
HdaCodecStop(controller, IoAudioSurpriseRemoval);
HdaCodecDestroy(controller);
device->PnpState = DevicePnpSurpriseRemoved;
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
if (!controller->SurpriseRemoved && controller->Registers)
HdaWrite32(controller, HDA_REG_INTCTL, 0);
controller->Started = false;
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
KSTATUS dpcStatus = HdaWaitForControllerInterruptWorkIdle(controller);
if (!KSUCCESS(dpcStatus)) {
KePanic(KE_PANIC_CODE_DRIVER,
"HDA: interrupt work did not quiesce during removal status=0x%x\n",
dpcStatus);
}
HdaCodecStop(controller,
controller->SurpriseRemoved
? IoAudioSurpriseRemoval
: IoAudioOrderlyRemoval);
HdaCodecDestroy(controller);
if (!controller->SurpriseRemoved)
HdaQuiesceController(controller);
controller->Removed = true;
HdaReleaseDma(controller);
if (!controller->SurpriseRemoved)
(void)PciEnableDevice(controller->Pdo, PCI_DISABLE_INTX);
device->PnpState = DevicePnpDeleted;
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
default:
return IoForwardIrpToLower(device, irp);
}
}
// Attaches one HDA function device object to a matching PCI PDO.
static KSTATUS HdaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject ||
physicalDeviceObject->Type != DeviceObjectPhysical)
return NTLSTATUS_INVALID_PARAMETER;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS status = IoCreateDevice(driver, sizeof(HDA_CONTROLLER), &fdo);
if (!KSUCCESS(status))
return status;
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("hda"));
HDA_CONTROLLER* controller = (HDA_CONTROLLER*)fdo->DeviceExtension;
memset(controller, 0, sizeof(*controller));
controller->Magic = HDA_CONTROLLER_MAGIC;
controller->Fdo = fdo;
controller->Pdo = physicalDeviceObject;
controller->Irq = 0xFFu;
controller->StateLock = (SPINLOCK)SPINLOCK_INIT;
controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG;
controller->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!controller->LowerDevice) {
IoDeleteDevice(fdo);
return NTLSTATUS_FAILURE;
}
return NTLSTATUS_SUCCESS;
}
// Registers the Intel HDA transport driver with the I/O manager.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Debug || !ddk->Debug->Print || !ddk->Dma || !ddk->Timer ||
!ddk->Timer->QueryTicks || !ddk->Timer->Stall || !ddk->Audio ||
!ddk->Scheduler || !ddk->Scheduler->DelayThread)
return NTLSTATUS_INVALID_PARAMETER;
gHdaDdk = ddk;
gHdaControllerCount = 0;
memset(gHdaControllers, 0, sizeof(gHdaControllers));
memset(&gHdaDriver, 0, sizeof(gHdaDriver));
IoInitializeDriverObject(&gHdaDriver, SYS_SV_LITERAL("hda"));
gHdaDriver.IdTable = &gHdaIdTable;
gHdaDriver.DriverAddDevice = HdaAddDevice;
gHdaDriver.MajorFunction[IRP_MJ_PNP] = HdaDispatchPnp;
KSTATUS status = IoRegisterDriverObject(&gHdaDriver);
if (!KSUCCESS(status))
gHdaDdk->Debug->Print("HDA: IoRegisterDriverObject failed status=0x%x\n", status);
return status;
}
code/drivers/audio/hda/hda.h
+389
View File
@@ -0,0 +1,389 @@
// Intel High Definition Audio transport declarations shared by the controller and stream implementation.
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddi/pci.h>
#include <ntlite/ddk/ddk.h>
#define HDA_CONTROLLER_MAGIC 0x4144484Eu
#define HDA_MAX_CONTROLLERS 8u
#define HDA_MAX_CODECS 15u
#define HDA_MAX_NODE_ID 0xFFu
#define HDA_MAX_UNSOLICITED_RESPONSES 32u
#define HDA_OUTPUT_SAMPLE_RATE 48000u
#define HDA_OUTPUT_CHANNELS 2u
#define HDA_OUTPUT_VALID_BITS 16u
#define HDA_OUTPUT_BYTES_PER_SAMPLE 2u
#define HDA_OUTPUT_BYTES_PER_FRAME 4u
#define HDA_OUTPUT_PERIOD_FRAMES 512u
#define HDA_OUTPUT_PERIOD_COUNT 4u
#define HDA_OUTPUT_PERIOD_BYTES (HDA_OUTPUT_PERIOD_FRAMES * HDA_OUTPUT_BYTES_PER_FRAME)
#define HDA_OUTPUT_BUFFER_BYTES (HDA_OUTPUT_PERIOD_COUNT * HDA_OUTPUT_PERIOD_BYTES)
#define HDA_OUTPUT_STREAM_FORMAT 0x0011u
#define HDA_OUTPUT_STREAM_TAG 1u
#define HDA_MAX_LOGICAL_ENDPOINTS 3u
#define HDA_REG_GCAP 0x00u
#define HDA_REG_VMIN 0x02u
#define HDA_REG_VMAJ 0x03u
#define HDA_REG_GCTL 0x08u
#define HDA_REG_STATESTS 0x0Eu
#define HDA_REG_INTCTL 0x20u
#define HDA_REG_INTSTS 0x24u
#define HDA_REG_WALLCLK 0x30u
#define HDA_REG_CORBLBASE 0x40u
#define HDA_REG_CORBUBASE 0x44u
#define HDA_REG_CORBWP 0x48u
#define HDA_REG_CORBRP 0x4Au
#define HDA_REG_CORBCTL 0x4Cu
#define HDA_REG_CORBSTS 0x4Du
#define HDA_REG_CORBSIZE 0x4Eu
#define HDA_REG_RIRBLBASE 0x50u
#define HDA_REG_RIRBUBASE 0x54u
#define HDA_REG_RIRBWP 0x58u
#define HDA_REG_RINTCNT 0x5Au
#define HDA_REG_RIRBCTL 0x5Cu
#define HDA_REG_RIRBSTS 0x5Du
#define HDA_REG_RIRBSIZE 0x5Eu
#define HDA_REG_DPLBASE 0x70u
#define HDA_REG_DPUBASE 0x74u
#define HDA_REG_STREAM_BASE 0x80u
#define HDA_REG_STREAM_STRIDE 0x20u
#define HDA_GCAP_64OK 0x0001u
#define HDA_GCAP_BSS_MASK 0x00F8u
#define HDA_GCAP_BSS_SHIFT 3u
#define HDA_GCAP_ISS_MASK 0x0F00u
#define HDA_GCAP_ISS_SHIFT 8u
#define HDA_GCAP_OSS_MASK 0xF000u
#define HDA_GCAP_OSS_SHIFT 12u
#define HDA_GCTL_RESET 0x00000001u
#define HDA_GCTL_UNSOLICITED_ENABLE 0x00000100u
#define HDA_INT_CONTROLLER_ENABLE 0x40000000u
#define HDA_INT_GLOBAL_ENABLE 0x80000000u
#define HDA_INT_CONTROLLER_STATUS 0x40000000u
#define HDA_INT_GLOBAL_STATUS 0x80000000u
#define HDA_CORBRP_POINTER_MASK 0x00FFu
#define HDA_CORBRP_RESET 0x8000u
#define HDA_CORBCTL_MEMORY_ERROR_INTERRUPT 0x01u
#define HDA_CORBCTL_RUN 0x02u
#define HDA_CORBSTS_MEMORY_ERROR 0x01u
#define HDA_RIRBWP_RESET 0x8000u
#define HDA_RIRBCTL_RESPONSE_INTERRUPT 0x01u
#define HDA_RIRBCTL_RUN 0x02u
#define HDA_RIRBCTL_OVERRUN_INTERRUPT 0x04u
#define HDA_RIRBSTS_RESPONSE 0x01u
#define HDA_RIRBSTS_OVERRUN 0x04u
#define HDA_RIRB_RESPONSE_CODEC_MASK 0x0000000Fu
#define HDA_RIRB_RESPONSE_UNSOLICITED 0x00000010u
#define HDA_RING_SIZE_CAP_2 0x10u
#define HDA_RING_SIZE_CAP_16 0x20u
#define HDA_RING_SIZE_CAP_256 0x40u
#define HDA_RING_SIZE_SELECT_2 0x00u
#define HDA_RING_SIZE_SELECT_16 0x01u
#define HDA_RING_SIZE_SELECT_256 0x02u
#define HDA_DPLBASE_ENABLE 0x00000001u
#define HDA_STREAM_REG_CTL 0x00u
#define HDA_STREAM_REG_STS 0x03u
#define HDA_STREAM_REG_LPIB 0x04u
#define HDA_STREAM_REG_CBL 0x08u
#define HDA_STREAM_REG_LVI 0x0Cu
#define HDA_STREAM_REG_FORMAT 0x12u
#define HDA_STREAM_REG_BDLPL 0x18u
#define HDA_STREAM_REG_BDLPU 0x1Cu
#define HDA_STREAM_CTL_RESET 0x000001u
#define HDA_STREAM_CTL_RUN 0x000002u
#define HDA_STREAM_CTL_INTERRUPT_COMPLETE 0x000004u
#define HDA_STREAM_CTL_INTERRUPT_FIFO_ERROR 0x000008u
#define HDA_STREAM_CTL_INTERRUPT_DESCRIPTOR_ERROR 0x000010u
#define HDA_STREAM_CTL_INTERRUPT_MASK 0x00001Cu
#define HDA_STREAM_CTL_TAG_SHIFT 20u
#define HDA_STREAM_CTL_TAG_MASK 0x00F00000u
#define HDA_STREAM_STS_COMPLETE 0x04u
#define HDA_STREAM_STS_FIFO_ERROR 0x08u
#define HDA_STREAM_STS_DESCRIPTOR_ERROR 0x10u
#define HDA_STREAM_STS_ERROR_MASK 0x18u
#define HDA_STREAM_STS_INTERRUPT_MASK 0x1Cu
#define HDA_BDL_FLAG_INTERRUPT_ON_COMPLETION 0x00000001u
#define HDA_VERB_GET_PARAMETER(parameterId) (0xF0000u | ((uint32_t)(parameterId) & 0xFFu))
#define HDA_PARAMETER_VENDOR_ID 0x00u
#define HDA_CONTROLLER_RESET_TIMEOUT_US 100000u
#define HDA_STREAM_RESET_TIMEOUT_US 10000u
#define HDA_RING_RESET_TIMEOUT_US 10000u
#define HDA_CODEC_COMMAND_TIMEOUT_US 100000u
#define HDA_CODEC_COMMAND_ATTEMPTS 2u
#define HDA_CODEC_RESET_SETTLE_US 1000u
#define HDA_STREAM_ERROR_WINDOW_US 1000000ull
#define HDA_STREAM_ERRORS_BEFORE_CONTROLLER_RESET 3u
#define HDA_RECOVERY_NONE 0u
#define HDA_RECOVERY_STREAM 1u
#define HDA_RECOVERY_CONTROLLER 2u
#pragma pack(push, 1)
typedef struct HDA_BDL_ENTRY {
uint64_t Address;
uint32_t Length;
uint32_t Flags;
} HDA_BDL_ENTRY;
typedef struct HDA_RIRB_ENTRY {
uint32_t Response;
uint32_t ResponseExtra;
} HDA_RIRB_ENTRY;
#pragma pack(pop)
_Static_assert(sizeof(HDA_BDL_ENTRY) == 16u, "HDA BDL entries must be 16 bytes");
_Static_assert(sizeof(HDA_RIRB_ENTRY) == 8u, "HDA RIRB entries must be 8 bytes");
typedef struct HDA_CONTROLLER HDA_CONTROLLER;
typedef struct HDA_CODEC_STATE HDA_CODEC_STATE;
typedef struct HDA_OUTPUT_DESCRIPTION {
void* PeriodBuffer;
size_t PeriodBufferBytes;
uint32_t PeriodCount;
uint32_t FramesPerPeriod;
size_t PeriodStrideBytes;
IO_AUDIO_FORMAT HardwareFormat;
uint8_t StreamTag;
} HDA_OUTPUT_DESCRIPTION;
typedef struct HDA_TRANSPORT_CALLBACKS {
// Reports one BDL-owned period after completion interrupt processing.
void (*PeriodComplete)(void* context,
uint32_t completedPeriodIndex,
uint64_t presentedFrames,
uint64_t finePositionFrames,
uint64_t interruptCount,
uint64_t timestampTicks);
// Delivers one unsolicited codec response outside the controller lock.
void (*UnsolicitedResponse)(void* context,
uint8_t codecAddress,
uint32_t response,
uint32_t responseExtra);
// Restores codec programming in a muted state while transport start and stop are serialized.
KSTATUS (*RestoreAfterControllerReset)(void* context);
// Unmutes the restored route only after controller recovery has restarted DMA.
KSTATUS (*ResumeAfterControllerReset)(void* context);
// Reports a recovery failure that leaves the transport stopped.
void (*TransportFailure)(void* context, KSTATUS status);
} HDA_TRANSPORT_CALLBACKS;
typedef struct HDA_UNSOLICITED_RESPONSE {
uint32_t Response;
uint32_t ResponseExtra;
} HDA_UNSOLICITED_RESPONSE;
typedef struct HDA_OUTPUT_RING {
IO_DMA_BUFFER* Dma;
uint8_t* Buffer;
uint64_t DeviceAddress;
uint8_t NextLogicalPeriod;
uint64_t PresentedFrames;
} HDA_OUTPUT_RING;
typedef struct HDA_OUTPUT_STREAM {
uint8_t DescriptorIndex;
uint8_t StreamTag;
uint8_t BdlPeriodMap[HDA_OUTPUT_PERIOD_COUNT];
uint8_t NextCompletionBdl;
bool Prepared;
bool Running;
volatile long PendingPeriodInterrupts;
volatile long PendingDmaErrors;
HDA_OUTPUT_RING Rings[HDA_MAX_LOGICAL_ENDPOINTS];
uint8_t ActiveRingIndex;
IO_DMA_BUFFER* BdlDma;
HDA_BDL_ENTRY* Bdl;
uint64_t BdlDeviceAddress;
uint64_t PositionRunBaseFrames;
uint64_t LastPositionFrames;
uint64_t PositionWrapFrames;
uint32_t LastHardwarePositionBytes;
bool PositionInitialized;
uint64_t InterruptCount;
uint64_t PeriodCompletionCount;
uint64_t DmaErrorCount;
uint64_t StreamResetCount;
uint64_t PositionBufferReadCount;
uint64_t LpibReadCount;
uint64_t PositionDiscontinuityCount;
} HDA_OUTPUT_STREAM;
struct HDA_CONTROLLER {
uint32_t Magic;
DEVICE_OBJECT* Fdo;
DEVICE_OBJECT* Pdo;
DEVICE_OBJECT* LowerDevice;
PCI_DEVICE_INFO PciInfo;
volatile uint8_t* Registers;
uint64_t RegisterBytes;
uint16_t Gcap;
uint8_t InputStreamCount;
uint8_t OutputStreamCount;
uint8_t BidirectionalStreamCount;
uint8_t TotalStreamCount;
uint16_t CodecMask;
uint8_t Irq;
bool UsingMsi;
bool Started;
bool Removed;
bool SurpriseRemoved;
SPINLOCK StateLock;
KDPC Dpc;
volatile long IsrActive;
volatile long DpcActive;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* CorbDma;
IO_DMA_BUFFER* RirbDma;
IO_DMA_BUFFER* PositionDma;
uint32_t* Corb;
HDA_RIRB_ENTRY* Rirb;
volatile uint64_t* PositionBuffer;
uint64_t CorbDeviceAddress;
uint64_t RirbDeviceAddress;
uint64_t PositionDeviceAddress;
uint16_t CorbEntryCount;
uint16_t RirbEntryCount;
uint16_t CorbWritePointer;
uint16_t RirbReadPointer;
MUTEX* CommandMutex;
MUTEX* StreamMutex;
bool CommandWaiting;
bool CommandResponseReady;
uint8_t CommandCodecAddress;
uint32_t CommandResponse;
uint32_t CommandResponseExtra;
uint64_t CodecCommandCount;
uint64_t CodecCommandRetryCount;
uint64_t CodecCommandFailureCount;
uint64_t RirbResponseCount;
uint64_t RirbUnsolicitedCount;
uint64_t RirbOverrunCount;
uint64_t CorbMemoryErrorCount;
HDA_UNSOLICITED_RESPONSE Unsolicited[HDA_MAX_UNSOLICITED_RESPONSES];
uint8_t UnsolicitedRead;
uint8_t UnsolicitedWrite;
uint8_t UnsolicitedCount;
HDA_OUTPUT_STREAM Output;
HDA_TRANSPORT_CALLBACKS Callbacks;
void* CallbackContext;
SEMAPHORE* RecoverySemaphore;
EVENT* RecoveryFinished;
THREAD* RecoveryThread;
bool RecoveryWorkerRunning;
bool RecoveryInProgress;
uint8_t PendingRecovery;
uint64_t StreamErrorWindowStart;
uint32_t StreamErrorsInWindow;
uint64_t ControllerResetCount;
HDA_CODEC_STATE* Codec;
uint64_t InterruptCount;
uint64_t ControllerInterruptCount;
};
extern const NTL_DDK* gHdaDdk;
// Returns one byte from the controller register aperture.
uint8_t HdaRead8(const HDA_CONTROLLER* controller, uint32_t offset);
// Returns one word from the controller register aperture.
uint16_t HdaRead16(const HDA_CONTROLLER* controller, uint32_t offset);
// Returns one dword from the controller register aperture.
uint32_t HdaRead32(const HDA_CONTROLLER* controller, uint32_t offset);
// Writes one byte to the controller register aperture.
void HdaWrite8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t value);
// Writes one word to the controller register aperture.
void HdaWrite16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t value);
// Writes one dword to the controller register aperture.
void HdaWrite32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t value);
// Returns the register offset of the controller's one output stream descriptor.
uint32_t HdaOutputStreamBase(const HDA_CONTROLLER* controller);
// Allocates the transport and output DMA resources for one controller.
KSTATUS HdaAllocateDma(HDA_CONTROLLER* controller);
// Releases all DMA and synchronization resources owned by one controller.
void HdaReleaseDma(HDA_CONTROLLER* controller);
// Resets and initializes the controller, command rings, position buffer, and output descriptor.
KSTATUS HdaInitializeController(HDA_CONTROLLER* controller);
// Arms controller and output-stream interrupts after ISR registration.
void HdaArmInterrupts(HDA_CONTROLLER* controller);
// Stops hardware access without releasing software resources.
void HdaQuiesceController(HDA_CONTROLLER* controller);
// Sends one 20-bit codec verb payload through CORB/RIRB with one retry.
KSTATUS HdaCodecVerb(HDA_CONTROLLER* controller,
uint8_t codecAddress,
uint8_t nodeId,
uint32_t verbPayload,
uint32_t* responseOut);
// Copies one logical endpoint ring contract for codec and endpoint setup.
KSTATUS HdaDescribeOutput(HDA_CONTROLLER* controller,
uint8_t ringIndex,
HDA_OUTPUT_DESCRIPTION* descriptionOut);
// Selects one logical endpoint ring while the stream is stopped.
KSTATUS HdaSelectOutputRing(HDA_CONTROLLER* controller, uint8_t ringIndex);
// Replaces the transport callbacks while the output stream is stopped.
KSTATUS HdaSetTransportCallbacks(HDA_CONTROLLER* controller,
const HDA_TRANSPORT_CALLBACKS* callbacks,
void* context);
// Starts the prepared fixed-format output stream.
KSTATUS HdaStartOutput(HDA_CONTROLLER* controller);
// Stops the output stream while retaining its DMA ring.
void HdaStopOutput(HDA_CONTROLLER* controller);
// Returns the monotonic fine position derived from the position buffer or LPIB.
uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller);
// Captures one controller interrupt and queues deferred processing.
void HdaHandleInterrupt(HDA_CONTROLLER* controller);
// Runs period delivery, unsolicited callbacks, and recovery scheduling at DPC level.
void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2);
// Starts the passive recovery worker for one controller.
KSTATUS HdaStartRecoveryWorker(HDA_CONTROLLER* controller);
// Stops and joins the passive recovery worker for one controller.
void HdaStopRecoveryWorker(HDA_CONTROLLER* controller);
// Discovers codec topology and registers usable analog render endpoints.
KSTATUS HdaCodecInitialize(HDA_CONTROLLER* controller);
// Stops jack workers and unregisters codec endpoints.
void HdaCodecStop(HDA_CONTROLLER* controller, IO_AUDIO_REMOVAL_MODE mode);
// Releases codec topology storage after interrupt work is quiescent.
void HdaCodecDestroy(HDA_CONTROLLER* controller);
code/drivers/audio/hda/hda_codec.c
+1928
View File
File diff suppressed because it is too large. Load diff
code/drivers/audio/hda/hda_transport.c
+1421
View File
File diff suppressed because it is too large. Load diff
code/drivers/bus/pci/pci.c
+1 -1
View File
@@ -33,7 +33,7 @@ static uint64_t PciAlignUp64(uint64_t value, uint64_t alignment) {
// Implements the default Mmio Flags operation.
static uint64_t PciDefaultMmioFlags(void) {
return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_WRITETHROUGH | NTL_DDK_PAGE_NOCACHE | NTL_DDK_PAGE_NX;
return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_CACHE_UC | NTL_DDK_PAGE_NX;
}
// Creates mmio Va.
code/drivers/bus/usb/usb.c
+1 -1
View File
@@ -37,7 +37,7 @@ static KSTATUS UsbInitializeXhciDriver(void) {
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER;
if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("usb: driver loading\n");
gUsbDdk = ddk;
code/drivers/bus/usb/usb.h
+83 -40
View File
@@ -124,6 +124,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define XHCI_TRB_SETUP_TRT_IN (3u << 16)
#define XHCI_TRB_STATUS_DIR_IN (1u << 16)
#define XHCI_HCC1_AC64 (1u << 0)
#define XHCI_HCC1_CSZ (1u << 2)
#define XHCI_CONTEXT_ENTRIES(n) ((uint32_t)(n) << 27)
@@ -199,6 +200,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define SCSI_INQUIRY 0x12u
#define SCSI_READ_10 0x28u
#define SCSI_WRITE_10 0x2Au
#define SCSI_SYNCHRONIZE_CACHE_10 0x35u
#define SCSI_READ_CAPACITY_10 0x25u
#define XHCI_RING_TRBS 256u
@@ -393,7 +395,11 @@ typedef struct USB_HCD_OPERATIONS {
KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb);
KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed);
KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port);
void* (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys);
void (*DestroyPipe)(USB_HCD* hcd, USB_PIPE* pipe);
void* (*AllocateDma)(USB_HCD* hcd,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer);
} USB_HCD_OPERATIONS;
struct USB_HCD {
@@ -454,7 +460,7 @@ struct USB_URB {
USB_PIPE* Pipe;
USB_SETUP_PACKET Setup;
void* Buffer;
PHYSICAL_ADDRESS BufferPhys;
uint64_t BufferDeviceAddress;
uint32_t Length;
uint32_t ActualLength;
uint32_t HcdCompletionCode;
@@ -521,8 +527,9 @@ typedef struct USB_HUB_EXTENSION {
uint32_t PendingHubPorts[1];
SPINLOCK PendingLock;
SEMAPHORE* WorkerSemaphore;
EVENT* WorkerFinished;
THREAD* WorkerThread;
bool WorkerRunning;
volatile bool WorkerRunning;
bool InitialScanPending;
bool InitialScanComplete;
bool InitialScanInProgress;
@@ -533,7 +540,8 @@ typedef struct USB_HUB_EXTENSION {
uint8_t InterruptInterval;
USB_PIPE* InterruptPipe;
uint8_t* ChangeBitmap;
PHYSICAL_ADDRESS ChangeBitmapPhys;
uint64_t ChangeBitmapDeviceAddress;
IO_DMA_BUFFER* ChangeBitmapDma;
USB_URB ChangeUrb;
bool ChangeUrbPending;
uint8_t PowerOnToPowerGoodMs;
@@ -573,7 +581,8 @@ typedef struct USB_HID_EXTENSION {
USB_PIPE* InterruptPipe;
uint8_t* ReportBuffer;
PHYSICAL_ADDRESS ReportBufferPhys;
uint64_t ReportBufferDeviceAddress;
IO_DMA_BUFFER* ReportBufferDma;
uint16_t ReportLength;
USB_URB InterruptUrb;
bool InterruptPending;
@@ -584,6 +593,12 @@ typedef struct USB_HID_EXTENSION {
uint32_t InputDeviceId;
} USB_HID_EXTENSION;
typedef enum USB_STORAGE_FLUSH_CAPABILITY {
UsbStorageFlushUnknown = 0,
UsbStorageFlushSupported,
UsbStorageFlushUnsupported,
} USB_STORAGE_FLUSH_CAPABILITY;
typedef enum USB_STORAGE_EXTENSION_KIND {
UsbStorageExtensionInterface = 0x55534946u,
UsbStorageExtensionDisk = 0x5553444Bu,
@@ -620,6 +635,7 @@ struct USB_STORAGE_EXTENSION {
uint32_t Tag;
uint32_t BlockSize;
uint64_t BlockCount;
USB_STORAGE_FLUSH_CAPABILITY FlushCapability;
volatile long IoBusy;
};
@@ -630,14 +646,15 @@ struct USB_STORAGE_DISK_EXTENSION {
typedef struct XHCI_PIPE_CONTEXT {
XHCI_TRB* Ring;
PHYSICAL_ADDRESS RingPhys;
uint64_t RingDeviceAddress;
IO_DMA_BUFFER* RingDma;
uint32_t Enqueue;
uint8_t Cycle;
uint8_t EndpointId;
USB_PIPE_TYPE Type;
bool Configured;
bool ActiveTransfer;
uint64_t ActiveTrbPhys;
uint64_t ActiveTrbDeviceAddress;
USB_URB* ActiveUrb;
} XHCI_PIPE_CONTEXT;
@@ -683,7 +700,7 @@ typedef struct XHCI_DEVICE {
uint8_t HidLastModifiers;
uint8_t HidLastKeys[6];
uint8_t HidLastButtons;
uint64_t HidLastTrbPhys;
uint64_t HidLastTrbDeviceAddress;
bool MassCandidate;
bool MassActive;
@@ -708,15 +725,25 @@ typedef struct XHCI_DEVICE {
bool MassRemoved;
volatile long MassIoBusy;
PHYSICAL_ADDRESS InputContextPhys;
PHYSICAL_ADDRESS DeviceContextPhys;
PHYSICAL_ADDRESS Ep0RingPhys;
PHYSICAL_ADDRESS HidRingPhys;
PHYSICAL_ADDRESS MassInRingPhys;
PHYSICAL_ADDRESS MassOutRingPhys;
PHYSICAL_ADDRESS DescriptorPhys;
PHYSICAL_ADDRESS HidBufferPhys;
PHYSICAL_ADDRESS MassBufferPhys;
uint64_t InputContextDeviceAddress;
uint64_t DeviceContextDeviceAddress;
uint64_t Ep0RingDeviceAddress;
uint64_t HidRingDeviceAddress;
uint64_t MassInRingDeviceAddress;
uint64_t MassOutRingDeviceAddress;
uint64_t DescriptorDeviceAddress;
uint64_t HidBufferDeviceAddress;
uint64_t MassBufferDeviceAddress;
IO_DMA_BUFFER* InputContextDma;
IO_DMA_BUFFER* DeviceContextDma;
IO_DMA_BUFFER* Ep0RingDma;
IO_DMA_BUFFER* HidRingDma;
IO_DMA_BUFFER* MassInRingDma;
IO_DMA_BUFFER* MassOutRingDma;
IO_DMA_BUFFER* DescriptorDma;
IO_DMA_BUFFER* HidBufferDma;
IO_DMA_BUFFER* MassBufferDma;
uint8_t* InputContext;
uint8_t* DeviceContext;
@@ -746,8 +773,8 @@ typedef struct XHCI_TRANSFER_RESULT {
typedef struct XHCI_WAIT_TARGET {
struct XHCI_WAIT_TARGET* Next;
uint64_t CommandPhys;
uint64_t TransferPhys;
uint64_t CommandDeviceAddress;
uint64_t TransferDeviceAddress;
uint8_t TransferSlotId;
uint8_t TransferEndpointId;
volatile bool CommandMatched;
@@ -793,12 +820,20 @@ typedef struct XHCI_CONTROLLER {
uint8_t PortProtocolMajor[XHCI_MAX_ROOT_PORTS + 1u];
uint8_t PortProtocolMinor[XHCI_MAX_ROOT_PORTS + 1u];
PHYSICAL_ADDRESS DcbaaPhys;
PHYSICAL_ADDRESS ScratchpadArrayPhys;
PHYSICAL_ADDRESS ScratchpadPhys[XHCI_MAX_SCRATCHPADS];
PHYSICAL_ADDRESS CommandRingPhys;
PHYSICAL_ADDRESS EventRingPhys;
PHYSICAL_ADDRESS ErstPhys;
uint64_t DcbaaDeviceAddress;
uint64_t ScratchpadArrayDeviceAddress;
uint64_t ScratchpadDeviceAddress[XHCI_MAX_SCRATCHPADS];
uint64_t CommandRingDeviceAddress;
uint64_t EventRingDeviceAddress;
uint64_t ErstDeviceAddress;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* DcbaaDma;
IO_DMA_BUFFER* ScratchpadArrayDma;
IO_DMA_BUFFER* ScratchpadDma[XHCI_MAX_SCRATCHPADS];
IO_DMA_BUFFER* CommandRingDma;
IO_DMA_BUFFER* EventRingDma;
IO_DMA_BUFFER* ErstDma;
uint64_t* Dcbaa;
uint64_t* ScratchpadArray;
@@ -825,6 +860,7 @@ typedef struct XHCI_CONTROLLER {
XHCI_WAIT_TARGET* WaiterList;
volatile bool Running;
bool SurpriseRemoved;
USB_HCD Hcd;
} XHCI_CONTROLLER;
@@ -867,7 +903,11 @@ void UsbDereferencePipe(USB_PIPE* pipe);
KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbCancelUrb(USB_URB* urb);
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys);
void* UsbAllocateDmaBuffer(USB_DEVICE* dev,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer);
void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer);
KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp);
@@ -882,7 +922,11 @@ void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value);
void UsbDelayUs(const NTL_DDK* ddk, uint32_t us);
bool UsbWaitSet32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys);
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer);
void XhciReleaseDeviceDma(XHCI_DEVICE* dev);
uint16_t XhciScratchpadCount(uint32_t hcs2);
uint32_t XhciTrbType(const XHCI_TRB* trb);
uint32_t XhciCompletionCode(const XHCI_TRB* trb);
@@ -918,7 +962,7 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci);
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci);
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci);
KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci);
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index);
uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index);
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, XHCI_COMMAND_RESULT* outResult);
KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci);
KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId);
@@ -931,17 +975,17 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed);
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId);
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index);
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs);
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length);
uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index);
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress);
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length, uint32_t timeoutUs);
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length);
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup);
KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs);
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc);
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength);
uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress);
uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval);
KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress);
bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key);
INPUT_KEY XhciHidModifierKey(uint8_t bit);
void XhciSubmitHidKey(INPUT_KEY key, bool pressed);
@@ -949,24 +993,23 @@ void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint
void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len);
KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void XhciArmAllHid(XHCI_CONTROLLER* xhci);
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue);
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue);
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci);
KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle);
void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint8_t cycle);
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe);
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t* enqueue, uint8_t* cycle, uint64_t bufferDeviceAddress, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, uint64_t dataDeviceAddress, uint32_t dataLength, bool dataIn);
uint32_t UsbBe32(const uint8_t* p);
void UsbStoreBe32(uint8_t* p, uint32_t v);
void UsbStoreBe16(uint8_t* p, uint16_t v);
KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer);
code/drivers/bus/usb/usb_bus.c
+19 -6
View File
@@ -24,9 +24,12 @@ static void UsbDeviceDestroy(OBJECT* object) {
}
}
// Implements the pipe Destroy operation.
// Destroys a pipe after the host controller releases its private context.
static void UsbPipeDestroy(OBJECT* object) {
USB_PIPE* pipe = (USB_PIPE*)object;
if (pipe->Device && pipe->Device->Hcd && pipe->Device->Hcd->Operations &&
pipe->Device->Hcd->Operations->DestroyPipe)
pipe->Device->Hcd->Operations->DestroyPipe(pipe->Device->Hcd, pipe);
if (pipe->Device) {
UsbDereferenceDevice(pipe->Device);
pipe->Device = (USB_DEVICE*)0;
@@ -53,11 +56,21 @@ void UsbDereferencePipe(USB_PIPE* pipe) {
if (pipe) ObDereferenceObject(&pipe->Header);
}
// Creates dma Buffer.
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma)
// Allocates a DMA buffer through the device host-controller adapter.
void* UsbAllocateDmaBuffer(USB_DEVICE* dev,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!dev || !dev->Hcd || !dev->Hcd->Operations ||
!dev->Hcd->Operations->AllocateDma || !outDeviceAddress || !outBuffer)
return (void*)0;
return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys);
return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outDeviceAddress, outBuffer);
}
// Frees a DMA buffer allocated for a USB class driver.
void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer) {
if (buffer && gUsbDdk && gUsbDdk->Dma)
gUsbDdk->Dma->FreeCommonBuffer(buffer);
}
// Creates device.
@@ -464,7 +477,7 @@ KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) {
if (!parent)
return NTLSTATUS_INVALID_PARAMETER;
for (;;) {
while (true) {
DEVICE_OBJECT* child = UsbReferenceFirstChild(parent);
if (!child)
break;
code/drivers/bus/usb/usb_core.c
+4 -3
View File
@@ -258,6 +258,9 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
XHCI_DEVICE* dev = &xhci->Devices[slotId];
if (dev->Allocated && xhci->Running)
(void)XhciDisableSlot(xhci, slotId);
if (dev->UsbPdo) {
(void)UsbRemovePdoSubtree(dev->UsbPdo, true);
dev->UsbPdo = (DEVICE_OBJECT*)0;
@@ -269,9 +272,7 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
dev->UsbDevice = (USB_DEVICE*)0;
}
if (dev->Allocated)
(void)XhciDisableSlot(xhci, slotId);
XhciReleaseDeviceDma(dev);
memset(dev, 0, sizeof(*dev));
}
code/drivers/bus/usb/usb_hid.c
+49 -20
View File
@@ -2,6 +2,16 @@
#include "usb.h"
// Releases the HID class DMA buffer after all pending transfers are cancelled.
static void UsbHidReleaseDma(USB_HID_EXTENSION* ext) {
if (!ext)
return;
UsbFreeDmaBuffer(ext->ReportBufferDma);
ext->ReportBufferDma = (IO_DMA_BUFFER*)0;
ext->ReportBuffer = (uint8_t*)0;
ext->ReportBufferDeviceAddress = 0;
}
// Implements the hid Key In Report operation.
bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) {
if (key == 0 || key == 1) return false;
@@ -91,21 +101,21 @@ KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
memset(dev->HidBuffer, 0, dev->HidReportLength);
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciRingAppend(dev->HidRing,
dev->HidRingPhys,
dev->HidRingDeviceAddress,
XHCI_HID_RING_TRBS,
&dev->HidEnqueue,
&dev->HidCycle,
dev->HidBufferPhys,
dev->HidBufferDeviceAddress,
dev->HidReportLength,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
dev->HidLastTrbPhys = trbPhys;
dev->HidLastTrbDeviceAddress = trbDeviceAddress;
dev->HidArmed = true;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, dev->HidEndpointId);
@@ -125,13 +135,13 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
}
}
static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
if (!dev->HidActive || endpointId != dev->HidEndpointId || trbPhys != dev->HidLastTrbPhys)
if (!dev->HidActive || endpointId != dev->HidEndpointId || trbDeviceAddress != dev->HidLastTrbDeviceAddress)
return;
dev->HidArmed = false;
@@ -200,17 +210,26 @@ KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->HidRing && dev->HidBuffer) return NTLSTATUS_SUCCESS;
dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingPhys);
dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferPhys);
if (!dev->HidRing || !dev->HidBuffer)
dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingDeviceAddress, &dev->HidRingDma);
dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferDeviceAddress, &dev->HidBufferDma);
if (!dev->HidRing || !dev->HidBuffer) {
if (gUsbDdk && gUsbDdk->Dma) {
gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma);
}
dev->HidBufferDma = (IO_DMA_BUFFER*)0;
dev->HidRingDma = (IO_DMA_BUFFER*)0;
dev->HidBuffer = (uint8_t*)0;
dev->HidRing = (XHCI_TRB*)0;
return NTLSTATUS_NO_MEMORY;
}
dev->HidEnqueue = 0;
dev->HidCycle = 1;
XHCI_TRB* link = &dev->HidRing[XHCI_HID_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->HidRingPhys;
link->ParameterHi = (uint32_t)(dev->HidRingPhys >> 32);
link->ParameterLo = (uint32_t)dev->HidRingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->HidRingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -253,15 +272,15 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
ep[0] = (uint32_t)interval << 16;
ep[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_INT_IN | ((uint32_t)maxPacket << 16);
ep[2] = (uint32_t)(dev->HidRingPhys | XHCI_EP_DCS);
ep[3] = (uint32_t)((dev->HidRingPhys | XHCI_EP_DCS) >> 32);
ep[2] = (uint32_t)(dev->HidRingDeviceAddress | XHCI_EP_DCS);
ep[3] = (uint32_t)((dev->HidRingDeviceAddress | XHCI_EP_DCS) >> 32);
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -499,7 +518,7 @@ static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) {
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed)
return NTLSTATUS_OBJECT_DEAD;
if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys)
if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferDeviceAddress)
return NTLSTATUS_INVALID_PARAMETER;
if (ext->InterruptPending)
return NTLSTATUS_SUCCESS;
@@ -511,7 +530,7 @@ static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
ext->InterruptUrb.Device = ext->Device;
ext->InterruptUrb.Pipe = ext->InterruptPipe;
ext->InterruptUrb.Buffer = ext->ReportBuffer;
ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys;
ext->InterruptUrb.BufferDeviceAddress = ext->ReportBufferDeviceAddress;
ext->InterruptUrb.Length = ext->ReportLength;
ext->InterruptUrb.Complete = UsbHidInterruptComplete;
ext->InterruptUrb.CompleteContext = ext;
@@ -565,7 +584,7 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext)
if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT;
ext->ReportLength = reportLength;
ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys);
ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferDeviceAddress, &ext->ReportBufferDma);
if (!ext->ReportBuffer)
return NTLSTATUS_NO_MEMORY;
@@ -575,13 +594,22 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext)
ext->MaxPacketSize,
ext->Interval,
&ext->InterruptPipe);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
UsbHidReleaseDma(ext);
return st;
}
if (!ext->AbsoluteMouse) {
st = UsbHidSendBootProtocol(ext);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
if (ext->InterruptPipe) {
ext->InterruptPipe->Open = false;
UsbDereferencePipe(ext->InterruptPipe);
ext->InterruptPipe = (USB_PIPE*)0;
}
UsbHidReleaseDma(ext);
return st;
}
}
memset(ext->LastKeys, 0, sizeof(ext->LastKeys));
@@ -678,6 +706,7 @@ KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
}
UsbHidReleaseDma(ext);
}
KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS;
code/drivers/bus/usb/usb_hub.c
+132 -53
View File
@@ -8,6 +8,16 @@
DRIVER_OBJECT gUsbHubDriver;
// Releases the external-hub change bitmap after transfer rundown.
static void UsbHubReleaseDma(USB_HUB_EXTENSION* hub) {
if (!hub)
return;
UsbFreeDmaBuffer(hub->ChangeBitmapDma);
hub->ChangeBitmapDma = (IO_DMA_BUFFER*)0;
hub->ChangeBitmap = (uint8_t*)0;
hub->ChangeBitmapDeviceAddress = 0;
}
static const IO_DEVICE_ID gUsbHubIds[] = {
{ IoBusUsb, { "USB\\ROOT_HUB", sizeof("USB\\ROOT_HUB") - 1 } },
{ IoBusUsb, { "USB\\CLASS_09", sizeof("USB\\CLASS_09") - 1 } },
@@ -61,8 +71,10 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub,
if (maxAttempts == 0) maxAttempts = 1;
for (uint8_t attempt = 1; attempt <= maxAttempts; attempt++) {
if (!UsbHubRootPortConnected(xhci, port))
return NTLSTATUS_NOT_FOUND;
if (!UsbHubRootPortConnected(xhci, port)) {
last = NTLSTATUS_NOT_FOUND;
break;
}
uint8_t slot = 0;
last = XhciEnumeratePortEx(xhci, port, &slot);
@@ -100,6 +112,40 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub,
return last;
}
// Enumerates one connected root port through the same debounce path at boot and hotplug.
static KSTATUS UsbHubEnumerateConnectedRootPort(USB_HUB_EXTENSION* hub,
XHCI_CONTROLLER* xhci,
uint8_t port,
uint8_t* outSlot) {
if (outSlot)
*outSlot = 0;
if (!hub || !xhci || port == 0 || port > hub->PortCount ||
port > USB_ROOT_HUB_MAX_PORTS)
return NTLSTATUS_INVALID_PARAMETER;
hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING;
hub->RootPortState[port] = UsbHubPortConnectedDebounce;
UsbDelayUs(xhci->Ddk, 30000u);
if (!UsbHubRootPortConnected(xhci, port)) {
UsbHubClearRootPortFailure(hub, port);
hub->RootPortSlot[port] = 0;
hub->RootPortState[port] = UsbHubPortDisconnected;
return NTLSTATUS_NOT_FOUND;
}
hub->RootPortState[port] = UsbHubPortEnumerating;
uint8_t slot = 0;
KSTATUS status = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot);
if (KSUCCESS(status)) {
hub->RootPortSlot[port] = slot;
hub->RootPortState[port] = UsbHubPortChildPresent;
if (outSlot)
*outSlot = slot;
}
return status;
}
// Implements the hub Pop Pending Root Port operation.
static bool UsbHubPopPendingRootPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) {
if (!hub || !outPort) return false;
@@ -184,17 +230,19 @@ static void UsbHubRemoveSlotChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t
DbgPrint("usb-hub: %s port %u removing slot=%u\n",
hub->Kind == UsbHubKindRoot ? "root" : "hub", port, slot);
if (xdev->Allocated && xhci->Running)
(void)XhciDisableSlot(xhci, slot);
if (xdev->UsbPdo)
(void)UsbRemovePdoSubtree(xdev->UsbPdo, true);
if (xdev->Allocated)
(void)XhciDisableSlot(xhci, slot);
if (xdev->UsbDevice) {
xdev->UsbDevice->State = UsbDeviceRemoved;
UsbDereferenceDevice(xdev->UsbDevice);
xdev->UsbDevice = (USB_DEVICE*)0;
}
XhciReleaseDeviceDma(xdev);
memset(xdev, 0, sizeof(*xdev));
if (hub->Kind == UsbHubKindRoot && port <= USB_ROOT_HUB_MAX_PORTS) {
hub->RootPortSlot[port] = 0;
@@ -216,9 +264,9 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub,
uint16_t value,
uint16_t index,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length) {
if (!hub || !hub->LowerDevice || !buffer || !bufferPhys || !length)
if (!hub || !hub->LowerDevice || !buffer || !bufferDeviceAddress || !length)
return NTLSTATUS_INVALID_PARAMETER;
USB_URB urb;
@@ -227,7 +275,7 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub,
urb.Direction = UsbTransferIn;
urb.Device = hub->Device;
urb.Buffer = buffer;
urb.BufferPhys = bufferPhys;
urb.BufferDeviceAddress = bufferDeviceAddress;
urb.Length = length;
urb.Setup.bmRequestType = requestType;
urb.Setup.bRequest = request;
@@ -260,7 +308,7 @@ static KSTATUS UsbHubControlNoData(USB_HUB_EXTENSION* hub,
// Implements the hub Read Descriptor operation.
static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* outDesc) {
if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapPhys)
if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapDeviceAddress)
return NTLSTATUS_INVALID_PARAMETER;
memset(hub->ChangeBitmap, 0, USB_HUB_CHANGE_BITMAP_BYTES);
@@ -270,7 +318,7 @@ static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR*
(uint16_t)(USB_DESC_HUB << 8),
0,
hub->ChangeBitmap,
hub->ChangeBitmapPhys,
hub->ChangeBitmapDeviceAddress,
USB_HUB_CHANGE_BITMAP_BYTES);
if (!KSUCCESS(st))
return st;
@@ -293,7 +341,7 @@ static KSTATUS UsbHubGetPortStatus(USB_HUB_EXTENSION* hub, uint8_t port, USB_HUB
0,
port,
hub->ChangeBitmap,
hub->ChangeBitmapPhys,
hub->ChangeBitmapDeviceAddress,
sizeof(USB_HUB_PORT_STATUS));
if (!KSUCCESS(st))
return st;
@@ -501,19 +549,8 @@ static void UsbHubHandleRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
return;
}
hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING;
hub->RootPortState[port] = UsbHubPortConnectedDebounce;
UsbDelayUs(xhci->Ddk, 30000);
portsc = XhciRead32(xhci->Op, XhciPortReg(port));
if (!(portsc & XHCI_PORTSC_CCS)) {
UsbHubClearRootPortFailure(hub, port);
return;
}
hub->RootPortState[port] = UsbHubPortEnumerating;
uint8_t slot = 0;
KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot);
KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot);
if (KSUCCESS(st)) {
hub->RootPortSlot[port] = slot;
hub->RootPortState[port] = UsbHubPortChildPresent;
@@ -544,9 +581,11 @@ static bool UsbHubTakeInitialRootScan(USB_HUB_EXTENSION* hub) {
static void UsbHubWorker(void* arg) {
USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)arg;
while (hub && hub->WorkerRunning) {
KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore);
KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
continue;
if (!hub->WorkerRunning)
break;
if (hub->Kind == UsbHubKindRoot) {
if (UsbHubTakeInitialRootScan(hub))
@@ -563,6 +602,61 @@ static void UsbHubWorker(void* arg) {
UsbHubArmExternalInterrupt(hub);
}
}
if (hub && hub->WorkerFinished)
KeSignalEvent(hub->WorkerFinished);
}
// Stops the hub worker and releases its synchronization objects.
static void UsbHubStopWorker(USB_HUB_EXTENSION* hub) {
if (!hub)
return;
hub->WorkerRunning = false;
if (hub->WorkerSemaphore)
KeSignalSemaphore(hub->WorkerSemaphore);
if (hub->WorkerThread && hub->WorkerFinished)
(void)KeWaitForEvent(hub->WorkerFinished, 0);
if (hub->WorkerThread) {
ObDereferenceObject((OBJECT*)hub->WorkerThread);
hub->WorkerThread = (THREAD*)0;
}
if (hub->WorkerFinished) {
ObDereferenceObject(&hub->WorkerFinished->Header);
hub->WorkerFinished = (EVENT*)0;
}
if (hub->WorkerSemaphore) {
ObDereferenceObject(&hub->WorkerSemaphore->Header);
hub->WorkerSemaphore = (SEMAPHORE*)0;
}
}
// Creates the synchronization objects and thread for one hub worker.
static KSTATUS UsbHubStartWorker(USB_HUB_EXTENSION* hub) {
if (!hub)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore);
if (!KSUCCESS(status))
return status;
status = KeEventCreate(EventManualReset, 0, &hub->WorkerFinished);
if (!KSUCCESS(status)) {
UsbHubStopWorker(hub);
return status;
}
hub->WorkerRunning = true;
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
status = KeCreateKernelThread((uintptr_t)UsbHubWorker,
(uintptr_t)hub,
&workerParameters,
&hub->WorkerThread);
if (!KSUCCESS(status)) {
hub->WorkerRunning = false;
UsbHubStopWorker(hub);
return status;
}
return NTLSTATUS_SUCCESS;
}
// Implements the hub Do Root Port Scan operation.
@@ -592,10 +686,8 @@ static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd) {
}
attempted++;
hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING;
hub->RootPortState[port] = UsbHubPortEnumerating;
uint8_t slot = 0;
KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot);
KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot);
if (KSUCCESS(st)) {
hub->RootPortSlot[port] = slot;
hub->RootPortState[port] = UsbHubPortChildPresent;
@@ -734,7 +826,7 @@ static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub) {
hub->ChangeUrb.Device = hub->Device;
hub->ChangeUrb.Pipe = hub->InterruptPipe;
hub->ChangeUrb.Buffer = hub->ChangeBitmap;
hub->ChangeUrb.BufferPhys = hub->ChangeBitmapPhys;
hub->ChangeUrb.BufferDeviceAddress = hub->ChangeBitmapDeviceAddress;
hub->ChangeUrb.Length = hub->InterruptMaxPacket;
if (hub->ChangeUrb.Length > USB_HUB_CHANGE_BITMAP_BYTES)
hub->ChangeUrb.Length = USB_HUB_CHANGE_BITMAP_BYTES;
@@ -756,20 +848,10 @@ static KSTATUS UsbHubStartRootDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* h
if (hub->PortCount > USB_ROOT_HUB_MAX_PORTS)
hub->PortCount = USB_ROOT_HUB_MAX_PORTS;
KSTATUS st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore);
KSTATUS st = UsbHubStartWorker(hub);
if (!KSUCCESS(st))
return st;
hub->WorkerRunning = true;
st = KeCreateKernelThread((uintptr_t)UsbHubWorker,
(uintptr_t)hub,
PriorityNormal,
&hub->WorkerThread);
if (!KSUCCESS(st)) {
hub->WorkerRunning = false;
return st;
}
hub->Started = true;
device->PnpState = DevicePnpStarted;
if (hub->HubPdo)
@@ -784,7 +866,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
if (!hub || !hub->Device)
return NTLSTATUS_INVALID_PARAMETER;
hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapPhys);
hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapDeviceAddress, &hub->ChangeBitmapDma);
if (!hub->ChangeBitmap)
return NTLSTATUS_NO_MEMORY;
@@ -793,6 +875,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
KSTATUS st = UsbHubReadDescriptor(hub, &desc);
if (!KSUCCESS(st)) {
DbgPrint("usb-hub: slot=%u descriptor read failed st=0x%x\n", hub->Device->SlotId, st);
UsbHubReleaseDma(hub);
return st;
}
@@ -809,17 +892,9 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
UsbDelayUs(((XHCI_CONTROLLER*)hub->Hcd->ControllerContext)->Ddk,
(uint32_t)hub->PowerOnToPowerGoodMs * 1000u);
st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore);
if (!KSUCCESS(st))
return st;
hub->WorkerRunning = true;
st = KeCreateKernelThread((uintptr_t)UsbHubWorker,
(uintptr_t)hub,
PriorityNormal,
&hub->WorkerThread);
st = UsbHubStartWorker(hub);
if (!KSUCCESS(st)) {
hub->WorkerRunning = false;
UsbHubReleaseDma(hub);
return st;
}
@@ -829,8 +904,11 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
hub->InterruptMaxPacket,
hub->InterruptInterval,
&hub->InterruptPipe);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
UsbHubStopWorker(hub);
UsbHubReleaseDma(hub);
return st;
}
hub->Started = true;
device->PnpState = DevicePnpStarted;
@@ -897,6 +975,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
(void)UsbCancelUrb(&hub->ChangeUrb);
if (hub->WorkerSemaphore)
KeSignalSemaphore(hub->WorkerSemaphore);
UsbHubStopWorker(hub);
// Remove downstream PDOs owned by this hub FDO. The explicit
// slot arrays are still used as the hardware lookup table; the
@@ -929,8 +1008,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
UsbDereferencePipe(hub->InterruptPipe);
hub->InterruptPipe = (USB_PIPE*)0;
}
if (hub->WorkerSemaphore)
KeSignalSemaphore(hub->WorkerSemaphore);
UsbHubStopWorker(hub);
if (hub->Kind == UsbHubKindRoot && hub->Hcd && hub->Hcd->RootHubContext == hub)
hub->Hcd->RootHubContext = (void*)0;
if (hub->RemoveLockReady) {
@@ -938,6 +1016,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
IoDestroyRemoveLock(&hub->RemoveLock);
hub->RemoveLockReady = false;
}
UsbHubReleaseDma(hub);
}
KSTATUS lower = hub && hub->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS;
code/drivers/bus/usb/usb_storage.c
+261 -50
View File
@@ -56,10 +56,10 @@ static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_E
static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
bool dataIn,
PHYSICAL_ADDRESS phys,
uint64_t deviceAddress,
uint32_t length,
uint32_t* outActual) {
if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0)
if (!storage || !storage->LowerDevice || !storage->Device || !deviceAddress || length == 0)
return NTLSTATUS_INVALID_PARAMETER;
USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe;
@@ -73,7 +73,7 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
urb.Device = storage->Device;
urb.Pipe = pipe;
urb.Buffer = (void*)0;
urb.BufferPhys = phys;
urb.BufferDeviceAddress = deviceAddress;
urb.Length = length;
KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb);
@@ -83,29 +83,42 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
}
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint64_t deviceAddress,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, true, phys, length, outActual);
return UsbStorageSubmitBulk(storage, true, deviceAddress, length, outActual);
}
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint64_t deviceAddress,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, false, phys, length, outActual);
return UsbStorageSubmitBulk(storage, false, deviceAddress, length, outActual);
}
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
PHYSICAL_ADDRESS dataPhys,
uint32_t dataLength,
bool dataIn) {
typedef struct USB_STORAGE_SENSE_DATA {
bool Valid;
uint8_t ResponseCode;
uint8_t Key;
uint8_t Asc;
uint8_t Ascq;
} USB_STORAGE_SENSE_DATA;
// Executes one BOT command without issuing follow-up commands.
static KSTATUS UsbStorageBotCommandRaw(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
uint64_t dataDeviceAddress,
uint32_t dataLength,
bool dataIn,
uint8_t* outCswStatus,
uint32_t* outResidue) {
if (outCswStatus) *outCswStatus = 0xFFu;
if (outResidue) *outResidue = 0;
if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16)
return NTLSTATUS_INVALID_PARAMETER;
if (dataLength && (!data || !dataPhys))
if (dataLength && (!data || !dataDeviceAddress))
return NTLSTATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
@@ -120,14 +133,14 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
memcpy(cbw->CommandBlock, cdb, cdbLen);
uint32_t actual = 0;
KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual);
KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorDeviceAddress, sizeof(*cbw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*cbw))
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
if (dataLength) {
actual = 0;
st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual)
: UsbStorageBulkOut(storage, dataPhys, dataLength, &actual);
st = dataIn ? UsbStorageBulkIn(storage, dataDeviceAddress, dataLength, &actual)
: UsbStorageBulkOut(storage, dataDeviceAddress, dataLength, &actual);
if (!KSUCCESS(st))
return st;
}
@@ -135,7 +148,7 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer;
memset(csw, 0, sizeof(*csw));
actual = 0;
st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual);
st = UsbStorageBulkIn(storage, dev->DescriptorDeviceAddress, sizeof(*csw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*csw))
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
@@ -144,15 +157,136 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
dev->SlotId, csw->Signature, csw->Tag, storage->Tag);
return NTLSTATUS_DEVICE_ERROR;
}
if (csw->Status != USB_MASS_BOT_CSW_PASSED) {
DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u\n",
cdb[0], dev->SlotId, csw->Status, csw->DataResidue);
return NTLSTATUS_DEVICE_ERROR;
if (outCswStatus) *outCswStatus = csw->Status;
if (outResidue) *outResidue = csw->DataResidue;
return csw->Status == USB_MASS_BOT_CSW_PASSED
? NTLSTATUS_SUCCESS
: NTLSTATUS_DEVICE_ERROR;
}
// Returns a readable SCSI sense-key name.
static const char* UsbStorageSenseKeyName(uint8_t key) {
switch (key) {
case 0x00u: return "NO SENSE";
case 0x01u: return "RECOVERED ERROR";
case 0x02u: return "NOT READY";
case 0x03u: return "MEDIUM ERROR";
case 0x04u: return "HARDWARE ERROR";
case 0x05u: return "ILLEGAL REQUEST";
case 0x06u: return "UNIT ATTENTION";
case 0x07u: return "DATA PROTECT";
case 0x08u: return "BLANK CHECK";
case 0x0Bu: return "ABORTED COMMAND";
case 0x0Eu: return "MISCOMPARE";
default: return "UNKNOWN";
}
}
// Decodes fixed or descriptor-format REQUEST SENSE data.
static void UsbStorageDecodeSense(const uint8_t* bytes,
uint32_t length,
USB_STORAGE_SENSE_DATA* sense) {
if (!sense) return;
memset(sense, 0, sizeof(*sense));
if (!bytes || length == 0) return;
sense->ResponseCode = bytes[0] & 0x7Fu;
if ((sense->ResponseCode == 0x70u || sense->ResponseCode == 0x71u) && length >= 14u) {
sense->Key = bytes[2] & 0x0Fu;
sense->Asc = bytes[12];
sense->Ascq = bytes[13];
sense->Valid = true;
return;
}
if ((sense->ResponseCode == 0x72u || sense->ResponseCode == 0x73u) && length >= 4u) {
sense->Key = bytes[1] & 0x0Fu;
sense->Asc = bytes[2];
sense->Ascq = bytes[3];
sense->Valid = true;
}
}
// Fetches and decodes sense data after a command failure.
static KSTATUS UsbStorageFetchSense(USB_STORAGE_EXTENSION* storage,
USB_STORAGE_SENSE_DATA* sense) {
if (!storage || !storage->XhciDevice || !sense)
return NTLSTATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t cdb[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
memset(dev->MassBuffer, 0, 18);
uint8_t cswStatus = 0xFFu;
uint32_t residue = 0;
KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferDeviceAddress,
18, true, &cswStatus, &residue);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: REQUEST SENSE failed slot=%u st=0x%x csw_status=%u residue=%u\n",
dev->SlotId, st, cswStatus, residue);
memset(sense, 0, sizeof(*sense));
return st;
}
uint32_t returned = residue < 18u ? 18u - residue : 0u;
UsbStorageDecodeSense(dev->MassBuffer, returned, sense);
return NTLSTATUS_SUCCESS;
}
// Executes one BOT command and reports SCSI sense data on command failure.
static KSTATUS UsbStorageExecuteCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
uint64_t dataDeviceAddress,
uint32_t dataLength,
bool dataIn,
USB_STORAGE_SENSE_DATA* outSense) {
if (outSense) memset(outSense, 0, sizeof(*outSense));
uint8_t cswStatus = 0xFFu;
uint32_t residue = 0;
KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, cdbLen, data,
dataDeviceAddress, dataLength, dataIn,
&cswStatus, &residue);
if (KSUCCESS(st)) return st;
XHCI_DEVICE* dev = storage ? storage->XhciDevice : (XHCI_DEVICE*)0;
uint32_t slot = dev ? dev->SlotId : 0;
if (cswStatus != USB_MASS_BOT_CSW_FAILED || !cdb || cdb[0] == SCSI_REQUEST_SENSE) {
DbgPrint("usb-storage: command %02x failed slot=%u st=0x%x csw_status=%u residue=%u\n",
cdb ? cdb[0] : 0, slot, st, cswStatus, residue);
return st;
}
USB_STORAGE_SENSE_DATA sense;
KSTATUS senseStatus = UsbStorageFetchSense(storage, &sense);
if (KSUCCESS(senseStatus) && sense.Valid) {
DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=%s key=%02x asc=%02x ascq=%02x\n",
cdb[0], slot, cswStatus, residue,
UsbStorageSenseKeyName(sense.Key), sense.Key, sense.Asc, sense.Ascq);
if (outSense) *outSense = sense;
} else {
DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=unavailable\n",
cdb[0], slot, cswStatus, residue);
}
return st;
}
// Executes one BOT command through the sense-reporting path.
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
uint64_t dataDeviceAddress,
uint32_t dataLength,
bool dataIn) {
return UsbStorageExecuteCommand(storage, cdb, cdbLen, data,
dataDeviceAddress, dataLength, dataIn,
(USB_STORAGE_SENSE_DATA*)0);
}
// Implements the be32 operation.
uint32_t UsbBe32(const uint8_t* p) {
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
@@ -182,7 +316,7 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 };
memset(dev->MassBuffer, 0, 36);
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
dev->MassBuffer, dev->MassBufferDeviceAddress,
36, true);
if (!KSUCCESS(st)) return st;
@@ -195,18 +329,6 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
return NTLSTATUS_SUCCESS;
}
// Implements the storage Request Sense operation.
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return NTLSTATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
memset(dev->MassBuffer, 0, 18);
return UsbStorageBotCommand(storage, rs, sizeof(rs),
dev->MassBuffer, dev->MassBufferPhys, 18, true);
}
// Implements the storage Test Unit Ready operation.
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
@@ -222,7 +344,6 @@ KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
if (KSUCCESS(st)) return NTLSTATUS_SUCCESS;
last = st;
(void)UsbStorageRequestSense(storage);
UsbDelayUs(xhci->Ddk, 10000u);
}
@@ -238,7 +359,7 @@ KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) {
uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
memset(dev->MassBuffer, 0, 8);
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
dev->MassBuffer, dev->MassBufferDeviceAddress,
8, true);
if (!KSUCCESS(st)) return st;
@@ -288,7 +409,7 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
memcpy(dev->MassBuffer, buffer, bytes);
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
dev->MassBuffer, dev->MassBufferDeviceAddress,
bytes, !write);
if (!KSUCCESS(st)) return st;
@@ -297,6 +418,48 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
return NTLSTATUS_SUCCESS;
}
// Returns whether sense data means SYNCHRONIZE CACHE is unsupported.
static bool UsbStorageSenseRejectsSynchronizeCache(const USB_STORAGE_SENSE_DATA* sense) {
if (!sense || !sense->Valid || sense->Key != 0x05u)
return false;
return sense->Asc == 0x20u || sense->Asc == 0x24u;
}
// Flushes device write caches through SCSI SYNCHRONIZE CACHE(10).
static KSTATUS UsbStorageSynchronizeCache(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return NTLSTATUS_INVALID_PARAMETER;
if (storage->FlushCapability == UsbStorageFlushUnsupported)
return NTLSTATUS_SUCCESS;
uint8_t cdb[10];
memset(cdb, 0, sizeof(cdb));
cdb[0] = SCSI_SYNCHRONIZE_CACHE_10;
for (uint32_t attempt = 0; attempt < 2; attempt++) {
USB_STORAGE_SENSE_DATA sense;
KSTATUS st = UsbStorageExecuteCommand(storage, cdb, sizeof(cdb),
(void*)0, 0, 0, false, &sense);
if (KSUCCESS(st)) {
storage->FlushCapability = UsbStorageFlushSupported;
return NTLSTATUS_SUCCESS;
}
if (UsbStorageSenseRejectsSynchronizeCache(&sense)) {
storage->FlushCapability = UsbStorageFlushUnsupported;
DbgPrint("usb-storage: slot=%u does not support SYNCHRONIZE CACHE; flushes use BOT command-completion semantics only\n",
storage->XhciDevice->SlotId);
return NTLSTATUS_SUCCESS;
}
if (sense.Valid && sense.Key == 0x06u && attempt == 0)
continue;
return st;
}
return NTLSTATUS_DEVICE_ERROR;
}
// Implements the storage Acquire Io operation.
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
if (!storage) return;
@@ -375,24 +538,70 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
return NTLSTATUS_PENDING;
}
// Flushes one USB mass-storage disk synchronously.
static KSTATUS UsbStorageDispatchFlush(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return NTLSTATUS_INVALID_PARAMETER;
USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device);
if (!storage || !storage->RemoveLockReady)
return NTLSTATUS_OBJECT_DEAD;
KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return NTLSTATUS_INVALID_PARAMETER;
}
if (!storage->Started || storage->SurpriseRemoved || storage->Removed) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, NTLSTATUS_OBJECT_DEAD, 0);
return NTLSTATUS_PENDING;
}
UsbStorageAcquireIo(storage);
st = UsbStorageSynchronizeCache(storage);
UsbStorageReleaseIo(storage);
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, st, 0);
return NTLSTATUS_PENDING;
}
// Creates mass Storage Memory.
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer)
return NTLSTATUS_SUCCESS;
dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingPhys);
dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingPhys);
dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferPhys);
if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer)
dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingDeviceAddress, &dev->MassInRingDma);
dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingDeviceAddress, &dev->MassOutRingDma);
dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferDeviceAddress, &dev->MassBufferDma);
if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer) {
if (gUsbDdk && gUsbDdk->Dma) {
gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma);
}
dev->MassBufferDma = (IO_DMA_BUFFER*)0;
dev->MassOutRingDma = (IO_DMA_BUFFER*)0;
dev->MassInRingDma = (IO_DMA_BUFFER*)0;
dev->MassBuffer = (uint8_t*)0;
dev->MassOutRing = (XHCI_TRB*)0;
dev->MassInRing = (XHCI_TRB*)0;
return NTLSTATUS_NO_MEMORY;
}
dev->MassInEnqueue = 0;
dev->MassInCycle = 1;
dev->MassOutEnqueue = 0;
dev->MassOutCycle = 1;
dev->MassTag = 0x1000u + dev->SlotId;
XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingPhys, XHCI_BULK_RING_TRBS, dev->MassInCycle);
XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingPhys, XHCI_BULK_RING_TRBS, dev->MassOutCycle);
XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassInCycle);
XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassOutCycle);
return NTLSTATUS_SUCCESS;
}
@@ -431,21 +640,21 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de
epIn[0] = 0;
epIn[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_IN | ((uint32_t)inMaxPacket << 16);
epIn[2] = (uint32_t)(dev->MassInRingPhys | XHCI_EP_DCS);
epIn[3] = (uint32_t)((dev->MassInRingPhys | XHCI_EP_DCS) >> 32);
epIn[2] = (uint32_t)(dev->MassInRingDeviceAddress | XHCI_EP_DCS);
epIn[3] = (uint32_t)((dev->MassInRingDeviceAddress | XHCI_EP_DCS) >> 32);
epIn[4] = inMaxPacket;
epOut[0] = 0;
epOut[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_OUT | ((uint32_t)outMaxPacket << 16);
epOut[2] = (uint32_t)(dev->MassOutRingPhys | XHCI_EP_DCS);
epOut[3] = (uint32_t)((dev->MassOutRingPhys | XHCI_EP_DCS) >> 32);
epOut[2] = (uint32_t)(dev->MassOutRingDeviceAddress | XHCI_EP_DCS);
epOut[3] = (uint32_t)((dev->MassOutRingDeviceAddress | XHCI_EP_DCS) >> 32);
epOut[4] = outMaxPacket;
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -549,6 +758,7 @@ static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSIO
storage->BulkInMaxPacket = bulkIn->wMaxPacketSize;
storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize;
storage->Tag = 0x1000u + storage->Device->SlotId;
storage->FlushCapability = UsbStorageFlushUnknown;
xdev->MassCandidate = true;
xdev->MassInterface = storage->InterfaceNumber;
@@ -844,6 +1054,7 @@ KSTATUS UsbInitializeStorageDriver(void) {
gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice;
gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = UsbStorageDispatchFlush;
gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp;
return IoRegisterDriverObject(&gUsbStorageDriver);
}
code/drivers/bus/usb/usb_xhci.c
+322 -151
View File
@@ -73,32 +73,105 @@ bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg,
return false;
}
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
PHYSICAL_ADDRESS* outPhys) {
if (!outPhys || pages == 0)
return NULL;
// Allocates page-aligned common-buffer memory from the controller adapter.
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!xhci || !xhci->DmaAdapter || !outDeviceAddress || !outBuffer || pages == 0 ||
pages > ((size_t)-1) / PAGE_SIZE)
return (void*)0;
// Most xHCI runtime objects are one page. Do not route those through the
// physically-contiguous allocator: on real machines that allocator is a
// scarce DMA primitive, while a single frame is already contiguous. QEMU
// hid this because it had simple low memory; laptop firmware does not.
PHYSICAL_ADDRESS phys = 0;
if (pages == 1) {
phys = MmPfnAllocate(MmPfnUseDma);
} else {
phys = MmPfnAllocateContiguous(pages, MmPfnUseDma);
IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
KSTATUS status = xhci->Ddk->Dma->AllocateCommonBuffer(xhci->DmaAdapter,
pages * (size_t)PAGE_SIZE,
PAGE_SIZE,
&buffer);
if (!KSUCCESS(status)) {
DbgPrint("usb: DMA allocation failed pages=%llu st=0x%x\n",
(unsigned long long)pages, status);
return (void*)0;
}
if (!phys) {
DbgPrint("usb: DMA allocation failed pages=%llu\n",
(unsigned long long)pages);
return NULL;
*outDeviceAddress = xhci->Ddk->Dma->GetCommonBufferDeviceAddress(buffer);
*outBuffer = buffer;
return xhci->Ddk->Dma->GetCommonBufferAddress(buffer);
}
// Releases all DMA allocations owned by one xHCI slot.
void XhciReleaseDeviceDma(XHCI_DEVICE* dev) {
if (!dev || !gUsbDdk || !gUsbDdk->Dma)
return;
for (uint32_t endpoint = 1; endpoint < 32; endpoint++) {
USB_PIPE* pipe = dev->EndpointPipes[endpoint];
dev->EndpointPipes[endpoint] = (USB_PIPE*)0;
if (pipe)
UsbDereferencePipe(pipe);
}
void* virt = xhci->Ddk->Memory->PhysToVirt(phys);
memset(virt, 0, pages * (size_t)PAGE_SIZE);
*outPhys = phys;
return virt;
gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DescriptorDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->Ep0RingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DeviceContextDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->InputContextDma);
dev->MassBufferDma = (IO_DMA_BUFFER*)0;
dev->HidBufferDma = (IO_DMA_BUFFER*)0;
dev->DescriptorDma = (IO_DMA_BUFFER*)0;
dev->MassOutRingDma = (IO_DMA_BUFFER*)0;
dev->MassInRingDma = (IO_DMA_BUFFER*)0;
dev->HidRingDma = (IO_DMA_BUFFER*)0;
dev->Ep0RingDma = (IO_DMA_BUFFER*)0;
dev->DeviceContextDma = (IO_DMA_BUFFER*)0;
dev->InputContextDma = (IO_DMA_BUFFER*)0;
dev->MassBuffer = (uint8_t*)0;
dev->HidBuffer = (uint8_t*)0;
dev->DescriptorBuffer = (uint8_t*)0;
dev->MassOutRing = (XHCI_TRB*)0;
dev->MassInRing = (XHCI_TRB*)0;
dev->HidRing = (XHCI_TRB*)0;
dev->Ep0Ring = (XHCI_TRB*)0;
dev->DeviceContext = (uint8_t*)0;
dev->InputContext = (uint8_t*)0;
}
// Releases controller-wide xHCI DMA allocations.
static void XhciReleaseControllerDma(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->Ddk || !xhci->Ddk->Dma)
return;
for (uint32_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++)
XhciReleaseDeviceDma(&xhci->Devices[slot]);
for (uint16_t i = 0; i < XHCI_MAX_SCRATCHPADS; i++) {
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadDma[i]);
xhci->ScratchpadDma[i] = (IO_DMA_BUFFER*)0;
xhci->ScratchpadDeviceAddress[i] = 0;
}
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadArrayDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ErstDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->EventRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->CommandRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->DcbaaDma);
xhci->ScratchpadArrayDma = (IO_DMA_BUFFER*)0;
xhci->ErstDma = (IO_DMA_BUFFER*)0;
xhci->EventRingDma = (IO_DMA_BUFFER*)0;
xhci->CommandRingDma = (IO_DMA_BUFFER*)0;
xhci->DcbaaDma = (IO_DMA_BUFFER*)0;
xhci->ScratchpadArray = (uint64_t*)0;
xhci->Erst = (XHCI_ERST_ENTRY*)0;
xhci->EventRing = (XHCI_TRB*)0;
xhci->CommandRing = (XHCI_TRB*)0;
xhci->Dcbaa = (uint64_t*)0;
xhci->Ddk->Dma->PutAdapter(xhci->DmaAdapter);
xhci->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
// Implements the scratchpad Count operation.
@@ -328,26 +401,26 @@ void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
// Updates erdp.
void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB);
uint64_t erdp = xhci->EventRingDeviceAddress + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB);
XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear
}
// Returns command Waiter Locked.
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandDeviceAddress) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->CommandPhys && w->CommandPhys == commandPhys)
if (w->CommandDeviceAddress && w->CommandDeviceAddress == commandDeviceAddress)
return w;
}
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint64_t transferDeviceAddress,
uint8_t slotId,
uint8_t endpointId) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->TransferPhys &&
w->TransferPhys == transferPhys &&
if (w->TransferDeviceAddress &&
w->TransferDeviceAddress == transferDeviceAddress &&
w->TransferSlotId == slotId &&
w->TransferEndpointId == endpointId) {
return w;
@@ -357,11 +430,11 @@ static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
}
static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
uint64_t commandPhys,
uint64_t commandDeviceAddress,
uint32_t completionCode,
uint8_t slotId) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandDeviceAddress);
if (w) {
w->Command.CompletionCode = completionCode;
w->Command.SlotId = slotId;
@@ -373,13 +446,13 @@ static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
}
static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint64_t transferDeviceAddress,
uint8_t slotId,
uint8_t endpointId,
uint32_t completionCode,
uint32_t residue) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferDeviceAddress, slotId, endpointId);
if (w) {
w->Transfer.CompletionCode = completionCode;
w->Transfer.SlotId = slotId;
@@ -397,7 +470,7 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
bool processed = false;
for (;;) {
while (true) {
XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue];
uint32_t control = ev->Control;
uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE);
@@ -460,8 +533,8 @@ bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
// Waits for target Complete.
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
if (!target) return false;
if (target->CommandPhys && !target->CommandMatched) return false;
if (target->TransferPhys && !target->TransferMatched) return false;
if (target->CommandDeviceAddress && !target->CommandMatched) return false;
if (target->TransferDeviceAddress && !target->TransferMatched) return false;
return true;
}
@@ -568,6 +641,21 @@ static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the interrupt-dispatch registry.
static void XhciUnregisterController(XHCI_CONTROLLER* xhci) {
if (!xhci)
return;
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
if (gXhciControllers[i] != xhci)
continue;
for (uint32_t next = i + 1; next < gXhciControllerCount; next++)
gXhciControllers[next - 1] = gXhciControllers[next];
gXhciControllerCount--;
gXhciControllers[gXhciControllerCount] = (XHCI_CONTROLLER*)0;
return;
}
}
// Implements the isr operation.
void XhciIsr(void) {
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
@@ -663,6 +751,34 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_SUCCESS;
}
// Halts the controller before any device-visible memory can be released.
static void XhciHaltForDmaRelease(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->DmaAdapter)
return;
xhci->InterruptsEnabled = false;
xhci->Running = false;
if (xhci->SurpriseRemoved || !xhci->Op)
return;
if (xhci->Runtime) {
volatile uint8_t* interrupter = xhci->Runtime + 0x20u;
XhciWrite32(interrupter, 0x00u, XHCI_IMAN_IP);
}
uint32_t command = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
command &= ~(XHCI_USBCMD_RS | XHCI_USBCMD_INTE);
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, command);
if (!UsbWaitSet32(xhci->Ddk,
xhci->Op,
XHCI_OP_USBSTS,
XHCI_USBSTS_HCH,
1000000u))
KePanic(KE_PANIC_CODE_DRIVER,
"usb: xHCI failed to halt before DMA release usbsts=%08x\n",
XhciRead32(xhci->Op, XHCI_OP_USBSTS));
}
// Creates scratchpads.
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
if (xhci->ScratchpadCount == 0) return NTLSTATUS_SUCCESS;
@@ -672,7 +788,7 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_NOT_SUPPORTED;
}
xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayPhys);
xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayDeviceAddress, &xhci->ScratchpadArrayDma);
if (!xhci->ScratchpadArray) {
DbgPrint("usb: xHCI scratchpad pointer-array allocation failed count=%u\n",
xhci->ScratchpadCount);
@@ -680,29 +796,29 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
}
for (uint16_t i = 0; i < xhci->ScratchpadCount; i++) {
void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadPhys[i]);
void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadDeviceAddress[i], &xhci->ScratchpadDma[i]);
if (!page) {
DbgPrint("usb: xHCI scratchpad buffer allocation failed index=%u/%u\n",
i, xhci->ScratchpadCount);
return NTLSTATUS_NO_MEMORY;
}
(void)page;
xhci->ScratchpadArray[i] = xhci->ScratchpadPhys[i];
xhci->ScratchpadArray[i] = xhci->ScratchpadDeviceAddress[i];
}
xhci->Dcbaa[0] = xhci->ScratchpadArrayPhys;
xhci->Dcbaa[0] = xhci->ScratchpadArrayDeviceAddress;
UsbTrace("usb: xHCI allocated %u scratchpad buffer(s) array=0x%016llX\n",
xhci->ScratchpadCount,
(unsigned long long)xhci->ScratchpadArrayPhys);
(unsigned long long)xhci->ScratchpadArrayDeviceAddress);
return NTLSTATUS_SUCCESS;
}
// Creates runtime Memory.
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys);
xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys);
xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingPhys);
xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstPhys);
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaDeviceAddress, &xhci->DcbaaDma);
xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingDeviceAddress, &xhci->CommandRingDma);
xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingDeviceAddress, &xhci->EventRingDma);
xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstDeviceAddress, &xhci->ErstDma);
if (!xhci->Dcbaa || !xhci->CommandRing || !xhci->EventRing || !xhci->Erst) {
DbgPrint("usb: xHCI runtime DMA allocation failed dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n",
@@ -716,14 +832,14 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->EventCycle = 1;
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress;
link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
xhci->CommandCycle;
xhci->Erst[0].RingSegmentBase = xhci->EventRingPhys;
xhci->Erst[0].RingSegmentBase = xhci->EventRingDeviceAddress;
xhci->Erst[0].RingSegmentSize = XHCI_EVENT_TRBS;
xhci->Erst[0].Reserved = 0;
@@ -731,10 +847,10 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
if (!KSUCCESS(st)) return st;
UsbTrace("usb: xHCI dma dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n",
(unsigned long long)xhci->DcbaaPhys,
(unsigned long long)xhci->CommandRingPhys,
(unsigned long long)xhci->EventRingPhys,
(unsigned long long)xhci->ErstPhys);
(unsigned long long)xhci->DcbaaDeviceAddress,
(unsigned long long)xhci->CommandRingDeviceAddress,
(unsigned long long)xhci->EventRingDeviceAddress,
(unsigned long long)xhci->ErstDeviceAddress);
return NTLSTATUS_SUCCESS;
}
@@ -743,8 +859,8 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
uint8_t slots = xhci->MaxSlots;
if (slots == 0) return NTLSTATUS_DEVICE_ERROR;
XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaPhys);
XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingPhys | xhci->CommandCycle);
XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaDeviceAddress);
XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingDeviceAddress | xhci->CommandCycle);
XhciWrite32(xhci->Op, XHCI_OP_DNCTRL, 0);
XhciWrite32(xhci->Op, XHCI_OP_CONFIG, slots);
@@ -752,30 +868,30 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
XhciWrite32(ir0, 0x00u, XHCI_IMAN_IP); // clear pending, leave disabled until smoke test passes
XhciWrite32(ir0, 0x04u, 0); // no interrupt moderation yet
XhciWrite32(ir0, 0x08u, 1); // one ERST entry
XhciWrite64(ir0, 0x10u, xhci->ErstPhys);
XhciWrite64(ir0, 0x18u, xhci->EventRingPhys);
XhciWrite64(ir0, 0x10u, xhci->ErstDeviceAddress);
XhciWrite64(ir0, 0x18u, xhci->EventRingDeviceAddress);
XhciBarrier();
return NTLSTATUS_SUCCESS;
}
// Implements the command Trb Phys operation.
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB);
// Returns the device address of one command TRB.
uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
}
static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
uint64_t* outDeviceAddress) {
if (!xhci || !xhci->CommandRing)
return NTLSTATUS_INVALID_PARAMETER;
if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) {
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress;
link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -787,7 +903,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint32_t index = xhci->CommandEnqueue++;
XHCI_TRB* trb = &xhci->CommandRing[index];
uint64_t trbPhys = XhciCommandTrbPhys(xhci, index);
uint64_t trbDeviceAddress = XhciCommandTrbDeviceAddress(xhci, index);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -795,7 +911,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
XhciBarrier();
trb->Control = controlWithoutCycle | xhci->CommandCycle;
if (outPhys) *outPhys = trbPhys;
if (outDeviceAddress) *outDeviceAddress = trbDeviceAddress;
return NTLSTATUS_SUCCESS;
}
@@ -811,14 +927,14 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
memset(&target, 0, sizeof(target));
uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock);
uint64_t trbPhys = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys);
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbDeviceAddress);
if (!KSUCCESS(st)) {
KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
return st;
}
target.CommandPhys = trbPhys;
target.CommandDeviceAddress = trbDeviceAddress;
XhciRegisterWaitTarget(xhci, &target);
XhciBarrier();
@@ -912,7 +1028,7 @@ KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_ADDRESS_DEVICE << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -1059,20 +1175,22 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
// Creates device Memory.
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys);
dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys);
dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingPhys);
dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorPhys);
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextDeviceAddress, &dev->InputContextDma);
dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextDeviceAddress, &dev->DeviceContextDma);
dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingDeviceAddress, &dev->Ep0RingDma);
dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorDeviceAddress, &dev->DescriptorDma);
if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer)
if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) {
XhciReleaseDeviceDma(dev);
return NTLSTATUS_NO_MEMORY;
}
dev->Ep0Enqueue = 0;
dev->Ep0Cycle = 1;
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -1116,31 +1234,31 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
ep0[0] = 0;
ep0[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_CONTROL | (maxPacket << 16);
ep0[2] = (uint32_t)(dev->Ep0RingPhys | XHCI_EP_DCS);
ep0[3] = (uint32_t)((dev->Ep0RingPhys | XHCI_EP_DCS) >> 32);
ep0[2] = (uint32_t)(dev->Ep0RingDeviceAddress | XHCI_EP_DCS);
ep0[3] = (uint32_t)((dev->Ep0RingDeviceAddress | XHCI_EP_DCS) >> 32);
ep0[4] = 8;
xhci->Dcbaa[dev->SlotId] = dev->DeviceContextPhys;
xhci->Dcbaa[dev->SlotId] = dev->DeviceContextDeviceAddress;
XhciBarrier();
}
// Implements the ep0 Trb Phys operation.
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB);
// Returns the device address of one endpoint-zero TRB.
uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
}
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
uint64_t* outDeviceAddress) {
if (!dev || !dev->Ep0Ring)
return NTLSTATUS_INVALID_PARAMETER;
if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) {
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -1152,7 +1270,7 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint32_t index = dev->Ep0Enqueue++;
XHCI_TRB* trb = &dev->Ep0Ring[index];
uint64_t phys = XhciEp0TrbPhys(dev, index);
uint64_t deviceAddress = XhciEp0TrbDeviceAddress(dev, index);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -1160,14 +1278,14 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
XhciBarrier();
trb->Control = controlWithoutCycle | dev->Ep0Cycle;
if (outPhys) *outPhys = phys;
if (outDeviceAddress) *outDeviceAddress = deviceAddress;
return NTLSTATUS_SUCCESS;
}
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length,
uint32_t timeoutUs) {
if (!buffer || length == 0 || length > PAGE_SIZE)
@@ -1188,26 +1306,26 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
st = XhciEp0AppendTrb(
dev,
bufferPhys,
bufferDeviceAddress,
length,
(XHCI_TRB_TYPE_DATA << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_DATA_DIR_IN,
NULL);
if (!KSUCCESS(st)) return st;
uint64_t statusTrbPhys = 0;
uint64_t statusTrbDeviceAddress = 0;
st = XhciEp0AppendTrb(
dev,
0,
0,
(XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_IOC,
&statusTrbPhys);
&statusTrbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferDeviceAddress = statusTrbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
@@ -1235,9 +1353,9 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length) {
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length,
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferDeviceAddress, length,
USB_XHCI_CONTROL_TIMEOUT_US);
}
@@ -1254,7 +1372,7 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
NULL);
if (!KSUCCESS(st)) return st;
uint64_t statusTrbPhys = 0;
uint64_t statusTrbDeviceAddress = 0;
st = XhciEp0AppendTrb(
dev,
0,
@@ -1262,12 +1380,12 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
(XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_STATUS_DIR_IN |
XHCI_TRB_IOC,
&statusTrbPhys);
&statusTrbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferDeviceAddress = statusTrbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
@@ -1304,7 +1422,7 @@ KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
KSTATUS st = XhciControlInTimeout(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
sizeof(USB_DEVICE_DESCRIPTOR),
timeoutUs);
if (!KSUCCESS(st)) return st;
@@ -1339,7 +1457,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
KSTATUS st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
sizeof(USB_CONFIGURATION_DESCRIPTOR));
if (!KSUCCESS(st)) return st;
@@ -1366,7 +1484,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
setup.wLength = total;
st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
total);
if (!KSUCCESS(st)) return st;
@@ -1415,17 +1533,17 @@ uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) {
return interval;
}
KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress,
uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle,
uint64_t parameter, uint32_t status,
uint32_t controlWithoutCycle, uint64_t* outPhys) {
uint32_t controlWithoutCycle, uint64_t* outDeviceAddress) {
if (!ring || !enqueue || !cycle || trbCount < 2)
return NTLSTATUS_INVALID_PARAMETER;
if (*enqueue >= trbCount - 1u) {
XHCI_TRB* link = &ring[trbCount - 1u];
link->ParameterLo = (uint32_t)ringPhys;
link->ParameterHi = (uint32_t)(ringPhys >> 32);
link->ParameterLo = (uint32_t)ringDeviceAddress;
link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -1437,7 +1555,7 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
uint32_t index = (*enqueue)++;
XHCI_TRB* trb = &ring[index];
uint64_t phys = ringPhys + (uint64_t)index * sizeof(XHCI_TRB);
uint64_t deviceAddress = ringDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -1445,16 +1563,16 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
XhciBarrier();
trb->Control = controlWithoutCycle | *cycle;
if (outPhys) *outPhys = phys;
if (outDeviceAddress) *outDeviceAddress = deviceAddress;
return NTLSTATUS_SUCCESS;
}
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress,
uint32_t trbCount, uint8_t cycle) {
memset(ring, 0, trbCount * sizeof(XHCI_TRB));
XHCI_TRB* link = &ring[trbCount - 1u];
link->ParameterLo = (uint32_t)ringPhys;
link->ParameterHi = (uint32_t)(ringPhys >> 32);
link->ParameterLo = (uint32_t)ringDeviceAddress;
link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -1464,34 +1582,34 @@ void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t endpointId,
XHCI_TRB* ring,
PHYSICAL_ADDRESS ringPhys,
uint64_t ringDeviceAddress,
uint32_t* enqueue,
uint8_t* cycle,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint32_t length,
uint32_t* outActual) {
if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferPhys || length == 0)
if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferDeviceAddress || length == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (length > USB_STORAGE_MAX_BYTES)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciRingAppend(ring,
ringPhys,
ringDeviceAddress,
XHCI_BULK_RING_TRBS,
enqueue,
cycle,
bufferPhys,
bufferDeviceAddress,
length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = trbPhys;
target.TransferDeviceAddress = trbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = endpointId;
@@ -1537,7 +1655,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
if (!ctx) return NTLSTATUS_NO_MEMORY;
memset(ctx, 0, sizeof(*ctx));
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys);
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingDeviceAddress, &ctx->RingDma);
if (!ctx->Ring) {
MmFreePool(ctx);
return NTLSTATUS_NO_MEMORY;
@@ -1546,7 +1664,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
ctx->Cycle = 1;
ctx->EndpointId = dci;
ctx->Type = pipe->Type;
XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle);
XhciInitialiseTransferRing(ctx->Ring, ctx->RingDeviceAddress, XHCI_PIPE_RING_TRBS, ctx->Cycle);
memset(dev->InputContext, 0, PAGE_SIZE);
uint32_t* icc = XhciInputControlContext(xhci, dev);
@@ -1574,21 +1692,22 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
} else if (pipe->Type == UsbPipeBulk) {
endpointType = XHCI_EP_TYPE_BULK_OUT;
} else {
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return NTLSTATUS_NOT_SUPPORTED;
}
ep[0] = (uint32_t)interval << 16;
ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16);
ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32);
ep[2] = (uint32_t)(ctx->RingDeviceAddress | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingDeviceAddress | XHCI_EP_DCS) >> 32);
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -1596,6 +1715,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
@@ -1609,7 +1729,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
// Implements the hcd Submit Interrupt Urb operation.
static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN))
return NTLSTATUS_NOT_SUPPORTED;
@@ -1623,22 +1743,22 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
if (ctx->ActiveTransfer)
return NTLSTATUS_QUEUE_FULL;
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
st = XhciRingAppend(ctx->Ring,
ctx->RingPhys,
ctx->RingDeviceAddress,
XHCI_PIPE_RING_TRBS,
&ctx->Enqueue,
&ctx->Cycle,
urb->BufferPhys,
urb->BufferDeviceAddress,
urb->Length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
ctx->ActiveTransfer = true;
ctx->ActiveTrbPhys = trbPhys;
ctx->ActiveTrbDeviceAddress = trbDeviceAddress;
ctx->ActiveUrb = urb;
urb->Pending = true;
urb->Cancelled = false;
@@ -1651,7 +1771,7 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
return NTLSTATUS_PENDING;
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
@@ -1663,12 +1783,12 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys)
if (!ctx->ActiveTransfer || ctx->ActiveTrbDeviceAddress != trbDeviceAddress)
return;
USB_URB* urb = ctx->ActiveUrb;
ctx->ActiveTransfer = false;
ctx->ActiveTrbPhys = 0;
ctx->ActiveTrbDeviceAddress = 0;
ctx->ActiveUrb = (USB_URB*)0;
if (!urb)
@@ -1704,23 +1824,23 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
// Implements the hcd Submit Bulk Urb operation.
static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0)
return NTLSTATUS_INVALID_PARAMETER;
uint8_t endpointId = urb->Pipe->EndpointId;
XHCI_TRB* ring = (XHCI_TRB*)0;
PHYSICAL_ADDRESS ringPhys = 0;
uint64_t ringDeviceAddress = 0;
uint32_t* enqueue = (uint32_t*)0;
uint8_t* cycle = (uint8_t*)0;
if (endpointId == dev->MassBulkInEndpointId) {
ring = dev->MassInRing;
ringPhys = dev->MassInRingPhys;
ringDeviceAddress = dev->MassInRingDeviceAddress;
enqueue = &dev->MassInEnqueue;
cycle = &dev->MassInCycle;
} else if (endpointId == dev->MassBulkOutEndpointId) {
ring = dev->MassOutRing;
ringPhys = dev->MassOutRingPhys;
ringDeviceAddress = dev->MassOutRingDeviceAddress;
enqueue = &dev->MassOutEnqueue;
cycle = &dev->MassOutCycle;
} else {
@@ -1732,10 +1852,10 @@ static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB
dev,
endpointId,
ring,
ringPhys,
ringDeviceAddress,
enqueue,
cycle,
urb->BufferPhys,
urb->BufferDeviceAddress,
urb->Length,
&actual);
urb->ActualLength = KSUCCESS(st) ? actual : 0;
@@ -1759,7 +1879,7 @@ static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
st = XhciControlNoData(xhci, dev, &urb->Setup);
urb->ActualLength = 0;
} else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) {
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length);
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferDeviceAddress, (uint16_t)urb->Length);
urb->ActualLength = KSUCCESS(st) ? urb->Length : 0;
} else {
st = NTLSTATUS_NOT_SUPPORTED;
@@ -1796,9 +1916,29 @@ static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
}
// Implements the hcd Allocate Dma operation.
static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
static void* XhciHcdAllocateDma(USB_HCD* hcd,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!hcd) return (void*)0;
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext,
pages,
outDeviceAddress,
outBuffer);
}
// Releases the host-controller context attached to a USB pipe.
static void XhciHcdDestroyPipe(USB_HCD* hcd, USB_PIPE* pipe) {
(void)hcd;
if (!pipe || !pipe->HcdPipeContext)
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (gUsbDdk && gUsbDdk->Dma)
gUsbDdk->Dma->FreeCommonBuffer(ctx->RingDma);
ctx->RingDma = (IO_DMA_BUFFER*)0;
ctx->Ring = (XHCI_TRB*)0;
pipe->HcdPipeContext = (void*)0;
MmFreePool(ctx);
}
// Implements the hcd Reset Root Port operation.
@@ -1819,6 +1959,7 @@ const USB_HCD_OPERATIONS gXhciHcdOperations = {
XhciHcdCancelUrb,
XhciHcdResetRootPort,
XhciHcdClearRootPortChanges,
XhciHcdDestroyPipe,
XhciHcdAllocateDma,
};
@@ -1886,7 +2027,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
uint64_t mappedLength = 0;
st = PciMapBar(xhci->Pdo,
0,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&xhci->Mmio,
&mappedLength);
if (!KSUCCESS(st)) {
@@ -1960,17 +2101,37 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_NOT_SUPPORTED;
}
IO_DMA_REQUIREMENTS dmaRequirements;
memset(&dmaRequirements, 0, sizeof(dmaRequirements));
dmaRequirements.MaximumDeviceAddress =
(xhci->Hcc1 & XHCI_HCC1_AC64) ? UINT64_MAX : UINT32_MAX;
dmaRequirements.Alignment = 1;
dmaRequirements.Boundary = 0x10000u;
dmaRequirements.MaximumSegmentLength = 0x10000u;
dmaRequirements.MaximumSegments = XHCI_PIPE_RING_TRBS - 1u;
st = xhci->Ddk->Dma->GetAdapter(xhci->Fdo,
&dmaRequirements,
&xhci->DmaAdapter);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI DMA adapter creation failed st=0x%x\n", st);
return st;
}
st = XhciAllocateRuntimeMemory(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciProgramRuntime(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciStart(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciSubmitNoopCommand(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
XhciPowerRootPorts(xhci);
XhciLogPortStatus(xhci);
@@ -1978,13 +2139,13 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
st = UsbCreateRootHubPdo(&xhci->Hcd);
if (!KSUCCESS(st) && st != NTLSTATUS_ALREADY_EXISTS) {
DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st);
return st;
goto failDma;
}
st = XhciRegisterController(xhci);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI controller registry full st=0x%x\n", st);
return st;
goto failDma;
}
if (xhci->Irq != 0xFFu) {
@@ -2009,6 +2170,16 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
xhci->HidDeviceCount);
return NTLSTATUS_SUCCESS;
failDma:
XhciHaltForDmaRelease(xhci);
XhciUnregisterController(xhci);
if (xhci->Hcd.RootHubPdo) {
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0;
}
XhciReleaseControllerDma(xhci);
return st;
}
// Dispatches pnp.
@@ -2037,22 +2208,22 @@ KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
xhci->SurpriseRemoved = true;
xhci->Running = false;
xhci->InterruptsEnabled = false;
if (xhci->Op) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS);
}
if (xhci->Hcd.RootHubPdo)
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
xhci->Running = false;
xhci->InterruptsEnabled = false;
if (xhci->Hcd.RootHubPdo)
XhciHaltForDmaRelease(xhci);
if (xhci->Hcd.RootHubPdo) {
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0;
}
XhciUnregisterController(xhci);
XhciReleaseControllerDma(xhci);
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
code/drivers/display/vmwaresvga/vmwaresvga.c
+166 -14
View File
@@ -748,10 +748,103 @@ static KSTATUS VmwSvgaDisplayGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMo
return NTLSTATUS_SUCCESS;
}
// Implements the display Blit operation.
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
// Streams one pixel row into the write-combined SVGA framebuffer.
static void VmwSvgaCopyToFramebuffer(uint32_t* destination,
const uint32_t* source,
size_t pixelCount) {
if (!destination || !source || pixelCount == 0)
return;
while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) {
*destination++ = *source++;
pixelCount--;
}
while (pixelCount >= 16u) {
__m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
__m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
__m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
__m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
_mm_stream_si128((__m128i*)(destination + 0), v0);
_mm_stream_si128((__m128i*)(destination + 4), v1);
_mm_stream_si128((__m128i*)(destination + 8), v2);
_mm_stream_si128((__m128i*)(destination + 12), v3);
source += 16;
destination += 16;
pixelCount -= 16;
}
while (pixelCount >= 4u) {
__m128i value = _mm_loadu_si128((const __m128i*)source);
_mm_stream_si128((__m128i*)destination, value);
source += 4;
destination += 4;
pixelCount -= 4;
}
while (pixelCount != 0) {
*destination++ = *source++;
pixelCount--;
}
}
// Copies one row into the retained SVGA shadow and framebuffer in one read pass.
static void VmwSvgaCopyToShadowAndFramebuffer(uint32_t* shadow,
uint32_t* framebuffer,
const uint32_t* source,
size_t pixelCount) {
if (!shadow || !framebuffer || !source || pixelCount == 0)
return;
while (pixelCount != 0 && ((uintptr_t)framebuffer & 15u) != 0) {
uint32_t value = *source++;
*shadow++ = value;
*framebuffer++ = value;
pixelCount--;
}
while (pixelCount >= 16u) {
__m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
__m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
__m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
__m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
_mm_storeu_si128((__m128i*)(shadow + 0), v0);
_mm_storeu_si128((__m128i*)(shadow + 4), v1);
_mm_storeu_si128((__m128i*)(shadow + 8), v2);
_mm_storeu_si128((__m128i*)(shadow + 12), v3);
_mm_stream_si128((__m128i*)(framebuffer + 0), v0);
_mm_stream_si128((__m128i*)(framebuffer + 4), v1);
_mm_stream_si128((__m128i*)(framebuffer + 8), v2);
_mm_stream_si128((__m128i*)(framebuffer + 12), v3);
source += 16;
shadow += 16;
framebuffer += 16;
pixelCount -= 16;
}
while (pixelCount >= 4u) {
__m128i value = _mm_loadu_si128((const __m128i*)source);
_mm_storeu_si128((__m128i*)shadow, value);
_mm_stream_si128((__m128i*)framebuffer, value);
source += 4;
shadow += 4;
framebuffer += 4;
pixelCount -= 4;
}
while (pixelCount != 0) {
uint32_t value = *source++;
*shadow++ = value;
*framebuffer++ = value;
pixelCount--;
}
}
// Implements the display pitched blit operation.
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0)
if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
DISPLAY_RECT r = { x, y, w, h };
@@ -766,12 +859,60 @@ static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, i
size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t);
for (int row = 0; row < r.H; row++) {
uint32_t* dst = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX;
const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
memcpy(dst, src, bytesPerRow);
}
return NTLSTATUS_SUCCESS;
}
// Presents one pitched rectangle without a shadow-to-framebuffer second pass.
static KSTATUS VmwSvgaDisplayPresentPitched(DISPLAY_DEVICE* device,
int x,
int y,
int w,
int h,
const uint32_t* pixels,
uint32_t pitchBytes) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Framebuffer || !v->Backbuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
return NTLSTATUS_SUCCESS;
int srcOffsetX = r.X - x;
int srcOffsetY = r.Y - y;
size_t pixelCount = (size_t)r.W;
uint64_t bytes = (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
v->Stats.BlitCount++;
v->Stats.PresentCount++;
v->Stats.PresentRectCount++;
v->Stats.DirectPresentCount++;
v->Stats.ScanoutBytes += bytes;
v->Stats.BytesCopied += bytes * 2u;
for (int row = 0; row < r.H; row++) {
const uint8_t* sourceRow = (const uint8_t*)pixels +
(size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
uint32_t* shadow = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
uint32_t* framebuffer = (uint32_t*)(uintptr_t)(v->Framebuffer +
((size_t)(r.Y + row) * v->BytesPerLine) +
((size_t)r.X * sizeof(uint32_t)));
VmwSvgaCopyToShadowAndFramebuffer(shadow, framebuffer, source, pixelCount);
}
_mm_sfence();
_ReadWriteBarrier();
KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H);
if (!KSUCCESS(st))
return st;
return VmwSvgaSyncDevice(v);
}
// Implements the display Fill Rect operation.
static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
@@ -869,13 +1010,19 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT*
DISPLAY_RECT r = rects[i];
if (!VmwSvgaClipRect(v, &r))
continue;
v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
size_t rowBytes = (size_t)r.W * sizeof(uint32_t);
uint64_t bytes = (uint64_t)(uint32_t)r.W *
(uint64_t)(uint32_t)r.H * sizeof(uint32_t);
v->Stats.BytesCopied += bytes;
v->Stats.ScanoutBytes += bytes;
size_t pixelCount = (size_t)r.W;
for (int row = 0; row < r.H; row++) {
volatile uint8_t* dstBytes = v->Framebuffer + ((size_t)(r.Y + row) * v->BytesPerLine) + ((size_t)r.X * sizeof(uint32_t));
uint32_t* dst = (uint32_t*)(uintptr_t)(v->Framebuffer +
((size_t)(r.Y + row) * v->BytesPerLine) +
((size_t)r.X * sizeof(uint32_t)));
uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
memcpy((void*)dstBytes, src, rowBytes);
VmwSvgaCopyToFramebuffer(dst, src, pixelCount);
}
_mm_sfence();
_ReadWriteBarrier();
KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H);
@@ -1019,6 +1166,7 @@ static KSTATUS VmwSvgaDisplayQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* o
static const DISPLAY_DEVICE_OPS gVmwSvgaDisplayOps = {
.GetMode = VmwSvgaDisplayGetMode,
.Blit = VmwSvgaDisplayBlit,
.PresentPitched = VmwSvgaDisplayPresentPitched,
.FillRect = VmwSvgaDisplayFillRect,
.CopyRect = VmwSvgaDisplayCopyRect,
.ReadRect = VmwSvgaDisplayReadRect,
@@ -1060,15 +1208,19 @@ static KSTATUS VmwSvgaSelectBars(VMWSVGA_DEVICE* v) {
return NTLSTATUS_SUCCESS;
}
// Implements the mmio Page Flags operation.
static uint64_t VmwSvgaMmioPageFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_WRITETHROUGH | PAGE_NOCACHE | PAGE_NX;
// Returns the write-combined mapping flags used for the framebuffer aperture.
static uint64_t VmwSvgaFramebufferPageFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_WC | PAGE_NX;
}
// Returns the uncached mapping flags used for device-control apertures.
static uint64_t VmwSvgaRegisterPageFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC | PAGE_NX;
}
// Maps bars.
static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) {
uint64_t mmioFlags = VmwSvgaMmioPageFlags();
KSTATUS st = PciMapBar(v->Pdo, 1, mmioFlags,
KSTATUS st = PciMapBar(v->Pdo, 1, VmwSvgaFramebufferPageFlags(),
(void**)&v->FramebufferMapping, &v->FramebufferBytes);
if (!KSUCCESS(st) || !v->FramebufferMapping) {
DbgPrint("VMWSVGA: PciMapBar framebuffer failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n",
@@ -1079,7 +1231,7 @@ static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) {
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
st = PciMapBar(v->Pdo, 2, mmioFlags,
st = PciMapBar(v->Pdo, 2, VmwSvgaRegisterPageFlags(),
(void**)&v->Fifo, &v->FifoBytes);
if (!KSUCCESS(st) || !v->Fifo) {
DbgPrint("VMWSVGA: PciMapBar FIFO failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n",
code/drivers/drivers.ini
+1
View File
@@ -1,5 +1,6 @@
[boot]
pci.sys
hda.sys
ps2.sys
serial.sys
ahci.sys
code/drivers/fs/fat/fat.c
+37 -12
View File
@@ -80,6 +80,11 @@ static void FatCacheEnsureMutex(void) {
gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
gFatCacheMutex.Owner = (struct THREAD*)0;
LIST_INIT(&gFatCacheMutex.WaitList);
LIST_INIT(&gFatCacheMutex.OwnerLink);
gFatCacheMutex.DonationClass = KeSchedulingNormal;
gFatCacheMutex.DonationPriority = PriorityLow;
gFatCacheMutex.HasDonation = 0;
gFatCacheMutex.Abandoned = 0;
gFatCacheMutexReady = 1;
}
KeReleaseSpinlock(&gFatCacheMutexInitLock, flags);
@@ -89,7 +94,7 @@ static int FatCacheAcquire(void) {
if (!KeCanWait())
return 0;
FatCacheEnsureMutex();
return KSUCCESS(KeAcquireMutex(&gFatCacheMutex));
return KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatCacheMutex, KE_WAIT_INFINITE));
}
static void FatCacheRelease(int held) {
@@ -1913,6 +1918,17 @@ KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
return NTLSTATUS_SUCCESS;
}
// Flushes FAT metadata and the backing device write cache.
KSTATUS FatFlush(IO_FILESYSTEM* fs, void* fileCtx) {
(void)fs;
FAT_FILE* f = (FAT_FILE*)fileCtx;
if (!f || !f->Vol)
return NTLSTATUS_INVALID_PARAMETER;
FlushFatCache(f->Vol);
return IoFlushDisk(f->Vol->DriveIndex);
}
// Releases the requested object.
void FatClose(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs;
@@ -2304,11 +2320,14 @@ KSTATUS FatDelete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (!KSUCCESS(s)) return s;
if (!direntLba) return NTLSTATUS_ACCESS_DENIED;
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
if (!sec) return NTLSTATUS_NO_MEMORY;
s = ReadSectors(v, direntLba, 1, sec);
if (KSUCCESS(s)) { sec[direntOffset] = 0xE5; s = WriteSectors(v, direntLba, 1, sec); }
MmFreePool(sec);
uint32_t parentCluster = 0;
SYS_STRING_VIEW filenameView;
s = FatResolveParentForMutation(v, path, &parentCluster, &filenameView);
if (!KSUCCESS(s)) return s;
// Delete the complete entry run so long-name metadata cannot survive the
// short entry and keep writable vvfat mirrors visibly present.
s = FatDeleteDirentRunV(v, parentCluster, filenameView);
if (!KSUCCESS(s)) return s;
if (cluster >= 2) FreeClusterChain(v, cluster);
@@ -2583,7 +2602,7 @@ KSTATUS FatReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
FAT_DIR_CTX *ctx = (FAT_DIR_CTX *)dirCtx;
FAT_VOL *v = ctx->Vol;
for (;;) {
while (true) {
if (ctx->Cursor != ctx->LoadedCursor) {
if (!KSUCCESS(ReadSectors(v, ctx->LoadedLba, ctx->LoadedSectors, ctx->ClusterBuf)))
return NTLSTATUS_NO_MORE_FILES;
@@ -2918,8 +2937,8 @@ static void FatAsyncWriteSubmitNext(FAT_ASYNC_RW* ctx);
static void FatAsyncWorker(void* arg) {
(void)arg;
for (;;) {
KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore);
while (true) {
KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(s)) {
KeYield();
continue;
@@ -2959,8 +2978,9 @@ static KSTATUS FatEnsureAsyncWorkerStarted(void) {
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gFatAsyncSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)FatAsyncWorker, 0,
PriorityNormal, &worker);
&parameters, &worker);
(void)worker;
}
@@ -3355,6 +3375,11 @@ static void FatMountEnsureMutex(void) {
gFatMountMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
gFatMountMutex.Owner = (struct THREAD*)0;
LIST_INIT(&gFatMountMutex.WaitList);
LIST_INIT(&gFatMountMutex.OwnerLink);
gFatMountMutex.DonationClass = KeSchedulingNormal;
gFatMountMutex.DonationPriority = PriorityLow;
gFatMountMutex.HasDonation = 0;
gFatMountMutex.Abandoned = 0;
gFatMountMutexReady = 1;
}
KeReleaseSpinlock(&gFatMountMutexInitLock, flags);
@@ -3375,7 +3400,7 @@ static uint32_t FatTryMountDisk(uint8_t disk) {
return 0;
FatMountEnsureMutex();
if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex)))
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE)))
return 0;
if (gDiskScanned[disk]) {
@@ -3509,7 +3534,7 @@ static void FatUnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) {
uint32_t removed = 0;
FatMountEnsureMutex();
if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex)))
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE)))
return;
for (uint8_t i = 0; i < FAT_MAX_VOLUMES; i++) {
code/drivers/fs/fat/fat.h
+1
View File
@@ -22,6 +22,7 @@
typedef struct FAT_VOL_S FAT_VOL;
KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
KSTATUS FatFlush(IO_FILESYSTEM *fs, void *fileCtx);
void FatClose(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS FatRead(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead);
KSTATUS FatWrite(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten);
code/drivers/fs/fat/fat_mount.c
+1
View File
@@ -19,6 +19,7 @@ void FatInitVtable(IO_FILESYSTEM* fs, FAT_VOL* vol) {
fs->QueryInfo = FatQueryInfo;
fs->Stat = FatStat;
fs->QueryIdentity = FatQueryIdentity;
fs->Flush = FatFlush;
fs->Close = FatClose;
fs->DriverCtx = vol;
fs->OpenDir = FatOpenDir;
code/drivers/net/lwip/lwip_stack.c
+106 -10
View File
@@ -4,6 +4,7 @@
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddk/ddk.h>
#include <ntlite/status.h>
@@ -74,6 +75,15 @@ typedef struct LWIP_ATTACH_CTX {
err_t Result;
} LWIP_ATTACH_CTX;
typedef struct LWIP_RESOLVE_CONTEXT {
SEMAPHORE* Done;
volatile long References;
err_t Result;
u8_t DnsType;
ip_addr_t Address;
char Name[DNS_MAX_NAME_LENGTH];
} LWIP_RESOLVE_CONTEXT;
typedef struct LWIP_PROVIDER_SOCKET {
SEMAPHORE* Lock;
struct netconn* Conn;
@@ -118,6 +128,9 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how);
static KSTATUS LwipProviderSetOption(void* socket, uint32_t option, const void* value, size_t valueSize);
static KSTATUS LwipProviderGetOption(void* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut);
static KSTATUS LwipProviderPoll(void* socket, uint32_t events, uint32_t* returnedEventsOut);
static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context);
static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument);
static void LwipResolveStartCallback(void* argument);
static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut);
static KSTATUS LwipDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp);
@@ -355,7 +368,7 @@ static KSTATUS LwipLoadNetifConfig(const NET_DEVICE* device, LWIP_NETIF_CONFIG*
static KSTATUS LwipSocketLock(LWIP_PROVIDER_SOCKET* s) {
if (!s || !s->Lock)
return NTLSTATUS_INVALID_PARAMETER;
return KeWaitSemaphore(s->Lock);
return KeWaitSemaphore(s->Lock, KE_WAIT_INFINITE);
}
// Implements the socket Unlock operation.
@@ -464,7 +477,7 @@ static KSTATUS LwipSocketWaitRecvNetbuf(LWIP_PROVIDER_SOCKET* s,
uint32_t originalTimeout = s->ReceiveTimeoutMs;
uint32_t elapsed = 0;
for (;;) {
while (true) {
if (LwipSocketCancelRequested(cancelRequested)) {
netconn_set_recvtimeout(s->Conn, originalTimeout);
return NTLSTATUS_CANCELLED;
@@ -510,7 +523,7 @@ static KSTATUS LwipSocketWaitAccept(LWIP_PROVIDER_SOCKET* s,
uint32_t originalTimeout = s->ReceiveTimeoutMs;
uint32_t elapsed = 0;
for (;;) {
while (true) {
if (LwipSocketCancelRequested(cancelRequested)) {
netconn_set_recvtimeout(s->Conn, originalTimeout);
return NTLSTATUS_CANCELLED;
@@ -778,7 +791,7 @@ static KSTATUS LwipStackInitialize(const NTL_DDK* ddk) {
return st;
tcpip_init(LwipInitDone, done);
st = KeWaitSemaphore(done);
st = KeWaitSemaphore(done, KE_WAIT_INFINITE);
ObDereferenceObject(&done->Header);
if (!KSUCCESS(st))
return st;
@@ -826,7 +839,7 @@ static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device) {
err_t err = tcpip_callback(LwipAttachCallback, &ctx);
if (err == ERR_OK)
st = KeWaitSemaphore(done);
st = KeWaitSemaphore(done, KE_WAIT_INFINITE);
else
st = NTLSTATUS_FAILURE;
@@ -1135,7 +1148,7 @@ static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_K* add
return LwipErrToStatus(err);
}
for (;;) {
while (true) {
if (LwipSocketCancelRequested(cancelRequested)) {
(void)netconn_close(s->Conn);
s->ShutdownRead = 1;
@@ -1595,6 +1608,51 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how) {
return LwipErrToStatus(err);
}
// Releases one resolver owner and destroys the request after the final callback or waiter leaves.
static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context) {
if (!context)
return;
if (_InterlockedDecrement(&context->References) != 0)
return;
if (context->Done)
ObDereferenceObject(&context->Done->Header);
MmFreePool(context);
}
// Completes an asynchronous DNS lookup without retaining the caller's kernel stack.
static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument) {
LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument;
(void)name;
if (address) {
context->Address = *address;
context->Result = ERR_OK;
} else {
context->Result = ERR_VAL;
}
KeSignalSemaphore(context->Done);
LwipResolveRelease(context);
}
// Starts one DNS lookup in the tcpip thread and transfers its resolver reference to the callback.
static void LwipResolveStartCallback(void* argument) {
LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument;
err_t err = dns_gethostbyname_addrtype(context->Name,
&context->Address,
LwipResolveDnsDoneCallback,
context,
context->DnsType);
context->Result = err;
if (err == ERR_INPROGRESS)
return;
KeSignalSemaphore(context->Done);
LwipResolveRelease(context);
}
// Implements the provider Resolve Host operation.
static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut) {
if (!name || !addressOut)
@@ -1610,10 +1668,48 @@ static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SO
else
return NTLSTATUS_NOT_SUPPORTED;
ip_addr_t resolved;
err_t err = netconn_gethostbyname_addrtype(name, &resolved, dnsType);
if (err != ERR_OK)
return LwipErrToStatus(err);
size_t nameLength = strlen(name);
if (nameLength == 0 || nameLength >= DNS_MAX_NAME_LENGTH)
return NTLSTATUS_INVALID_PARAMETER;
LWIP_RESOLVE_CONTEXT* context =
(LWIP_RESOLVE_CONTEXT*)MmAllocatePool(sizeof(*context));
if (!context)
return NTLSTATUS_NO_MEMORY;
memset(context, 0, sizeof(*context));
memcpy(context->Name, name, nameLength + 1);
context->References = 2;
context->Result = ERR_INPROGRESS;
context->DnsType = dnsType;
KSTATUS status = KeSemaphoreCreate(0, 1, &context->Done);
if (!KSUCCESS(status)) {
MmFreePool(context);
return status;
}
err_t queueError = tcpip_try_callback(LwipResolveStartCallback, context);
if (queueError != ERR_OK) {
LwipResolveRelease(context);
LwipResolveRelease(context);
return LwipErrToStatus(queueError);
}
status = KeWaitSemaphore(context->Done, KE_WAIT_INFINITE);
if (!KSUCCESS(status)) {
DbgPrint("lwIP: resolve waiter left name=%s status=0x%x; DNS request remains active\n",
context->Name,
status);
LwipResolveRelease(context);
return status;
}
err_t result = context->Result;
ip_addr_t resolved = context->Address;
LwipResolveRelease(context);
if (result != ERR_OK)
return LwipErrToStatus(result);
if ((family == SYS_AF_INET && !IP_IS_V4(&resolved)) ||
(family == SYS_AF_INET6 && !IP_IS_V6(&resolved)))
return NTLSTATUS_NOT_FOUND;
code/drivers/net/lwip/lwip_sys_arch.c
+14 -23
View File
@@ -117,19 +117,14 @@ u32_t sys_arch_sem_wait(sys_sem_t* sem, u32_t timeout) {
u32_t start = sys_now();
if (timeout == 0) {
KSTATUS st = KeWaitSemaphore((*sem)->Sem);
KSTATUS st = KeWaitSemaphore((*sem)->Sem, KE_WAIT_INFINITE);
return KSUCCESS(st) ? (u32_t)(sys_now() - start) : SYS_ARCH_TIMEOUT;
}
for (;;) {
if (KSUCCESS(KeTrySemaphore((*sem)->Sem)))
return (u32_t)(sys_now() - start);
if ((u32_t)(sys_now() - start) >= timeout)
return SYS_ARCH_TIMEOUT;
KeYield();
}
KSTATUS status = KeWaitSemaphore((*sem)->Sem, timeout);
return KSUCCESS(status)
? (u32_t)(sys_now() - start)
: SYS_ARCH_TIMEOUT;
}
// Implements the sys sem free operation.
@@ -165,7 +160,7 @@ err_t sys_mutex_new(sys_mutex_t* mutex) {
// Implements the sys mutex lock operation.
void sys_mutex_lock(sys_mutex_t* mutex) {
if (mutex && *mutex && (*mutex)->Sem)
(void)KeWaitSemaphore((*mutex)->Sem);
(void)KeWaitSemaphore((*mutex)->Sem, KE_WAIT_INFINITE);
}
// Implements the sys mutex unlock operation.
@@ -279,28 +274,23 @@ u32_t sys_arch_mbox_fetch(sys_mbox_t* mbox, void** msg, u32_t timeout) {
u32_t start = sys_now();
if (timeout == 0) {
KSTATUS st = KeWaitSemaphore((*mbox)->Items);
KSTATUS st = KeWaitSemaphore((*mbox)->Items, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return SYS_ARCH_TIMEOUT;
return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start));
}
for (;;) {
if (KSUCCESS(KeTrySemaphore((*mbox)->Items)))
return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start));
if ((u32_t)(sys_now() - start) >= timeout)
return SYS_ARCH_TIMEOUT;
KeYield();
}
KSTATUS status = KeWaitSemaphore((*mbox)->Items, timeout);
if (!KSUCCESS(status))
return SYS_ARCH_TIMEOUT;
return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start));
}
u32_t sys_arch_mbox_tryfetch(sys_mbox_t* mbox, void** msg) {
if (!mbox || !*mbox || !(*mbox)->Valid)
return SYS_MBOX_EMPTY;
if (!KSUCCESS(KeTrySemaphore((*mbox)->Items)))
if (!KSUCCESS(KeWaitSemaphore((*mbox)->Items, 0)))
return SYS_MBOX_EMPTY;
return LwipMboxPopAfterItem(mbox, msg, 0);
@@ -347,9 +337,10 @@ sys_thread_t sys_thread_new(const char* name, lwip_thread_fn thread, void* arg,
// function pointer that lwIP supplied, with exactly the argument it supplied.
THREAD* t = 0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(kp);
KSTATUS st = KeCreateKernelThread((uintptr_t)thread,
(uintptr_t)arg,
kp,
&parameters,
&t);
if (!KSUCCESS(st))
KePanic(KE_PANIC_CODE_DRIVER, "lwIP: failed to create kernel thread: 0x%x\n", st);
code/drivers/net/rtl8139/rtl8139.c
+81 -30
View File
@@ -82,29 +82,79 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) {
r->NetDevice.HardwareAddressLength = 6;
}
// Creates dma.
static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma);
r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma);
if (!r->RxPhys || !r->TxPhys)
return NTLSTATUS_NO_MEMORY;
// Stops bus mastering before DMA memory can be released.
static void Rtl8139StopDma(RTL8139_DEVICE* r) {
if (!r || !r->Started)
return;
r->Started = false;
if (r->SurpriseRemoved)
return;
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
Rtl8139Write8(r, RTL8139_REG_CMD, 0);
KSTATUS status = Rtl8139Reset(r);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_DRIVER,
"RTL8139: failed to stop DMA before release status=0x%x\n",
status);
}
// RTL8139 has 32-bit DMA address registers. This is fine for typical QEMU
// and low-memory hobby OS configs, but make failure explicit.
if ((r->RxPhys + RTL8139_RX_ALLOC_SIZE - 1) > 0xFFFFFFFFULL ||
(r->TxPhys + RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE - 1) > 0xFFFFFFFFULL) {
DbgPrint("RTL8139: DMA allocation above 4GiB is not supported by this NIC\n");
return NTLSTATUS_NOT_SUPPORTED;
// Releases the common buffers and adapter owned by one RTL8139 device.
static void Rtl8139ReleaseDma(RTL8139_DEVICE* r) {
if (!r || !gRtl8139Ddk || !gRtl8139Ddk->Dma)
return;
gRtl8139Ddk->Dma->FreeCommonBuffer(r->TxDma);
gRtl8139Ddk->Dma->FreeCommonBuffer(r->RxDma);
r->TxDma = (IO_DMA_BUFFER*)0;
r->RxDma = (IO_DMA_BUFFER*)0;
r->TxBuffers = (uint8_t*)0;
r->RxBuffer = (uint8_t*)0;
r->TxDeviceAddress = 0;
r->RxDeviceAddress = 0;
gRtl8139Ddk->Dma->PutAdapter(r->DmaAdapter);
r->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
// Allocates the receive ring and transmit slots through the device DMA adapter.
static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
IO_DMA_REQUIREMENTS requirements;
memset(&requirements, 0, sizeof(requirements));
requirements.MaximumDeviceAddress = 0xFFFFFFFFULL;
requirements.Alignment = 4;
requirements.MaximumSegmentLength = RTL8139_TX_BUF_SIZE;
requirements.MaximumSegments = 1;
KSTATUS status = gRtl8139Ddk->Dma->GetAdapter(r->Fdo,
&requirements,
&r->DmaAdapter);
if (!KSUCCESS(status))
return status;
status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter,
RTL8139_RX_ALLOC_SIZE,
PAGE_SIZE,
&r->RxDma);
if (!KSUCCESS(status)) {
Rtl8139ReleaseDma(r);
return status;
}
r->RxBuffer = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->RxPhys);
r->TxBuffers = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->TxPhys);
memset(r->RxBuffer, 0, PAGE_SIZE * RTL8139_RX_PAGES);
memset(r->TxBuffers, 0, PAGE_SIZE * RTL8139_TX_PAGES);
status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter,
RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE,
PAGE_SIZE,
&r->TxDma);
if (!KSUCCESS(status)) {
Rtl8139ReleaseDma(r);
return status;
}
r->RxBuffer = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->RxDma);
r->TxBuffers = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->TxDma);
r->RxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->RxDma);
r->TxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->TxDma);
DbgPrint("RTL8139: rx=0x%016llX tx=0x%016llX\n",
(unsigned long long)r->RxPhys,
(unsigned long long)r->TxPhys);
(unsigned long long)r->RxDeviceAddress,
(unsigned long long)r->TxDeviceAddress);
return NTLSTATUS_SUCCESS;
}
@@ -136,13 +186,13 @@ static void Rtl8139ProgramHardware(RTL8139_DEVICE* r) {
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
Rtl8139Write16(r, RTL8139_REG_ISR, 0xFFFFu);
Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxPhys);
Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxDeviceAddress);
r->RxOffset = 0;
Rtl8139Write16(r, RTL8139_REG_CAPR, 0xFFF0u); // CAPR is rxOffset - 16.
for (uint32_t i = 0; i < RTL8139_TX_SLOTS; i++) {
PHYSICAL_ADDRESS p = r->TxPhys + (PHYSICAL_ADDRESS)i * RTL8139_TX_BUF_SIZE;
Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)p);
uint64_t address = r->TxDeviceAddress + (uint64_t)i * RTL8139_TX_BUF_SIZE;
Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)address);
}
Rtl8139Write32(r, RTL8139_REG_TCR,
@@ -360,7 +410,7 @@ static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) {
KSTATUS st = PciMapBar(r->Pdo,
best.Index,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&r->Mmio,
&r->RegisterBytes);
if (!KSUCCESS(st))
@@ -460,8 +510,10 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) {
r->NetDevice.SendFrame = Rtl8139SendFrame;
st = Rtl8139RegisterDevice(r);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
Rtl8139ReleaseDma(r);
return st;
}
KeIrqRegister(r->Irq, Rtl8139Isr);
r->Started = true;
@@ -470,9 +522,9 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) {
st = NetRegisterDevice(&r->NetDevice);
if (!KSUCCESS(st)) {
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
r->Started = false;
Rtl8139StopDma(r);
Rtl8139UnregisterDevice(r);
Rtl8139ReleaseDma(r);
return st;
}
@@ -503,17 +555,16 @@ KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
r->SurpriseRemoved = true;
r->Started = false;
device->PnpState = DevicePnpSurpriseRemoved;
if (r->Started)
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
r->Removed = true;
device->PnpState = DevicePnpDeleted;
if (r->Started)
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
Rtl8139StopDma(r);
Rtl8139UnregisterDevice(r);
Rtl8139ReleaseDma(r);
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
@@ -555,7 +606,7 @@ KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer)
if (!ddk || !ddk->Dma || !ddk->Timer)
return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("RTL8139: driver loading...\n");
code/drivers/net/rtl8139/rtl8139.h
+5 -2
View File
@@ -115,8 +115,11 @@ typedef struct RTL8139_DEVICE {
volatile uint8_t* Mmio;
uint16_t IoBase;
PHYSICAL_ADDRESS RxPhys;
PHYSICAL_ADDRESS TxPhys;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* RxDma;
IO_DMA_BUFFER* TxDma;
uint64_t RxDeviceAddress;
uint64_t TxDeviceAddress;
uint8_t* RxBuffer;
uint8_t* TxBuffers;
uint8_t RxScratch[RTL8139_TX_BUF_SIZE];
code/drivers/net/rtl816x/rtl816x.c
+158 -49
View File
@@ -83,48 +83,117 @@ static void RtlReadMac(RTL816X_DEVICE* r) {
r->NetDevice.HardwareAddressLength = 6;
}
// Implements the rtl Allocate Dma operation.
// Stops bus mastering before DMA memory can be released.
static void RtlStopDma(RTL816X_DEVICE* r) {
if (!r || !r->Started)
return;
r->Started = false;
if (r->SurpriseRemoved)
return;
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlWrite8(r, RTL_REG_CHIP_CMD, 0);
KSTATUS status = RtlReset(r);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_DRIVER,
"RTL816X: failed to stop DMA before release status=0x%x\n",
status);
}
// Releases the common buffers and adapter owned by one RTL816x device.
static void RtlReleaseDma(RTL816X_DEVICE* r) {
if (!r || !gRtl816xDdk || !gRtl816xDdk->Dma)
return;
gRtl816xDdk->Dma->FreeCommonBuffer(r->TxBuffersDma);
gRtl816xDdk->Dma->FreeCommonBuffer(r->RxBuffersDma);
gRtl816xDdk->Dma->FreeCommonBuffer(r->TxRingDma);
gRtl816xDdk->Dma->FreeCommonBuffer(r->RxRingDma);
r->TxBuffersDma = (IO_DMA_BUFFER*)0;
r->RxBuffersDma = (IO_DMA_BUFFER*)0;
r->TxRingDma = (IO_DMA_BUFFER*)0;
r->RxRingDma = (IO_DMA_BUFFER*)0;
r->TxBuffers = (uint8_t*)0;
r->RxBuffers = (uint8_t*)0;
r->TxRing = (RTL_DESC*)0;
r->RxRing = (RTL_DESC*)0;
r->TxBuffersDeviceAddress = 0;
r->RxBuffersDeviceAddress = 0;
r->TxRingDeviceAddress = 0;
r->RxRingDeviceAddress = 0;
gRtl816xDdk->Dma->PutAdapter(r->DmaAdapter);
r->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
// Allocates descriptor rings and packet buffers through the device DMA adapter.
static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma);
r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma);
IO_DMA_REQUIREMENTS requirements;
memset(&requirements, 0, sizeof(requirements));
requirements.MaximumDeviceAddress = (uint64_t)-1;
requirements.Alignment = 1;
requirements.MaximumSegmentLength = RTL_BUF_SIZE;
requirements.MaximumSegments = RTL_TX_COUNT;
if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys)
return NTLSTATUS_NO_MEMORY;
KSTATUS status = gRtl816xDdk->Dma->GetAdapter(r->Fdo,
&requirements,
&r->DmaAdapter);
if (!KSUCCESS(status))
return status;
r->RxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->RxRingPhys);
r->TxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->TxRingPhys);
r->RxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->RxBuffersPhys);
r->TxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->TxBuffersPhys);
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_RING_PAGES,
PAGE_SIZE,
&r->RxRingDma);
if (!KSUCCESS(status)) goto fail;
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_RING_PAGES,
PAGE_SIZE,
&r->TxRingDma);
if (!KSUCCESS(status)) goto fail;
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_RXBUF_PAGES,
PAGE_SIZE,
&r->RxBuffersDma);
if (!KSUCCESS(status)) goto fail;
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_TXBUF_PAGES,
PAGE_SIZE,
&r->TxBuffersDma);
if (!KSUCCESS(status)) goto fail;
memset(r->RxRing, 0, PAGE_SIZE * RTL_RING_PAGES);
memset(r->TxRing, 0, PAGE_SIZE * RTL_RING_PAGES);
memset(r->RxBuffers, 0, PAGE_SIZE * RTL_RXBUF_PAGES);
memset(r->TxBuffers, 0, PAGE_SIZE * RTL_TXBUF_PAGES);
r->RxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxRingDma);
r->TxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxRingDma);
r->RxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxBuffersDma);
r->TxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxBuffersDma);
r->RxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxRingDma);
r->TxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxRingDma);
r->RxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxBuffersDma);
r->TxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxBuffersDma);
DbgPrint("RTL816X: rxRing=0x%016llX txRing=0x%016llX rxBuf=0x%016llX txBuf=0x%016llX\n",
(unsigned long long)r->RxRingPhys,
(unsigned long long)r->TxRingPhys,
(unsigned long long)r->RxBuffersPhys,
(unsigned long long)r->TxBuffersPhys);
(unsigned long long)r->RxRingDeviceAddress,
(unsigned long long)r->TxRingDeviceAddress,
(unsigned long long)r->RxBuffersDeviceAddress,
(unsigned long long)r->TxBuffersDeviceAddress);
return NTLSTATUS_SUCCESS;
fail:
RtlReleaseDma(r);
return status;
}
// Implements the rtl Init Descriptors operation.
static void RtlInitDescriptors(RTL816X_DEVICE* r) {
for (uint32_t i = 0; i < RTL_RX_COUNT; i++) {
PHYSICAL_ADDRESS p = r->RxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE;
r->RxRing[i].addrLo = (uint32_t)p;
r->RxRing[i].addrHi = (uint32_t)(p >> 32);
uint64_t address = r->RxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE;
r->RxRing[i].addrLo = (uint32_t)address;
r->RxRing[i].addrHi = (uint32_t)(address >> 32);
r->RxRing[i].opts2 = 0;
r->RxRing[i].opts1 = RTL_DESC_OWN | RTL_BUF_SIZE | ((i == RTL_RX_COUNT - 1) ? RTL_DESC_EOR : 0);
}
for (uint32_t i = 0; i < RTL_TX_COUNT; i++) {
PHYSICAL_ADDRESS p = r->TxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE;
r->TxRing[i].addrLo = (uint32_t)p;
r->TxRing[i].addrHi = (uint32_t)(p >> 32);
uint64_t address = r->TxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE;
r->TxRing[i].addrLo = (uint32_t)address;
r->TxRing[i].addrHi = (uint32_t)(address >> 32);
r->TxRing[i].opts2 = 0;
r->TxRing[i].opts1 = (i == RTL_TX_COUNT - 1) ? RTL_DESC_EOR : 0;
}
@@ -141,12 +210,12 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) {
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingPhys);
RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingPhys >> 32));
RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingPhys);
RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingPhys >> 32));
RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingPhys);
RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingPhys >> 32));
RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingDeviceAddress);
RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32));
RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingDeviceAddress);
RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32));
RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingDeviceAddress);
RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingDeviceAddress >> 32));
RtlWrite16(r, RTL_REG_CPLUS_CMD, RtlRead16(r, RTL_REG_CPLUS_CMD) & ~(uint16_t)0x00E0u);
RtlWrite16(r, RTL_REG_RX_MAXSIZE, RTL_BUF_SIZE);
@@ -366,7 +435,7 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) {
KSTATUS st = PciMapBar(r->Pdo,
best.Index,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&r->Mmio,
&r->RegisterBytes);
if (!KSUCCESS(st))
@@ -400,6 +469,41 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
return NTLSTATUS_NOT_FOUND;
}
// Configures a dedicated MSI vector and falls back to a routed INTx line.
static KSTATUS RtlConfigureInterrupt(RTL816X_DEVICE* r) {
if (!r || !r->Pdo)
return NTLSTATUS_INVALID_PARAMETER;
r->Irq = 0xFFu;
r->UsingMsi = false;
KSTATUS msiStatus = PciEnableMsiAuto(r->Pdo, &r->Irq);
if (KSUCCESS(msiStatus)) {
r->UsingMsi = true;
DbgPrint("RTL816X: using MSI IRQ slot %u\n", r->Irq);
return NTLSTATUS_SUCCESS;
}
DbgPrint("RTL816X: MSI setup failed status=0x%x; attempting explicit INTx fallback\n",
msiStatus);
KSTATUS intxStatus = PciGetInterrupt(r->Pdo, &r->Irq);
if (!KSUCCESS(intxStatus))
return intxStatus;
if (r->Irq == 0xFFu || r->Irq >= 24u)
return NTLSTATUS_NOT_SUPPORTED;
intxStatus = PciEnableDevice(r->Pdo,
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX |
(r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
if (!KSUCCESS(intxStatus))
return intxStatus;
DbgPrint("RTL816X: using legacy INTx IRQ %u\n", r->Irq);
return NTLSTATUS_SUCCESS;
}
// Implements the rtl Start Device operation.
static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
if (!r || r->Magic != RTL816X_DEVICE_MAGIC || !r->Pdo || !r->Fdo)
@@ -428,18 +532,15 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
st = PciEnableDevice(r->Pdo,
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX |
PCI_DISABLE_INTX |
(r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
if (!KSUCCESS(st)) {
DbgPrint("RTL816X: PciEnableDevice failed: 0x%x\n", st);
return st;
}
st = PciGetInterrupt(r->Pdo, &r->Irq);
if (!KSUCCESS(st) || r->Irq == 0xFFu) {
DbgPrint("RTL816X: no legacy PCI IRQ line wired\n");
return KSUCCESS(st) ? NTLSTATUS_NOT_SUPPORTED : st;
}
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
st = RtlReset(r);
if (!KSUCCESS(st)) {
@@ -447,12 +548,18 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
return st;
}
st = RtlConfigureInterrupt(r);
if (!KSUCCESS(st)) {
DbgPrint("RTL816X: no usable MSI or INTx interrupt status=0x%x\n", st);
return st;
}
RtlReadMac(r);
DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u\n",
DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u mode=%s\n",
r->NetDevice.HardwareAddress[0], r->NetDevice.HardwareAddress[1],
r->NetDevice.HardwareAddress[2], r->NetDevice.HardwareAddress[3],
r->NetDevice.HardwareAddress[4], r->NetDevice.HardwareAddress[5],
r->Irq);
r->Irq, r->UsingMsi ? "MSI" : "INTx");
st = RtlAllocateDma(r);
if (!KSUCCESS(st))
@@ -466,8 +573,10 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
r->NetDevice.SendFrame = RtlSendFrame;
st = RtlRegisterDevice(r);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
RtlReleaseDma(r);
return st;
}
KeIrqRegister(r->Irq, RtlIsr);
r->Started = true;
@@ -476,13 +585,14 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
st = NetRegisterDevice(&r->NetDevice);
if (!KSUCCESS(st)) {
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
r->Started = false;
RtlStopDma(r);
RtlUnregisterDevice(r);
RtlReleaseDma(r);
return st;
}
DbgPrint("RTL816X: initialized; RX/TX enabled on IRQ %u\n", r->Irq);
DbgPrint("RTL816X: initialized; RX/TX enabled on %s IRQ %u\n",
r->UsingMsi ? "MSI" : "INTx", r->Irq);
return NTLSTATUS_SUCCESS;
}
@@ -509,17 +619,16 @@ KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
r->SurpriseRemoved = true;
r->Started = false;
device->PnpState = DevicePnpSurpriseRemoved;
if (r->Started)
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
r->Removed = true;
device->PnpState = DevicePnpDeleted;
if (r->Started)
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlStopDma(r);
RtlUnregisterDevice(r);
RtlReleaseDma(r);
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
@@ -561,7 +670,7 @@ KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer)
if (!ddk || !ddk->Dma || !ddk->Timer)
return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("RTL816X: driver loading...\n");
code/drivers/net/rtl816x/rtl816x.h
+10 -4
View File
@@ -117,6 +117,7 @@ typedef struct RTL816X_DEVICE {
PCI_BAR RegisterBar;
uint64_t RegisterBytes;
uint8_t Irq;
bool UsingMsi;
bool Started;
bool Removed;
@@ -126,15 +127,20 @@ typedef struct RTL816X_DEVICE {
volatile uint8_t* Mmio;
uint16_t IoBase;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* RxRingDma;
IO_DMA_BUFFER* TxRingDma;
IO_DMA_BUFFER* RxBuffersDma;
IO_DMA_BUFFER* TxBuffersDma;
RTL_DESC* RxRing;
RTL_DESC* TxRing;
PHYSICAL_ADDRESS RxRingPhys;
PHYSICAL_ADDRESS TxRingPhys;
uint64_t RxRingDeviceAddress;
uint64_t TxRingDeviceAddress;
uint8_t* RxBuffers;
uint8_t* TxBuffers;
PHYSICAL_ADDRESS RxBuffersPhys;
PHYSICAL_ADDRESS TxBuffersPhys;
uint64_t RxBuffersDeviceAddress;
uint64_t TxBuffersDeviceAddress;
uint32_t RxIndex;
uint32_t TxIndex;
code/drivers/storage/ahci/ahci.c
+366 -104
View File
@@ -9,6 +9,11 @@ static AHCI_CONTROLLER* g_controllers[MAX_AHCI_CONTROLLERS];
static int g_controllerCount;
static uint8_t g_nextDiskIndex;
#define AHCI_COMMAND_TIMEOUT_MS 10000ull
#define AHCI_PORT_ERROR_MASK \
(PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define AHCI_PORT_TERMINAL_MASK (PORT_IS_DHRS | AHCI_PORT_ERROR_MASK)
static const IO_DEVICE_ID g_ahciIds[] = {
{ IoBusPci, { "PCI\\CC_010601", sizeof("PCI\\CC_010601") - 1 } },
};
@@ -21,20 +26,21 @@ static const IO_DRIVER_ID_TABLE g_ahciIdTable = {
// Helpers.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl);
static void AhciUnregisterController(AHCI_CONTROLLER* ctl);
// Accessors for port structure memory at known offsets from structVirt/Phys.
// Accessors for port structure memory at known CPU and device-address offsets.
static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) {
return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST);
}
// Implements the cmd list phys operation.
static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_CMDLIST;
// Returns the command-list device address.
static inline uint64_t cmd_list_device_address(AHCI_DRIVE *drv) {
return drv->structDeviceAddress + PORT_OFF_CMDLIST;
}
// Implements the fis phys operation.
static inline uint64_t fis_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_FIS;
// Returns the received-FIS device address.
static inline uint64_t fis_device_address(AHCI_DRIVE *drv) {
return drv->structDeviceAddress + PORT_OFF_FIS;
}
static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
@@ -43,9 +49,9 @@ static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
+ (uint32_t)slot * PORT_CMDTBL_SIZE);
}
// Implements the cmd tbl phys operation.
static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
return drv->structPhys + PORT_OFF_CMDTBLS
// Returns one command-table device address.
static inline uint64_t cmd_tbl_device_address(AHCI_DRIVE *drv, int slot) {
return drv->structDeviceAddress + PORT_OFF_CMDTBLS
+ (uint64_t)slot * PORT_CMDTBL_SIZE;
}
@@ -53,53 +59,108 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
// Port memory initialisation.
// Stops one drive worker before its queue and DMA storage are released.
static void AhciStopWorker(AHCI_DRIVE* drv) {
if (!drv)
return;
drv->workerRunning = false;
if (drv->irpQueue.Semaphore)
KeSignalSemaphore(drv->irpQueue.Semaphore);
if (drv->workerThread && drv->workerFinished)
(void)KeWaitForEvent(drv->workerFinished, 0);
if (drv->workerThread) {
ObDereferenceObject((OBJECT*)drv->workerThread);
drv->workerThread = (THREAD*)0;
}
if (drv->workerFinished) {
ObDereferenceObject(&drv->workerFinished->Header);
drv->workerFinished = (EVENT*)0;
}
}
// Releases the non-DMA synchronization objects owned by one AHCI drive.
static void AhciReleaseDriveRuntime(AHCI_DRIVE* drv) {
if (!drv)
return;
AhciStopWorker(drv);
if (drv->removeLockReady) {
IoDestroyRemoveLock(&drv->removeLock);
drv->removeLockReady = false;
}
if (drv->irpQueue.Semaphore) {
ObDereferenceObject(&drv->irpQueue.Semaphore->Header);
drv->irpQueue.Semaphore = (SEMAPHORE*)0;
}
if (drv->completionEvent) {
ObDereferenceObject(&drv->completionEvent->Header);
drv->completionEvent = (EVENT*)0;
}
}
// Releases the DMA buffers owned by one AHCI drive.
static void AhciReleaseDriveDma(AHCI_DRIVE* drv) {
if (!drv || !gAhciDdk || !gAhciDdk->Dma)
return;
if (drv->port && drv->controller &&
!drv->surpriseRemoved && !drv->controller->surpriseRemoved)
port_stop(drv->port);
gAhciDdk->Dma->FreeCommonBuffer(drv->bounceDma);
gAhciDdk->Dma->FreeCommonBuffer(drv->structDma);
drv->bounceDma = (IO_DMA_BUFFER*)0;
drv->structDma = (IO_DMA_BUFFER*)0;
drv->bounceDeviceAddress = 0;
drv->structDeviceAddress = 0;
drv->bounceVirt = (void*)0;
drv->structVirt = (void*)0;
}
// Allocates and programs one port's command, FIS, and bounce buffers.
static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
if (!drv || !drv->controller || !drv->controller->dmaAdapter)
return NTLSTATUS_INVALID_PARAMETER;
drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma);
if (!drv->structPhys) return NTLSTATUS_NO_MEMORY;
drv->structVirt = gAhciDdk->Memory->PhysToVirt(drv->structPhys);
// Force NOCACHE on DMA descriptor memory.
MmPteProtectRange((uintptr_t)drv->structVirt,
PORT_PAGES_STRUCT * 4096, flags);
memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096);
KSTATUS status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter,
PORT_PAGES_STRUCT * PAGE_SIZE,
PAGE_SIZE,
&drv->structDma);
if (!KSUCCESS(status))
return status;
drv->structVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->structDma);
drv->structDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->structDma);
drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma);
if (!drv->bouncePhys) return NTLSTATUS_NO_MEMORY;
drv->bounceVirt = gAhciDdk->Memory->PhysToVirt(drv->bouncePhys);
// Bounce buffer: NOCACHE to ensure CPU writes reach physical memory
// before the HBA's DMA engine reads them.
MmPteProtectRange((uintptr_t)drv->bounceVirt,
BOUNCE_PAGES * 4096, flags);
memset(drv->bounceVirt, 0, BOUNCE_BYTES);
status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter,
BOUNCE_BYTES,
PAGE_SIZE,
&drv->bounceDma);
if (!KSUCCESS(status)) {
AhciReleaseDriveDma(drv);
return status;
}
drv->bounceVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->bounceDma);
drv->bounceDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->bounceDma);
HBA_PORT *port = drv->port;
port_stop(port);
// Set command list base.
port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF);
port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32);
port->clb = (uint32_t)(cmd_list_device_address(drv) & 0xFFFFFFFF);
port->clbu = (uint32_t)(cmd_list_device_address(drv) >> 32);
port->fb = (uint32_t)(fis_device_address(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_device_address(drv) >> 32);
// Set FIS base.
port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_phys(drv) >> 32);
// Initialise all 32 command headers, pointing each to its command table.
HBA_CMD_HEADER *hdrs = cmd_list(drv);
for (int i = 0; i < NUM_CMD_SLOTS; i++) {
hdrs[i].prdtl = PORT_PRDT_COUNT;
uint64_t tbl = cmd_tbl_phys(drv, i);
hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF);
hdrs[i].ctbau = (uint32_t)(tbl >> 32);
uint64_t table = cmd_tbl_device_address(drv, i);
hdrs[i].ctba = (uint32_t)(table & 0xFFFFFFFF);
hdrs[i].ctbau = (uint32_t)(table >> 32);
}
// Clear any stale interrupts then start the port engine.
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
port_start(port);
return NTLSTATUS_SUCCESS;
}
@@ -125,16 +186,22 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
// Issue slot 0.
port->ci = (1u << 0);
// Poll until slot 0 clears.
while (true) {
if (!(port->ci & (1u << 0))) break;
if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) {
// Poll until slot 0 clears, but never let early-init I/O hang forever.
spin = 0;
while ((port->ci & (1u << 0)) && spin < POLL_TIMEOUT) {
if (port->is & AHCI_PORT_ERROR_MASK) {
DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n",
port->is, port->tfd, port->serr);
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
return NTLSTATUS_DEVICE_ERROR;
}
spin++;
}
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: command timed out while polling CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, port->is, port->serr);
return NTLSTATUS_TIMEOUT;
}
if (port->is & PORT_IS_TFES) {
@@ -146,6 +213,30 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
return NTLSTATUS_SUCCESS;
}
// Recovers a port after a command that cannot be trusted to have retired.
static void AhciRecoverPort(AHCI_DRIVE *drv) {
if (!drv || !drv->port)
return;
HBA_PORT *port = drv->port;
uint32_t savedIe = port->ie;
// Keep reset-generated FIS traffic from satisfying a command waiter.
port->ie = 0;
KSTATUS status = port_comreset(drv);
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
drv->lastIs = 0;
if (drv->completionEvent)
KeClearEvent(drv->completionEvent);
port->ie = savedIe;
if (!KSUCCESS(status)) {
DbgPrint("AHCI: port recovery failed st=0x%x CI=0x%x TFD=0x%x SSTS=0x%x\n",
status, port->ci, port->tfd, port->ssts);
}
}
// Interrupt bottom half. The ISR owns hardware acknowledgement and status
// capture; this DPC wakes the worker thread after the hard IRQ has unwound.
@@ -191,7 +282,11 @@ static void AhciIsr(void) {
// Clear the HBA-level bit.
ctl->hba->is = portBit;
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
// DMA Setup, PIO Setup, and other progress interrupts are not
// command completion. Only wake the worker for D2H completion or
// an error that terminates the command.
if (portIs & AHCI_PORT_TERMINAL_MASK)
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
}
}
}
@@ -206,8 +301,19 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
// Clear stale IS and snapshot before issuing so we don't mis-read a
// leftover value from a prior command.
// A serialized slot must be idle before reuse. A stale active bit means a
// prior command escaped normal completion and must be recovered first.
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: slot 0 still active before issue CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, port->is, port->serr);
AhciRecoverPort(drv);
if (port->ci & (1u << 0))
return NTLSTATUS_DEVICE_ERROR;
}
// Clear stale software and hardware completion state before submission.
// The event can otherwise retain a late DPC signal from an older command.
KeClearEvent(drv->completionEvent);
port->is = (uint32_t)-1;
drv->lastIs = 0;
@@ -216,20 +322,54 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT)
spin++;
if (spin == POLL_TIMEOUT) {
DbgPrint("AHCI: port hung (pre-issue)\n");
DbgPrint("AHCI: port hung before issue TFD=0x%x IS=0x%x SERR=0x%x\n",
port->tfd, port->is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_TIMEOUT;
}
// Issue slot 0, then block on the event.
// Issue slot 0, then block with a finite deadline.
port->ci = (1u << 0);
KeWaitForEvent(drv->completionEvent, 0);
EVENT *events[1] = { drv->completionEvent };
KSTATUS waitStatus = KeWaitForMultipleEvents(events, 1, 0,
AHCI_COMMAND_TIMEOUT_MS,
(uint32_t *)0);
// The ISR has cleared IS - check for errors in the snapshot.
uint32_t is = drv->lastIs;
if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) {
DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n",
is, port->tfd, port->serr);
port->serr = (uint32_t)-1;
// Merge any live status that was not consumed by the ISR. This lets a
// missed IRQ still report the hardware error that accompanied it.
uint32_t liveIs = port->is;
uint32_t is = drv->lastIs | liveIs;
if (liveIs)
port->is = liveIs;
if (is & AHCI_PORT_ERROR_MASK) {
DbgPrint("AHCI: command error IS=0x%x CI=0x%x TFD=0x%x SERR=0x%x\n",
is, port->ci, port->tfd, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_DEVICE_ERROR;
}
if (waitStatus == NTLSTATUS_TIMEOUT) {
if (!(port->ci & (1u << 0))) {
DbgPrint("AHCI: completion IRQ missed; command retired IS=0x%x TFD=0x%x SERR=0x%x\n",
is, port->tfd, port->serr);
return NTLSTATUS_SUCCESS;
}
DbgPrint("AHCI: command timeout CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_TIMEOUT;
}
if (waitStatus != NTLSTATUS_SUCCESS)
return waitStatus;
// An interrupt is only completion if hardware has also retired slot 0.
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: premature completion IRQ CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_DEVICE_ERROR;
}
@@ -239,7 +379,7 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
// Build a standard DMA read or write command in slot 0.
static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
uint64_t bufPhys, bool write) {
uint64_t bufferDeviceAddress, bool write) {
HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0
HBA_CMD_TBL *tbl = cmd_tbl(drv, 0);
@@ -256,7 +396,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
// Fill PRDT entries.
uint32_t remaining = sectors;
uint64_t buf = bufPhys;
uint64_t buf = bufferDeviceAddress;
for (uint32_t i = 0; i < prdtl; i++) {
uint32_t sects = (remaining > 16) ? 16 : remaining;
uint32_t bytes = sects * 512;
@@ -289,6 +429,31 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
fis->counth = (uint8_t)(sectors >> 8);
}
// Builds a non-data ATA FLUSH CACHE EXT command in slot 0.
static void build_flush_cmd(AHCI_DRIVE* drv) {
HBA_CMD_HEADER* hdr = cmd_list(drv);
HBA_CMD_TBL* tbl = cmd_tbl(drv, 0);
memset(tbl, 0, PORT_CMDTBL_SIZE);
hdr->cfl = sizeof(FIS_REG_H2D) / 4;
hdr->w = 0;
hdr->prdtl = 0;
hdr->prdbc = 0;
FIS_REG_H2D* fis = (FIS_REG_H2D*)tbl->cfis;
memset(fis, 0, sizeof(*fis));
fis->fis_type = FIS_TYPE_REG_H2D;
fis->c = 1;
fis->command = ATA_CMD_FLUSH_EXT;
}
// Flushes the drive write cache through the serialized command worker.
static KSTATUS do_flush(AHCI_DRIVE* drv) {
if (!drv) return NTLSTATUS_INVALID_PARAMETER;
build_flush_cmd(drv);
return port_issue_cmd(drv);
}
// IDENTIFY command - fills a 512-byte buffer and returns it.
static KSTATUS port_identify(AHCI_DRIVE *drv) {
@@ -305,8 +470,8 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
hdr->prdtl = 1;
hdr->prdbc = 0;
tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF);
tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32);
tbl->prdt_entry[0].dba = (uint32_t)(drv->bounceDeviceAddress & 0xFFFFFFFF);
tbl->prdt_entry[0].dbau = (uint32_t)(drv->bounceDeviceAddress >> 32);
tbl->prdt_entry[0].dbc = 512 - 1;
tbl->prdt_entry[0].i = 1;
@@ -347,7 +512,7 @@ static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba,
while (count > 0) {
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
build_rw_cmd(drv, lba, batch, drv->bouncePhys, false);
build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, false);
KSTATUS st = port_issue_cmd(drv);
if (!KSUCCESS(st)) return st;
@@ -368,7 +533,7 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
memcpy(drv->bounceVirt, in, (size_t)batch * 512);
build_rw_cmd(drv, lba, batch, drv->bouncePhys, true);
build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, true);
KSTATUS st = port_issue_cmd(drv);
if (!KSUCCESS(st)) return st;
@@ -463,13 +628,16 @@ static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
drv->surpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
drv->removed = true;
device->PnpState = DevicePnpDeleted;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->removeLockReady)
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
IoDeleteDevice(device);
drv->deviceObject = (DEVICE_OBJECT*)0;
return NTLSTATUS_SUCCESS;
@@ -511,7 +679,10 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
for (int i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
drv->surpriseRemoved = true;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->deviceObject)
(void)IoNotifyDiskSurpriseRemoved(drv->deviceObject->u.Disk.DiskIndex);
else
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
}
return NTLSTATUS_SUCCESS;
@@ -520,9 +691,18 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
device->PnpState = DevicePnpDeleted;
for (int i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
drv->removed = true;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->deviceObject) {
(void)IoUnregisterDiskDevice(drv->deviceObject->u.Disk.DiskIndex);
} else {
drv->removed = true;
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
}
}
AhciUnregisterController(ctl);
gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter);
ctl->dmaAdapter = (IO_DMA_ADAPTER*)0;
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
@@ -532,6 +712,29 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Queues one write-cache flush on the per-drive worker.
static KSTATUS AhciDispatchFlush(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER;
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
if (!drv || drv->magic != AHCI_DRIVE_MAGIC)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) {
IoReleaseRemoveLock(&drv->removeLock, irp);
return NTLSTATUS_INVALID_PARAMETER;
}
irp->Status = NTLSTATUS_FAILURE;
irp->Information = 0;
IrpQueueEnqueue(&drv->irpQueue, irp);
return NTLSTATUS_PENDING;
}
// Dispatches pnp.
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DeviceExtension)
@@ -552,7 +755,7 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
// ever having blocked in a driver call.
// NCQ remains serialized until the driver grows a multi-slot dispatcher:
// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP.
// 1. Wait for a free slot and a pending IRP through the queue semaphore.
// 2. Pick the oldest IRP from the queue.
// 3. Grab a free bit in drv->slotMask (under the slot spinlock).
// 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA),
@@ -561,30 +764,45 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
static void AhciWorker(uintptr_t arg) {
AHCI_DRIVE *drv = (AHCI_DRIVE *)arg;
while (true) {
while (drv && drv->workerRunning) {
// Block until AHCI dispatch signals a new IRP.
KeWaitSemaphore(drv->irpQueue.Semaphore);
KeWaitSemaphore(drv->irpQueue.Semaphore, KE_WAIT_INFINITE);
if (!drv->workerRunning)
break;
IRP *irp = IrpQueueDequeue(&drv->irpQueue);
if (!irp)
continue;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read (drv, lba, sectors, buffer);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
KSTATUS st = NTLSTATUS_INVALID_PARAMETER;
size_t information = 0;
if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) {
st = do_flush(drv);
} else {
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read(drv, lba, sectors, buffer);
if (KSUCCESS(st))
information = (size_t)sectors * 512u;
}
}
// Publish result then fire completion hooks through the common I/O path.
IoReleaseRemoveLock(&drv->removeLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
IoCompleteRequest(irp, st, information);
}
if (drv && drv->workerFinished)
KeSignalEvent(drv->workerFinished);
}
// AHCI controller initialisation.
@@ -597,18 +815,8 @@ static void AhciWorker(uintptr_t arg) {
// FRE must already be set (port_init_memory does this) so the HBA can
// capture the signature FIS the device sends after the reset.
// PORT_IE bits we care about:
// DHRS (bit 0) - D2H Register FIS received - normal completion
// PSS (bit 1) - PIO Setup FIS received
// DSS (bit 2) - DMA Setup FIS received
// SDBS (bit 3) - Set Device Bits FIS received
// TFES (bit 30) - Task File Error Status - error
// HBFS (bit 29) - Host Bus Fatal error
// HBDS (bit 28) - Host Bus Data error
// IFS (bit 27) - Interface Fatal error.
#define PORT_IE_NORMAL (0x0000000Fu)
#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS)
// Runtime commands are non-NCQ DMA commands. DHRS is their completion signal;
// setup/progress FIS interrupts must not wake the serialized command waiter.
// Registers controller.
static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
@@ -623,6 +831,20 @@ static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the interrupt registry.
static void AhciUnregisterController(AHCI_CONTROLLER* ctl) {
if (!ctl)
return;
for (int i = 0; i < g_controllerCount; i++) {
if (g_controllers[i] != ctl)
continue;
g_controllerCount--;
g_controllers[i] = g_controllers[g_controllerCount];
g_controllers[g_controllerCount] = (AHCI_CONTROLLER*)0;
return;
}
}
// Starts controller.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC || !ctl->pdo || !ctl->fdo)
@@ -676,7 +898,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = PciMapBar(ctl->pdo,
AHCI_PCI_BAR_ABAR,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&ctl->hba,
&ctl->abarLength);
if (!KSUCCESS(st)) {
@@ -712,9 +934,23 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n",
cap, nPorts, nSlots, (int)s64a);
IO_DMA_REQUIREMENTS dmaRequirements;
memset(&dmaRequirements, 0, sizeof(dmaRequirements));
dmaRequirements.MaximumDeviceAddress = s64a ? (uint64_t)-1 : 0xFFFFFFFFULL;
dmaRequirements.Alignment = 2;
dmaRequirements.MaximumSegmentLength = 1u << 22;
dmaRequirements.MaximumSegments = PORT_PRDT_COUNT;
st = gAhciDdk->Dma->GetAdapter(ctl->fdo, &dmaRequirements, &ctl->dmaAdapter);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: DMA adapter creation failed (0x%x)\n", st);
return st;
}
st = AhciRegisterController(ctl);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: controller registry full (0x%x)\n", st);
gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter);
ctl->dmaAdapter = (IO_DMA_ADAPTER*)0;
return st;
}
@@ -751,6 +987,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = port_comreset(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: COMRESET failed\n", portno);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -758,6 +996,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
if (!port_is_sata_disk(port)) {
DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n",
portno, port->sig);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -765,6 +1005,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
KeInitializeDpc(&drv->completionDpc, AhciCompletionDpc, drv);
@@ -775,19 +1017,26 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
IrpQueueInit(&drv->irpQueue, irpSem);
st = IoInitializeRemoveLock(&drv->removeLock);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
drv->removeLockReady = true;
// Phase 6: IDENTIFY (polled - scheduler not yet running).
st = port_identify(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -799,12 +1048,14 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
// Phase 7: enable per-port interrupts.
// Clear any stale IS first so we don't get a spurious signal immediately.
port->is = (uint32_t)-1;
port->ie = PORT_IE_ALL;
port->ie = AHCI_PORT_TERMINAL_MASK;
// Phase 8: create and register a disk DEVICE_OBJECT.
st = IoCreateDevice(&g_driver, 0, &drv->deviceObject);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", g_nextDiskIndex, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
drv->deviceObject->DeviceExtension = drv;
@@ -822,19 +1073,29 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st);
IoDeleteDevice(drv->deviceObject);
drv->deviceObject = (DEVICE_OBJECT*)0;
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
// Phase 9: spawn the per-drive IRP worker thread.
// The worker owns do_read / do_write exclusively once the scheduler
// is live; it is the only thread that may call port_issue_cmd().
st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv,
PriorityHigh, &drv->workerThread);
st = KeEventCreate(EventManualReset, 0, &drv->workerFinished);
if (KSUCCESS(st)) {
drv->workerRunning = true;
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh);
st = KeCreateKernelThread((uintptr_t)AhciWorker,
(uintptr_t)drv,
&workerParameters,
&drv->workerThread);
}
if (!KSUCCESS(st)) {
DbgPrint("AHCI: drive %u: worker thread create failed (0x%x)\n",
diskIndex, st);
// Fatal for this device: all disk access is IRP-based now, so
// leave the device registered but clearly fail during testing.
drv->workerRunning = false;
DbgPrint("AHCI: drive %u: worker creation failed (0x%x)\n", diskIndex, st);
(void)IoUnregisterDiskDevice(diskIndex);
ctl->driveCount--;
continue;
}
}
@@ -889,7 +1150,7 @@ static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevic
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt)
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Dma)
return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("AHCI: driver loading...\n");
@@ -903,6 +1164,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
g_driver.DriverAddDevice = AhciAddDevice;
g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AhciDispatchFlush;
g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
KSTATUS st = IoRegisterDriverObject(&g_driver);
code/drivers/storage/ahci/ahci.h
+11 -4
View File
@@ -250,6 +250,7 @@ typedef volatile struct {
#define PORT_CMD_ICC_ACTIVE (0x1u << 28)
// Port IS register bits (interrupt status).
#define PORT_IS_DHRS (1u << 0) // D2H Register FIS Received.
#define PORT_IS_TFES (1u << 30) // Task File Error Status
#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error
#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error
@@ -321,11 +322,13 @@ typedef struct {
uint8_t portNo; // Physical port index (for ISR dispatch)
// Structures block (command list + FIS + cmd tables).
uint64_t structPhys; // Physical base
void* structVirt; // Virtual base
IO_DMA_BUFFER* structDma;
uint64_t structDeviceAddress; // Device base.
void* structVirt; // CPU base.
// Bounce buffer for DMA data.
uint64_t bouncePhys;
IO_DMA_BUFFER* bounceDma;
uint64_t bounceDeviceAddress;
void* bounceVirt;
// Signaled by the completion DPC when the port raises an interrupt.
@@ -345,13 +348,16 @@ typedef struct {
IO_REMOVE_LOCK removeLock;
bool surpriseRemoved;
bool removed;
bool removeLockReady;
volatile bool workerRunning;
THREAD* workerThread;
EVENT* workerFinished;
DEVICE_OBJECT* deviceObject;
// Multi-slot NCQ needs per-slot ownership state in addition to the serialized queue:
// uint32_t slotMask; // bitmask of in-flight slots
// IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP
// uint64_t slotBouncePhys[NUM_CMD_SLOTS];
// uint64_t slotBounceDeviceAddress[NUM_CMD_SLOTS];
// void* slotBounceVirt[NUM_CMD_SLOTS];
// volatile uint32_t prevSact; // previous port->sact snapshot.
} AHCI_DRIVE;
@@ -361,6 +367,7 @@ struct AHCI_CONTROLLER {
DEVICE_OBJECT* fdo;
DEVICE_OBJECT* pdo;
DEVICE_OBJECT* lowerDevice;
IO_DMA_ADAPTER* dmaAdapter;
PCI_DEVICE_INFO pciInfo;
PCI_BAR abar;
code/drivers/storage/ahci/ahci_port.c
+6 -9
View File
@@ -11,16 +11,13 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
int pageIndex = 0;
for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) {
uintptr_t virt = (uintptr_t)gAhciDdk->Memory->PhysToVirt(phys);
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent);
// Ignore NTLSTATUS_ALREADY_EXISTS - a prior mapping with different flags
// is possible for the first page if the bootloader was overly generous.
// Use MmPteProtectRange to force NOCACHE in that case.
if (st == NTLSTATUS_ALREADY_EXISTS) {
st = MmPteProtectRange(virt, 4096, flags);
}
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC;
KSTATUS st = gAhciDdk->Memory->MapPage(virt,
phys,
flags,
NtlDdkMapReplaceExisting);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
DbgPrint("AHCI: MMIO map failed for page %d (phys=0x%016llX, st=0x%x)\n",
pageIndex, (unsigned long long)phys, st);
return st;
}
code/drivers/storage/ata/ata.c
+74 -20
View File
@@ -92,7 +92,7 @@ static KSTATUS AtaIdentify(ATA_DEVICE* dev) {
uint16_t ident[256];
memset(ident, 0, sizeof(ident));
KSTATUS st = KeWaitSemaphore(ch->Lock);
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
@@ -193,7 +193,7 @@ static KSTATUS AtaReadPio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, void* b
ATA_CHANNEL* ch = dev->Channel;
uint16_t* out = (uint16_t*)buffer;
KSTATUS st = KeWaitSemaphore(ch->Lock);
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
@@ -234,6 +234,22 @@ static KSTATUS AtaFlush(ATA_DEVICE* dev) {
return AtaWaitNotBusy(ch);
}
// Flushes one device while holding the shared channel lock.
static KSTATUS AtaFlushSerialized(ATA_DEVICE* dev) {
if (!dev || !dev->Channel)
return NTLSTATUS_INVALID_PARAMETER;
ATA_CHANNEL* ch = dev->Channel;
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
AtaSelectDrive(dev);
st = AtaFlush(dev);
KeSignalSemaphore(ch->Lock);
return st;
}
// Updates pio.
static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const void* buffer) {
if (!dev || !buffer)
@@ -246,7 +262,7 @@ static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const
ATA_CHANNEL* ch = dev->Channel;
const uint16_t* in = (const uint16_t*)buffer;
KSTATUS st = KeWaitSemaphore(ch->Lock);
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
@@ -308,27 +324,38 @@ static void AtaWorker(uintptr_t arg) {
return;
while (true) {
KeWaitSemaphore(dev->IrpQueue.Semaphore);
KeWaitSemaphore(dev->IrpQueue.Semaphore, KE_WAIT_INFINITE);
IRP* irp = IrpQueueDequeue(&dev->IrpQueue);
if (!irp)
continue;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = AtaGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (lba + sectors > dev->SectorCount)
st = NTLSTATUS_INVALID_PARAMETER;
else
st = write ? AtaWritePio(dev, lba, sectors, buffer)
: AtaReadPio(dev, lba, sectors, buffer);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
KSTATUS st = NTLSTATUS_INVALID_PARAMETER;
size_t information = 0;
if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) {
st = AtaFlushSerialized(dev);
} else {
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = AtaGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (lba + sectors > dev->SectorCount) {
st = NTLSTATUS_INVALID_PARAMETER;
} else {
st = write ? AtaWritePio(dev, lba, sectors, buffer)
: AtaReadPio(dev, lba, sectors, buffer);
if (KSUCCESS(st))
information = (size_t)sectors * ATA_SECTOR_SIZE;
}
}
}
IoReleaseRemoveLock(&dev->RemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * ATA_SECTOR_SIZE) : 0);
IoCompleteRequest(irp, st, information);
}
}
@@ -363,6 +390,31 @@ KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
return NTLSTATUS_PENDING;
}
// Queues one cache flush on the device worker.
static KSTATUS AtaDispatchFlush(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return NTLSTATUS_INVALID_PARAMETER;
ATA_DEVICE* dev = (ATA_DEVICE*)device->DeviceExtension;
if (dev->Magic != ATA_DEVICE_MAGIC || !dev->Present)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&dev->RemoveLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) {
IoReleaseRemoveLock(&dev->RemoveLock, irp);
return NTLSTATUS_INVALID_PARAMETER;
}
irp->Status = NTLSTATUS_FAILURE;
irp->Information = 0;
IrpQueueEnqueue(&dev->IrpQueue, irp);
return NTLSTATUS_PENDING;
}
// Returns free Disk Index.
static uint8_t AtaFindFreeDiskIndex(void) {
for (uint8_t i = 0; i < 8; i++) {
@@ -486,8 +538,9 @@ static KSTATUS AtaProbeDrive(ATA_CONTROLLER* ctl, ATA_CHANNEL* ch, uint8_t drive
return st;
}
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh);
st = KeCreateKernelThread((uintptr_t)AtaWorker, (uintptr_t)dev,
PriorityHigh, &dev->WorkerThread);
&workerParameters, &dev->WorkerThread);
if (!KSUCCESS(st))
DbgPrint("ATA: drive %u worker thread create failed (0x%x)\n", dev->DiskIndex, st);
return NTLSTATUS_SUCCESS;
@@ -580,7 +633,7 @@ static void AtaRemoveControllerDevices(ATA_CONTROLLER* ctl, IO_DEVICE_CHANGE cha
if (!dev->Present)
continue;
IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
if (dev->DiskIndex != 0xFFu) {
if (change == IoDeviceSurpriseRemoved)
IoNotifyDiskSurpriseRemoved(dev->DiskIndex);
@@ -647,12 +700,12 @@ KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
dev->Present = false;
IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
dev->Present = false;
IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IoReleaseRemoveLockAndWait(&dev->RemoveLock, irp);
IoDeleteDevice(device);
dev->DeviceObject = (DEVICE_OBJECT*)0;
@@ -708,6 +761,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
gAtaDriver.DriverAddDevice = AtaAddDevice;
gAtaDriver.MajorFunction[IRP_MJ_READ] = AtaDispatchReadWrite;
gAtaDriver.MajorFunction[IRP_MJ_WRITE] = AtaDispatchReadWrite;
gAtaDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AtaDispatchFlush;
gAtaDriver.MajorFunction[IRP_MJ_PNP] = AtaDispatchPnp;
KSTATUS st = IoRegisterDriverObject(&gAtaDriver);