diff --git a/OVMF/OVMF_VARS_4M.fd b/OVMF/OVMF_VARS_4M.fd index 71d485c..7ab8cfc 100644 Binary files a/OVMF/OVMF_VARS_4M.fd and b/OVMF/OVMF_VARS_4M.fd differ diff --git a/OVMF/OVMF_VARS_4M_d.fd b/OVMF/OVMF_VARS_4M_d.fd index 57dd62c..8b73f92 100644 Binary files a/OVMF/OVMF_VARS_4M_d.fd and b/OVMF/OVMF_VARS_4M_d.fd differ diff --git a/code/boot.ini b/code/boot.ini index 8c0cd3c..1d0fb60 100644 --- a/code/boot.ini +++ b/code/boot.ini @@ -1,5 +1,5 @@ -[kernel] -; bitmask: 1 = debug prints, 2 = self check -debug = 1 -[init] -commandline = apps/shell_native.exe +[kernel] +; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file +debug = 4 +[init] +commandline = apps/shell_native.exe diff --git a/code/boot/uefi/main.c b/code/boot/uefi/main.c index 5ed1eb6..9389ccf 100644 --- a/code/boot/uefi/main.c +++ b/code/boot/uefi/main.c @@ -264,12 +264,35 @@ typedef struct { char commandLine[64]; } PARSE_STATE; + +static int ParseBootIniInteger(const char* val) { + int out = 0; + if (!val) return 0; + while (*val == ' ' || *val == '\t') val++; + int base = 10; + if (val[0] == '0' && (val[1] == 'x' || val[1] == 'X')) { + base = 16; + val += 2; + } + while (*val) { + char c = *val++; + int digit; + if (c >= '0' && c <= '9') digit = c - '0'; + else if (c >= 'a' && c <= 'f') digit = 10 + c - 'a'; + else if (c >= 'A' && c <= 'F') digit = 10 + c - 'A'; + else break; + if (digit >= base) break; + out = out * base + digit; + } + return out; +} + static void BootIniCb(const char* section, const char* key, const char* val, void* ud) { PARSE_STATE* s = (PARSE_STATE *)ud; - // [kernel] debug = 1 + // [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug")) - s->debug = (int)(val[0] - '0'); + s->debug = ParseBootIniInteger(val); // [init] commandline = hello.exe else if (!AsciiStrCmp(section, "init") && !AsciiStrCmp(key, "commandline")) AsciiStrCpyS(s->commandLine, 64, val); diff --git a/code/build/build.c b/code/build/build.c index edfc4fe..20aa019 100644 --- a/code/build/build.c +++ b/code/build/build.c @@ -28,7 +28,7 @@ static const Driver_Target STANDALONE_DRIVERS[] = { DRIVER_TARGET("ide", "obj\\drivers\\ide", "code\\drivers\\storage\\ide\\ide.c code\\drivers\\storage\\ide\\ide_bus.c"), DRIVER_TARGET("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c"), DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"), - DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c"), + DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_bus.c code\\drivers\\bus\\usb\\usb_composite.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c code\\drivers\\bus\\usb\\usb_hub.c"), DRIVER_TARGET("rtl816x", "obj\\drivers\\rtl816x", "code\\drivers\\net\\rtl816x\\rtl816x.c code\\drivers\\net\\rtl816x\\rtl816x_pci.c"), DRIVER_TARGET("rtl8139", "obj\\drivers\\rtl8139", "code\\drivers\\net\\rtl8139\\rtl8139.c code\\drivers\\net\\rtl8139\\rtl8139_pci.c"), }; diff --git a/code/drivers/bus/usb/usb.c b/code/drivers/bus/usb/usb.c index a84e49d..433da83 100644 --- a/code/drivers/bus/usb/usb.c +++ b/code/drivers/bus/usb/usb.c @@ -75,9 +75,33 @@ KSTATUS DriverEntry(const HAL_API* hal) { if (!hal) return STATUS_INVALID_PARAMETER; DbgPrint("usb: discovery starting\n"); - KSTATUS st = UsbRegisterClassMatchers(); + KSTATUS st = UsbInitializeBusModel(); if (!KSUCCESS(st)) { - DbgPrint("usb: class matcher registration failed st=0x%x\n", st); + DbgPrint("usb: bus model initialisation failed st=0x%x\n", st); + return st; + } + + st = UsbInitializeCompositeDriver(); + if (!KSUCCESS(st)) { + DbgPrint("usb: composite driver initialisation failed st=0x%x\n", st); + return st; + } + + st = UsbInitializeHidDriver(); + if (!KSUCCESS(st)) { + DbgPrint("usb: HID driver initialisation failed st=0x%x\n", st); + return st; + } + + st = UsbInitializeStorageDriver(); + if (!KSUCCESS(st)) { + DbgPrint("usb: storage driver initialisation failed st=0x%x\n", st); + return st; + } + + st = UsbInitializeHubDriver(); + if (!KSUCCESS(st)) { + DbgPrint("usb: hub driver initialisation failed st=0x%x\n", st); return st; } diff --git a/code/drivers/bus/usb/usb.h b/code/drivers/bus/usb/usb.h index d0bba62..ec20b35 100644 --- a/code/drivers/bus/usb/usb.h +++ b/code/drivers/bus/usb/usb.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include @@ -114,6 +116,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define XHCI_TRB_TYPE_DATA 3u #define XHCI_TRB_TYPE_STATUS 4u #define XHCI_TRB_TYPE_ENABLE_SLOT 9u +#define XHCI_TRB_TYPE_DISABLE_SLOT 10u #define XHCI_TRB_TYPE_ADDRESS_DEVICE 11u #define XHCI_TRB_TYPE_CONFIGURE_ENDPOINT 12u @@ -144,10 +147,15 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define USB_REQ_SET_CONFIGURATION 0x09u #define USB_REQ_SET_IDLE 0x0Au #define USB_REQ_SET_PROTOCOL 0x0Bu +#define USB_REQ_GET_STATUS 0x00u +#define USB_REQ_CLEAR_FEATURE 0x01u +#define USB_REQ_SET_FEATURE 0x03u #define USB_DESC_DEVICE 0x01u #define USB_DESC_CONFIGURATION 0x02u #define USB_DESC_INTERFACE 0x04u #define USB_DESC_ENDPOINT 0x05u +#define USB_DESC_HUB 0x29u +#define USB_DESC_SUPERSPEED_HUB 0x2Au #define USB_DESC_HID 0x21u #define USB_DESC_CS_INTERFACE 0x24u #define USB_DIR_OUT 0x00u @@ -155,6 +163,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define USB_TYPE_STANDARD 0x00u #define USB_TYPE_CLASS 0x20u #define USB_RECIP_DEVICE 0x00u +#define USB_RECIP_OTHER 0x03u #define USB_RECIP_INTERFACE 0x01u #define USB_ENDPOINT_DIR_IN 0x80u #define USB_ENDPOINT_NUMBER_MASK 0x0Fu @@ -165,6 +174,23 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define USB_ENDPOINT_TYPE_INTERRUPT 0x03u #define USB_CLASS_HID 0x03u #define USB_CLASS_MASS_STORAGE 0x08u +#define USB_CLASS_HUB 0x09u +#define USB_HUB_PORT_FEAT_CONNECTION 0u +#define USB_HUB_PORT_FEAT_ENABLE 1u +#define USB_HUB_PORT_FEAT_SUSPEND 2u +#define USB_HUB_PORT_FEAT_OVER_CURRENT 3u +#define USB_HUB_PORT_FEAT_RESET 4u +#define USB_HUB_PORT_FEAT_POWER 8u +#define USB_HUB_PORT_FEAT_C_CONNECTION 16u +#define USB_HUB_PORT_FEAT_C_ENABLE 17u +#define USB_HUB_PORT_FEAT_C_RESET 20u +#define USB_HUB_PORT_STATUS_CONNECTION (1u << 0) +#define USB_HUB_PORT_STATUS_ENABLE (1u << 1) +#define USB_HUB_PORT_STATUS_LOW_SPEED (1u << 9) +#define USB_HUB_PORT_STATUS_HIGH_SPEED (1u << 10) +#define USB_HUB_PORT_CHANGE_CONNECTION (1u << 0) +#define USB_HUB_PORT_CHANGE_ENABLE (1u << 1) +#define USB_HUB_PORT_CHANGE_RESET (1u << 4) #define USB_MASS_SUBCLASS_SCSI 0x06u #define USB_MASS_PROTOCOL_BOT 0x50u #define USB_HID_SUBCLASS_BOOT 0x01u @@ -193,12 +219,22 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define XHCI_EP0_RING_TRBS 256u #define XHCI_HID_RING_TRBS 256u #define XHCI_BULK_RING_TRBS 256u +#define XHCI_PIPE_RING_TRBS 256u #define XHCI_MAX_HID_REPORT 16u #define USB_STORAGE_DMA_PAGES 1u #define USB_STORAGE_MAX_BYTES (USB_STORAGE_DMA_PAGES * PAGE_SIZE) #define USB_STORAGE_SECTOR_SIZE 512u #define USB_MAX_TRACKED_INTERFACES 8u #define USB_MAX_TRACKED_ENDPOINTS 8u +#define USB_MAX_DEVICE_ID_TEXT 96u +#define USB_MAX_PIPES 32u +#define USB_IOCTL_SUBMIT_URB 0x55534201u /* USB1 */ +#define USB_ROOT_HUB_MAX_PORTS XHCI_MAX_ROOT_PORTS +#define USB_EXTERNAL_HUB_MAX_PORTS 31u +#define USB_ROOT_PORT_ENUM_RETRIES 3u +#define USB_XHCI_CONTROL_TIMEOUT_US 1000000u +#define USB_XHCI_ENUM_CONTROL_TIMEOUT_US 120000u +#define USB_HUB_CHANGE_BITMAP_BYTES 8u #pragma pack(push, 1) typedef struct XHCI_TRB { @@ -271,6 +307,21 @@ typedef struct USB_ENDPOINT_DESCRIPTOR { uint8_t bInterval; } USB_ENDPOINT_DESCRIPTOR; +typedef struct USB_HUB_DESCRIPTOR { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bNbrPorts; + uint16_t wHubCharacteristics; + uint8_t bPwrOn2PwrGood; + uint8_t bHubContrCurrent; + uint8_t DeviceRemovable[1]; +} USB_HUB_DESCRIPTOR; + +typedef struct USB_HUB_PORT_STATUS { + uint16_t Status; + uint16_t Change; +} USB_HUB_PORT_STATUS; + typedef struct USB_MASS_CBW { uint32_t Signature; uint32_t Tag; @@ -308,6 +359,309 @@ typedef struct USB_BAR { uint8_t Index; } USB_BAR; + +typedef enum USB_DEVICE_STATE { + UsbDeviceAttached = 0, + UsbDevicePowered, + UsbDeviceDefault, + UsbDeviceAddressed, + UsbDeviceConfigured, + UsbDeviceSuspended, + UsbDeviceSurpriseRemoved, + UsbDeviceRemoved, +} USB_DEVICE_STATE; + +typedef struct USB_PORT_PATH { + uint8_t RootPort; + uint8_t Depth; + uint32_t RouteString; + uint8_t ParentHubSlot; + uint8_t ParentHubPort; + bool BehindHub; +} USB_PORT_PATH; + +typedef enum USB_PIPE_TYPE { + UsbPipeControl = 0, + UsbPipeBulk, + UsbPipeInterrupt, + UsbPipeIsochronous, +} USB_PIPE_TYPE; + +typedef enum USB_TRANSFER_DIRECTION { + UsbTransferNoData = 0, + UsbTransferOut, + UsbTransferIn, +} USB_TRANSFER_DIRECTION; + +typedef enum USB_URB_FUNCTION { + UrbFunctionControlTransfer = 0, + UrbFunctionBulkTransfer, + UrbFunctionInterruptTransfer, + UrbFunctionSelectConfiguration, + UrbFunctionSelectInterface, + UrbFunctionResetPipe, + UrbFunctionAbortPipe, +} USB_URB_FUNCTION; + +typedef struct USB_HCD USB_HCD; +typedef struct USB_DEVICE USB_DEVICE; +typedef struct USB_PIPE USB_PIPE; +typedef struct USB_URB USB_URB; +typedef struct XHCI_DEVICE XHCI_DEVICE; +typedef void (*USB_URB_COMPLETION)(USB_URB* urb, void* context); + +typedef struct USB_HCD_OPERATIONS { + KSTATUS (*SubmitUrb)(USB_HCD* hcd, USB_URB* urb); + KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb); + KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed); + KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port); + void* (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys); +} USB_HCD_OPERATIONS; + +struct USB_HCD { + const USB_HCD_OPERATIONS* Operations; + void* ControllerContext; + DRIVER_OBJECT* DriverObject; + DEVICE_OBJECT* ControllerPdo; + DEVICE_OBJECT* RootHubPdo; + void* RootHubContext; + uint8_t ControllerIndex; + uint8_t MaxRootPorts; +}; + +struct USB_DEVICE { + OBJECT Header; + USB_HCD* Hcd; + DEVICE_OBJECT* Pdo; + void* HcdDeviceContext; + USB_DEVICE_STATE State; + USB_PORT_PATH Path; + + uint8_t Address; + uint8_t SlotId; + uint8_t Speed; + uint16_t MaxPacketSize0; + + uint16_t VendorId; + uint16_t ProductId; + uint16_t BcdUsb; + uint16_t BcdDevice; + uint8_t DeviceClass; + uint8_t DeviceSubclass; + uint8_t DeviceProtocol; + uint8_t ConfigurationValue; + + IO_REMOVE_LOCK RemoveLock; + bool RemoveLockReady; +}; + +struct USB_PIPE { + OBJECT Header; + USB_DEVICE* Device; + USB_PIPE_TYPE Type; + uint8_t EndpointAddress; + uint8_t EndpointId; + uint16_t MaxPacketSize; + uint8_t Interval; + bool Open; + bool Halted; + void* HcdPipeContext; +}; + +struct USB_URB { + LIST_ENTRY Link; + USB_URB_FUNCTION Function; + USB_TRANSFER_DIRECTION Direction; + USB_DEVICE* Device; + USB_PIPE* Pipe; + USB_SETUP_PACKET Setup; + void* Buffer; + PHYSICAL_ADDRESS BufferPhys; + uint32_t Length; + uint32_t ActualLength; + KSTATUS Status; + IRP* Irp; + USB_URB_COMPLETION Complete; + void* CompleteContext; + bool Pending; + bool Cancelled; +}; + +typedef enum USB_PDO_KIND { + UsbPdoController = 1, + UsbPdoDevice = 2, + UsbPdoInterface = 3, + UsbPdoRootHub = 4, +} USB_PDO_KIND; + +typedef struct USB_PDO_EXTENSION { + USB_PDO_KIND Kind; + USB_HCD* Hcd; + USB_DEVICE* Device; + uint8_t InterfaceIndex; + USB_INTERFACE_DESCRIPTOR InterfaceDescriptor; + uint8_t EndpointCount; + USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS]; +} USB_PDO_EXTENSION; + + + +typedef enum USB_HUB_KIND { + UsbHubKindRoot = 1, + UsbHubKindExternal = 2, +} USB_HUB_KIND; + +typedef enum USB_HUB_PORT_STATE { + UsbHubPortDisconnected = 0, + UsbHubPortPowered, + UsbHubPortConnectedDebounce, + UsbHubPortResetting, + UsbHubPortEnabled, + UsbHubPortEnumerating, + UsbHubPortChildPresent, + UsbHubPortFailedConnected, + UsbHubPortRemoving, +} USB_HUB_PORT_STATE; + +typedef struct USB_HUB_EXTENSION { + DEVICE_OBJECT* Self; + DEVICE_OBJECT* LowerDevice; + DEVICE_OBJECT* HubPdo; + USB_HUB_KIND Kind; + USB_HCD* Hcd; + USB_DEVICE* Device; + IO_REMOVE_LOCK RemoveLock; + bool RemoveLockReady; + bool Started; + bool SurpriseRemoved; + bool Removed; + uint8_t PortCount; + uint8_t RootPortSlot[USB_ROOT_HUB_MAX_PORTS + 1u]; + uint8_t RootPortFailCount[USB_ROOT_HUB_MAX_PORTS + 1u]; + USB_HUB_PORT_STATE RootPortState[USB_ROOT_HUB_MAX_PORTS + 1u]; + uint8_t ChildSlot[USB_EXTERNAL_HUB_MAX_PORTS + 1u]; + USB_HUB_PORT_STATE ChildPortState[USB_EXTERNAL_HUB_MAX_PORTS + 1u]; + uint32_t PendingRootPorts[8]; + uint32_t PendingHubPorts[1]; + SPINLOCK PendingLock; + SEMAPHORE* WorkerSemaphore; + THREAD* WorkerThread; + bool WorkerRunning; + bool InitialScanPending; + bool InitialScanComplete; + bool InitialScanInProgress; + + uint8_t InterfaceNumber; + uint8_t InterruptEndpointAddress; + uint16_t InterruptMaxPacket; + uint8_t InterruptInterval; + USB_PIPE* InterruptPipe; + uint8_t* ChangeBitmap; + PHYSICAL_ADDRESS ChangeBitmapPhys; + USB_URB ChangeUrb; + bool ChangeUrbPending; + uint8_t PowerOnToPowerGoodMs; +} USB_HUB_EXTENSION; + +typedef struct USB_COMPOSITE_EXTENSION { + DEVICE_OBJECT* Self; + DEVICE_OBJECT* LowerDevice; + DEVICE_OBJECT* DevicePdo; + USB_DEVICE* Device; + bool Started; + bool InterfacesCreated; +} USB_COMPOSITE_EXTENSION; + +typedef struct USB_HID_EXTENSION { + DEVICE_OBJECT* Self; + DEVICE_OBJECT* LowerDevice; + DEVICE_OBJECT* InterfacePdo; + USB_DEVICE* Device; + IO_REMOVE_LOCK RemoveLock; + bool RemoveLockReady; + bool Started; + bool SurpriseRemoved; + bool Removed; + bool Keyboard; + bool Mouse; + uint8_t InterfaceNumber; + uint8_t Protocol; + uint8_t EndpointAddress; + uint16_t MaxPacketSize; + uint8_t Interval; + + USB_PIPE* InterruptPipe; + uint8_t* ReportBuffer; + PHYSICAL_ADDRESS ReportBufferPhys; + uint16_t ReportLength; + USB_URB InterruptUrb; + bool InterruptPending; + + uint8_t LastModifiers; + uint8_t LastKeys[6]; + uint8_t LastButtons; + uint32_t InputDeviceId; +} USB_HID_EXTENSION; + + + +typedef enum USB_STORAGE_EXTENSION_KIND { + UsbStorageExtensionInterface = 0x55534946u, + UsbStorageExtensionDisk = 0x5553444Bu, +} USB_STORAGE_EXTENSION_KIND; + +typedef struct USB_STORAGE_EXTENSION USB_STORAGE_EXTENSION; +typedef struct USB_STORAGE_DISK_EXTENSION USB_STORAGE_DISK_EXTENSION; + +struct USB_STORAGE_EXTENSION { + USB_STORAGE_EXTENSION_KIND Kind; + DEVICE_OBJECT* Self; + DEVICE_OBJECT* LowerDevice; + DEVICE_OBJECT* InterfacePdo; + DEVICE_OBJECT* DiskDevice; + + USB_DEVICE* Device; + XHCI_DEVICE* XhciDevice; + + USB_PIPE* BulkInPipe; + USB_PIPE* BulkOutPipe; + + IO_REMOVE_LOCK RemoveLock; + bool RemoveLockReady; + bool Started; + bool SurpriseRemoved; + bool Removed; + + uint8_t InterfaceNumber; + uint8_t BulkInAddress; + uint8_t BulkOutAddress; + uint16_t BulkInMaxPacket; + uint16_t BulkOutMaxPacket; + + uint32_t Tag; + uint32_t BlockSize; + uint64_t BlockCount; + volatile long IoBusy; +}; + +struct USB_STORAGE_DISK_EXTENSION { + USB_STORAGE_EXTENSION_KIND Kind; + USB_STORAGE_EXTENSION* Storage; +}; + +typedef struct XHCI_PIPE_CONTEXT { + XHCI_TRB* Ring; + PHYSICAL_ADDRESS RingPhys; + uint32_t Enqueue; + uint8_t Cycle; + uint8_t EndpointId; + USB_PIPE_TYPE Type; + bool Configured; + bool ActiveTransfer; + uint64_t ActiveTrbPhys; + USB_URB* ActiveUrb; +} XHCI_PIPE_CONTEXT; + typedef struct XHCI_DEVICE { bool Allocated; bool Addressed; @@ -324,6 +678,11 @@ typedef struct XHCI_DEVICE { uint8_t DeviceProtocol; uint8_t ConfigurationValue; bool Configured; + USB_DEVICE* UsbDevice; + DEVICE_OBJECT* UsbPdo; + DEVICE_OBJECT* ParentDeviceObject; + USB_PORT_PATH Path; + USB_PIPE* EndpointPipes[32]; uint8_t InterfaceCount; USB_TRACKED_INTERFACE Interfaces[USB_MAX_TRACKED_INTERFACES]; @@ -364,6 +723,10 @@ typedef struct XHCI_DEVICE { uint32_t MassBlockSize; uint64_t MassBlockCount; DEVICE_OBJECT* MassDeviceObject; + IO_REMOVE_LOCK MassRemoveLock; + bool MassRemoveLockReady; + bool MassSurpriseRemoved; + bool MassRemoved; volatile long MassIoBusy; PHYSICAL_ADDRESS InputContextPhys; @@ -403,12 +766,14 @@ typedef struct XHCI_TRANSFER_RESULT { } XHCI_TRANSFER_RESULT; typedef struct XHCI_WAIT_TARGET { + struct XHCI_WAIT_TARGET* Next; uint64_t CommandPhys; uint64_t TransferPhys; uint8_t TransferSlotId; uint8_t TransferEndpointId; - bool CommandMatched; - bool TransferMatched; + volatile bool CommandMatched; + volatile bool TransferMatched; + volatile bool Registered; XHCI_COMMAND_RESULT Command; XHCI_TRANSFER_RESULT Transfer; } XHCI_WAIT_TARGET; @@ -470,15 +835,58 @@ typedef struct XHCI_CONTROLLER { volatile uint32_t InterruptCount; volatile uint32_t EventDpcCount; SPINLOCK EventLock; + SPINLOCK WaitLock; + SPINLOCK CommandLock; + XHCI_WAIT_TARGET* WaiterList; - volatile XHCI_WAIT_TARGET* ActiveWait; volatile bool Running; + + USB_HCD Hcd; } XHCI_CONTROLLER; extern XHCI_CONTROLLER gXhci; extern bool gHaveXhci; extern DRIVER_OBJECT gUsbStorageDriver; +extern DRIVER_OBJECT gUsbBusDriver; +extern DRIVER_OBJECT gUsbCompositeDriver; +extern DRIVER_OBJECT gUsbHidDriver; +extern DRIVER_OBJECT gUsbHubDriver; + +KSTATUS UsbInitializeBusModel(void); +KSTATUS UsbInitializeCompositeDriver(void); +KSTATUS UsbInitializeHidDriver(void); +KSTATUS UsbInitializeStorageDriver(void); +KSTATUS UsbInitializeHubDriver(void); +KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject); +KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp); +void UsbInitializeHcd(USB_HCD* hcd, + const USB_HCD_OPERATIONS* ops, + void* controllerContext, + uint8_t controllerIndex, + uint8_t maxRootPorts); +KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device); +KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd); +KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise); +KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise); +KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent); +KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent); +KSTATUS UsbCreatePipe(USB_DEVICE* dev, + USB_PIPE_TYPE type, + uint8_t endpointAddress, + uint16_t maxPacketSize, + uint8_t interval, + USB_PIPE** outPipe); +void UsbReferenceDevice(USB_DEVICE* dev); +void UsbDereferenceDevice(USB_DEVICE* dev); +void UsbReferencePipe(USB_PIPE* pipe); +void UsbDereferencePipe(USB_PIPE* pipe); +KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb); +KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb); +KSTATUS UsbCancelUrb(USB_URB* urb); +void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys); +KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp); +KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp); uint32_t UsbAlignUp32(uint32_t value, uint32_t align); void XhciBarrier(void); @@ -512,11 +920,15 @@ const char* UsbEndpointTypeName(uint8_t type); const char* UsbDirectionName(uint8_t endpointAddress); const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto); void XhciLogPortStatus(XHCI_CONTROLLER* xhci); +void XhciPowerRootPorts(XHCI_CONTROLLER* xhci); void XhciAckInterrupter0(XHCI_CONTROLLER* xhci); void XhciUpdateErdp(XHCI_CONTROLLER* xhci); bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target); bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target); bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target); +void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target); +void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target); +KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs); KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs); void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2); void XhciIsr(void); @@ -535,12 +947,15 @@ void XhciEnableInterrupts(XHCI_CONTROLLER* xhci); uint32_t XhciPortReg(uint8_t port); void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port); KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed); +KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId); KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index); KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys); +KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs); KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length); KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup); +KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs); KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc); KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength); uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress); @@ -561,33 +976,42 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub, uint8_t endpointType, bool wantIn); bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context); KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context); +KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject); +KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp); void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle); +KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe); KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn); +KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn); uint32_t UsbBe32(const uint8_t* p); void UsbStoreBe32(uint8_t* p, uint32_t v); void UsbStoreBe16(uint8_t* p, uint16_t v); -KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); -KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); -KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); -KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); -KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, bool write, uint64_t lba, uint32_t sectors, void* buffer); -void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); -void UsbStorageReleaseIo(XHCI_DEVICE* dev); +KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage); +KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage); +KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage); +KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage); +KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer); +void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage); +void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage); KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp); +KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp); +KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject); KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); uint8_t UsbFindFreeDiskIndex(void); -KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev); +KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage); bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context); KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context); -KSTATUS UsbRegisterClassMatchers(void); KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationValue); void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_CONFIGURATION_DESCRIPTOR* config, uint16_t totalLength); -void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_DEVICE_DESCRIPTOR* devDesc); +KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId); +KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci, USB_HUB_EXTENSION* hub, uint8_t hubPort, uint8_t speed, uint8_t* outSlotId); KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port); void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci); +KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd); +void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port); +void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port); +extern const USB_HCD_OPERATIONS gXhciHcdOperations; KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device); diff --git a/code/drivers/bus/usb/usb_bus.c b/code/drivers/bus/usb/usb_bus.c new file mode 100644 index 0000000..c044e1b --- /dev/null +++ b/code/drivers/bus/usb/usb_bus.c @@ -0,0 +1,656 @@ +// usb_bus.c - USB bus/HCD/PDO/URB core model. +// +// This is the structural USB bus layer used by the upcoming composite, hub, +// and hotplug stages. The legacy direct xHCI class path is still present for +// boot HID/storage during migration, but newly enumerated xHCI devices now also +// get first-class USB PDOs and can accept internal URB IRPs. +#include "usb.h" + +DRIVER_OBJECT gUsbBusDriver; + +static bool gUsbBusModelReady; +static uint8_t gUsbNextControllerIndex; + +static void UsbDeviceDestroy(OBJECT* object) { + USB_DEVICE* dev = (USB_DEVICE*)object; + if (dev->RemoveLockReady) { + IoDestroyRemoveLock(&dev->RemoveLock); + dev->RemoveLockReady = false; + } + if (dev->Pdo) { + IoDereferenceDeviceObject(dev->Pdo); + dev->Pdo = (DEVICE_OBJECT*)0; + } +} + +static void UsbPipeDestroy(OBJECT* object) { + USB_PIPE* pipe = (USB_PIPE*)object; + if (pipe->Device) { + UsbDereferenceDevice(pipe->Device); + pipe->Device = (USB_DEVICE*)0; + } +} + +void UsbReferenceDevice(USB_DEVICE* dev) { + if (dev) ObReferenceObject(&dev->Header); +} + +void UsbDereferenceDevice(USB_DEVICE* dev) { + if (dev) ObDereferenceObject(&dev->Header); +} + +void UsbReferencePipe(USB_PIPE* pipe) { + if (pipe) ObReferenceObject(&pipe->Header); +} + +void UsbDereferencePipe(USB_PIPE* pipe) { + if (pipe) ObDereferenceObject(&pipe->Header); +} + +void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) { + if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma) + return (void*)0; + return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys); +} + +static USB_DEVICE* UsbAllocateDevice(void) { + USB_DEVICE* dev = (USB_DEVICE*)ObAllocateObject(ObjectTypeUsbDevice, sizeof(USB_DEVICE)); + if (!dev) return (USB_DEVICE*)0; + dev->Header.Destroy = UsbDeviceDestroy; + KSTATUS st = IoInitializeRemoveLock(&dev->RemoveLock); + if (!KSUCCESS(st)) { + ObDereferenceObject(&dev->Header); + return (USB_DEVICE*)0; + } + dev->RemoveLockReady = true; + dev->State = UsbDeviceAttached; + return dev; +} + +KSTATUS UsbCreatePipe(USB_DEVICE* dev, + USB_PIPE_TYPE type, + uint8_t endpointAddress, + uint16_t maxPacketSize, + uint8_t interval, + USB_PIPE** outPipe) { + if (!dev || !outPipe) return STATUS_INVALID_PARAMETER; + + USB_PIPE* pipe = (USB_PIPE*)ObAllocateObject(ObjectTypeUsbPipe, sizeof(USB_PIPE)); + if (!pipe) return STATUS_NO_MEMORY; + pipe->Header.Destroy = UsbPipeDestroy; + pipe->Device = dev; + UsbReferenceDevice(dev); + pipe->Type = type; + pipe->EndpointAddress = endpointAddress; + pipe->EndpointId = XhciEndpointIdFromAddress(endpointAddress); + pipe->MaxPacketSize = maxPacketSize; + pipe->Interval = interval; + pipe->Open = true; + *outPipe = pipe; + return STATUS_SUCCESS; +} + +static SYS_STRING_VIEW UsbIdFromBuffer(char* buf) { + return SysSvFromCstr(buf); +} + +static void UsbIdPutChar(char* buf, size_t cap, size_t* pos, char ch) { + if (!buf || !pos || cap == 0) + return; + if (*pos + 1 < cap) + buf[*pos] = ch; + (*pos)++; + if (*pos < cap) + buf[*pos] = '\0'; + else + buf[cap - 1] = '\0'; +} + +static void UsbIdAppendLiteral(char* buf, size_t cap, size_t* pos, const char* text) { + if (!text) + return; + while (*text) + UsbIdPutChar(buf, cap, pos, *text++); +} + +static void UsbIdAppendHex(char* buf, size_t cap, size_t* pos, uint32_t value, uint8_t digits) { + static const char kHex[] = "0123456789ABCDEF"; + while (digits) { + digits--; + UsbIdPutChar(buf, cap, pos, kHex[(value >> (digits * 4u)) & 0xFu]); + } +} + +static void UsbFormatVidPidRev(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint16_t rev) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_"); + UsbIdAppendHex(buf, cap, &pos, vid, 4); + UsbIdAppendLiteral(buf, cap, &pos, "&PID_"); + UsbIdAppendHex(buf, cap, &pos, pid, 4); + UsbIdAppendLiteral(buf, cap, &pos, "&REV_"); + UsbIdAppendHex(buf, cap, &pos, rev, 4); +} + +static void UsbFormatVidPid(char* buf, size_t cap, uint16_t vid, uint16_t pid) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_"); + UsbIdAppendHex(buf, cap, &pos, vid, 4); + UsbIdAppendLiteral(buf, cap, &pos, "&PID_"); + UsbIdAppendHex(buf, cap, &pos, pid, 4); +} + +static void UsbFormatVidPidMi(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint8_t mi) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_"); + UsbIdAppendHex(buf, cap, &pos, vid, 4); + UsbIdAppendLiteral(buf, cap, &pos, "&PID_"); + UsbIdAppendHex(buf, cap, &pos, pid, 4); + UsbIdAppendLiteral(buf, cap, &pos, "&MI_"); + UsbIdAppendHex(buf, cap, &pos, mi, 2); +} + +static void UsbFormatClassSubProt(char* buf, size_t cap, uint8_t cls, uint8_t sub, uint8_t proto) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_"); + UsbIdAppendHex(buf, cap, &pos, cls, 2); + UsbIdAppendLiteral(buf, cap, &pos, "&SUBCLASS_"); + UsbIdAppendHex(buf, cap, &pos, sub, 2); + UsbIdAppendLiteral(buf, cap, &pos, "&PROT_"); + UsbIdAppendHex(buf, cap, &pos, proto, 2); +} + +static void UsbFormatClass(char* buf, size_t cap, uint8_t cls) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_"); + UsbIdAppendHex(buf, cap, &pos, cls, 2); +} + + +static void UsbFormatComposite(char* buf, size_t cap) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "USB\\COMPOSITE"); +} + +static void UsbFormatPciXhci(char* buf, size_t cap, uint16_t vendor, uint16_t device) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "PCI\\VEN_"); + UsbIdAppendHex(buf, cap, &pos, vendor, 4); + UsbIdAppendLiteral(buf, cap, &pos, "&DEV_"); + UsbIdAppendHex(buf, cap, &pos, device, 4); + UsbIdAppendLiteral(buf, cap, &pos, "&CC_0C0330"); +} + +static void UsbFormatRootHub(char* buf, size_t cap) { + size_t pos = 0; + if (cap) buf[0] = '\0'; + UsbIdAppendLiteral(buf, cap, &pos, "USB\\ROOT_HUB"); +} + +static void UsbMakeDeviceIds(const XHCI_DEVICE* xdev, IO_DEVICE_ID* ids, char idbuf[][USB_MAX_DEVICE_ID_TEXT], size_t* outCount) { + size_t n = 0; + UsbFormatVidPidRev(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, xdev->BcdDevice); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + + UsbFormatVidPid(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + + UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass, xdev->DeviceSubclass, xdev->DeviceProtocol); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + + UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + + if (xdev->DeviceClass == 0 || xdev->InterfaceCount > 1) { + UsbFormatComposite(idbuf[n], USB_MAX_DEVICE_ID_TEXT); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + } + + *outCount = n; +} + +static void UsbMakeInterfaceIds(const XHCI_DEVICE* xdev, + const USB_TRACKED_INTERFACE* iface, + IO_DEVICE_ID* ids, + char idbuf[][USB_MAX_DEVICE_ID_TEXT], + size_t* outCount) { + size_t n = 0; + const USB_INTERFACE_DESCRIPTOR* d = &iface->Descriptor; + + UsbFormatVidPidMi(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, d->bInterfaceNumber); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + + UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass, d->bInterfaceSubClass, d->bInterfaceProtocol); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + + UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass); + ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) }; + n++; + + *outCount = n; +} + +KSTATUS UsbInitializeBusModel(void) { + if (gUsbBusModelReady) + return STATUS_SUCCESS; + + memset(&gUsbBusDriver, 0, sizeof(gUsbBusDriver)); + IoInitializeDriverObject(&gUsbBusDriver, SYS_SV_LITERAL("usb-bus")); + gUsbBusDriver.MajorFunction[IRP_MJ_PNP] = UsbDispatchPdoPnp; + gUsbBusDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl; + + KSTATUS st = IoRegisterDriverObject(&gUsbBusDriver); + if (!KSUCCESS(st)) return st; + + gUsbBusModelReady = true; + return STATUS_SUCCESS; +} + +void UsbInitializeHcd(USB_HCD* hcd, + const USB_HCD_OPERATIONS* ops, + void* controllerContext, + uint8_t controllerIndex, + uint8_t maxRootPorts) { + if (!hcd) return; + memset(hcd, 0, sizeof(*hcd)); + hcd->Operations = ops; + hcd->ControllerContext = controllerContext; + hcd->DriverObject = &gUsbBusDriver; + hcd->ControllerIndex = controllerIndex ? controllerIndex : ++gUsbNextControllerIndex; + hcd->MaxRootPorts = maxRootPorts; +} + +KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device) { + if (!hcd) return STATUS_INVALID_PARAMETER; + if (hcd->ControllerPdo) return STATUS_ALREADY_EXISTS; + + IO_DEVICE_ID ids[2]; + char idbuf[2][USB_MAX_DEVICE_ID_TEXT]; + UsbFormatPciXhci(idbuf[0], USB_MAX_DEVICE_ID_TEXT, vendor, device); + { + size_t pos = 0; + idbuf[1][0] = '\0'; + UsbIdAppendLiteral(idbuf[1], USB_MAX_DEVICE_ID_TEXT, &pos, "PCI\\CC_0C0330"); + } + ids[0] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[0]) }; + ids[1] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[1]) }; + + DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver, (DEVICE_OBJECT*)0, ids, 2, sizeof(USB_PDO_EXTENSION), &pdo); + if (!KSUCCESS(st)) return st; + (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("xhci-controller")); + + USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension; + ext->Kind = UsbPdoController; + ext->Hcd = hcd; + hcd->ControllerPdo = pdo; + + st = IoRegisterPhysicalDevice(pdo); + if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) + return st; + return STATUS_SUCCESS; +} + + +KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) { + if (!hcd) return STATUS_INVALID_PARAMETER; + if (hcd->RootHubPdo) return STATUS_ALREADY_EXISTS; + + IO_DEVICE_ID ids[2]; + char idbuf[2][USB_MAX_DEVICE_ID_TEXT]; + UsbFormatRootHub(idbuf[0], USB_MAX_DEVICE_ID_TEXT); + UsbFormatClass(idbuf[1], USB_MAX_DEVICE_ID_TEXT, USB_CLASS_HUB); + ids[0] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[0]) }; + ids[1] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[1]) }; + + DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver, + hcd->ControllerPdo, + ids, + 2, + sizeof(USB_PDO_EXTENSION), + &pdo); + if (!KSUCCESS(st)) return st; + (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-root-hub")); + + USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension; + ext->Kind = UsbPdoRootHub; + ext->Hcd = hcd; + ext->Device = (USB_DEVICE*)0; + hcd->RootHubPdo = pdo; + + st = IoRegisterPhysicalDevice(pdo); + if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) + return st; + + DbgPrint("usb: created root hub PDO ports=%u\n", hcd->MaxRootPorts); + return STATUS_SUCCESS; +} + +KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) { + if (!hcd || !xdev) return STATUS_INVALID_PARAMETER; + if (xdev->UsbPdo) return STATUS_ALREADY_EXISTS; + + USB_DEVICE* udev = UsbAllocateDevice(); + if (!udev) return STATUS_NO_MEMORY; + + udev->Hcd = hcd; + udev->HcdDeviceContext = xdev; + udev->State = xdev->Configured ? UsbDeviceConfigured : (xdev->Addressed ? UsbDeviceAddressed : UsbDeviceAttached); + udev->SlotId = xdev->SlotId; + udev->Speed = xdev->Speed; + udev->Address = xdev->SlotId; + udev->MaxPacketSize0 = xdev->MaxPacketSize0; + udev->VendorId = xdev->VendorId; + udev->ProductId = xdev->ProductId; + udev->BcdUsb = xdev->BcdUsb; + udev->BcdDevice = xdev->BcdDevice; + udev->DeviceClass = xdev->DeviceClass; + udev->DeviceSubclass = xdev->DeviceSubclass; + udev->DeviceProtocol = xdev->DeviceProtocol; + udev->ConfigurationValue = xdev->ConfigurationValue; + udev->Path = xdev->Path; + if (udev->Path.RootPort == 0) { + udev->Path.RootPort = xdev->Port; + udev->Path.Depth = 0; + udev->Path.RouteString = 0; + } + + IO_DEVICE_ID ids[5]; + char idbuf[5][USB_MAX_DEVICE_ID_TEXT]; + size_t idCount = 0; + UsbMakeDeviceIds(xdev, ids, idbuf, &idCount); + + DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver, + parent ? parent : (hcd->RootHubPdo ? hcd->RootHubPdo : hcd->ControllerPdo), + ids, + idCount, + sizeof(USB_PDO_EXTENSION), + &pdo); + if (!KSUCCESS(st)) { + UsbDereferenceDevice(udev); + return st; + } + (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-device")); + + udev->Pdo = pdo; + IoReferenceDeviceObject(pdo); + + USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension; + ext->Kind = UsbPdoDevice; + ext->Hcd = hcd; + ext->Device = udev; + + xdev->UsbDevice = udev; + xdev->UsbPdo = pdo; + + st = IoRegisterPhysicalDevice(pdo); + if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) + return st; + + DbgPrint("usb: created device PDO slot=%u vid=%04x pid=%04x ids=%llu\n", + xdev->SlotId, xdev->VendorId, xdev->ProductId, (unsigned long long)idCount); + return STATUS_SUCCESS; +} + +KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) { + if (!hcd || !xdev || !xdev->UsbDevice) return STATUS_INVALID_PARAMETER; + DEVICE_OBJECT* deviceParent = parent ? parent : xdev->UsbPdo; + if (!deviceParent) return STATUS_INVALID_PARAMETER; + + for (uint8_t i = 0; i < xdev->InterfaceCount; i++) { + USB_TRACKED_INTERFACE* iface = &xdev->Interfaces[i]; + if (!iface->Present) + continue; + + IO_DEVICE_ID ids[3]; + char idbuf[3][USB_MAX_DEVICE_ID_TEXT]; + size_t idCount = 0; + UsbMakeInterfaceIds(xdev, iface, ids, idbuf, &idCount); + + DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreatePhysicalDevice(&gUsbCompositeDriver, + deviceParent, + ids, + idCount, + sizeof(USB_PDO_EXTENSION), + &pdo); + if (!KSUCCESS(st)) { + DbgPrint("usb: create interface PDO slot=%u if=%u failed st=0x%x\n", + xdev->SlotId, iface->Descriptor.bInterfaceNumber, st); + return st; + } + (void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-interface")); + + USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension; + ext->Kind = UsbPdoInterface; + ext->Hcd = hcd; + ext->Device = xdev->UsbDevice; + ext->InterfaceIndex = i; + memcpy(&ext->InterfaceDescriptor, &iface->Descriptor, sizeof(iface->Descriptor)); + ext->EndpointCount = iface->EndpointCount; + for (uint8_t ep = 0; ep < iface->EndpointCount && ep < USB_MAX_TRACKED_ENDPOINTS; ep++) + memcpy(&ext->Endpoints[ep], &iface->Endpoints[ep], sizeof(USB_ENDPOINT_DESCRIPTOR)); + + st = IoRegisterPhysicalDevice(pdo); + if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) + return st; + + DbgPrint("usb: created interface PDO slot=%u if=%u class=%02x subclass=%02x proto=%02x\n", + xdev->SlotId, + iface->Descriptor.bInterfaceNumber, + iface->Descriptor.bInterfaceClass, + iface->Descriptor.bInterfaceSubClass, + iface->Descriptor.bInterfaceProtocol); + } + return STATUS_SUCCESS; +} + + +static DEVICE_OBJECT* UsbReferenceFirstChild(DEVICE_OBJECT* parent) { + if (!parent) return (DEVICE_OBJECT*)0; + uintptr_t flags = KeAcquireSpinlock(&parent->Lock); + DEVICE_OBJECT* child = parent->FirstChild; + if (child) + IoReferenceDeviceObject(child); + KeReleaseSpinlock(&parent->Lock, flags); + return child; +} + +static void UsbSendPnpIfAlive(DEVICE_OBJECT* pdo, IO_PNP_MINOR_FUNCTION minor) { + if (!pdo || pdo->DeletePending) + return; + (void)IoSendPnpIrpSync(pdo, minor); +} + +KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) { + if (!parent) + return STATUS_INVALID_PARAMETER; + + for (;;) { + DEVICE_OBJECT* child = UsbReferenceFirstChild(parent); + if (!child) + break; + + if (child->Type == DeviceObjectPhysical) + (void)UsbRemovePdoSubtree(child, surprise); + else + (void)UsbRemoveDeviceObjectChildren(child, surprise); + + IoDereferenceDeviceObject(child); + } + return STATUS_SUCCESS; +} + +KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise) { + if (!pdo || pdo->Type != DeviceObjectPhysical) + return STATUS_INVALID_PARAMETER; + + // Child PDOs may be owned either directly by the PDO or by an attached FDO + // such as usb-composite or usb-hub. The attached driver's PnP handler is + // therefore responsible for removing its own child PDOs before forwarding + // REMOVE_DEVICE to the lower PDO. + (void)UsbRemoveDeviceObjectChildren(pdo, surprise); + + if (surprise) + UsbSendPnpIfAlive(pdo, IRP_MN_SURPRISE_REMOVAL); + + UsbSendPnpIfAlive(pdo, IRP_MN_REMOVE_DEVICE); + return STATUS_SUCCESS; +} + +static KSTATUS UsbSubmitUrbToHcd(USB_PDO_EXTENSION* ext, USB_URB* urb) { + if (!ext || !urb) return STATUS_INVALID_PARAMETER; + USB_HCD* hcd = ext->Hcd; + if (!hcd || !hcd->Operations || !hcd->Operations->SubmitUrb) + return STATUS_NOT_SUPPORTED; + + if (!urb->Device) + urb->Device = ext->Device; + if (!urb->Device) + return STATUS_INVALID_PARAMETER; + + return hcd->Operations->SubmitUrb(hcd, urb); +} + +KSTATUS UsbCancelUrb(USB_URB* urb) { + if (!urb || !urb->Device || !urb->Device->Hcd) + return STATUS_INVALID_PARAMETER; + USB_HCD* hcd = urb->Device->Hcd; + if (!hcd->Operations || !hcd->Operations->CancelUrb) + return STATUS_NOT_SUPPORTED; + return hcd->Operations->CancelUrb(hcd, urb); +} + +KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb) { + if (!target || !urb) return STATUS_INVALID_PARAMETER; + + IRP irp; + IoInitializeIrp(&irp, 1); + IoBuildDeviceIoControlIrp(&irp, + IRP_MJ_INTERNAL_DEVICE_CONTROL, + USB_IOCTL_SUBMIT_URB, + urb, + sizeof(*urb), + urb, + sizeof(*urb)); + KSTATUS st = IoSubmitIrpSync(target, &irp); + return KSUCCESS(st) ? urb->Status : st; +} + +static void UsbAsyncUrbIrpComplete(IRP* irp, void* context) { + (void)context; + IoFreeIrp(irp); +} + +KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb) { + if (!target || !urb) return STATUS_INVALID_PARAMETER; + + IRP* irp = (IRP*)0; + KSTATUS st = IoAllocateIrp(1, &irp); + if (!KSUCCESS(st)) return st; + + IoBuildDeviceIoControlIrp(irp, + IRP_MJ_INTERNAL_DEVICE_CONTROL, + USB_IOCTL_SUBMIT_URB, + urb, + sizeof(*urb), + urb, + sizeof(*urb)); + IoSetCompletionRoutine(irp, UsbAsyncUrbIrpComplete, (void*)0); + + st = IoSubmitIrp(target, irp); + if (st != STATUS_PENDING && !(irp->Flags & IRP_FLAG_COMPLETED)) { + IoFreeIrp(irp); + return st; + } + return st; +} + +KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return STATUS_INVALID_PARAMETER; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) return STATUS_INVALID_PARAMETER; + + if (ios->Parameters.DeviceIoControl.IoControlCode != USB_IOCTL_SUBMIT_URB) { + IoCompleteRequest(irp, STATUS_NOT_SUPPORTED, 0); + return STATUS_PENDING; + } + if (ios->Parameters.DeviceIoControl.InputLength < sizeof(USB_URB) || + !ios->Parameters.DeviceIoControl.InputBuffer) { + IoCompleteRequest(irp, STATUS_INVALID_PARAMETER, 0); + return STATUS_PENDING; + } + + USB_URB* urb = (USB_URB*)ios->Parameters.DeviceIoControl.InputBuffer; + USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension; + urb->Irp = irp; + KSTATUS st = UsbSubmitUrbToHcd(ext, urb); + urb->Status = st; + + if (st == STATUS_PENDING) { + urb->Pending = true; + return STATUS_PENDING; + } + + urb->Pending = false; + IoCompleteRequest(irp, st, KSUCCESS(st) ? urb->ActualLength : 0); + return STATUS_PENDING; +} + +KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return STATUS_INVALID_PARAMETER; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_PNP) + return STATUS_INVALID_PARAMETER; + + USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension; + KSTATUS st = STATUS_SUCCESS; + + switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { + case IRP_MN_START_DEVICE: + device->PnpState = DevicePnpStarted; + if (ext && ext->Device && ext->Device->State < UsbDeviceConfigured) + ext->Device->State = UsbDeviceConfigured; + break; + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + break; + case IRP_MN_CANCEL_REMOVE_DEVICE: + device->PnpState = DevicePnpStarted; + break; + case IRP_MN_SURPRISE_REMOVAL: + device->PnpState = DevicePnpSurpriseRemoved; + if (ext && ext->Device) + ext->Device->State = UsbDeviceSurpriseRemoved; + break; + case IRP_MN_REMOVE_DEVICE: + device->PnpState = DevicePnpDeleted; + if (ext && ext->Device) + ext->Device->State = UsbDeviceRemoved; + IoDeleteDevice(device); + break; + default: + st = STATUS_NOT_SUPPORTED; + break; + } + + if (st == STATUS_NOT_SUPPORTED) + return st; + IoCompleteRequest(irp, st, 0); + return STATUS_PENDING; +} diff --git a/code/drivers/bus/usb/usb_composite.c b/code/drivers/bus/usb/usb_composite.c new file mode 100644 index 0000000..e9487b1 --- /dev/null +++ b/code/drivers/bus/usb/usb_composite.c @@ -0,0 +1,150 @@ +// usb_composite.c - USB composite function driver and interface-PDO bus. +#include "usb.h" + +DRIVER_OBJECT gUsbCompositeDriver; + +static const IO_DEVICE_ID gUsbCompositeIds[] = { + { IoBusUsb, { "USB\\COMPOSITE", sizeof("USB\\COMPOSITE") - 1 } }, +}; + +static const IO_DRIVER_ID_TABLE gUsbCompositeIdTable = { + gUsbCompositeIds, + sizeof(gUsbCompositeIds) / sizeof(gUsbCompositeIds[0]), +}; + +static KSTATUS UsbCompositeCreateInterfaces(USB_COMPOSITE_EXTENSION* ext) { + if (!ext || !ext->Device || ext->InterfacesCreated) + return STATUS_SUCCESS; + + XHCI_DEVICE* xdev = (XHCI_DEVICE*)ext->Device->HcdDeviceContext; + if (!xdev) + return STATUS_INVALID_PARAMETER; + + KSTATUS st = UsbCreateInterfacePdos(ext->Device->Hcd, xdev, ext->Self); + if (KSUCCESS(st)) + ext->InterfacesCreated = true; + return st; +} + +static KSTATUS UsbCompositeForwardPnp(DEVICE_OBJECT* device, IRP* irp, KSTATUS localStatus) { + if (!KSUCCESS(localStatus)) { + IoCompleteRequest(irp, localStatus, 0); + return STATUS_PENDING; + } + return IoForwardIrpToLower(device, irp); +} + +KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) + return STATUS_INVALID_PARAMETER; + + // The composite driver owns two different device-object roles: + // * an FDO attached above a USB device PDO, and + // * interface PDOs that it creates for class drivers. + // Forwarded PnP IRPs from HID/storage land on the interface PDO, so the + // PDO must use the generic USB PDO PnP handler. Treating its + // USB_PDO_EXTENSION as USB_COMPOSITE_EXTENSION corrupts the dispatch path. + if (device->Type == DeviceObjectPhysical) + return UsbDispatchPdoPnp(device, irp); + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_PNP) + return STATUS_INVALID_PARAMETER; + + USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)device->DeviceExtension; + KSTATUS st = STATUS_SUCCESS; + + switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { + case IRP_MN_START_DEVICE: + device->PnpState = DevicePnpStarted; + if (ext) { + if (ext->DevicePdo) + ext->DevicePdo->PnpState = DevicePnpStarted; + ext->Started = true; + st = UsbCompositeCreateInterfaces(ext); + } + return UsbCompositeForwardPnp(device, irp, st); + + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_CANCEL_REMOVE_DEVICE: + device->PnpState = DevicePnpStarted; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_SURPRISE_REMOVAL: + device->PnpState = DevicePnpSurpriseRemoved; + if (ext) + ext->Started = false; + (void)UsbRemoveDeviceObjectChildren(device, true); + return IoForwardIrpToLower(device, irp); + + case IRP_MN_REMOVE_DEVICE: + device->PnpState = DevicePnpDeleted; + if (ext) { + ext->Started = false; + (void)UsbRemoveDeviceObjectChildren(device, false); + if (ext->LowerDevice) { + KSTATUS lower = IoForwardIrpToLower(device, irp); + IoDetachDevice(device); + ext->LowerDevice = (DEVICE_OBJECT*)0; + IoDeleteDevice(device); + if (lower == STATUS_NOT_SUPPORTED) { + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + } + if (lower != STATUS_PENDING && !KSUCCESS(lower)) { + IoCompleteRequest(irp, lower, 0); + return STATUS_PENDING; + } + return lower; + } + } + IoDeleteDevice(device); + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + + default: + return IoForwardIrpToLower(device, irp); + } +} + +KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) { + if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical) + return STATUS_INVALID_PARAMETER; + + USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension; + if (!pdoExt || pdoExt->Kind != UsbPdoDevice || !pdoExt->Device) + return STATUS_INVALID_PARAMETER; + + DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreateDevice(driver, sizeof(USB_COMPOSITE_EXTENSION), &fdo); + if (!KSUCCESS(st)) + return st; + + (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-composite")); + + USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)fdo->DeviceExtension; + ext->Self = fdo; + ext->DevicePdo = physicalDeviceObject; + ext->Device = pdoExt->Device; + ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject); + if (!ext->LowerDevice) { + IoDeleteDevice(fdo); + return STATUS_FAILURE; + } + + DbgPrint("usb: composite attached slot=%u\n", ext->Device->SlotId); + return STATUS_SUCCESS; +} + +KSTATUS UsbInitializeCompositeDriver(void) { + memset(&gUsbCompositeDriver, 0, sizeof(gUsbCompositeDriver)); + IoInitializeDriverObject(&gUsbCompositeDriver, SYS_SV_LITERAL("usb-composite")); + gUsbCompositeDriver.IdTable = &gUsbCompositeIdTable; + gUsbCompositeDriver.DriverAddDevice = UsbCompositeAddDevice; + gUsbCompositeDriver.MajorFunction[IRP_MJ_PNP] = UsbCompositeDispatchPnp; + gUsbCompositeDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl; + return IoRegisterDriverObject(&gUsbCompositeDriver); +} diff --git a/code/drivers/bus/usb/usb_core.c b/code/drivers/bus/usb/usb_core.c index 52a25b3..252d882 100644 --- a/code/drivers/bus/usb/usb_core.c +++ b/code/drivers/bus/usb/usb_core.c @@ -25,6 +25,16 @@ KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, } +static USB_TRACKED_INTERFACE* UsbFindTrackedInterfaceByNumber(XHCI_DEVICE* dev, uint8_t interfaceNumber) { + if (!dev) return (USB_TRACKED_INTERFACE*)0; + for (uint8_t i = 0; i < dev->InterfaceCount && i < USB_MAX_TRACKED_INTERFACES; i++) { + if (dev->Interfaces[i].Present && + dev->Interfaces[i].Descriptor.bInterfaceNumber == interfaceNumber) + return &dev->Interfaces[i]; + } + return (USB_TRACKED_INTERFACE*)0; +} + void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_CONFIGURATION_DESCRIPTOR* config, uint16_t totalLength) { @@ -79,17 +89,38 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, massBulkOut = 0; tracked = (USB_TRACKED_INTERFACE*)0; - if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) { - tracked = &dev->Interfaces[dev->InterfaceCount++]; - memset(tracked, 0, sizeof(*tracked)); - tracked->Present = true; - memcpy(&tracked->Descriptor, iface, sizeof(*iface)); + if (iface->bAlternateSetting == 0) { + USB_TRACKED_INTERFACE* existing = UsbFindTrackedInterfaceByNumber(dev, iface->bInterfaceNumber); + if (existing) { + tracked = existing; + memset(tracked->Endpoints, 0, sizeof(tracked->Endpoints)); + tracked->EndpointCount = 0; + memcpy(&tracked->Descriptor, iface, sizeof(*iface)); + UsbTrace("usb: slot=%u duplicate alt0 interface %u replaced existing PDO candidate\n", + dev->SlotId, iface->bInterfaceNumber); + } else if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) { + tracked = &dev->Interfaces[dev->InterfaceCount++]; + memset(tracked, 0, sizeof(*tracked)); + tracked->Present = true; + memcpy(&tracked->Descriptor, iface, sizeof(*iface)); + } else { + DbgPrint("usb: slot=%u too many interfaces, interface %u alt=0 not published\n", + dev->SlotId, iface->bInterfaceNumber); + } } else { - DbgPrint("usb: slot=%u too many interfaces, interface %u not published\n", - dev->SlotId, iface->bInterfaceNumber); + /* + * Alternate settings are not independent USB interfaces. Real + * UVC cameras often expose one VideoStreaming interface with + * many alternate settings, and publishing each alternate setting + * as a separate PDO causes duplicate interface devices and can + * exhaust USB_MAX_TRACKED_INTERFACES. Track only alternate + * setting zero until a function driver explicitly selects a + * different setting. + */ + tracked = (USB_TRACKED_INTERFACE*)0; } - UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)\n", + UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)%s\n", dev->SlotId, iface->bInterfaceNumber, iface->bAlternateSetting, @@ -97,7 +128,8 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, currentClass, currentSubClass, currentProtocol, - UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol)); + UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol), + iface->bAlternateSetting ? " alt-not-published" : ""); } else if (type == USB_DESC_ENDPOINT && len >= sizeof(USB_ENDPOINT_DESCRIPTOR)) { const USB_ENDPOINT_DESCRIPTOR* ep = (const USB_ENDPOINT_DESCRIPTOR*)desc; uint8_t epType = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK; @@ -178,54 +210,6 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, } -void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { - for (uint8_t i = 0; i < dev->InterfaceCount; i++) { - USB_TRACKED_INTERFACE* iface = &dev->Interfaces[i]; - if (!iface->Present) - continue; - - IO_PUBLISHED_DEVICE pub; - memset(&pub, 0, sizeof(pub)); - pub.BusType = IoBusUsb; - pub.Name = "usb-interface"; - pub.BusContext = xhci; - pub.DeviceContext = dev; - - pub.u.Usb.VendorId = dev->VendorId; - pub.u.Usb.ProductId = dev->ProductId; - pub.u.Usb.BcdUsb = dev->BcdUsb; - pub.u.Usb.BcdDevice = dev->BcdDevice; - pub.u.Usb.DeviceClass = dev->DeviceClass; - pub.u.Usb.DeviceSubclass = dev->DeviceSubclass; - pub.u.Usb.DeviceProtocol = dev->DeviceProtocol; - pub.u.Usb.SlotId = dev->SlotId; - pub.u.Usb.Port = dev->Port; - pub.u.Usb.Speed = dev->Speed; - pub.u.Usb.ConfigurationValue = dev->ConfigurationValue; - pub.u.Usb.InterfaceNumber = iface->Descriptor.bInterfaceNumber; - pub.u.Usb.AlternateSetting = iface->Descriptor.bAlternateSetting; - pub.u.Usb.InterfaceClass = iface->Descriptor.bInterfaceClass; - pub.u.Usb.InterfaceSubclass = iface->Descriptor.bInterfaceSubClass; - pub.u.Usb.InterfaceProtocol = iface->Descriptor.bInterfaceProtocol; - pub.u.Usb.EndpointCount = iface->EndpointCount; - if (pub.u.Usb.EndpointCount > IO_PUBLISHED_USB_MAX_ENDPOINTS) - pub.u.Usb.EndpointCount = IO_PUBLISHED_USB_MAX_ENDPOINTS; - - for (uint8_t ep = 0; ep < pub.u.Usb.EndpointCount; ep++) { - pub.u.Usb.Endpoints[ep].Address = iface->Endpoints[ep].bEndpointAddress; - pub.u.Usb.Endpoints[ep].Attributes = iface->Endpoints[ep].bmAttributes; - pub.u.Usb.Endpoints[ep].MaxPacketSize = iface->Endpoints[ep].wMaxPacketSize; - pub.u.Usb.Endpoints[ep].Interval = iface->Endpoints[ep].bInterval; - } - - KSTATUS st = IoPublishDevice(&pub); - if (!KSUCCESS(st)) { - DbgPrint("usb: publish interface slot=%u if=%u failed st=0x%x\n", - dev->SlotId, iface->Descriptor.bInterfaceNumber, st); - } - } -} - KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_DEVICE_DESCRIPTOR* devDesc) { @@ -255,43 +239,89 @@ KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, st = XhciSetConfiguration(xhci, dev, config.bConfigurationValue); if (!KSUCCESS(st)) return st; - XhciPublishUsbInterfaces(xhci, dev); + if (!dev->UsbPdo) { + st = UsbCreateDevicePdo(&xhci->Hcd, dev, + dev->ParentDeviceObject ? dev->ParentDeviceObject : + (xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : xhci->Hcd.ControllerPdo)); + if (!KSUCCESS(st)) { + DbgPrint("usb: create device PDO slot=%u failed st=0x%x\n", dev->SlotId, st); + return st; + } + } return STATUS_SUCCESS; } -KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) { - uint8_t speed = 0; - KSTATUS st = XhciResetPort(xhci, port, &speed); - if (!KSUCCESS(st)) return st; +static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) { + if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) + return; + + XHCI_DEVICE* dev = &xhci->Devices[slotId]; + + if (dev->UsbPdo) { + (void)UsbRemovePdoSubtree(dev->UsbPdo, true); + dev->UsbPdo = (DEVICE_OBJECT*)0; + } + + if (dev->UsbDevice) { + dev->UsbDevice->State = UsbDeviceRemoved; + UsbDereferenceDevice(dev->UsbDevice); + dev->UsbDevice = (USB_DEVICE*)0; + } + + if (dev->Allocated) + (void)XhciDisableSlot(xhci, slotId); + + memset(dev, 0, sizeof(*dev)); +} + + +static KSTATUS XhciEnumeratePathEx(XHCI_CONTROLLER* xhci, + uint8_t logicalPort, + uint8_t speed, + const USB_PORT_PATH* path, + DEVICE_OBJECT* parentDevice, + uint8_t* outSlotId) { + if (outSlotId) *outSlotId = 0; + if (!xhci || !path || path->RootPort == 0) + return STATUS_INVALID_PARAMETER; uint8_t slotId = 0; - st = XhciEnableSlot(xhci, &slotId); + KSTATUS st = XhciEnableSlot(xhci, &slotId); if (!KSUCCESS(st)) return st; XHCI_DEVICE* dev = &xhci->Devices[slotId]; memset(dev, 0, sizeof(*dev)); dev->Allocated = true; dev->SlotId = slotId; - dev->Port = port; + dev->Port = path->RootPort; dev->Speed = speed; + dev->Path = *path; + dev->ParentDeviceObject = parentDevice; st = XhciAllocateDeviceMemory(xhci, dev); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) { + XhciRollbackEnumeratedSlot(xhci, slotId); + return st; + } XhciBuildDefaultControlContext(xhci, dev); st = XhciAddressDevice(xhci, dev); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) { + XhciRollbackEnumeratedSlot(xhci, slotId); + return st; + } dev->Addressed = true; USB_DEVICE_DESCRIPTOR desc; memset(&desc, 0, sizeof(desc)); - st = XhciGetDeviceDescriptor(xhci, dev, &desc); + st = XhciGetDeviceDescriptorTimeout(xhci, dev, &desc, USB_XHCI_ENUM_CONTROL_TIMEOUT_US); if (!KSUCCESS(st)) { DbgPrint("usb: port %u slot %u descriptor read failed st=0x%x\n", - port, slotId, st); + logicalPort, slotId, st); + XhciRollbackEnumeratedSlot(xhci, slotId); return st; } @@ -304,24 +334,100 @@ KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) { dev->DeviceSubclass = desc.bDeviceSubClass; dev->DeviceProtocol = desc.bDeviceProtocol; - DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n", - port, - slotId, - XhciPortSpeedName(speed), - speed, - desc.idVendor, - desc.idProduct, - desc.bDeviceClass, - desc.bDeviceSubClass, - desc.bDeviceProtocol, - desc.bMaxPacketSize0, - desc.bNumConfigurations, - desc.bcdUSB); + if (path->BehindHub) { + DbgPrint("usb: device root-port=%u hub-slot=%u hub-port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x route=%05x\n", + path->RootPort, + path->ParentHubSlot, + path->ParentHubPort, + slotId, + XhciPortSpeedName(speed), + speed, + desc.idVendor, + desc.idProduct, + desc.bDeviceClass, + desc.bDeviceSubClass, + desc.bDeviceProtocol, + desc.bMaxPacketSize0, + desc.bNumConfigurations, + desc.bcdUSB, + path->RouteString); + } else { + DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n", + logicalPort, + slotId, + XhciPortSpeedName(speed), + speed, + desc.idVendor, + desc.idProduct, + desc.bDeviceClass, + desc.bDeviceSubClass, + desc.bDeviceProtocol, + desc.bMaxPacketSize0, + desc.bNumConfigurations, + desc.bcdUSB); + } st = XhciDiscoverAndSetConfiguration(xhci, dev, &desc); + if (!KSUCCESS(st)) { + XhciRollbackEnumeratedSlot(xhci, slotId); + return st; + } + + if (outSlotId) *outSlotId = slotId; + return STATUS_SUCCESS; +} + +KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId) { + if (outSlotId) *outSlotId = 0; + + uint8_t speed = 0; + KSTATUS st = XhciResetPort(xhci, port, &speed); if (!KSUCCESS(st)) return st; - return STATUS_SUCCESS; + USB_PORT_PATH path; + memset(&path, 0, sizeof(path)); + path.RootPort = port; + path.Depth = 0; + path.RouteString = 0; + path.BehindHub = false; + + DEVICE_OBJECT* parent = xhci && xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : (DEVICE_OBJECT*)0; + return XhciEnumeratePathEx(xhci, port, speed, &path, parent, outSlotId); +} + +KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci, + USB_HUB_EXTENSION* hub, + uint8_t hubPort, + uint8_t speed, + uint8_t* outSlotId) { + if (outSlotId) *outSlotId = 0; + if (!xhci || !hub || !hub->Device || !hub->Self || hubPort == 0) + return STATUS_INVALID_PARAMETER; + + USB_DEVICE* parent = hub->Device; + if (parent->Path.Depth >= 5u) { + DbgPrint("usb-hub: hub slot=%u port=%u exceeds xHCI route depth\n", + parent->SlotId, hubPort); + return STATUS_NOT_SUPPORTED; + } + if (hubPort > 15u) { + DbgPrint("usb-hub: hub slot=%u port=%u cannot be represented in xHCI route string yet\n", + parent->SlotId, hubPort); + return STATUS_NOT_SUPPORTED; + } + + USB_PORT_PATH path = parent->Path; + path.BehindHub = true; + path.ParentHubSlot = parent->SlotId; + path.ParentHubPort = hubPort; + path.RouteString |= ((uint32_t)hubPort << (path.Depth * 4u)); + path.Depth++; + + return XhciEnumeratePathEx(xhci, hubPort, speed, &path, hub->Self, outSlotId); +} + +KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) { + return XhciEnumeratePortEx(xhci, port, (uint8_t*)0); } @@ -335,7 +441,8 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) { continue; attempted++; - KSTATUS st = XhciEnumeratePort(xhci, port); + uint8_t slot = 0; + KSTATUS st = XhciEnumeratePortEx(xhci, port, &slot); if (KSUCCESS(st)) configured++; else DbgPrint("usb: root-port %u enumeration failed st=0x%x\n", port, st); } @@ -343,19 +450,3 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) { DbgPrint("usb: xHCI root-port enumeration complete; attempted=%u configured=%u\n", attempted, configured); } - - -KSTATUS UsbRegisterClassMatchers(void) { - memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver)); - gUsbStorageDriver.Name = "usbstor"; - gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite; - gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite; - - KSTATUS st = IoRegisterDriverMatcher("usbhid", IoBusUsb, UsbHidMatch, UsbHidBind, NULL); - if (!KSUCCESS(st)) return st; - - st = IoRegisterDriverMatcher("usbstor", IoBusUsb, UsbStorageMatch, UsbStorageBind, NULL); - if (!KSUCCESS(st)) return st; - - return STATUS_SUCCESS; -} diff --git a/code/drivers/bus/usb/usb_hid.c b/code/drivers/bus/usb/usb_hid.c index 0cf00f9..41cdf3b 100644 --- a/code/drivers/bus/usb/usb_hid.c +++ b/code/drivers/bus/usb/usb_hid.c @@ -125,7 +125,7 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) { } -void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, +static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue) { if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return; @@ -158,11 +158,10 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) { - if (xhci->HidDeviceCount == 0) + if (!xhci || xhci->HidDeviceCount == 0) return STATUS_SUCCESS; - XhciArmAllHid(xhci); - DbgPrint("usb: HID interrupt input armed devices=%u\n", xhci->HidDeviceCount); + DbgPrint("usb: HID interrupt URBs active devices=%u\n", xhci->HidDeviceCount); return STATUS_SUCCESS; } @@ -365,3 +364,412 @@ KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context) { } return st; } + +// ------------------------------------------------------------------------- +// PDO/FDO boot HID function driver path. +// ------------------------------------------------------------------------- + +DRIVER_OBJECT gUsbHidDriver; + +static const IO_DEVICE_ID gUsbHidIds[] = { + { IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_01", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_01") - 1 } }, + { IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_02", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_02") - 1 } }, +}; + +static const IO_DRIVER_ID_TABLE gUsbHidIdTable = { + gUsbHidIds, + sizeof(gUsbHidIds) / sizeof(gUsbHidIds[0]), +}; + +static const USB_ENDPOINT_DESCRIPTOR* UsbHidFindInterruptInPdo(const USB_PDO_EXTENSION* ext) { + if (!ext) return (const USB_ENDPOINT_DESCRIPTOR*)0; + + for (uint8_t i = 0; i < ext->EndpointCount; i++) { + const USB_ENDPOINT_DESCRIPTOR* ep = &ext->Endpoints[i]; + uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK; + bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0; + if (type == USB_ENDPOINT_TYPE_INTERRUPT && in) + return ep; + } + return (const USB_ENDPOINT_DESCRIPTOR*)0; +} + + +static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) { + if (!ext || len < 8) return; + + uint8_t modifiers = report[0]; + for (uint8_t bit = 0; bit < 8; bit++) { + uint8_t mask = (uint8_t)(1u << bit); + bool wasDown = (ext->LastModifiers & mask) != 0; + bool isDown = (modifiers & mask) != 0; + if (wasDown != isDown) + XhciSubmitHidKey(XhciHidModifierKey(bit), isDown); + } + + const uint8_t* keys = &report[2]; + for (uint8_t i = 0; i < 6; i++) { + uint8_t oldKey = ext->LastKeys[i]; + if (oldKey > 1 && !XhciHidKeyInReport(keys, oldKey)) + XhciSubmitHidKey((INPUT_KEY)oldKey, false); + } + for (uint8_t i = 0; i < 6; i++) { + uint8_t newKey = keys[i]; + if (newKey > 1 && !XhciHidKeyInReport(ext->LastKeys, newKey)) + XhciSubmitHidKey((INPUT_KEY)newKey, true); + } + + ext->LastModifiers = modifiers; + memcpy(ext->LastKeys, keys, sizeof(ext->LastKeys)); +} + +static void UsbHidProcessBootMouseReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) { + if (!ext || len < 3) return; + + int dx = (int)(int8_t)report[1]; + int dy = (int)(int8_t)report[2]; + int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0; + uint8_t buttons = report[0] & 0x07u; + + if (dx || dy || wheel || buttons != ext->LastButtons) { + INPUT_MOUSE_EVENT ev; + ev.dx = dx; + ev.dy = dy; + ev.wheel = wheel; + ev.buttons = buttons; + KeInputSubmitMouse(&ev); + } + + ext->LastButtons = buttons; +} + +static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext); + +static void UsbHidInterruptComplete(USB_URB* urb, void* context) { + USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)context; + if (!ext || urb != &ext->InterruptUrb) + return; + + ext->InterruptPending = false; + if (ext->SurpriseRemoved || ext->Removed || !ext->Started) + return; + + if (KSUCCESS(urb->Status)) { + uint16_t length = (uint16_t)urb->ActualLength; + if (length > ext->ReportLength) length = ext->ReportLength; + + if (ext->Keyboard) + UsbHidProcessBootKeyboardReport(ext, ext->ReportBuffer, length); + else if (ext->Mouse) + UsbHidProcessBootMouseReport(ext, ext->ReportBuffer, length); + } else if (urb->Status != STATUS_CANCELLED) { + DbgPrint("usb: HID interrupt URB failed slot=%u if=%u ep=%02x st=0x%x\n", + ext->Device ? ext->Device->SlotId : 0, + ext->InterfaceNumber, + ext->EndpointAddress, + urb->Status); + } + + if (!ext->SurpriseRemoved && !ext->Removed) + (void)UsbHidArmInterrupt(ext); +} + +static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) { + if (!ext || !ext->LowerDevice || !ext->Device) + return STATUS_INVALID_PARAMETER; + + USB_URB urb; + memset(&urb, 0, sizeof(urb)); + urb.Function = UrbFunctionControlTransfer; + urb.Direction = UsbTransferNoData; + urb.Device = ext->Device; + urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE; + urb.Setup.bRequest = USB_REQ_SET_PROTOCOL; + urb.Setup.wValue = 0; + urb.Setup.wIndex = ext->InterfaceNumber; + urb.Setup.wLength = 0; + + KSTATUS st = UsbSubmitUrbSync(ext->LowerDevice, &urb); + if (!KSUCCESS(st)) { + DbgPrint("usb: HID SET_PROTOCOL boot failed slot=%u if=%u st=0x%x\n", + ext->Device->SlotId, ext->InterfaceNumber, st); + return st; + } + + memset(&urb, 0, sizeof(urb)); + urb.Function = UrbFunctionControlTransfer; + urb.Direction = UsbTransferNoData; + urb.Device = ext->Device; + urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE; + urb.Setup.bRequest = USB_REQ_SET_IDLE; + urb.Setup.wValue = 0; + urb.Setup.wIndex = ext->InterfaceNumber; + urb.Setup.wLength = 0; + + st = UsbSubmitUrbSync(ext->LowerDevice, &urb); + if (!KSUCCESS(st)) { + DbgPrint("usb: HID SET_IDLE failed slot=%u if=%u st=0x%x; continuing\n", + ext->Device->SlotId, ext->InterfaceNumber, st); + } + + DbgPrint("usb: HID boot protocol selected slot=%u if=%u proto=%u\n", + ext->Device->SlotId, ext->InterfaceNumber, ext->Protocol); + return STATUS_SUCCESS; +} + +static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) { + if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed) + return STATUS_OBJECT_DEAD; + if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys) + return STATUS_INVALID_PARAMETER; + if (ext->InterruptPending) + return STATUS_SUCCESS; + + memset(ext->ReportBuffer, 0, ext->ReportLength); + memset(&ext->InterruptUrb, 0, sizeof(ext->InterruptUrb)); + ext->InterruptUrb.Function = UrbFunctionInterruptTransfer; + ext->InterruptUrb.Direction = UsbTransferIn; + ext->InterruptUrb.Device = ext->Device; + ext->InterruptUrb.Pipe = ext->InterruptPipe; + ext->InterruptUrb.Buffer = ext->ReportBuffer; + ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys; + ext->InterruptUrb.Length = ext->ReportLength; + ext->InterruptUrb.Complete = UsbHidInterruptComplete; + ext->InterruptUrb.CompleteContext = ext; + + KSTATUS st = UsbSubmitUrbAsync(ext->LowerDevice, &ext->InterruptUrb); + if (st != STATUS_PENDING && !KSUCCESS(st)) + return st; + if (ext->InterruptUrb.Status != STATUS_PENDING && ext->InterruptUrb.Status != STATUS_SUCCESS) + return ext->InterruptUrb.Status; + + ext->InterruptPending = true; + return STATUS_SUCCESS; +} + +static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) { + if (!device || !ext || !ext->Device || !ext->InterfacePdo) + return STATUS_INVALID_PARAMETER; + if (ext->Started) + return STATUS_SUCCESS; + if (ext->SurpriseRemoved || ext->Removed) + return STATUS_OBJECT_DEAD; + + USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)ext->InterfacePdo->DeviceExtension; + if (!pdoExt || pdoExt->Kind != UsbPdoInterface) + return STATUS_INVALID_PARAMETER; + + const USB_ENDPOINT_DESCRIPTOR* ep = UsbHidFindInterruptInPdo(pdoExt); + if (!ep) + return STATUS_NOT_FOUND; + + ext->Keyboard = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD); + ext->Mouse = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_MOUSE); + ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber; + ext->Protocol = pdoExt->InterfaceDescriptor.bInterfaceProtocol; + ext->EndpointAddress = ep->bEndpointAddress; + ext->MaxPacketSize = ep->wMaxPacketSize; + ext->Interval = ep->bInterval; + + uint16_t reportLength = ext->MaxPacketSize & 0x07FFu; + if (reportLength == 0) reportLength = ext->Keyboard ? 8u : 4u; + if (ext->Keyboard && reportLength < 8) reportLength = 8; + if (ext->Mouse && reportLength < 3) reportLength = 3; + if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT; + ext->ReportLength = reportLength; + + ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys); + if (!ext->ReportBuffer) + return STATUS_NO_MEMORY; + + KSTATUS st = UsbCreatePipe(ext->Device, + UsbPipeInterrupt, + ext->EndpointAddress, + ext->MaxPacketSize, + ext->Interval, + &ext->InterruptPipe); + if (!KSUCCESS(st)) + return st; + + st = UsbHidSendBootProtocol(ext); + if (!KSUCCESS(st)) + return st; + + memset(ext->LastKeys, 0, sizeof(ext->LastKeys)); + ext->LastModifiers = 0; + ext->LastButtons = 0; + + KeInputRegisterDevice(ext->Keyboard ? "usb-hid-keyboard" : "usb-hid-mouse", 0, &ext->InputDeviceId); + + ext->Started = true; + device->PnpState = DevicePnpStarted; + if (ext->InterfacePdo) + ext->InterfacePdo->PnpState = DevicePnpStarted; + + XHCI_CONTROLLER* xhci = ext->Device && ext->Device->Hcd ? + (XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0; + + st = UsbHidArmInterrupt(ext); + if (!KSUCCESS(st)) + return st; + + if (xhci) + xhci->HidDeviceCount++; + + DbgPrint("usb: HID boot %s bound slot=%u if=%u ep=%02x maxpkt=%u interval=%u input=%u urb\n", + ext->Keyboard ? "keyboard" : "mouse", + ext->Device->SlotId, + ext->InterfaceNumber, + ext->EndpointAddress, + ext->MaxPacketSize, + ext->Interval, + ext->InputDeviceId); + return STATUS_SUCCESS; +} + + +KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) + return STATUS_INVALID_PARAMETER; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_PNP) + return STATUS_INVALID_PARAMETER; + + USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)device->DeviceExtension; + KSTATUS st = STATUS_SUCCESS; + + switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { + case IRP_MN_START_DEVICE: + st = UsbHidStartDevice(device, ext); + if (!KSUCCESS(st)) { + IoCompleteRequest(irp, st, 0); + return STATUS_PENDING; + } + return IoForwardIrpToLower(device, irp); + + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_CANCEL_REMOVE_DEVICE: + if (!ext || !ext->SurpriseRemoved) + device->PnpState = DevicePnpStarted; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_SURPRISE_REMOVAL: + device->PnpState = DevicePnpSurpriseRemoved; + if (ext) { + ext->SurpriseRemoved = true; + ext->Started = false; + if (ext->InterruptPending) + (void)UsbCancelUrb(&ext->InterruptUrb); + } + return IoForwardIrpToLower(device, irp); + + case IRP_MN_REMOVE_DEVICE: { + device->PnpState = DevicePnpDeleted; + if (ext) { + ext->Removed = true; + ext->Started = false; + if (ext->InterruptPending) + (void)UsbCancelUrb(&ext->InterruptUrb); + if (ext->InterruptPipe) { + ext->InterruptPipe->Open = false; + UsbDereferencePipe(ext->InterruptPipe); + ext->InterruptPipe = (USB_PIPE*)0; + } + if (ext->Device && ext->Device->Hcd) { + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext; + if (xhci && xhci->HidDeviceCount) + xhci->HidDeviceCount--; + } + if (ext->RemoveLockReady) { + IoReleaseRemoveLockAndWait(&ext->RemoveLock, irp); + IoDestroyRemoveLock(&ext->RemoveLock); + ext->RemoveLockReady = false; + } + } + + KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS; + if (ext && ext->LowerDevice) { + IoDetachDevice(device); + ext->LowerDevice = (DEVICE_OBJECT*)0; + } + IoDeleteDevice(device); + if (lower == STATUS_NOT_SUPPORTED) { + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + } + if (lower != STATUS_PENDING && !KSUCCESS(lower)) { + IoCompleteRequest(irp, lower, 0); + return STATUS_PENDING; + } + return lower; + } + + default: + return IoForwardIrpToLower(device, irp); + } +} + +KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) { + if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical) + return STATUS_INVALID_PARAMETER; + + USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension; + if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device) + return STATUS_INVALID_PARAMETER; + if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_HID || + pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_HID_SUBCLASS_BOOT) + return STATUS_INVALID_PARAMETER; + if (pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_KEYBOARD && + pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_MOUSE) + return STATUS_INVALID_PARAMETER; + if (!UsbHidFindInterruptInPdo(pdoExt)) + return STATUS_NOT_FOUND; + + DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreateDevice(driver, sizeof(USB_HID_EXTENSION), &fdo); + if (!KSUCCESS(st)) + return st; + + (void)IoSetDeviceName(fdo, + pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD + ? SYS_SV_LITERAL("usb-hid-keyboard") + : SYS_SV_LITERAL("usb-hid-mouse")); + + USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)fdo->DeviceExtension; + ext->Self = fdo; + ext->InterfacePdo = physicalDeviceObject; + ext->Device = pdoExt->Device; + st = IoInitializeRemoveLock(&ext->RemoveLock); + if (!KSUCCESS(st)) { + IoDeleteDevice(fdo); + return st; + } + ext->RemoveLockReady = true; + + ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject); + if (!ext->LowerDevice) { + IoDestroyRemoveLock(&ext->RemoveLock); + ext->RemoveLockReady = false; + IoDeleteDevice(fdo); + return STATUS_FAILURE; + } + + DbgPrint("usb: HID FDO attached slot=%u if=%u proto=%u\n", + ext->Device->SlotId, + pdoExt->InterfaceDescriptor.bInterfaceNumber, + pdoExt->InterfaceDescriptor.bInterfaceProtocol); + return STATUS_SUCCESS; +} + +KSTATUS UsbInitializeHidDriver(void) { + memset(&gUsbHidDriver, 0, sizeof(gUsbHidDriver)); + IoInitializeDriverObject(&gUsbHidDriver, SYS_SV_LITERAL("usb-hid")); + gUsbHidDriver.IdTable = &gUsbHidIdTable; + gUsbHidDriver.DriverAddDevice = UsbHidAddDevice; + gUsbHidDriver.MajorFunction[IRP_MJ_PNP] = UsbHidDispatchPnp; + return IoRegisterDriverObject(&gUsbHidDriver); +} diff --git a/code/drivers/bus/usb/usb_hub.c b/code/drivers/bus/usb/usb_hub.c new file mode 100644 index 0000000..0812615 --- /dev/null +++ b/code/drivers/bus/usb/usb_hub.c @@ -0,0 +1,1027 @@ +// usb_hub.c - USB hub function driver. +// +// Root hubs and external hubs are represented by the same FDO abstraction. The +// xHCI root hub feeds port-change events directly into this driver, while an +// external hub feeds the same port state machine from its interrupt-IN change +// bitmap. +#include "usb.h" + +DRIVER_OBJECT gUsbHubDriver; + +static const IO_DEVICE_ID gUsbHubIds[] = { + { IoBusUsb, { "USB\\ROOT_HUB", sizeof("USB\\ROOT_HUB") - 1 } }, + { IoBusUsb, { "USB\\CLASS_09", sizeof("USB\\CLASS_09") - 1 } }, +}; + +static const IO_DRIVER_ID_TABLE gUsbHubIdTable = { + gUsbHubIds, + sizeof(gUsbHubIds) / sizeof(gUsbHubIds[0]), +}; + +#define USB_HUB_PORT_ENUMERATING 0xffu +#define USB_HUB_PORT_FAILED 0xfeu + +static bool UsbHubSlotBusy(uint8_t slot) { + return slot != 0 && slot != USB_HUB_PORT_ENUMERATING && slot != USB_HUB_PORT_FAILED; +} + +static bool UsbHubRootPortConnected(XHCI_CONTROLLER* xhci, uint8_t port) { + if (!xhci || !xhci->Op || port == 0) return false; + uint32_t portsc = XhciRead32(xhci->Op, XhciPortReg(port)); + return (portsc & XHCI_PORTSC_CCS) != 0; +} + +static void UsbHubClearRootPortFailure(USB_HUB_EXTENSION* hub, uint8_t port) { + if (!hub || port == 0 || port > USB_ROOT_HUB_MAX_PORTS) return; + hub->RootPortFailCount[port] = 0; + if (hub->RootPortSlot[port] == USB_HUB_PORT_FAILED) + hub->RootPortSlot[port] = 0; +} + +static void UsbHubMarkRootPortFailed(USB_HUB_EXTENSION* hub, uint8_t port) { + if (!hub || port == 0 || port > USB_ROOT_HUB_MAX_PORTS) return; + hub->RootPortSlot[port] = USB_HUB_PORT_FAILED; + hub->RootPortState[port] = UsbHubPortFailedConnected; +} + +static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub, + XHCI_CONTROLLER* xhci, + uint8_t port, + uint8_t* outSlot) { + if (outSlot) *outSlot = 0; + if (!hub || !xhci || port == 0 || port > USB_ROOT_HUB_MAX_PORTS) + return STATUS_INVALID_PARAMETER; + + KSTATUS last = STATUS_DEVICE_ERROR; + uint8_t maxAttempts = USB_ROOT_PORT_ENUM_RETRIES; + if (maxAttempts == 0) maxAttempts = 1; + + for (uint8_t attempt = 1; attempt <= maxAttempts; attempt++) { + if (!UsbHubRootPortConnected(xhci, port)) + return STATUS_NOT_FOUND; + + uint8_t slot = 0; + last = XhciEnumeratePortEx(xhci, port, &slot); + if (KSUCCESS(last)) { + if (outSlot) *outSlot = slot; + hub->RootPortFailCount[port] = 0; + hub->RootPortState[port] = UsbHubPortChildPresent; + return STATUS_SUCCESS; + } + + hub->RootPortFailCount[port] = attempt; + + /* + * On real Intel laptop controllers some internal full-speed devices + * respond with xHCI USB Transaction Error during Address Device while + * the port is still settling. Retry that boundedly while the port is + * held in ENUMERATING state so queued PSC events cannot start parallel + * enumerations. Descriptor/control timeouts are not retried here; they + * are usually dead/firmware-owned ports and retrying them stalls boot. + */ + if (last != STATUS_DEVICE_ERROR || attempt == maxAttempts) + break; + + DbgPrint("usb-hub: root port %u enumeration attempt %u failed st=0x%x; retrying\n", + port, attempt, last); + UsbDelayUs(xhci->Hal, 50000); + } + + if (UsbHubRootPortConnected(xhci, port)) { + UsbHubMarkRootPortFailed(hub, port); + } else { + UsbHubClearRootPortFailure(hub, port); + hub->RootPortState[port] = UsbHubPortDisconnected; + } + + return last; +} + +static bool UsbHubPopPendingRootPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) { + if (!hub || !outPort) return false; + + uintptr_t flags = KeAcquireSpinlock(&hub->PendingLock); + for (uint32_t word = 0; word < 8; word++) { + uint32_t bits = hub->PendingRootPorts[word]; + if (!bits) + continue; + for (uint8_t bit = 0; bit < 32; bit++) { + if (!(bits & (1u << bit))) + continue; + hub->PendingRootPorts[word] &= ~(1u << bit); + KeReleaseSpinlock(&hub->PendingLock, flags); + *outPort = (uint8_t)(word * 32u + bit); + return true; + } + } + KeReleaseSpinlock(&hub->PendingLock, flags); + return false; +} + +static bool UsbHubPopPendingExternalPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) { + if (!hub || !outPort) return false; + + uintptr_t flags = KeAcquireSpinlock(&hub->PendingLock); + uint32_t bits = hub->PendingHubPorts[0]; + if (!bits) { + KeReleaseSpinlock(&hub->PendingLock, flags); + return false; + } + + for (uint8_t port = 1; port <= hub->PortCount && port <= USB_EXTERNAL_HUB_MAX_PORTS; port++) { + if (!(bits & (1u << port))) + continue; + hub->PendingHubPorts[0] &= ~(1u << port); + KeReleaseSpinlock(&hub->PendingLock, flags); + *outPort = port; + return true; + } + + hub->PendingHubPorts[0] = 0; + KeReleaseSpinlock(&hub->PendingLock, flags); + return false; +} + +static void UsbHubRecordRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) { + if (!hub || port == 0 || port > hub->PortCount) + return; + uint32_t word = (uint32_t)port / 32u; + uint32_t bit = (uint32_t)port & 31u; + uintptr_t flags = KeAcquireSpinlock(&hub->PendingLock); + hub->PendingRootPorts[word] |= (1u << bit); + KeReleaseSpinlock(&hub->PendingLock, flags); + if (hub->WorkerSemaphore) + KeSignalSemaphore(hub->WorkerSemaphore); +} + +void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port) { + if (!hub || port == 0 || port > hub->PortCount || port > USB_EXTERNAL_HUB_MAX_PORTS) + return; + uintptr_t flags = KeAcquireSpinlock(&hub->PendingLock); + hub->PendingHubPorts[0] |= (1u << port); + KeReleaseSpinlock(&hub->PendingLock, flags); + if (hub->WorkerSemaphore) + KeSignalSemaphore(hub->WorkerSemaphore); +} + +static void UsbHubRemoveSlotChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t slot) { + if (!hub || !hub->Hcd || slot == 0) + return; + + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hub->Hcd->ControllerContext; + if (!xhci || slot > XHCI_MAX_TRACKED_SLOTS) + return; + + XHCI_DEVICE* xdev = &xhci->Devices[slot]; + DbgPrint("usb-hub: %s port %u removing slot=%u\n", + hub->Kind == UsbHubKindRoot ? "root" : "hub", port, slot); + + if (xdev->UsbPdo) + (void)UsbRemovePdoSubtree(xdev->UsbPdo, true); + + if (xdev->Allocated) + (void)XhciDisableSlot(xhci, slot); + + if (xdev->UsbDevice) { + xdev->UsbDevice->State = UsbDeviceRemoved; + UsbDereferenceDevice(xdev->UsbDevice); + } + + memset(xdev, 0, sizeof(*xdev)); + if (hub->Kind == UsbHubKindRoot && port <= USB_ROOT_HUB_MAX_PORTS) { + hub->RootPortSlot[port] = 0; + hub->RootPortState[port] = UsbHubPortDisconnected; + } else if (port <= USB_EXTERNAL_HUB_MAX_PORTS) { + hub->ChildSlot[port] = 0; + hub->ChildPortState[port] = UsbHubPortDisconnected; + } +} + +static void UsbHubRemoveRootChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t slot) { + UsbHubRemoveSlotChild(hub, port, slot); +} + +static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub, + uint8_t requestType, + uint8_t request, + uint16_t value, + uint16_t index, + void* buffer, + PHYSICAL_ADDRESS bufferPhys, + uint16_t length) { + if (!hub || !hub->LowerDevice || !buffer || !bufferPhys || !length) + return STATUS_INVALID_PARAMETER; + + USB_URB urb; + memset(&urb, 0, sizeof(urb)); + urb.Function = UrbFunctionControlTransfer; + urb.Direction = UsbTransferIn; + urb.Device = hub->Device; + urb.Buffer = buffer; + urb.BufferPhys = bufferPhys; + urb.Length = length; + urb.Setup.bmRequestType = requestType; + urb.Setup.bRequest = request; + urb.Setup.wValue = value; + urb.Setup.wIndex = index; + urb.Setup.wLength = length; + return UsbSubmitUrbSync(hub->LowerDevice, &urb); +} + +static KSTATUS UsbHubControlNoData(USB_HUB_EXTENSION* hub, + uint8_t requestType, + uint8_t request, + uint16_t value, + uint16_t index) { + if (!hub || !hub->LowerDevice) + return STATUS_INVALID_PARAMETER; + + USB_URB urb; + memset(&urb, 0, sizeof(urb)); + urb.Function = UrbFunctionControlTransfer; + urb.Direction = UsbTransferNoData; + urb.Device = hub->Device; + urb.Setup.bmRequestType = requestType; + urb.Setup.bRequest = request; + urb.Setup.wValue = value; + urb.Setup.wIndex = index; + urb.Setup.wLength = 0; + return UsbSubmitUrbSync(hub->LowerDevice, &urb); +} + +static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* outDesc) { + if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapPhys) + return STATUS_INVALID_PARAMETER; + + memset(hub->ChangeBitmap, 0, USB_HUB_CHANGE_BITMAP_BYTES); + KSTATUS st = UsbHubControlIn(hub, + USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_DEVICE, + USB_REQ_GET_DESCRIPTOR, + (uint16_t)(USB_DESC_HUB << 8), + 0, + hub->ChangeBitmap, + hub->ChangeBitmapPhys, + USB_HUB_CHANGE_BITMAP_BYTES); + if (!KSUCCESS(st)) + return st; + + memcpy(outDesc, hub->ChangeBitmap, sizeof(USB_HUB_DESCRIPTOR)); + if (outDesc->bDescriptorType != USB_DESC_HUB || outDesc->bNbrPorts == 0) + return STATUS_DEVICE_ERROR; + return STATUS_SUCCESS; +} + +static KSTATUS UsbHubGetPortStatus(USB_HUB_EXTENSION* hub, uint8_t port, USB_HUB_PORT_STATUS* outStatus) { + if (!hub || !outStatus || port == 0) + return STATUS_INVALID_PARAMETER; + + memset(hub->ChangeBitmap, 0, USB_HUB_CHANGE_BITMAP_BYTES); + KSTATUS st = UsbHubControlIn(hub, + USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_OTHER, + USB_REQ_GET_STATUS, + 0, + port, + hub->ChangeBitmap, + hub->ChangeBitmapPhys, + sizeof(USB_HUB_PORT_STATUS)); + if (!KSUCCESS(st)) + return st; + + memcpy(outStatus, hub->ChangeBitmap, sizeof(USB_HUB_PORT_STATUS)); + return STATUS_SUCCESS; +} + +static KSTATUS UsbHubSetPortFeature(USB_HUB_EXTENSION* hub, uint8_t port, uint16_t feature) { + return UsbHubControlNoData(hub, + USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_OTHER, + USB_REQ_SET_FEATURE, + feature, + port); +} + +static KSTATUS UsbHubClearPortFeature(USB_HUB_EXTENSION* hub, uint8_t port, uint16_t feature) { + return UsbHubControlNoData(hub, + USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_OTHER, + USB_REQ_CLEAR_FEATURE, + feature, + port); +} + +static uint8_t UsbHubSpeedFromPortStatus(uint16_t status) { + if (status & USB_HUB_PORT_STATUS_LOW_SPEED) + return 2u; + if (status & USB_HUB_PORT_STATUS_HIGH_SPEED) + return 3u; + return 1u; +} + +static KSTATUS UsbHubResetExternalPort(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t* outSpeed) { + if (outSpeed) *outSpeed = 0; + if (!hub || port == 0) + return STATUS_INVALID_PARAMETER; + + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_CONNECTION); + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_ENABLE); + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_RESET); + + KSTATUS st = UsbHubSetPortFeature(hub, port, USB_HUB_PORT_FEAT_RESET); + if (!KSUCCESS(st)) + return st; + + USB_HUB_PORT_STATUS ps; + memset(&ps, 0, sizeof(ps)); + for (uint32_t i = 0; i < 500u; i++) { + UsbDelayUs(((XHCI_CONTROLLER*)hub->Hcd->ControllerContext)->Hal, 1000u); + st = UsbHubGetPortStatus(hub, port, &ps); + if (!KSUCCESS(st)) + return st; + if (ps.Change & USB_HUB_PORT_CHANGE_RESET) + break; + } + + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_RESET); + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_ENABLE); + + st = UsbHubGetPortStatus(hub, port, &ps); + if (!KSUCCESS(st)) + return st; + if (!(ps.Status & USB_HUB_PORT_STATUS_CONNECTION)) + return STATUS_NOT_FOUND; + if (!(ps.Status & USB_HUB_PORT_STATUS_ENABLE)) + return STATUS_DEVICE_ERROR; + + if (outSpeed) + *outSpeed = UsbHubSpeedFromPortStatus(ps.Status); + return STATUS_SUCCESS; +} + +static void UsbHubHandleExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port) { + if (!hub || hub->Kind != UsbHubKindExternal || port == 0 || port > hub->PortCount || port > USB_EXTERNAL_HUB_MAX_PORTS) + return; + + USB_HUB_PORT_STATUS ps; + memset(&ps, 0, sizeof(ps)); + KSTATUS st = UsbHubGetPortStatus(hub, port, &ps); + if (!KSUCCESS(st)) { + DbgPrint("usb-hub: slot=%u port=%u GET_STATUS failed st=0x%x\n", + hub->Device ? hub->Device->SlotId : 0, port, st); + return; + } + + if (ps.Change & USB_HUB_PORT_CHANGE_CONNECTION) + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_CONNECTION); + if (ps.Change & USB_HUB_PORT_CHANGE_ENABLE) + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_ENABLE); + if (ps.Change & USB_HUB_PORT_CHANGE_RESET) + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_RESET); + + uint8_t oldSlot = hub->ChildSlot[port]; + if (!(ps.Status & USB_HUB_PORT_STATUS_CONNECTION)) { + if (UsbHubSlotBusy(oldSlot)) { + hub->ChildPortState[port] = UsbHubPortRemoving; + UsbHubRemoveSlotChild(hub, port, oldSlot); + } else { + hub->ChildPortState[port] = UsbHubPortDisconnected; + UsbTrace("usb-hub: slot=%u port=%u disconnected\n", hub->Device ? hub->Device->SlotId : 0, port); + } + return; + } + + if (oldSlot == USB_HUB_PORT_ENUMERATING) { + UsbTrace("usb-hub: slot=%u port=%u change ignored; enumeration already in progress\n", + hub->Device ? hub->Device->SlotId : 0, port); + return; + } + if (UsbHubSlotBusy(oldSlot)) { + UsbTrace("usb-hub: slot=%u port=%u change ignored; child slot=%u still present\n", + hub->Device ? hub->Device->SlotId : 0, port, oldSlot); + return; + } + + hub->ChildSlot[port] = USB_HUB_PORT_ENUMERATING; + hub->ChildPortState[port] = UsbHubPortConnectedDebounce; + UsbDelayUs(((XHCI_CONTROLLER*)hub->Hcd->ControllerContext)->Hal, 30000u); + + uint8_t speed = 0; + hub->ChildPortState[port] = UsbHubPortResetting; + st = UsbHubResetExternalPort(hub, port, &speed); + if (!KSUCCESS(st)) { + hub->ChildSlot[port] = 0; + hub->ChildPortState[port] = UsbHubPortFailedConnected; + DbgPrint("usb-hub: slot=%u port=%u reset failed st=0x%x\n", + hub->Device ? hub->Device->SlotId : 0, port, st); + return; + } + + hub->ChildPortState[port] = UsbHubPortEnumerating; + uint8_t slot = 0; + st = XhciEnumerateHubPortEx((XHCI_CONTROLLER*)hub->Hcd->ControllerContext, hub, port, speed, &slot); + if (KSUCCESS(st)) { + hub->ChildSlot[port] = slot; + hub->ChildPortState[port] = UsbHubPortChildPresent; + DbgPrint("usb-hub: slot=%u port=%u attached child-slot=%u speed=%s(%u)\n", + hub->Device ? hub->Device->SlotId : 0, + port, + slot, + XhciPortSpeedName(speed), + speed); + } else { + hub->ChildSlot[port] = 0; + hub->ChildPortState[port] = UsbHubPortFailedConnected; + DbgPrint("usb-hub: slot=%u port=%u enumeration failed st=0x%x\n", + hub->Device ? hub->Device->SlotId : 0, port, st); + } +} + +static void UsbHubHandleRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) { + if (!hub || !hub->Hcd || port == 0 || port > hub->PortCount || port > USB_ROOT_HUB_MAX_PORTS) + return; + + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hub->Hcd->ControllerContext; + if (!xhci || !xhci->Op) + return; + + uint32_t portsc = XhciRead32(xhci->Op, XhciPortReg(port)); + uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK; + bool connected = (portsc & XHCI_PORTSC_CCS) != 0; + uint8_t oldSlot = hub->RootPortSlot[port]; + + if (changes) + XhciClearPortChanges(xhci, port); + + if (!connected) { + if (UsbHubSlotBusy(oldSlot)) { + hub->RootPortState[port] = UsbHubPortRemoving; + UsbHubRemoveRootChild(hub, port, oldSlot); + } else if (oldSlot == USB_HUB_PORT_ENUMERATING) + UsbTrace("usb-hub: root port %u disconnect while enumeration is already in progress\n", port); + else { + hub->RootPortState[port] = UsbHubPortDisconnected; + DbgPrint("usb-hub: root port %u disconnected\n", port); + } + UsbHubClearRootPortFailure(hub, port); + return; + } + + if (oldSlot == USB_HUB_PORT_ENUMERATING) { + UsbTrace("usb-hub: root port %u change ignored; enumeration already in progress\n", port); + return; + } + + if (oldSlot == USB_HUB_PORT_FAILED) { + if (changes & XHCI_PORTSC_CSC) { + UsbTrace("usb-hub: root port %u retrying after connection-change on failed port\n", port); + UsbHubClearRootPortFailure(hub, port); + oldSlot = 0; + } else { + UsbTrace("usb-hub: root port %u change ignored; previous enumeration failed and device is still connected\n", port); + return; + } + } + + if (UsbHubSlotBusy(oldSlot)) { + UsbTrace("usb-hub: root port %u change ignored; slot=%u still present\n", port, oldSlot); + return; + } + + hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; + hub->RootPortState[port] = UsbHubPortConnectedDebounce; + + UsbDelayUs(xhci->Hal, 30000); + portsc = XhciRead32(xhci->Op, XhciPortReg(port)); + if (!(portsc & XHCI_PORTSC_CCS)) { + UsbHubClearRootPortFailure(hub, port); + return; + } + + hub->RootPortState[port] = UsbHubPortEnumerating; + uint8_t slot = 0; + KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + if (KSUCCESS(st)) { + hub->RootPortSlot[port] = slot; + hub->RootPortState[port] = UsbHubPortChildPresent; + DbgPrint("usb-hub: root port %u attached slot=%u\n", port, slot); + } else { + DbgPrint("usb-hub: root port %u hotplug enumeration failed st=0x%x after %u attempt(s)\n", + port, st, hub->RootPortFailCount[port]); + } +} +static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub); +static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd); + +static bool UsbHubTakeInitialRootScan(USB_HUB_EXTENSION* hub) { + if (!hub || hub->Kind != UsbHubKindRoot) + return false; + uintptr_t flags = KeAcquireSpinlock(&hub->PendingLock); + bool take = hub->InitialScanPending && !hub->InitialScanInProgress && !hub->InitialScanComplete; + if (take) { + hub->InitialScanPending = false; + hub->InitialScanInProgress = true; + } + KeReleaseSpinlock(&hub->PendingLock, flags); + return take; +} + +static void UsbHubWorker(void* arg) { + USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)arg; + while (hub && hub->WorkerRunning) { + KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore); + if (!KSUCCESS(st)) + continue; + + if (hub->Kind == UsbHubKindRoot) { + if (UsbHubTakeInitialRootScan(hub)) + (void)UsbHubDoRootPortScan(hub->Hcd); + + uint8_t port = 0; + while (UsbHubPopPendingRootPort(hub, &port)) + UsbHubHandleRootPortChange(hub, port); + } else { + uint8_t port = 0; + while (UsbHubPopPendingExternalPort(hub, &port)) + UsbHubHandleExternalPortChange(hub, port); + if (hub->Started && !hub->Removed && !hub->SurpriseRemoved && !hub->ChangeUrbPending) + UsbHubArmExternalInterrupt(hub); + } + } +} + +static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd) { + if (!hcd || !hcd->RootHubContext) + return STATUS_INVALID_PARAMETER; + + USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)hcd->RootHubContext; + if (!hub->Started) + return STATUS_OBJECT_DEAD; + + uint32_t attempted = 0; + uint32_t configured = 0; + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext; + if (!xhci || !xhci->Op) + return STATUS_INVALID_PARAMETER; + + for (uint8_t port = 1; port <= hub->PortCount; port++) { + uint32_t portsc = XhciRead32(xhci->Op, XhciPortReg(port)); + if (!(portsc & XHCI_PORTSC_CCS)) + continue; + + if (hub->RootPortSlot[port]) { + UsbTrace("usb-hub: root port %u initial scan skipped; state=%u\n", + port, hub->RootPortSlot[port]); + continue; + } + + attempted++; + hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; + hub->RootPortState[port] = UsbHubPortEnumerating; + uint8_t slot = 0; + KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + if (KSUCCESS(st)) { + hub->RootPortSlot[port] = slot; + hub->RootPortState[port] = UsbHubPortChildPresent; + configured++; + } else { + DbgPrint("usb-hub: root port %u enumeration failed st=0x%x after %u attempt(s)\n", + port, st, hub->RootPortFailCount[port]); + } + } + + DbgPrint("usb-hub: root-port scan complete; attempted=%u configured=%u\n", attempted, configured); + hub->InitialScanComplete = true; + hub->InitialScanInProgress = false; + return STATUS_SUCCESS; +} + +KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd) { + if (!hcd || !hcd->RootHubContext) + return STATUS_INVALID_PARAMETER; + USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)hcd->RootHubContext; + if (hub->Kind != UsbHubKindRoot || !hub->Started) + return STATUS_OBJECT_DEAD; + + uintptr_t flags = KeAcquireSpinlock(&hub->PendingLock); + if (!hub->InitialScanComplete && !hub->InitialScanInProgress) + hub->InitialScanPending = true; + KeReleaseSpinlock(&hub->PendingLock, flags); + + if (hub->WorkerSemaphore) + KeSignalSemaphore(hub->WorkerSemaphore); + return STATUS_SUCCESS; +} + +void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port) { + if (!hcd || !hcd->RootHubContext) + return; + USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)hcd->RootHubContext; + UsbHubRecordRootPortChange(hub, port); +} + +static KSTATUS UsbHubScanExternalPorts(USB_HUB_EXTENSION* hub) { + if (!hub || hub->Kind != UsbHubKindExternal) + return STATUS_INVALID_PARAMETER; + + uint32_t attempted = 0; + uint32_t configured = 0; + for (uint8_t port = 1; port <= hub->PortCount && port <= USB_EXTERNAL_HUB_MAX_PORTS; port++) { + USB_HUB_PORT_STATUS ps; + memset(&ps, 0, sizeof(ps)); + KSTATUS st = UsbHubGetPortStatus(hub, port, &ps); + if (!KSUCCESS(st)) + continue; + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_CONNECTION); + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_ENABLE); + (void)UsbHubClearPortFeature(hub, port, USB_HUB_PORT_FEAT_C_RESET); + if (!(ps.Status & USB_HUB_PORT_STATUS_CONNECTION)) + continue; + attempted++; + UsbHubHandleExternalPortChange(hub, port); + if (UsbHubSlotBusy(hub->ChildSlot[port])) + configured++; + } + DbgPrint("usb-hub: external hub slot=%u scan complete; attempted=%u configured=%u\n", + hub->Device ? hub->Device->SlotId : 0, attempted, configured); + return STATUS_SUCCESS; +} + +static const USB_ENDPOINT_DESCRIPTOR* UsbHubFindInterruptIn(USB_PDO_EXTENSION* pdoExt) { + if (!pdoExt) + return (const USB_ENDPOINT_DESCRIPTOR*)0; + + if (pdoExt->Kind == UsbPdoInterface) { + for (uint8_t i = 0; i < pdoExt->EndpointCount; i++) { + const USB_ENDPOINT_DESCRIPTOR* ep = &pdoExt->Endpoints[i]; + if ((ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) && + ((ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK) == USB_ENDPOINT_TYPE_INTERRUPT)) + return ep; + } + return (const USB_ENDPOINT_DESCRIPTOR*)0; + } + + if (pdoExt->Kind == UsbPdoDevice && pdoExt->Device) { + XHCI_DEVICE* xdev = (XHCI_DEVICE*)pdoExt->Device->HcdDeviceContext; + if (!xdev) + return (const USB_ENDPOINT_DESCRIPTOR*)0; + for (uint8_t i = 0; i < xdev->InterfaceCount; i++) { + USB_TRACKED_INTERFACE* iface = &xdev->Interfaces[i]; + if (!iface->Present || iface->Descriptor.bInterfaceClass != USB_CLASS_HUB) + continue; + for (uint8_t epn = 0; epn < iface->EndpointCount; epn++) { + const USB_ENDPOINT_DESCRIPTOR* ep = &iface->Endpoints[epn]; + if ((ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) && + ((ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK) == USB_ENDPOINT_TYPE_INTERRUPT)) + return ep; + } + } + } + return (const USB_ENDPOINT_DESCRIPTOR*)0; +} + +static void UsbHubExternalChangeComplete(USB_URB* urb, void* context) { + USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)context; + if (!hub || urb != &hub->ChangeUrb) + return; + + hub->ChangeUrbPending = false; + if (!KSUCCESS(urb->Status) || hub->Removed || hub->SurpriseRemoved) + return; + + uint32_t actual = urb->ActualLength; + if (actual > USB_HUB_CHANGE_BITMAP_BYTES) + actual = USB_HUB_CHANGE_BITMAP_BYTES; + + for (uint8_t port = 1; port <= hub->PortCount && port <= USB_EXTERNAL_HUB_MAX_PORTS; port++) { + uint8_t byteIndex = port / 8u; + uint8_t bit = port & 7u; + if (byteIndex < actual && (hub->ChangeBitmap[byteIndex] & (1u << bit))) + UsbHubQueueExternalPortChange(hub, port); + } +} + +static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub) { + if (!hub || !hub->InterruptPipe || !hub->ChangeBitmap || hub->ChangeUrbPending) + return; + + memset(hub->ChangeBitmap, 0, USB_HUB_CHANGE_BITMAP_BYTES); + memset(&hub->ChangeUrb, 0, sizeof(hub->ChangeUrb)); + hub->ChangeUrb.Function = UrbFunctionInterruptTransfer; + hub->ChangeUrb.Direction = UsbTransferIn; + hub->ChangeUrb.Device = hub->Device; + hub->ChangeUrb.Pipe = hub->InterruptPipe; + hub->ChangeUrb.Buffer = hub->ChangeBitmap; + hub->ChangeUrb.BufferPhys = hub->ChangeBitmapPhys; + hub->ChangeUrb.Length = hub->InterruptMaxPacket; + if (hub->ChangeUrb.Length > USB_HUB_CHANGE_BITMAP_BYTES) + hub->ChangeUrb.Length = USB_HUB_CHANGE_BITMAP_BYTES; + hub->ChangeUrb.Complete = UsbHubExternalChangeComplete; + hub->ChangeUrb.CompleteContext = hub; + + KSTATUS st = UsbSubmitUrbAsync(hub->LowerDevice, &hub->ChangeUrb); + if (st == STATUS_PENDING) { + hub->ChangeUrbPending = true; + } else if (!KSUCCESS(st)) { + DbgPrint("usb-hub: slot=%u interrupt arm failed st=0x%x\n", + hub->Device ? hub->Device->SlotId : 0, st); + } +} + +static KSTATUS UsbHubStartRootDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* hub) { + hub->PortCount = hub->Hcd ? hub->Hcd->MaxRootPorts : 0; + if (hub->PortCount > USB_ROOT_HUB_MAX_PORTS) + hub->PortCount = USB_ROOT_HUB_MAX_PORTS; + + KSTATUS st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); + if (!KSUCCESS(st)) + return st; + + hub->WorkerRunning = true; + st = KeCreateKernelThread((uintptr_t)UsbHubWorker, + (uintptr_t)hub, + PriorityNormal, + &hub->WorkerThread); + if (!KSUCCESS(st)) { + hub->WorkerRunning = false; + return st; + } + + hub->Started = true; + device->PnpState = DevicePnpStarted; + if (hub->HubPdo) + hub->HubPdo->PnpState = DevicePnpStarted; + + DbgPrint("usb-hub: root hub started ports=%u\n", hub->PortCount); + return STATUS_SUCCESS; +} + +static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* hub) { + if (!hub || !hub->Device) + return STATUS_INVALID_PARAMETER; + + hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapPhys); + if (!hub->ChangeBitmap) + return STATUS_NO_MEMORY; + + USB_HUB_DESCRIPTOR desc; + memset(&desc, 0, sizeof(desc)); + KSTATUS st = UsbHubReadDescriptor(hub, &desc); + if (!KSUCCESS(st)) { + DbgPrint("usb-hub: slot=%u descriptor read failed st=0x%x\n", hub->Device->SlotId, st); + return st; + } + + hub->PortCount = desc.bNbrPorts; + if (hub->PortCount > USB_EXTERNAL_HUB_MAX_PORTS) { + DbgPrint("usb-hub: slot=%u reports %u ports; limiting to %u\n", + hub->Device->SlotId, hub->PortCount, USB_EXTERNAL_HUB_MAX_PORTS); + hub->PortCount = USB_EXTERNAL_HUB_MAX_PORTS; + } + hub->PowerOnToPowerGoodMs = desc.bPwrOn2PwrGood ? (uint8_t)(desc.bPwrOn2PwrGood * 2u) : 100u; + + for (uint8_t port = 1; port <= hub->PortCount; port++) + (void)UsbHubSetPortFeature(hub, port, USB_HUB_PORT_FEAT_POWER); + UsbDelayUs(((XHCI_CONTROLLER*)hub->Hcd->ControllerContext)->Hal, + (uint32_t)hub->PowerOnToPowerGoodMs * 1000u); + + st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); + if (!KSUCCESS(st)) + return st; + + hub->WorkerRunning = true; + st = KeCreateKernelThread((uintptr_t)UsbHubWorker, + (uintptr_t)hub, + PriorityNormal, + &hub->WorkerThread); + if (!KSUCCESS(st)) { + hub->WorkerRunning = false; + return st; + } + + st = UsbCreatePipe(hub->Device, + UsbPipeInterrupt, + hub->InterruptEndpointAddress, + hub->InterruptMaxPacket, + hub->InterruptInterval, + &hub->InterruptPipe); + if (!KSUCCESS(st)) + return st; + + hub->Started = true; + device->PnpState = DevicePnpStarted; + if (hub->HubPdo) + hub->HubPdo->PnpState = DevicePnpStarted; + + DbgPrint("usb-hub: external hub started slot=%u ports=%u ep=%02x maxpkt=%u interval=%u\n", + hub->Device->SlotId, + hub->PortCount, + hub->InterruptEndpointAddress, + hub->InterruptMaxPacket, + hub->InterruptInterval); + + (void)UsbHubScanExternalPorts(hub); + UsbHubArmExternalInterrupt(hub); + return STATUS_SUCCESS; +} + +static KSTATUS UsbHubStartDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* hub) { + if (!device || !hub || hub->Started) + return STATUS_SUCCESS; + + if (hub->Kind == UsbHubKindRoot) + return UsbHubStartRootDevice(device, hub); + return UsbHubStartExternalDevice(device, hub); +} + +KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) + return STATUS_INVALID_PARAMETER; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_PNP) + return STATUS_INVALID_PARAMETER; + + USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)device->DeviceExtension; + KSTATUS st = STATUS_SUCCESS; + + switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { + case IRP_MN_START_DEVICE: + st = UsbHubStartDevice(device, hub); + if (!KSUCCESS(st)) { + IoCompleteRequest(irp, st, 0); + return STATUS_PENDING; + } + return IoForwardIrpToLower(device, irp); + + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_CANCEL_REMOVE_DEVICE: + if (!hub || !hub->SurpriseRemoved) + device->PnpState = DevicePnpStarted; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_SURPRISE_REMOVAL: + device->PnpState = DevicePnpSurpriseRemoved; + if (hub) { + hub->SurpriseRemoved = true; + hub->WorkerRunning = false; + if (hub->ChangeUrbPending) + (void)UsbCancelUrb(&hub->ChangeUrb); + if (hub->WorkerSemaphore) + KeSignalSemaphore(hub->WorkerSemaphore); + + // Remove downstream PDOs owned by this hub FDO. The explicit + // slot arrays are still used as the hardware lookup table; the + // child PDO tree is the authoritative software lifetime tree. + (void)UsbRemoveDeviceObjectChildren(device, true); + if (hub->Kind == UsbHubKindExternal) { + for (uint8_t port = 1; port <= hub->PortCount && port <= USB_EXTERNAL_HUB_MAX_PORTS; port++) { + if (UsbHubSlotBusy(hub->ChildSlot[port])) + UsbHubRemoveSlotChild(hub, port, hub->ChildSlot[port]); + } + } else if (hub->Kind == UsbHubKindRoot) { + for (uint8_t port = 1; port <= hub->PortCount && port <= USB_ROOT_HUB_MAX_PORTS; port++) { + if (UsbHubSlotBusy(hub->RootPortSlot[port])) + UsbHubRemoveRootChild(hub, port, hub->RootPortSlot[port]); + } + } + } + return IoForwardIrpToLower(device, irp); + + case IRP_MN_REMOVE_DEVICE: { + device->PnpState = DevicePnpDeleted; + if (hub) { + hub->Removed = true; + hub->WorkerRunning = false; + if (hub->ChangeUrbPending) + (void)UsbCancelUrb(&hub->ChangeUrb); + (void)UsbRemoveDeviceObjectChildren(device, false); + if (hub->InterruptPipe) { + hub->InterruptPipe->Open = false; + UsbDereferencePipe(hub->InterruptPipe); + hub->InterruptPipe = (USB_PIPE*)0; + } + if (hub->WorkerSemaphore) + KeSignalSemaphore(hub->WorkerSemaphore); + if (hub->Kind == UsbHubKindRoot && hub->Hcd && hub->Hcd->RootHubContext == hub) + hub->Hcd->RootHubContext = (void*)0; + if (hub->RemoveLockReady) { + IoReleaseRemoveLockAndWait(&hub->RemoveLock, irp); + IoDestroyRemoveLock(&hub->RemoveLock); + hub->RemoveLockReady = false; + } + } + + KSTATUS lower = hub && hub->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS; + if (hub && hub->LowerDevice) { + IoDetachDevice(device); + hub->LowerDevice = (DEVICE_OBJECT*)0; + } + IoDeleteDevice(device); + if (lower == STATUS_NOT_SUPPORTED) { + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + } + if (lower != STATUS_PENDING && !KSUCCESS(lower)) { + IoCompleteRequest(irp, lower, 0); + return STATUS_PENDING; + } + return lower; + } + + default: + return IoForwardIrpToLower(device, irp); + } +} + +KSTATUS UsbHubAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) { + if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical) + return STATUS_INVALID_PARAMETER; + + USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension; + if (!pdoExt) + return STATUS_INVALID_PARAMETER; + + USB_HUB_KIND kind; + USB_DEVICE* usbDevice = (USB_DEVICE*)0; + const USB_ENDPOINT_DESCRIPTOR* intr = (const USB_ENDPOINT_DESCRIPTOR*)0; + + if (pdoExt->Kind == UsbPdoRootHub && pdoExt->Hcd) { + kind = UsbHubKindRoot; + } else if (pdoExt->Kind == UsbPdoDevice && pdoExt->Device && + pdoExt->Device->DeviceClass == USB_CLASS_HUB) { + kind = UsbHubKindExternal; + usbDevice = pdoExt->Device; + intr = UsbHubFindInterruptIn(pdoExt); + if (!intr) + return STATUS_NOT_FOUND; + } else if (pdoExt->Kind == UsbPdoInterface && pdoExt->Device && + pdoExt->InterfaceDescriptor.bInterfaceClass == USB_CLASS_HUB) { + kind = UsbHubKindExternal; + usbDevice = pdoExt->Device; + intr = UsbHubFindInterruptIn(pdoExt); + if (!intr) + return STATUS_NOT_FOUND; + } else { + return STATUS_INVALID_PARAMETER; + } + + DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreateDevice(driver, sizeof(USB_HUB_EXTENSION), &fdo); + if (!KSUCCESS(st)) + return st; + (void)IoSetDeviceName(fdo, kind == UsbHubKindRoot ? SYS_SV_LITERAL("usb-root-hub-driver") + : SYS_SV_LITERAL("usb-hub")); + + USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)fdo->DeviceExtension; + hub->Self = fdo; + hub->HubPdo = physicalDeviceObject; + hub->Kind = kind; + hub->Hcd = pdoExt->Hcd; + hub->Device = usbDevice; + if (kind == UsbHubKindExternal && hub->Device) + hub->Hcd = hub->Device->Hcd; + hub->PendingLock = (SPINLOCK)SPINLOCK_INIT; + + if (intr) { + hub->InterruptEndpointAddress = intr->bEndpointAddress; + hub->InterruptMaxPacket = intr->wMaxPacketSize; + if (hub->InterruptMaxPacket == 0 || hub->InterruptMaxPacket > USB_HUB_CHANGE_BITMAP_BYTES) + hub->InterruptMaxPacket = USB_HUB_CHANGE_BITMAP_BYTES; + hub->InterruptInterval = intr->bInterval ? intr->bInterval : 12u; + if (pdoExt->Kind == UsbPdoInterface) + hub->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber; + } + + st = IoInitializeRemoveLock(&hub->RemoveLock); + if (!KSUCCESS(st)) { + IoDeleteDevice(fdo); + return st; + } + hub->RemoveLockReady = true; + + hub->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject); + if (!hub->LowerDevice) { + IoDestroyRemoveLock(&hub->RemoveLock); + hub->RemoveLockReady = false; + IoDeleteDevice(fdo); + return STATUS_FAILURE; + } + + if (kind == UsbHubKindRoot) { + hub->Hcd->RootHubContext = hub; + DbgPrint("usb-hub: root hub FDO attached\n"); + } else { + DbgPrint("usb-hub: external hub FDO attached slot=%u ep=%02x\n", + hub->Device ? hub->Device->SlotId : 0, + hub->InterruptEndpointAddress); + } + return STATUS_SUCCESS; +} + +KSTATUS UsbInitializeHubDriver(void) { + memset(&gUsbHubDriver, 0, sizeof(gUsbHubDriver)); + IoInitializeDriverObject(&gUsbHubDriver, SYS_SV_LITERAL("usb-hub")); + gUsbHubDriver.IdTable = &gUsbHubIdTable; + gUsbHubDriver.DriverAddDevice = UsbHubAddDevice; + gUsbHubDriver.MajorFunction[IRP_MJ_PNP] = UsbHubDispatchPnp; + return IoRegisterDriverObject(&gUsbHubDriver); +} diff --git a/code/drivers/bus/usb/usb_storage.c b/code/drivers/bus/usb/usb_storage.c index be13c16..9f74339 100644 --- a/code/drivers/bus/usb/usb_storage.c +++ b/code/drivers/bus/usb/usb_storage.c @@ -1,53 +1,115 @@ -// usb_storage.c - USB mass-storage BOT/SCSI class binding and disk bridge. +// usb_storage.c - USB mass-storage BOT/SCSI class driver. +// +// The storage driver now binds as a USB function driver to an interface PDO +// (USB\CLASS_08&SUBCLASS_06&PROT_50). It creates a disk FDO above that +// interface stack and issues BOT transfers through USB URBs. The xHCI HCD +// still backs the bulk URBs with the existing per-device rings; later stages +// will make endpoint rings fully pipe-owned and asynchronous. #include "usb.h" DRIVER_OBJECT gUsbStorageDriver; -KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, - PHYSICAL_ADDRESS phys, uint32_t length, - uint32_t* outActual) { - return XhciBulkTransfer(xhci, - dev, - dev->MassBulkInEndpointId, - dev->MassInRing, - dev->MassInRingPhys, - &dev->MassInEnqueue, - &dev->MassInCycle, - phys, - length, - outActual); +static const IO_DEVICE_ID gUsbStorageIds[] = { + { IoBusUsb, { "USB\\CLASS_08&SUBCLASS_06&PROT_50", sizeof("USB\\CLASS_08&SUBCLASS_06&PROT_50") - 1 } }, +}; + +static const IO_DRIVER_ID_TABLE gUsbStorageIdTable = { + gUsbStorageIds, + sizeof(gUsbStorageIds) / sizeof(gUsbStorageIds[0]), +}; + +static USB_STORAGE_EXTENSION* UsbStorageFromDiskDevice(DEVICE_OBJECT* device) { + if (!device || !device->DeviceExtension) + return (USB_STORAGE_EXTENSION*)0; + + USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)device->DeviceExtension; + if (diskExt->Kind != UsbStorageExtensionDisk) + return (USB_STORAGE_EXTENSION*)0; + return diskExt->Storage; } +static USB_STORAGE_EXTENSION* UsbStorageFromInterfaceDevice(DEVICE_OBJECT* device) { + if (!device || !device->DeviceExtension) + return (USB_STORAGE_EXTENSION*)0; -KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, - PHYSICAL_ADDRESS phys, uint32_t length, - uint32_t* outActual) { - return XhciBulkTransfer(xhci, - dev, - dev->MassBulkOutEndpointId, - dev->MassOutRing, - dev->MassOutRingPhys, - &dev->MassOutEnqueue, - &dev->MassOutCycle, - phys, - length, - outActual); + USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)device->DeviceExtension; + if (ext->Kind != UsbStorageExtensionInterface) + return (USB_STORAGE_EXTENSION*)0; + return ext; } +static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_EXTENSION* pdoExt, bool wantIn) { + if (!pdoExt) return (const USB_ENDPOINT_DESCRIPTOR*)0; -KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, - const uint8_t* cdb, uint8_t cdbLen, - void* data, PHYSICAL_ADDRESS dataPhys, - uint32_t dataLength, bool dataIn) { - if (!xhci || !dev || !cdb || cdbLen == 0 || cdbLen > 16) + for (uint8_t i = 0; i < pdoExt->EndpointCount; i++) { + const USB_ENDPOINT_DESCRIPTOR* ep = &pdoExt->Endpoints[i]; + uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK; + bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0; + if (type == USB_ENDPOINT_TYPE_BULK && in == wantIn) + return ep; + } + return (const USB_ENDPOINT_DESCRIPTOR*)0; +} + +static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage, + bool dataIn, + PHYSICAL_ADDRESS phys, + uint32_t length, + uint32_t* outActual) { + if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0) + return STATUS_INVALID_PARAMETER; + + USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe; + if (!pipe || !pipe->Open) + return STATUS_OBJECT_DEAD; + + USB_URB urb; + memset(&urb, 0, sizeof(urb)); + urb.Function = UrbFunctionBulkTransfer; + urb.Direction = dataIn ? UsbTransferIn : UsbTransferOut; + urb.Device = storage->Device; + urb.Pipe = pipe; + urb.Buffer = (void*)0; + urb.BufferPhys = phys; + urb.Length = length; + + KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb); + if (outActual) + *outActual = KSUCCESS(st) ? urb.ActualLength : 0; + return st; +} + +KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, + PHYSICAL_ADDRESS phys, + uint32_t length, + uint32_t* outActual) { + return UsbStorageSubmitBulk(storage, true, phys, length, outActual); +} + +KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, + PHYSICAL_ADDRESS phys, + uint32_t length, + uint32_t* outActual) { + return UsbStorageSubmitBulk(storage, false, phys, length, outActual); +} + +KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, + const uint8_t* cdb, + uint8_t cdbLen, + void* data, + PHYSICAL_ADDRESS dataPhys, + uint32_t dataLength, + bool dataIn) { + if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16) return STATUS_INVALID_PARAMETER; if (dataLength && (!data || !dataPhys)) return STATUS_INVALID_PARAMETER; + XHCI_DEVICE* dev = storage->XhciDevice; USB_MASS_CBW* cbw = (USB_MASS_CBW*)dev->DescriptorBuffer; memset(cbw, 0, sizeof(*cbw)); cbw->Signature = USB_MASS_BOT_CBW_SIGNATURE; - cbw->Tag = ++dev->MassTag; + cbw->Tag = ++storage->Tag; cbw->DataTransferLength = dataLength; cbw->Flags = dataIn ? USB_DIR_IN : USB_DIR_OUT; cbw->Lun = 0; @@ -55,16 +117,14 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, memcpy(cbw->CommandBlock, cdb, cdbLen); uint32_t actual = 0; - KSTATUS st = UsbStorageBulkOut(xhci, dev, dev->DescriptorPhys, sizeof(*cbw), &actual); + KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual); if (!KSUCCESS(st) || actual != sizeof(*cbw)) return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st; if (dataLength) { actual = 0; - if (dataIn) - st = UsbStorageBulkIn(xhci, dev, dataPhys, dataLength, &actual); - else - st = UsbStorageBulkOut(xhci, dev, dataPhys, dataLength, &actual); + st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual) + : UsbStorageBulkOut(storage, dataPhys, dataLength, &actual); if (!KSUCCESS(st)) return st; } @@ -72,13 +132,13 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer; memset(csw, 0, sizeof(*csw)); actual = 0; - st = UsbStorageBulkIn(xhci, dev, dev->DescriptorPhys, sizeof(*csw), &actual); + st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual); if (!KSUCCESS(st) || actual != sizeof(*csw)) return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st; - if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != dev->MassTag) { + if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != storage->Tag) { DbgPrint("usb-storage: invalid CSW slot=%u sig=%08x tag=%08x expected=%08x\n", - dev->SlotId, csw->Signature, csw->Tag, dev->MassTag); + dev->SlotId, csw->Signature, csw->Tag, storage->Tag); return STATUS_DEVICE_ERROR; } if (csw->Status != USB_MASS_BOT_CSW_PASSED) { @@ -90,13 +150,11 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, return STATUS_SUCCESS; } - uint32_t UsbBe32(const uint8_t* p) { return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | ((uint32_t)p[2] << 8) | (uint32_t)p[3]; } - void UsbStoreBe32(uint8_t* p, uint32_t v) { p[0] = (uint8_t)(v >> 24); p[1] = (uint8_t)(v >> 16); @@ -104,17 +162,19 @@ void UsbStoreBe32(uint8_t* p, uint32_t v) { p[3] = (uint8_t)v; } - void UsbStoreBe16(uint8_t* p, uint16_t v) { p[0] = (uint8_t)(v >> 8); p[1] = (uint8_t)v; } +KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) { + if (!storage || !storage->XhciDevice) + return STATUS_INVALID_PARAMETER; -KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + XHCI_DEVICE* dev = storage->XhciDevice; uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 }; memset(dev->MassBuffer, 0, 36); - KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), dev->MassBuffer, dev->MassBufferPhys, 36, true); if (!KSUCCESS(st)) return st; @@ -128,37 +188,46 @@ KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { return STATUS_SUCCESS; } +KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) { + if (!storage || !storage->XhciDevice) + return STATUS_INVALID_PARAMETER; -KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + XHCI_DEVICE* dev = storage->XhciDevice; uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 }; memset(dev->MassBuffer, 0, 18); - return UsbStorageBotCommand(xhci, dev, rs, sizeof(rs), + return UsbStorageBotCommand(storage, rs, sizeof(rs), dev->MassBuffer, dev->MassBufferPhys, 18, true); } +KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) { + if (!storage || !storage->XhciDevice) + return STATUS_INVALID_PARAMETER; -KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { uint8_t cdb[6] = { SCSI_TEST_UNIT_READY, 0, 0, 0, 0, 0 }; - KSTATUS last = STATUS_DEVICE_ERROR; + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext; + for (uint32_t attempt = 0; attempt < 10; attempt++) { - KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), (void*)0, 0, 0, false); if (KSUCCESS(st)) return STATUS_SUCCESS; last = st; - (void)UsbStorageRequestSense(xhci, dev); + (void)UsbStorageRequestSense(storage); UsbDelayUs(xhci->Hal, 10000u); } return last; } +KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) { + if (!storage || !storage->XhciDevice) + return STATUS_INVALID_PARAMETER; -KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + XHCI_DEVICE* dev = storage->XhciDevice; uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; memset(dev->MassBuffer, 0, 8); - KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), dev->MassBuffer, dev->MassBufferPhys, 8, true); if (!KSUCCESS(st)) return st; @@ -168,27 +237,33 @@ KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { if (blockSize == 0) return STATUS_DEVICE_ERROR; + storage->BlockSize = blockSize; + storage->BlockCount = (uint64_t)lastLba + 1ull; + dev->MassBlockSize = blockSize; - dev->MassBlockCount = (uint64_t)lastLba + 1ull; + dev->MassBlockCount = storage->BlockCount; + DbgPrint("usb-storage: capacity slot=%u blocks=%llu block_size=%u (%llu MB)\n", dev->SlotId, - (unsigned long long)dev->MassBlockCount, - dev->MassBlockSize, - (unsigned long long)((dev->MassBlockCount * (uint64_t)dev->MassBlockSize) / (1024ull * 1024ull))); + (unsigned long long)storage->BlockCount, + storage->BlockSize, + (unsigned long long)((storage->BlockCount * (uint64_t)storage->BlockSize) / (1024ull * 1024ull))); return STATUS_SUCCESS; } - -KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, - bool write, uint64_t lba, - uint32_t sectors, void* buffer) { - if (!xhci || !dev || !buffer || sectors == 0) +KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, + bool write, + uint64_t lba, + uint32_t sectors, + void* buffer) { + if (!storage || !storage->XhciDevice || !buffer || sectors == 0) return STATUS_INVALID_PARAMETER; - if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) + if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE) return STATUS_NOT_SUPPORTED; if (lba > 0xFFFFFFFFull || sectors > 0xFFFFu) return STATUS_INVALID_PARAMETER; + XHCI_DEVICE* dev = storage->XhciDevice; uint32_t bytes = sectors * USB_STORAGE_SECTOR_SIZE; if (bytes > USB_STORAGE_MAX_BYTES) return STATUS_INVALID_PARAMETER; @@ -202,7 +277,7 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, if (write) memcpy(dev->MassBuffer, buffer, bytes); - KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), dev->MassBuffer, dev->MassBufferPhys, bytes, !write); if (!KSUCCESS(st)) return st; @@ -212,42 +287,67 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, return STATUS_SUCCESS; } - -void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { - while (_InterlockedCompareExchange(&dev->MassIoBusy, 1, 0) != 0) { - UsbDelayUs(xhci->Hal, 10); +void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) { + if (!storage) return; + XHCI_CONTROLLER* xhci = storage->Device ? (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0; + while (_InterlockedCompareExchange(&storage->IoBusy, 1, 0) != 0) { + if (xhci) UsbDelayUs(xhci->Hal, 10); } } - -void UsbStorageReleaseIo(XHCI_DEVICE* dev) { - _InterlockedExchange(&dev->MassIoBusy, 0); +void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage) { + if (storage) _InterlockedExchange(&storage->IoBusy, 0); } - KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) { - if (!device || !irp || !device->DeviceExtension) + if (!device || !irp) return STATUS_INVALID_PARAMETER; - XHCI_DEVICE* dev = (XHCI_DEVICE*)device->DeviceExtension; - if (!dev->MassActive || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) { - IoCompleteRequest(irp, STATUS_DEVICE_ERROR, 0); + USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device); + if (!storage || !storage->RemoveLockReady) + return STATUS_OBJECT_DEAD; + + KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp); + if (!KSUCCESS(st)) + return st; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + return STATUS_INVALID_PARAMETER; + } + if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + return STATUS_INVALID_PARAMETER; + } + if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) || + (ios->Parameters.ReadWrite.Length & 511u) || + ios->Parameters.ReadWrite.Length == 0 || + !irp->UserBuffer) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + return STATUS_INVALID_PARAMETER; + } + + if (!storage->Started || storage->SurpriseRemoved || storage->Removed || + storage->BlockSize != USB_STORAGE_SECTOR_SIZE) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + IoCompleteRequest(irp, STATUS_OBJECT_DEAD, 0); return STATUS_PENDING; } - uint8_t* buffer = (uint8_t*)irp->Buffer; - uint64_t lba = irp->Lba; - uint32_t remaining = irp->SectorCount; + uint8_t* buffer = (uint8_t*)irp->UserBuffer; + uint64_t lba = ios->Parameters.ReadWrite.ByteOffset >> 9; + uint32_t remaining = ios->Parameters.ReadWrite.Length >> 9; uint32_t done = 0; - bool write = (irp->Flags & IRP_FLAG_WRITE) != 0; + bool write = (ios->MajorFunction == IRP_MJ_WRITE); uint32_t maxSectors = USB_STORAGE_MAX_BYTES / USB_STORAGE_SECTOR_SIZE; - KSTATUS st = STATUS_SUCCESS; - UsbStorageAcquireIo(&gXhci, dev); + st = STATUS_SUCCESS; + UsbStorageAcquireIo(storage); while (remaining) { uint32_t chunk = remaining; if (chunk > maxSectors) chunk = maxSectors; - st = UsbStorageReadWriteSectors(&gXhci, dev, write, lba, chunk, + st = UsbStorageReadWriteSectors(storage, write, lba, chunk, buffer + (done * USB_STORAGE_SECTOR_SIZE)); if (!KSUCCESS(st)) break; @@ -255,13 +355,13 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) { done += chunk; remaining -= chunk; } - UsbStorageReleaseIo(dev); + UsbStorageReleaseIo(storage); + IoReleaseRemoveLock(&storage->RemoveLock, irp); IoCompleteRequest(irp, st, KSUCCESS(st) ? (size_t)done * USB_STORAGE_SECTOR_SIZE : 0); return STATUS_PENDING; } - KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer) return STATUS_SUCCESS; @@ -282,7 +382,6 @@ KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { return STATUS_SUCCESS; } - KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { if (!dev->MassCandidate) return STATUS_SUCCESS; @@ -352,7 +451,6 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de return STATUS_SUCCESS; } - uint8_t UsbFindFreeDiskIndex(void) { for (uint8_t i = 0; i < 8; i++) { if (!IoDiskDevicePresent(i)) @@ -361,17 +459,23 @@ uint8_t UsbFindFreeDiskIndex(void) { return 0xFFu; } - -KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) { - if (!dev || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE || dev->MassBlockCount == 0) +KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage) { + if (!storage || storage->BlockSize != USB_STORAGE_SECTOR_SIZE || storage->BlockCount == 0) return STATUS_NOT_SUPPORTED; + if (storage->DiskDevice) + return STATUS_ALREADY_EXISTS; DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0; - KSTATUS st = IoCreateDevice(&gUsbStorageDriver, 0, &disk); + KSTATUS st = IoCreateDevice(&gUsbStorageDriver, sizeof(USB_STORAGE_DISK_EXTENSION), &disk); if (!KSUCCESS(st)) return st; + (void)IoSetDeviceName(disk, SYS_SV_LITERAL("usb-storage-disk")); - disk->DeviceExtension = dev; - disk->SectorCount = dev->MassBlockCount; + USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)disk->DeviceExtension; + diskExt->Kind = UsbStorageExtensionDisk; + diskExt->Storage = storage; + + disk->u.Disk.SectorCount = storage->BlockCount; + disk->u.Disk.SectorSize = USB_STORAGE_SECTOR_SIZE; disk->AlignmentRequirement = USB_STORAGE_SECTOR_SIZE; disk->Flags |= DEVICE_OBJECT_REMOVABLE; @@ -387,12 +491,278 @@ KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) { return st; } - dev->MassDeviceObject = disk; + storage->DiskDevice = disk; + if (storage->XhciDevice) + storage->XhciDevice->MassDeviceObject = disk; + DbgPrint("usb-storage: registered disk %u slot=%u sectors=%llu\n", - index, dev->SlotId, (unsigned long long)dev->MassBlockCount); + index, + storage->Device ? storage->Device->SlotId : 0, + (unsigned long long)storage->BlockCount); return STATUS_SUCCESS; } +static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSION* storage) { + if (!device || !storage || !storage->InterfacePdo || !storage->Device) + return STATUS_INVALID_PARAMETER; + if (storage->Started) + return STATUS_SUCCESS; + if (storage->SurpriseRemoved || storage->Removed) + return STATUS_OBJECT_DEAD; + + USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)storage->InterfacePdo->DeviceExtension; + if (!pdoExt || pdoExt->Kind != UsbPdoInterface) + return STATUS_INVALID_PARAMETER; + + const USB_ENDPOINT_DESCRIPTOR* bulkIn = UsbStorageFindBulkEndpoint(pdoExt, true); + const USB_ENDPOINT_DESCRIPTOR* bulkOut = UsbStorageFindBulkEndpoint(pdoExt, false); + if (!bulkIn || !bulkOut) + return STATUS_NOT_FOUND; + + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext; + XHCI_DEVICE* xdev = (XHCI_DEVICE*)storage->Device->HcdDeviceContext; + if (!xhci || !xdev) + return STATUS_INVALID_PARAMETER; + + storage->XhciDevice = xdev; + storage->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber; + storage->BulkInAddress = bulkIn->bEndpointAddress; + storage->BulkOutAddress = bulkOut->bEndpointAddress; + storage->BulkInMaxPacket = bulkIn->wMaxPacketSize; + storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize; + storage->Tag = 0x1000u + storage->Device->SlotId; + + xdev->MassCandidate = true; + xdev->MassInterface = storage->InterfaceNumber; + xdev->MassBulkInAddress = storage->BulkInAddress; + xdev->MassBulkOutAddress = storage->BulkOutAddress; + xdev->MassBulkInEndpointId = XhciEndpointIdFromAddress(storage->BulkInAddress); + xdev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(storage->BulkOutAddress); + xdev->MassBulkInMaxPacket = storage->BulkInMaxPacket; + xdev->MassBulkOutMaxPacket = storage->BulkOutMaxPacket; + + KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, xdev); + if (!KSUCCESS(st)) goto fail; + + st = UsbCreatePipe(storage->Device, + UsbPipeBulk, + storage->BulkInAddress, + storage->BulkInMaxPacket, + bulkIn->bInterval, + &storage->BulkInPipe); + if (!KSUCCESS(st)) goto fail; + + st = UsbCreatePipe(storage->Device, + UsbPipeBulk, + storage->BulkOutAddress, + storage->BulkOutMaxPacket, + bulkOut->bInterval, + &storage->BulkOutPipe); + if (!KSUCCESS(st)) goto fail; + + st = UsbStorageInquiry(storage); + if (!KSUCCESS(st)) goto fail; + + st = UsbStorageTestUnitReady(storage); + if (!KSUCCESS(st)) { + DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", storage->Device->SlotId, st); + goto fail; + } + + st = UsbStorageReadCapacity(storage); + if (!KSUCCESS(st)) goto fail; + if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE) { + DbgPrint("usb-storage: unsupported block size slot=%u size=%u\n", + storage->Device->SlotId, storage->BlockSize); + st = STATUS_NOT_SUPPORTED; + goto fail; + } + + // The disk namespace publishes an arrival notification synchronously from + // IoRegisterDiskDevice(). Mark the storage FDO usable before publishing the + // disk so filesystem hotplug callbacks can read sector 0 immediately. + xdev->MassActive = true; + storage->Started = true; + device->PnpState = DevicePnpStarted; + if (storage->InterfacePdo) + storage->InterfacePdo->PnpState = DevicePnpStarted; + + st = UsbStorageRegisterDisk(storage); + if (!KSUCCESS(st)) { + storage->Started = false; + device->PnpState = DevicePnpNotStarted; + if (storage->InterfacePdo) + storage->InterfacePdo->PnpState = DevicePnpNotStarted; + goto fail; + } + + DbgPrint("usb-storage: FDO bound slot=%u if=%u disk-ready\n", + storage->Device->SlotId, storage->InterfaceNumber); + return STATUS_SUCCESS; + +fail: + xdev->MassCandidate = false; + xdev->MassActive = false; + if (storage->BulkInPipe) { + UsbDereferencePipe(storage->BulkInPipe); + storage->BulkInPipe = (USB_PIPE*)0; + } + if (storage->BulkOutPipe) { + UsbDereferencePipe(storage->BulkOutPipe); + storage->BulkOutPipe = (USB_PIPE*)0; + } + DbgPrint("usb-storage: start failed slot=%u if=%u st=0x%x\n", + storage->Device ? storage->Device->SlotId : 0, + storage->InterfaceNumber, + st); + return st; +} + +static KSTATUS UsbStorageDiskPnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) { + if (!device || !irp || !storage) + return STATUS_INVALID_PARAMETER; + + switch (minor) { + case IRP_MN_START_DEVICE: + device->PnpState = DevicePnpStarted; + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + + case IRP_MN_CANCEL_REMOVE_DEVICE: + device->PnpState = DevicePnpStarted; + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + + case IRP_MN_SURPRISE_REMOVAL: + storage->SurpriseRemoved = true; + if (storage->XhciDevice) { + storage->XhciDevice->MassSurpriseRemoved = true; + storage->XhciDevice->MassActive = false; + } + device->PnpState = DevicePnpSurpriseRemoved; + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + + case IRP_MN_REMOVE_DEVICE: + storage->Removed = true; + if (storage->XhciDevice) { + storage->XhciDevice->MassRemoved = true; + storage->XhciDevice->MassActive = false; + if (storage->XhciDevice->MassDeviceObject == device) + storage->XhciDevice->MassDeviceObject = (DEVICE_OBJECT*)0; + } + device->PnpState = DevicePnpDeleted; + if (storage->DiskDevice == device) + storage->DiskDevice = (DEVICE_OBJECT*)0; + IoDeleteDevice(device); + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + + default: + return STATUS_NOT_SUPPORTED; + } +} + +static KSTATUS UsbStorageInterfacePnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) { + if (!device || !irp || !storage) + return STATUS_INVALID_PARAMETER; + + KSTATUS st = STATUS_SUCCESS; + switch (minor) { + case IRP_MN_START_DEVICE: + st = UsbStorageStartDevice(device, storage); + if (!KSUCCESS(st)) { + IoCompleteRequest(irp, st, 0); + return STATUS_PENDING; + } + return IoForwardIrpToLower(device, irp); + + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_CANCEL_REMOVE_DEVICE: + if (!storage->SurpriseRemoved) + device->PnpState = DevicePnpStarted; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_SURPRISE_REMOVAL: + storage->SurpriseRemoved = true; + storage->Started = false; + if (storage->XhciDevice) { + storage->XhciDevice->MassSurpriseRemoved = true; + storage->XhciDevice->MassActive = false; + } + if (storage->DiskDevice) + (void)IoNotifyDiskSurpriseRemoved(storage->DiskDevice->u.Disk.DiskIndex); + device->PnpState = DevicePnpSurpriseRemoved; + return IoForwardIrpToLower(device, irp); + + case IRP_MN_REMOVE_DEVICE: { + storage->Removed = true; + storage->Started = false; + if (storage->DiskDevice) + (void)IoUnregisterDiskDevice(storage->DiskDevice->u.Disk.DiskIndex); + if (storage->BulkInPipe) { + UsbDereferencePipe(storage->BulkInPipe); + storage->BulkInPipe = (USB_PIPE*)0; + } + if (storage->BulkOutPipe) { + UsbDereferencePipe(storage->BulkOutPipe); + storage->BulkOutPipe = (USB_PIPE*)0; + } + if (storage->RemoveLockReady) { + IoReleaseRemoveLockAndWait(&storage->RemoveLock, irp); + IoDestroyRemoveLock(&storage->RemoveLock); + storage->RemoveLockReady = false; + } + device->PnpState = DevicePnpDeleted; + + KSTATUS lower = storage->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS; + if (storage->LowerDevice) { + IoDetachDevice(device); + storage->LowerDevice = (DEVICE_OBJECT*)0; + } + IoDeleteDevice(device); + if (lower == STATUS_NOT_SUPPORTED) { + IoCompleteRequest(irp, STATUS_SUCCESS, 0); + return STATUS_PENDING; + } + if (lower != STATUS_PENDING && !KSUCCESS(lower)) { + IoCompleteRequest(irp, lower, 0); + return STATUS_PENDING; + } + return lower; + } + + default: + return IoForwardIrpToLower(device, irp); + } +} + +KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) + return STATUS_INVALID_PARAMETER; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_PNP) + return STATUS_INVALID_PARAMETER; + + USB_STORAGE_EXTENSION* iface = UsbStorageFromInterfaceDevice(device); + if (iface) + return UsbStorageInterfacePnp(device, irp, iface, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction); + + USB_STORAGE_EXTENSION* disk = UsbStorageFromDiskDevice(device); + if (disk) + return UsbStorageDiskPnp(device, irp, disk, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction); + + return STATUS_INVALID_PARAMETER; +} bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) { (void)context; @@ -409,64 +779,69 @@ bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) { UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false); } - KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context) { + (void)pub; (void)context; - if (!pub || !pub->BusContext || !pub->DeviceContext) - return STATUS_INVALID_PARAMETER; - - XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext; - XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext; - if (dev->MassActive || dev->MassCandidate) - return STATUS_ALREADY_EXISTS; - - const IO_PUBLISHED_USB_ENDPOINT* bulkIn = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true); - const IO_PUBLISHED_USB_ENDPOINT* bulkOut = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false); - if (!bulkIn || !bulkOut) - return STATUS_NOT_FOUND; - - dev->MassCandidate = true; - dev->MassInterface = pub->u.Usb.InterfaceNumber; - dev->MassBulkInAddress = bulkIn->Address; - dev->MassBulkOutAddress = bulkOut->Address; - dev->MassBulkInEndpointId = XhciEndpointIdFromAddress(bulkIn->Address); - dev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(bulkOut->Address); - dev->MassBulkInMaxPacket = bulkIn->MaxPacketSize; - dev->MassBulkOutMaxPacket = bulkOut->MaxPacketSize; - - KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, dev); - if (!KSUCCESS(st)) goto fail; - - st = UsbStorageInquiry(xhci, dev); - if (!KSUCCESS(st)) goto fail; - - st = UsbStorageTestUnitReady(xhci, dev); - if (!KSUCCESS(st)) { - DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", dev->SlotId, st); - goto fail; - } - - st = UsbStorageReadCapacity(xhci, dev); - if (!KSUCCESS(st)) goto fail; - if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) { - DbgPrint("usb-storage: unsupported block size slot=%u block_size=%u\n", - dev->SlotId, dev->MassBlockSize); - st = STATUS_NOT_SUPPORTED; - goto fail; - } - - dev->MassActive = true; - st = UsbStorageRegisterDisk(dev); - if (!KSUCCESS(st)) goto fail; - - DbgPrint("usb-storage: bound slot=%u if=%u\n", dev->SlotId, dev->MassInterface); - return STATUS_SUCCESS; - -fail: - dev->MassCandidate = false; - dev->MassActive = false; - DbgPrint("usb-storage: bind failed slot=%u if=%u st=0x%x\n", - dev->SlotId, pub->u.Usb.InterfaceNumber, st); - return st; + return STATUS_NOT_SUPPORTED; } +KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) { + if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical) + return STATUS_INVALID_PARAMETER; + + USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension; + if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device) + return STATUS_INVALID_PARAMETER; + if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_MASS_STORAGE || + pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_MASS_SUBCLASS_SCSI || + pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_MASS_PROTOCOL_BOT) + return STATUS_INVALID_PARAMETER; + if (!UsbStorageFindBulkEndpoint(pdoExt, true) || !UsbStorageFindBulkEndpoint(pdoExt, false)) + return STATUS_NOT_FOUND; + + DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreateDevice(driver, sizeof(USB_STORAGE_EXTENSION), &fdo); + if (!KSUCCESS(st)) + return st; + (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-storage")); + + USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)fdo->DeviceExtension; + ext->Kind = UsbStorageExtensionInterface; + ext->Self = fdo; + ext->InterfacePdo = physicalDeviceObject; + ext->Device = pdoExt->Device; + ext->XhciDevice = (XHCI_DEVICE*)pdoExt->Device->HcdDeviceContext; + ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber; + ext->IoBusy = 0; + + st = IoInitializeRemoveLock(&ext->RemoveLock); + if (!KSUCCESS(st)) { + IoDeleteDevice(fdo); + return st; + } + ext->RemoveLockReady = true; + + ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject); + if (!ext->LowerDevice) { + IoDestroyRemoveLock(&ext->RemoveLock); + ext->RemoveLockReady = false; + IoDeleteDevice(fdo); + return STATUS_FAILURE; + } + + DbgPrint("usb-storage: FDO attached slot=%u if=%u\n", + ext->Device->SlotId, + pdoExt->InterfaceDescriptor.bInterfaceNumber); + return STATUS_SUCCESS; +} + +KSTATUS UsbInitializeStorageDriver(void) { + memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver)); + IoInitializeDriverObject(&gUsbStorageDriver, SYS_SV_LITERAL("usb-storage")); + gUsbStorageDriver.IdTable = &gUsbStorageIdTable; + gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice; + gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite; + gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite; + gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp; + return IoRegisterDriverObject(&gUsbStorageDriver); +} diff --git a/code/drivers/bus/usb/usb_xhci.c b/code/drivers/bus/usb/usb_xhci.c index dc66833..9d593c2 100644 --- a/code/drivers/bus/usb/usb_xhci.c +++ b/code/drivers/bus/usb/usb_xhci.c @@ -353,6 +353,31 @@ void XhciLogPortStatus(XHCI_CONTROLLER* xhci) { } } +void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) { + if (!xhci || !xhci->Op) + return; + + uint32_t powered = 0; + for (uint8_t port = 1; port <= xhci->MaxPorts; port++) { + uint32_t off = XhciPortReg(port); + uint32_t portsc = XhciRead32(xhci->Op, off); + uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK; + + if (!(portsc & XHCI_PORTSC_PP)) { + XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes); + powered++; + UsbTrace("usb: xHCI powered root port %u old-portsc=%08x\n", port, portsc); + } else if (changes) { + XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes); + } + } + + if (powered) { + UsbDelayUs(xhci->Hal, 20000u); + DbgPrint("usb: xHCI powered %u previously-off root port(s)\n", powered); + } +} + void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) { if (!xhci->Runtime) return; @@ -369,7 +394,71 @@ void XhciUpdateErdp(XHCI_CONTROLLER* xhci) { } +static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) { + for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { + if (w->CommandPhys && w->CommandPhys == commandPhys) + return w; + } + return (XHCI_WAIT_TARGET*)0; +} + + +static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci, + uint64_t transferPhys, + uint8_t slotId, + uint8_t endpointId) { + for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { + if (w->TransferPhys && + w->TransferPhys == transferPhys && + w->TransferSlotId == slotId && + w->TransferEndpointId == endpointId) { + return w; + } + } + return (XHCI_WAIT_TARGET*)0; +} + + +static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci, + uint64_t commandPhys, + uint32_t completionCode, + uint8_t slotId) { + uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); + XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys); + if (w) { + w->Command.CompletionCode = completionCode; + w->Command.SlotId = slotId; + XhciBarrier(); + w->CommandMatched = true; + } + KeReleaseSpinlock(&xhci->WaitLock, flags); + return w; +} + + +static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci, + uint64_t transferPhys, + uint8_t slotId, + uint8_t endpointId, + uint32_t completionCode, + uint32_t residue) { + uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); + XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId); + if (w) { + w->Transfer.CompletionCode = completionCode; + w->Transfer.SlotId = slotId; + w->Transfer.EndpointId = endpointId; + w->Transfer.Residue = residue; + XhciBarrier(); + w->TransferMatched = true; + } + KeReleaseSpinlock(&xhci->WaitLock, flags); + return w; +} + + bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { + (void)target; bool processed = false; for (;;) { @@ -384,41 +473,28 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { uint32_t cc = XhciCompletionCode(ev); uint64_t param = ((uint64_t)ev->ParameterHi << 32) | ev->ParameterLo; - XHCI_WAIT_TARGET* waiter = target ? target : (XHCI_WAIT_TARGET*)xhci->ActiveWait; - if (type == XHCI_TRB_TYPE_CMD_DONE) { uint8_t slotId = XhciEventSlotId(ev); UsbTrace("usb: xHCI command completion ptr=0x%016llX cc=%u slot=%u\n", (unsigned long long)param, cc, slotId); - if (waiter && waiter->CommandPhys && param == waiter->CommandPhys) { - waiter->CommandMatched = true; - waiter->Command.CompletionCode = cc; - waiter->Command.SlotId = slotId; - } + (void)XhciClaimCommandWaiter(xhci, param, cc, slotId); } else if (type == XHCI_TRB_TYPE_TRANSFER) { uint8_t slotId = XhciEventSlotId(ev); uint8_t epId = XhciEventEndpointId(ev); uint32_t residue = XhciTransferResidue(ev); - bool claimedByWaiter = false; - bool targetMatches = waiter && waiter->TransferPhys && param == waiter->TransferPhys && - slotId == waiter->TransferSlotId && epId == waiter->TransferEndpointId; if (cc != XHCI_CC_SUCCESS) { UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n", (unsigned long long)param, cc, slotId, epId, residue); } - if (targetMatches) { - waiter->TransferMatched = true; - waiter->Transfer.CompletionCode = cc; - waiter->Transfer.SlotId = slotId; - waiter->Transfer.EndpointId = epId; - waiter->Transfer.Residue = residue; - claimedByWaiter = true; - } - if (!claimedByWaiter) + + XHCI_WAIT_TARGET* waiter = XhciClaimTransferWaiter(xhci, param, slotId, epId, cc, residue); + if (!waiter) XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue); } else if (type == XHCI_TRB_TYPE_PORT_SC) { uint32_t portId = ev->ParameterLo >> 24; UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc); + if (portId && portId <= xhci->MaxPorts) + UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)portId); } else { DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n", type, cc, (unsigned long long)param, ev->Status, ev->Control); @@ -454,22 +530,53 @@ bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) { } -KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, - uint32_t timeoutUs) { +void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { + if (!xhci || !target || target->Registered) + return; + + uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); + target->Next = xhci->WaiterList; + target->Registered = true; + xhci->WaiterList = target; + KeReleaseSpinlock(&xhci->WaitLock, flags); +} + + +void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { + if (!xhci || !target || !target->Registered) + return; + + uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); + XHCI_WAIT_TARGET** link = &xhci->WaiterList; + while (*link) { + if (*link == target) { + *link = target->Next; + target->Next = (XHCI_WAIT_TARGET*)0; + target->Registered = false; + break; + } + link = &(*link)->Next; + } + KeReleaseSpinlock(&xhci->WaitLock, flags); +} + + +KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, + uint32_t timeoutUs) { if (!xhci || !target) return STATUS_INVALID_PARAMETER; - xhci->ActiveWait = target; for (uint32_t i = 0; i < timeoutUs; i++) { if (XhciWaitTargetComplete(target)) { - xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0; + XhciUnregisterWaitTarget(xhci, target); return STATUS_SUCCESS; } // The IRQ/DPC path normally owns the event ring after enumeration, but // synchronous storage/control waits must not depend exclusively on an - // interrupt being delivered. Opportunistically drain the ring from the - // waiter too; EventLock serializes this with the DPC. + // interrupt being delivered. Opportunistically drain the ring too; + // EventLock serializes this with the DPC while WaitLock protects waiter + // registration/matching. if (!xhci->InterruptsEnabled || (i & 0x7u) == 0) { XhciDrainEventRingLocked(xhci, target); } else if ((XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_EINT) && (i & 0x3Fu) == 0) { @@ -479,11 +586,20 @@ KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, UsbDelayUs(xhci->Hal, 1); } - xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0; + XhciUnregisterWaitTarget(xhci, target); return STATUS_TIMEOUT; } +KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, + uint32_t timeoutUs) { + if (!xhci || !target) + return STATUS_INVALID_PARAMETER; + XhciRegisterWaitTarget(xhci, target); + return XhciWaitForArmedTarget(xhci, target, timeoutUs); +} + + void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { (void)dpc; @@ -695,13 +811,26 @@ uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) { } -KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, - uint64_t parameter, - uint32_t status, - uint32_t controlWithoutCycle, - XHCI_COMMAND_RESULT* outResult) { - if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) - return STATUS_QUEUE_FULL; +static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, + uint64_t parameter, + uint32_t status, + uint32_t controlWithoutCycle, + uint64_t* outPhys) { + if (!xhci || !xhci->CommandRing) + return STATUS_INVALID_PARAMETER; + + if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) { + XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; + link->ParameterLo = (uint32_t)xhci->CommandRingPhys; + link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); + link->Status = 0; + XhciBarrier(); + link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_LINK_TC | + xhci->CommandCycle; + xhci->CommandEnqueue = 0; + xhci->CommandCycle ^= 1u; + } uint32_t index = xhci->CommandEnqueue++; XHCI_TRB* trb = &xhci->CommandRing[index]; @@ -713,15 +842,38 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, XhciBarrier(); trb->Control = controlWithoutCycle | xhci->CommandCycle; + if (outPhys) *outPhys = trbPhys; + return STATUS_SUCCESS; +} + + +KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, + uint64_t parameter, + uint32_t status, + uint32_t controlWithoutCycle, + XHCI_COMMAND_RESULT* outResult) { + if (!xhci) + return STATUS_INVALID_PARAMETER; + XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); + + uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock); + uint64_t trbPhys = 0; + KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys); + if (!KSUCCESS(st)) { + KeReleaseSpinlock(&xhci->CommandLock, cmdFlags); + return st; + } + target.CommandPhys = trbPhys; - xhci->ActiveWait = ⌖ + XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); XhciWrite32(xhci->Doorbells, 0, 0); // host-controller doorbell, command ring + KeReleaseSpinlock(&xhci->CommandLock, cmdFlags); - KSTATUS waitStatus = XhciWaitForTarget(xhci, &target, 500000u); + KSTATUS waitStatus = XhciWaitForArmedTarget(xhci, &target, 500000u); if (KSUCCESS(waitStatus) && target.CommandMatched) { if (outResult) *outResult = target.Command; return (target.Command.CompletionCode == XHCI_CC_SUCCESS) @@ -781,6 +933,28 @@ KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) { } + +KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) { + if (!xhci || slotId == 0) + return STATUS_INVALID_PARAMETER; + + XHCI_COMMAND_RESULT res; + memset(&res, 0, sizeof(res)); + KSTATUS st = XhciSubmitCommand(xhci, + 0, + 0, + (XHCI_TRB_TYPE_DISABLE_SLOT << XHCI_TRB_TYPE_SHIFT) | + ((uint32_t)slotId << 24), + &res); + if (!KSUCCESS(st)) + return st; + if (res.CompletionCode != XHCI_CC_SUCCESS) { + DbgPrint("usb: Disable Slot slot=%u failed cc=%u\n", slotId, res.CompletionCode); + return STATUS_DEVICE_ERROR; + } + return STATUS_SUCCESS; +} + KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); @@ -970,12 +1144,22 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { uint32_t maxPacket = XhciEpMaxPacket(dev->Speed); dev->MaxPacketSize0 = (uint16_t)maxPacket; - slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed); + uint8_t rootPort = dev->Path.RootPort ? dev->Path.RootPort : dev->Port; + uint32_t routeString = dev->Path.RouteString & 0x000fffffu; + + slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed) | routeString; // Slot context dword 1 owns Root Hub Port Number bits 23:16. // Putting the port number in dword 2 makes Address Device fail with a // context/TRB error on QEMU xHCI. - slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port); + slot[1] = XHCI_SLOT_ROOT_PORT(rootPort); slot[2] = 0; + if (dev->Path.BehindHub && dev->Speed <= 2u && dev->Path.ParentHubSlot) { + // TT Hub Slot ID bits 7:0 and TT Port Number bits 15:8. + // This is required for low/full-speed devices behind high-speed hubs; + // harmless for older full-speed hub paths on the controllers we target. + slot[2] = (uint32_t)dev->Path.ParentHubSlot | + ((uint32_t)dev->Path.ParentHubPort << 8); + } slot[3] = 0; ep0[0] = 0; @@ -999,8 +1183,21 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys) { - if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) - return STATUS_QUEUE_FULL; + if (!dev || !dev->Ep0Ring) + return STATUS_INVALID_PARAMETER; + + if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) { + XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; + link->ParameterLo = (uint32_t)dev->Ep0RingPhys; + link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); + link->Status = 0; + XhciBarrier(); + link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_LINK_TC | + dev->Ep0Cycle; + dev->Ep0Enqueue = 0; + dev->Ep0Cycle ^= 1u; + } uint32_t index = dev->Ep0Enqueue++; XHCI_TRB* trb = &dev->Ep0Ring[index]; @@ -1017,11 +1214,12 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, } -KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, - const USB_SETUP_PACKET* setup, - void* buffer, - PHYSICAL_ADDRESS bufferPhys, - uint16_t length) { +KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + const USB_SETUP_PACKET* setup, + void* buffer, + PHYSICAL_ADDRESS bufferPhys, + uint16_t length, + uint32_t timeoutUs) { if (!buffer || length == 0 || length > PAGE_SIZE) return STATUS_INVALID_PARAMETER; @@ -1057,18 +1255,21 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, &statusTrbPhys); if (!KSUCCESS(st)) return st; - XhciBarrier(); - XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0 - XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); target.TransferPhys = statusTrbPhys; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; + XhciRegisterWaitTarget(xhci, &target); - st = XhciWaitForTarget(xhci, &target, 1000000u); + XhciBarrier(); + XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0 + + if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US; + st = XhciWaitForArmedTarget(xhci, &target, timeoutUs); if (!KSUCCESS(st)) { - DbgPrint("usb: xHCI control IN timeout slot=%u len=%u\n", dev->SlotId, length); + DbgPrint("usb: xHCI control IN timeout slot=%u len=%u timeout=%uus\n", + dev->SlotId, length, timeoutUs); return st; } if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS) @@ -1082,6 +1283,16 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, } +KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + const USB_SETUP_PACKET* setup, + void* buffer, + PHYSICAL_ADDRESS bufferPhys, + uint16_t length) { + return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length, + USB_XHCI_CONTROL_TIMEOUT_US); +} + + KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup) { uint64_t setupValue = UsbSetupPacketToU64(setup); @@ -1106,16 +1317,17 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, &statusTrbPhys); if (!KSUCCESS(st)) return st; - XhciBarrier(); - XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); - XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); target.TransferPhys = statusTrbPhys; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; + XhciRegisterWaitTarget(xhci, &target); - st = XhciWaitForTarget(xhci, &target, 1000000u); + XhciBarrier(); + XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); + + st = XhciWaitForArmedTarget(xhci, &target, 1000000u); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI control no-data timeout slot=%u req=%u value=%04x\n", dev->SlotId, setup->bRequest, setup->wValue); @@ -1132,8 +1344,9 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, } -KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, - USB_DEVICE_DESCRIPTOR* outDesc) { +KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + USB_DEVICE_DESCRIPTOR* outDesc, + uint32_t timeoutUs) { USB_SETUP_PACKET setup; setup.bmRequestType = USB_DIR_IN; setup.bRequest = USB_REQ_GET_DESCRIPTOR; @@ -1141,10 +1354,12 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, setup.wIndex = 0; setup.wLength = sizeof(USB_DEVICE_DESCRIPTOR); - KSTATUS st = XhciControlIn(xhci, dev, &setup, - dev->DescriptorBuffer, - dev->DescriptorPhys, - sizeof(USB_DEVICE_DESCRIPTOR)); + if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US; + KSTATUS st = XhciControlInTimeout(xhci, dev, &setup, + dev->DescriptorBuffer, + dev->DescriptorPhys, + sizeof(USB_DEVICE_DESCRIPTOR), + timeoutUs); if (!KSUCCESS(st)) return st; memcpy(outDesc, dev->DescriptorBuffer, sizeof(*outDesc)); @@ -1159,6 +1374,13 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, } +KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + USB_DEVICE_DESCRIPTOR* outDesc) { + return XhciGetDeviceDescriptorTimeout(xhci, dev, outDesc, + USB_XHCI_CONTROL_TIMEOUT_US); +} + + KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, @@ -1331,11 +1553,11 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, target.TransferSlotId = dev->SlotId; target.TransferEndpointId = endpointId; - xhci->ActiveWait = ⌖ + XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, endpointId); - st = XhciWaitForTarget(xhci, &target, 1000000u); + st = XhciWaitForArmedTarget(xhci, &target, 1000000u); if (!KSUCCESS(st)) { DbgPrint("usb-storage: bulk transfer timeout slot=%u ep=%u len=%u\n", dev->SlotId, endpointId, length); @@ -1356,6 +1578,300 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, } + + +KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) { + if (!xhci || !dev || !pipe || !pipe->Open) + return STATUS_INVALID_PARAMETER; + + uint8_t dci = pipe->EndpointId; + if (dci < 2 || dci > 31) + return STATUS_INVALID_PARAMETER; + + if (pipe->HcdPipeContext) { + XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; + return ctx->Configured ? STATUS_SUCCESS : STATUS_DEVICE_ERROR; + } + + XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)MmAllocatePool(sizeof(XHCI_PIPE_CONTEXT)); + if (!ctx) return STATUS_NO_MEMORY; + memset(ctx, 0, sizeof(*ctx)); + + ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys); + if (!ctx->Ring) { + MmFreePool(ctx); + return STATUS_NO_MEMORY; + } + ctx->Enqueue = 0; + ctx->Cycle = 1; + ctx->EndpointId = dci; + ctx->Type = pipe->Type; + XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle); + + memset(dev->InputContext, 0, PAGE_SIZE); + uint32_t* icc = XhciInputControlContext(xhci, dev); + uint32_t* slot = XhciInputSlotContext(xhci, dev); + uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci); + + icc[0] = 0; + icc[1] = (1u << 0) | (1u << dci); + + slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed); + slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port); + slot[2] = 0; + slot[3] = 0; + + uint16_t maxPacket = pipe->MaxPacketSize & 0x07FFu; + if (maxPacket == 0) maxPacket = 8u; + + uint32_t endpointType = 0; + uint8_t interval = 0; + if (pipe->Type == UsbPipeInterrupt && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) { + endpointType = XHCI_EP_TYPE_INT_IN; + interval = XhciInterruptInterval(dev->Speed, pipe->Interval); + } else if (pipe->Type == UsbPipeBulk && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) { + endpointType = XHCI_EP_TYPE_BULK_IN; + } else if (pipe->Type == UsbPipeBulk) { + endpointType = XHCI_EP_TYPE_BULK_OUT; + } else { + MmFreePool(ctx); + return STATUS_NOT_SUPPORTED; + } + + ep[0] = (uint32_t)interval << 16; + ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16); + ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS); + ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32); + ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16); + + XHCI_COMMAND_RESULT result; + memset(&result, 0, sizeof(result)); + KSTATUS st = XhciSubmitCommand( + xhci, + dev->InputContextPhys, + 0, + (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | + ((uint32_t)dev->SlotId << 24), + &result); + if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) { + DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n", + dev->SlotId, dci, pipe->Type, st, result.CompletionCode); + MmFreePool(ctx); + return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st; + } + + ctx->Configured = true; + pipe->HcdPipeContext = ctx; + dev->EndpointPipes[dci] = pipe; + UsbReferencePipe(pipe); + return STATUS_SUCCESS; +} + +static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { + if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) + return STATUS_INVALID_PARAMETER; + if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) + return STATUS_NOT_SUPPORTED; + + KSTATUS st = XhciConfigurePipeEndpoint(xhci, dev, urb->Pipe); + if (!KSUCCESS(st)) return st; + + XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext; + if (!ctx || !ctx->Configured) + return STATUS_DEVICE_ERROR; + if (ctx->ActiveTransfer) + return STATUS_QUEUE_FULL; + + uint64_t trbPhys = 0; + st = XhciRingAppend(ctx->Ring, + ctx->RingPhys, + XHCI_PIPE_RING_TRBS, + &ctx->Enqueue, + &ctx->Cycle, + urb->BufferPhys, + urb->Length, + (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_ISP | + XHCI_TRB_IOC, + &trbPhys); + if (!KSUCCESS(st)) return st; + + ctx->ActiveTransfer = true; + ctx->ActiveTrbPhys = trbPhys; + ctx->ActiveUrb = urb; + urb->Pending = true; + urb->Cancelled = false; + urb->Status = STATUS_PENDING; + urb->ActualLength = 0; + + XhciBarrier(); + XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, ctx->EndpointId); + return STATUS_PENDING; +} + +void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, + uint32_t completionCode, uint8_t slotId, + uint8_t endpointId, uint32_t residue) { + if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32) + return; + + XHCI_DEVICE* dev = &xhci->Devices[slotId]; + USB_PIPE* pipe = dev->EndpointPipes[endpointId]; + if (!pipe || !pipe->HcdPipeContext) + return; + + XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; + if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys) + return; + + USB_URB* urb = ctx->ActiveUrb; + ctx->ActiveTransfer = false; + ctx->ActiveTrbPhys = 0; + ctx->ActiveUrb = (USB_URB*)0; + + if (!urb) + return; + + KSTATUS st = STATUS_SUCCESS; + if (urb->Cancelled) { + st = STATUS_CANCELLED; + } else if (completionCode != XHCI_CC_SUCCESS) { + st = STATUS_DEVICE_ERROR; + } + + uint32_t actual = 0; + if (KSUCCESS(st)) { + actual = (residue <= urb->Length) ? (urb->Length - residue) : 0; + } + + urb->Pending = false; + urb->Status = st; + urb->ActualLength = actual; + + IRP* completeIrp = urb->Irp; + urb->Irp = (IRP*)0; + + if (urb->Complete) + urb->Complete(urb, urb->CompleteContext); + + if (completeIrp) + IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0); +} + +static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { + if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) + return STATUS_INVALID_PARAMETER; + + uint8_t endpointId = urb->Pipe->EndpointId; + XHCI_TRB* ring = (XHCI_TRB*)0; + PHYSICAL_ADDRESS ringPhys = 0; + uint32_t* enqueue = (uint32_t*)0; + uint8_t* cycle = (uint8_t*)0; + + if (endpointId == dev->MassBulkInEndpointId) { + ring = dev->MassInRing; + ringPhys = dev->MassInRingPhys; + enqueue = &dev->MassInEnqueue; + cycle = &dev->MassInCycle; + } else if (endpointId == dev->MassBulkOutEndpointId) { + ring = dev->MassOutRing; + ringPhys = dev->MassOutRingPhys; + enqueue = &dev->MassOutEnqueue; + cycle = &dev->MassOutCycle; + } else { + return STATUS_NOT_SUPPORTED; + } + + uint32_t actual = 0; + KSTATUS st = XhciBulkTransfer(xhci, + dev, + endpointId, + ring, + ringPhys, + enqueue, + cycle, + urb->BufferPhys, + urb->Length, + &actual); + urb->ActualLength = KSUCCESS(st) ? actual : 0; + return st; +} + +static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) { + if (!hcd || !urb || !urb->Device) + return STATUS_INVALID_PARAMETER; + + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext; + XHCI_DEVICE* dev = (XHCI_DEVICE*)urb->Device->HcdDeviceContext; + if (!xhci || !dev || !dev->Allocated) + return STATUS_INVALID_PARAMETER; + + KSTATUS st; + switch (urb->Function) { + case UrbFunctionControlTransfer: + if (urb->Length == 0) { + st = XhciControlNoData(xhci, dev, &urb->Setup); + urb->ActualLength = 0; + } else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) { + st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length); + urb->ActualLength = KSUCCESS(st) ? urb->Length : 0; + } else { + st = STATUS_NOT_SUPPORTED; + urb->ActualLength = 0; + } + break; + + case UrbFunctionBulkTransfer: + st = XhciHcdSubmitBulkUrb(xhci, dev, urb); + break; + + case UrbFunctionInterruptTransfer: + st = XhciHcdSubmitInterruptUrb(xhci, dev, urb); + break; + + default: + st = STATUS_NOT_SUPPORTED; + urb->ActualLength = 0; + break; + } + + urb->Status = st; + return st; +} + +static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) { + (void)hcd; + if (!urb) return STATUS_INVALID_PARAMETER; + urb->Cancelled = true; + if (!urb->Pending) + urb->Status = STATUS_CANCELLED; + return STATUS_SUCCESS; +} + +static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) { + if (!hcd) return (void*)0; + return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys); +} + +static KSTATUS XhciHcdResetRootPort(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed) { + if (!hcd) return STATUS_INVALID_PARAMETER; + return XhciResetPort((XHCI_CONTROLLER*)hcd->ControllerContext, port, outSpeed); +} + +static KSTATUS XhciHcdClearRootPortChanges(USB_HCD* hcd, uint8_t port) { + if (!hcd) return STATUS_INVALID_PARAMETER; + XhciClearPortChanges((XHCI_CONTROLLER*)hcd->ControllerContext, port); + return STATUS_SUCCESS; +} + +const USB_HCD_OPERATIONS gXhciHcdOperations = { + XhciHcdSubmitUrb, + XhciHcdCancelUrb, + XhciHcdResetRootPort, + XhciHcdClearRootPortChanges, + XhciHcdAllocateDma, +}; + KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device) { @@ -1438,6 +1954,13 @@ KSTATUS XhciInitialise(const HAL_API* hal, return STATUS_DEVICE_ERROR; } + UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts); + KSTATUS hcdPdoStatus = UsbCreateControllerPdo(&xhci->Hcd, vendor, device); + if (!KSUCCESS(hcdPdoStatus)) { + DbgPrint("usb: xHCI controller PDO creation failed st=0x%x\n", hcdPdoStatus); + return hcdPdoStatus; + } + uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE + (uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE; uint32_t needBytes = portEnd; @@ -1489,12 +2012,13 @@ KSTATUS XhciInitialise(const HAL_API* hal, st = XhciSubmitNoopCommand(xhci); if (!KSUCCESS(st)) return st; + XhciPowerRootPorts(xhci); XhciLogPortStatus(xhci); - XhciEnumerateRootPorts(xhci); - if (xhci->HidDeviceCount > 0) { - st = XhciStartHidInterrupts(xhci); - if (!KSUCCESS(st)) return st; + st = UsbCreateRootHubPdo(&xhci->Hcd); + if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) { + DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st); + return st; } KeIrqRegister(xhci->Irq, XhciIsr); @@ -1504,8 +2028,14 @@ KSTATUS XhciInitialise(const HAL_API* hal, DbgPrint("usb: xHCI ISR registered on IRQ %u%s; event ring handled by DPC\n", xhci->Irq, xhci->UsingMsi ? "(msi)" : ""); + st = UsbHubScheduleRootScan(&xhci->Hcd); + if (!KSUCCESS(st)) + DbgPrint("usb: root hub async scan scheduling failed st=0x%x\n", st); + else + DbgPrint("usb: root hub initial scan queued asynchronously\n"); + gHaveXhci = true; - DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage is next\n", + DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n", xhci->HidDeviceCount); return STATUS_SUCCESS; } diff --git a/code/drivers/fs/fat32/fat32.c b/code/drivers/fs/fat32/fat32.c index 87cda9e..8d13913 100644 --- a/code/drivers/fs/fat32/fat32.c +++ b/code/drivers/fs/fat32/fat32.c @@ -1612,7 +1612,10 @@ static void Fat32AddCandidate(FAT32_VOL* probe, static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex, FAT32_PARTITION_CANDIDATE* out, - uint32_t maxCount) { + uint32_t maxCount, + bool* outMediaReadable) { + if (outMediaReadable) + *outMediaReadable = false; if (!out || maxCount == 0) return 0; FAT32_VOL probe; @@ -1623,6 +1626,8 @@ static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex, uint8_t mbr[512]; if (!KSUCCESS(ReadSectors(&probe, 0, 1, mbr))) return 0; + if (outMediaReadable) + *outMediaReadable = true; // A disk image may be a raw FAT32 volume with no partition table. Fat32AddCandidate(&probe, out, &count, maxCount, 0, IoDiskSectorCount(diskIndex)); @@ -1734,6 +1739,7 @@ typedef struct FAT32_ASYNC_RW { size_t CurrentCopy; IRP Irp; + DEVICE_OBJECT* CurrentDevice; IO_FS_ASYNC_COMPLETE Complete; void* CompleteContext; } FAT32_ASYNC_RW; @@ -1811,6 +1817,10 @@ static void Fat32AsyncQueueContinuation(FAT32_ASYNC_RW* ctx) { static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t information) { if (!ctx) return; + if (ctx->CurrentDevice) { + IoDereferenceDeviceObject(ctx->CurrentDevice); + ctx->CurrentDevice = (DEVICE_OBJECT*)0; + } if (ctx->RunBuf) MmFreePool(ctx->RunBuf); IO_FS_ASYNC_COMPLETE complete = ctx->Complete; void* completeContext = ctx->CompleteContext; @@ -1821,6 +1831,10 @@ static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t informa static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) { FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx; if (!ctx) return; + if (ctx->CurrentDevice) { + IoDereferenceDeviceObject(ctx->CurrentDevice); + ctx->CurrentDevice = (DEVICE_OBJECT*)0; + } if (!KSUCCESS(irp->Status)) { Fat32AsyncFinish(ctx, irp->Status, ctx->Done); @@ -1872,17 +1886,25 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) { ctx->CurrentOffset = ctx->OffsetInCluster; ctx->CurrentCopy = copy; - memset(&ctx->Irp, 0, sizeof(ctx->Irp)); - ctx->Irp.DriveIndex = v->DriveIndex; - ctx->Irp.Lba = ClusterToLba(v, runStart); - ctx->Irp.SectorCount = runSectors; - ctx->Irp.Buffer = ctx->RunBuf; + ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex); + if (!ctx->CurrentDevice) { + Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done); + return; + } + + IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_READ, + ClusterToLba(v, runStart) * 512ull, + runSectors * 512u, + ctx->RunBuf); ctx->Irp.CompletionFn = Fat32AsyncReadIrpComplete; ctx->Irp.CompletionCtx = ctx; - KSTATUS s = IoSubmitIrp(&ctx->Irp); - if (s != STATUS_PENDING) + KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp); + if (s != STATUS_PENDING) { + IoDereferenceDeviceObject(ctx->CurrentDevice); + ctx->CurrentDevice = (DEVICE_OBJECT*)0; Fat32AsyncFinish(ctx, s, ctx->Done); + } } KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, @@ -1949,6 +1971,10 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) { FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx; if (!ctx) return; + if (ctx->CurrentDevice) { + IoDereferenceDeviceObject(ctx->CurrentDevice); + ctx->CurrentDevice = (DEVICE_OBJECT*)0; + } if (!KSUCCESS(irp->Status)) { Fat32AsyncFinish(ctx, irp->Status, ctx->Done); @@ -2024,18 +2050,25 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) { ctx->CurrentOffset = ctx->OffsetInCluster; ctx->CurrentCopy = copy; - memset(&ctx->Irp, 0, sizeof(ctx->Irp)); - ctx->Irp.DriveIndex = v->DriveIndex; - ctx->Irp.Flags = IRP_FLAG_WRITE; - ctx->Irp.Lba = ClusterToLba(v, runStart); - ctx->Irp.SectorCount = runSectors; - ctx->Irp.Buffer = ctx->RunBuf; + ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex); + if (!ctx->CurrentDevice) { + Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done); + return; + } + + IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_WRITE, + ClusterToLba(v, runStart) * 512ull, + runSectors * 512u, + ctx->RunBuf); ctx->Irp.CompletionFn = Fat32AsyncWriteIrpComplete; ctx->Irp.CompletionCtx = ctx; - KSTATUS s = IoSubmitIrp(&ctx->Irp); - if (s != STATUS_PENDING) + KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp); + if (s != STATUS_PENDING) { + IoDereferenceDeviceObject(ctx->CurrentDevice); + ctx->CurrentDevice = (DEVICE_OBJECT*)0; Fat32AsyncFinish(ctx, s, ctx->Done); + } } KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, @@ -2124,31 +2157,55 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, static FAT32_VOL gVols[FAT32_MAX_VOLUMES]; static IO_FILESYSTEM gFs[FAT32_MAX_VOLUMES]; - +static bool gVolumeMounted[FAT32_MAX_VOLUMES]; static uint32_t gMountedCount; static bool gDiskScanned[FAT32_MAX_VOLUMES]; +static int Fat32FindFreeVolumeIndex(void) { + for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) { + if (!gVolumeMounted[i] && !IoFilesystemPresent(i)) + return i; + } + return -1; +} + static uint32_t Fat32TryMountDisk(uint8_t disk) { if (disk >= FAT32_MAX_VOLUMES) return 0; if (gDiskScanned[disk]) return 0; - gDiskScanned[disk] = true; if (!IoDiskDevicePresent(disk)) return 0; uint32_t mountedHere = 0; FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK]; - uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK); + bool mediaReadable = false; + uint32_t partCount = Fat32EnumerateDiskVolumes(disk, + parts, + FAT32_MAX_PARTITIONS_PER_DISK, + &mediaReadable); + if (!mediaReadable) { + DbgPrint("FAT32: disk %u: media probe deferred; sector 0 unreadable\n", disk); + return 0; + } + + // Only remember a completed probe. A transient hotplug/readiness failure + // must not permanently suppress a later mount attempt for the same disk + // index after a device is unplugged and replugged. + gDiskScanned[disk] = true; + if (!partCount) { DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk); return 0; } - for (uint32_t part = 0; part < partCount && gMountedCount < FAT32_MAX_VOLUMES; part++) { - uint8_t volumeIndex = (uint8_t)gMountedCount; + for (uint32_t part = 0; part < partCount; part++) { + int freeVolume = Fat32FindFreeVolumeIndex(); + if (freeVolume < 0) + break; + uint8_t volumeIndex = (uint8_t)freeVolume; FAT32_VOL* v = &gVols[volumeIndex]; memset(v, 0, sizeof(*v)); v->DriveIndex = disk; @@ -2192,6 +2249,7 @@ static uint32_t Fat32TryMountDisk(uint8_t disk) { KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams); if (KSUCCESS(status)) { + gVolumeMounted[volumeIndex] = true; gMountedCount++; mountedHere++; DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n", @@ -2222,6 +2280,60 @@ static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* co } } +static void Fat32UnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) { + uint32_t removed = 0; + + for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) { + if (!gVolumeMounted[i]) + continue; + if (gVols[i].DriveIndex != diskIndex) + continue; + + // Surprise removal means the backing device may already be gone; do not + // attempt writeback here. Drop cached FAT sectors instead. + FatCacheInvalidate(); + (void)IoUnregisterFilesystem(i); + memset(&gVols[i], 0, sizeof(gVols[i])); + memset(&gFs[i], 0, sizeof(gFs[i])); + gVolumeMounted[i] = false; + if (gMountedCount) + gMountedCount--; + removed++; + } + + if (diskIndex < FAT32_MAX_VOLUMES) + gDiskScanned[diskIndex] = false; + + if (removed) { + DbgPrint("FAT32: disk %u %s; unmounted %u volume(s); total=%u\n", + diskIndex, + change == IoDeviceSurpriseRemoved ? "surprise removed" : "removed", + removed, + gMountedCount); + } +} + +static void Fat32DiskChange(uint8_t driveIndex, + DEVICE_OBJECT* device, + IO_DEVICE_CHANGE change, + void* context) { + (void)device; + (void)context; + + switch (change) { + case IoDeviceArrived: + Fat32DiskArrival(driveIndex, device, context); + break; + case IoDeviceSurpriseRemoved: + case IoDeviceRemoved: + Fat32UnmountDisk(driveIndex, change); + break; + case IoDeviceQueryRemove: + default: + break; + } +} + KSTATUS DriverEntry(const HAL_API *hal) { DbgPrint("FAT32: Init\n"); if (!hal) return STATUS_INVALID_PARAMETER; @@ -2231,9 +2343,14 @@ KSTATUS DriverEntry(const HAL_API *hal) { gMountedCount = 0; memset(gDiskScanned, 0, sizeof(gDiskScanned)); - KSTATUS cbStatus = IoRegisterDiskArrivalCallback(Fat32DiskArrival, (void*)0); + KSTATUS cbStatus = IoRegisterDiskChangeCallback(Fat32DiskChange, (void*)0); if (!KSUCCESS(cbStatus)) { - DbgPrint("FAT32: disk-arrival callback registration failed st=0x%x\n", cbStatus); + DbgPrint("FAT32: disk-change callback registration failed st=0x%x\n", cbStatus); + } + + cbStatus = IoEnumerateDiskDevices(Fat32DiskArrival, (void*)0); + if (!KSUCCESS(cbStatus)) { + DbgPrint("FAT32: initial disk enumeration failed st=0x%x\n", cbStatus); } if (!gMountedCount) { diff --git a/code/drivers/storage/ahci/ahci.c b/code/drivers/storage/ahci/ahci.c index 940509a..8917a8e 100644 --- a/code/drivers/storage/ahci/ahci.c +++ b/code/drivers/storage/ahci/ahci.c @@ -382,6 +382,28 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba, // Async IRP dispatch // --------------------------------------------------------------------------- + +static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba, + uint32_t* outSectors, void** outBuffer) { + if (!irp || !outWrite || !outLba || !outSectors || !outBuffer) + return STATUS_INVALID_PARAMETER; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) return STATUS_INVALID_PARAMETER; + if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE) + return STATUS_INVALID_PARAMETER; + if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) || + (ios->Parameters.ReadWrite.Length & 511u)) + return STATUS_INVALID_PARAMETER; + if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0) + return STATUS_INVALID_PARAMETER; + + *outWrite = (ios->MajorFunction == IRP_MJ_WRITE); + *outLba = ios->Parameters.ReadWrite.ByteOffset >> 9; + *outSectors = ios->Parameters.ReadWrite.Length >> 9; + *outBuffer = irp->UserBuffer; + return STATUS_SUCCESS; +} + // AhciDispatchReadWrite - DEVICE_OBJECT dispatch entry for disk IRPs. // Accepted IRPs are queued to the per-drive worker and completed later with // IoCompleteRequest(). Rejected IRPs are returned synchronously and are not @@ -390,8 +412,22 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { if (!device || !irp) return STATUS_INVALID_PARAMETER; AHCI_DRIVE *drv = (AHCI_DRIVE *)device->DeviceExtension; if (!drv) return STATUS_INVALID_PARAMETER; - if (irp->SectorCount == 0 || !irp->Buffer) return STATUS_INVALID_PARAMETER; - if (irp->Lba + irp->SectorCount > drv->sectorCount) return STATUS_INVALID_PARAMETER; + + KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp); + if (!KSUCCESS(st)) + return st; + + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + (void)write; + (void)buffer; + if (!KSUCCESS(st) || lba + sectors > drv->sectorCount) { + IoReleaseRemoveLock(&drv->removeLock, irp); + return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st; + } irp->Status = STATUS_FAILURE; irp->Information = 0; @@ -399,6 +435,49 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { return STATUS_PENDING; } +static void AhciCancelReleaseRemoveLock(IRP* irp, void* context) { + AHCI_DRIVE* drv = (AHCI_DRIVE*)context; + if (drv) + IoReleaseRemoveLock(&drv->removeLock, irp); +} + +static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return STATUS_INVALID_PARAMETER; + AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension; + if (!drv) return STATUS_INVALID_PARAMETER; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_PNP) + return STATUS_INVALID_PARAMETER; + + switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { + case IRP_MN_START_DEVICE: + device->PnpState = DevicePnpStarted; + return STATUS_SUCCESS; + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + return STATUS_SUCCESS; + case IRP_MN_CANCEL_REMOVE_DEVICE: + device->PnpState = DevicePnpStarted; + return STATUS_SUCCESS; + case IRP_MN_SURPRISE_REMOVAL: + drv->surpriseRemoved = true; + device->PnpState = DevicePnpSurpriseRemoved; + IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + return STATUS_SUCCESS; + case IRP_MN_REMOVE_DEVICE: + drv->removed = true; + device->PnpState = DevicePnpDeleted; + IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + IoReleaseRemoveLockAndWait(&drv->removeLock, irp); + IoDeleteDevice(device); + drv->deviceObject = (DEVICE_OBJECT*)0; + return STATUS_SUCCESS; + default: + return STATUS_NOT_SUPPORTED; + } +} + // AhciWorker - kernel thread, one per drive. // Dequeues IRPs and executes them one at a time using the existing // single-slot interrupt-driven path. Completion fires the IRP's event @@ -420,17 +499,24 @@ static void AhciWorker(uintptr_t arg) { KeWaitSemaphore(drv->irpQueue.Semaphore); IRP *irp = IrpQueueDequeue(&drv->irpQueue); - if (!irp) KePanic(KE_PANIC_CODE_DRIVER, "AhciWorker: semaphore fired but queue is empty - IRP accounting bug\n"); + if (!irp) + continue; - bool write = !!(irp->Flags & IRP_FLAG_WRITE); - KSTATUS st; - if (write) - st = do_write(drv, irp->Lba, irp->SectorCount, irp->Buffer); - else - st = do_read (drv, irp->Lba, irp->SectorCount, irp->Buffer); + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + if (KSUCCESS(st)) { + if (write) + st = do_write(drv, lba, sectors, buffer); + else + st = do_read (drv, lba, sectors, buffer); + } // Publish result then fire completion hooks through the common I/O path. - IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0); + IoReleaseRemoveLock(&drv->removeLock, irp); + IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0); } } @@ -467,9 +553,10 @@ KSTATUS AhciInitialize(const HAL_API *hal) { g_driveCount = 0; memset(&g_driver, 0, sizeof(g_driver)); - g_driver.Name = "ahci"; + IoInitializeDriverObject(&g_driver, SYS_SV_LITERAL("ahci")); g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite; g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite; + g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp; // ----------------------------------------------------------------------- // Locate AHCI controller via PCI class scan @@ -588,6 +675,11 @@ KSTATUS AhciInitialize(const HAL_API *hal) { continue; } IrpQueueInit(&drv->irpQueue, irpSem); + st = IoInitializeRemoveLock(&drv->removeLock); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st); + continue; + } // Phase 6: IDENTIFY (polled - scheduler not yet running) st = port_identify(drv); @@ -613,7 +705,8 @@ KSTATUS AhciInitialize(const HAL_API *hal) { continue; } drv->deviceObject->DeviceExtension = drv; - drv->deviceObject->SectorCount = drv->sectorCount; + drv->deviceObject->u.Disk.SectorCount = drv->sectorCount; + drv->deviceObject->u.Disk.SectorSize = 512; drv->deviceObject->AlignmentRequirement = 512; st = IoRegisterDiskDevice((uint8_t)driveIdx, drv->deviceObject); diff --git a/code/drivers/storage/ahci/ahci.h b/code/drivers/storage/ahci/ahci.h index abedc39..4bc33f5 100644 --- a/code/drivers/storage/ahci/ahci.h +++ b/code/drivers/storage/ahci/ahci.h @@ -344,6 +344,9 @@ typedef struct { // AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP // and signals the semaphore to wake the worker. IRP_QUEUE irpQueue; + IO_REMOVE_LOCK removeLock; + bool surpriseRemoved; + bool removed; THREAD* workerThread; DEVICE_OBJECT* deviceObject; diff --git a/code/drivers/storage/ide/ide.c b/code/drivers/storage/ide/ide.c index 142d52c..8def9b2 100644 --- a/code/drivers/storage/ide/ide.c +++ b/code/drivers/storage/ide/ide.c @@ -21,6 +21,9 @@ IRP_QUEUE gIrpQueue; SEMAPHORE *gIrpSemaphore = NULL; EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer KDPC gDmaDpc; // bottom half for DMA completion +IO_REMOVE_LOCK gRemoveLock; +bool gSurpriseRemoved = false; +bool gRemoved = false; // --------------------------------------------------------------------------- // Bus Master I/O helpers @@ -307,6 +310,28 @@ static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) { return STATUS_SUCCESS; } + +static KSTATUS IdeGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba, + uint32_t* outSectors, void** outBuffer) { + if (!irp || !outWrite || !outLba || !outSectors || !outBuffer) + return STATUS_INVALID_PARAMETER; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) return STATUS_INVALID_PARAMETER; + if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE) + return STATUS_INVALID_PARAMETER; + if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) || + (ios->Parameters.ReadWrite.Length & 511u)) + return STATUS_INVALID_PARAMETER; + if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0) + return STATUS_INVALID_PARAMETER; + + *outWrite = (ios->MajorFunction == IRP_MJ_WRITE); + *outLba = ios->Parameters.ReadWrite.ByteOffset >> 9; + *outSectors = ios->Parameters.ReadWrite.Length >> 9; + *outBuffer = irp->UserBuffer; + return STATUS_SUCCESS; +} + // --------------------------------------------------------------------------- // IdeWorker — kernel thread, one per IDE controller. // @@ -320,17 +345,22 @@ void IdeWorker(uintptr_t arg) { IRP *irp = IrpQueueDequeue(&gIrpQueue); if (!irp) - KePanic(KE_PANIC_CODE_DRIVER, "IdeWorker: semaphore fired but IRP queue is empty — accounting bug\n"); + continue; - KSTATUS st; - if (irp->Lba + irp->SectorCount > gSectorCount) { - st = STATUS_INVALID_PARAMETER; - } else { - bool write = !!(irp->Flags & IRP_FLAG_WRITE); - st = IdeDmaRw(irp->Lba, irp->SectorCount, irp->Buffer, write); + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + KSTATUS st = IdeGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + if (KSUCCESS(st)) { + if (lba + sectors > gSectorCount) + st = STATUS_INVALID_PARAMETER; + else + st = IdeDmaRw(lba, sectors, buffer, write); } - IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0); + IoReleaseRemoveLock(&gRemoveLock, irp); + IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0); } } @@ -344,8 +374,20 @@ void IdeWorker(uintptr_t arg) { // --------------------------------------------------------------------------- static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { (void)device; - if (!irp || !irp->Buffer || irp->SectorCount == 0) return STATUS_INVALID_PARAMETER; - if (irp->Lba + irp->SectorCount > gSectorCount) return STATUS_INVALID_PARAMETER; + + KSTATUS st = IoAcquireRemoveLock(&gRemoveLock, irp); + if (!KSUCCESS(st)) + return st; + + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + st = IdeGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + if (!KSUCCESS(st) || lba + sectors > gSectorCount) { + IoReleaseRemoveLock(&gRemoveLock, irp); + return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st; + } irp->Status = STATUS_FAILURE; irp->Information = 0; @@ -355,13 +397,53 @@ static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { return STATUS_PENDING; } - bool write = !!(irp->Flags & IRP_FLAG_WRITE); - KSTATUS st = write ? IdeWrite(NULL, irp->Lba, irp->SectorCount, irp->Buffer) - : IdeRead (NULL, irp->Lba, irp->SectorCount, irp->Buffer); - IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0); + st = write ? IdeWrite(NULL, lba, sectors, buffer) + : IdeRead (NULL, lba, sectors, buffer); + IoReleaseRemoveLock(&gRemoveLock, irp); + IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0); return STATUS_PENDING; } +static void IdeCancelReleaseRemoveLock(IRP* irp, void* context) { + (void)context; + IoReleaseRemoveLock(&gRemoveLock, irp); +} + +static KSTATUS IdeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return STATUS_INVALID_PARAMETER; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_PNP) + return STATUS_INVALID_PARAMETER; + + switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { + case IRP_MN_START_DEVICE: + device->PnpState = DevicePnpStarted; + return STATUS_SUCCESS; + case IRP_MN_QUERY_REMOVE_DEVICE: + device->PnpState = DevicePnpRemovePending; + return STATUS_SUCCESS; + case IRP_MN_CANCEL_REMOVE_DEVICE: + device->PnpState = DevicePnpStarted; + return STATUS_SUCCESS; + case IRP_MN_SURPRISE_REMOVAL: + gSurpriseRemoved = true; + device->PnpState = DevicePnpSurpriseRemoved; + IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0); + return STATUS_SUCCESS; + case IRP_MN_REMOVE_DEVICE: + gRemoved = true; + device->PnpState = DevicePnpDeleted; + IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0); + IoReleaseRemoveLockAndWait(&gRemoveLock, irp); + IoDeleteDevice(device); + gDevice = (DEVICE_OBJECT*)0; + return STATUS_SUCCESS; + default: + return STATUS_NOT_SUPPORTED; + } +} + // --------------------------------------------------------------------------- // IdeDmaInit — locate the PCI IDE controller, grab BAR4, allocate the PRDT, // create sync objects, and register IRQ 14. @@ -427,13 +509,18 @@ KSTATUS DriverEntry(const HAL_API *hal) { // ---- Register definitive DEVICE_OBJECT -------------------------------- memset(&gDriver, 0, sizeof(gDriver)); - gDriver.Name = "ide"; + IoInitializeDriverObject(&gDriver, SYS_SV_LITERAL("ide")); gDriver.MajorFunction[IRP_MJ_READ] = IdeDispatchReadWrite; gDriver.MajorFunction[IRP_MJ_WRITE] = IdeDispatchReadWrite; + gDriver.MajorFunction[IRP_MJ_PNP] = IdeDispatchPnp; - KSTATUS st = IoCreateDevice(&gDriver, 0, &gDevice); + KSTATUS st = IoInitializeRemoveLock(&gRemoveLock); if (!KSUCCESS(st)) return st; - gDevice->SectorCount = gSectorCount; + + st = IoCreateDevice(&gDriver, 0, &gDevice); + if (!KSUCCESS(st)) return st; + gDevice->u.Disk.SectorCount = gSectorCount; + gDevice->u.Disk.SectorSize = 512; gDevice->AlignmentRequirement = 512; // ---- Try to initialise Bus Master DMA --------------------------------- diff --git a/code/kernel/hal/hal_main.c b/code/kernel/hal/hal_main.c index ee89a26..18a0a9d 100644 --- a/code/kernel/hal/hal_main.c +++ b/code/kernel/hal/hal_main.c @@ -82,11 +82,13 @@ static uintptr_t gIoapicPhys = 0xFEC00000; static uintptr_t gHpetPhys = 0xFED00000; static HAL_CPU_TOPOLOGY gCpuTopology; -// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5) +// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5). +// Do not guess SLP_TYP on real hardware: these values are firmware-specific. static uint16_t gPm1aCntBlk = 0; static uint16_t gPm1bCntBlk = 0; -static uint8_t gSlpTypA = 5; // S5 SLP_TYP for PM1a; 5 is correct for QEMU/most VMs -static uint8_t gSlpTypB = 5; // S5 SLP_TYP for PM1b +static uint8_t gSlpTypA = 0; +static uint8_t gSlpTypB = 0; +static int gS5Found = 0; int debugEnabled = 0; int _fltused; // MSVC floating-point helper anchor @@ -106,73 +108,183 @@ static void *HalPhysToVirt(uint64_t phys) { return (void *)(phys + gPhysBase); } -// FADT (Fixed ACPI Description Table) - partial layout up to PM1b_CNT_BLK. -// We only need offsets 40 (DSDT ptr), 64 (PM1a_CNT_BLK), 68 (PM1b_CNT_BLK). -// The full table is ~276 bytes (ACPI 2.0+); we never access beyond offset 68. +// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+ +// systems may put the real DSDT and PM1 control registers in the extended +// X_* fields, so do not rely only on the legacy 32-bit fields. +#define FADT_DSDT_OFFSET 40u +#define FADT_PM1A_CNT_BLK_OFFSET 64u +#define FADT_PM1B_CNT_BLK_OFFSET 68u +#define FADT_X_DSDT_OFFSET 140u +#define FADT_X_PM1A_CNT_BLK_OFFSET 172u +#define FADT_X_PM1B_CNT_BLK_OFFSET 184u + #pragma pack(push, 1) typedef struct { - ACPISDTHeader Header; // 0 (36 bytes) - uint32_t FirmwareCtrl; // 36 - uint32_t Dsdt; // 40 - physical addr of DSDT - uint8_t Ignored44; // 44 - uint8_t PreferredPmProfile; // 45 - uint16_t SciInt; // 46 - uint32_t SmiCmd; // 48 - uint8_t AcpiEnable; // 52 - uint8_t AcpiDisable; // 53 - uint8_t S4BiosReq; // 54 - uint8_t PstateCnt; // 55 - uint32_t Pm1aEvtBlk; // 56 - uint32_t Pm1bEvtBlk; // 60 - uint32_t Pm1aCntBlk; // 64 - I/O port for PM1a control register - uint32_t Pm1bCntBlk; // 68 - I/O port for PM1b control register -} FADT; + uint8_t AddressSpaceId; + uint8_t RegisterBitWidth; + uint8_t RegisterBitOffset; + uint8_t AccessSize; + uint64_t Address; +} ACPI_GAS; #pragma pack(pop) -// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. -// The encoding looks like: -// 0x08 '_' 'S' '5' '_' ; NameOp + "_S5_" -// 0x12 ; PackageOp + encoded length -// ; usually 0x04 -// 0x0A ; BytePrefix + value (or 0x00 for zero) -// 0x0A -static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) { - for (uint32_t i = 0; i + 9 < len; i++) { - // Match NameOp + "_S5_" + PackageOp - if (aml[i] != 0x08) continue; - if (aml[i+1] != 0x5F || aml[i+2] != 0x53 || - aml[i+3] != 0x35 || aml[i+4] != 0x5F) continue; - if (aml[i+5] != 0x12) continue; +static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) { + uint32_t value = 0; + if (off + sizeof(value) <= tableLen) + memcpy(&value, (const uint8_t*)table + off, sizeof(value)); + return value; +} - // PkgLength encoding: low 6 bits = byte count (0 → only this byte); - // bits [7:6] = number of following bytes in the length field. - uint8_t pkgByte = aml[i+6]; - uint32_t pkgExtra = pkgByte >> 6; // 0..3 extra bytes - uint32_t j = i + 7 + pkgExtra; // skip PkgLen field - if (j + 2 >= len) break; - j++; // skip NumElements byte +static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off) { + uint64_t value = 0; + if (off + sizeof(value) <= tableLen) + memcpy(&value, (const uint8_t*)table + off, sizeof(value)); + return value; +} - // SLP_TYP_A - if (aml[j] == 0x0A && j + 1 < len) { - gSlpTypA = aml[j + 1]; j += 2; - } else if (aml[j] == 0x00) { - gSlpTypA = 0; j++; - } else { - break; // unexpected encoding - keep defaults +static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t off) { + ACPI_GAS gas; + if (off + sizeof(gas) > tableLen) return 0; + + memcpy(&gas, (const uint8_t*)table + off, sizeof(gas)); + + // 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(), + // so ignore MMIO GAS entries until the HAL grows MMIO register access here. + if (gas.AddressSpaceId != 0x01 || gas.Address == 0 || gas.Address > 0xFFFF) + return 0; + + return (uint16_t)gas.Address; +} + +static int AmlParsePkgLength(const uint8_t* p, const uint8_t* end, + const uint8_t** body, const uint8_t** pkgEnd) { + if (p >= end) return 0; + + const uint8_t* start = p; + uint8_t first = *p++; + uint32_t follow = first >> 6; + uint32_t length; + + if (follow == 0) { + length = first & 0x3F; + } else { + length = first & 0x0F; + for (uint32_t i = 0; i < follow; i++) { + if (p >= end) return 0; + length |= ((uint32_t)*p++) << (4 + i * 8); } + } - // SLP_TYP_B - if (j < len && aml[j] == 0x0A && j + 1 < len) - gSlpTypB = aml[j + 1]; - else if (j < len && aml[j] == 0x00) - gSlpTypB = 0; + if (start + length < p || start + length > end) return 0; + *body = p; + *pkgEnd = start + length; + return 1; +} + +static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end, + int* lastSegmentIsS5) { + uint32_t segments = 1; + *lastSegmentIsS5 = 0; + + // Optional root or parent prefixes: '\\', '^', '^^', ... + while (p < end && (*p == 0x5C || *p == 0x5E)) p++; + if (p >= end) return NULL; + + if (*p == 0x00) { // NullName + return p + 1; + } else if (*p == 0x2E) { // DualNamePrefix + p++; + segments = 2; + } else if (*p == 0x2F) { // MultiNamePrefix + p++; + if (p >= end) return NULL; + segments = *p++; + } + + for (uint32_t s = 0; s < segments; s++) { + if (p + 4 > end) return NULL; + if (s == segments - 1 && memcmp(p, "_S5_", 4) == 0) + *lastSegmentIsS5 = 1; + p += 4; + } + + return p; +} + +static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) { + const uint8_t* p = *pp; + if (p >= end) return 0; + + switch (*p++) { + case 0x00: // ZeroOp + *out = 0; + break; + case 0x01: // OneOp + *out = 1; + break; + case 0x0A: // BytePrefix + if (p >= end) return 0; + *out = *p++; + break; + case 0x0B: // WordPrefix + if (p + 2 > end) return 0; + *out = p[0]; + p += 2; + break; + case 0x0C: // DWordPrefix + if (p + 4 > end) return 0; + *out = p[0]; + p += 4; + break; + case 0x0E: // QWordPrefix + if (p + 8 > end) return 0; + *out = p[0]; + p += 8; + break; + default: + return 0; + } + + *pp = p; + return 1; +} + +// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real +// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and +// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants. +static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) { + const uint8_t* end = aml + len; + + for (const uint8_t* cur = aml; cur < end; cur++) { + if (*cur != 0x08) continue; // NameOp + + const uint8_t* p = cur + 1; + int isS5 = 0; + p = AmlParseNameString(p, end, &isS5); + if (!p || !isS5 || p >= end || *p != 0x12) continue; // PackageOp + + const uint8_t* body; + const uint8_t* pkgEnd; + if (!AmlParsePkgLength(p + 1, end, &body, &pkgEnd)) continue; + if (body >= pkgEnd) continue; + + uint8_t elements = *body++; + if (elements < 2) continue; + + uint8_t typA, typB; + if (!AmlReadInteger(&body, pkgEnd, &typA)) continue; + if (!AmlReadInteger(&body, pkgEnd, &typB)) continue; + + gSlpTypA = typA & 0x07; + gSlpTypB = typB & 0x07; + gS5Found = 1; HalDbgPrint("HAL: _S5_ SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n", gSlpTypA, gSlpTypB); return; } - HalDbgPrint("HAL: _S5_ not found in DSDT, using defaults (A=0x%02x B=0x%02x)\n", - gSlpTypA, gSlpTypB); + + HalDbgPrint("HAL: _S5_ not found in DSDT; ACPI shutdown disabled\n"); } static int validate_checksum(void* table, uint32_t length) { @@ -184,19 +296,33 @@ static int validate_checksum(void* table, uint32_t length) { return sum == 0; } -static void ParseFadt(const FADT* fadt) { - gPm1aCntBlk = (uint16_t)fadt->Pm1aCntBlk; - gPm1bCntBlk = (uint16_t)fadt->Pm1bCntBlk; - HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%08x\n", - gPm1aCntBlk, gPm1bCntBlk, fadt->Dsdt); +static void ParseFadt(const ACPISDTHeader* header) { + const void* fadt = header; + uint32_t len = header->Length; - if (fadt->Dsdt) { - const ACPISDTHeader* dsdt = - (const ACPISDTHeader*)((uintptr_t)fadt->Dsdt + gPhysBase); + gPm1aCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1A_CNT_BLK_OFFSET); + gPm1bCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1B_CNT_BLK_OFFSET); + + if (!gPm1aCntBlk) + gPm1aCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1A_CNT_BLK_OFFSET); + if (!gPm1bCntBlk) + gPm1bCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1B_CNT_BLK_OFFSET); + + uint64_t dsdtPhys = AcpiReadU64(fadt, len, FADT_X_DSDT_OFFSET); + if (!dsdtPhys) + dsdtPhys = AcpiReadU32(fadt, len, FADT_DSDT_OFFSET); + + HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%016llx\n", + gPm1aCntBlk, gPm1bCntBlk, dsdtPhys); + + if (dsdtPhys) { + const ACPISDTHeader* dsdt = (const ACPISDTHeader*)(dsdtPhys + gPhysBase); if (memcmp(dsdt->Signature, "DSDT", 4) == 0 && validate_checksum((void*)dsdt, dsdt->Length)) { const uint8_t* aml = (const uint8_t*)(dsdt + 1); ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader)); + } else { + HalDbgPrint("HAL: DSDT invalid or checksum failed\n"); } } } @@ -319,7 +445,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) { } // Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer else if (memcmp(header->Signature, "FACP", 4) == 0) { - ParseFadt((const FADT*)header); + ParseFadt(header); } // Parse HPET else if (memcmp(header->Signature, "HPET", 4) == 0) { @@ -337,11 +463,17 @@ static void HalShutdown(void) { // Silence! Stop the world! _disable(); - // SLP_EN (bit 13) | SLP_TYPx (bits 12:10) - uint16_t valA = (uint16_t)((gSlpTypA << 10) | (1 << 13)); - uint16_t valB = (uint16_t)((gSlpTypB << 10) | (1 << 13)); + if (!gS5Found || !gPm1aCntBlk) { + // Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets + // put into S3/S4 sleep instead of S5 soft-off. + while (true) __halt(); + } - if (gPm1aCntBlk) __outword(gPm1aCntBlk, valA); + // SLP_EN (bit 13) | SLP_TYPx (bits 12:10) + uint16_t valA = (uint16_t)(((uint16_t)gSlpTypA << 10) | (1 << 13)); + uint16_t valB = (uint16_t)(((uint16_t)gSlpTypB << 10) | (1 << 13)); + + __outword(gPm1aCntBlk, valA); if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB); // Die! diff --git a/code/kernel/include/kernel/io_file.h b/code/kernel/include/kernel/io_file.h index b6ced40..ae941cf 100644 --- a/code/kernel/include/kernel/io_file.h +++ b/code/kernel/include/kernel/io_file.h @@ -4,6 +4,7 @@ // Kernel and user code open files through IoOpenFile. #include #include +#include #include #include #include @@ -143,6 +144,9 @@ typedef struct IO_VOLUME_PARAMS { // --------------------------------------------------------------------------- // driveIndex matches the index passed to IoRegisterDiskDevice. __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol); +__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex); +__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex); +__declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex); // --------------------------------------------------------------------------- // Kernel-side API diff --git a/code/kernel/include/kernel/ke_cpu.h b/code/kernel/include/kernel/ke_cpu.h index 9e6da58..5b83bd2 100644 --- a/code/kernel/include/kernel/ke_cpu.h +++ b/code/kernel/include/kernel/ke_cpu.h @@ -30,6 +30,24 @@ typedef struct KCPU { uint8_t Online; uint8_t IsBsp; + // CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded + // from the APIC ID using CPUID leaf 0x1F/0x0B when available, with a + // conservative CPUID(1)/CPUID(4) fallback. PhysicalCoreIndex is a compact + // kernel-local core number used by the scheduler. CoreSiblingMask contains + // all logical CPUs that share this physical core. PackageMask contains all + // logical CPUs in the same package. IsPrimaryThread is the scheduler's + // preferred logical CPU for a physical core, currently the lowest CpuIndex + // in that core. + uint32_t PackageId; + uint32_t CoreId; + uint32_t ThreadId; + uint32_t PhysicalCoreIndex; + uint64_t CoreSiblingMask; + uint64_t PackageMask; + uint8_t IsPrimaryThread; + uint8_t SmtThreadCount; + uint16_t TopologyFlags; + struct THREAD* CurrentThread; struct THREAD* IdleThread; struct THREAD* OutgoingThread; diff --git a/code/kernel/include/kernel/ke_debug.h b/code/kernel/include/kernel/ke_debug.h index d83b0ff..a10788b 100644 --- a/code/kernel/include/kernel/ke_debug.h +++ b/code/kernel/include/kernel/ke_debug.h @@ -3,10 +3,18 @@ #include #include +#include +#include -void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v); +#define KE_DEBUG_CONSOLE 0x00000001u +#define KE_DEBUG_SELFTEST 0x00000002u +#define KE_DEBUG_FILE_LOG 0x00000004u + +void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags); +uint32_t KeGetDebugFlags(void); __declspec(dllexport) void DbgPrint(const char* fmt, ...); __declspec(dllexport) void KePrintConsole(const char* buf, size_t len); +__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path); typedef enum KE_PANIC_CODE { KE_PANIC_CODE_GENERIC = 0, diff --git a/code/kernel/include/kernel/ke_irp.h b/code/kernel/include/kernel/ke_irp.h index a9288b6..c59510c 100644 --- a/code/kernel/include/kernel/ke_irp.h +++ b/code/kernel/include/kernel/ke_irp.h @@ -1,83 +1,157 @@ #pragma once -// ke_irp.h - I/O Request Packet +// ke_irp.h - I/O Request Packet and remove-lock primitives // -// This header intentionally includes only ke_list.h, ke_spinlock.h, and -// ke_status.h - all of which are pure-inline / macro headers with no -// external symbol references. ke_event.h and ke_semaphore.h are forward- -// declared here so the HAL layer (hal_disk.c, io_driver.h) can include this -// header without dragging in kernel sync-object link dependencies. -// -// Files that actually call KeEventCreate / KeSignalEvent / KeWaitForEvent -// (ke_irp.c, ahci.c) include ke_event.h and ke_semaphore.h themselves. +// IRPs are stack-based. A caller builds an IRP with one or more +// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O +// manager/driver stack advances CurrentLocation as the IRP is forwarded. #include #include #include #include #include +#include #include -// Forward declarations - full types live in ke_event.h / ke_semaphore.h. -// We only ever hold pointers in IRP and IRP_QUEUE, so a forward decl suffices. struct EVENT; struct SEMAPHORE; struct KAPC; +typedef struct DEVICE_OBJECT DEVICE_OBJECT; + +// ------------------------------------------------------------------------- +// IRP major/minor functions +// ------------------------------------------------------------------------- + +typedef enum IO_MAJOR_FUNCTION { + IRP_MJ_CREATE = 0, + IRP_MJ_CLOSE = 1, + IRP_MJ_READ = 2, + IRP_MJ_WRITE = 3, + IRP_MJ_DEVICE_CONTROL = 4, + IRP_MJ_INTERNAL_DEVICE_CONTROL = 5, + IRP_MJ_PNP = 6, + IRP_MJ_POWER = 7, + IRP_MJ_MAXIMUM_FUNCTION = 8, +} IO_MAJOR_FUNCTION; + +typedef enum IO_PNP_MINOR_FUNCTION { + IRP_MN_START_DEVICE = 0, + IRP_MN_QUERY_REMOVE_DEVICE = 1, + IRP_MN_CANCEL_REMOVE_DEVICE = 2, + IRP_MN_SURPRISE_REMOVAL = 3, + IRP_MN_REMOVE_DEVICE = 4, + IRP_MN_QUERY_DEVICE_RELATIONS = 5, + IRP_MN_QUERY_ID = 6, + IRP_MN_QUERY_CAPABILITIES = 7, +} IO_PNP_MINOR_FUNCTION; + +typedef enum IO_DEVICE_RELATION_TYPE { + IoBusRelations = 0, + IoRemovalRelations = 1, +} IO_DEVICE_RELATION_TYPE; // ------------------------------------------------------------------------- // IRP flags // ------------------------------------------------------------------------- -#define IRP_FLAG_WRITE (1u << 0) // direction: write to device -#define IRP_FLAG_PRIORITY (1u << 1) // insert at head of queue (elevated) +#define IRP_FLAG_PRIORITY (1u << 0) // insert at head of a driver queue +#define IRP_FLAG_CANCELLED (1u << 1) +#define IRP_FLAG_COMPLETED (1u << 2) // ------------------------------------------------------------------------- -// Forward declaration (IRP is self-referential via CompletionFn) +// IRP stack parameters +// ------------------------------------------------------------------------- + +typedef struct IO_STACK_LOCATION { + IO_MAJOR_FUNCTION MajorFunction; + uint8_t MinorFunction; + DEVICE_OBJECT* DeviceObject; + + union { + struct { + uint64_t ByteOffset; + uint32_t Length; + } ReadWrite; + + struct { + uint32_t IoControlCode; + void* InputBuffer; + uint32_t InputLength; + void* OutputBuffer; + uint32_t OutputLength; + } DeviceIoControl; + + struct { + IO_DEVICE_RELATION_TYPE RelationType; + } QueryDeviceRelations; + + struct { + uint32_t IdType; + } QueryId; + } Parameters; +} IO_STACK_LOCATION; + +// ------------------------------------------------------------------------- +// Forward declaration / callbacks // ------------------------------------------------------------------------- typedef struct IRP IRP; -// Completion callback: invoked by IoCompleteRequest() in the completing -// context. Driver worker, synchronous fallback, DPC, and APC completion paths -// all centralize policy here. -typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx); +typedef void (*IRP_COMPLETION_FN)(IRP* irp, void* ctx); +typedef void (*IRP_CANCEL_ROUTINE)(DEVICE_OBJECT* device, IRP* irp); // ------------------------------------------------------------------------- // I/O Request Packet // ------------------------------------------------------------------------- typedef struct IRP { - // Embedded list node - driver-internal; do not touch after submit. LIST_ENTRY Link; - // ---- Request parameters (caller fills before IoSubmitIrp) ------------ - - uint8_t DriveIndex; - uint32_t Flags; // IRP_FLAG_* - uint64_t Lba; - uint32_t SectorCount; - void* Buffer; // kernel VA; valid until completion - - // ---- Completion (caller fills before submit; driver writes Status) ---- - volatile KSTATUS Status; - volatile size_t Information; // bytes transferred on success + volatile size_t Information; - // Forward-declared pointer: callers that create events include ke_event.h. - struct EVENT *CompletionEvent; - - // Optional kernel APC. If set, IoCompleteRequest queues it to its - // target thread with arg1=irp and arg2=(void*)(uintptr_t)status. - struct KAPC *CompletionApc; + uint32_t Flags; + volatile bool Cancel; + SPINLOCK CancelLock; + KIRQL CancelIrql; + IRP_CANCEL_ROUTINE CancelRoutine; + struct EVENT* CompletionEvent; + struct KAPC* CompletionApc; IRP_COMPLETION_FN CompletionFn; void* CompletionCtx; - // ---- Driver-internal -------------------------------------------------- + void* UserBuffer; + void* SystemBuffer; + uint8_t StackCount; + uint8_t CurrentLocation; + + // Driver-owned scratch fields. They are deliberately not interpreted by + // the I/O manager and are preserved across queueing/completion. uint8_t HwSlot; void* DriverCtx; + + IO_STACK_LOCATION Stack[1]; } IRP; +// ------------------------------------------------------------------------- +// Remove lock +// ------------------------------------------------------------------------- + +typedef struct IO_REMOVE_LOCK { + SPINLOCK Lock; + volatile long IoCount; + bool Removing; + struct EVENT* RemovedEvent; +} IO_REMOVE_LOCK; + +__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock); +__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock); +__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp); +__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp); +__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp); + // ------------------------------------------------------------------------- // IRP_QUEUE (per-device, driver-internal) // ------------------------------------------------------------------------- @@ -85,24 +159,56 @@ typedef struct IRP { typedef struct IRP_QUEUE { SPINLOCK Lock; LIST_ENTRY Head; - struct SEMAPHORE *Semaphore; // forward-declared pointer + struct SEMAPHORE* Semaphore; } IRP_QUEUE; -// Non-inline: defined in ke_irp.c, which can pull in ke_semaphore.h safely. -__declspec(dllexport) void IrpQueueInit (IRP_QUEUE *q, struct SEMAPHORE *sem); -__declspec(dllexport) void IrpQueueEnqueue (IRP_QUEUE *q, IRP *irp); -__declspec(dllexport) IRP *IrpQueueDequeue (IRP_QUEUE *q); +__declspec(dllexport) void IrpQueueInit (IRP_QUEUE* q, struct SEMAPHORE* sem); +__declspec(dllexport) void IrpQueueEnqueue (IRP_QUEUE* q, IRP* irp); +__declspec(dllexport) IRP* IrpQueueDequeue (IRP_QUEUE* q); +typedef void (*IRP_QUEUE_CANCEL_CALLBACK)(IRP* irp, void* context); +__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, + KSTATUS status, + IRP_QUEUE_CANCEL_CALLBACK callback, + void* context); +__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status); // ------------------------------------------------------------------------- -// Public dispatch API - implemented in kernel/ke_irp.c +// IRP allocation / stack helpers // ------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp); +__declspec(dllexport) void IoFreeIrp(IRP* irp); +__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount); + +__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp); +__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp); +__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp); +__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp); +__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx); +__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine); +__declspec(dllexport) void IoCancelIrp(IRP* irp); + +__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp, + IO_MAJOR_FUNCTION major, + uint64_t byteOffset, + uint32_t length, + void* buffer); +__declspec(dllexport) void IoBuildPnpIrp(IRP* irp, + IO_PNP_MINOR_FUNCTION minor); +__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp, + IO_MAJOR_FUNCTION major, + uint32_t ioControlCode, + void* inputBuffer, + uint32_t inputLength, + void* outputBuffer, + uint32_t outputLength); + // Complete an IRP exactly once: publish status/information, signal the -// optional completion event, then invoke the optional completion callback. -__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information); +// optional completion event, queue the optional APC, then invoke the optional +// completion callback. +__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information); -// Submit asynchronously. Returns STATUS_PENDING on success. -__declspec(dllexport) KSTATUS IoSubmitIrp (IRP *irp); - -// Submit and block until complete. Returns irp->Status. -__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp); +// Submit to a specific top-of-stack device object. The current stack location +// is stamped with the target device before dispatch. +__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp); +__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp); diff --git a/code/kernel/include/kernel/ke_sched.h b/code/kernel/include/kernel/ke_sched.h index 62e3a15..dc94d95 100644 --- a/code/kernel/include/kernel/ke_sched.h +++ b/code/kernel/include/kernel/ke_sched.h @@ -23,6 +23,8 @@ typedef struct KE_SCHED_STATS { uint64_t PriorityPreemptions; uint64_t VoluntaryBlocks; uint64_t CpuBurstDemotions; + uint64_t SmtSiblingDeferrals; + uint64_t SmtPrimaryPokes; } KE_SCHED_STATS; // Called once by KernelMain before any threads are created. diff --git a/code/kernel/include/kernel/ke_smp.h b/code/kernel/include/kernel/ke_smp.h index f1e0162..993fe2a 100644 --- a/code/kernel/include/kernel/ke_smp.h +++ b/code/kernel/include/kernel/ke_smp.h @@ -13,6 +13,12 @@ KSTATUS KeStartApplicationProcessors(void); // park loop into their idle threads. KSTATUS KeAdmitApplicationProcessorsToScheduler(void); + +// Decode CPUID topology and build per-KCPU package/core/SMT sibling masks. +// KeStartApplicationProcessors calls this after MADT enumeration; it is also +// safe to call again after failed AP startup disables a CPU record. +void KeBuildProcessorTopology(void); + // Run explicit AP-affine kernel workers that exercise the global scheduler, // reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver // threads remain BSP-affine unless their affinity is explicitly widened. diff --git a/code/kernel/include/kernel/ob_object.h b/code/kernel/include/kernel/ob_object.h index 9f3fa8e..ae8fde8 100644 --- a/code/kernel/include/kernel/ob_object.h +++ b/code/kernel/include/kernel/ob_object.h @@ -17,6 +17,10 @@ typedef enum OBJECT_TYPE { ObjectTypeDir = 8, ObjectTypeAsyncIo = 9, ObjectTypeSocket = 10, + ObjectTypeDevice = 11, + ObjectTypeDriver = 12, + ObjectTypeUsbDevice = 13, + ObjectTypeUsbPipe = 14, } OBJECT_TYPE; typedef struct OBJECT { diff --git a/code/kernel/io_device.c b/code/kernel/io_device.c index bcd08f1..5f0ad39 100644 --- a/code/kernel/io_device.c +++ b/code/kernel/io_device.c @@ -1,13 +1,423 @@ -// io_device.c - kernel driver/device dispatch and disk registry +// io_device.c - I/O manager: driver/device objects, stacks, PnP skeleton, disk namespace #include +#include #include #include #include #include - +#include #define MAX_PUBLISHED_DEVICES 64 -#define MAX_DRIVER_MATCHERS 32 +#define MAX_DRIVER_MATCHERS 32 +#define MAX_DRIVER_OBJECTS 64 +#define MAX_PDO_OBJECTS 128 + +static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) { + return SysSvIsEmpty(name) ? SYS_SV_LITERAL("") : name; +} + +static const char* IoBusTypeName(IO_BUS_TYPE busType) { + switch (busType) { + case IoBusPci: return "pci"; + case IoBusUsb: return "usb"; + default: return "unknown"; + } +} + +// ------------------------------------------------------------------------- +// Driver object registry / PDO AddDevice binding +// ------------------------------------------------------------------------- + +static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS]; +static uint32_t gDriverObjectCount; +static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS]; +static uint32_t gPhysicalDeviceCount; + +static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) { + if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical) + return false; + + for (size_t d = 0; d < driver->IdTable->Count; d++) { + const IO_DEVICE_ID* want = &driver->IdTable->Ids[d]; + if (want->BusType != IoBusUnknown && want->BusType != pdo->u.Pdo.BusType) + continue; + for (uint8_t p = 0; p < pdo->u.Pdo.IdCount; p++) { + if (SysSvEqual(want->Id, pdo->u.Pdo.Ids[p].Id)) + return true; + } + } + return false; +} + +static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) { + if (!driver || !pdo || !driver->DriverAddDevice) + return false; + if (!IoDriverMatchesPdo(driver, pdo)) + return false; + + KSTATUS st = driver->DriverAddDevice(driver, pdo); + if (KSUCCESS(st)) { + DbgPrint("io: AddDevice driver=%v pdo=%v bus=%s\n", + IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), IoBusTypeName(pdo->u.Pdo.BusType)); + + KSTATUS start = IoSendPnpIrpSync(pdo, IRP_MN_START_DEVICE); + if (!KSUCCESS(start) && start != STATUS_NOT_SUPPORTED) { + DbgPrint("io: START_DEVICE failed driver=%v pdo=%v st=0x%x\n", + IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), start); + } + return true; + } else { + DbgPrint("io: AddDevice failed driver=%v pdo=%v st=0x%x\n", + IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), st); + return true; + } +} + +__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver, + SYS_STRING_VIEW name) { + if (!driver) return; + memset(driver, 0, sizeof(*driver)); + driver->Header.Type = ObjectTypeDriver; + driver->Header.RefCount = 1; + driver->Header.Destroy = (void*)0; + driver->Name = name; +} + +__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) { + if (!driver) return STATUS_INVALID_PARAMETER; + if (gDriverObjectCount >= MAX_DRIVER_OBJECTS) + return STATUS_NO_MEMORY; + + if (driver->Header.Type != ObjectTypeDriver) { + driver->Header.Type = ObjectTypeDriver; + driver->Header.RefCount = 1; + } + + gDriverObjects[gDriverObjectCount++] = driver; + DbgPrint("io: registered driver object %v\n", IoSvOrUnnamed(driver->Name)); + + for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) + (void)IoTryAddDeviceForDriver(driver, gPhysicalDevices[i]); + + return STATUS_SUCCESS; +} + +// ------------------------------------------------------------------------- +// Device object lifetime +// ------------------------------------------------------------------------- + +static void IoDeviceDestroy(OBJECT* object) { + DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object; + + for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) { + if (gPhysicalDevices[i] == dev) { + gPhysicalDevices[i] = gPhysicalDevices[gPhysicalDeviceCount - 1u]; + gPhysicalDevices[gPhysicalDeviceCount - 1u] = (DEVICE_OBJECT*)0; + gPhysicalDeviceCount--; + break; + } + } + + if (dev->LowerDevice) + IoDetachDevice(dev); + + if (dev->Parent) { + DEVICE_OBJECT* parent = dev->Parent; + uintptr_t flags = KeAcquireSpinlock(&parent->Lock); + DEVICE_OBJECT** link = &parent->FirstChild; + while (*link) { + if (*link == dev) { + *link = dev->NextSibling; + break; + } + link = &(*link)->NextSibling; + } + KeReleaseSpinlock(&parent->Lock, flags); + dev->Parent = (DEVICE_OBJECT*)0; + dev->NextSibling = (DEVICE_OBJECT*)0; + IoDereferenceDeviceObject(parent); + } +} + +static KSTATUS IoCreateDeviceInternal(DRIVER_OBJECT* driver, + DEVICE_OBJECT_TYPE type, + size_t extensionSize, + DEVICE_OBJECT** outDevice) { + if (!driver || !outDevice) return STATUS_INVALID_PARAMETER; + + size_t total = sizeof(DEVICE_OBJECT) + extensionSize; + DEVICE_OBJECT* dev = (DEVICE_OBJECT*)ObAllocateObject(ObjectTypeDevice, total); + if (!dev) return STATUS_NO_MEMORY; + + dev->Header.Destroy = IoDeviceDestroy; + dev->DriverObject = driver; + dev->Type = type; + dev->PnpState = DevicePnpNotStarted; + dev->AlignmentRequirement = 1; + dev->Lock = (SPINLOCK)SPINLOCK_INIT; + + // The per-device union is interpreted differently for PDOs and disk FDOs. + // Do not write disk defaults into a PDO: u.Disk.DiskIndex aliases the low + // byte of u.Pdo.BusType, which prevents ID-table AddDevice matching. + if (type != DeviceObjectPhysical) { + dev->u.Disk.DiskIndex = 0xFF; + dev->u.Disk.SectorSize = 512; + } + + if (extensionSize) + dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT)); + + *outDevice = dev; + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver, + size_t extensionSize, + DEVICE_OBJECT** outDevice) { + return IoCreateDeviceInternal(driver, DeviceObjectFunction, extensionSize, outDevice); +} + +__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver, + DEVICE_OBJECT* parent, + const IO_DEVICE_ID* ids, + size_t idCount, + size_t extensionSize, + DEVICE_OBJECT** outPdo) { + if (!busDriver || !outPdo) return STATUS_INVALID_PARAMETER; + if (idCount > IO_MAX_DEVICE_IDS) return STATUS_INVALID_PARAMETER; + + DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreateDeviceInternal(busDriver, DeviceObjectPhysical, extensionSize, &pdo); + if (!KSUCCESS(st)) return st; + + if (idCount && ids) { + for (size_t i = 0; i < idCount; i++) { + if (!SysSvIsValid(ids[i].Id) || ids[i].Id.Length >= IO_MAX_DEVICE_ID_CHARS) { + IoDeleteDevice(pdo); + return STATUS_INVALID_PARAMETER; + } + + char* dst = pdo->u.Pdo.IdStorage[i]; + if (ids[i].Id.Length) + memcpy(dst, ids[i].Id.Data, ids[i].Id.Length); + dst[ids[i].Id.Length] = '\0'; + + pdo->u.Pdo.Ids[i].BusType = ids[i].BusType; + pdo->u.Pdo.Ids[i].Id = SysSvFromParts(dst, ids[i].Id.Length); + if (pdo->u.Pdo.BusType == IoBusUnknown) + pdo->u.Pdo.BusType = ids[i].BusType; + } + pdo->u.Pdo.IdCount = (uint8_t)idCount; + } + + if (parent) { + IoReferenceDeviceObject(parent); + pdo->Parent = parent; + uintptr_t flags = KeAcquireSpinlock(&parent->Lock); + pdo->NextSibling = parent->FirstChild; + parent->FirstChild = pdo; + KeReleaseSpinlock(&parent->Lock, flags); + } + + *outPdo = pdo; + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) { + if (!pdo || pdo->Type != DeviceObjectPhysical) + return STATUS_INVALID_PARAMETER; + + for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) { + if (gPhysicalDevices[i] == pdo) + return STATUS_ALREADY_EXISTS; + } + + if (gPhysicalDeviceCount >= MAX_PDO_OBJECTS) + return STATUS_NO_MEMORY; + + gPhysicalDevices[gPhysicalDeviceCount++] = pdo; + + bool anyMatched = false; + for (uint32_t i = 0; i < gDriverObjectCount; i++) { + if (IoTryAddDeviceForDriver(gDriverObjects[i], pdo)) + anyMatched = true; + } + + if (!anyMatched && pdo->u.Pdo.BusType == IoBusUsb) { + SYS_STRING_VIEW id = pdo->u.Pdo.IdCount ? pdo->u.Pdo.Ids[0].Id : SYS_SV_LITERAL(""); + DbgPrint("io: no function driver for pdo=%v id=%v bus=%s\n", + IoSvOrUnnamed(pdo->Name), id, IoBusTypeName(pdo->u.Pdo.BusType)); + } + return STATUS_SUCCESS; +} + + +__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device, + SYS_STRING_VIEW name) { + if (!device || !SysSvIsValid(name)) + return STATUS_INVALID_PARAMETER; + if (name.Length >= IO_MAX_DEVICE_NAME_CHARS) + return STATUS_BUFFER_TOO_SMALL; + + if (name.Length) + memcpy(device->NameStorage, name.Data, name.Length); + device->NameStorage[name.Length] = '\0'; + device->Name = SysSvFromParts(device->NameStorage, name.Length); + return STATUS_SUCCESS; +} + +__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) { + if (device) ObReferenceObject(&device->Header); +} + +__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) { + if (device) ObDereferenceObject(&device->Header); +} + +static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) { + if (!device || device->Type != DeviceObjectPhysical) + return; + + for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) { + if (gPhysicalDevices[i] == device) { + for (uint32_t j = i + 1; j < gPhysicalDeviceCount; j++) + gPhysicalDevices[j - 1] = gPhysicalDevices[j]; + gPhysicalDeviceCount--; + if (gPhysicalDeviceCount < MAX_PDO_OBJECTS) + gPhysicalDevices[gPhysicalDeviceCount] = (DEVICE_OBJECT*)0; + return; + } + } +} + +static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) { + if (!device || !device->Parent) + return; + + DEVICE_OBJECT* parent = device->Parent; + uintptr_t flags = KeAcquireSpinlock(&parent->Lock); + DEVICE_OBJECT** link = &parent->FirstChild; + while (*link) { + if (*link == device) { + *link = device->NextSibling; + break; + } + link = &(*link)->NextSibling; + } + device->Parent = (DEVICE_OBJECT*)0; + device->NextSibling = (DEVICE_OBJECT*)0; + KeReleaseSpinlock(&parent->Lock, flags); + + IoDereferenceDeviceObject(parent); +} + +__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) { + if (!device) return; + if (device->DeletePending) + return; + + device->DeletePending = true; + device->PnpState = DevicePnpDeleted; + IoUnlinkDeviceFromParent(device); + IoRemovePhysicalDeviceRegistryEntry(device); + IoDereferenceDeviceObject(device); +} + +// ------------------------------------------------------------------------- +// Device stack management +// ------------------------------------------------------------------------- + +__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source, + DEVICE_OBJECT* target) { + if (!source || !target) return (DEVICE_OBJECT*)0; + + DEVICE_OBJECT* top = target; + while (top->AttachedDevice) + top = top->AttachedDevice; + + top->AttachedDevice = source; + source->LowerDevice = top; + IoReferenceDeviceObject(top); + return top; +} + +__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) { + if (!source || !source->LowerDevice) return; + + DEVICE_OBJECT* lower = source->LowerDevice; + if (lower->AttachedDevice == source) + lower->AttachedDevice = (DEVICE_OBJECT*)0; + source->LowerDevice = (DEVICE_OBJECT*)0; + IoDereferenceDeviceObject(lower); +} + +__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device) { + if (!device) return (DEVICE_OBJECT*)0; + DEVICE_OBJECT* top = device; + while (top->AttachedDevice) + top = top->AttachedDevice; + IoReferenceDeviceObject(top); + return top; +} + +// ------------------------------------------------------------------------- +// Device-stack helpers / PnP dispatch +// ------------------------------------------------------------------------- + +__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) { + uint8_t count = 0; + for (DEVICE_OBJECT* cur = device; cur && count < 0xFEu; cur = cur->LowerDevice) + count++; + return count ? count : 1; +} + +__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp || !device->LowerDevice) + return STATUS_NOT_SUPPORTED; + + IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp); + IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp); + if (!cur || !next) + return STATUS_INVALID_PARAMETER; + + DEVICE_OBJECT* currentDevice = cur->DeviceObject; + memcpy(next, cur, sizeof(*next)); + + if (currentDevice == device) { + cur->DeviceObject = (DEVICE_OBJECT*)0; + _InterlockedDecrement(&device->OutstandingIrps); + IoDereferenceDeviceObject(device); + } + + irp->CurrentLocation++; + return IoCallDriver(device->LowerDevice, irp); +} + +__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device, + IO_PNP_MINOR_FUNCTION minor) { + if (!device) return STATUS_INVALID_PARAMETER; + + DEVICE_OBJECT* top = IoGetAttachedDeviceReference(device); + if (!top) return STATUS_INVALID_PARAMETER; + + uint8_t stackCount = IoGetDeviceStackSize(top); + IRP* irp = (IRP*)0; + KSTATUS st = IoAllocateIrp(stackCount, &irp); + if (!KSUCCESS(st)) { + IoDereferenceDeviceObject(top); + return st; + } + + IoBuildPnpIrp(irp, minor); + st = IoSubmitIrpSync(top, irp); + IoFreeIrp(irp); + IoDereferenceDeviceObject(top); + return st; +} + +// ------------------------------------------------------------------------- +// Legacy publish/matcher path +// ------------------------------------------------------------------------- typedef struct IO_DRIVER_MATCHER_ENTRY { bool Used; @@ -23,14 +433,6 @@ static uint32_t gPublishedDeviceCount; static IO_DRIVER_MATCHER_ENTRY gDriverMatchers[MAX_DRIVER_MATCHERS]; static uint32_t gDriverMatcherCount; -static const char* IoBusTypeName(IO_BUS_TYPE busType) { - switch (busType) { - case IoBusPci: return "pci"; - case IoBusUsb: return "usb"; - default: return "unknown"; - } -} - static bool IoTryBindDevice(IO_PUBLISHED_DEVICE* device, IO_DRIVER_MATCHER_ENTRY* matcher) { if (!device || !matcher || !matcher->Used || device->Bound) @@ -80,12 +482,10 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName, entry->Bind = bind; entry->Context = context; - DbgPrint("io: registered %s matcher %s\n", - IoBusTypeName(busType), driverName); + DbgPrint("io: registered %s matcher %s\n", IoBusTypeName(busType), driverName); - for (uint32_t i = 0; i < gPublishedDeviceCount; i++) { + for (uint32_t i = 0; i < gPublishedDeviceCount; i++) IoTryBindDevice(&gPublishedDevices[i], entry); - } return STATUS_SUCCESS; } @@ -101,7 +501,6 @@ __declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device) stored->Bound = false; stored->BoundDriverName = (const char*)0; - for (uint32_t i = 0; i < gDriverMatcherCount; i++) { if (IoTryBindDevice(stored, &gDriverMatchers[i])) break; @@ -114,50 +513,44 @@ __declspec(dllexport) uint32_t IoPublishedDeviceCount(void) { return gPublishedDeviceCount; } -#define MAX_DISK_DEVICES 8 -#define MAX_DISK_ARRIVAL_CALLBACKS 16 +// ------------------------------------------------------------------------- +// Disk namespace +// ------------------------------------------------------------------------- -typedef struct IO_DISK_CALLBACK_ENTRY { +#define MAX_DISK_DEVICES 8 +#define MAX_DISK_CALLBACKS 16 + +typedef struct IO_DISK_CHANGE_CALLBACK_ENTRY { + bool Used; + IO_DISK_CHANGE_CALLBACK Callback; + void* Context; +} IO_DISK_CHANGE_CALLBACK_ENTRY; + +typedef struct IO_DISK_ARRIVAL_CALLBACK_ENTRY { bool Used; IO_DISK_ARRIVAL_CALLBACK Callback; void* Context; -} IO_DISK_CALLBACK_ENTRY; +} IO_DISK_ARRIVAL_CALLBACK_ENTRY; static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES]; -static IO_DISK_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_ARRIVAL_CALLBACKS]; +static IO_DISK_CHANGE_CALLBACK_ENTRY gDiskChangeCallbacks[MAX_DISK_CALLBACKS]; +static uint32_t gDiskChangeCallbackCount; +static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS]; static uint32_t gDiskArrivalCallbackCount; -static void IoNotifyDiskArrived(uint8_t driveIndex, DEVICE_OBJECT* device) { - for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) { - IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i]; +static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) { + for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) { + IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i]; if (entry->Used && entry->Callback) - entry->Callback(driveIndex, device, entry->Context); + entry->Callback(driveIndex, device, change, entry->Context); + } + if (change == IoDeviceArrived) { + for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) { + IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i]; + if (entry->Used && entry->Callback) + entry->Callback(driveIndex, device, entry->Context); + } } -} - -__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver, - size_t extensionSize, - DEVICE_OBJECT** outDevice) { - if (!driver || !outDevice) return STATUS_INVALID_PARAMETER; - - size_t total = sizeof(DEVICE_OBJECT) + extensionSize; - DEVICE_OBJECT* dev = (DEVICE_OBJECT*)MmAllocatePool(total); - if (!dev) return STATUS_NO_MEMORY; - - memset(dev, 0, total); - dev->DriverObject = driver; - if (extensionSize) - dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT)); - dev->AlignmentRequirement = 1; - dev->DiskIndex = 0xFF; - - *outDevice = dev; - return STATUS_SUCCESS; -} - -__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) { - if (!device) return; - MmFreePool(device); } __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, @@ -168,9 +561,57 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, return STATUS_ALREADY_EXISTS; device->Flags |= DEVICE_OBJECT_DISK; - device->DiskIndex = driveIndex; + device->u.Disk.DiskIndex = driveIndex; + if (device->u.Disk.SectorSize == 0) + device->u.Disk.SectorSize = 512; + + KSTATUS pnpStatus = IoSendPnpIrpSync(device, IRP_MN_START_DEVICE); + if (!KSUCCESS(pnpStatus) && pnpStatus != STATUS_NOT_SUPPORTED) + return pnpStatus; + if (pnpStatus == STATUS_NOT_SUPPORTED) + device->PnpState = DevicePnpStarted; + + IoReferenceDeviceObject(device); gDiskDevices[driveIndex] = device; - IoNotifyDiskArrived(driveIndex, device); + IoNotifyDiskChange(driveIndex, device, IoDeviceArrived); + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) { + if (driveIndex >= MAX_DISK_DEVICES) + return STATUS_INVALID_PARAMETER; + DEVICE_OBJECT* dev = gDiskDevices[driveIndex]; + if (!dev) return STATUS_NOT_FOUND; + + dev->PnpState = DevicePnpSurpriseRemoved; + IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved); + (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL); + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) { + if (driveIndex >= MAX_DISK_DEVICES) + return STATUS_INVALID_PARAMETER; + DEVICE_OBJECT* dev = gDiskDevices[driveIndex]; + if (!dev) return STATUS_NOT_FOUND; + + if (dev->PnpState != DevicePnpSurpriseRemoved) { + IoNotifyDiskChange(driveIndex, dev, IoDeviceQueryRemove); + KSTATUS queryStatus = IoSendPnpIrpSync(dev, IRP_MN_QUERY_REMOVE_DEVICE); + if (!KSUCCESS(queryStatus) && queryStatus != STATUS_NOT_SUPPORTED) + return queryStatus; + + dev->PnpState = DevicePnpSurpriseRemoved; + IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved); + (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL); + } + + dev->PnpState = DevicePnpRemovePending; + (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE); + + gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0; + IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved); + IoDereferenceDeviceObject(dev); return STATUS_SUCCESS; } @@ -178,10 +619,10 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL void* context) { if (!callback) return STATUS_INVALID_PARAMETER; - if (gDiskArrivalCallbackCount >= MAX_DISK_ARRIVAL_CALLBACKS) + if (gDiskArrivalCallbackCount >= MAX_DISK_CALLBACKS) return STATUS_NO_MEMORY; - IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++]; + IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++]; memset(entry, 0, sizeof(*entry)); entry->Used = true; entry->Callback = callback; @@ -191,7 +632,21 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL if (gDiskDevices[i]) callback(i, gDiskDevices[i], context); } + return STATUS_SUCCESS; +} +__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback, + void* context) { + if (!callback) + return STATUS_INVALID_PARAMETER; + if (gDiskChangeCallbackCount >= MAX_DISK_CALLBACKS) + return STATUS_NO_MEMORY; + + IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[gDiskChangeCallbackCount++]; + memset(entry, 0, sizeof(*entry)); + entry->Used = true; + entry->Callback = callback; + entry->Context = context; return STATUS_SUCCESS; } @@ -204,7 +659,6 @@ __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK ca if (gDiskDevices[i]) callback(i, gDiskDevices[i], context); } - return STATUS_SUCCESS; } @@ -213,28 +667,59 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) { return gDiskDevices[driveIndex]; } +__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) { + DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex); + if (dev) IoReferenceDeviceObject(dev); + return dev; +} + __declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) { return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0; } __declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) { DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex); - return dev ? dev->SectorCount : 0; + return dev ? dev->u.Disk.SectorCount : 0; } +// ------------------------------------------------------------------------- +// Dispatch +// ------------------------------------------------------------------------- + __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) { if (!device || !device->DriverObject || !irp) return STATUS_INVALID_PARAMETER; - IO_MAJOR_FUNCTION major = (irp->Flags & IRP_FLAG_WRITE) ? IRP_MJ_WRITE : IRP_MJ_READ; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) + return STATUS_INVALID_PARAMETER; + + IO_MAJOR_FUNCTION major = ios->MajorFunction; if (major >= IRP_MJ_MAXIMUM_FUNCTION) return STATUS_INVALID_PARAMETER; + if (major != IRP_MJ_PNP && + (device->DeletePending || device->PnpState == DevicePnpSurpriseRemoved || + device->PnpState == DevicePnpRemovePending || device->PnpState == DevicePnpDeleted)) + return STATUS_OBJECT_DEAD; + PDRIVER_DISPATCH dispatch = device->DriverObject->MajorFunction[major]; if (!dispatch) return STATUS_NOT_SUPPORTED; - return dispatch(device, irp); + ios->DeviceObject = device; + IoReferenceDeviceObject(device); + _InterlockedIncrement(&device->OutstandingIrps); + + KSTATUS st = dispatch(device, irp); + if (st != STATUS_PENDING) { + if (ios->DeviceObject == device) { + ios->DeviceObject = (DEVICE_OBJECT*)0; + _InterlockedDecrement(&device->OutstandingIrps); + IoDereferenceDeviceObject(device); + } + } + return st; } __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, @@ -243,17 +728,15 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, void* buffer) { if (!buffer || count == 0) return STATUS_INVALID_PARAMETER; - IRP irp; - memset(&irp, 0, sizeof(irp)); - irp.DriveIndex = driveIndex; - irp.Flags = 0; - irp.Lba = lba; - irp.SectorCount = count; - irp.Buffer = buffer; - irp.Status = STATUS_FAILURE; - irp.Information = 0; + DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex); + if (!dev) return STATUS_NOT_FOUND; - return IoSubmitIrpSync(&irp); + IRP irp; + IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer); + KSTATUS st = IoSubmitIrpSync(dev, &irp); + + IoDereferenceDeviceObject(dev); + return st; } __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, @@ -262,15 +745,13 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, const void* buffer) { if (!buffer || count == 0) return STATUS_INVALID_PARAMETER; - IRP irp; - memset(&irp, 0, sizeof(irp)); - irp.DriveIndex = driveIndex; - irp.Flags = IRP_FLAG_WRITE; - irp.Lba = lba; - irp.SectorCount = count; - irp.Buffer = (void*)buffer; - irp.Status = STATUS_FAILURE; - irp.Information = 0; + DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex); + if (!dev) return STATUS_NOT_FOUND; - return IoSubmitIrpSync(&irp); + IRP irp; + IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer); + KSTATUS st = IoSubmitIrpSync(dev, &irp); + + IoDereferenceDeviceObject(dev); + return st; } diff --git a/code/kernel/io_file.c b/code/kernel/io_file.c index da2029b..56a7ee0 100644 --- a/code/kernel/io_file.c +++ b/code/kernel/io_file.c @@ -586,6 +586,33 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex, return STATUS_SUCCESS; } +__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) { + if (volumeIndex >= MAX_DRIVES) + return STATUS_INVALID_PARAMETER; + if (!gVolumes[volumeIndex].Fs) + return STATUS_NOT_FOUND; + + memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex])); + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) { + KSTATUS last = STATUS_NOT_FOUND; + for (uint8_t i = 0; i < MAX_DRIVES; i++) { + if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) { + memset(&gVolumes[i], 0, sizeof(gVolumes[i])); + last = STATUS_SUCCESS; + } + } + return last; +} + +__declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) { + if (volumeIndex >= MAX_DRIVES) + return false; + return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0; +} + // --------------------------------------------------------------------------- // Destructor // --------------------------------------------------------------------------- diff --git a/code/kernel/ke_cpu.c b/code/kernel/ke_cpu.c index 85570ff..126a56e 100644 --- a/code/kernel/ke_cpu.c +++ b/code/kernel/ke_cpu.c @@ -48,6 +48,15 @@ void KeInitializeBootstrapCpu(void) { cpu->Parked = 0; cpu->Online = 1; cpu->IsBsp = 1; + cpu->PackageId = 0; + cpu->CoreId = 0; + cpu->ThreadId = 0; + cpu->PhysicalCoreIndex = 0; + cpu->CoreSiblingMask = 1ULL; + cpu->PackageMask = 1ULL; + cpu->IsPrimaryThread = 1; + cpu->SmtThreadCount = 1; + cpu->TopologyFlags = 0; cpu->CurrentIrql = PASSIVE_LEVEL; cpu->InterruptDepth = 0; cpu->ReschedulePending = 0; diff --git a/code/kernel/ke_debug.c b/code/kernel/ke_debug.c index 50fa77a..a23c57f 100644 --- a/code/kernel/ke_debug.c +++ b/code/kernel/ke_debug.c @@ -13,12 +13,53 @@ #include #include #include +#include #include int _fltused; // MSVC floating-point helper anchor int gDebugEnabled = 0; +static uint32_t gDebugFlags = 0; void (*PrintChar)(char, uint32_t, uint32_t); +#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u) +#define KE_DEBUG_LOG_LINE_SIZE 1024u + +static SPINLOCK gDebugPrintLock = SPINLOCK_INIT; +static char* gDebugLogBuffer; +static size_t gDebugLogSize; +static size_t gDebugLogWrite; +static size_t gDebugLogUsed; +static volatile long gDebugLogFlushing; + +static void KiDebugLogAppendLocked(const char* text, size_t len) { + if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0) + return; + + if (len >= gDebugLogSize) { + text += len - (gDebugLogSize - 1u); + len = gDebugLogSize - 1u; + } + + size_t first = gDebugLogSize - gDebugLogWrite; + if (first > len) first = len; + memcpy(gDebugLogBuffer + gDebugLogWrite, text, first); + if (len > first) + memcpy(gDebugLogBuffer, text + first, len - first); + + gDebugLogWrite = (gDebugLogWrite + len) % gDebugLogSize; + if (gDebugLogUsed + len >= gDebugLogSize) + gDebugLogUsed = gDebugLogSize; + else + gDebugLogUsed += len; +} + +static void KiDebugConsoleWrite(const char* text, size_t len) { + if (!PrintChar || !text) + return; + for (size_t i = 0; i < len; i++) + PrintChar(text[i], 0x00FF00, 0x000000); +} + static volatile long gPanicInProgress = 0; static volatile long gPanicStoppedCpuCount = 0; @@ -50,22 +91,98 @@ void KiKernelPutChar(char c) { PrintChar(c, 0x00FF00, 0x000000); } -void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) { - gDebugEnabled = v; +void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) { + gDebugFlags = flags; + gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0; PrintChar = PrintPtr; printf_set_putchar(KiKernelPutChar); + + if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) { + gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE); + if (gDebugLogBuffer) { + gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE; + gDebugLogWrite = 0; + gDebugLogUsed = 0; + memset(gDebugLogBuffer, 0, gDebugLogSize); + } + } +} + +uint32_t KeGetDebugFlags(void) { + return gDebugFlags; } __declspec(dllexport) void DbgPrint(const char* fmt, ...) { - if (!gDebugEnabled) return; + if (!gDebugEnabled || !fmt) return; - static SPINLOCK sl = {0}; - uintptr_t flags = KeAcquireSpinlock(&sl); + char line[KE_DEBUG_LOG_LINE_SIZE]; va_list va; va_start(va, fmt); - vprintf(fmt, va); + int n = vsnprintf_(line, sizeof(line), fmt, va); va_end(va); - KeReleaseSpinlock(&sl, flags); + + size_t len = 0; + if (n > 0) { + len = (size_t)n; + if (len >= sizeof(line)) + len = sizeof(line) - 1u; + } + + uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock); + if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0)) + KiDebugLogAppendLocked(line, len); + if (gDebugFlags & KE_DEBUG_CONSOLE) + KiDebugConsoleWrite(line, len); + KeReleaseSpinlock(&gDebugPrintLock, flags); +} + +__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { + if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0) + return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) + return STATUS_INVALID_IRQL; + + if (_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 1, 0) != 0) + return STATUS_PENDING; + + size_t used; + size_t start; + uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock); + used = gDebugLogUsed; + start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0; + KeReleaseSpinlock(&gDebugPrintLock, flags); + + if (used == 0) { + _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0); + return STATUS_SUCCESS; + } + + (void)IoDeleteFile(path); + + FILE_OBJECT* file = (FILE_OBJECT*)0; + KSTATUS st = IoCreateFile(path, &file); + if (!KSUCCESS(st)) { + _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0); + return st; + } + + size_t written = 0; + size_t totalWritten = 0; + size_t first = gDebugLogSize - start; + if (first > used) first = used; + st = IoWriteFile(file, gDebugLogBuffer + start, first, &written); + totalWritten += written; + + if (KSUCCESS(st) && used > first) { + written = 0; + st = IoWriteFile(file, gDebugLogBuffer, used - first, &written); + totalWritten += written; + } + + IoCloseFile(file); + + _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0); + return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st; } // KePanic diff --git a/code/kernel/ke_irp.c b/code/kernel/ke_irp.c index 43532c9..f1cbdda 100644 --- a/code/kernel/ke_irp.c +++ b/code/kernel/ke_irp.c @@ -1,130 +1,321 @@ -// ke_irp.c - I/O Request Packet dispatch and queue helpers -// -// This file lives in the kernel layer and is the ONLY place that bridges the -// IRP abstraction to kernel sync objects. All calls to KeEventCreate, -// KeSignalEvent, KeWaitForEvent, KeSignalSemaphore, and ObDereferenceObject -// are confined here; hal_disk.c and io_driver.h remain free of those symbols. -// -// Dependency graph (→ = includes / calls): -// ke_irp.c → ke_irp.h (IRP type, IRP_QUEUE, forward decls) -// → ke_event.h (KeEventCreate, KeSignalEvent, KeWaitForEvent) -// → ke_semaphore.h (KeSignalSemaphore) -// → ob_object.h (ObDereferenceObject) -// → io_driver.h (DEVICE_OBJECT / IoCallDriver) +// ke_irp.c - I/O Request Packet dispatch, queue, cancellation, remove-locks +#include +#include #include #include #include #include #include +#include #include // ------------------------------------------------------------------------- // IRP_QUEUE operations // ------------------------------------------------------------------------- -// These are defined here (not as static inlines in ke_irp.h) so that -// ke_irp.h can avoid including ke_semaphore.h, keeping the header safe -// for inclusion from the HAL layer. -__declspec(dllexport) void IrpQueueInit(IRP_QUEUE *q, struct SEMAPHORE *sem) { +__declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) { q->Lock = (SPINLOCK)SPINLOCK_INIT; q->Semaphore = sem; LIST_INIT(&q->Head); } -__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE *q, IRP *irp) { +__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) { uintptr_t flags = KeAcquireSpinlock(&q->Lock); if (irp->Flags & IRP_FLAG_PRIORITY) ListInsertHead(&q->Head, &irp->Link); else ListInsertTail(&q->Head, &irp->Link); KeReleaseSpinlock(&q->Lock, flags); - // Signal outside the spinlock - safe, and avoids priority-inversion risk. - KeSignalSemaphore((SEMAPHORE *)q->Semaphore); + KeSignalSemaphore((SEMAPHORE*)q->Semaphore); } -__declspec(dllexport) IRP *IrpQueueDequeue(IRP_QUEUE *q) { +__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) { uintptr_t flags = KeAcquireSpinlock(&q->Lock); - LIST_ENTRY *node = ListPopHead(&q->Head); + LIST_ENTRY* node = ListPopHead(&q->Head); KeReleaseSpinlock(&q->Lock, flags); - if (!node) return (IRP *)0; + if (!node) return (IRP*)0; return CONTAINING_RECORD(node, IRP, Link); } -// ------------------------------------------------------------------------- -// IoCompleteRequest -// ------------------------------------------------------------------------- +__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, + KSTATUS status, + IRP_QUEUE_CANCEL_CALLBACK callback, + void* context) { + if (!q) return; + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&q->Lock); + LIST_ENTRY* node = ListPopHead(&q->Head); + KeReleaseSpinlock(&q->Lock, flags); + if (!node) break; -static size_t IoIrpTransferBytes(const IRP *irp, KSTATUS status) { - if (!irp || !KSUCCESS(status)) return 0; - return (size_t)irp->SectorCount * 512u; + IRP* irp = CONTAINING_RECORD(node, IRP, Link); + irp->Cancel = true; + irp->Flags |= IRP_FLAG_CANCELLED; + if (callback) + callback(irp, context); + IoCompleteRequest(irp, status, 0); + } } -__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information) { +__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) { + IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0); +} + +// ------------------------------------------------------------------------- +// Remove locks +// ------------------------------------------------------------------------- + +__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) { + if (!lock) return STATUS_INVALID_PARAMETER; + memset(lock, 0, sizeof(*lock)); + lock->Lock = (SPINLOCK)SPINLOCK_INIT; + lock->IoCount = 1; // sentinel reference released by IoReleaseRemoveLockAndWait + KSTATUS st = KeEventCreate(EventManualReset, 0, &lock->RemovedEvent); + if (!KSUCCESS(st)) return st; + return STATUS_SUCCESS; +} + +__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) { + if (!lock) return; + if (lock->RemovedEvent) { + ObDereferenceObject(&lock->RemovedEvent->Header); + lock->RemovedEvent = (EVENT*)0; + } +} + +__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) { + (void)irp; + if (!lock) return STATUS_INVALID_PARAMETER; + uintptr_t flags = KeAcquireSpinlock(&lock->Lock); + if (lock->Removing) { + KeReleaseSpinlock(&lock->Lock, flags); + return STATUS_OBJECT_DEAD; + } + _InterlockedIncrement(&lock->IoCount); + KeReleaseSpinlock(&lock->Lock, flags); + return STATUS_SUCCESS; +} + +__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) { + (void)irp; + if (!lock) return; + long left = _InterlockedDecrement(&lock->IoCount); + if (left == 0 && lock->Removing && lock->RemovedEvent) + KeSignalEvent(lock->RemovedEvent); +} + +__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp) { + (void)removeIrp; + if (!lock) return; + + uintptr_t flags = KeAcquireSpinlock(&lock->Lock); + lock->Removing = true; + long left = _InterlockedDecrement(&lock->IoCount); // release sentinel + bool done = (left == 0); + KeReleaseSpinlock(&lock->Lock, flags); + + if (done) { + if (lock->RemovedEvent) KeSignalEvent(lock->RemovedEvent); + return; + } + + if (lock->RemovedEvent) + KeWaitForEvent(lock->RemovedEvent, 0); +} + +// ------------------------------------------------------------------------- +// IRP helpers +// ------------------------------------------------------------------------- + +static size_t IoIrpSize(uint8_t stackCount) { + if (stackCount == 0) stackCount = 1; + return sizeof(IRP) + ((size_t)stackCount - 1u) * sizeof(IO_STACK_LOCATION); +} + +__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) { + if (!outIrp) return STATUS_INVALID_PARAMETER; + if (stackCount == 0) stackCount = 1; + IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount)); + if (!irp) return STATUS_NO_MEMORY; + IoInitializeIrp(irp, stackCount); + *outIrp = irp; + return STATUS_SUCCESS; +} + +__declspec(dllexport) void IoFreeIrp(IRP* irp) { + if (irp) MmFreePool(irp); +} + +__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) { + if (!irp) return; + if (stackCount == 0) stackCount = 1; + memset(irp, 0, IoIrpSize(stackCount)); + LIST_INIT(&irp->Link); + irp->Status = STATUS_FAILURE; + irp->Information = 0; + irp->CancelLock = (SPINLOCK)SPINLOCK_INIT; + irp->StackCount = stackCount; + irp->CurrentLocation = 0; +} + +__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp) { + if (!irp || irp->CurrentLocation >= irp->StackCount) return (IO_STACK_LOCATION*)0; + return &irp->Stack[irp->CurrentLocation]; +} + +__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) { + if (!irp || irp->CurrentLocation + 1u >= irp->StackCount) return (IO_STACK_LOCATION*)0; + return &irp->Stack[irp->CurrentLocation + 1u]; +} + +__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) { + if (!irp) return; + if (irp->CurrentLocation + 1u < irp->StackCount) + irp->CurrentLocation++; +} + +__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) { + IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp); + IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp); + if (!cur || !next) return; + memcpy(next, cur, sizeof(*next)); + irp->CurrentLocation++; +} + +__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx) { + if (!irp) return; + irp->CompletionFn = fn; + irp->CompletionCtx = ctx; +} + +__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine) { + if (!irp) return (IRP_CANCEL_ROUTINE)0; + uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock); + IRP_CANCEL_ROUTINE old = irp->CancelRoutine; + irp->CancelRoutine = routine; + KeReleaseSpinlock(&irp->CancelLock, flags); + return old; +} + +__declspec(dllexport) void IoCancelIrp(IRP* irp) { if (!irp) return; + uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock); + irp->Cancel = true; + irp->Flags |= IRP_FLAG_CANCELLED; + IRP_CANCEL_ROUTINE routine = irp->CancelRoutine; + irp->CancelRoutine = (IRP_CANCEL_ROUTINE)0; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0; + KeReleaseSpinlock(&irp->CancelLock, flags); + + if (routine) + routine(dev, irp); +} + +__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp, + IO_MAJOR_FUNCTION major, + uint64_t byteOffset, + uint32_t length, + void* buffer) { + if (!irp) return; + IoInitializeIrp(irp, 1); + irp->UserBuffer = buffer; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) return; + ios->MajorFunction = major; + ios->MinorFunction = 0; + ios->Parameters.ReadWrite.ByteOffset = byteOffset; + ios->Parameters.ReadWrite.Length = length; +} + +__declspec(dllexport) void IoBuildPnpIrp(IRP* irp, + IO_PNP_MINOR_FUNCTION minor) { + if (!irp) return; + uint8_t stackCount = irp->StackCount ? irp->StackCount : 1; + IoInitializeIrp(irp, stackCount); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) return; + ios->MajorFunction = IRP_MJ_PNP; + ios->MinorFunction = (uint8_t)minor; +} + +__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp, + IO_MAJOR_FUNCTION major, + uint32_t ioControlCode, + void* inputBuffer, + uint32_t inputLength, + void* outputBuffer, + uint32_t outputLength) { + if (!irp) return; + uint8_t stackCount = irp->StackCount ? irp->StackCount : 1; + IoInitializeIrp(irp, stackCount); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios) return; + ios->MajorFunction = major; + ios->MinorFunction = 0; + ios->Parameters.DeviceIoControl.IoControlCode = ioControlCode; + ios->Parameters.DeviceIoControl.InputBuffer = inputBuffer; + ios->Parameters.DeviceIoControl.InputLength = inputLength; + ios->Parameters.DeviceIoControl.OutputBuffer = outputBuffer; + ios->Parameters.DeviceIoControl.OutputLength = outputLength; +} + +// ------------------------------------------------------------------------- +// Completion/submission +// ------------------------------------------------------------------------- + +__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information) { + if (!irp) return STATUS_INVALID_PARAMETER; + if (irp->Flags & IRP_FLAG_COMPLETED) + return STATUS_ALREADY_EXISTS; + + irp->Flags |= IRP_FLAG_COMPLETED; irp->Status = status; irp->Information = information; - // Queue the APC before waking event waiters so a thread that waits on the - // completion event will see and deliver the APC on its wait-return path. + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0; + if (dev) { + _InterlockedDecrement(&dev->OutstandingIrps); + IoDereferenceDeviceObject(dev); + ios->DeviceObject = (DEVICE_OBJECT*)0; + } + if (irp->CompletionApc) - KeInsertQueueApc((KAPC *)irp->CompletionApc, irp, (void *)(uintptr_t)status); + KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status); if (irp->CompletionEvent) - KeSignalEvent((EVENT *)irp->CompletionEvent); + KeSignalEvent((EVENT*)irp->CompletionEvent); if (irp->CompletionFn) irp->CompletionFn(irp, irp->CompletionCtx); + + return STATUS_SUCCESS; } -// ------------------------------------------------------------------------- -// IoSubmitIrp -// ------------------------------------------------------------------------- -// Dispatches through the registered DEVICE_OBJECT. -// Accepted IRPs are always completed by the target driver via IoCompleteRequest(). - -__declspec(dllexport) KSTATUS IoSubmitIrp(IRP *irp) { - if (!irp) return STATUS_INVALID_PARAMETER; - +__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return STATUS_INVALID_PARAMETER; irp->Status = STATUS_FAILURE; irp->Information = 0; - - DEVICE_OBJECT *device = IoGetDiskDevice(irp->DriveIndex); - if (!device) return STATUS_NOT_FOUND; - - KSTATUS st = IoCallDriver(device, irp); - - // Definitive driver contract: - // STATUS_PENDING means the IRP was accepted and will be completed via - // IoCompleteRequest(), possibly inline before IoCallDriver returns. - // Any other status means the IRP was rejected and no completion occurs. - return st; + irp->Flags &= ~IRP_FLAG_COMPLETED; + return IoCallDriver(device, irp); } -// ------------------------------------------------------------------------- -// IoSubmitIrpSync -// ------------------------------------------------------------------------- -// Creates a stack-allocated auto-reset event, attaches it to the IRP, -// submits, and blocks. Callers that already provided a CompletionEvent -// should use IoSubmitIrp() directly and wait on their own event instead. +__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return STATUS_INVALID_PARAMETER; -__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp) { - if (!irp) return STATUS_INVALID_PARAMETER; - - EVENT *ev = (EVENT *)0; + EVENT* ev = (EVENT*)0; KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev); if (!KSUCCESS(st)) return st; - // Attach our event; preserve any APC/callback the caller may have set. irp->CompletionEvent = ev; - st = IoSubmitIrp(irp); + st = IoSubmitIrp(device, irp); if (st == STATUS_PENDING) { KeWaitForEvent(ev, 0); st = irp->Status; } - // Release our reference to the temporary event object. ObDereferenceObject(&ev->Header); - return st; } diff --git a/code/kernel/ke_sched.c b/code/kernel/ke_sched.c index 09f998b..b6d285e 100644 --- a/code/kernel/ke_sched.c +++ b/code/kernel/ke_sched.c @@ -36,6 +36,44 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) { return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0; } +static int KiCpuIsIdleLocked(const KCPU* cpu) { + if (!cpu || !cpu->Online) + return 0; + THREAD* cur = cpu->CurrentThread; + return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning); +} + +static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) { + if (!thread || thread->IsIdleThread) + return 0; + + for (uint32_t i = 0; i < gCpuCount; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present || !cpu->Started || !cpu->Online) + continue; + if (!cpu->IsPrimaryThread) + continue; + if (!KiCpuIsIdleLocked(cpu)) + continue; + if (KiThreadCanRunOnCpuLocked(thread, cpu)) + return 1; + } + return 0; +} + +static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) { + if (!cpu || !thread || thread->IsIdleThread) + return 0; + if (cpu->IsPrimaryThread || cpu->SmtThreadCount <= 1) + return 0; + + // Do not fill a secondary SMT sibling while any primary hardware thread + // that can run this thread is idle. KiPokeRunnableCpusLocked targets those + // primary CPUs first, so this CPU can safely remain idle and wait for more + // work or for physical cores to fill. + return KiAnyIdlePrimaryForThreadLocked(thread); +} + static void KiRequestRescheduleCpuLocked(KCPU* target) { if (!target || !target->Online) return; @@ -52,6 +90,9 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) { if (!thread || thread->IsIdleThread) return; + int haveIdlePrimary = KiAnyIdlePrimaryForThreadLocked(thread); + int requestedIdle = 0; + for (uint32_t i = 0; i < gCpuCount; i++) { KCPU* cpu = &gCpus[i]; if (!cpu->Present || !cpu->Started || !cpu->Online) @@ -61,12 +102,35 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) { THREAD* cur = cpu->CurrentThread; if (!cur || cur == cpu->IdleThread) { + if (haveIdlePrimary && !cpu->IsPrimaryThread) + continue; + if (cpu->IsPrimaryThread) + gSchedStats.SmtPrimaryPokes++; KiRequestRescheduleCpuLocked(cpu); + requestedIdle = 1; } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) { gSchedStats.PriorityPreemptions++; KiRequestRescheduleCpuLocked(cpu); } } + + // If no preferred idle primary was actually targetable, allow an idle SMT + // sibling to take the work. This covers explicit affinity to a secondary + // logical CPU and partially-online systems. + if (!requestedIdle && haveIdlePrimary == 0) { + for (uint32_t i = 0; i < gCpuCount; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present || !cpu->Started || !cpu->Online) + continue; + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + THREAD* cur = cpu->CurrentThread; + if (!cur || cur == cpu->IdleThread) { + KiRequestRescheduleCpuLocked(cpu); + break; + } + } + } } static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) { @@ -369,8 +433,11 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) { LIST_ENTRY* head = &gRunQueue[p]; for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); - if (KiThreadCanRunOnCpuLocked(thread, cpu)) - return 1; + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) + continue; + return 1; } } return 0; @@ -385,6 +452,10 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) { THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); if (!KiThreadCanRunOnCpuLocked(thread, cpu)) continue; + if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) { + gSchedStats.SmtSiblingDeferrals++; + continue; + } ListRemove(node); thread->Queued = 0; diff --git a/code/kernel/ke_selftest.c b/code/kernel/ke_selftest.c index 2acfa14..431ee17 100644 --- a/code/kernel/ke_selftest.c +++ b/code/kernel/ke_selftest.c @@ -352,6 +352,7 @@ static void KiRunKernelApcSelfTest(void) { typedef struct KI_IRP_APC_SELFTEST_STATE { EVENT* CompletionEvent; + DEVICE_OBJECT* DiskDevice; IRP Irp; KAPC Apc; uint8_t* Buffer; @@ -430,23 +431,18 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) { KiIrpApcSelfTestNormalRoutine, (void*)&st); - LIST_INIT(&st.Irp.Link); - st.Irp.DriveIndex = 0; - st.Irp.Flags = 0; - st.Irp.Lba = 0; - st.Irp.SectorCount = 1; - st.Irp.Buffer = st.Buffer; - st.Irp.Status = STATUS_FAILURE; - st.Irp.Information = 0; + st.DiskDevice = IoReferenceDiskDevice(0); + if (!st.DiskDevice) + KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: drive 0 disappeared before submit\n"); + + IoBuildReadWriteIrp(&st.Irp, IRP_MJ_READ, 0, 512, st.Buffer); st.Irp.CompletionEvent = st.CompletionEvent; st.Irp.CompletionApc = &st.Apc; st.Irp.CompletionFn = (IRP_COMPLETION_FN)0; st.Irp.CompletionCtx = (void*)0; - st.Irp.HwSlot = 0; - st.Irp.DriverCtx = (void*)0; DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n"); - s = IoSubmitIrp(&st.Irp); + s = IoSubmitIrp(st.DiskDevice, &st.Irp); if (s != STATUS_PENDING) KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: IoSubmitIrp returned %u\n", s); @@ -472,6 +468,7 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) { st.Buffer[510], st.Buffer[511]); MmFreePool(st.Buffer); + if (st.DiskDevice) IoDereferenceDeviceObject(st.DiskDevice); ObDereferenceObject(&st.CompletionEvent->Header); DbgPrint("IRP/APC selftest: PASS\n"); diff --git a/code/kernel/ke_smp.c b/code/kernel/ke_smp.c index 7951889..780d201 100644 --- a/code/kernel/ke_smp.c +++ b/code/kernel/ke_smp.c @@ -321,6 +321,15 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) { cpu->Parked = 0; cpu->Online = 0; cpu->IsBsp = info->IsBsp ? 1 : 0; + cpu->PackageId = 0; + cpu->CoreId = cpuIndex; + cpu->ThreadId = 0; + cpu->PhysicalCoreIndex = cpuIndex; + cpu->CoreSiblingMask = (cpuIndex < 64) ? (1ULL << cpuIndex) : 0ULL; + cpu->PackageMask = cpu->CoreSiblingMask; + cpu->IsPrimaryThread = 1; + cpu->SmtThreadCount = 1; + cpu->TopologyFlags = 0; cpu->CurrentIrql = PASSIVE_LEVEL; cpu->InterruptDepth = 0; cpu->ReschedulePending = 0; @@ -328,6 +337,223 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) { cpu->SchedulerActive = 0; } + +static uint32_t KiCeilLog2U32(uint32_t value) { + if (value <= 1u) + return 0u; + uint32_t shift = 0; + uint32_t n = value - 1u; + while (n) { + shift++; + n >>= 1; + } + return shift; +} + +static uint32_t KiMaskWidth(uint32_t shift) { + if (shift >= 32u) + return 0xFFFFFFFFu; + if (shift == 0u) + return 0u; + return (1u << shift) - 1u; +} + +typedef struct KI_CPUID_TOPOLOGY_PARAMS { + uint32_t SmtShift; + uint32_t CoreShift; + uint32_t ThreadsPerCore; + uint32_t LogicalPerPackage; + const char* Source; +} KI_CPUID_TOPOLOGY_PARAMS; + +static void KiReadCpuidTopologyLeaf(uint32_t leaf, KI_CPUID_TOPOLOGY_PARAMS* out) { + if (!out) + return; + + for (uint32_t sub = 0; sub < 32u; sub++) { + int r[4] = {0, 0, 0, 0}; + __cpuidex(r, (int)leaf, (int)sub); + + uint32_t levelShift = (uint32_t)r[0] & 0x1Fu; + uint32_t logicals = (uint32_t)r[1] & 0xFFFFu; + uint32_t levelType = ((uint32_t)r[2] >> 8) & 0xFFu; + + if (levelType == 0u || logicals == 0u) + break; + + if (levelType == 1u) { // SMT level + out->SmtShift = levelShift; + out->ThreadsPerCore = logicals ? logicals : 1u; + } else if (levelType == 2u) { // Core/package level + out->CoreShift = levelShift; + out->LogicalPerPackage = logicals ? logicals : 1u; + break; + } + } +} + +static KI_CPUID_TOPOLOGY_PARAMS KiDetectCpuidTopologyParams(void) { + KI_CPUID_TOPOLOGY_PARAMS topo; + memset(&topo, 0, sizeof(topo)); + topo.ThreadsPerCore = 1; + topo.LogicalPerPackage = 1; + topo.Source = "flat"; + + int maxRegs[4] = {0, 0, 0, 0}; + __cpuid(maxRegs, 0); + uint32_t maxLeaf = (uint32_t)maxRegs[0]; + + if (maxLeaf >= 0x1Fu) { + KiReadCpuidTopologyLeaf(0x1Fu, &topo); + if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) { + topo.Source = "cpuid.1f"; + return topo; + } + memset(&topo, 0, sizeof(topo)); + topo.ThreadsPerCore = 1; + topo.LogicalPerPackage = 1; + topo.Source = "flat"; + } + + if (maxLeaf >= 0x0Bu) { + KiReadCpuidTopologyLeaf(0x0Bu, &topo); + if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) { + topo.Source = "cpuid.0b"; + return topo; + } + memset(&topo, 0, sizeof(topo)); + topo.ThreadsPerCore = 1; + topo.LogicalPerPackage = 1; + topo.Source = "flat"; + } + + int r1[4] = {0, 0, 0, 0}; + __cpuid(r1, 1); + uint32_t logicalPerPackage = ((uint32_t)r1[1] >> 16) & 0xFFu; + if (logicalPerPackage == 0u) + logicalPerPackage = 1u; + + uint32_t coresPerPackage = 1u; + if (maxLeaf >= 4u) { + int r4[4] = {0, 0, 0, 0}; + __cpuidex(r4, 4, 0); + coresPerPackage = (((uint32_t)r4[0] >> 26) & 0x3Fu) + 1u; + if (coresPerPackage == 0u) + coresPerPackage = 1u; + } + + uint32_t threadsPerCore = logicalPerPackage / coresPerPackage; + if (threadsPerCore == 0u) + threadsPerCore = 1u; + + topo.ThreadsPerCore = threadsPerCore; + topo.LogicalPerPackage = logicalPerPackage; + topo.SmtShift = KiCeilLog2U32(threadsPerCore); + topo.CoreShift = KiCeilLog2U32(logicalPerPackage); + if (topo.CoreShift < topo.SmtShift) + topo.CoreShift = topo.SmtShift; + topo.Source = "cpuid.1/4"; + return topo; +} + +void KeBuildProcessorTopology(void) { + KI_CPUID_TOPOLOGY_PARAMS topo = KiDetectCpuidTopologyParams(); + + uint32_t smtMask = KiMaskWidth(topo.SmtShift); + uint32_t coreBits = (topo.CoreShift > topo.SmtShift) ? + (topo.CoreShift - topo.SmtShift) : 0u; + uint32_t coreMask = KiMaskWidth(coreBits); + + for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present) + continue; + + uint32_t apic = cpu->LapicId; + cpu->ThreadId = smtMask ? (apic & smtMask) : 0u; + cpu->CoreId = coreBits ? ((apic >> topo.SmtShift) & coreMask) : + (apic >> topo.SmtShift); + cpu->PackageId = (topo.CoreShift < 32u) ? (apic >> topo.CoreShift) : 0u; + cpu->PhysicalCoreIndex = KE_INVALID_CPU; + cpu->CoreSiblingMask = 0; + cpu->PackageMask = 0; + cpu->IsPrimaryThread = 0; + cpu->SmtThreadCount = 1; + cpu->TopologyFlags = (topo.ThreadsPerCore > 1u) ? 1u : 0u; + } + + uint32_t nextCoreIndex = 0; + for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present) + continue; + if (cpu->PhysicalCoreIndex != KE_INVALID_CPU) + continue; + + uint32_t coreIndex = nextCoreIndex++; + uint64_t coreMaskBits = 0; + uint64_t packageMaskBits = 0; + uint32_t primaryCpuIndex = i; + uint32_t siblingCount = 0; + + for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) { + KCPU* peer = &gCpus[j]; + if (!peer->Present) + continue; + if (peer->PackageId == cpu->PackageId) { + if (j < 64) + packageMaskBits |= (1ULL << j); + } + if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) { + peer->PhysicalCoreIndex = coreIndex; + if (j < 64) + coreMaskBits |= (1ULL << j); + if (j < primaryCpuIndex) + primaryCpuIndex = j; + siblingCount++; + } + } + + if (siblingCount == 0) + siblingCount = 1; + + for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) { + KCPU* peer = &gCpus[j]; + if (!peer->Present) + continue; + if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) { + peer->CoreSiblingMask = coreMaskBits; + peer->PackageMask = packageMaskBits; + peer->SmtThreadCount = (uint8_t)((siblingCount > 255u) ? 255u : siblingCount); + peer->IsPrimaryThread = (j == primaryCpuIndex) ? 1u : 0u; + } + } + } + + DbgPrint("SMP: SMT topology source=%s smtShift=%u coreShift=%u threads/core=%u logical/package=%u physicalCores=%u\n", + topo.Source ? topo.Source : "?", + topo.SmtShift, + topo.CoreShift, + topo.ThreadsPerCore, + topo.LogicalPerPackage, + nextCoreIndex); + + for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present) + continue; + DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n", + i, + cpu->LapicId, + cpu->PackageId, + cpu->CoreId, + cpu->ThreadId, + cpu->IsPrimaryThread, + (unsigned long long)cpu->CoreSiblingMask, + (unsigned long long)cpu->PackageMask); + } +} + static KSTATUS KiAllocateApStack(KCPU* cpu) { // AP kernel stacks do not need physically contiguous memory. Requiring // MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being @@ -510,6 +736,7 @@ KSTATUS KeStartApplicationProcessors(void) { if (claimed == 0 && gCpuCount > 1) DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n"); + KeBuildProcessorTopology(); DbgPrint("SMP: kernel claimed %u APs\n", claimed); return STATUS_SUCCESS; } diff --git a/code/kernel/kmain.c b/code/kernel/kmain.c index 72a7af1..06398c5 100644 --- a/code/kernel/kmain.c +++ b/code/kernel/kmain.c @@ -341,7 +341,7 @@ void SystemInitThread(void* arg) { KiResolveSystemImports(args->bootInfo); KiReclaimBootloaderMemory(args->bootInfo); - if (args->bootInfo->Debug & 2) + if (args->bootInfo->Debug & KE_DEBUG_SELFTEST) KiRunKernelSelfTests(args->hal); // Launch User-Mode init.exe and restart it until it exits cleanly. @@ -363,6 +363,11 @@ void SystemInitThread(void* arg) { } DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n"); + if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) { + KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log")); + if (!KSUCCESS(logStatus)) + DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus); + } gKmainHal->Power.Shutdown(); MmFreePool(args); } @@ -416,7 +421,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) { const HAL_API* hal = HalGetApi(); gKmainHal = hal; - KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1); + KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug); // CPU startup intentionally lives after HAL initialization but before IRQs // and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI; diff --git a/code/public/ntlite/ddi/io_driver.h b/code/public/ntlite/ddi/io_driver.h index b1d401e..38bc717 100644 --- a/code/public/ntlite/ddi/io_driver.h +++ b/code/public/ntlite/ddi/io_driver.h @@ -1,39 +1,35 @@ #pragma once -// io_driver.h - definitive kernel driver/device I/O model +// io_driver.h - kernel driver/device I/O model // -// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through -// IoCallDriver()/IoSubmitIrp(); drivers complete accepted IRPs by calling -// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table. +// Device lifetime is owned by the I/O manager. Bus drivers create PDOs, +// function/filter drivers attach FDOs, and PnP/remove IRPs tear stacks down. +// Disk indexes are only a compatibility namespace above DEVICE_OBJECTs. #include #include #include #include +#include +#include +#include +#include +#include typedef struct HAL_API HAL_API; typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API* hal); typedef void (*DRIVER_UNLOAD_FN)(void); -struct IRP; typedef struct DRIVER_OBJECT DRIVER_OBJECT; typedef struct DEVICE_OBJECT DEVICE_OBJECT; +typedef struct IRP IRP; -typedef enum IO_MAJOR_FUNCTION { - IRP_MJ_READ = 0, - IRP_MJ_WRITE = 1, - IRP_MJ_DEVICE_CONTROL = 2, - IRP_MJ_MAXIMUM_FUNCTION = 3, -} IO_MAJOR_FUNCTION; - -typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, struct IRP* irp); - -struct DRIVER_OBJECT { - const char* Name; - PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION]; - DRIVER_UNLOAD_FN DriverUnload; -}; - +typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, IRP* irp); +typedef KSTATUS (*PDRIVER_ADD_DEVICE)(DRIVER_OBJECT* driver, + DEVICE_OBJECT* physicalDeviceObject); +// ------------------------------------------------------------------------- +// Bus / identity +// ------------------------------------------------------------------------- typedef enum IO_BUS_TYPE { IoBusUnknown = 0, @@ -41,6 +37,139 @@ typedef enum IO_BUS_TYPE { IoBusUsb = 2, } IO_BUS_TYPE; +typedef struct IO_DEVICE_ID { + IO_BUS_TYPE BusType; + SYS_STRING_VIEW Id; +} IO_DEVICE_ID; + +typedef struct IO_DRIVER_ID_TABLE { + const IO_DEVICE_ID* Ids; + size_t Count; +} IO_DRIVER_ID_TABLE; + +#define IO_MAX_DEVICE_IDS 8u +#define IO_MAX_DEVICE_ID_CHARS 96u +#define IO_MAX_DEVICE_NAME_CHARS 64u + +// ------------------------------------------------------------------------- +// Driver/device objects +// ------------------------------------------------------------------------- + +typedef enum DEVICE_OBJECT_TYPE { + DeviceObjectPhysical = 1, + DeviceObjectFunction = 2, + DeviceObjectFilter = 3, +} DEVICE_OBJECT_TYPE; + +typedef enum DEVICE_PNP_STATE { + DevicePnpNotStarted = 0, + DevicePnpStarted, + DevicePnpStopPending, + DevicePnpStopped, + DevicePnpRemovePending, + DevicePnpSurpriseRemoved, + DevicePnpDeleted, +} DEVICE_PNP_STATE; + +struct DRIVER_OBJECT { + OBJECT Header; + SYS_STRING_VIEW Name; + PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION]; + PDRIVER_ADD_DEVICE DriverAddDevice; + DRIVER_UNLOAD_FN DriverUnload; + const IO_DRIVER_ID_TABLE* IdTable; +}; + +struct DEVICE_OBJECT { + OBJECT Header; + + DRIVER_OBJECT* DriverObject; + DEVICE_OBJECT_TYPE Type; + DEVICE_PNP_STATE PnpState; + + DEVICE_OBJECT* Parent; + DEVICE_OBJECT* FirstChild; + DEVICE_OBJECT* NextSibling; + + DEVICE_OBJECT* AttachedDevice; // device directly above this object + DEVICE_OBJECT* LowerDevice; // device directly below this FDO/filter + + void* DeviceExtension; + + uint32_t Flags; + uint32_t Characteristics; + uint32_t AlignmentRequirement; + + SPINLOCK Lock; + volatile long OutstandingIrps; + bool DeletePending; + + SYS_STRING_VIEW Name; + char NameStorage[IO_MAX_DEVICE_NAME_CHARS]; + + union { + struct { + IO_BUS_TYPE BusType; + void* BusContext; + void* DeviceContext; + IO_DEVICE_ID Ids[IO_MAX_DEVICE_IDS]; + char IdStorage[IO_MAX_DEVICE_IDS][IO_MAX_DEVICE_ID_CHARS]; + uint8_t IdCount; + } Pdo; + + struct { + uint8_t DiskIndex; + uint64_t SectorCount; + uint32_t SectorSize; + } Disk; + } u; +}; + +#define DEVICE_OBJECT_DISK 0x00000001u +#define DEVICE_OBJECT_REMOVABLE 0x00000002u + +// ------------------------------------------------------------------------- +// Object/device creation and stack management +// ------------------------------------------------------------------------- + +__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver, + SYS_STRING_VIEW name); +__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver); + +__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver, + size_t extensionSize, + DEVICE_OBJECT** outDevice); + +__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver, + DEVICE_OBJECT* parent, + const IO_DEVICE_ID* ids, + size_t idCount, + size_t extensionSize, + DEVICE_OBJECT** outPdo); +__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo); + +__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device, + SYS_STRING_VIEW name); +__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device); +__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device); +__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device); + +__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source, + DEVICE_OBJECT* target); +__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source); +__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device); + +__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp); +__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp); +__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device); +__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device, + IO_PNP_MINOR_FUNCTION minor); + +// ------------------------------------------------------------------------- +// Legacy USB publish/matcher path. This remains only until USB is migrated to +// PDO/interface-PDO enumeration in the later USB stages. +// ------------------------------------------------------------------------- + #define IO_PUBLISHED_USB_MAX_ENDPOINTS 8u typedef struct IO_PUBLISHED_USB_ENDPOINT { @@ -102,41 +231,42 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName, __declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device); __declspec(dllexport) uint32_t IoPublishedDeviceCount(void); -struct DEVICE_OBJECT { - DRIVER_OBJECT* DriverObject; - void* DeviceExtension; - uint32_t Flags; - uint32_t AlignmentRequirement; - uint64_t SectorCount; - uint8_t DiskIndex; -}; +// ------------------------------------------------------------------------- +// Disk namespace above storage DEVICE_OBJECTs +// ------------------------------------------------------------------------- -#define DEVICE_OBJECT_DISK 0x00000001u -#define DEVICE_OBJECT_REMOVABLE 0x00000002u +typedef enum IO_DEVICE_CHANGE { + IoDeviceArrived = 0, + IoDeviceQueryRemove, + IoDeviceSurpriseRemoved, + IoDeviceRemoved, +} IO_DEVICE_CHANGE; -__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver, - size_t extensionSize, - DEVICE_OBJECT** outDevice); -__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device); - -__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, - DEVICE_OBJECT* device); -__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex); -__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex); -__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex); +typedef void (*IO_DISK_CHANGE_CALLBACK)(uint8_t driveIndex, + DEVICE_OBJECT* device, + IO_DEVICE_CHANGE change, + void* context); typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex, DEVICE_OBJECT* device, void* context); +__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, + DEVICE_OBJECT* device); +__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex); +__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex); +__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex); +__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex); +__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex); +__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex); + __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback, void* context); +__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback, + void* context); __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback, void* context); -__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, - struct IRP* irp); - __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, uint64_t lba, uint32_t count, diff --git a/code/public/ntlite/syscall.h b/code/public/ntlite/syscall.h index bd51fc6..5cc800b 100644 --- a/code/public/ntlite/syscall.h +++ b/code/public/ntlite/syscall.h @@ -106,7 +106,6 @@ typedef struct SYS_USER_APC_FRAME { void SysUserApcDispatcher(void); - // --------------------------------------------------------------------------- // Trap-frame query ABI // --------------------------------------------------------------------------- diff --git a/readme.md b/readme.md index 6200951..51ede1f 100644 --- a/readme.md +++ b/readme.md @@ -10,7 +10,7 @@ NT-Lite is a 64-bit operating system kernel written from scratch. It has a UEFI It boots into a shell, launches user processes, survives application crashes, and can run things it probably should not be able to run yet. -The codebase is still small enough to understand, but large enough to hurt your feelings. The entire project is about 40k lines. +The codebase is still small enough to understand, but large enough to hurt your feelings. ## features diff --git a/runa.bat b/runa.bat index 771ead2..8bd5d12 100644 --- a/runa.bat +++ b/runa.bat @@ -1 +1,18 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=vpc,file=disk.vhd,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -serial stdio -M q35 -m 256M -smp cpus=4,cores=4,threads=1,sockets=1 -netdev user,id=net0,hostfwd=tcp::8080-:8080 -device rtl8139,netdev=net0 -device qemu-xhci,id=usb -device usb-kbd,bus=usb.0 -device usb-mouse,bus=usb.0 -drive if=none,id=usbflash,format=vpc,file=usb.vhd -device usb-storage,drive=usbflash,bus=usb.0 +@start qemu-system-x86_64 ^ +-drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd ^ +-drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd ^ +-device ahci,id=ahci ^ +-drive format=vpc,file=disk.vhd,if=none,id=disk0 ^ +-device ide-hd,bus=ahci.0,drive=disk0 ^ +-serial stdio ^ +-M q35 ^ +-m 256M ^ +-smp cpus=4,cores=4,threads=1,sockets=1 ^ +-netdev user,id=net0,hostfwd=tcp::8080-:8080 ^ +-device rtl8139,netdev=net0 ^ +-device qemu-xhci,id=usb ^ +-device usb-kbd,bus=usb.0 ^ +-device usb-mouse,bus=usb.0 ^ +-device usb-hub,id=myhub,bus=usb.0,port=3 ^ +-drive if=none,id=usbflash,format=vpc,file=usb.vhd ^ +-device usb-storage,drive=usbflash,bus=usb.0,port=3.1,id=myusbflash