maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

usb hubs, hotplug support, unplug support, saving log files to disk, smt

Browse Source
This commit is contained in:
maksw committed 2026-06-09 09:14:24 +02:00
1 parent b838f8d83d
commit 27b31f62b6
39 files changed
+6294 -735
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/boot.ini
+5 -5
View File
@@ -1,5 +1,5 @@
[kernel]
; bitmask: 1 = debug prints, 2 = self check
debug = 1
[init]
commandline = apps/shell_native.exe
[kernel]
; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file
debug = 4
[init]
commandline = apps/shell_native.exe
code/boot/uefi/main.c
+25 -2
View File
@@ -264,12 +264,35 @@ typedef struct {
char commandLine[64];
} PARSE_STATE;
static int ParseBootIniInteger(const char* val) {
int out = 0;
if (!val) return 0;
while (*val == ' ' || *val == '\t') val++;
int base = 10;
if (val[0] == '0' && (val[1] == 'x' || val[1] == 'X')) {
base = 16;
val += 2;
}
while (*val) {
char c = *val++;
int digit;
if (c >= '0' && c <= '9') digit = c - '0';
else if (c >= 'a' && c <= 'f') digit = 10 + c - 'a';
else if (c >= 'A' && c <= 'F') digit = 10 + c - 'A';
else break;
if (digit >= base) break;
out = out * base + digit;
}
return out;
}
static void BootIniCb(const char* section, const char* key,
const char* val, void* ud) {
PARSE_STATE* s = (PARSE_STATE *)ud;
// [kernel] debug = 1
// [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log
if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug"))
s->debug = (int)(val[0] - '0');
s->debug = ParseBootIniInteger(val);
// [init] commandline = hello.exe
else if (!AsciiStrCmp(section, "init") && !AsciiStrCmp(key, "commandline"))
AsciiStrCpyS(s->commandLine, 64, val);
code/build/build.c
+1 -1
View File
@@ -28,7 +28,7 @@ static const Driver_Target STANDALONE_DRIVERS[] = {
DRIVER_TARGET("ide", "obj\\drivers\\ide", "code\\drivers\\storage\\ide\\ide.c code\\drivers\\storage\\ide\\ide_bus.c"),
DRIVER_TARGET("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c"),
DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c"),
DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_bus.c code\\drivers\\bus\\usb\\usb_composite.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c code\\drivers\\bus\\usb\\usb_hub.c"),
DRIVER_TARGET("rtl816x", "obj\\drivers\\rtl816x", "code\\drivers\\net\\rtl816x\\rtl816x.c code\\drivers\\net\\rtl816x\\rtl816x_pci.c"),
DRIVER_TARGET("rtl8139", "obj\\drivers\\rtl8139", "code\\drivers\\net\\rtl8139\\rtl8139.c code\\drivers\\net\\rtl8139\\rtl8139_pci.c"),
};
code/drivers/bus/usb/usb.c
+26 -2
View File
@@ -75,9 +75,33 @@ KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal) return STATUS_INVALID_PARAMETER;
DbgPrint("usb: discovery starting\n");
KSTATUS st = UsbRegisterClassMatchers();
KSTATUS st = UsbInitializeBusModel();
if (!KSUCCESS(st)) {
DbgPrint("usb: class matcher registration failed st=0x%x\n", st);
DbgPrint("usb: bus model initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeCompositeDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: composite driver initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeHidDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: HID driver initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeStorageDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: storage driver initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeHubDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: hub driver initialisation failed st=0x%x\n", st);
return st;
}
code/drivers/bus/usb/usb.h
+440 -16
View File
@@ -16,6 +16,8 @@
#include <kernel/ke_dpc.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_virt.h>
@@ -114,6 +116,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define XHCI_TRB_TYPE_DATA 3u
#define XHCI_TRB_TYPE_STATUS 4u
#define XHCI_TRB_TYPE_ENABLE_SLOT 9u
#define XHCI_TRB_TYPE_DISABLE_SLOT 10u
#define XHCI_TRB_TYPE_ADDRESS_DEVICE 11u
#define XHCI_TRB_TYPE_CONFIGURE_ENDPOINT 12u
@@ -144,10 +147,15 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define USB_REQ_SET_CONFIGURATION 0x09u
#define USB_REQ_SET_IDLE 0x0Au
#define USB_REQ_SET_PROTOCOL 0x0Bu
#define USB_REQ_GET_STATUS 0x00u
#define USB_REQ_CLEAR_FEATURE 0x01u
#define USB_REQ_SET_FEATURE 0x03u
#define USB_DESC_DEVICE 0x01u
#define USB_DESC_CONFIGURATION 0x02u
#define USB_DESC_INTERFACE 0x04u
#define USB_DESC_ENDPOINT 0x05u
#define USB_DESC_HUB 0x29u
#define USB_DESC_SUPERSPEED_HUB 0x2Au
#define USB_DESC_HID 0x21u
#define USB_DESC_CS_INTERFACE 0x24u
#define USB_DIR_OUT 0x00u
@@ -155,6 +163,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define USB_TYPE_STANDARD 0x00u
#define USB_TYPE_CLASS 0x20u
#define USB_RECIP_DEVICE 0x00u
#define USB_RECIP_OTHER 0x03u
#define USB_RECIP_INTERFACE 0x01u
#define USB_ENDPOINT_DIR_IN 0x80u
#define USB_ENDPOINT_NUMBER_MASK 0x0Fu
@@ -165,6 +174,23 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define USB_ENDPOINT_TYPE_INTERRUPT 0x03u
#define USB_CLASS_HID 0x03u
#define USB_CLASS_MASS_STORAGE 0x08u
#define USB_CLASS_HUB 0x09u
#define USB_HUB_PORT_FEAT_CONNECTION 0u
#define USB_HUB_PORT_FEAT_ENABLE 1u
#define USB_HUB_PORT_FEAT_SUSPEND 2u
#define USB_HUB_PORT_FEAT_OVER_CURRENT 3u
#define USB_HUB_PORT_FEAT_RESET 4u
#define USB_HUB_PORT_FEAT_POWER 8u
#define USB_HUB_PORT_FEAT_C_CONNECTION 16u
#define USB_HUB_PORT_FEAT_C_ENABLE 17u
#define USB_HUB_PORT_FEAT_C_RESET 20u
#define USB_HUB_PORT_STATUS_CONNECTION (1u << 0)
#define USB_HUB_PORT_STATUS_ENABLE (1u << 1)
#define USB_HUB_PORT_STATUS_LOW_SPEED (1u << 9)
#define USB_HUB_PORT_STATUS_HIGH_SPEED (1u << 10)
#define USB_HUB_PORT_CHANGE_CONNECTION (1u << 0)
#define USB_HUB_PORT_CHANGE_ENABLE (1u << 1)
#define USB_HUB_PORT_CHANGE_RESET (1u << 4)
#define USB_MASS_SUBCLASS_SCSI 0x06u
#define USB_MASS_PROTOCOL_BOT 0x50u
#define USB_HID_SUBCLASS_BOOT 0x01u
@@ -193,12 +219,22 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define XHCI_EP0_RING_TRBS 256u
#define XHCI_HID_RING_TRBS 256u
#define XHCI_BULK_RING_TRBS 256u
#define XHCI_PIPE_RING_TRBS 256u
#define XHCI_MAX_HID_REPORT 16u
#define USB_STORAGE_DMA_PAGES 1u
#define USB_STORAGE_MAX_BYTES (USB_STORAGE_DMA_PAGES * PAGE_SIZE)
#define USB_STORAGE_SECTOR_SIZE 512u
#define USB_MAX_TRACKED_INTERFACES 8u
#define USB_MAX_TRACKED_ENDPOINTS 8u
#define USB_MAX_DEVICE_ID_TEXT 96u
#define USB_MAX_PIPES 32u
#define USB_IOCTL_SUBMIT_URB 0x55534201u /* USB1 */
#define USB_ROOT_HUB_MAX_PORTS XHCI_MAX_ROOT_PORTS
#define USB_EXTERNAL_HUB_MAX_PORTS 31u
#define USB_ROOT_PORT_ENUM_RETRIES 3u
#define USB_XHCI_CONTROL_TIMEOUT_US 1000000u
#define USB_XHCI_ENUM_CONTROL_TIMEOUT_US 120000u
#define USB_HUB_CHANGE_BITMAP_BYTES 8u
#pragma pack(push, 1)
typedef struct XHCI_TRB {
@@ -271,6 +307,21 @@ typedef struct USB_ENDPOINT_DESCRIPTOR {
uint8_t bInterval;
} USB_ENDPOINT_DESCRIPTOR;
typedef struct USB_HUB_DESCRIPTOR {
uint8_t bLength;
uint8_t bDescriptorType;
uint8_t bNbrPorts;
uint16_t wHubCharacteristics;
uint8_t bPwrOn2PwrGood;
uint8_t bHubContrCurrent;
uint8_t DeviceRemovable[1];
} USB_HUB_DESCRIPTOR;
typedef struct USB_HUB_PORT_STATUS {
uint16_t Status;
uint16_t Change;
} USB_HUB_PORT_STATUS;
typedef struct USB_MASS_CBW {
uint32_t Signature;
uint32_t Tag;
@@ -308,6 +359,309 @@ typedef struct USB_BAR {
uint8_t Index;
} USB_BAR;
typedef enum USB_DEVICE_STATE {
UsbDeviceAttached = 0,
UsbDevicePowered,
UsbDeviceDefault,
UsbDeviceAddressed,
UsbDeviceConfigured,
UsbDeviceSuspended,
UsbDeviceSurpriseRemoved,
UsbDeviceRemoved,
} USB_DEVICE_STATE;
typedef struct USB_PORT_PATH {
uint8_t RootPort;
uint8_t Depth;
uint32_t RouteString;
uint8_t ParentHubSlot;
uint8_t ParentHubPort;
bool BehindHub;
} USB_PORT_PATH;
typedef enum USB_PIPE_TYPE {
UsbPipeControl = 0,
UsbPipeBulk,
UsbPipeInterrupt,
UsbPipeIsochronous,
} USB_PIPE_TYPE;
typedef enum USB_TRANSFER_DIRECTION {
UsbTransferNoData = 0,
UsbTransferOut,
UsbTransferIn,
} USB_TRANSFER_DIRECTION;
typedef enum USB_URB_FUNCTION {
UrbFunctionControlTransfer = 0,
UrbFunctionBulkTransfer,
UrbFunctionInterruptTransfer,
UrbFunctionSelectConfiguration,
UrbFunctionSelectInterface,
UrbFunctionResetPipe,
UrbFunctionAbortPipe,
} USB_URB_FUNCTION;
typedef struct USB_HCD USB_HCD;
typedef struct USB_DEVICE USB_DEVICE;
typedef struct USB_PIPE USB_PIPE;
typedef struct USB_URB USB_URB;
typedef struct XHCI_DEVICE XHCI_DEVICE;
typedef void (*USB_URB_COMPLETION)(USB_URB* urb, void* context);
typedef struct USB_HCD_OPERATIONS {
KSTATUS (*SubmitUrb)(USB_HCD* hcd, USB_URB* urb);
KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb);
KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed);
KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port);
void* (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys);
} USB_HCD_OPERATIONS;
struct USB_HCD {
const USB_HCD_OPERATIONS* Operations;
void* ControllerContext;
DRIVER_OBJECT* DriverObject;
DEVICE_OBJECT* ControllerPdo;
DEVICE_OBJECT* RootHubPdo;
void* RootHubContext;
uint8_t ControllerIndex;
uint8_t MaxRootPorts;
};
struct USB_DEVICE {
OBJECT Header;
USB_HCD* Hcd;
DEVICE_OBJECT* Pdo;
void* HcdDeviceContext;
USB_DEVICE_STATE State;
USB_PORT_PATH Path;
uint8_t Address;
uint8_t SlotId;
uint8_t Speed;
uint16_t MaxPacketSize0;
uint16_t VendorId;
uint16_t ProductId;
uint16_t BcdUsb;
uint16_t BcdDevice;
uint8_t DeviceClass;
uint8_t DeviceSubclass;
uint8_t DeviceProtocol;
uint8_t ConfigurationValue;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
};
struct USB_PIPE {
OBJECT Header;
USB_DEVICE* Device;
USB_PIPE_TYPE Type;
uint8_t EndpointAddress;
uint8_t EndpointId;
uint16_t MaxPacketSize;
uint8_t Interval;
bool Open;
bool Halted;
void* HcdPipeContext;
};
struct USB_URB {
LIST_ENTRY Link;
USB_URB_FUNCTION Function;
USB_TRANSFER_DIRECTION Direction;
USB_DEVICE* Device;
USB_PIPE* Pipe;
USB_SETUP_PACKET Setup;
void* Buffer;
PHYSICAL_ADDRESS BufferPhys;
uint32_t Length;
uint32_t ActualLength;
KSTATUS Status;
IRP* Irp;
USB_URB_COMPLETION Complete;
void* CompleteContext;
bool Pending;
bool Cancelled;
};
typedef enum USB_PDO_KIND {
UsbPdoController = 1,
UsbPdoDevice = 2,
UsbPdoInterface = 3,
UsbPdoRootHub = 4,
} USB_PDO_KIND;
typedef struct USB_PDO_EXTENSION {
USB_PDO_KIND Kind;
USB_HCD* Hcd;
USB_DEVICE* Device;
uint8_t InterfaceIndex;
USB_INTERFACE_DESCRIPTOR InterfaceDescriptor;
uint8_t EndpointCount;
USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS];
} USB_PDO_EXTENSION;
typedef enum USB_HUB_KIND {
UsbHubKindRoot = 1,
UsbHubKindExternal = 2,
} USB_HUB_KIND;
typedef enum USB_HUB_PORT_STATE {
UsbHubPortDisconnected = 0,
UsbHubPortPowered,
UsbHubPortConnectedDebounce,
UsbHubPortResetting,
UsbHubPortEnabled,
UsbHubPortEnumerating,
UsbHubPortChildPresent,
UsbHubPortFailedConnected,
UsbHubPortRemoving,
} USB_HUB_PORT_STATE;
typedef struct USB_HUB_EXTENSION {
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* HubPdo;
USB_HUB_KIND Kind;
USB_HCD* Hcd;
USB_DEVICE* Device;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
bool Started;
bool SurpriseRemoved;
bool Removed;
uint8_t PortCount;
uint8_t RootPortSlot[USB_ROOT_HUB_MAX_PORTS + 1u];
uint8_t RootPortFailCount[USB_ROOT_HUB_MAX_PORTS + 1u];
USB_HUB_PORT_STATE RootPortState[USB_ROOT_HUB_MAX_PORTS + 1u];
uint8_t ChildSlot[USB_EXTERNAL_HUB_MAX_PORTS + 1u];
USB_HUB_PORT_STATE ChildPortState[USB_EXTERNAL_HUB_MAX_PORTS + 1u];
uint32_t PendingRootPorts[8];
uint32_t PendingHubPorts[1];
SPINLOCK PendingLock;
SEMAPHORE* WorkerSemaphore;
THREAD* WorkerThread;
bool WorkerRunning;
bool InitialScanPending;
bool InitialScanComplete;
bool InitialScanInProgress;
uint8_t InterfaceNumber;
uint8_t InterruptEndpointAddress;
uint16_t InterruptMaxPacket;
uint8_t InterruptInterval;
USB_PIPE* InterruptPipe;
uint8_t* ChangeBitmap;
PHYSICAL_ADDRESS ChangeBitmapPhys;
USB_URB ChangeUrb;
bool ChangeUrbPending;
uint8_t PowerOnToPowerGoodMs;
} USB_HUB_EXTENSION;
typedef struct USB_COMPOSITE_EXTENSION {
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* DevicePdo;
USB_DEVICE* Device;
bool Started;
bool InterfacesCreated;
} USB_COMPOSITE_EXTENSION;
typedef struct USB_HID_EXTENSION {
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* InterfacePdo;
USB_DEVICE* Device;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
bool Started;
bool SurpriseRemoved;
bool Removed;
bool Keyboard;
bool Mouse;
uint8_t InterfaceNumber;
uint8_t Protocol;
uint8_t EndpointAddress;
uint16_t MaxPacketSize;
uint8_t Interval;
USB_PIPE* InterruptPipe;
uint8_t* ReportBuffer;
PHYSICAL_ADDRESS ReportBufferPhys;
uint16_t ReportLength;
USB_URB InterruptUrb;
bool InterruptPending;
uint8_t LastModifiers;
uint8_t LastKeys[6];
uint8_t LastButtons;
uint32_t InputDeviceId;
} USB_HID_EXTENSION;
typedef enum USB_STORAGE_EXTENSION_KIND {
UsbStorageExtensionInterface = 0x55534946u,
UsbStorageExtensionDisk = 0x5553444Bu,
} USB_STORAGE_EXTENSION_KIND;
typedef struct USB_STORAGE_EXTENSION USB_STORAGE_EXTENSION;
typedef struct USB_STORAGE_DISK_EXTENSION USB_STORAGE_DISK_EXTENSION;
struct USB_STORAGE_EXTENSION {
USB_STORAGE_EXTENSION_KIND Kind;
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* InterfacePdo;
DEVICE_OBJECT* DiskDevice;
USB_DEVICE* Device;
XHCI_DEVICE* XhciDevice;
USB_PIPE* BulkInPipe;
USB_PIPE* BulkOutPipe;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
bool Started;
bool SurpriseRemoved;
bool Removed;
uint8_t InterfaceNumber;
uint8_t BulkInAddress;
uint8_t BulkOutAddress;
uint16_t BulkInMaxPacket;
uint16_t BulkOutMaxPacket;
uint32_t Tag;
uint32_t BlockSize;
uint64_t BlockCount;
volatile long IoBusy;
};
struct USB_STORAGE_DISK_EXTENSION {
USB_STORAGE_EXTENSION_KIND Kind;
USB_STORAGE_EXTENSION* Storage;
};
typedef struct XHCI_PIPE_CONTEXT {
XHCI_TRB* Ring;
PHYSICAL_ADDRESS RingPhys;
uint32_t Enqueue;
uint8_t Cycle;
uint8_t EndpointId;
USB_PIPE_TYPE Type;
bool Configured;
bool ActiveTransfer;
uint64_t ActiveTrbPhys;
USB_URB* ActiveUrb;
} XHCI_PIPE_CONTEXT;
typedef struct XHCI_DEVICE {
bool Allocated;
bool Addressed;
@@ -324,6 +678,11 @@ typedef struct XHCI_DEVICE {
uint8_t DeviceProtocol;
uint8_t ConfigurationValue;
bool Configured;
USB_DEVICE* UsbDevice;
DEVICE_OBJECT* UsbPdo;
DEVICE_OBJECT* ParentDeviceObject;
USB_PORT_PATH Path;
USB_PIPE* EndpointPipes[32];
uint8_t InterfaceCount;
USB_TRACKED_INTERFACE Interfaces[USB_MAX_TRACKED_INTERFACES];
@@ -364,6 +723,10 @@ typedef struct XHCI_DEVICE {
uint32_t MassBlockSize;
uint64_t MassBlockCount;
DEVICE_OBJECT* MassDeviceObject;
IO_REMOVE_LOCK MassRemoveLock;
bool MassRemoveLockReady;
bool MassSurpriseRemoved;
bool MassRemoved;
volatile long MassIoBusy;
PHYSICAL_ADDRESS InputContextPhys;
@@ -403,12 +766,14 @@ typedef struct XHCI_TRANSFER_RESULT {
} XHCI_TRANSFER_RESULT;
typedef struct XHCI_WAIT_TARGET {
struct XHCI_WAIT_TARGET* Next;
uint64_t CommandPhys;
uint64_t TransferPhys;
uint8_t TransferSlotId;
uint8_t TransferEndpointId;
bool CommandMatched;
bool TransferMatched;
volatile bool CommandMatched;
volatile bool TransferMatched;
volatile bool Registered;
XHCI_COMMAND_RESULT Command;
XHCI_TRANSFER_RESULT Transfer;
} XHCI_WAIT_TARGET;
@@ -470,15 +835,58 @@ typedef struct XHCI_CONTROLLER {
volatile uint32_t InterruptCount;
volatile uint32_t EventDpcCount;
SPINLOCK EventLock;
SPINLOCK WaitLock;
SPINLOCK CommandLock;
XHCI_WAIT_TARGET* WaiterList;
volatile XHCI_WAIT_TARGET* ActiveWait;
volatile bool Running;
USB_HCD Hcd;
} XHCI_CONTROLLER;
extern XHCI_CONTROLLER gXhci;
extern bool gHaveXhci;
extern DRIVER_OBJECT gUsbStorageDriver;
extern DRIVER_OBJECT gUsbBusDriver;
extern DRIVER_OBJECT gUsbCompositeDriver;
extern DRIVER_OBJECT gUsbHidDriver;
extern DRIVER_OBJECT gUsbHubDriver;
KSTATUS UsbInitializeBusModel(void);
KSTATUS UsbInitializeCompositeDriver(void);
KSTATUS UsbInitializeHidDriver(void);
KSTATUS UsbInitializeStorageDriver(void);
KSTATUS UsbInitializeHubDriver(void);
KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
void UsbInitializeHcd(USB_HCD* hcd,
const USB_HCD_OPERATIONS* ops,
void* controllerContext,
uint8_t controllerIndex,
uint8_t maxRootPorts);
KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device);
KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd);
KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise);
KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise);
KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent);
KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent);
KSTATUS UsbCreatePipe(USB_DEVICE* dev,
USB_PIPE_TYPE type,
uint8_t endpointAddress,
uint16_t maxPacketSize,
uint8_t interval,
USB_PIPE** outPipe);
void UsbReferenceDevice(USB_DEVICE* dev);
void UsbDereferenceDevice(USB_DEVICE* dev);
void UsbReferencePipe(USB_PIPE* pipe);
void UsbDereferencePipe(USB_PIPE* pipe);
KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbCancelUrb(USB_URB* urb);
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys);
KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp);
uint32_t UsbAlignUp32(uint32_t value, uint32_t align);
void XhciBarrier(void);
@@ -512,11 +920,15 @@ const char* UsbEndpointTypeName(uint8_t type);
const char* UsbDirectionName(uint8_t endpointAddress);
const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto);
void XhciLogPortStatus(XHCI_CONTROLLER* xhci);
void XhciPowerRootPorts(XHCI_CONTROLLER* xhci);
void XhciAckInterrupter0(XHCI_CONTROLLER* xhci);
void XhciUpdateErdp(XHCI_CONTROLLER* xhci);
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target);
void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs);
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs);
void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2);
void XhciIsr(void);
@@ -535,12 +947,15 @@ void XhciEnableInterrupts(XHCI_CONTROLLER* xhci);
uint32_t XhciPortReg(uint8_t port);
void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port);
KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed);
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId);
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index);
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs);
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length);
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup);
KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs);
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc);
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength);
uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress);
@@ -561,33 +976,42 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub, uint8_t endpointType, bool wantIn);
bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle);
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe);
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
uint32_t UsbBe32(const uint8_t* p);
void UsbStoreBe32(uint8_t* p, uint32_t v);
void UsbStoreBe16(uint8_t* p, uint16_t v);
KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, bool write, uint64_t lba, uint32_t sectors, void* buffer);
void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void UsbStorageReleaseIo(XHCI_DEVICE* dev);
KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer);
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage);
void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
uint8_t UsbFindFreeDiskIndex(void);
KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev);
KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage);
bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbRegisterClassMatchers(void);
KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationValue);
void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_CONFIGURATION_DESCRIPTOR* config, uint16_t totalLength);
void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_DEVICE_DESCRIPTOR* devDesc);
KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId);
KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci, USB_HUB_EXTENSION* hub, uint8_t hubPort, uint8_t speed, uint8_t* outSlotId);
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port);
void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci);
KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd);
void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port);
void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port);
extern const USB_HCD_OPERATIONS gXhciHcdOperations;
KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device);
code/drivers/bus/usb/usb_bus.c
+656
View File
@@ -0,0 +1,656 @@
// usb_bus.c - USB bus/HCD/PDO/URB core model.
//
// This is the structural USB bus layer used by the upcoming composite, hub,
// and hotplug stages. The legacy direct xHCI class path is still present for
// boot HID/storage during migration, but newly enumerated xHCI devices now also
// get first-class USB PDOs and can accept internal URB IRPs.
#include "usb.h"
DRIVER_OBJECT gUsbBusDriver;
static bool gUsbBusModelReady;
static uint8_t gUsbNextControllerIndex;
static void UsbDeviceDestroy(OBJECT* object) {
USB_DEVICE* dev = (USB_DEVICE*)object;
if (dev->RemoveLockReady) {
IoDestroyRemoveLock(&dev->RemoveLock);
dev->RemoveLockReady = false;
}
if (dev->Pdo) {
IoDereferenceDeviceObject(dev->Pdo);
dev->Pdo = (DEVICE_OBJECT*)0;
}
}
static void UsbPipeDestroy(OBJECT* object) {
USB_PIPE* pipe = (USB_PIPE*)object;
if (pipe->Device) {
UsbDereferenceDevice(pipe->Device);
pipe->Device = (USB_DEVICE*)0;
}
}
void UsbReferenceDevice(USB_DEVICE* dev) {
if (dev) ObReferenceObject(&dev->Header);
}
void UsbDereferenceDevice(USB_DEVICE* dev) {
if (dev) ObDereferenceObject(&dev->Header);
}
void UsbReferencePipe(USB_PIPE* pipe) {
if (pipe) ObReferenceObject(&pipe->Header);
}
void UsbDereferencePipe(USB_PIPE* pipe) {
if (pipe) ObDereferenceObject(&pipe->Header);
}
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma)
return (void*)0;
return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys);
}
static USB_DEVICE* UsbAllocateDevice(void) {
USB_DEVICE* dev = (USB_DEVICE*)ObAllocateObject(ObjectTypeUsbDevice, sizeof(USB_DEVICE));
if (!dev) return (USB_DEVICE*)0;
dev->Header.Destroy = UsbDeviceDestroy;
KSTATUS st = IoInitializeRemoveLock(&dev->RemoveLock);
if (!KSUCCESS(st)) {
ObDereferenceObject(&dev->Header);
return (USB_DEVICE*)0;
}
dev->RemoveLockReady = true;
dev->State = UsbDeviceAttached;
return dev;
}
KSTATUS UsbCreatePipe(USB_DEVICE* dev,
USB_PIPE_TYPE type,
uint8_t endpointAddress,
uint16_t maxPacketSize,
uint8_t interval,
USB_PIPE** outPipe) {
if (!dev || !outPipe) return STATUS_INVALID_PARAMETER;
USB_PIPE* pipe = (USB_PIPE*)ObAllocateObject(ObjectTypeUsbPipe, sizeof(USB_PIPE));
if (!pipe) return STATUS_NO_MEMORY;
pipe->Header.Destroy = UsbPipeDestroy;
pipe->Device = dev;
UsbReferenceDevice(dev);
pipe->Type = type;
pipe->EndpointAddress = endpointAddress;
pipe->EndpointId = XhciEndpointIdFromAddress(endpointAddress);
pipe->MaxPacketSize = maxPacketSize;
pipe->Interval = interval;
pipe->Open = true;
*outPipe = pipe;
return STATUS_SUCCESS;
}
static SYS_STRING_VIEW UsbIdFromBuffer(char* buf) {
return SysSvFromCstr(buf);
}
static void UsbIdPutChar(char* buf, size_t cap, size_t* pos, char ch) {
if (!buf || !pos || cap == 0)
return;
if (*pos + 1 < cap)
buf[*pos] = ch;
(*pos)++;
if (*pos < cap)
buf[*pos] = '\0';
else
buf[cap - 1] = '\0';
}
static void UsbIdAppendLiteral(char* buf, size_t cap, size_t* pos, const char* text) {
if (!text)
return;
while (*text)
UsbIdPutChar(buf, cap, pos, *text++);
}
static void UsbIdAppendHex(char* buf, size_t cap, size_t* pos, uint32_t value, uint8_t digits) {
static const char kHex[] = "0123456789ABCDEF";
while (digits) {
digits--;
UsbIdPutChar(buf, cap, pos, kHex[(value >> (digits * 4u)) & 0xFu]);
}
}
static void UsbFormatVidPidRev(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint16_t rev) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
UsbIdAppendHex(buf, cap, &pos, vid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
UsbIdAppendHex(buf, cap, &pos, pid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&REV_");
UsbIdAppendHex(buf, cap, &pos, rev, 4);
}
static void UsbFormatVidPid(char* buf, size_t cap, uint16_t vid, uint16_t pid) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
UsbIdAppendHex(buf, cap, &pos, vid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
UsbIdAppendHex(buf, cap, &pos, pid, 4);
}
static void UsbFormatVidPidMi(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint8_t mi) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
UsbIdAppendHex(buf, cap, &pos, vid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
UsbIdAppendHex(buf, cap, &pos, pid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&MI_");
UsbIdAppendHex(buf, cap, &pos, mi, 2);
}
static void UsbFormatClassSubProt(char* buf, size_t cap, uint8_t cls, uint8_t sub, uint8_t proto) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_");
UsbIdAppendHex(buf, cap, &pos, cls, 2);
UsbIdAppendLiteral(buf, cap, &pos, "&SUBCLASS_");
UsbIdAppendHex(buf, cap, &pos, sub, 2);
UsbIdAppendLiteral(buf, cap, &pos, "&PROT_");
UsbIdAppendHex(buf, cap, &pos, proto, 2);
}
static void UsbFormatClass(char* buf, size_t cap, uint8_t cls) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_");
UsbIdAppendHex(buf, cap, &pos, cls, 2);
}
static void UsbFormatComposite(char* buf, size_t cap) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\COMPOSITE");
}
static void UsbFormatPciXhci(char* buf, size_t cap, uint16_t vendor, uint16_t device) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "PCI\\VEN_");
UsbIdAppendHex(buf, cap, &pos, vendor, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&DEV_");
UsbIdAppendHex(buf, cap, &pos, device, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&CC_0C0330");
}
static void UsbFormatRootHub(char* buf, size_t cap) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\ROOT_HUB");
}
static void UsbMakeDeviceIds(const XHCI_DEVICE* xdev, IO_DEVICE_ID* ids, char idbuf[][USB_MAX_DEVICE_ID_TEXT], size_t* outCount) {
size_t n = 0;
UsbFormatVidPidRev(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, xdev->BcdDevice);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatVidPid(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass, xdev->DeviceSubclass, xdev->DeviceProtocol);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
if (xdev->DeviceClass == 0 || xdev->InterfaceCount > 1) {
UsbFormatComposite(idbuf[n], USB_MAX_DEVICE_ID_TEXT);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
}
*outCount = n;
}
static void UsbMakeInterfaceIds(const XHCI_DEVICE* xdev,
const USB_TRACKED_INTERFACE* iface,
IO_DEVICE_ID* ids,
char idbuf[][USB_MAX_DEVICE_ID_TEXT],
size_t* outCount) {
size_t n = 0;
const USB_INTERFACE_DESCRIPTOR* d = &iface->Descriptor;
UsbFormatVidPidMi(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, d->bInterfaceNumber);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass, d->bInterfaceSubClass, d->bInterfaceProtocol);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
*outCount = n;
}
KSTATUS UsbInitializeBusModel(void) {
if (gUsbBusModelReady)
return STATUS_SUCCESS;
memset(&gUsbBusDriver, 0, sizeof(gUsbBusDriver));
IoInitializeDriverObject(&gUsbBusDriver, SYS_SV_LITERAL("usb-bus"));
gUsbBusDriver.MajorFunction[IRP_MJ_PNP] = UsbDispatchPdoPnp;
gUsbBusDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl;
KSTATUS st = IoRegisterDriverObject(&gUsbBusDriver);
if (!KSUCCESS(st)) return st;
gUsbBusModelReady = true;
return STATUS_SUCCESS;
}
void UsbInitializeHcd(USB_HCD* hcd,
const USB_HCD_OPERATIONS* ops,
void* controllerContext,
uint8_t controllerIndex,
uint8_t maxRootPorts) {
if (!hcd) return;
memset(hcd, 0, sizeof(*hcd));
hcd->Operations = ops;
hcd->ControllerContext = controllerContext;
hcd->DriverObject = &gUsbBusDriver;
hcd->ControllerIndex = controllerIndex ? controllerIndex : ++gUsbNextControllerIndex;
hcd->MaxRootPorts = maxRootPorts;
}
KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device) {
if (!hcd) return STATUS_INVALID_PARAMETER;
if (hcd->ControllerPdo) return STATUS_ALREADY_EXISTS;
IO_DEVICE_ID ids[2];
char idbuf[2][USB_MAX_DEVICE_ID_TEXT];
UsbFormatPciXhci(idbuf[0], USB_MAX_DEVICE_ID_TEXT, vendor, device);
{
size_t pos = 0;
idbuf[1][0] = '\0';
UsbIdAppendLiteral(idbuf[1], USB_MAX_DEVICE_ID_TEXT, &pos, "PCI\\CC_0C0330");
}
ids[0] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[0]) };
ids[1] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[1]) };
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver, (DEVICE_OBJECT*)0, ids, 2, sizeof(USB_PDO_EXTENSION), &pdo);
if (!KSUCCESS(st)) return st;
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("xhci-controller"));
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoController;
ext->Hcd = hcd;
hcd->ControllerPdo = pdo;
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
return STATUS_SUCCESS;
}
KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) {
if (!hcd) return STATUS_INVALID_PARAMETER;
if (hcd->RootHubPdo) return STATUS_ALREADY_EXISTS;
IO_DEVICE_ID ids[2];
char idbuf[2][USB_MAX_DEVICE_ID_TEXT];
UsbFormatRootHub(idbuf[0], USB_MAX_DEVICE_ID_TEXT);
UsbFormatClass(idbuf[1], USB_MAX_DEVICE_ID_TEXT, USB_CLASS_HUB);
ids[0] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[0]) };
ids[1] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[1]) };
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
hcd->ControllerPdo,
ids,
2,
sizeof(USB_PDO_EXTENSION),
&pdo);
if (!KSUCCESS(st)) return st;
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-root-hub"));
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoRootHub;
ext->Hcd = hcd;
ext->Device = (USB_DEVICE*)0;
hcd->RootHubPdo = pdo;
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
DbgPrint("usb: created root hub PDO ports=%u\n", hcd->MaxRootPorts);
return STATUS_SUCCESS;
}
KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
if (!hcd || !xdev) return STATUS_INVALID_PARAMETER;
if (xdev->UsbPdo) return STATUS_ALREADY_EXISTS;
USB_DEVICE* udev = UsbAllocateDevice();
if (!udev) return STATUS_NO_MEMORY;
udev->Hcd = hcd;
udev->HcdDeviceContext = xdev;
udev->State = xdev->Configured ? UsbDeviceConfigured : (xdev->Addressed ? UsbDeviceAddressed : UsbDeviceAttached);
udev->SlotId = xdev->SlotId;
udev->Speed = xdev->Speed;
udev->Address = xdev->SlotId;
udev->MaxPacketSize0 = xdev->MaxPacketSize0;
udev->VendorId = xdev->VendorId;
udev->ProductId = xdev->ProductId;
udev->BcdUsb = xdev->BcdUsb;
udev->BcdDevice = xdev->BcdDevice;
udev->DeviceClass = xdev->DeviceClass;
udev->DeviceSubclass = xdev->DeviceSubclass;
udev->DeviceProtocol = xdev->DeviceProtocol;
udev->ConfigurationValue = xdev->ConfigurationValue;
udev->Path = xdev->Path;
if (udev->Path.RootPort == 0) {
udev->Path.RootPort = xdev->Port;
udev->Path.Depth = 0;
udev->Path.RouteString = 0;
}
IO_DEVICE_ID ids[5];
char idbuf[5][USB_MAX_DEVICE_ID_TEXT];
size_t idCount = 0;
UsbMakeDeviceIds(xdev, ids, idbuf, &idCount);
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
parent ? parent : (hcd->RootHubPdo ? hcd->RootHubPdo : hcd->ControllerPdo),
ids,
idCount,
sizeof(USB_PDO_EXTENSION),
&pdo);
if (!KSUCCESS(st)) {
UsbDereferenceDevice(udev);
return st;
}
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-device"));
udev->Pdo = pdo;
IoReferenceDeviceObject(pdo);
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoDevice;
ext->Hcd = hcd;
ext->Device = udev;
xdev->UsbDevice = udev;
xdev->UsbPdo = pdo;
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
DbgPrint("usb: created device PDO slot=%u vid=%04x pid=%04x ids=%llu\n",
xdev->SlotId, xdev->VendorId, xdev->ProductId, (unsigned long long)idCount);
return STATUS_SUCCESS;
}
KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
if (!hcd || !xdev || !xdev->UsbDevice) return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* deviceParent = parent ? parent : xdev->UsbPdo;
if (!deviceParent) return STATUS_INVALID_PARAMETER;
for (uint8_t i = 0; i < xdev->InterfaceCount; i++) {
USB_TRACKED_INTERFACE* iface = &xdev->Interfaces[i];
if (!iface->Present)
continue;
IO_DEVICE_ID ids[3];
char idbuf[3][USB_MAX_DEVICE_ID_TEXT];
size_t idCount = 0;
UsbMakeInterfaceIds(xdev, iface, ids, idbuf, &idCount);
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbCompositeDriver,
deviceParent,
ids,
idCount,
sizeof(USB_PDO_EXTENSION),
&pdo);
if (!KSUCCESS(st)) {
DbgPrint("usb: create interface PDO slot=%u if=%u failed st=0x%x\n",
xdev->SlotId, iface->Descriptor.bInterfaceNumber, st);
return st;
}
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-interface"));
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoInterface;
ext->Hcd = hcd;
ext->Device = xdev->UsbDevice;
ext->InterfaceIndex = i;
memcpy(&ext->InterfaceDescriptor, &iface->Descriptor, sizeof(iface->Descriptor));
ext->EndpointCount = iface->EndpointCount;
for (uint8_t ep = 0; ep < iface->EndpointCount && ep < USB_MAX_TRACKED_ENDPOINTS; ep++)
memcpy(&ext->Endpoints[ep], &iface->Endpoints[ep], sizeof(USB_ENDPOINT_DESCRIPTOR));
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
DbgPrint("usb: created interface PDO slot=%u if=%u class=%02x subclass=%02x proto=%02x\n",
xdev->SlotId,
iface->Descriptor.bInterfaceNumber,
iface->Descriptor.bInterfaceClass,
iface->Descriptor.bInterfaceSubClass,
iface->Descriptor.bInterfaceProtocol);
}
return STATUS_SUCCESS;
}
static DEVICE_OBJECT* UsbReferenceFirstChild(DEVICE_OBJECT* parent) {
if (!parent) return (DEVICE_OBJECT*)0;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
DEVICE_OBJECT* child = parent->FirstChild;
if (child)
IoReferenceDeviceObject(child);
KeReleaseSpinlock(&parent->Lock, flags);
return child;
}
static void UsbSendPnpIfAlive(DEVICE_OBJECT* pdo, IO_PNP_MINOR_FUNCTION minor) {
if (!pdo || pdo->DeletePending)
return;
(void)IoSendPnpIrpSync(pdo, minor);
}
KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) {
if (!parent)
return STATUS_INVALID_PARAMETER;
for (;;) {
DEVICE_OBJECT* child = UsbReferenceFirstChild(parent);
if (!child)
break;
if (child->Type == DeviceObjectPhysical)
(void)UsbRemovePdoSubtree(child, surprise);
else
(void)UsbRemoveDeviceObjectChildren(child, surprise);
IoDereferenceDeviceObject(child);
}
return STATUS_SUCCESS;
}
KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise) {
if (!pdo || pdo->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
// Child PDOs may be owned either directly by the PDO or by an attached FDO
// such as usb-composite or usb-hub. The attached driver's PnP handler is
// therefore responsible for removing its own child PDOs before forwarding
// REMOVE_DEVICE to the lower PDO.
(void)UsbRemoveDeviceObjectChildren(pdo, surprise);
if (surprise)
UsbSendPnpIfAlive(pdo, IRP_MN_SURPRISE_REMOVAL);
UsbSendPnpIfAlive(pdo, IRP_MN_REMOVE_DEVICE);
return STATUS_SUCCESS;
}
static KSTATUS UsbSubmitUrbToHcd(USB_PDO_EXTENSION* ext, USB_URB* urb) {
if (!ext || !urb) return STATUS_INVALID_PARAMETER;
USB_HCD* hcd = ext->Hcd;
if (!hcd || !hcd->Operations || !hcd->Operations->SubmitUrb)
return STATUS_NOT_SUPPORTED;
if (!urb->Device)
urb->Device = ext->Device;
if (!urb->Device)
return STATUS_INVALID_PARAMETER;
return hcd->Operations->SubmitUrb(hcd, urb);
}
KSTATUS UsbCancelUrb(USB_URB* urb) {
if (!urb || !urb->Device || !urb->Device->Hcd)
return STATUS_INVALID_PARAMETER;
USB_HCD* hcd = urb->Device->Hcd;
if (!hcd->Operations || !hcd->Operations->CancelUrb)
return STATUS_NOT_SUPPORTED;
return hcd->Operations->CancelUrb(hcd, urb);
}
KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb) {
if (!target || !urb) return STATUS_INVALID_PARAMETER;
IRP irp;
IoInitializeIrp(&irp, 1);
IoBuildDeviceIoControlIrp(&irp,
IRP_MJ_INTERNAL_DEVICE_CONTROL,
USB_IOCTL_SUBMIT_URB,
urb,
sizeof(*urb),
urb,
sizeof(*urb));
KSTATUS st = IoSubmitIrpSync(target, &irp);
return KSUCCESS(st) ? urb->Status : st;
}
static void UsbAsyncUrbIrpComplete(IRP* irp, void* context) {
(void)context;
IoFreeIrp(irp);
}
KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb) {
if (!target || !urb) return STATUS_INVALID_PARAMETER;
IRP* irp = (IRP*)0;
KSTATUS st = IoAllocateIrp(1, &irp);
if (!KSUCCESS(st)) return st;
IoBuildDeviceIoControlIrp(irp,
IRP_MJ_INTERNAL_DEVICE_CONTROL,
USB_IOCTL_SUBMIT_URB,
urb,
sizeof(*urb),
urb,
sizeof(*urb));
IoSetCompletionRoutine(irp, UsbAsyncUrbIrpComplete, (void*)0);
st = IoSubmitIrp(target, irp);
if (st != STATUS_PENDING && !(irp->Flags & IRP_FLAG_COMPLETED)) {
IoFreeIrp(irp);
return st;
}
return st;
}
KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return STATUS_INVALID_PARAMETER;
if (ios->Parameters.DeviceIoControl.IoControlCode != USB_IOCTL_SUBMIT_URB) {
IoCompleteRequest(irp, STATUS_NOT_SUPPORTED, 0);
return STATUS_PENDING;
}
if (ios->Parameters.DeviceIoControl.InputLength < sizeof(USB_URB) ||
!ios->Parameters.DeviceIoControl.InputBuffer) {
IoCompleteRequest(irp, STATUS_INVALID_PARAMETER, 0);
return STATUS_PENDING;
}
USB_URB* urb = (USB_URB*)ios->Parameters.DeviceIoControl.InputBuffer;
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension;
urb->Irp = irp;
KSTATUS st = UsbSubmitUrbToHcd(ext, urb);
urb->Status = st;
if (st == STATUS_PENDING) {
urb->Pending = true;
return STATUS_PENDING;
}
urb->Pending = false;
IoCompleteRequest(irp, st, KSUCCESS(st) ? urb->ActualLength : 0);
return STATUS_PENDING;
}
KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension;
KSTATUS st = STATUS_SUCCESS;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
if (ext && ext->Device && ext->Device->State < UsbDeviceConfigured)
ext->Device->State = UsbDeviceConfigured;
break;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
break;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
break;
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
if (ext && ext->Device)
ext->Device->State = UsbDeviceSurpriseRemoved;
break;
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
if (ext && ext->Device)
ext->Device->State = UsbDeviceRemoved;
IoDeleteDevice(device);
break;
default:
st = STATUS_NOT_SUPPORTED;
break;
}
if (st == STATUS_NOT_SUPPORTED)
return st;
IoCompleteRequest(irp, st, 0);
return STATUS_PENDING;
}
code/drivers/bus/usb/usb_composite.c
+150
View File
@@ -0,0 +1,150 @@
// usb_composite.c - USB composite function driver and interface-PDO bus.
#include "usb.h"
DRIVER_OBJECT gUsbCompositeDriver;
static const IO_DEVICE_ID gUsbCompositeIds[] = {
{ IoBusUsb, { "USB\\COMPOSITE", sizeof("USB\\COMPOSITE") - 1 } },
};
static const IO_DRIVER_ID_TABLE gUsbCompositeIdTable = {
gUsbCompositeIds,
sizeof(gUsbCompositeIds) / sizeof(gUsbCompositeIds[0]),
};
static KSTATUS UsbCompositeCreateInterfaces(USB_COMPOSITE_EXTENSION* ext) {
if (!ext || !ext->Device || ext->InterfacesCreated)
return STATUS_SUCCESS;
XHCI_DEVICE* xdev = (XHCI_DEVICE*)ext->Device->HcdDeviceContext;
if (!xdev)
return STATUS_INVALID_PARAMETER;
KSTATUS st = UsbCreateInterfacePdos(ext->Device->Hcd, xdev, ext->Self);
if (KSUCCESS(st))
ext->InterfacesCreated = true;
return st;
}
static KSTATUS UsbCompositeForwardPnp(DEVICE_OBJECT* device, IRP* irp, KSTATUS localStatus) {
if (!KSUCCESS(localStatus)) {
IoCompleteRequest(irp, localStatus, 0);
return STATUS_PENDING;
}
return IoForwardIrpToLower(device, irp);
}
KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
// The composite driver owns two different device-object roles:
// * an FDO attached above a USB device PDO, and
// * interface PDOs that it creates for class drivers.
// Forwarded PnP IRPs from HID/storage land on the interface PDO, so the
// PDO must use the generic USB PDO PnP handler. Treating its
// USB_PDO_EXTENSION as USB_COMPOSITE_EXTENSION corrupts the dispatch path.
if (device->Type == DeviceObjectPhysical)
return UsbDispatchPdoPnp(device, irp);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)device->DeviceExtension;
KSTATUS st = STATUS_SUCCESS;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
if (ext) {
if (ext->DevicePdo)
ext->DevicePdo->PnpState = DevicePnpStarted;
ext->Started = true;
st = UsbCompositeCreateInterfaces(ext);
}
return UsbCompositeForwardPnp(device, irp, st);
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return IoForwardIrpToLower(device, irp);
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
return IoForwardIrpToLower(device, irp);
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
if (ext)
ext->Started = false;
(void)UsbRemoveDeviceObjectChildren(device, true);
return IoForwardIrpToLower(device, irp);
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
if (ext) {
ext->Started = false;
(void)UsbRemoveDeviceObjectChildren(device, false);
if (ext->LowerDevice) {
KSTATUS lower = IoForwardIrpToLower(device, irp);
IoDetachDevice(device);
ext->LowerDevice = (DEVICE_OBJECT*)0;
IoDeleteDevice(device);
if (lower == STATUS_NOT_SUPPORTED) {
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
}
if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
IoCompleteRequest(irp, lower, 0);
return STATUS_PENDING;
}
return lower;
}
}
IoDeleteDevice(device);
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
default:
return IoForwardIrpToLower(device, irp);
}
}
KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoDevice || !pdoExt->Device)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(driver, sizeof(USB_COMPOSITE_EXTENSION), &fdo);
if (!KSUCCESS(st))
return st;
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-composite"));
USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)fdo->DeviceExtension;
ext->Self = fdo;
ext->DevicePdo = physicalDeviceObject;
ext->Device = pdoExt->Device;
ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!ext->LowerDevice) {
IoDeleteDevice(fdo);
return STATUS_FAILURE;
}
DbgPrint("usb: composite attached slot=%u\n", ext->Device->SlotId);
return STATUS_SUCCESS;
}
KSTATUS UsbInitializeCompositeDriver(void) {
memset(&gUsbCompositeDriver, 0, sizeof(gUsbCompositeDriver));
IoInitializeDriverObject(&gUsbCompositeDriver, SYS_SV_LITERAL("usb-composite"));
gUsbCompositeDriver.IdTable = &gUsbCompositeIdTable;
gUsbCompositeDriver.DriverAddDevice = UsbCompositeAddDevice;
gUsbCompositeDriver.MajorFunction[IRP_MJ_PNP] = UsbCompositeDispatchPnp;
gUsbCompositeDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl;
return IoRegisterDriverObject(&gUsbCompositeDriver);
}
code/drivers/bus/usb/usb_core.c
+190 -99
View File
@@ -25,6 +25,16 @@ KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
static USB_TRACKED_INTERFACE* UsbFindTrackedInterfaceByNumber(XHCI_DEVICE* dev, uint8_t interfaceNumber) {
if (!dev) return (USB_TRACKED_INTERFACE*)0;
for (uint8_t i = 0; i < dev->InterfaceCount && i < USB_MAX_TRACKED_INTERFACES; i++) {
if (dev->Interfaces[i].Present &&
dev->Interfaces[i].Descriptor.bInterfaceNumber == interfaceNumber)
return &dev->Interfaces[i];
}
return (USB_TRACKED_INTERFACE*)0;
}
void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_CONFIGURATION_DESCRIPTOR* config,
uint16_t totalLength) {
@@ -79,17 +89,38 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
massBulkOut = 0;
tracked = (USB_TRACKED_INTERFACE*)0;
if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) {
tracked = &dev->Interfaces[dev->InterfaceCount++];
memset(tracked, 0, sizeof(*tracked));
tracked->Present = true;
memcpy(&tracked->Descriptor, iface, sizeof(*iface));
if (iface->bAlternateSetting == 0) {
USB_TRACKED_INTERFACE* existing = UsbFindTrackedInterfaceByNumber(dev, iface->bInterfaceNumber);
if (existing) {
tracked = existing;
memset(tracked->Endpoints, 0, sizeof(tracked->Endpoints));
tracked->EndpointCount = 0;
memcpy(&tracked->Descriptor, iface, sizeof(*iface));
UsbTrace("usb: slot=%u duplicate alt0 interface %u replaced existing PDO candidate\n",
dev->SlotId, iface->bInterfaceNumber);
} else if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) {
tracked = &dev->Interfaces[dev->InterfaceCount++];
memset(tracked, 0, sizeof(*tracked));
tracked->Present = true;
memcpy(&tracked->Descriptor, iface, sizeof(*iface));
} else {
DbgPrint("usb: slot=%u too many interfaces, interface %u alt=0 not published\n",
dev->SlotId, iface->bInterfaceNumber);
}
} else {
DbgPrint("usb: slot=%u too many interfaces, interface %u not published\n",
dev->SlotId, iface->bInterfaceNumber);
/*
* Alternate settings are not independent USB interfaces. Real
* UVC cameras often expose one VideoStreaming interface with
* many alternate settings, and publishing each alternate setting
* as a separate PDO causes duplicate interface devices and can
* exhaust USB_MAX_TRACKED_INTERFACES. Track only alternate
* setting zero until a function driver explicitly selects a
* different setting.
*/
tracked = (USB_TRACKED_INTERFACE*)0;
}
UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)\n",
UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)%s\n",
dev->SlotId,
iface->bInterfaceNumber,
iface->bAlternateSetting,
@@ -97,7 +128,8 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
currentClass,
currentSubClass,
currentProtocol,
UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol));
UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol),
iface->bAlternateSetting ? " alt-not-published" : "");
} else if (type == USB_DESC_ENDPOINT && len >= sizeof(USB_ENDPOINT_DESCRIPTOR)) {
const USB_ENDPOINT_DESCRIPTOR* ep = (const USB_ENDPOINT_DESCRIPTOR*)desc;
uint8_t epType = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
@@ -178,54 +210,6 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
for (uint8_t i = 0; i < dev->InterfaceCount; i++) {
USB_TRACKED_INTERFACE* iface = &dev->Interfaces[i];
if (!iface->Present)
continue;
IO_PUBLISHED_DEVICE pub;
memset(&pub, 0, sizeof(pub));
pub.BusType = IoBusUsb;
pub.Name = "usb-interface";
pub.BusContext = xhci;
pub.DeviceContext = dev;
pub.u.Usb.VendorId = dev->VendorId;
pub.u.Usb.ProductId = dev->ProductId;
pub.u.Usb.BcdUsb = dev->BcdUsb;
pub.u.Usb.BcdDevice = dev->BcdDevice;
pub.u.Usb.DeviceClass = dev->DeviceClass;
pub.u.Usb.DeviceSubclass = dev->DeviceSubclass;
pub.u.Usb.DeviceProtocol = dev->DeviceProtocol;
pub.u.Usb.SlotId = dev->SlotId;
pub.u.Usb.Port = dev->Port;
pub.u.Usb.Speed = dev->Speed;
pub.u.Usb.ConfigurationValue = dev->ConfigurationValue;
pub.u.Usb.InterfaceNumber = iface->Descriptor.bInterfaceNumber;
pub.u.Usb.AlternateSetting = iface->Descriptor.bAlternateSetting;
pub.u.Usb.InterfaceClass = iface->Descriptor.bInterfaceClass;
pub.u.Usb.InterfaceSubclass = iface->Descriptor.bInterfaceSubClass;
pub.u.Usb.InterfaceProtocol = iface->Descriptor.bInterfaceProtocol;
pub.u.Usb.EndpointCount = iface->EndpointCount;
if (pub.u.Usb.EndpointCount > IO_PUBLISHED_USB_MAX_ENDPOINTS)
pub.u.Usb.EndpointCount = IO_PUBLISHED_USB_MAX_ENDPOINTS;
for (uint8_t ep = 0; ep < pub.u.Usb.EndpointCount; ep++) {
pub.u.Usb.Endpoints[ep].Address = iface->Endpoints[ep].bEndpointAddress;
pub.u.Usb.Endpoints[ep].Attributes = iface->Endpoints[ep].bmAttributes;
pub.u.Usb.Endpoints[ep].MaxPacketSize = iface->Endpoints[ep].wMaxPacketSize;
pub.u.Usb.Endpoints[ep].Interval = iface->Endpoints[ep].bInterval;
}
KSTATUS st = IoPublishDevice(&pub);
if (!KSUCCESS(st)) {
DbgPrint("usb: publish interface slot=%u if=%u failed st=0x%x\n",
dev->SlotId, iface->Descriptor.bInterfaceNumber, st);
}
}
}
KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_DEVICE_DESCRIPTOR* devDesc) {
@@ -255,43 +239,89 @@ KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
st = XhciSetConfiguration(xhci, dev, config.bConfigurationValue);
if (!KSUCCESS(st)) return st;
XhciPublishUsbInterfaces(xhci, dev);
if (!dev->UsbPdo) {
st = UsbCreateDevicePdo(&xhci->Hcd, dev,
dev->ParentDeviceObject ? dev->ParentDeviceObject :
(xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : xhci->Hcd.ControllerPdo));
if (!KSUCCESS(st)) {
DbgPrint("usb: create device PDO slot=%u failed st=0x%x\n", dev->SlotId, st);
return st;
}
}
return STATUS_SUCCESS;
}
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
uint8_t speed = 0;
KSTATUS st = XhciResetPort(xhci, port, &speed);
if (!KSUCCESS(st)) return st;
static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS)
return;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
if (dev->UsbPdo) {
(void)UsbRemovePdoSubtree(dev->UsbPdo, true);
dev->UsbPdo = (DEVICE_OBJECT*)0;
}
if (dev->UsbDevice) {
dev->UsbDevice->State = UsbDeviceRemoved;
UsbDereferenceDevice(dev->UsbDevice);
dev->UsbDevice = (USB_DEVICE*)0;
}
if (dev->Allocated)
(void)XhciDisableSlot(xhci, slotId);
memset(dev, 0, sizeof(*dev));
}
static KSTATUS XhciEnumeratePathEx(XHCI_CONTROLLER* xhci,
uint8_t logicalPort,
uint8_t speed,
const USB_PORT_PATH* path,
DEVICE_OBJECT* parentDevice,
uint8_t* outSlotId) {
if (outSlotId) *outSlotId = 0;
if (!xhci || !path || path->RootPort == 0)
return STATUS_INVALID_PARAMETER;
uint8_t slotId = 0;
st = XhciEnableSlot(xhci, &slotId);
KSTATUS st = XhciEnableSlot(xhci, &slotId);
if (!KSUCCESS(st)) return st;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
memset(dev, 0, sizeof(*dev));
dev->Allocated = true;
dev->SlotId = slotId;
dev->Port = port;
dev->Port = path->RootPort;
dev->Speed = speed;
dev->Path = *path;
dev->ParentDeviceObject = parentDevice;
st = XhciAllocateDeviceMemory(xhci, dev);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
XhciBuildDefaultControlContext(xhci, dev);
st = XhciAddressDevice(xhci, dev);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
dev->Addressed = true;
USB_DEVICE_DESCRIPTOR desc;
memset(&desc, 0, sizeof(desc));
st = XhciGetDeviceDescriptor(xhci, dev, &desc);
st = XhciGetDeviceDescriptorTimeout(xhci, dev, &desc, USB_XHCI_ENUM_CONTROL_TIMEOUT_US);
if (!KSUCCESS(st)) {
DbgPrint("usb: port %u slot %u descriptor read failed st=0x%x\n",
port, slotId, st);
logicalPort, slotId, st);
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
@@ -304,24 +334,100 @@ KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
dev->DeviceSubclass = desc.bDeviceSubClass;
dev->DeviceProtocol = desc.bDeviceProtocol;
DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n",
port,
slotId,
XhciPortSpeedName(speed),
speed,
desc.idVendor,
desc.idProduct,
desc.bDeviceClass,
desc.bDeviceSubClass,
desc.bDeviceProtocol,
desc.bMaxPacketSize0,
desc.bNumConfigurations,
desc.bcdUSB);
if (path->BehindHub) {
DbgPrint("usb: device root-port=%u hub-slot=%u hub-port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x route=%05x\n",
path->RootPort,
path->ParentHubSlot,
path->ParentHubPort,
slotId,
XhciPortSpeedName(speed),
speed,
desc.idVendor,
desc.idProduct,
desc.bDeviceClass,
desc.bDeviceSubClass,
desc.bDeviceProtocol,
desc.bMaxPacketSize0,
desc.bNumConfigurations,
desc.bcdUSB,
path->RouteString);
} else {
DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n",
logicalPort,
slotId,
XhciPortSpeedName(speed),
speed,
desc.idVendor,
desc.idProduct,
desc.bDeviceClass,
desc.bDeviceSubClass,
desc.bDeviceProtocol,
desc.bMaxPacketSize0,
desc.bNumConfigurations,
desc.bcdUSB);
}
st = XhciDiscoverAndSetConfiguration(xhci, dev, &desc);
if (!KSUCCESS(st)) {
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
if (outSlotId) *outSlotId = slotId;
return STATUS_SUCCESS;
}
KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId) {
if (outSlotId) *outSlotId = 0;
uint8_t speed = 0;
KSTATUS st = XhciResetPort(xhci, port, &speed);
if (!KSUCCESS(st)) return st;
return STATUS_SUCCESS;
USB_PORT_PATH path;
memset(&path, 0, sizeof(path));
path.RootPort = port;
path.Depth = 0;
path.RouteString = 0;
path.BehindHub = false;
DEVICE_OBJECT* parent = xhci && xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : (DEVICE_OBJECT*)0;
return XhciEnumeratePathEx(xhci, port, speed, &path, parent, outSlotId);
}
KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci,
USB_HUB_EXTENSION* hub,
uint8_t hubPort,
uint8_t speed,
uint8_t* outSlotId) {
if (outSlotId) *outSlotId = 0;
if (!xhci || !hub || !hub->Device || !hub->Self || hubPort == 0)
return STATUS_INVALID_PARAMETER;
USB_DEVICE* parent = hub->Device;
if (parent->Path.Depth >= 5u) {
DbgPrint("usb-hub: hub slot=%u port=%u exceeds xHCI route depth\n",
parent->SlotId, hubPort);
return STATUS_NOT_SUPPORTED;
}
if (hubPort > 15u) {
DbgPrint("usb-hub: hub slot=%u port=%u cannot be represented in xHCI route string yet\n",
parent->SlotId, hubPort);
return STATUS_NOT_SUPPORTED;
}
USB_PORT_PATH path = parent->Path;
path.BehindHub = true;
path.ParentHubSlot = parent->SlotId;
path.ParentHubPort = hubPort;
path.RouteString |= ((uint32_t)hubPort << (path.Depth * 4u));
path.Depth++;
return XhciEnumeratePathEx(xhci, hubPort, speed, &path, hub->Self, outSlotId);
}
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
return XhciEnumeratePortEx(xhci, port, (uint8_t*)0);
}
@@ -335,7 +441,8 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
continue;
attempted++;
KSTATUS st = XhciEnumeratePort(xhci, port);
uint8_t slot = 0;
KSTATUS st = XhciEnumeratePortEx(xhci, port, &slot);
if (KSUCCESS(st)) configured++;
else DbgPrint("usb: root-port %u enumeration failed st=0x%x\n", port, st);
}
@@ -343,19 +450,3 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
DbgPrint("usb: xHCI root-port enumeration complete; attempted=%u configured=%u\n",
attempted, configured);
}
KSTATUS UsbRegisterClassMatchers(void) {
memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
gUsbStorageDriver.Name = "usbstor";
gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
KSTATUS st = IoRegisterDriverMatcher("usbhid", IoBusUsb, UsbHidMatch, UsbHidBind, NULL);
if (!KSUCCESS(st)) return st;
st = IoRegisterDriverMatcher("usbstor", IoBusUsb, UsbStorageMatch, UsbStorageBind, NULL);
if (!KSUCCESS(st)) return st;
return STATUS_SUCCESS;
}
code/drivers/bus/usb/usb_hid.c
+412 -4
View File
@@ -125,7 +125,7 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return;
@@ -158,11 +158,10 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
if (xhci->HidDeviceCount == 0)
if (!xhci || xhci->HidDeviceCount == 0)
return STATUS_SUCCESS;
XhciArmAllHid(xhci);
DbgPrint("usb: HID interrupt input armed devices=%u\n", xhci->HidDeviceCount);
DbgPrint("usb: HID interrupt URBs active devices=%u\n", xhci->HidDeviceCount);
return STATUS_SUCCESS;
}
@@ -365,3 +364,412 @@ KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context) {
}
return st;
}
// -------------------------------------------------------------------------
// PDO/FDO boot HID function driver path.
// -------------------------------------------------------------------------
DRIVER_OBJECT gUsbHidDriver;
static const IO_DEVICE_ID gUsbHidIds[] = {
{ IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_01", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_01") - 1 } },
{ IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_02", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_02") - 1 } },
};
static const IO_DRIVER_ID_TABLE gUsbHidIdTable = {
gUsbHidIds,
sizeof(gUsbHidIds) / sizeof(gUsbHidIds[0]),
};
static const USB_ENDPOINT_DESCRIPTOR* UsbHidFindInterruptInPdo(const USB_PDO_EXTENSION* ext) {
if (!ext) return (const USB_ENDPOINT_DESCRIPTOR*)0;
for (uint8_t i = 0; i < ext->EndpointCount; i++) {
const USB_ENDPOINT_DESCRIPTOR* ep = &ext->Endpoints[i];
uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0;
if (type == USB_ENDPOINT_TYPE_INTERRUPT && in)
return ep;
}
return (const USB_ENDPOINT_DESCRIPTOR*)0;
}
static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 8) return;
uint8_t modifiers = report[0];
for (uint8_t bit = 0; bit < 8; bit++) {
uint8_t mask = (uint8_t)(1u << bit);
bool wasDown = (ext->LastModifiers & mask) != 0;
bool isDown = (modifiers & mask) != 0;
if (wasDown != isDown)
XhciSubmitHidKey(XhciHidModifierKey(bit), isDown);
}
const uint8_t* keys = &report[2];
for (uint8_t i = 0; i < 6; i++) {
uint8_t oldKey = ext->LastKeys[i];
if (oldKey > 1 && !XhciHidKeyInReport(keys, oldKey))
XhciSubmitHidKey((INPUT_KEY)oldKey, false);
}
for (uint8_t i = 0; i < 6; i++) {
uint8_t newKey = keys[i];
if (newKey > 1 && !XhciHidKeyInReport(ext->LastKeys, newKey))
XhciSubmitHidKey((INPUT_KEY)newKey, true);
}
ext->LastModifiers = modifiers;
memcpy(ext->LastKeys, keys, sizeof(ext->LastKeys));
}
static void UsbHidProcessBootMouseReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 3) return;
int dx = (int)(int8_t)report[1];
int dy = (int)(int8_t)report[2];
int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0;
uint8_t buttons = report[0] & 0x07u;
if (dx || dy || wheel || buttons != ext->LastButtons) {
INPUT_MOUSE_EVENT ev;
ev.dx = dx;
ev.dy = dy;
ev.wheel = wheel;
ev.buttons = buttons;
KeInputSubmitMouse(&ev);
}
ext->LastButtons = buttons;
}
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext);
static void UsbHidInterruptComplete(USB_URB* urb, void* context) {
USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)context;
if (!ext || urb != &ext->InterruptUrb)
return;
ext->InterruptPending = false;
if (ext->SurpriseRemoved || ext->Removed || !ext->Started)
return;
if (KSUCCESS(urb->Status)) {
uint16_t length = (uint16_t)urb->ActualLength;
if (length > ext->ReportLength) length = ext->ReportLength;
if (ext->Keyboard)
UsbHidProcessBootKeyboardReport(ext, ext->ReportBuffer, length);
else if (ext->Mouse)
UsbHidProcessBootMouseReport(ext, ext->ReportBuffer, length);
} else if (urb->Status != STATUS_CANCELLED) {
DbgPrint("usb: HID interrupt URB failed slot=%u if=%u ep=%02x st=0x%x\n",
ext->Device ? ext->Device->SlotId : 0,
ext->InterfaceNumber,
ext->EndpointAddress,
urb->Status);
}
if (!ext->SurpriseRemoved && !ext->Removed)
(void)UsbHidArmInterrupt(ext);
}
static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) {
if (!ext || !ext->LowerDevice || !ext->Device)
return STATUS_INVALID_PARAMETER;
USB_URB urb;
memset(&urb, 0, sizeof(urb));
urb.Function = UrbFunctionControlTransfer;
urb.Direction = UsbTransferNoData;
urb.Device = ext->Device;
urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
urb.Setup.bRequest = USB_REQ_SET_PROTOCOL;
urb.Setup.wValue = 0;
urb.Setup.wIndex = ext->InterfaceNumber;
urb.Setup.wLength = 0;
KSTATUS st = UsbSubmitUrbSync(ext->LowerDevice, &urb);
if (!KSUCCESS(st)) {
DbgPrint("usb: HID SET_PROTOCOL boot failed slot=%u if=%u st=0x%x\n",
ext->Device->SlotId, ext->InterfaceNumber, st);
return st;
}
memset(&urb, 0, sizeof(urb));
urb.Function = UrbFunctionControlTransfer;
urb.Direction = UsbTransferNoData;
urb.Device = ext->Device;
urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
urb.Setup.bRequest = USB_REQ_SET_IDLE;
urb.Setup.wValue = 0;
urb.Setup.wIndex = ext->InterfaceNumber;
urb.Setup.wLength = 0;
st = UsbSubmitUrbSync(ext->LowerDevice, &urb);
if (!KSUCCESS(st)) {
DbgPrint("usb: HID SET_IDLE failed slot=%u if=%u st=0x%x; continuing\n",
ext->Device->SlotId, ext->InterfaceNumber, st);
}
DbgPrint("usb: HID boot protocol selected slot=%u if=%u proto=%u\n",
ext->Device->SlotId, ext->InterfaceNumber, ext->Protocol);
return STATUS_SUCCESS;
}
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed)
return STATUS_OBJECT_DEAD;
if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys)
return STATUS_INVALID_PARAMETER;
if (ext->InterruptPending)
return STATUS_SUCCESS;
memset(ext->ReportBuffer, 0, ext->ReportLength);
memset(&ext->InterruptUrb, 0, sizeof(ext->InterruptUrb));
ext->InterruptUrb.Function = UrbFunctionInterruptTransfer;
ext->InterruptUrb.Direction = UsbTransferIn;
ext->InterruptUrb.Device = ext->Device;
ext->InterruptUrb.Pipe = ext->InterruptPipe;
ext->InterruptUrb.Buffer = ext->ReportBuffer;
ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys;
ext->InterruptUrb.Length = ext->ReportLength;
ext->InterruptUrb.Complete = UsbHidInterruptComplete;
ext->InterruptUrb.CompleteContext = ext;
KSTATUS st = UsbSubmitUrbAsync(ext->LowerDevice, &ext->InterruptUrb);
if (st != STATUS_PENDING && !KSUCCESS(st))
return st;
if (ext->InterruptUrb.Status != STATUS_PENDING && ext->InterruptUrb.Status != STATUS_SUCCESS)
return ext->InterruptUrb.Status;
ext->InterruptPending = true;
return STATUS_SUCCESS;
}
static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) {
if (!device || !ext || !ext->Device || !ext->InterfacePdo)
return STATUS_INVALID_PARAMETER;
if (ext->Started)
return STATUS_SUCCESS;
if (ext->SurpriseRemoved || ext->Removed)
return STATUS_OBJECT_DEAD;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)ext->InterfacePdo->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface)
return STATUS_INVALID_PARAMETER;
const USB_ENDPOINT_DESCRIPTOR* ep = UsbHidFindInterruptInPdo(pdoExt);
if (!ep)
return STATUS_NOT_FOUND;
ext->Keyboard = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD);
ext->Mouse = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_MOUSE);
ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
ext->Protocol = pdoExt->InterfaceDescriptor.bInterfaceProtocol;
ext->EndpointAddress = ep->bEndpointAddress;
ext->MaxPacketSize = ep->wMaxPacketSize;
ext->Interval = ep->bInterval;
uint16_t reportLength = ext->MaxPacketSize & 0x07FFu;
if (reportLength == 0) reportLength = ext->Keyboard ? 8u : 4u;
if (ext->Keyboard && reportLength < 8) reportLength = 8;
if (ext->Mouse && reportLength < 3) reportLength = 3;
if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT;
ext->ReportLength = reportLength;
ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys);
if (!ext->ReportBuffer)
return STATUS_NO_MEMORY;
KSTATUS st = UsbCreatePipe(ext->Device,
UsbPipeInterrupt,
ext->EndpointAddress,
ext->MaxPacketSize,
ext->Interval,
&ext->InterruptPipe);
if (!KSUCCESS(st))
return st;
st = UsbHidSendBootProtocol(ext);
if (!KSUCCESS(st))
return st;
memset(ext->LastKeys, 0, sizeof(ext->LastKeys));
ext->LastModifiers = 0;
ext->LastButtons = 0;
KeInputRegisterDevice(ext->Keyboard ? "usb-hid-keyboard" : "usb-hid-mouse", 0, &ext->InputDeviceId);
ext->Started = true;
device->PnpState = DevicePnpStarted;
if (ext->InterfacePdo)
ext->InterfacePdo->PnpState = DevicePnpStarted;
XHCI_CONTROLLER* xhci = ext->Device && ext->Device->Hcd ?
(XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
st = UsbHidArmInterrupt(ext);
if (!KSUCCESS(st))
return st;
if (xhci)
xhci->HidDeviceCount++;
DbgPrint("usb: HID boot %s bound slot=%u if=%u ep=%02x maxpkt=%u interval=%u input=%u urb\n",
ext->Keyboard ? "keyboard" : "mouse",
ext->Device->SlotId,
ext->InterfaceNumber,
ext->EndpointAddress,
ext->MaxPacketSize,
ext->Interval,
ext->InputDeviceId);
return STATUS_SUCCESS;
}
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)device->DeviceExtension;
KSTATUS st = STATUS_SUCCESS;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
st = UsbHidStartDevice(device, ext);
if (!KSUCCESS(st)) {
IoCompleteRequest(irp, st, 0);
return STATUS_PENDING;
}
return IoForwardIrpToLower(device, irp);
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return IoForwardIrpToLower(device, irp);
case IRP_MN_CANCEL_REMOVE_DEVICE:
if (!ext || !ext->SurpriseRemoved)
device->PnpState = DevicePnpStarted;
return IoForwardIrpToLower(device, irp);
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
if (ext) {
ext->SurpriseRemoved = true;
ext->Started = false;
if (ext->InterruptPending)
(void)UsbCancelUrb(&ext->InterruptUrb);
}
return IoForwardIrpToLower(device, irp);
case IRP_MN_REMOVE_DEVICE: {
device->PnpState = DevicePnpDeleted;
if (ext) {
ext->Removed = true;
ext->Started = false;
if (ext->InterruptPending)
(void)UsbCancelUrb(&ext->InterruptUrb);
if (ext->InterruptPipe) {
ext->InterruptPipe->Open = false;
UsbDereferencePipe(ext->InterruptPipe);
ext->InterruptPipe = (USB_PIPE*)0;
}
if (ext->Device && ext->Device->Hcd) {
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext;
if (xhci && xhci->HidDeviceCount)
xhci->HidDeviceCount--;
}
if (ext->RemoveLockReady) {
IoReleaseRemoveLockAndWait(&ext->RemoveLock, irp);
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
}
}
KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS;
if (ext && ext->LowerDevice) {
IoDetachDevice(device);
ext->LowerDevice = (DEVICE_OBJECT*)0;
}
IoDeleteDevice(device);
if (lower == STATUS_NOT_SUPPORTED) {
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
}
if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
IoCompleteRequest(irp, lower, 0);
return STATUS_PENDING;
}
return lower;
}
default:
return IoForwardIrpToLower(device, irp);
}
}
KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device)
return STATUS_INVALID_PARAMETER;
if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_HID ||
pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_HID_SUBCLASS_BOOT)
return STATUS_INVALID_PARAMETER;
if (pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_KEYBOARD &&
pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_MOUSE)
return STATUS_INVALID_PARAMETER;
if (!UsbHidFindInterruptInPdo(pdoExt))
return STATUS_NOT_FOUND;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(driver, sizeof(USB_HID_EXTENSION), &fdo);
if (!KSUCCESS(st))
return st;
(void)IoSetDeviceName(fdo,
pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD
? SYS_SV_LITERAL("usb-hid-keyboard")
: SYS_SV_LITERAL("usb-hid-mouse"));
USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)fdo->DeviceExtension;
ext->Self = fdo;
ext->InterfacePdo = physicalDeviceObject;
ext->Device = pdoExt->Device;
st = IoInitializeRemoveLock(&ext->RemoveLock);
if (!KSUCCESS(st)) {
IoDeleteDevice(fdo);
return st;
}
ext->RemoveLockReady = true;
ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!ext->LowerDevice) {
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
IoDeleteDevice(fdo);
return STATUS_FAILURE;
}
DbgPrint("usb: HID FDO attached slot=%u if=%u proto=%u\n",
ext->Device->SlotId,
pdoExt->InterfaceDescriptor.bInterfaceNumber,
pdoExt->InterfaceDescriptor.bInterfaceProtocol);
return STATUS_SUCCESS;
}
KSTATUS UsbInitializeHidDriver(void) {
memset(&gUsbHidDriver, 0, sizeof(gUsbHidDriver));
IoInitializeDriverObject(&gUsbHidDriver, SYS_SV_LITERAL("usb-hid"));
gUsbHidDriver.IdTable = &gUsbHidIdTable;
gUsbHidDriver.DriverAddDevice = UsbHidAddDevice;
gUsbHidDriver.MajorFunction[IRP_MJ_PNP] = UsbHidDispatchPnp;
return IoRegisterDriverObject(&gUsbHidDriver);
}
code/drivers/bus/usb/usb_hub.c
+1027
View File
File diff suppressed because it is too large. Load diff
code/drivers/bus/usb/usb_storage.c
+528 -153
View File
@@ -1,53 +1,115 @@
// usb_storage.c - USB mass-storage BOT/SCSI class binding and disk bridge.
// usb_storage.c - USB mass-storage BOT/SCSI class driver.
//
// The storage driver now binds as a USB function driver to an interface PDO
// (USB\CLASS_08&SUBCLASS_06&PROT_50). It creates a disk FDO above that
// interface stack and issues BOT transfers through USB URBs. The xHCI HCD
// still backs the bulk URBs with the existing per-device rings; later stages
// will make endpoint rings fully pipe-owned and asynchronous.
#include "usb.h"
DRIVER_OBJECT gUsbStorageDriver;
KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
PHYSICAL_ADDRESS phys, uint32_t length,
uint32_t* outActual) {
return XhciBulkTransfer(xhci,
dev,
dev->MassBulkInEndpointId,
dev->MassInRing,
dev->MassInRingPhys,
&dev->MassInEnqueue,
&dev->MassInCycle,
phys,
length,
outActual);
static const IO_DEVICE_ID gUsbStorageIds[] = {
{ IoBusUsb, { "USB\\CLASS_08&SUBCLASS_06&PROT_50", sizeof("USB\\CLASS_08&SUBCLASS_06&PROT_50") - 1 } },
};
static const IO_DRIVER_ID_TABLE gUsbStorageIdTable = {
gUsbStorageIds,
sizeof(gUsbStorageIds) / sizeof(gUsbStorageIds[0]),
};
static USB_STORAGE_EXTENSION* UsbStorageFromDiskDevice(DEVICE_OBJECT* device) {
if (!device || !device->DeviceExtension)
return (USB_STORAGE_EXTENSION*)0;
USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)device->DeviceExtension;
if (diskExt->Kind != UsbStorageExtensionDisk)
return (USB_STORAGE_EXTENSION*)0;
return diskExt->Storage;
}
static USB_STORAGE_EXTENSION* UsbStorageFromInterfaceDevice(DEVICE_OBJECT* device) {
if (!device || !device->DeviceExtension)
return (USB_STORAGE_EXTENSION*)0;
KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
PHYSICAL_ADDRESS phys, uint32_t length,
uint32_t* outActual) {
return XhciBulkTransfer(xhci,
dev,
dev->MassBulkOutEndpointId,
dev->MassOutRing,
dev->MassOutRingPhys,
&dev->MassOutEnqueue,
&dev->MassOutCycle,
phys,
length,
outActual);
USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)device->DeviceExtension;
if (ext->Kind != UsbStorageExtensionInterface)
return (USB_STORAGE_EXTENSION*)0;
return ext;
}
static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_EXTENSION* pdoExt, bool wantIn) {
if (!pdoExt) return (const USB_ENDPOINT_DESCRIPTOR*)0;
KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const uint8_t* cdb, uint8_t cdbLen,
void* data, PHYSICAL_ADDRESS dataPhys,
uint32_t dataLength, bool dataIn) {
if (!xhci || !dev || !cdb || cdbLen == 0 || cdbLen > 16)
for (uint8_t i = 0; i < pdoExt->EndpointCount; i++) {
const USB_ENDPOINT_DESCRIPTOR* ep = &pdoExt->Endpoints[i];
uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0;
if (type == USB_ENDPOINT_TYPE_BULK && in == wantIn)
return ep;
}
return (const USB_ENDPOINT_DESCRIPTOR*)0;
}
static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
bool dataIn,
PHYSICAL_ADDRESS phys,
uint32_t length,
uint32_t* outActual) {
if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0)
return STATUS_INVALID_PARAMETER;
USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe;
if (!pipe || !pipe->Open)
return STATUS_OBJECT_DEAD;
USB_URB urb;
memset(&urb, 0, sizeof(urb));
urb.Function = UrbFunctionBulkTransfer;
urb.Direction = dataIn ? UsbTransferIn : UsbTransferOut;
urb.Device = storage->Device;
urb.Pipe = pipe;
urb.Buffer = (void*)0;
urb.BufferPhys = phys;
urb.Length = length;
KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb);
if (outActual)
*outActual = KSUCCESS(st) ? urb.ActualLength : 0;
return st;
}
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, true, phys, length, outActual);
}
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, false, phys, length, outActual);
}
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
PHYSICAL_ADDRESS dataPhys,
uint32_t dataLength,
bool dataIn) {
if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16)
return STATUS_INVALID_PARAMETER;
if (dataLength && (!data || !dataPhys))
return STATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
USB_MASS_CBW* cbw = (USB_MASS_CBW*)dev->DescriptorBuffer;
memset(cbw, 0, sizeof(*cbw));
cbw->Signature = USB_MASS_BOT_CBW_SIGNATURE;
cbw->Tag = ++dev->MassTag;
cbw->Tag = ++storage->Tag;
cbw->DataTransferLength = dataLength;
cbw->Flags = dataIn ? USB_DIR_IN : USB_DIR_OUT;
cbw->Lun = 0;
@@ -55,16 +117,14 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
memcpy(cbw->CommandBlock, cdb, cdbLen);
uint32_t actual = 0;
KSTATUS st = UsbStorageBulkOut(xhci, dev, dev->DescriptorPhys, sizeof(*cbw), &actual);
KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*cbw))
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
if (dataLength) {
actual = 0;
if (dataIn)
st = UsbStorageBulkIn(xhci, dev, dataPhys, dataLength, &actual);
else
st = UsbStorageBulkOut(xhci, dev, dataPhys, dataLength, &actual);
st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual)
: UsbStorageBulkOut(storage, dataPhys, dataLength, &actual);
if (!KSUCCESS(st))
return st;
}
@@ -72,13 +132,13 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer;
memset(csw, 0, sizeof(*csw));
actual = 0;
st = UsbStorageBulkIn(xhci, dev, dev->DescriptorPhys, sizeof(*csw), &actual);
st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*csw))
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != dev->MassTag) {
if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != storage->Tag) {
DbgPrint("usb-storage: invalid CSW slot=%u sig=%08x tag=%08x expected=%08x\n",
dev->SlotId, csw->Signature, csw->Tag, dev->MassTag);
dev->SlotId, csw->Signature, csw->Tag, storage->Tag);
return STATUS_DEVICE_ERROR;
}
if (csw->Status != USB_MASS_BOT_CSW_PASSED) {
@@ -90,13 +150,11 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
uint32_t UsbBe32(const uint8_t* p) {
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
((uint32_t)p[2] << 8) | (uint32_t)p[3];
}
void UsbStoreBe32(uint8_t* p, uint32_t v) {
p[0] = (uint8_t)(v >> 24);
p[1] = (uint8_t)(v >> 16);
@@ -104,17 +162,19 @@ void UsbStoreBe32(uint8_t* p, uint32_t v) {
p[3] = (uint8_t)v;
}
void UsbStoreBe16(uint8_t* p, uint16_t v) {
p[0] = (uint8_t)(v >> 8);
p[1] = (uint8_t)v;
}
KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 };
memset(dev->MassBuffer, 0, 36);
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
36, true);
if (!KSUCCESS(st)) return st;
@@ -128,37 +188,46 @@ KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
memset(dev->MassBuffer, 0, 18);
return UsbStorageBotCommand(xhci, dev, rs, sizeof(rs),
return UsbStorageBotCommand(storage, rs, sizeof(rs),
dev->MassBuffer, dev->MassBufferPhys, 18, true);
}
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
uint8_t cdb[6] = { SCSI_TEST_UNIT_READY, 0, 0, 0, 0, 0 };
KSTATUS last = STATUS_DEVICE_ERROR;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext;
for (uint32_t attempt = 0; attempt < 10; attempt++) {
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
(void*)0, 0, 0, false);
if (KSUCCESS(st)) return STATUS_SUCCESS;
last = st;
(void)UsbStorageRequestSense(xhci, dev);
(void)UsbStorageRequestSense(storage);
UsbDelayUs(xhci->Hal, 10000u);
}
return last;
}
KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
memset(dev->MassBuffer, 0, 8);
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
8, true);
if (!KSUCCESS(st)) return st;
@@ -168,27 +237,33 @@ KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (blockSize == 0)
return STATUS_DEVICE_ERROR;
storage->BlockSize = blockSize;
storage->BlockCount = (uint64_t)lastLba + 1ull;
dev->MassBlockSize = blockSize;
dev->MassBlockCount = (uint64_t)lastLba + 1ull;
dev->MassBlockCount = storage->BlockCount;
DbgPrint("usb-storage: capacity slot=%u blocks=%llu block_size=%u (%llu MB)\n",
dev->SlotId,
(unsigned long long)dev->MassBlockCount,
dev->MassBlockSize,
(unsigned long long)((dev->MassBlockCount * (uint64_t)dev->MassBlockSize) / (1024ull * 1024ull)));
(unsigned long long)storage->BlockCount,
storage->BlockSize,
(unsigned long long)((storage->BlockCount * (uint64_t)storage->BlockSize) / (1024ull * 1024ull)));
return STATUS_SUCCESS;
}
KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
bool write, uint64_t lba,
uint32_t sectors, void* buffer) {
if (!xhci || !dev || !buffer || sectors == 0)
KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
bool write,
uint64_t lba,
uint32_t sectors,
void* buffer) {
if (!storage || !storage->XhciDevice || !buffer || sectors == 0)
return STATUS_INVALID_PARAMETER;
if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE)
if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE)
return STATUS_NOT_SUPPORTED;
if (lba > 0xFFFFFFFFull || sectors > 0xFFFFu)
return STATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
uint32_t bytes = sectors * USB_STORAGE_SECTOR_SIZE;
if (bytes > USB_STORAGE_MAX_BYTES)
return STATUS_INVALID_PARAMETER;
@@ -202,7 +277,7 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
if (write)
memcpy(dev->MassBuffer, buffer, bytes);
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
bytes, !write);
if (!KSUCCESS(st)) return st;
@@ -212,42 +287,67 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
while (_InterlockedCompareExchange(&dev->MassIoBusy, 1, 0) != 0) {
UsbDelayUs(xhci->Hal, 10);
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
if (!storage) return;
XHCI_CONTROLLER* xhci = storage->Device ? (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
while (_InterlockedCompareExchange(&storage->IoBusy, 1, 0) != 0) {
if (xhci) UsbDelayUs(xhci->Hal, 10);
}
}
void UsbStorageReleaseIo(XHCI_DEVICE* dev) {
_InterlockedExchange(&dev->MassIoBusy, 0);
void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage) {
if (storage) _InterlockedExchange(&storage->IoBusy, 0);
}
KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = (XHCI_DEVICE*)device->DeviceExtension;
if (!dev->MassActive || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
IoCompleteRequest(irp, STATUS_DEVICE_ERROR, 0);
USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device);
if (!storage || !storage->RemoveLockReady)
return STATUS_OBJECT_DEAD;
KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return STATUS_INVALID_PARAMETER;
}
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return STATUS_INVALID_PARAMETER;
}
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
(ios->Parameters.ReadWrite.Length & 511u) ||
ios->Parameters.ReadWrite.Length == 0 ||
!irp->UserBuffer) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return STATUS_INVALID_PARAMETER;
}
if (!storage->Started || storage->SurpriseRemoved || storage->Removed ||
storage->BlockSize != USB_STORAGE_SECTOR_SIZE) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, STATUS_OBJECT_DEAD, 0);
return STATUS_PENDING;
}
uint8_t* buffer = (uint8_t*)irp->Buffer;
uint64_t lba = irp->Lba;
uint32_t remaining = irp->SectorCount;
uint8_t* buffer = (uint8_t*)irp->UserBuffer;
uint64_t lba = ios->Parameters.ReadWrite.ByteOffset >> 9;
uint32_t remaining = ios->Parameters.ReadWrite.Length >> 9;
uint32_t done = 0;
bool write = (irp->Flags & IRP_FLAG_WRITE) != 0;
bool write = (ios->MajorFunction == IRP_MJ_WRITE);
uint32_t maxSectors = USB_STORAGE_MAX_BYTES / USB_STORAGE_SECTOR_SIZE;
KSTATUS st = STATUS_SUCCESS;
UsbStorageAcquireIo(&gXhci, dev);
st = STATUS_SUCCESS;
UsbStorageAcquireIo(storage);
while (remaining) {
uint32_t chunk = remaining;
if (chunk > maxSectors) chunk = maxSectors;
st = UsbStorageReadWriteSectors(&gXhci, dev, write, lba, chunk,
st = UsbStorageReadWriteSectors(storage, write, lba, chunk,
buffer + (done * USB_STORAGE_SECTOR_SIZE));
if (!KSUCCESS(st))
break;
@@ -255,13 +355,13 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
done += chunk;
remaining -= chunk;
}
UsbStorageReleaseIo(dev);
UsbStorageReleaseIo(storage);
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? (size_t)done * USB_STORAGE_SECTOR_SIZE : 0);
return STATUS_PENDING;
}
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer)
return STATUS_SUCCESS;
@@ -282,7 +382,6 @@ KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (!dev->MassCandidate)
return STATUS_SUCCESS;
@@ -352,7 +451,6 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de
return STATUS_SUCCESS;
}
uint8_t UsbFindFreeDiskIndex(void) {
for (uint8_t i = 0; i < 8; i++) {
if (!IoDiskDevicePresent(i))
@@ -361,17 +459,23 @@ uint8_t UsbFindFreeDiskIndex(void) {
return 0xFFu;
}
KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) {
if (!dev || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE || dev->MassBlockCount == 0)
KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage) {
if (!storage || storage->BlockSize != USB_STORAGE_SECTOR_SIZE || storage->BlockCount == 0)
return STATUS_NOT_SUPPORTED;
if (storage->DiskDevice)
return STATUS_ALREADY_EXISTS;
DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(&gUsbStorageDriver, 0, &disk);
KSTATUS st = IoCreateDevice(&gUsbStorageDriver, sizeof(USB_STORAGE_DISK_EXTENSION), &disk);
if (!KSUCCESS(st)) return st;
(void)IoSetDeviceName(disk, SYS_SV_LITERAL("usb-storage-disk"));
disk->DeviceExtension = dev;
disk->SectorCount = dev->MassBlockCount;
USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)disk->DeviceExtension;
diskExt->Kind = UsbStorageExtensionDisk;
diskExt->Storage = storage;
disk->u.Disk.SectorCount = storage->BlockCount;
disk->u.Disk.SectorSize = USB_STORAGE_SECTOR_SIZE;
disk->AlignmentRequirement = USB_STORAGE_SECTOR_SIZE;
disk->Flags |= DEVICE_OBJECT_REMOVABLE;
@@ -387,12 +491,278 @@ KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) {
return st;
}
dev->MassDeviceObject = disk;
storage->DiskDevice = disk;
if (storage->XhciDevice)
storage->XhciDevice->MassDeviceObject = disk;
DbgPrint("usb-storage: registered disk %u slot=%u sectors=%llu\n",
index, dev->SlotId, (unsigned long long)dev->MassBlockCount);
index,
storage->Device ? storage->Device->SlotId : 0,
(unsigned long long)storage->BlockCount);
return STATUS_SUCCESS;
}
static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSION* storage) {
if (!device || !storage || !storage->InterfacePdo || !storage->Device)
return STATUS_INVALID_PARAMETER;
if (storage->Started)
return STATUS_SUCCESS;
if (storage->SurpriseRemoved || storage->Removed)
return STATUS_OBJECT_DEAD;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)storage->InterfacePdo->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface)
return STATUS_INVALID_PARAMETER;
const USB_ENDPOINT_DESCRIPTOR* bulkIn = UsbStorageFindBulkEndpoint(pdoExt, true);
const USB_ENDPOINT_DESCRIPTOR* bulkOut = UsbStorageFindBulkEndpoint(pdoExt, false);
if (!bulkIn || !bulkOut)
return STATUS_NOT_FOUND;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext;
XHCI_DEVICE* xdev = (XHCI_DEVICE*)storage->Device->HcdDeviceContext;
if (!xhci || !xdev)
return STATUS_INVALID_PARAMETER;
storage->XhciDevice = xdev;
storage->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
storage->BulkInAddress = bulkIn->bEndpointAddress;
storage->BulkOutAddress = bulkOut->bEndpointAddress;
storage->BulkInMaxPacket = bulkIn->wMaxPacketSize;
storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize;
storage->Tag = 0x1000u + storage->Device->SlotId;
xdev->MassCandidate = true;
xdev->MassInterface = storage->InterfaceNumber;
xdev->MassBulkInAddress = storage->BulkInAddress;
xdev->MassBulkOutAddress = storage->BulkOutAddress;
xdev->MassBulkInEndpointId = XhciEndpointIdFromAddress(storage->BulkInAddress);
xdev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(storage->BulkOutAddress);
xdev->MassBulkInMaxPacket = storage->BulkInMaxPacket;
xdev->MassBulkOutMaxPacket = storage->BulkOutMaxPacket;
KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, xdev);
if (!KSUCCESS(st)) goto fail;
st = UsbCreatePipe(storage->Device,
UsbPipeBulk,
storage->BulkInAddress,
storage->BulkInMaxPacket,
bulkIn->bInterval,
&storage->BulkInPipe);
if (!KSUCCESS(st)) goto fail;
st = UsbCreatePipe(storage->Device,
UsbPipeBulk,
storage->BulkOutAddress,
storage->BulkOutMaxPacket,
bulkOut->bInterval,
&storage->BulkOutPipe);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageInquiry(storage);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageTestUnitReady(storage);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", storage->Device->SlotId, st);
goto fail;
}
st = UsbStorageReadCapacity(storage);
if (!KSUCCESS(st)) goto fail;
if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE) {
DbgPrint("usb-storage: unsupported block size slot=%u size=%u\n",
storage->Device->SlotId, storage->BlockSize);
st = STATUS_NOT_SUPPORTED;
goto fail;
}
// The disk namespace publishes an arrival notification synchronously from
// IoRegisterDiskDevice(). Mark the storage FDO usable before publishing the
// disk so filesystem hotplug callbacks can read sector 0 immediately.
xdev->MassActive = true;
storage->Started = true;
device->PnpState = DevicePnpStarted;
if (storage->InterfacePdo)
storage->InterfacePdo->PnpState = DevicePnpStarted;
st = UsbStorageRegisterDisk(storage);
if (!KSUCCESS(st)) {
storage->Started = false;
device->PnpState = DevicePnpNotStarted;
if (storage->InterfacePdo)
storage->InterfacePdo->PnpState = DevicePnpNotStarted;
goto fail;
}
DbgPrint("usb-storage: FDO bound slot=%u if=%u disk-ready\n",
storage->Device->SlotId, storage->InterfaceNumber);
return STATUS_SUCCESS;
fail:
xdev->MassCandidate = false;
xdev->MassActive = false;
if (storage->BulkInPipe) {
UsbDereferencePipe(storage->BulkInPipe);
storage->BulkInPipe = (USB_PIPE*)0;
}
if (storage->BulkOutPipe) {
UsbDereferencePipe(storage->BulkOutPipe);
storage->BulkOutPipe = (USB_PIPE*)0;
}
DbgPrint("usb-storage: start failed slot=%u if=%u st=0x%x\n",
storage->Device ? storage->Device->SlotId : 0,
storage->InterfaceNumber,
st);
return st;
}
static KSTATUS UsbStorageDiskPnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
if (!device || !irp || !storage)
return STATUS_INVALID_PARAMETER;
switch (minor) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_SURPRISE_REMOVAL:
storage->SurpriseRemoved = true;
if (storage->XhciDevice) {
storage->XhciDevice->MassSurpriseRemoved = true;
storage->XhciDevice->MassActive = false;
}
device->PnpState = DevicePnpSurpriseRemoved;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_REMOVE_DEVICE:
storage->Removed = true;
if (storage->XhciDevice) {
storage->XhciDevice->MassRemoved = true;
storage->XhciDevice->MassActive = false;
if (storage->XhciDevice->MassDeviceObject == device)
storage->XhciDevice->MassDeviceObject = (DEVICE_OBJECT*)0;
}
device->PnpState = DevicePnpDeleted;
if (storage->DiskDevice == device)
storage->DiskDevice = (DEVICE_OBJECT*)0;
IoDeleteDevice(device);
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
default:
return STATUS_NOT_SUPPORTED;
}
}
static KSTATUS UsbStorageInterfacePnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
if (!device || !irp || !storage)
return STATUS_INVALID_PARAMETER;
KSTATUS st = STATUS_SUCCESS;
switch (minor) {
case IRP_MN_START_DEVICE:
st = UsbStorageStartDevice(device, storage);
if (!KSUCCESS(st)) {
IoCompleteRequest(irp, st, 0);
return STATUS_PENDING;
}
return IoForwardIrpToLower(device, irp);
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return IoForwardIrpToLower(device, irp);
case IRP_MN_CANCEL_REMOVE_DEVICE:
if (!storage->SurpriseRemoved)
device->PnpState = DevicePnpStarted;
return IoForwardIrpToLower(device, irp);
case IRP_MN_SURPRISE_REMOVAL:
storage->SurpriseRemoved = true;
storage->Started = false;
if (storage->XhciDevice) {
storage->XhciDevice->MassSurpriseRemoved = true;
storage->XhciDevice->MassActive = false;
}
if (storage->DiskDevice)
(void)IoNotifyDiskSurpriseRemoved(storage->DiskDevice->u.Disk.DiskIndex);
device->PnpState = DevicePnpSurpriseRemoved;
return IoForwardIrpToLower(device, irp);
case IRP_MN_REMOVE_DEVICE: {
storage->Removed = true;
storage->Started = false;
if (storage->DiskDevice)
(void)IoUnregisterDiskDevice(storage->DiskDevice->u.Disk.DiskIndex);
if (storage->BulkInPipe) {
UsbDereferencePipe(storage->BulkInPipe);
storage->BulkInPipe = (USB_PIPE*)0;
}
if (storage->BulkOutPipe) {
UsbDereferencePipe(storage->BulkOutPipe);
storage->BulkOutPipe = (USB_PIPE*)0;
}
if (storage->RemoveLockReady) {
IoReleaseRemoveLockAndWait(&storage->RemoveLock, irp);
IoDestroyRemoveLock(&storage->RemoveLock);
storage->RemoveLockReady = false;
}
device->PnpState = DevicePnpDeleted;
KSTATUS lower = storage->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS;
if (storage->LowerDevice) {
IoDetachDevice(device);
storage->LowerDevice = (DEVICE_OBJECT*)0;
}
IoDeleteDevice(device);
if (lower == STATUS_NOT_SUPPORTED) {
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
}
if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
IoCompleteRequest(irp, lower, 0);
return STATUS_PENDING;
}
return lower;
}
default:
return IoForwardIrpToLower(device, irp);
}
}
KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_STORAGE_EXTENSION* iface = UsbStorageFromInterfaceDevice(device);
if (iface)
return UsbStorageInterfacePnp(device, irp, iface, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction);
USB_STORAGE_EXTENSION* disk = UsbStorageFromDiskDevice(device);
if (disk)
return UsbStorageDiskPnp(device, irp, disk, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction);
return STATUS_INVALID_PARAMETER;
}
bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
(void)context;
@@ -409,64 +779,69 @@ bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
}
KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context) {
(void)pub;
(void)context;
if (!pub || !pub->BusContext || !pub->DeviceContext)
return STATUS_INVALID_PARAMETER;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext;
XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext;
if (dev->MassActive || dev->MassCandidate)
return STATUS_ALREADY_EXISTS;
const IO_PUBLISHED_USB_ENDPOINT* bulkIn = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true);
const IO_PUBLISHED_USB_ENDPOINT* bulkOut = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
if (!bulkIn || !bulkOut)
return STATUS_NOT_FOUND;
dev->MassCandidate = true;
dev->MassInterface = pub->u.Usb.InterfaceNumber;
dev->MassBulkInAddress = bulkIn->Address;
dev->MassBulkOutAddress = bulkOut->Address;
dev->MassBulkInEndpointId = XhciEndpointIdFromAddress(bulkIn->Address);
dev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(bulkOut->Address);
dev->MassBulkInMaxPacket = bulkIn->MaxPacketSize;
dev->MassBulkOutMaxPacket = bulkOut->MaxPacketSize;
KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, dev);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageInquiry(xhci, dev);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageTestUnitReady(xhci, dev);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", dev->SlotId, st);
goto fail;
}
st = UsbStorageReadCapacity(xhci, dev);
if (!KSUCCESS(st)) goto fail;
if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
DbgPrint("usb-storage: unsupported block size slot=%u block_size=%u\n",
dev->SlotId, dev->MassBlockSize);
st = STATUS_NOT_SUPPORTED;
goto fail;
}
dev->MassActive = true;
st = UsbStorageRegisterDisk(dev);
if (!KSUCCESS(st)) goto fail;
DbgPrint("usb-storage: bound slot=%u if=%u\n", dev->SlotId, dev->MassInterface);
return STATUS_SUCCESS;
fail:
dev->MassCandidate = false;
dev->MassActive = false;
DbgPrint("usb-storage: bind failed slot=%u if=%u st=0x%x\n",
dev->SlotId, pub->u.Usb.InterfaceNumber, st);
return st;
return STATUS_NOT_SUPPORTED;
}
KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device)
return STATUS_INVALID_PARAMETER;
if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_MASS_STORAGE ||
pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_MASS_SUBCLASS_SCSI ||
pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_MASS_PROTOCOL_BOT)
return STATUS_INVALID_PARAMETER;
if (!UsbStorageFindBulkEndpoint(pdoExt, true) || !UsbStorageFindBulkEndpoint(pdoExt, false))
return STATUS_NOT_FOUND;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(driver, sizeof(USB_STORAGE_EXTENSION), &fdo);
if (!KSUCCESS(st))
return st;
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-storage"));
USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)fdo->DeviceExtension;
ext->Kind = UsbStorageExtensionInterface;
ext->Self = fdo;
ext->InterfacePdo = physicalDeviceObject;
ext->Device = pdoExt->Device;
ext->XhciDevice = (XHCI_DEVICE*)pdoExt->Device->HcdDeviceContext;
ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
ext->IoBusy = 0;
st = IoInitializeRemoveLock(&ext->RemoveLock);
if (!KSUCCESS(st)) {
IoDeleteDevice(fdo);
return st;
}
ext->RemoveLockReady = true;
ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!ext->LowerDevice) {
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
IoDeleteDevice(fdo);
return STATUS_FAILURE;
}
DbgPrint("usb-storage: FDO attached slot=%u if=%u\n",
ext->Device->SlotId,
pdoExt->InterfaceDescriptor.bInterfaceNumber);
return STATUS_SUCCESS;
}
KSTATUS UsbInitializeStorageDriver(void) {
memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
IoInitializeDriverObject(&gUsbStorageDriver, SYS_SV_LITERAL("usb-storage"));
gUsbStorageDriver.IdTable = &gUsbStorageIdTable;
gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice;
gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp;
return IoRegisterDriverObject(&gUsbStorageDriver);
}
code/drivers/bus/usb/usb_xhci.c
+596 -66
View File
@@ -353,6 +353,31 @@ void XhciLogPortStatus(XHCI_CONTROLLER* xhci) {
}
}
void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->Op)
return;
uint32_t powered = 0;
for (uint8_t port = 1; port <= xhci->MaxPorts; port++) {
uint32_t off = XhciPortReg(port);
uint32_t portsc = XhciRead32(xhci->Op, off);
uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK;
if (!(portsc & XHCI_PORTSC_PP)) {
XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
powered++;
UsbTrace("usb: xHCI powered root port %u old-portsc=%08x\n", port, portsc);
} else if (changes) {
XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
}
}
if (powered) {
UsbDelayUs(xhci->Hal, 20000u);
DbgPrint("usb: xHCI powered %u previously-off root port(s)\n", powered);
}
}
void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
if (!xhci->Runtime) return;
@@ -369,7 +394,71 @@ void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
}
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->CommandPhys && w->CommandPhys == commandPhys)
return w;
}
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint8_t slotId,
uint8_t endpointId) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->TransferPhys &&
w->TransferPhys == transferPhys &&
w->TransferSlotId == slotId &&
w->TransferEndpointId == endpointId) {
return w;
}
}
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
uint64_t commandPhys,
uint32_t completionCode,
uint8_t slotId) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys);
if (w) {
w->Command.CompletionCode = completionCode;
w->Command.SlotId = slotId;
XhciBarrier();
w->CommandMatched = true;
}
KeReleaseSpinlock(&xhci->WaitLock, flags);
return w;
}
static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint8_t slotId,
uint8_t endpointId,
uint32_t completionCode,
uint32_t residue) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId);
if (w) {
w->Transfer.CompletionCode = completionCode;
w->Transfer.SlotId = slotId;
w->Transfer.EndpointId = endpointId;
w->Transfer.Residue = residue;
XhciBarrier();
w->TransferMatched = true;
}
KeReleaseSpinlock(&xhci->WaitLock, flags);
return w;
}
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
bool processed = false;
for (;;) {
@@ -384,41 +473,28 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
uint32_t cc = XhciCompletionCode(ev);
uint64_t param = ((uint64_t)ev->ParameterHi << 32) | ev->ParameterLo;
XHCI_WAIT_TARGET* waiter = target ? target : (XHCI_WAIT_TARGET*)xhci->ActiveWait;
if (type == XHCI_TRB_TYPE_CMD_DONE) {
uint8_t slotId = XhciEventSlotId(ev);
UsbTrace("usb: xHCI command completion ptr=0x%016llX cc=%u slot=%u\n",
(unsigned long long)param, cc, slotId);
if (waiter && waiter->CommandPhys && param == waiter->CommandPhys) {
waiter->CommandMatched = true;
waiter->Command.CompletionCode = cc;
waiter->Command.SlotId = slotId;
}
(void)XhciClaimCommandWaiter(xhci, param, cc, slotId);
} else if (type == XHCI_TRB_TYPE_TRANSFER) {
uint8_t slotId = XhciEventSlotId(ev);
uint8_t epId = XhciEventEndpointId(ev);
uint32_t residue = XhciTransferResidue(ev);
bool claimedByWaiter = false;
bool targetMatches = waiter && waiter->TransferPhys && param == waiter->TransferPhys &&
slotId == waiter->TransferSlotId && epId == waiter->TransferEndpointId;
if (cc != XHCI_CC_SUCCESS) {
UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n",
(unsigned long long)param, cc, slotId, epId, residue);
}
if (targetMatches) {
waiter->TransferMatched = true;
waiter->Transfer.CompletionCode = cc;
waiter->Transfer.SlotId = slotId;
waiter->Transfer.EndpointId = epId;
waiter->Transfer.Residue = residue;
claimedByWaiter = true;
}
if (!claimedByWaiter)
XHCI_WAIT_TARGET* waiter = XhciClaimTransferWaiter(xhci, param, slotId, epId, cc, residue);
if (!waiter)
XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue);
} else if (type == XHCI_TRB_TYPE_PORT_SC) {
uint32_t portId = ev->ParameterLo >> 24;
UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc);
if (portId && portId <= xhci->MaxPorts)
UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)portId);
} else {
DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n",
type, cc, (unsigned long long)param, ev->Status, ev->Control);
@@ -454,22 +530,53 @@ bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
}
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || target->Registered)
return;
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
target->Next = xhci->WaiterList;
target->Registered = true;
xhci->WaiterList = target;
KeReleaseSpinlock(&xhci->WaitLock, flags);
}
void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || !target->Registered)
return;
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET** link = &xhci->WaiterList;
while (*link) {
if (*link == target) {
*link = target->Next;
target->Next = (XHCI_WAIT_TARGET*)0;
target->Registered = false;
break;
}
link = &(*link)->Next;
}
KeReleaseSpinlock(&xhci->WaitLock, flags);
}
KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
if (!xhci || !target)
return STATUS_INVALID_PARAMETER;
xhci->ActiveWait = target;
for (uint32_t i = 0; i < timeoutUs; i++) {
if (XhciWaitTargetComplete(target)) {
xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
XhciUnregisterWaitTarget(xhci, target);
return STATUS_SUCCESS;
}
// The IRQ/DPC path normally owns the event ring after enumeration, but
// synchronous storage/control waits must not depend exclusively on an
// interrupt being delivered. Opportunistically drain the ring from the
// waiter too; EventLock serializes this with the DPC.
// interrupt being delivered. Opportunistically drain the ring too;
// EventLock serializes this with the DPC while WaitLock protects waiter
// registration/matching.
if (!xhci->InterruptsEnabled || (i & 0x7u) == 0) {
XhciDrainEventRingLocked(xhci, target);
} else if ((XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_EINT) && (i & 0x3Fu) == 0) {
@@ -479,11 +586,20 @@ KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
UsbDelayUs(xhci->Hal, 1);
}
xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
XhciUnregisterWaitTarget(xhci, target);
return STATUS_TIMEOUT;
}
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
if (!xhci || !target)
return STATUS_INVALID_PARAMETER;
XhciRegisterWaitTarget(xhci, target);
return XhciWaitForArmedTarget(xhci, target, timeoutUs);
}
void XhciEventDpcRoutine(KDPC* dpc, void* context,
void* systemArgument1, void* systemArgument2) {
(void)dpc;
@@ -695,13 +811,26 @@ uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
}
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
XHCI_COMMAND_RESULT* outResult) {
if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u)
return STATUS_QUEUE_FULL;
static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
if (!xhci || !xhci->CommandRing)
return STATUS_INVALID_PARAMETER;
if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) {
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
xhci->CommandCycle;
xhci->CommandEnqueue = 0;
xhci->CommandCycle ^= 1u;
}
uint32_t index = xhci->CommandEnqueue++;
XHCI_TRB* trb = &xhci->CommandRing[index];
@@ -713,15 +842,38 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
XhciBarrier();
trb->Control = controlWithoutCycle | xhci->CommandCycle;
if (outPhys) *outPhys = trbPhys;
return STATUS_SUCCESS;
}
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
XHCI_COMMAND_RESULT* outResult) {
if (!xhci)
return STATUS_INVALID_PARAMETER;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock);
uint64_t trbPhys = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys);
if (!KSUCCESS(st)) {
KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
return st;
}
target.CommandPhys = trbPhys;
xhci->ActiveWait = &target;
XhciRegisterWaitTarget(xhci, &target);
XhciBarrier();
XhciWrite32(xhci->Doorbells, 0, 0); // host-controller doorbell, command ring
KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
KSTATUS waitStatus = XhciWaitForTarget(xhci, &target, 500000u);
KSTATUS waitStatus = XhciWaitForArmedTarget(xhci, &target, 500000u);
if (KSUCCESS(waitStatus) && target.CommandMatched) {
if (outResult) *outResult = target.Command;
return (target.Command.CompletionCode == XHCI_CC_SUCCESS)
@@ -781,6 +933,28 @@ KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
}
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0)
return STATUS_INVALID_PARAMETER;
XHCI_COMMAND_RESULT res;
memset(&res, 0, sizeof(res));
KSTATUS st = XhciSubmitCommand(xhci,
0,
0,
(XHCI_TRB_TYPE_DISABLE_SLOT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)slotId << 24),
&res);
if (!KSUCCESS(st))
return st;
if (res.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Disable Slot slot=%u failed cc=%u\n", slotId, res.CompletionCode);
return STATUS_DEVICE_ERROR;
}
return STATUS_SUCCESS;
}
KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
@@ -970,12 +1144,22 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
uint32_t maxPacket = XhciEpMaxPacket(dev->Speed);
dev->MaxPacketSize0 = (uint16_t)maxPacket;
slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed);
uint8_t rootPort = dev->Path.RootPort ? dev->Path.RootPort : dev->Port;
uint32_t routeString = dev->Path.RouteString & 0x000fffffu;
slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed) | routeString;
// Slot context dword 1 owns Root Hub Port Number bits 23:16.
// Putting the port number in dword 2 makes Address Device fail with a
// context/TRB error on QEMU xHCI.
slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
slot[1] = XHCI_SLOT_ROOT_PORT(rootPort);
slot[2] = 0;
if (dev->Path.BehindHub && dev->Speed <= 2u && dev->Path.ParentHubSlot) {
// TT Hub Slot ID bits 7:0 and TT Port Number bits 15:8.
// This is required for low/full-speed devices behind high-speed hubs;
// harmless for older full-speed hub paths on the controllers we target.
slot[2] = (uint32_t)dev->Path.ParentHubSlot |
((uint32_t)dev->Path.ParentHubPort << 8);
}
slot[3] = 0;
ep0[0] = 0;
@@ -999,8 +1183,21 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u)
return STATUS_QUEUE_FULL;
if (!dev || !dev->Ep0Ring)
return STATUS_INVALID_PARAMETER;
if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) {
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
dev->Ep0Cycle;
dev->Ep0Enqueue = 0;
dev->Ep0Cycle ^= 1u;
}
uint32_t index = dev->Ep0Enqueue++;
XHCI_TRB* trb = &dev->Ep0Ring[index];
@@ -1017,11 +1214,12 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
}
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint16_t length) {
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint16_t length,
uint32_t timeoutUs) {
if (!buffer || length == 0 || length > PAGE_SIZE)
return STATUS_INVALID_PARAMETER;
@@ -1057,18 +1255,21 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
&statusTrbPhys);
if (!KSUCCESS(st)) return st;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
st = XhciWaitForTarget(xhci, &target, 1000000u);
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
st = XhciWaitForArmedTarget(xhci, &target, timeoutUs);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI control IN timeout slot=%u len=%u\n", dev->SlotId, length);
DbgPrint("usb: xHCI control IN timeout slot=%u len=%u timeout=%uus\n",
dev->SlotId, length, timeoutUs);
return st;
}
if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS)
@@ -1082,6 +1283,16 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint16_t length) {
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length,
USB_XHCI_CONTROL_TIMEOUT_US);
}
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup) {
uint64_t setupValue = UsbSetupPacketToU64(setup);
@@ -1106,16 +1317,17 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
&statusTrbPhys);
if (!KSUCCESS(st)) return st;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
st = XhciWaitForTarget(xhci, &target, 1000000u);
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI control no-data timeout slot=%u req=%u value=%04x\n",
dev->SlotId, setup->bRequest, setup->wValue);
@@ -1132,8 +1344,9 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc) {
KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc,
uint32_t timeoutUs) {
USB_SETUP_PACKET setup;
setup.bmRequestType = USB_DIR_IN;
setup.bRequest = USB_REQ_GET_DESCRIPTOR;
@@ -1141,10 +1354,12 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
setup.wIndex = 0;
setup.wLength = sizeof(USB_DEVICE_DESCRIPTOR);
KSTATUS st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
sizeof(USB_DEVICE_DESCRIPTOR));
if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
KSTATUS st = XhciControlInTimeout(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
sizeof(USB_DEVICE_DESCRIPTOR),
timeoutUs);
if (!KSUCCESS(st)) return st;
memcpy(outDesc, dev->DescriptorBuffer, sizeof(*outDesc));
@@ -1159,6 +1374,13 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc) {
return XhciGetDeviceDescriptorTimeout(xhci, dev, outDesc,
USB_XHCI_CONTROL_TIMEOUT_US);
}
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t configurationIndex,
USB_CONFIGURATION_DESCRIPTOR* outConfig,
@@ -1331,11 +1553,11 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = endpointId;
xhci->ActiveWait = &target;
XhciRegisterWaitTarget(xhci, &target);
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, endpointId);
st = XhciWaitForTarget(xhci, &target, 1000000u);
st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: bulk transfer timeout slot=%u ep=%u len=%u\n",
dev->SlotId, endpointId, length);
@@ -1356,6 +1578,300 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) {
if (!xhci || !dev || !pipe || !pipe->Open)
return STATUS_INVALID_PARAMETER;
uint8_t dci = pipe->EndpointId;
if (dci < 2 || dci > 31)
return STATUS_INVALID_PARAMETER;
if (pipe->HcdPipeContext) {
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
return ctx->Configured ? STATUS_SUCCESS : STATUS_DEVICE_ERROR;
}
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)MmAllocatePool(sizeof(XHCI_PIPE_CONTEXT));
if (!ctx) return STATUS_NO_MEMORY;
memset(ctx, 0, sizeof(*ctx));
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys);
if (!ctx->Ring) {
MmFreePool(ctx);
return STATUS_NO_MEMORY;
}
ctx->Enqueue = 0;
ctx->Cycle = 1;
ctx->EndpointId = dci;
ctx->Type = pipe->Type;
XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle);
memset(dev->InputContext, 0, PAGE_SIZE);
uint32_t* icc = XhciInputControlContext(xhci, dev);
uint32_t* slot = XhciInputSlotContext(xhci, dev);
uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci);
icc[0] = 0;
icc[1] = (1u << 0) | (1u << dci);
slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed);
slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
slot[2] = 0;
slot[3] = 0;
uint16_t maxPacket = pipe->MaxPacketSize & 0x07FFu;
if (maxPacket == 0) maxPacket = 8u;
uint32_t endpointType = 0;
uint8_t interval = 0;
if (pipe->Type == UsbPipeInterrupt && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
endpointType = XHCI_EP_TYPE_INT_IN;
interval = XhciInterruptInterval(dev->Speed, pipe->Interval);
} else if (pipe->Type == UsbPipeBulk && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
endpointType = XHCI_EP_TYPE_BULK_IN;
} else if (pipe->Type == UsbPipeBulk) {
endpointType = XHCI_EP_TYPE_BULK_OUT;
} else {
MmFreePool(ctx);
return STATUS_NOT_SUPPORTED;
}
ep[0] = (uint32_t)interval << 16;
ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16);
ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32);
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
&result);
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
MmFreePool(ctx);
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
}
ctx->Configured = true;
pipe->HcdPipeContext = ctx;
dev->EndpointPipes[dci] = pipe;
UsbReferencePipe(pipe);
return STATUS_SUCCESS;
}
static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
return STATUS_INVALID_PARAMETER;
if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN))
return STATUS_NOT_SUPPORTED;
KSTATUS st = XhciConfigurePipeEndpoint(xhci, dev, urb->Pipe);
if (!KSUCCESS(st)) return st;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext;
if (!ctx || !ctx->Configured)
return STATUS_DEVICE_ERROR;
if (ctx->ActiveTransfer)
return STATUS_QUEUE_FULL;
uint64_t trbPhys = 0;
st = XhciRingAppend(ctx->Ring,
ctx->RingPhys,
XHCI_PIPE_RING_TRBS,
&ctx->Enqueue,
&ctx->Cycle,
urb->BufferPhys,
urb->Length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
if (!KSUCCESS(st)) return st;
ctx->ActiveTransfer = true;
ctx->ActiveTrbPhys = trbPhys;
ctx->ActiveUrb = urb;
urb->Pending = true;
urb->Cancelled = false;
urb->Status = STATUS_PENDING;
urb->ActualLength = 0;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, ctx->EndpointId);
return STATUS_PENDING;
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
return;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
USB_PIPE* pipe = dev->EndpointPipes[endpointId];
if (!pipe || !pipe->HcdPipeContext)
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys)
return;
USB_URB* urb = ctx->ActiveUrb;
ctx->ActiveTransfer = false;
ctx->ActiveTrbPhys = 0;
ctx->ActiveUrb = (USB_URB*)0;
if (!urb)
return;
KSTATUS st = STATUS_SUCCESS;
if (urb->Cancelled) {
st = STATUS_CANCELLED;
} else if (completionCode != XHCI_CC_SUCCESS) {
st = STATUS_DEVICE_ERROR;
}
uint32_t actual = 0;
if (KSUCCESS(st)) {
actual = (residue <= urb->Length) ? (urb->Length - residue) : 0;
}
urb->Pending = false;
urb->Status = st;
urb->ActualLength = actual;
IRP* completeIrp = urb->Irp;
urb->Irp = (IRP*)0;
if (urb->Complete)
urb->Complete(urb, urb->CompleteContext);
if (completeIrp)
IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0);
}
static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
return STATUS_INVALID_PARAMETER;
uint8_t endpointId = urb->Pipe->EndpointId;
XHCI_TRB* ring = (XHCI_TRB*)0;
PHYSICAL_ADDRESS ringPhys = 0;
uint32_t* enqueue = (uint32_t*)0;
uint8_t* cycle = (uint8_t*)0;
if (endpointId == dev->MassBulkInEndpointId) {
ring = dev->MassInRing;
ringPhys = dev->MassInRingPhys;
enqueue = &dev->MassInEnqueue;
cycle = &dev->MassInCycle;
} else if (endpointId == dev->MassBulkOutEndpointId) {
ring = dev->MassOutRing;
ringPhys = dev->MassOutRingPhys;
enqueue = &dev->MassOutEnqueue;
cycle = &dev->MassOutCycle;
} else {
return STATUS_NOT_SUPPORTED;
}
uint32_t actual = 0;
KSTATUS st = XhciBulkTransfer(xhci,
dev,
endpointId,
ring,
ringPhys,
enqueue,
cycle,
urb->BufferPhys,
urb->Length,
&actual);
urb->ActualLength = KSUCCESS(st) ? actual : 0;
return st;
}
static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
if (!hcd || !urb || !urb->Device)
return STATUS_INVALID_PARAMETER;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext;
XHCI_DEVICE* dev = (XHCI_DEVICE*)urb->Device->HcdDeviceContext;
if (!xhci || !dev || !dev->Allocated)
return STATUS_INVALID_PARAMETER;
KSTATUS st;
switch (urb->Function) {
case UrbFunctionControlTransfer:
if (urb->Length == 0) {
st = XhciControlNoData(xhci, dev, &urb->Setup);
urb->ActualLength = 0;
} else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) {
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length);
urb->ActualLength = KSUCCESS(st) ? urb->Length : 0;
} else {
st = STATUS_NOT_SUPPORTED;
urb->ActualLength = 0;
}
break;
case UrbFunctionBulkTransfer:
st = XhciHcdSubmitBulkUrb(xhci, dev, urb);
break;
case UrbFunctionInterruptTransfer:
st = XhciHcdSubmitInterruptUrb(xhci, dev, urb);
break;
default:
st = STATUS_NOT_SUPPORTED;
urb->ActualLength = 0;
break;
}
urb->Status = st;
return st;
}
static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
(void)hcd;
if (!urb) return STATUS_INVALID_PARAMETER;
urb->Cancelled = true;
if (!urb->Pending)
urb->Status = STATUS_CANCELLED;
return STATUS_SUCCESS;
}
static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!hcd) return (void*)0;
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
}
static KSTATUS XhciHcdResetRootPort(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed) {
if (!hcd) return STATUS_INVALID_PARAMETER;
return XhciResetPort((XHCI_CONTROLLER*)hcd->ControllerContext, port, outSpeed);
}
static KSTATUS XhciHcdClearRootPortChanges(USB_HCD* hcd, uint8_t port) {
if (!hcd) return STATUS_INVALID_PARAMETER;
XhciClearPortChanges((XHCI_CONTROLLER*)hcd->ControllerContext, port);
return STATUS_SUCCESS;
}
const USB_HCD_OPERATIONS gXhciHcdOperations = {
XhciHcdSubmitUrb,
XhciHcdCancelUrb,
XhciHcdResetRootPort,
XhciHcdClearRootPortChanges,
XhciHcdAllocateDma,
};
KSTATUS XhciInitialise(const HAL_API* hal,
uint8_t bus, uint8_t dev, uint8_t fn,
uint16_t vendor, uint16_t device) {
@@ -1438,6 +1954,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
return STATUS_DEVICE_ERROR;
}
UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts);
KSTATUS hcdPdoStatus = UsbCreateControllerPdo(&xhci->Hcd, vendor, device);
if (!KSUCCESS(hcdPdoStatus)) {
DbgPrint("usb: xHCI controller PDO creation failed st=0x%x\n", hcdPdoStatus);
return hcdPdoStatus;
}
uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE +
(uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE;
uint32_t needBytes = portEnd;
@@ -1489,12 +2012,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
st = XhciSubmitNoopCommand(xhci);
if (!KSUCCESS(st)) return st;
XhciPowerRootPorts(xhci);
XhciLogPortStatus(xhci);
XhciEnumerateRootPorts(xhci);
if (xhci->HidDeviceCount > 0) {
st = XhciStartHidInterrupts(xhci);
if (!KSUCCESS(st)) return st;
st = UsbCreateRootHubPdo(&xhci->Hcd);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) {
DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st);
return st;
}
KeIrqRegister(xhci->Irq, XhciIsr);
@@ -1504,8 +2028,14 @@ KSTATUS XhciInitialise(const HAL_API* hal,
DbgPrint("usb: xHCI ISR registered on IRQ %u%s; event ring handled by DPC\n",
xhci->Irq, xhci->UsingMsi ? "(msi)" : "");
st = UsbHubScheduleRootScan(&xhci->Hcd);
if (!KSUCCESS(st))
DbgPrint("usb: root hub async scan scheduling failed st=0x%x\n", st);
else
DbgPrint("usb: root hub initial scan queued asynchronously\n");
gHaveXhci = true;
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage is next\n",
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
xhci->HidDeviceCount);
return STATUS_SUCCESS;
}
code/drivers/fs/fat32/fat32.c
+140 -23
View File
@@ -1612,7 +1612,10 @@ static void Fat32AddCandidate(FAT32_VOL* probe,
static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
FAT32_PARTITION_CANDIDATE* out,
uint32_t maxCount) {
uint32_t maxCount,
bool* outMediaReadable) {
if (outMediaReadable)
*outMediaReadable = false;
if (!out || maxCount == 0) return 0;
FAT32_VOL probe;
@@ -1623,6 +1626,8 @@ static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
uint8_t mbr[512];
if (!KSUCCESS(ReadSectors(&probe, 0, 1, mbr)))
return 0;
if (outMediaReadable)
*outMediaReadable = true;
// A disk image may be a raw FAT32 volume with no partition table.
Fat32AddCandidate(&probe, out, &count, maxCount, 0, IoDiskSectorCount(diskIndex));
@@ -1734,6 +1739,7 @@ typedef struct FAT32_ASYNC_RW {
size_t CurrentCopy;
IRP Irp;
DEVICE_OBJECT* CurrentDevice;
IO_FS_ASYNC_COMPLETE Complete;
void* CompleteContext;
} FAT32_ASYNC_RW;
@@ -1811,6 +1817,10 @@ static void Fat32AsyncQueueContinuation(FAT32_ASYNC_RW* ctx) {
static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t information) {
if (!ctx) return;
if (ctx->CurrentDevice) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
}
if (ctx->RunBuf) MmFreePool(ctx->RunBuf);
IO_FS_ASYNC_COMPLETE complete = ctx->Complete;
void* completeContext = ctx->CompleteContext;
@@ -1821,6 +1831,10 @@ static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t informa
static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
if (!ctx) return;
if (ctx->CurrentDevice) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
}
if (!KSUCCESS(irp->Status)) {
Fat32AsyncFinish(ctx, irp->Status, ctx->Done);
@@ -1872,17 +1886,25 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
ctx->CurrentOffset = ctx->OffsetInCluster;
ctx->CurrentCopy = copy;
memset(&ctx->Irp, 0, sizeof(ctx->Irp));
ctx->Irp.DriveIndex = v->DriveIndex;
ctx->Irp.Lba = ClusterToLba(v, runStart);
ctx->Irp.SectorCount = runSectors;
ctx->Irp.Buffer = ctx->RunBuf;
ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex);
if (!ctx->CurrentDevice) {
Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done);
return;
}
IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_READ,
ClusterToLba(v, runStart) * 512ull,
runSectors * 512u,
ctx->RunBuf);
ctx->Irp.CompletionFn = Fat32AsyncReadIrpComplete;
ctx->Irp.CompletionCtx = ctx;
KSTATUS s = IoSubmitIrp(&ctx->Irp);
if (s != STATUS_PENDING)
KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp);
if (s != STATUS_PENDING) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
Fat32AsyncFinish(ctx, s, ctx->Done);
}
}
KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
@@ -1949,6 +1971,10 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) {
FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
if (!ctx) return;
if (ctx->CurrentDevice) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
}
if (!KSUCCESS(irp->Status)) {
Fat32AsyncFinish(ctx, irp->Status, ctx->Done);
@@ -2024,18 +2050,25 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
ctx->CurrentOffset = ctx->OffsetInCluster;
ctx->CurrentCopy = copy;
memset(&ctx->Irp, 0, sizeof(ctx->Irp));
ctx->Irp.DriveIndex = v->DriveIndex;
ctx->Irp.Flags = IRP_FLAG_WRITE;
ctx->Irp.Lba = ClusterToLba(v, runStart);
ctx->Irp.SectorCount = runSectors;
ctx->Irp.Buffer = ctx->RunBuf;
ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex);
if (!ctx->CurrentDevice) {
Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done);
return;
}
IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_WRITE,
ClusterToLba(v, runStart) * 512ull,
runSectors * 512u,
ctx->RunBuf);
ctx->Irp.CompletionFn = Fat32AsyncWriteIrpComplete;
ctx->Irp.CompletionCtx = ctx;
KSTATUS s = IoSubmitIrp(&ctx->Irp);
if (s != STATUS_PENDING)
KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp);
if (s != STATUS_PENDING) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
Fat32AsyncFinish(ctx, s, ctx->Done);
}
}
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
@@ -2124,31 +2157,55 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
static FAT32_VOL gVols[FAT32_MAX_VOLUMES];
static IO_FILESYSTEM gFs[FAT32_MAX_VOLUMES];
static bool gVolumeMounted[FAT32_MAX_VOLUMES];
static uint32_t gMountedCount;
static bool gDiskScanned[FAT32_MAX_VOLUMES];
static int Fat32FindFreeVolumeIndex(void) {
for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) {
if (!gVolumeMounted[i] && !IoFilesystemPresent(i))
return i;
}
return -1;
}
static uint32_t Fat32TryMountDisk(uint8_t disk) {
if (disk >= FAT32_MAX_VOLUMES)
return 0;
if (gDiskScanned[disk])
return 0;
gDiskScanned[disk] = true;
if (!IoDiskDevicePresent(disk))
return 0;
uint32_t mountedHere = 0;
FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK];
uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK);
bool mediaReadable = false;
uint32_t partCount = Fat32EnumerateDiskVolumes(disk,
parts,
FAT32_MAX_PARTITIONS_PER_DISK,
&mediaReadable);
if (!mediaReadable) {
DbgPrint("FAT32: disk %u: media probe deferred; sector 0 unreadable\n", disk);
return 0;
}
// Only remember a completed probe. A transient hotplug/readiness failure
// must not permanently suppress a later mount attempt for the same disk
// index after a device is unplugged and replugged.
gDiskScanned[disk] = true;
if (!partCount) {
DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk);
return 0;
}
for (uint32_t part = 0; part < partCount && gMountedCount < FAT32_MAX_VOLUMES; part++) {
uint8_t volumeIndex = (uint8_t)gMountedCount;
for (uint32_t part = 0; part < partCount; part++) {
int freeVolume = Fat32FindFreeVolumeIndex();
if (freeVolume < 0)
break;
uint8_t volumeIndex = (uint8_t)freeVolume;
FAT32_VOL* v = &gVols[volumeIndex];
memset(v, 0, sizeof(*v));
v->DriveIndex = disk;
@@ -2192,6 +2249,7 @@ static uint32_t Fat32TryMountDisk(uint8_t disk) {
KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams);
if (KSUCCESS(status)) {
gVolumeMounted[volumeIndex] = true;
gMountedCount++;
mountedHere++;
DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n",
@@ -2222,6 +2280,60 @@ static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* co
}
}
static void Fat32UnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) {
uint32_t removed = 0;
for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) {
if (!gVolumeMounted[i])
continue;
if (gVols[i].DriveIndex != diskIndex)
continue;
// Surprise removal means the backing device may already be gone; do not
// attempt writeback here. Drop cached FAT sectors instead.
FatCacheInvalidate();
(void)IoUnregisterFilesystem(i);
memset(&gVols[i], 0, sizeof(gVols[i]));
memset(&gFs[i], 0, sizeof(gFs[i]));
gVolumeMounted[i] = false;
if (gMountedCount)
gMountedCount--;
removed++;
}
if (diskIndex < FAT32_MAX_VOLUMES)
gDiskScanned[diskIndex] = false;
if (removed) {
DbgPrint("FAT32: disk %u %s; unmounted %u volume(s); total=%u\n",
diskIndex,
change == IoDeviceSurpriseRemoved ? "surprise removed" : "removed",
removed,
gMountedCount);
}
}
static void Fat32DiskChange(uint8_t driveIndex,
DEVICE_OBJECT* device,
IO_DEVICE_CHANGE change,
void* context) {
(void)device;
(void)context;
switch (change) {
case IoDeviceArrived:
Fat32DiskArrival(driveIndex, device, context);
break;
case IoDeviceSurpriseRemoved:
case IoDeviceRemoved:
Fat32UnmountDisk(driveIndex, change);
break;
case IoDeviceQueryRemove:
default:
break;
}
}
KSTATUS DriverEntry(const HAL_API *hal) {
DbgPrint("FAT32: Init\n");
if (!hal) return STATUS_INVALID_PARAMETER;
@@ -2231,9 +2343,14 @@ KSTATUS DriverEntry(const HAL_API *hal) {
gMountedCount = 0;
memset(gDiskScanned, 0, sizeof(gDiskScanned));
KSTATUS cbStatus = IoRegisterDiskArrivalCallback(Fat32DiskArrival, (void*)0);
KSTATUS cbStatus = IoRegisterDiskChangeCallback(Fat32DiskChange, (void*)0);
if (!KSUCCESS(cbStatus)) {
DbgPrint("FAT32: disk-arrival callback registration failed st=0x%x\n", cbStatus);
DbgPrint("FAT32: disk-change callback registration failed st=0x%x\n", cbStatus);
}
cbStatus = IoEnumerateDiskDevices(Fat32DiskArrival, (void*)0);
if (!KSUCCESS(cbStatus)) {
DbgPrint("FAT32: initial disk enumeration failed st=0x%x\n", cbStatus);
}
if (!gMountedCount) {
code/drivers/storage/ahci/ahci.c
+105 -12
View File
@@ -382,6 +382,28 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
// Async IRP dispatch
// ---------------------------------------------------------------------------
static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
uint32_t* outSectors, void** outBuffer) {
if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return STATUS_INVALID_PARAMETER;
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
return STATUS_INVALID_PARAMETER;
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
(ios->Parameters.ReadWrite.Length & 511u))
return STATUS_INVALID_PARAMETER;
if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
return STATUS_INVALID_PARAMETER;
*outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
*outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
*outSectors = ios->Parameters.ReadWrite.Length >> 9;
*outBuffer = irp->UserBuffer;
return STATUS_SUCCESS;
}
// AhciDispatchReadWrite - DEVICE_OBJECT dispatch entry for disk IRPs.
// Accepted IRPs are queued to the per-drive worker and completed later with
// IoCompleteRequest(). Rejected IRPs are returned synchronously and are not
@@ -390,8 +412,22 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
AHCI_DRIVE *drv = (AHCI_DRIVE *)device->DeviceExtension;
if (!drv) return STATUS_INVALID_PARAMETER;
if (irp->SectorCount == 0 || !irp->Buffer) return STATUS_INVALID_PARAMETER;
if (irp->Lba + irp->SectorCount > drv->sectorCount) return STATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp);
if (!KSUCCESS(st))
return st;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
(void)write;
(void)buffer;
if (!KSUCCESS(st) || lba + sectors > drv->sectorCount) {
IoReleaseRemoveLock(&drv->removeLock, irp);
return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
}
irp->Status = STATUS_FAILURE;
irp->Information = 0;
@@ -399,6 +435,49 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
return STATUS_PENDING;
}
static void AhciCancelReleaseRemoveLock(IRP* irp, void* context) {
AHCI_DRIVE* drv = (AHCI_DRIVE*)context;
if (drv)
IoReleaseRemoveLock(&drv->removeLock, irp);
}
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
if (!drv) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return STATUS_SUCCESS;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_SURPRISE_REMOVAL:
drv->surpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
return STATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
drv->removed = true;
device->PnpState = DevicePnpDeleted;
IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
IoDeleteDevice(device);
drv->deviceObject = (DEVICE_OBJECT*)0;
return STATUS_SUCCESS;
default:
return STATUS_NOT_SUPPORTED;
}
}
// AhciWorker - kernel thread, one per drive.
// Dequeues IRPs and executes them one at a time using the existing
// single-slot interrupt-driven path. Completion fires the IRP's event
@@ -420,17 +499,24 @@ static void AhciWorker(uintptr_t arg) {
KeWaitSemaphore(drv->irpQueue.Semaphore);
IRP *irp = IrpQueueDequeue(&drv->irpQueue);
if (!irp) KePanic(KE_PANIC_CODE_DRIVER, "AhciWorker: semaphore fired but queue is empty - IRP accounting bug\n");
if (!irp)
continue;
bool write = !!(irp->Flags & IRP_FLAG_WRITE);
KSTATUS st;
if (write)
st = do_write(drv, irp->Lba, irp->SectorCount, irp->Buffer);
else
st = do_read (drv, irp->Lba, irp->SectorCount, irp->Buffer);
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read (drv, lba, sectors, buffer);
}
// Publish result then fire completion hooks through the common I/O path.
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
IoReleaseRemoveLock(&drv->removeLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
}
}
@@ -467,9 +553,10 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
g_driveCount = 0;
memset(&g_driver, 0, sizeof(g_driver));
g_driver.Name = "ahci";
IoInitializeDriverObject(&g_driver, SYS_SV_LITERAL("ahci"));
g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
// -----------------------------------------------------------------------
// Locate AHCI controller via PCI class scan
@@ -588,6 +675,11 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
continue;
}
IrpQueueInit(&drv->irpQueue, irpSem);
st = IoInitializeRemoveLock(&drv->removeLock);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st);
continue;
}
// Phase 6: IDENTIFY (polled - scheduler not yet running)
st = port_identify(drv);
@@ -613,7 +705,8 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
continue;
}
drv->deviceObject->DeviceExtension = drv;
drv->deviceObject->SectorCount = drv->sectorCount;
drv->deviceObject->u.Disk.SectorCount = drv->sectorCount;
drv->deviceObject->u.Disk.SectorSize = 512;
drv->deviceObject->AlignmentRequirement = 512;
st = IoRegisterDiskDevice((uint8_t)driveIdx, drv->deviceObject);
code/drivers/storage/ahci/ahci.h
+3
View File
@@ -344,6 +344,9 @@ typedef struct {
// AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP
// and signals the semaphore to wake the worker.
IRP_QUEUE irpQueue;
IO_REMOVE_LOCK removeLock;
bool surpriseRemoved;
bool removed;
THREAD* workerThread;
DEVICE_OBJECT* deviceObject;
code/drivers/storage/ide/ide.c
+104 -17
View File
@@ -21,6 +21,9 @@ IRP_QUEUE gIrpQueue;
SEMAPHORE *gIrpSemaphore = NULL;
EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer
KDPC gDmaDpc; // bottom half for DMA completion
IO_REMOVE_LOCK gRemoveLock;
bool gSurpriseRemoved = false;
bool gRemoved = false;
// ---------------------------------------------------------------------------
// Bus Master I/O helpers
@@ -307,6 +310,28 @@ static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) {
return STATUS_SUCCESS;
}
static KSTATUS IdeGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
uint32_t* outSectors, void** outBuffer) {
if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return STATUS_INVALID_PARAMETER;
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
return STATUS_INVALID_PARAMETER;
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
(ios->Parameters.ReadWrite.Length & 511u))
return STATUS_INVALID_PARAMETER;
if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
return STATUS_INVALID_PARAMETER;
*outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
*outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
*outSectors = ios->Parameters.ReadWrite.Length >> 9;
*outBuffer = irp->UserBuffer;
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// IdeWorker — kernel thread, one per IDE controller.
//
@@ -320,17 +345,22 @@ void IdeWorker(uintptr_t arg) {
IRP *irp = IrpQueueDequeue(&gIrpQueue);
if (!irp)
KePanic(KE_PANIC_CODE_DRIVER, "IdeWorker: semaphore fired but IRP queue is empty — accounting bug\n");
continue;
KSTATUS st;
if (irp->Lba + irp->SectorCount > gSectorCount) {
st = STATUS_INVALID_PARAMETER;
} else {
bool write = !!(irp->Flags & IRP_FLAG_WRITE);
st = IdeDmaRw(irp->Lba, irp->SectorCount, irp->Buffer, write);
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = IdeGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (lba + sectors > gSectorCount)
st = STATUS_INVALID_PARAMETER;
else
st = IdeDmaRw(lba, sectors, buffer, write);
}
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
IoReleaseRemoveLock(&gRemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
}
}
@@ -344,8 +374,20 @@ void IdeWorker(uintptr_t arg) {
// ---------------------------------------------------------------------------
static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
(void)device;
if (!irp || !irp->Buffer || irp->SectorCount == 0) return STATUS_INVALID_PARAMETER;
if (irp->Lba + irp->SectorCount > gSectorCount) return STATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&gRemoveLock, irp);
if (!KSUCCESS(st))
return st;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = IdeGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (!KSUCCESS(st) || lba + sectors > gSectorCount) {
IoReleaseRemoveLock(&gRemoveLock, irp);
return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
}
irp->Status = STATUS_FAILURE;
irp->Information = 0;
@@ -355,13 +397,53 @@ static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
return STATUS_PENDING;
}
bool write = !!(irp->Flags & IRP_FLAG_WRITE);
KSTATUS st = write ? IdeWrite(NULL, irp->Lba, irp->SectorCount, irp->Buffer)
: IdeRead (NULL, irp->Lba, irp->SectorCount, irp->Buffer);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
st = write ? IdeWrite(NULL, lba, sectors, buffer)
: IdeRead (NULL, lba, sectors, buffer);
IoReleaseRemoveLock(&gRemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
return STATUS_PENDING;
}
static void IdeCancelReleaseRemoveLock(IRP* irp, void* context) {
(void)context;
IoReleaseRemoveLock(&gRemoveLock, irp);
}
static KSTATUS IdeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return STATUS_SUCCESS;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_SURPRISE_REMOVAL:
gSurpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
return STATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
gRemoved = true;
device->PnpState = DevicePnpDeleted;
IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
IoReleaseRemoveLockAndWait(&gRemoveLock, irp);
IoDeleteDevice(device);
gDevice = (DEVICE_OBJECT*)0;
return STATUS_SUCCESS;
default:
return STATUS_NOT_SUPPORTED;
}
}
// ---------------------------------------------------------------------------
// IdeDmaInit — locate the PCI IDE controller, grab BAR4, allocate the PRDT,
// create sync objects, and register IRQ 14.
@@ -427,13 +509,18 @@ KSTATUS DriverEntry(const HAL_API *hal) {
// ---- Register definitive DEVICE_OBJECT --------------------------------
memset(&gDriver, 0, sizeof(gDriver));
gDriver.Name = "ide";
IoInitializeDriverObject(&gDriver, SYS_SV_LITERAL("ide"));
gDriver.MajorFunction[IRP_MJ_READ] = IdeDispatchReadWrite;
gDriver.MajorFunction[IRP_MJ_WRITE] = IdeDispatchReadWrite;
gDriver.MajorFunction[IRP_MJ_PNP] = IdeDispatchPnp;
KSTATUS st = IoCreateDevice(&gDriver, 0, &gDevice);
KSTATUS st = IoInitializeRemoveLock(&gRemoveLock);
if (!KSUCCESS(st)) return st;
gDevice->SectorCount = gSectorCount;
st = IoCreateDevice(&gDriver, 0, &gDevice);
if (!KSUCCESS(st)) return st;
gDevice->u.Disk.SectorCount = gSectorCount;
gDevice->u.Disk.SectorSize = 512;
gDevice->AlignmentRequirement = 512;
// ---- Try to initialise Bus Master DMA ---------------------------------
code/kernel/hal/hal_main.c
+202 -70
View File
@@ -82,11 +82,13 @@ static uintptr_t gIoapicPhys = 0xFEC00000;
static uintptr_t gHpetPhys = 0xFED00000;
static HAL_CPU_TOPOLOGY gCpuTopology;
// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5)
// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5).
// Do not guess SLP_TYP on real hardware: these values are firmware-specific.
static uint16_t gPm1aCntBlk = 0;
static uint16_t gPm1bCntBlk = 0;
static uint8_t gSlpTypA = 5; // S5 SLP_TYP for PM1a; 5 is correct for QEMU/most VMs
static uint8_t gSlpTypB = 5; // S5 SLP_TYP for PM1b
static uint8_t gSlpTypA = 0;
static uint8_t gSlpTypB = 0;
static int gS5Found = 0;
int debugEnabled = 0;
int _fltused; // MSVC floating-point helper anchor
@@ -106,73 +108,183 @@ static void *HalPhysToVirt(uint64_t phys) {
return (void *)(phys + gPhysBase);
}
// FADT (Fixed ACPI Description Table) - partial layout up to PM1b_CNT_BLK.
// We only need offsets 40 (DSDT ptr), 64 (PM1a_CNT_BLK), 68 (PM1b_CNT_BLK).
// The full table is ~276 bytes (ACPI 2.0+); we never access beyond offset 68.
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
// systems may put the real DSDT and PM1 control registers in the extended
// X_* fields, so do not rely only on the legacy 32-bit fields.
#define FADT_DSDT_OFFSET 40u
#define FADT_PM1A_CNT_BLK_OFFSET 64u
#define FADT_PM1B_CNT_BLK_OFFSET 68u
#define FADT_X_DSDT_OFFSET 140u
#define FADT_X_PM1A_CNT_BLK_OFFSET 172u
#define FADT_X_PM1B_CNT_BLK_OFFSET 184u
#pragma pack(push, 1)
typedef struct {
ACPISDTHeader Header; // 0 (36 bytes)
uint32_t FirmwareCtrl; // 36
uint32_t Dsdt; // 40 - physical addr of DSDT
uint8_t Ignored44; // 44
uint8_t PreferredPmProfile; // 45
uint16_t SciInt; // 46
uint32_t SmiCmd; // 48
uint8_t AcpiEnable; // 52
uint8_t AcpiDisable; // 53
uint8_t S4BiosReq; // 54
uint8_t PstateCnt; // 55
uint32_t Pm1aEvtBlk; // 56
uint32_t Pm1bEvtBlk; // 60
uint32_t Pm1aCntBlk; // 64 - I/O port for PM1a control register
uint32_t Pm1bCntBlk; // 68 - I/O port for PM1b control register
} FADT;
uint8_t AddressSpaceId;
uint8_t RegisterBitWidth;
uint8_t RegisterBitOffset;
uint8_t AccessSize;
uint64_t Address;
} ACPI_GAS;
#pragma pack(pop)
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values.
// The encoding looks like:
// 0x08 '_' 'S' '5' '_' ; NameOp + "_S5_"
// 0x12 <PkgLen> ; PackageOp + encoded length
// <NumElements> ; usually 0x04
// 0x0A <slp_typ_a> ; BytePrefix + value (or 0x00 for zero)
// 0x0A <slp_typ_b>
static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
for (uint32_t i = 0; i + 9 < len; i++) {
// Match NameOp + "_S5_" + PackageOp
if (aml[i] != 0x08) continue;
if (aml[i+1] != 0x5F || aml[i+2] != 0x53 ||
aml[i+3] != 0x35 || aml[i+4] != 0x5F) continue;
if (aml[i+5] != 0x12) continue;
static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) {
uint32_t value = 0;
if (off + sizeof(value) <= tableLen)
memcpy(&value, (const uint8_t*)table + off, sizeof(value));
return value;
}
// PkgLength encoding: low 6 bits = byte count (0 → only this byte);
// bits [7:6] = number of following bytes in the length field.
uint8_t pkgByte = aml[i+6];
uint32_t pkgExtra = pkgByte >> 6; // 0..3 extra bytes
uint32_t j = i + 7 + pkgExtra; // skip PkgLen field
if (j + 2 >= len) break;
j++; // skip NumElements byte
static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off) {
uint64_t value = 0;
if (off + sizeof(value) <= tableLen)
memcpy(&value, (const uint8_t*)table + off, sizeof(value));
return value;
}
// SLP_TYP_A
if (aml[j] == 0x0A && j + 1 < len) {
gSlpTypA = aml[j + 1]; j += 2;
} else if (aml[j] == 0x00) {
gSlpTypA = 0; j++;
} else {
break; // unexpected encoding - keep defaults
static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t off) {
ACPI_GAS gas;
if (off + sizeof(gas) > tableLen) return 0;
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
// 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(),
// so ignore MMIO GAS entries until the HAL grows MMIO register access here.
if (gas.AddressSpaceId != 0x01 || gas.Address == 0 || gas.Address > 0xFFFF)
return 0;
return (uint16_t)gas.Address;
}
static int AmlParsePkgLength(const uint8_t* p, const uint8_t* end,
const uint8_t** body, const uint8_t** pkgEnd) {
if (p >= end) return 0;
const uint8_t* start = p;
uint8_t first = *p++;
uint32_t follow = first >> 6;
uint32_t length;
if (follow == 0) {
length = first & 0x3F;
} else {
length = first & 0x0F;
for (uint32_t i = 0; i < follow; i++) {
if (p >= end) return 0;
length |= ((uint32_t)*p++) << (4 + i * 8);
}
}
// SLP_TYP_B
if (j < len && aml[j] == 0x0A && j + 1 < len)
gSlpTypB = aml[j + 1];
else if (j < len && aml[j] == 0x00)
gSlpTypB = 0;
if (start + length < p || start + length > end) return 0;
*body = p;
*pkgEnd = start + length;
return 1;
}
static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
int* lastSegmentIsS5) {
uint32_t segments = 1;
*lastSegmentIsS5 = 0;
// Optional root or parent prefixes: '\\', '^', '^^', ...
while (p < end && (*p == 0x5C || *p == 0x5E)) p++;
if (p >= end) return NULL;
if (*p == 0x00) { // NullName
return p + 1;
} else if (*p == 0x2E) { // DualNamePrefix
p++;
segments = 2;
} else if (*p == 0x2F) { // MultiNamePrefix
p++;
if (p >= end) return NULL;
segments = *p++;
}
for (uint32_t s = 0; s < segments; s++) {
if (p + 4 > end) return NULL;
if (s == segments - 1 && memcmp(p, "_S5_", 4) == 0)
*lastSegmentIsS5 = 1;
p += 4;
}
return p;
}
static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
const uint8_t* p = *pp;
if (p >= end) return 0;
switch (*p++) {
case 0x00: // ZeroOp
*out = 0;
break;
case 0x01: // OneOp
*out = 1;
break;
case 0x0A: // BytePrefix
if (p >= end) return 0;
*out = *p++;
break;
case 0x0B: // WordPrefix
if (p + 2 > end) return 0;
*out = p[0];
p += 2;
break;
case 0x0C: // DWordPrefix
if (p + 4 > end) return 0;
*out = p[0];
p += 4;
break;
case 0x0E: // QWordPrefix
if (p + 8 > end) return 0;
*out = p[0];
p += 8;
break;
default:
return 0;
}
*pp = p;
return 1;
}
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and
// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants.
static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
const uint8_t* end = aml + len;
for (const uint8_t* cur = aml; cur < end; cur++) {
if (*cur != 0x08) continue; // NameOp
const uint8_t* p = cur + 1;
int isS5 = 0;
p = AmlParseNameString(p, end, &isS5);
if (!p || !isS5 || p >= end || *p != 0x12) continue; // PackageOp
const uint8_t* body;
const uint8_t* pkgEnd;
if (!AmlParsePkgLength(p + 1, end, &body, &pkgEnd)) continue;
if (body >= pkgEnd) continue;
uint8_t elements = *body++;
if (elements < 2) continue;
uint8_t typA, typB;
if (!AmlReadInteger(&body, pkgEnd, &typA)) continue;
if (!AmlReadInteger(&body, pkgEnd, &typB)) continue;
gSlpTypA = typA & 0x07;
gSlpTypB = typB & 0x07;
gS5Found = 1;
HalDbgPrint("HAL: _S5_ SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n",
gSlpTypA, gSlpTypB);
return;
}
HalDbgPrint("HAL: _S5_ not found in DSDT, using defaults (A=0x%02x B=0x%02x)\n",
gSlpTypA, gSlpTypB);
HalDbgPrint("HAL: _S5_ not found in DSDT; ACPI shutdown disabled\n");
}
static int validate_checksum(void* table, uint32_t length) {
@@ -184,19 +296,33 @@ static int validate_checksum(void* table, uint32_t length) {
return sum == 0;
}
static void ParseFadt(const FADT* fadt) {
gPm1aCntBlk = (uint16_t)fadt->Pm1aCntBlk;
gPm1bCntBlk = (uint16_t)fadt->Pm1bCntBlk;
HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%08x\n",
gPm1aCntBlk, gPm1bCntBlk, fadt->Dsdt);
static void ParseFadt(const ACPISDTHeader* header) {
const void* fadt = header;
uint32_t len = header->Length;
if (fadt->Dsdt) {
const ACPISDTHeader* dsdt =
(const ACPISDTHeader*)((uintptr_t)fadt->Dsdt + gPhysBase);
gPm1aCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1A_CNT_BLK_OFFSET);
gPm1bCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1B_CNT_BLK_OFFSET);
if (!gPm1aCntBlk)
gPm1aCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1A_CNT_BLK_OFFSET);
if (!gPm1bCntBlk)
gPm1bCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1B_CNT_BLK_OFFSET);
uint64_t dsdtPhys = AcpiReadU64(fadt, len, FADT_X_DSDT_OFFSET);
if (!dsdtPhys)
dsdtPhys = AcpiReadU32(fadt, len, FADT_DSDT_OFFSET);
HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%016llx\n",
gPm1aCntBlk, gPm1bCntBlk, dsdtPhys);
if (dsdtPhys) {
const ACPISDTHeader* dsdt = (const ACPISDTHeader*)(dsdtPhys + gPhysBase);
if (memcmp(dsdt->Signature, "DSDT", 4) == 0 &&
validate_checksum((void*)dsdt, dsdt->Length)) {
const uint8_t* aml = (const uint8_t*)(dsdt + 1);
ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader));
} else {
HalDbgPrint("HAL: DSDT invalid or checksum failed\n");
}
}
}
@@ -319,7 +445,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
}
// Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer
else if (memcmp(header->Signature, "FACP", 4) == 0) {
ParseFadt((const FADT*)header);
ParseFadt(header);
}
// Parse HPET
else if (memcmp(header->Signature, "HPET", 4) == 0) {
@@ -337,11 +463,17 @@ static void HalShutdown(void) {
// Silence! Stop the world!
_disable();
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
uint16_t valA = (uint16_t)((gSlpTypA << 10) | (1 << 13));
uint16_t valB = (uint16_t)((gSlpTypB << 10) | (1 << 13));
if (!gS5Found || !gPm1aCntBlk) {
// Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets
// put into S3/S4 sleep instead of S5 soft-off.
while (true) __halt();
}
if (gPm1aCntBlk) __outword(gPm1aCntBlk, valA);
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
uint16_t valA = (uint16_t)(((uint16_t)gSlpTypA << 10) | (1 << 13));
uint16_t valB = (uint16_t)(((uint16_t)gSlpTypB << 10) | (1 << 13));
__outword(gPm1aCntBlk, valA);
if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB);
// Die!
code/kernel/include/kernel/io_file.h
+4
View File
@@ -4,6 +4,7 @@
// Kernel and user code open files through IoOpenFile.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <ntlite/status.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
@@ -143,6 +144,9 @@ typedef struct IO_VOLUME_PARAMS {
// ---------------------------------------------------------------------------
// driveIndex matches the index passed to IoRegisterDiskDevice.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol);
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
__declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex);
// ---------------------------------------------------------------------------
// Kernel-side API
code/kernel/include/kernel/ke_cpu.h
+18
View File
@@ -30,6 +30,24 @@ typedef struct KCPU {
uint8_t Online;
uint8_t IsBsp;
// CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded
// from the APIC ID using CPUID leaf 0x1F/0x0B when available, with a
// conservative CPUID(1)/CPUID(4) fallback. PhysicalCoreIndex is a compact
// kernel-local core number used by the scheduler. CoreSiblingMask contains
// all logical CPUs that share this physical core. PackageMask contains all
// logical CPUs in the same package. IsPrimaryThread is the scheduler's
// preferred logical CPU for a physical core, currently the lowest CpuIndex
// in that core.
uint32_t PackageId;
uint32_t CoreId;
uint32_t ThreadId;
uint32_t PhysicalCoreIndex;
uint64_t CoreSiblingMask;
uint64_t PackageMask;
uint8_t IsPrimaryThread;
uint8_t SmtThreadCount;
uint16_t TopologyFlags;
struct THREAD* CurrentThread;
struct THREAD* IdleThread;
struct THREAD* OutgoingThread;
code/kernel/include/kernel/ke_debug.h
+9 -1
View File
@@ -3,10 +3,18 @@
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v);
#define KE_DEBUG_CONSOLE 0x00000001u
#define KE_DEBUG_SELFTEST 0x00000002u
#define KE_DEBUG_FILE_LOG 0x00000004u
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
uint32_t KeGetDebugFlags(void);
__declspec(dllexport) void DbgPrint(const char* fmt, ...);
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len);
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path);
typedef enum KE_PANIC_CODE {
KE_PANIC_CODE_GENERIC = 0,
code/kernel/include/kernel/ke_irp.h
+156 -50
View File
@@ -1,83 +1,157 @@
#pragma once
// ke_irp.h - I/O Request Packet
// ke_irp.h - I/O Request Packet and remove-lock primitives
//
// This header intentionally includes only ke_list.h, ke_spinlock.h, and
// ke_status.h - all of which are pure-inline / macro headers with no
// external symbol references. ke_event.h and ke_semaphore.h are forward-
// declared here so the HAL layer (hal_disk.c, io_driver.h) can include this
// header without dragging in kernel sync-object link dependencies.
//
// Files that actually call KeEventCreate / KeSignalEvent / KeWaitForEvent
// (ke_irp.c, ahci.c) include ke_event.h and ke_semaphore.h themselves.
// IRPs are stack-based. A caller builds an IRP with one or more
// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O
// manager/driver stack advances CurrentLocation as the IRP is forwarded.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <ntlite/status.h>
// Forward declarations - full types live in ke_event.h / ke_semaphore.h.
// We only ever hold pointers in IRP and IRP_QUEUE, so a forward decl suffices.
struct EVENT;
struct SEMAPHORE;
struct KAPC;
typedef struct DEVICE_OBJECT DEVICE_OBJECT;
// -------------------------------------------------------------------------
// IRP major/minor functions
// -------------------------------------------------------------------------
typedef enum IO_MAJOR_FUNCTION {
IRP_MJ_CREATE = 0,
IRP_MJ_CLOSE = 1,
IRP_MJ_READ = 2,
IRP_MJ_WRITE = 3,
IRP_MJ_DEVICE_CONTROL = 4,
IRP_MJ_INTERNAL_DEVICE_CONTROL = 5,
IRP_MJ_PNP = 6,
IRP_MJ_POWER = 7,
IRP_MJ_MAXIMUM_FUNCTION = 8,
} IO_MAJOR_FUNCTION;
typedef enum IO_PNP_MINOR_FUNCTION {
IRP_MN_START_DEVICE = 0,
IRP_MN_QUERY_REMOVE_DEVICE = 1,
IRP_MN_CANCEL_REMOVE_DEVICE = 2,
IRP_MN_SURPRISE_REMOVAL = 3,
IRP_MN_REMOVE_DEVICE = 4,
IRP_MN_QUERY_DEVICE_RELATIONS = 5,
IRP_MN_QUERY_ID = 6,
IRP_MN_QUERY_CAPABILITIES = 7,
} IO_PNP_MINOR_FUNCTION;
typedef enum IO_DEVICE_RELATION_TYPE {
IoBusRelations = 0,
IoRemovalRelations = 1,
} IO_DEVICE_RELATION_TYPE;
// -------------------------------------------------------------------------
// IRP flags
// -------------------------------------------------------------------------
#define IRP_FLAG_WRITE (1u << 0) // direction: write to device
#define IRP_FLAG_PRIORITY (1u << 1) // insert at head of queue (elevated)
#define IRP_FLAG_PRIORITY (1u << 0) // insert at head of a driver queue
#define IRP_FLAG_CANCELLED (1u << 1)
#define IRP_FLAG_COMPLETED (1u << 2)
// -------------------------------------------------------------------------
// Forward declaration (IRP is self-referential via CompletionFn)
// IRP stack parameters
// -------------------------------------------------------------------------
typedef struct IO_STACK_LOCATION {
IO_MAJOR_FUNCTION MajorFunction;
uint8_t MinorFunction;
DEVICE_OBJECT* DeviceObject;
union {
struct {
uint64_t ByteOffset;
uint32_t Length;
} ReadWrite;
struct {
uint32_t IoControlCode;
void* InputBuffer;
uint32_t InputLength;
void* OutputBuffer;
uint32_t OutputLength;
} DeviceIoControl;
struct {
IO_DEVICE_RELATION_TYPE RelationType;
} QueryDeviceRelations;
struct {
uint32_t IdType;
} QueryId;
} Parameters;
} IO_STACK_LOCATION;
// -------------------------------------------------------------------------
// Forward declaration / callbacks
// -------------------------------------------------------------------------
typedef struct IRP IRP;
// Completion callback: invoked by IoCompleteRequest() in the completing
// context. Driver worker, synchronous fallback, DPC, and APC completion paths
// all centralize policy here.
typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx);
typedef void (*IRP_COMPLETION_FN)(IRP* irp, void* ctx);
typedef void (*IRP_CANCEL_ROUTINE)(DEVICE_OBJECT* device, IRP* irp);
// -------------------------------------------------------------------------
// I/O Request Packet
// -------------------------------------------------------------------------
typedef struct IRP {
// Embedded list node - driver-internal; do not touch after submit.
LIST_ENTRY Link;
// ---- Request parameters (caller fills before IoSubmitIrp) ------------
uint8_t DriveIndex;
uint32_t Flags; // IRP_FLAG_*
uint64_t Lba;
uint32_t SectorCount;
void* Buffer; // kernel VA; valid until completion
// ---- Completion (caller fills before submit; driver writes Status) ----
volatile KSTATUS Status;
volatile size_t Information; // bytes transferred on success
volatile size_t Information;
// Forward-declared pointer: callers that create events include ke_event.h.
struct EVENT *CompletionEvent;
// Optional kernel APC. If set, IoCompleteRequest queues it to its
// target thread with arg1=irp and arg2=(void*)(uintptr_t)status.
struct KAPC *CompletionApc;
uint32_t Flags;
volatile bool Cancel;
SPINLOCK CancelLock;
KIRQL CancelIrql;
IRP_CANCEL_ROUTINE CancelRoutine;
struct EVENT* CompletionEvent;
struct KAPC* CompletionApc;
IRP_COMPLETION_FN CompletionFn;
void* CompletionCtx;
// ---- Driver-internal --------------------------------------------------
void* UserBuffer;
void* SystemBuffer;
uint8_t StackCount;
uint8_t CurrentLocation;
// Driver-owned scratch fields. They are deliberately not interpreted by
// the I/O manager and are preserved across queueing/completion.
uint8_t HwSlot;
void* DriverCtx;
IO_STACK_LOCATION Stack[1];
} IRP;
// -------------------------------------------------------------------------
// Remove lock
// -------------------------------------------------------------------------
typedef struct IO_REMOVE_LOCK {
SPINLOCK Lock;
volatile long IoCount;
bool Removing;
struct EVENT* RemovedEvent;
} IO_REMOVE_LOCK;
__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock);
__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock);
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp);
// -------------------------------------------------------------------------
// IRP_QUEUE (per-device, driver-internal)
// -------------------------------------------------------------------------
@@ -85,24 +159,56 @@ typedef struct IRP {
typedef struct IRP_QUEUE {
SPINLOCK Lock;
LIST_ENTRY Head;
struct SEMAPHORE *Semaphore; // forward-declared pointer
struct SEMAPHORE* Semaphore;
} IRP_QUEUE;
// Non-inline: defined in ke_irp.c, which can pull in ke_semaphore.h safely.
__declspec(dllexport) void IrpQueueInit (IRP_QUEUE *q, struct SEMAPHORE *sem);
__declspec(dllexport) void IrpQueueEnqueue (IRP_QUEUE *q, IRP *irp);
__declspec(dllexport) IRP *IrpQueueDequeue (IRP_QUEUE *q);
__declspec(dllexport) void IrpQueueInit (IRP_QUEUE* q, struct SEMAPHORE* sem);
__declspec(dllexport) void IrpQueueEnqueue (IRP_QUEUE* q, IRP* irp);
__declspec(dllexport) IRP* IrpQueueDequeue (IRP_QUEUE* q);
typedef void (*IRP_QUEUE_CANCEL_CALLBACK)(IRP* irp, void* context);
__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context);
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status);
// -------------------------------------------------------------------------
// Public dispatch API - implemented in kernel/ke_irp.c
// IRP allocation / stack helpers
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp);
__declspec(dllexport) void IoFreeIrp(IRP* irp);
__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount);
__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp);
__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp);
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp);
__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp);
__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx);
__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine);
__declspec(dllexport) void IoCancelIrp(IRP* irp);
__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint64_t byteOffset,
uint32_t length,
void* buffer);
__declspec(dllexport) void IoBuildPnpIrp(IRP* irp,
IO_PNP_MINOR_FUNCTION minor);
__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint32_t ioControlCode,
void* inputBuffer,
uint32_t inputLength,
void* outputBuffer,
uint32_t outputLength);
// Complete an IRP exactly once: publish status/information, signal the
// optional completion event, then invoke the optional completion callback.
__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information);
// optional completion event, queue the optional APC, then invoke the optional
// completion callback.
__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information);
// Submit asynchronously. Returns STATUS_PENDING on success.
__declspec(dllexport) KSTATUS IoSubmitIrp (IRP *irp);
// Submit and block until complete. Returns irp->Status.
__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp);
// Submit to a specific top-of-stack device object. The current stack location
// is stamped with the target device before dispatch.
__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp);
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp);
code/kernel/include/kernel/ke_sched.h
+2
View File
@@ -23,6 +23,8 @@ typedef struct KE_SCHED_STATS {
uint64_t PriorityPreemptions;
uint64_t VoluntaryBlocks;
uint64_t CpuBurstDemotions;
uint64_t SmtSiblingDeferrals;
uint64_t SmtPrimaryPokes;
} KE_SCHED_STATS;
// Called once by KernelMain before any threads are created.
code/kernel/include/kernel/ke_smp.h
+6
View File
@@ -13,6 +13,12 @@ KSTATUS KeStartApplicationProcessors(void);
// park loop into their idle threads.
KSTATUS KeAdmitApplicationProcessorsToScheduler(void);
// Decode CPUID topology and build per-KCPU package/core/SMT sibling masks.
// KeStartApplicationProcessors calls this after MADT enumeration; it is also
// safe to call again after failed AP startup disables a CPU record.
void KeBuildProcessorTopology(void);
// Run explicit AP-affine kernel workers that exercise the global scheduler,
// reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver
// threads remain BSP-affine unless their affinity is explicitly widened.
code/kernel/include/kernel/ob_object.h
+4
View File
@@ -17,6 +17,10 @@ typedef enum OBJECT_TYPE {
ObjectTypeDir = 8,
ObjectTypeAsyncIo = 9,
ObjectTypeSocket = 10,
ObjectTypeDevice = 11,
ObjectTypeDriver = 12,
ObjectTypeUsbDevice = 13,
ObjectTypeUsbPipe = 14,
} OBJECT_TYPE;
typedef struct OBJECT {
code/kernel/io_device.c
+559 -78
View File
@@ -1,13 +1,423 @@
// io_device.c - kernel driver/device dispatch and disk registry
// io_device.c - I/O manager: driver/device objects, stacks, PnP skeleton, disk namespace
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_irp.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
#define MAX_PUBLISHED_DEVICES 64
#define MAX_DRIVER_MATCHERS 32
#define MAX_DRIVER_MATCHERS 32
#define MAX_DRIVER_OBJECTS 64
#define MAX_PDO_OBJECTS 128
static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) {
return SysSvIsEmpty(name) ? SYS_SV_LITERAL("<unnamed>") : name;
}
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
switch (busType) {
case IoBusPci: return "pci";
case IoBusUsb: return "usb";
default: return "unknown";
}
}
// -------------------------------------------------------------------------
// Driver object registry / PDO AddDevice binding
// -------------------------------------------------------------------------
static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS];
static uint32_t gDriverObjectCount;
static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS];
static uint32_t gPhysicalDeviceCount;
static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical)
return false;
for (size_t d = 0; d < driver->IdTable->Count; d++) {
const IO_DEVICE_ID* want = &driver->IdTable->Ids[d];
if (want->BusType != IoBusUnknown && want->BusType != pdo->u.Pdo.BusType)
continue;
for (uint8_t p = 0; p < pdo->u.Pdo.IdCount; p++) {
if (SysSvEqual(want->Id, pdo->u.Pdo.Ids[p].Id))
return true;
}
}
return false;
}
static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) {
if (!driver || !pdo || !driver->DriverAddDevice)
return false;
if (!IoDriverMatchesPdo(driver, pdo))
return false;
KSTATUS st = driver->DriverAddDevice(driver, pdo);
if (KSUCCESS(st)) {
DbgPrint("io: AddDevice driver=%v pdo=%v bus=%s\n",
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), IoBusTypeName(pdo->u.Pdo.BusType));
KSTATUS start = IoSendPnpIrpSync(pdo, IRP_MN_START_DEVICE);
if (!KSUCCESS(start) && start != STATUS_NOT_SUPPORTED) {
DbgPrint("io: START_DEVICE failed driver=%v pdo=%v st=0x%x\n",
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), start);
}
return true;
} else {
DbgPrint("io: AddDevice failed driver=%v pdo=%v st=0x%x\n",
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), st);
return true;
}
}
__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
SYS_STRING_VIEW name) {
if (!driver) return;
memset(driver, 0, sizeof(*driver));
driver->Header.Type = ObjectTypeDriver;
driver->Header.RefCount = 1;
driver->Header.Destroy = (void*)0;
driver->Name = name;
}
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
if (!driver) return STATUS_INVALID_PARAMETER;
if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
return STATUS_NO_MEMORY;
if (driver->Header.Type != ObjectTypeDriver) {
driver->Header.Type = ObjectTypeDriver;
driver->Header.RefCount = 1;
}
gDriverObjects[gDriverObjectCount++] = driver;
DbgPrint("io: registered driver object %v\n", IoSvOrUnnamed(driver->Name));
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++)
(void)IoTryAddDeviceForDriver(driver, gPhysicalDevices[i]);
return STATUS_SUCCESS;
}
// -------------------------------------------------------------------------
// Device object lifetime
// -------------------------------------------------------------------------
static void IoDeviceDestroy(OBJECT* object) {
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == dev) {
gPhysicalDevices[i] = gPhysicalDevices[gPhysicalDeviceCount - 1u];
gPhysicalDevices[gPhysicalDeviceCount - 1u] = (DEVICE_OBJECT*)0;
gPhysicalDeviceCount--;
break;
}
}
if (dev->LowerDevice)
IoDetachDevice(dev);
if (dev->Parent) {
DEVICE_OBJECT* parent = dev->Parent;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
DEVICE_OBJECT** link = &parent->FirstChild;
while (*link) {
if (*link == dev) {
*link = dev->NextSibling;
break;
}
link = &(*link)->NextSibling;
}
KeReleaseSpinlock(&parent->Lock, flags);
dev->Parent = (DEVICE_OBJECT*)0;
dev->NextSibling = (DEVICE_OBJECT*)0;
IoDereferenceDeviceObject(parent);
}
}
static KSTATUS IoCreateDeviceInternal(DRIVER_OBJECT* driver,
DEVICE_OBJECT_TYPE type,
size_t extensionSize,
DEVICE_OBJECT** outDevice) {
if (!driver || !outDevice) return STATUS_INVALID_PARAMETER;
size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)ObAllocateObject(ObjectTypeDevice, total);
if (!dev) return STATUS_NO_MEMORY;
dev->Header.Destroy = IoDeviceDestroy;
dev->DriverObject = driver;
dev->Type = type;
dev->PnpState = DevicePnpNotStarted;
dev->AlignmentRequirement = 1;
dev->Lock = (SPINLOCK)SPINLOCK_INIT;
// The per-device union is interpreted differently for PDOs and disk FDOs.
// Do not write disk defaults into a PDO: u.Disk.DiskIndex aliases the low
// byte of u.Pdo.BusType, which prevents ID-table AddDevice matching.
if (type != DeviceObjectPhysical) {
dev->u.Disk.DiskIndex = 0xFF;
dev->u.Disk.SectorSize = 512;
}
if (extensionSize)
dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));
*outDevice = dev;
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice) {
return IoCreateDeviceInternal(driver, DeviceObjectFunction, extensionSize, outDevice);
}
__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
DEVICE_OBJECT* parent,
const IO_DEVICE_ID* ids,
size_t idCount,
size_t extensionSize,
DEVICE_OBJECT** outPdo) {
if (!busDriver || !outPdo) return STATUS_INVALID_PARAMETER;
if (idCount > IO_MAX_DEVICE_IDS) return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDeviceInternal(busDriver, DeviceObjectPhysical, extensionSize, &pdo);
if (!KSUCCESS(st)) return st;
if (idCount && ids) {
for (size_t i = 0; i < idCount; i++) {
if (!SysSvIsValid(ids[i].Id) || ids[i].Id.Length >= IO_MAX_DEVICE_ID_CHARS) {
IoDeleteDevice(pdo);
return STATUS_INVALID_PARAMETER;
}
char* dst = pdo->u.Pdo.IdStorage[i];
if (ids[i].Id.Length)
memcpy(dst, ids[i].Id.Data, ids[i].Id.Length);
dst[ids[i].Id.Length] = '\0';
pdo->u.Pdo.Ids[i].BusType = ids[i].BusType;
pdo->u.Pdo.Ids[i].Id = SysSvFromParts(dst, ids[i].Id.Length);
if (pdo->u.Pdo.BusType == IoBusUnknown)
pdo->u.Pdo.BusType = ids[i].BusType;
}
pdo->u.Pdo.IdCount = (uint8_t)idCount;
}
if (parent) {
IoReferenceDeviceObject(parent);
pdo->Parent = parent;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
pdo->NextSibling = parent->FirstChild;
parent->FirstChild = pdo;
KeReleaseSpinlock(&parent->Lock, flags);
}
*outPdo = pdo;
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
if (!pdo || pdo->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == pdo)
return STATUS_ALREADY_EXISTS;
}
if (gPhysicalDeviceCount >= MAX_PDO_OBJECTS)
return STATUS_NO_MEMORY;
gPhysicalDevices[gPhysicalDeviceCount++] = pdo;
bool anyMatched = false;
for (uint32_t i = 0; i < gDriverObjectCount; i++) {
if (IoTryAddDeviceForDriver(gDriverObjects[i], pdo))
anyMatched = true;
}
if (!anyMatched && pdo->u.Pdo.BusType == IoBusUsb) {
SYS_STRING_VIEW id = pdo->u.Pdo.IdCount ? pdo->u.Pdo.Ids[0].Id : SYS_SV_LITERAL("<no-id>");
DbgPrint("io: no function driver for pdo=%v id=%v bus=%s\n",
IoSvOrUnnamed(pdo->Name), id, IoBusTypeName(pdo->u.Pdo.BusType));
}
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
SYS_STRING_VIEW name) {
if (!device || !SysSvIsValid(name))
return STATUS_INVALID_PARAMETER;
if (name.Length >= IO_MAX_DEVICE_NAME_CHARS)
return STATUS_BUFFER_TOO_SMALL;
if (name.Length)
memcpy(device->NameStorage, name.Data, name.Length);
device->NameStorage[name.Length] = '\0';
device->Name = SysSvFromParts(device->NameStorage, name.Length);
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObReferenceObject(&device->Header);
}
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObDereferenceObject(&device->Header);
}
static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
if (!device || device->Type != DeviceObjectPhysical)
return;
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == device) {
for (uint32_t j = i + 1; j < gPhysicalDeviceCount; j++)
gPhysicalDevices[j - 1] = gPhysicalDevices[j];
gPhysicalDeviceCount--;
if (gPhysicalDeviceCount < MAX_PDO_OBJECTS)
gPhysicalDevices[gPhysicalDeviceCount] = (DEVICE_OBJECT*)0;
return;
}
}
}
static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
if (!device || !device->Parent)
return;
DEVICE_OBJECT* parent = device->Parent;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
DEVICE_OBJECT** link = &parent->FirstChild;
while (*link) {
if (*link == device) {
*link = device->NextSibling;
break;
}
link = &(*link)->NextSibling;
}
device->Parent = (DEVICE_OBJECT*)0;
device->NextSibling = (DEVICE_OBJECT*)0;
KeReleaseSpinlock(&parent->Lock, flags);
IoDereferenceDeviceObject(parent);
}
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
if (device->DeletePending)
return;
device->DeletePending = true;
device->PnpState = DevicePnpDeleted;
IoUnlinkDeviceFromParent(device);
IoRemovePhysicalDeviceRegistryEntry(device);
IoDereferenceDeviceObject(device);
}
// -------------------------------------------------------------------------
// Device stack management
// -------------------------------------------------------------------------
__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
DEVICE_OBJECT* target) {
if (!source || !target) return (DEVICE_OBJECT*)0;
DEVICE_OBJECT* top = target;
while (top->AttachedDevice)
top = top->AttachedDevice;
top->AttachedDevice = source;
source->LowerDevice = top;
IoReferenceDeviceObject(top);
return top;
}
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
if (!source || !source->LowerDevice) return;
DEVICE_OBJECT* lower = source->LowerDevice;
if (lower->AttachedDevice == source)
lower->AttachedDevice = (DEVICE_OBJECT*)0;
source->LowerDevice = (DEVICE_OBJECT*)0;
IoDereferenceDeviceObject(lower);
}
__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device) {
if (!device) return (DEVICE_OBJECT*)0;
DEVICE_OBJECT* top = device;
while (top->AttachedDevice)
top = top->AttachedDevice;
IoReferenceDeviceObject(top);
return top;
}
// -------------------------------------------------------------------------
// Device-stack helpers / PnP dispatch
// -------------------------------------------------------------------------
__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
uint8_t count = 0;
for (DEVICE_OBJECT* cur = device; cur && count < 0xFEu; cur = cur->LowerDevice)
count++;
return count ? count : 1;
}
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->LowerDevice)
return STATUS_NOT_SUPPORTED;
IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
if (!cur || !next)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* currentDevice = cur->DeviceObject;
memcpy(next, cur, sizeof(*next));
if (currentDevice == device) {
cur->DeviceObject = (DEVICE_OBJECT*)0;
_InterlockedDecrement(&device->OutstandingIrps);
IoDereferenceDeviceObject(device);
}
irp->CurrentLocation++;
return IoCallDriver(device->LowerDevice, irp);
}
__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
IO_PNP_MINOR_FUNCTION minor) {
if (!device) return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* top = IoGetAttachedDeviceReference(device);
if (!top) return STATUS_INVALID_PARAMETER;
uint8_t stackCount = IoGetDeviceStackSize(top);
IRP* irp = (IRP*)0;
KSTATUS st = IoAllocateIrp(stackCount, &irp);
if (!KSUCCESS(st)) {
IoDereferenceDeviceObject(top);
return st;
}
IoBuildPnpIrp(irp, minor);
st = IoSubmitIrpSync(top, irp);
IoFreeIrp(irp);
IoDereferenceDeviceObject(top);
return st;
}
// -------------------------------------------------------------------------
// Legacy publish/matcher path
// -------------------------------------------------------------------------
typedef struct IO_DRIVER_MATCHER_ENTRY {
bool Used;
@@ -23,14 +433,6 @@ static uint32_t gPublishedDeviceCount;
static IO_DRIVER_MATCHER_ENTRY gDriverMatchers[MAX_DRIVER_MATCHERS];
static uint32_t gDriverMatcherCount;
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
switch (busType) {
case IoBusPci: return "pci";
case IoBusUsb: return "usb";
default: return "unknown";
}
}
static bool IoTryBindDevice(IO_PUBLISHED_DEVICE* device,
IO_DRIVER_MATCHER_ENTRY* matcher) {
if (!device || !matcher || !matcher->Used || device->Bound)
@@ -80,12 +482,10 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
entry->Bind = bind;
entry->Context = context;
DbgPrint("io: registered %s matcher %s\n",
IoBusTypeName(busType), driverName);
DbgPrint("io: registered %s matcher %s\n", IoBusTypeName(busType), driverName);
for (uint32_t i = 0; i < gPublishedDeviceCount; i++) {
for (uint32_t i = 0; i < gPublishedDeviceCount; i++)
IoTryBindDevice(&gPublishedDevices[i], entry);
}
return STATUS_SUCCESS;
}
@@ -101,7 +501,6 @@ __declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device)
stored->Bound = false;
stored->BoundDriverName = (const char*)0;
for (uint32_t i = 0; i < gDriverMatcherCount; i++) {
if (IoTryBindDevice(stored, &gDriverMatchers[i]))
break;
@@ -114,50 +513,44 @@ __declspec(dllexport) uint32_t IoPublishedDeviceCount(void) {
return gPublishedDeviceCount;
}
#define MAX_DISK_DEVICES 8
#define MAX_DISK_ARRIVAL_CALLBACKS 16
// -------------------------------------------------------------------------
// Disk namespace
// -------------------------------------------------------------------------
typedef struct IO_DISK_CALLBACK_ENTRY {
#define MAX_DISK_DEVICES 8
#define MAX_DISK_CALLBACKS 16
typedef struct IO_DISK_CHANGE_CALLBACK_ENTRY {
bool Used;
IO_DISK_CHANGE_CALLBACK Callback;
void* Context;
} IO_DISK_CHANGE_CALLBACK_ENTRY;
typedef struct IO_DISK_ARRIVAL_CALLBACK_ENTRY {
bool Used;
IO_DISK_ARRIVAL_CALLBACK Callback;
void* Context;
} IO_DISK_CALLBACK_ENTRY;
} IO_DISK_ARRIVAL_CALLBACK_ENTRY;
static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES];
static IO_DISK_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_ARRIVAL_CALLBACKS];
static IO_DISK_CHANGE_CALLBACK_ENTRY gDiskChangeCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
static void IoNotifyDiskArrived(uint8_t driveIndex, DEVICE_OBJECT* device) {
for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i];
if (entry->Used && entry->Callback)
entry->Callback(driveIndex, device, entry->Context);
entry->Callback(driveIndex, device, change, entry->Context);
}
if (change == IoDeviceArrived) {
for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
if (entry->Used && entry->Callback)
entry->Callback(driveIndex, device, entry->Context);
}
}
}
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice) {
if (!driver || !outDevice) return STATUS_INVALID_PARAMETER;
size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)MmAllocatePool(total);
if (!dev) return STATUS_NO_MEMORY;
memset(dev, 0, total);
dev->DriverObject = driver;
if (extensionSize)
dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));
dev->AlignmentRequirement = 1;
dev->DiskIndex = 0xFF;
*outDevice = dev;
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
MmFreePool(device);
}
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
@@ -168,9 +561,57 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
return STATUS_ALREADY_EXISTS;
device->Flags |= DEVICE_OBJECT_DISK;
device->DiskIndex = driveIndex;
device->u.Disk.DiskIndex = driveIndex;
if (device->u.Disk.SectorSize == 0)
device->u.Disk.SectorSize = 512;
KSTATUS pnpStatus = IoSendPnpIrpSync(device, IRP_MN_START_DEVICE);
if (!KSUCCESS(pnpStatus) && pnpStatus != STATUS_NOT_SUPPORTED)
return pnpStatus;
if (pnpStatus == STATUS_NOT_SUPPORTED)
device->PnpState = DevicePnpStarted;
IoReferenceDeviceObject(device);
gDiskDevices[driveIndex] = device;
IoNotifyDiskArrived(driveIndex, device);
IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return STATUS_NOT_FOUND;
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return STATUS_NOT_FOUND;
if (dev->PnpState != DevicePnpSurpriseRemoved) {
IoNotifyDiskChange(driveIndex, dev, IoDeviceQueryRemove);
KSTATUS queryStatus = IoSendPnpIrpSync(dev, IRP_MN_QUERY_REMOVE_DEVICE);
if (!KSUCCESS(queryStatus) && queryStatus != STATUS_NOT_SUPPORTED)
return queryStatus;
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
}
dev->PnpState = DevicePnpRemovePending;
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
}
@@ -178,10 +619,10 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL
void* context) {
if (!callback)
return STATUS_INVALID_PARAMETER;
if (gDiskArrivalCallbackCount >= MAX_DISK_ARRIVAL_CALLBACKS)
if (gDiskArrivalCallbackCount >= MAX_DISK_CALLBACKS)
return STATUS_NO_MEMORY;
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
memset(entry, 0, sizeof(*entry));
entry->Used = true;
entry->Callback = callback;
@@ -191,7 +632,21 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL
if (gDiskDevices[i])
callback(i, gDiskDevices[i], context);
}
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback,
void* context) {
if (!callback)
return STATUS_INVALID_PARAMETER;
if (gDiskChangeCallbackCount >= MAX_DISK_CALLBACKS)
return STATUS_NO_MEMORY;
IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[gDiskChangeCallbackCount++];
memset(entry, 0, sizeof(*entry));
entry->Used = true;
entry->Callback = callback;
entry->Context = context;
return STATUS_SUCCESS;
}
@@ -204,7 +659,6 @@ __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK ca
if (gDiskDevices[i])
callback(i, gDiskDevices[i], context);
}
return STATUS_SUCCESS;
}
@@ -213,28 +667,59 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
return gDiskDevices[driveIndex];
}
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
if (dev) IoReferenceDeviceObject(dev);
return dev;
}
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
}
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
return dev ? dev->SectorCount : 0;
return dev ? dev->u.Disk.SectorCount : 0;
}
// -------------------------------------------------------------------------
// Dispatch
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DriverObject || !irp)
return STATUS_INVALID_PARAMETER;
IO_MAJOR_FUNCTION major = (irp->Flags & IRP_FLAG_WRITE) ? IRP_MJ_WRITE : IRP_MJ_READ;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios)
return STATUS_INVALID_PARAMETER;
IO_MAJOR_FUNCTION major = ios->MajorFunction;
if (major >= IRP_MJ_MAXIMUM_FUNCTION)
return STATUS_INVALID_PARAMETER;
if (major != IRP_MJ_PNP &&
(device->DeletePending || device->PnpState == DevicePnpSurpriseRemoved ||
device->PnpState == DevicePnpRemovePending || device->PnpState == DevicePnpDeleted))
return STATUS_OBJECT_DEAD;
PDRIVER_DISPATCH dispatch = device->DriverObject->MajorFunction[major];
if (!dispatch)
return STATUS_NOT_SUPPORTED;
return dispatch(device, irp);
ios->DeviceObject = device;
IoReferenceDeviceObject(device);
_InterlockedIncrement(&device->OutstandingIrps);
KSTATUS st = dispatch(device, irp);
if (st != STATUS_PENDING) {
if (ios->DeviceObject == device) {
ios->DeviceObject = (DEVICE_OBJECT*)0;
_InterlockedDecrement(&device->OutstandingIrps);
IoDereferenceDeviceObject(device);
}
}
return st;
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
@@ -243,17 +728,15 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
void* buffer) {
if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
memset(&irp, 0, sizeof(irp));
irp.DriveIndex = driveIndex;
irp.Flags = 0;
irp.Lba = lba;
irp.SectorCount = count;
irp.Buffer = buffer;
irp.Status = STATUS_FAILURE;
irp.Information = 0;
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
return IoSubmitIrpSync(&irp);
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
IoDereferenceDeviceObject(dev);
return st;
}
__declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
@@ -262,15 +745,13 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
const void* buffer) {
if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
memset(&irp, 0, sizeof(irp));
irp.DriveIndex = driveIndex;
irp.Flags = IRP_FLAG_WRITE;
irp.Lba = lba;
irp.SectorCount = count;
irp.Buffer = (void*)buffer;
irp.Status = STATUS_FAILURE;
irp.Information = 0;
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
return IoSubmitIrpSync(&irp);
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
IoDereferenceDeviceObject(dev);
return st;
}
code/kernel/io_file.c
+27
View File
@@ -586,6 +586,33 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return STATUS_INVALID_PARAMETER;
if (!gVolumes[volumeIndex].Fs)
return STATUS_NOT_FOUND;
memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex]));
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
KSTATUS last = STATUS_NOT_FOUND;
for (uint8_t i = 0; i < MAX_DRIVES; i++) {
if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) {
memset(&gVolumes[i], 0, sizeof(gVolumes[i]));
last = STATUS_SUCCESS;
}
}
return last;
}
__declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return false;
return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
}
// ---------------------------------------------------------------------------
// Destructor
// ---------------------------------------------------------------------------
code/kernel/ke_cpu.c
+9
View File
@@ -48,6 +48,15 @@ void KeInitializeBootstrapCpu(void) {
cpu->Parked = 0;
cpu->Online = 1;
cpu->IsBsp = 1;
cpu->PackageId = 0;
cpu->CoreId = 0;
cpu->ThreadId = 0;
cpu->PhysicalCoreIndex = 0;
cpu->CoreSiblingMask = 1ULL;
cpu->PackageMask = 1ULL;
cpu->IsPrimaryThread = 1;
cpu->SmtThreadCount = 1;
cpu->TopologyFlags = 0;
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
code/kernel/ke_debug.c
+124 -7
View File
@@ -13,12 +13,53 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/mm_virt.h>
#include <kernel/io_file.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor
int gDebugEnabled = 0;
static uint32_t gDebugFlags = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile long gDebugLogFlushing;
static void KiDebugLogAppendLocked(const char* text, size_t len) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
return;
if (len >= gDebugLogSize) {
text += len - (gDebugLogSize - 1u);
len = gDebugLogSize - 1u;
}
size_t first = gDebugLogSize - gDebugLogWrite;
if (first > len) first = len;
memcpy(gDebugLogBuffer + gDebugLogWrite, text, first);
if (len > first)
memcpy(gDebugLogBuffer, text + first, len - first);
gDebugLogWrite = (gDebugLogWrite + len) % gDebugLogSize;
if (gDebugLogUsed + len >= gDebugLogSize)
gDebugLogUsed = gDebugLogSize;
else
gDebugLogUsed += len;
}
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
return;
for (size_t i = 0; i < len; i++)
PrintChar(text[i], 0x00FF00, 0x000000);
}
static volatile long gPanicInProgress = 0;
static volatile long gPanicStoppedCpuCount = 0;
@@ -50,22 +91,98 @@ void KiKernelPutChar(char c) {
PrintChar(c, 0x00FF00, 0x000000);
}
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) {
gDebugEnabled = v;
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
gDebugFlags = flags;
gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE);
if (gDebugLogBuffer) {
gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
gDebugLogWrite = 0;
gDebugLogUsed = 0;
memset(gDebugLogBuffer, 0, gDebugLogSize);
}
}
}
uint32_t KeGetDebugFlags(void) {
return gDebugFlags;
}
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled) return;
if (!gDebugEnabled || !fmt) return;
static SPINLOCK sl = {0};
uintptr_t flags = KeAcquireSpinlock(&sl);
char line[KE_DEBUG_LOG_LINE_SIZE];
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
int n = vsnprintf_(line, sizeof(line), fmt, va);
va_end(va);
KeReleaseSpinlock(&sl, flags);
size_t len = 0;
if (n > 0) {
len = (size_t)n;
if (len >= sizeof(line))
len = sizeof(line) - 1u;
}
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE)
KiDebugConsoleWrite(line, len);
KeReleaseSpinlock(&gDebugPrintLock, flags);
}
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
if (!KeCanWait())
return STATUS_INVALID_IRQL;
if (_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 1, 0) != 0)
return STATUS_PENDING;
size_t used;
size_t start;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
used = gDebugLogUsed;
start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (used == 0) {
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return STATUS_SUCCESS;
}
(void)IoDeleteFile(path);
FILE_OBJECT* file = (FILE_OBJECT*)0;
KSTATUS st = IoCreateFile(path, &file);
if (!KSUCCESS(st)) {
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return st;
}
size_t written = 0;
size_t totalWritten = 0;
size_t first = gDebugLogSize - start;
if (first > used) first = used;
st = IoWriteFile(file, gDebugLogBuffer + start, first, &written);
totalWritten += written;
if (KSUCCESS(st) && used > first) {
written = 0;
st = IoWriteFile(file, gDebugLogBuffer, used - first, &written);
totalWritten += written;
}
IoCloseFile(file);
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// KePanic
code/kernel/ke_irp.c
+259 -68
View File
@@ -1,130 +1,321 @@
// ke_irp.c - I/O Request Packet dispatch and queue helpers
//
// This file lives in the kernel layer and is the ONLY place that bridges the
// IRP abstraction to kernel sync objects. All calls to KeEventCreate,
// KeSignalEvent, KeWaitForEvent, KeSignalSemaphore, and ObDereferenceObject
// are confined here; hal_disk.c and io_driver.h remain free of those symbols.
//
// Dependency graph (→ = includes / calls):
// ke_irp.c → ke_irp.h (IRP type, IRP_QUEUE, forward decls)
// → ke_event.h (KeEventCreate, KeSignalEvent, KeWaitForEvent)
// → ke_semaphore.h (KeSignalSemaphore)
// → ob_object.h (ObDereferenceObject)
// → io_driver.h (DEVICE_OBJECT / IoCallDriver)
// ke_irp.c - I/O Request Packet dispatch, queue, cancellation, remove-locks
#include <string.h>
#include <intrin.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_event.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_apc.h>
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
#include <ntlite/ddi/io_driver.h>
// -------------------------------------------------------------------------
// IRP_QUEUE operations
// -------------------------------------------------------------------------
// These are defined here (not as static inlines in ke_irp.h) so that
// ke_irp.h can avoid including ke_semaphore.h, keeping the header safe
// for inclusion from the HAL layer.
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE *q, struct SEMAPHORE *sem) {
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
q->Lock = (SPINLOCK)SPINLOCK_INIT;
q->Semaphore = sem;
LIST_INIT(&q->Head);
}
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE *q, IRP *irp) {
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
if (irp->Flags & IRP_FLAG_PRIORITY)
ListInsertHead(&q->Head, &irp->Link);
else
ListInsertTail(&q->Head, &irp->Link);
KeReleaseSpinlock(&q->Lock, flags);
// Signal outside the spinlock - safe, and avoids priority-inversion risk.
KeSignalSemaphore((SEMAPHORE *)q->Semaphore);
KeSignalSemaphore((SEMAPHORE*)q->Semaphore);
}
__declspec(dllexport) IRP *IrpQueueDequeue(IRP_QUEUE *q) {
__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY *node = ListPopHead(&q->Head);
LIST_ENTRY* node = ListPopHead(&q->Head);
KeReleaseSpinlock(&q->Lock, flags);
if (!node) return (IRP *)0;
if (!node) return (IRP*)0;
return CONTAINING_RECORD(node, IRP, Link);
}
// -------------------------------------------------------------------------
// IoCompleteRequest
// -------------------------------------------------------------------------
__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context) {
if (!q) return;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY* node = ListPopHead(&q->Head);
KeReleaseSpinlock(&q->Lock, flags);
if (!node) break;
static size_t IoIrpTransferBytes(const IRP *irp, KSTATUS status) {
if (!irp || !KSUCCESS(status)) return 0;
return (size_t)irp->SectorCount * 512u;
IRP* irp = CONTAINING_RECORD(node, IRP, Link);
irp->Cancel = true;
irp->Flags |= IRP_FLAG_CANCELLED;
if (callback)
callback(irp, context);
IoCompleteRequest(irp, status, 0);
}
}
__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information) {
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
}
// -------------------------------------------------------------------------
// Remove locks
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return STATUS_INVALID_PARAMETER;
memset(lock, 0, sizeof(*lock));
lock->Lock = (SPINLOCK)SPINLOCK_INIT;
lock->IoCount = 1; // sentinel reference released by IoReleaseRemoveLockAndWait
KSTATUS st = KeEventCreate(EventManualReset, 0, &lock->RemovedEvent);
if (!KSUCCESS(st)) return st;
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return;
if (lock->RemovedEvent) {
ObDereferenceObject(&lock->RemovedEvent->Header);
lock->RemovedEvent = (EVENT*)0;
}
}
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
if (lock->Removing) {
KeReleaseSpinlock(&lock->Lock, flags);
return STATUS_OBJECT_DEAD;
}
_InterlockedIncrement(&lock->IoCount);
KeReleaseSpinlock(&lock->Lock, flags);
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return;
long left = _InterlockedDecrement(&lock->IoCount);
if (left == 0 && lock->Removing && lock->RemovedEvent)
KeSignalEvent(lock->RemovedEvent);
}
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp) {
(void)removeIrp;
if (!lock) return;
uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
lock->Removing = true;
long left = _InterlockedDecrement(&lock->IoCount); // release sentinel
bool done = (left == 0);
KeReleaseSpinlock(&lock->Lock, flags);
if (done) {
if (lock->RemovedEvent) KeSignalEvent(lock->RemovedEvent);
return;
}
if (lock->RemovedEvent)
KeWaitForEvent(lock->RemovedEvent, 0);
}
// -------------------------------------------------------------------------
// IRP helpers
// -------------------------------------------------------------------------
static size_t IoIrpSize(uint8_t stackCount) {
if (stackCount == 0) stackCount = 1;
return sizeof(IRP) + ((size_t)stackCount - 1u) * sizeof(IO_STACK_LOCATION);
}
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
if (!outIrp) return STATUS_INVALID_PARAMETER;
if (stackCount == 0) stackCount = 1;
IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount));
if (!irp) return STATUS_NO_MEMORY;
IoInitializeIrp(irp, stackCount);
*outIrp = irp;
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoFreeIrp(IRP* irp) {
if (irp) MmFreePool(irp);
}
__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
if (!irp) return;
if (stackCount == 0) stackCount = 1;
memset(irp, 0, IoIrpSize(stackCount));
LIST_INIT(&irp->Link);
irp->Status = STATUS_FAILURE;
irp->Information = 0;
irp->CancelLock = (SPINLOCK)SPINLOCK_INIT;
irp->StackCount = stackCount;
irp->CurrentLocation = 0;
}
__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation];
}
__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation + 1u >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation + 1u];
}
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) {
if (!irp) return;
if (irp->CurrentLocation + 1u < irp->StackCount)
irp->CurrentLocation++;
}
__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
if (!cur || !next) return;
memcpy(next, cur, sizeof(*next));
irp->CurrentLocation++;
}
__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx) {
if (!irp) return;
irp->CompletionFn = fn;
irp->CompletionCtx = ctx;
}
__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine) {
if (!irp) return (IRP_CANCEL_ROUTINE)0;
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
IRP_CANCEL_ROUTINE old = irp->CancelRoutine;
irp->CancelRoutine = routine;
KeReleaseSpinlock(&irp->CancelLock, flags);
return old;
}
__declspec(dllexport) void IoCancelIrp(IRP* irp) {
if (!irp) return;
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
irp->Cancel = true;
irp->Flags |= IRP_FLAG_CANCELLED;
IRP_CANCEL_ROUTINE routine = irp->CancelRoutine;
irp->CancelRoutine = (IRP_CANCEL_ROUTINE)0;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
KeReleaseSpinlock(&irp->CancelLock, flags);
if (routine)
routine(dev, irp);
}
__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint64_t byteOffset,
uint32_t length,
void* buffer) {
if (!irp) return;
IoInitializeIrp(irp, 1);
irp->UserBuffer = buffer;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return;
ios->MajorFunction = major;
ios->MinorFunction = 0;
ios->Parameters.ReadWrite.ByteOffset = byteOffset;
ios->Parameters.ReadWrite.Length = length;
}
__declspec(dllexport) void IoBuildPnpIrp(IRP* irp,
IO_PNP_MINOR_FUNCTION minor) {
if (!irp) return;
uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
IoInitializeIrp(irp, stackCount);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return;
ios->MajorFunction = IRP_MJ_PNP;
ios->MinorFunction = (uint8_t)minor;
}
__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint32_t ioControlCode,
void* inputBuffer,
uint32_t inputLength,
void* outputBuffer,
uint32_t outputLength) {
if (!irp) return;
uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
IoInitializeIrp(irp, stackCount);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return;
ios->MajorFunction = major;
ios->MinorFunction = 0;
ios->Parameters.DeviceIoControl.IoControlCode = ioControlCode;
ios->Parameters.DeviceIoControl.InputBuffer = inputBuffer;
ios->Parameters.DeviceIoControl.InputLength = inputLength;
ios->Parameters.DeviceIoControl.OutputBuffer = outputBuffer;
ios->Parameters.DeviceIoControl.OutputLength = outputLength;
}
// -------------------------------------------------------------------------
// Completion/submission
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information) {
if (!irp) return STATUS_INVALID_PARAMETER;
if (irp->Flags & IRP_FLAG_COMPLETED)
return STATUS_ALREADY_EXISTS;
irp->Flags |= IRP_FLAG_COMPLETED;
irp->Status = status;
irp->Information = information;
// Queue the APC before waking event waiters so a thread that waits on the
// completion event will see and deliver the APC on its wait-return path.
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
if (dev) {
_InterlockedDecrement(&dev->OutstandingIrps);
IoDereferenceDeviceObject(dev);
ios->DeviceObject = (DEVICE_OBJECT*)0;
}
if (irp->CompletionApc)
KeInsertQueueApc((KAPC *)irp->CompletionApc, irp, (void *)(uintptr_t)status);
KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status);
if (irp->CompletionEvent)
KeSignalEvent((EVENT *)irp->CompletionEvent);
KeSignalEvent((EVENT*)irp->CompletionEvent);
if (irp->CompletionFn)
irp->CompletionFn(irp, irp->CompletionCtx);
return STATUS_SUCCESS;
}
// -------------------------------------------------------------------------
// IoSubmitIrp
// -------------------------------------------------------------------------
// Dispatches through the registered DEVICE_OBJECT.
// Accepted IRPs are always completed by the target driver via IoCompleteRequest().
__declspec(dllexport) KSTATUS IoSubmitIrp(IRP *irp) {
if (!irp) return STATUS_INVALID_PARAMETER;
__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
irp->Status = STATUS_FAILURE;
irp->Information = 0;
DEVICE_OBJECT *device = IoGetDiskDevice(irp->DriveIndex);
if (!device) return STATUS_NOT_FOUND;
KSTATUS st = IoCallDriver(device, irp);
// Definitive driver contract:
// STATUS_PENDING means the IRP was accepted and will be completed via
// IoCompleteRequest(), possibly inline before IoCallDriver returns.
// Any other status means the IRP was rejected and no completion occurs.
return st;
irp->Flags &= ~IRP_FLAG_COMPLETED;
return IoCallDriver(device, irp);
}
// -------------------------------------------------------------------------
// IoSubmitIrpSync
// -------------------------------------------------------------------------
// Creates a stack-allocated auto-reset event, attaches it to the IRP,
// submits, and blocks. Callers that already provided a CompletionEvent
// should use IoSubmitIrp() directly and wait on their own event instead.
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp) {
if (!irp) return STATUS_INVALID_PARAMETER;
EVENT *ev = (EVENT *)0;
EVENT* ev = (EVENT*)0;
KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev);
if (!KSUCCESS(st)) return st;
// Attach our event; preserve any APC/callback the caller may have set.
irp->CompletionEvent = ev;
st = IoSubmitIrp(irp);
st = IoSubmitIrp(device, irp);
if (st == STATUS_PENDING) {
KeWaitForEvent(ev, 0);
st = irp->Status;
}
// Release our reference to the temporary event object.
ObDereferenceObject(&ev->Header);
return st;
}
code/kernel/ke_sched.c
+73 -2
View File
@@ -36,6 +36,44 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
}
static int KiCpuIsIdleLocked(const KCPU* cpu) {
if (!cpu || !cpu->Online)
return 0;
THREAD* cur = cpu->CurrentThread;
return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
}
static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
if (!thread || thread->IsIdleThread)
return 0;
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
if (!cpu->IsPrimaryThread)
continue;
if (!KiCpuIsIdleLocked(cpu))
continue;
if (KiThreadCanRunOnCpuLocked(thread, cpu))
return 1;
}
return 0;
}
static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
if (!cpu || !thread || thread->IsIdleThread)
return 0;
if (cpu->IsPrimaryThread || cpu->SmtThreadCount <= 1)
return 0;
// Do not fill a secondary SMT sibling while any primary hardware thread
// that can run this thread is idle. KiPokeRunnableCpusLocked targets those
// primary CPUs first, so this CPU can safely remain idle and wait for more
// work or for physical cores to fill.
return KiAnyIdlePrimaryForThreadLocked(thread);
}
static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || !target->Online)
return;
@@ -52,6 +90,9 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
if (!thread || thread->IsIdleThread)
return;
int haveIdlePrimary = KiAnyIdlePrimaryForThreadLocked(thread);
int requestedIdle = 0;
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
@@ -61,12 +102,35 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
THREAD* cur = cpu->CurrentThread;
if (!cur || cur == cpu->IdleThread) {
if (haveIdlePrimary && !cpu->IsPrimaryThread)
continue;
if (cpu->IsPrimaryThread)
gSchedStats.SmtPrimaryPokes++;
KiRequestRescheduleCpuLocked(cpu);
requestedIdle = 1;
} else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) {
gSchedStats.PriorityPreemptions++;
KiRequestRescheduleCpuLocked(cpu);
}
}
// If no preferred idle primary was actually targetable, allow an idle SMT
// sibling to take the work. This covers explicit affinity to a secondary
// logical CPU and partially-online systems.
if (!requestedIdle && haveIdlePrimary == 0) {
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
THREAD* cur = cpu->CurrentThread;
if (!cur || cur == cpu->IdleThread) {
KiRequestRescheduleCpuLocked(cpu);
break;
}
}
}
}
static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
@@ -369,8 +433,11 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
LIST_ENTRY* head = &gRunQueue[p];
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (KiThreadCanRunOnCpuLocked(thread, cpu))
return 1;
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
if (KiShouldDeferToIdlePrimaryLocked(cpu, thread))
continue;
return 1;
}
}
return 0;
@@ -385,6 +452,10 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) {
gSchedStats.SmtSiblingDeferrals++;
continue;
}
ListRemove(node);
thread->Queued = 0;
code/kernel/ke_selftest.c
+8 -11
View File
@@ -352,6 +352,7 @@ static void KiRunKernelApcSelfTest(void) {
typedef struct KI_IRP_APC_SELFTEST_STATE {
EVENT* CompletionEvent;
DEVICE_OBJECT* DiskDevice;
IRP Irp;
KAPC Apc;
uint8_t* Buffer;
@@ -430,23 +431,18 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
KiIrpApcSelfTestNormalRoutine,
(void*)&st);
LIST_INIT(&st.Irp.Link);
st.Irp.DriveIndex = 0;
st.Irp.Flags = 0;
st.Irp.Lba = 0;
st.Irp.SectorCount = 1;
st.Irp.Buffer = st.Buffer;
st.Irp.Status = STATUS_FAILURE;
st.Irp.Information = 0;
st.DiskDevice = IoReferenceDiskDevice(0);
if (!st.DiskDevice)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: drive 0 disappeared before submit\n");
IoBuildReadWriteIrp(&st.Irp, IRP_MJ_READ, 0, 512, st.Buffer);
st.Irp.CompletionEvent = st.CompletionEvent;
st.Irp.CompletionApc = &st.Apc;
st.Irp.CompletionFn = (IRP_COMPLETION_FN)0;
st.Irp.CompletionCtx = (void*)0;
st.Irp.HwSlot = 0;
st.Irp.DriverCtx = (void*)0;
DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n");
s = IoSubmitIrp(&st.Irp);
s = IoSubmitIrp(st.DiskDevice, &st.Irp);
if (s != STATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: IoSubmitIrp returned %u\n", s);
@@ -472,6 +468,7 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
st.Buffer[510], st.Buffer[511]);
MmFreePool(st.Buffer);
if (st.DiskDevice) IoDereferenceDeviceObject(st.DiskDevice);
ObDereferenceObject(&st.CompletionEvent->Header);
DbgPrint("IRP/APC selftest: PASS\n");
code/kernel/ke_smp.c
+227
View File
@@ -321,6 +321,15 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->Parked = 0;
cpu->Online = 0;
cpu->IsBsp = info->IsBsp ? 1 : 0;
cpu->PackageId = 0;
cpu->CoreId = cpuIndex;
cpu->ThreadId = 0;
cpu->PhysicalCoreIndex = cpuIndex;
cpu->CoreSiblingMask = (cpuIndex < 64) ? (1ULL << cpuIndex) : 0ULL;
cpu->PackageMask = cpu->CoreSiblingMask;
cpu->IsPrimaryThread = 1;
cpu->SmtThreadCount = 1;
cpu->TopologyFlags = 0;
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
@@ -328,6 +337,223 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->SchedulerActive = 0;
}
static uint32_t KiCeilLog2U32(uint32_t value) {
if (value <= 1u)
return 0u;
uint32_t shift = 0;
uint32_t n = value - 1u;
while (n) {
shift++;
n >>= 1;
}
return shift;
}
static uint32_t KiMaskWidth(uint32_t shift) {
if (shift >= 32u)
return 0xFFFFFFFFu;
if (shift == 0u)
return 0u;
return (1u << shift) - 1u;
}
typedef struct KI_CPUID_TOPOLOGY_PARAMS {
uint32_t SmtShift;
uint32_t CoreShift;
uint32_t ThreadsPerCore;
uint32_t LogicalPerPackage;
const char* Source;
} KI_CPUID_TOPOLOGY_PARAMS;
static void KiReadCpuidTopologyLeaf(uint32_t leaf, KI_CPUID_TOPOLOGY_PARAMS* out) {
if (!out)
return;
for (uint32_t sub = 0; sub < 32u; sub++) {
int r[4] = {0, 0, 0, 0};
__cpuidex(r, (int)leaf, (int)sub);
uint32_t levelShift = (uint32_t)r[0] & 0x1Fu;
uint32_t logicals = (uint32_t)r[1] & 0xFFFFu;
uint32_t levelType = ((uint32_t)r[2] >> 8) & 0xFFu;
if (levelType == 0u || logicals == 0u)
break;
if (levelType == 1u) { // SMT level
out->SmtShift = levelShift;
out->ThreadsPerCore = logicals ? logicals : 1u;
} else if (levelType == 2u) { // Core/package level
out->CoreShift = levelShift;
out->LogicalPerPackage = logicals ? logicals : 1u;
break;
}
}
}
static KI_CPUID_TOPOLOGY_PARAMS KiDetectCpuidTopologyParams(void) {
KI_CPUID_TOPOLOGY_PARAMS topo;
memset(&topo, 0, sizeof(topo));
topo.ThreadsPerCore = 1;
topo.LogicalPerPackage = 1;
topo.Source = "flat";
int maxRegs[4] = {0, 0, 0, 0};
__cpuid(maxRegs, 0);
uint32_t maxLeaf = (uint32_t)maxRegs[0];
if (maxLeaf >= 0x1Fu) {
KiReadCpuidTopologyLeaf(0x1Fu, &topo);
if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) {
topo.Source = "cpuid.1f";
return topo;
}
memset(&topo, 0, sizeof(topo));
topo.ThreadsPerCore = 1;
topo.LogicalPerPackage = 1;
topo.Source = "flat";
}
if (maxLeaf >= 0x0Bu) {
KiReadCpuidTopologyLeaf(0x0Bu, &topo);
if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) {
topo.Source = "cpuid.0b";
return topo;
}
memset(&topo, 0, sizeof(topo));
topo.ThreadsPerCore = 1;
topo.LogicalPerPackage = 1;
topo.Source = "flat";
}
int r1[4] = {0, 0, 0, 0};
__cpuid(r1, 1);
uint32_t logicalPerPackage = ((uint32_t)r1[1] >> 16) & 0xFFu;
if (logicalPerPackage == 0u)
logicalPerPackage = 1u;
uint32_t coresPerPackage = 1u;
if (maxLeaf >= 4u) {
int r4[4] = {0, 0, 0, 0};
__cpuidex(r4, 4, 0);
coresPerPackage = (((uint32_t)r4[0] >> 26) & 0x3Fu) + 1u;
if (coresPerPackage == 0u)
coresPerPackage = 1u;
}
uint32_t threadsPerCore = logicalPerPackage / coresPerPackage;
if (threadsPerCore == 0u)
threadsPerCore = 1u;
topo.ThreadsPerCore = threadsPerCore;
topo.LogicalPerPackage = logicalPerPackage;
topo.SmtShift = KiCeilLog2U32(threadsPerCore);
topo.CoreShift = KiCeilLog2U32(logicalPerPackage);
if (topo.CoreShift < topo.SmtShift)
topo.CoreShift = topo.SmtShift;
topo.Source = "cpuid.1/4";
return topo;
}
void KeBuildProcessorTopology(void) {
KI_CPUID_TOPOLOGY_PARAMS topo = KiDetectCpuidTopologyParams();
uint32_t smtMask = KiMaskWidth(topo.SmtShift);
uint32_t coreBits = (topo.CoreShift > topo.SmtShift) ?
(topo.CoreShift - topo.SmtShift) : 0u;
uint32_t coreMask = KiMaskWidth(coreBits);
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
uint32_t apic = cpu->LapicId;
cpu->ThreadId = smtMask ? (apic & smtMask) : 0u;
cpu->CoreId = coreBits ? ((apic >> topo.SmtShift) & coreMask) :
(apic >> topo.SmtShift);
cpu->PackageId = (topo.CoreShift < 32u) ? (apic >> topo.CoreShift) : 0u;
cpu->PhysicalCoreIndex = KE_INVALID_CPU;
cpu->CoreSiblingMask = 0;
cpu->PackageMask = 0;
cpu->IsPrimaryThread = 0;
cpu->SmtThreadCount = 1;
cpu->TopologyFlags = (topo.ThreadsPerCore > 1u) ? 1u : 0u;
}
uint32_t nextCoreIndex = 0;
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
if (cpu->PhysicalCoreIndex != KE_INVALID_CPU)
continue;
uint32_t coreIndex = nextCoreIndex++;
uint64_t coreMaskBits = 0;
uint64_t packageMaskBits = 0;
uint32_t primaryCpuIndex = i;
uint32_t siblingCount = 0;
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
continue;
if (peer->PackageId == cpu->PackageId) {
if (j < 64)
packageMaskBits |= (1ULL << j);
}
if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
peer->PhysicalCoreIndex = coreIndex;
if (j < 64)
coreMaskBits |= (1ULL << j);
if (j < primaryCpuIndex)
primaryCpuIndex = j;
siblingCount++;
}
}
if (siblingCount == 0)
siblingCount = 1;
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
continue;
if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
peer->CoreSiblingMask = coreMaskBits;
peer->PackageMask = packageMaskBits;
peer->SmtThreadCount = (uint8_t)((siblingCount > 255u) ? 255u : siblingCount);
peer->IsPrimaryThread = (j == primaryCpuIndex) ? 1u : 0u;
}
}
}
DbgPrint("SMP: SMT topology source=%s smtShift=%u coreShift=%u threads/core=%u logical/package=%u physicalCores=%u\n",
topo.Source ? topo.Source : "?",
topo.SmtShift,
topo.CoreShift,
topo.ThreadsPerCore,
topo.LogicalPerPackage,
nextCoreIndex);
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
i,
cpu->LapicId,
cpu->PackageId,
cpu->CoreId,
cpu->ThreadId,
cpu->IsPrimaryThread,
(unsigned long long)cpu->CoreSiblingMask,
(unsigned long long)cpu->PackageMask);
}
}
static KSTATUS KiAllocateApStack(KCPU* cpu) {
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
@@ -510,6 +736,7 @@ KSTATUS KeStartApplicationProcessors(void) {
if (claimed == 0 && gCpuCount > 1)
DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n");
KeBuildProcessorTopology();
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
return STATUS_SUCCESS;
}
code/kernel/kmain.c
+7 -2
View File
@@ -341,7 +341,7 @@ void SystemInitThread(void* arg) {
KiResolveSystemImports(args->bootInfo);
KiReclaimBootloaderMemory(args->bootInfo);
if (args->bootInfo->Debug & 2)
if (args->bootInfo->Debug & KE_DEBUG_SELFTEST)
KiRunKernelSelfTests(args->hal);
// Launch User-Mode init.exe and restart it until it exits cleanly.
@@ -363,6 +363,11 @@ void SystemInitThread(void* arg) {
}
DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n");
if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log"));
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus);
}
gKmainHal->Power.Shutdown();
MmFreePool(args);
}
@@ -416,7 +421,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
const HAL_API* hal = HalGetApi();
gKmainHal = hal;
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1);
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug);
// CPU startup intentionally lives after HAL initialization but before IRQs
// and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI;
code/public/ntlite/ddi/io_driver.h
+173 -43
View File
@@ -1,39 +1,35 @@
#pragma once
// io_driver.h - definitive kernel driver/device I/O model
// io_driver.h - kernel driver/device I/O model
//
// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through
// IoCallDriver()/IoSubmitIrp(); drivers complete accepted IRPs by calling
// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table.
// Device lifetime is owned by the I/O manager. Bus drivers create PDOs,
// function/filter drivers attach FDOs, and PnP/remove IRPs tear stacks down.
// Disk indexes are only a compatibility namespace above DEVICE_OBJECTs.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <kernel/ob_object.h>
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irp.h>
typedef struct HAL_API HAL_API;
typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API* hal);
typedef void (*DRIVER_UNLOAD_FN)(void);
struct IRP;
typedef struct DRIVER_OBJECT DRIVER_OBJECT;
typedef struct DEVICE_OBJECT DEVICE_OBJECT;
typedef struct IRP IRP;
typedef enum IO_MAJOR_FUNCTION {
IRP_MJ_READ = 0,
IRP_MJ_WRITE = 1,
IRP_MJ_DEVICE_CONTROL = 2,
IRP_MJ_MAXIMUM_FUNCTION = 3,
} IO_MAJOR_FUNCTION;
typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, struct IRP* irp);
struct DRIVER_OBJECT {
const char* Name;
PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION];
DRIVER_UNLOAD_FN DriverUnload;
};
typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, IRP* irp);
typedef KSTATUS (*PDRIVER_ADD_DEVICE)(DRIVER_OBJECT* driver,
DEVICE_OBJECT* physicalDeviceObject);
// -------------------------------------------------------------------------
// Bus / identity
// -------------------------------------------------------------------------
typedef enum IO_BUS_TYPE {
IoBusUnknown = 0,
@@ -41,6 +37,139 @@ typedef enum IO_BUS_TYPE {
IoBusUsb = 2,
} IO_BUS_TYPE;
typedef struct IO_DEVICE_ID {
IO_BUS_TYPE BusType;
SYS_STRING_VIEW Id;
} IO_DEVICE_ID;
typedef struct IO_DRIVER_ID_TABLE {
const IO_DEVICE_ID* Ids;
size_t Count;
} IO_DRIVER_ID_TABLE;
#define IO_MAX_DEVICE_IDS 8u
#define IO_MAX_DEVICE_ID_CHARS 96u
#define IO_MAX_DEVICE_NAME_CHARS 64u
// -------------------------------------------------------------------------
// Driver/device objects
// -------------------------------------------------------------------------
typedef enum DEVICE_OBJECT_TYPE {
DeviceObjectPhysical = 1,
DeviceObjectFunction = 2,
DeviceObjectFilter = 3,
} DEVICE_OBJECT_TYPE;
typedef enum DEVICE_PNP_STATE {
DevicePnpNotStarted = 0,
DevicePnpStarted,
DevicePnpStopPending,
DevicePnpStopped,
DevicePnpRemovePending,
DevicePnpSurpriseRemoved,
DevicePnpDeleted,
} DEVICE_PNP_STATE;
struct DRIVER_OBJECT {
OBJECT Header;
SYS_STRING_VIEW Name;
PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION];
PDRIVER_ADD_DEVICE DriverAddDevice;
DRIVER_UNLOAD_FN DriverUnload;
const IO_DRIVER_ID_TABLE* IdTable;
};
struct DEVICE_OBJECT {
OBJECT Header;
DRIVER_OBJECT* DriverObject;
DEVICE_OBJECT_TYPE Type;
DEVICE_PNP_STATE PnpState;
DEVICE_OBJECT* Parent;
DEVICE_OBJECT* FirstChild;
DEVICE_OBJECT* NextSibling;
DEVICE_OBJECT* AttachedDevice; // device directly above this object
DEVICE_OBJECT* LowerDevice; // device directly below this FDO/filter
void* DeviceExtension;
uint32_t Flags;
uint32_t Characteristics;
uint32_t AlignmentRequirement;
SPINLOCK Lock;
volatile long OutstandingIrps;
bool DeletePending;
SYS_STRING_VIEW Name;
char NameStorage[IO_MAX_DEVICE_NAME_CHARS];
union {
struct {
IO_BUS_TYPE BusType;
void* BusContext;
void* DeviceContext;
IO_DEVICE_ID Ids[IO_MAX_DEVICE_IDS];
char IdStorage[IO_MAX_DEVICE_IDS][IO_MAX_DEVICE_ID_CHARS];
uint8_t IdCount;
} Pdo;
struct {
uint8_t DiskIndex;
uint64_t SectorCount;
uint32_t SectorSize;
} Disk;
} u;
};
#define DEVICE_OBJECT_DISK 0x00000001u
#define DEVICE_OBJECT_REMOVABLE 0x00000002u
// -------------------------------------------------------------------------
// Object/device creation and stack management
// -------------------------------------------------------------------------
__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
SYS_STRING_VIEW name);
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver);
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice);
__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
DEVICE_OBJECT* parent,
const IO_DEVICE_ID* ids,
size_t idCount,
size_t extensionSize,
DEVICE_OBJECT** outPdo);
__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo);
__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
SYS_STRING_VIEW name);
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device);
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device);
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device);
__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
DEVICE_OBJECT* target);
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source);
__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp);
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp);
__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
IO_PNP_MINOR_FUNCTION minor);
// -------------------------------------------------------------------------
// Legacy USB publish/matcher path. This remains only until USB is migrated to
// PDO/interface-PDO enumeration in the later USB stages.
// -------------------------------------------------------------------------
#define IO_PUBLISHED_USB_MAX_ENDPOINTS 8u
typedef struct IO_PUBLISHED_USB_ENDPOINT {
@@ -102,41 +231,42 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
__declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device);
__declspec(dllexport) uint32_t IoPublishedDeviceCount(void);
struct DEVICE_OBJECT {
DRIVER_OBJECT* DriverObject;
void* DeviceExtension;
uint32_t Flags;
uint32_t AlignmentRequirement;
uint64_t SectorCount;
uint8_t DiskIndex;
};
// -------------------------------------------------------------------------
// Disk namespace above storage DEVICE_OBJECTs
// -------------------------------------------------------------------------
#define DEVICE_OBJECT_DISK 0x00000001u
#define DEVICE_OBJECT_REMOVABLE 0x00000002u
typedef enum IO_DEVICE_CHANGE {
IoDeviceArrived = 0,
IoDeviceQueryRemove,
IoDeviceSurpriseRemoved,
IoDeviceRemoved,
} IO_DEVICE_CHANGE;
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice);
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
DEVICE_OBJECT* device);
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex);
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex);
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex);
typedef void (*IO_DISK_CHANGE_CALLBACK)(uint8_t driveIndex,
DEVICE_OBJECT* device,
IO_DEVICE_CHANGE change,
void* context);
typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex,
DEVICE_OBJECT* device,
void* context);
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex);
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex);
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex);
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex);
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback,
void* context);
__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback,
void* context);
__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback,
void* context);
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device,
struct IRP* irp);
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
code/public/ntlite/syscall.h
-1
View File
@@ -106,7 +106,6 @@ typedef struct SYS_USER_APC_FRAME {
void SysUserApcDispatcher(void);
// ---------------------------------------------------------------------------
// Trap-frame query ABI
// ---------------------------------------------------------------------------