maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

usb hubs, hotplug support, unplug support, saving log files to disk, smt

Browse Source
This commit is contained in:
maksw committed 2026-06-09 09:14:24 +02:00
1 parent b838f8d83d
commit 27b31f62b6
39 files changed
+6294 -735
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

OVMF/OVMF_VARS_4M.fd
BIN
View File
Binary file not shown.
OVMF/OVMF_VARS_4M_d.fd
BIN
View File
Binary file not shown.
code/boot.ini
+5 -5
View File
@@ -1,5 +1,5 @@
[kernel]
; bitmask: 1 = debug prints, 2 = self check
debug = 1
[init]
commandline = apps/shell_native.exe
[kernel]
; bitmask: 1 = debug prints, 2 = self check, 4 = write debug prints to file
debug = 4
[init]
commandline = apps/shell_native.exe
code/boot/uefi/main.c
+25 -2
View File
@@ -264,12 +264,35 @@ typedef struct {
char commandLine[64];
} PARSE_STATE;
static int ParseBootIniInteger(const char* val) {
int out = 0;
if (!val) return 0;
while (*val == ' ' || *val == '\t') val++;
int base = 10;
if (val[0] == '0' && (val[1] == 'x' || val[1] == 'X')) {
base = 16;
val += 2;
}
while (*val) {
char c = *val++;
int digit;
if (c >= '0' && c <= '9') digit = c - '0';
else if (c >= 'a' && c <= 'f') digit = 10 + c - 'a';
else if (c >= 'A' && c <= 'F') digit = 10 + c - 'A';
else break;
if (digit >= base) break;
out = out * base + digit;
}
return out;
}
static void BootIniCb(const char* section, const char* key,
const char* val, void* ud) {
PARSE_STATE* s = (PARSE_STATE *)ud;
// [kernel] debug = 1
// [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log
if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug"))
s->debug = (int)(val[0] - '0');
s->debug = ParseBootIniInteger(val);
// [init] commandline = hello.exe
else if (!AsciiStrCmp(section, "init") && !AsciiStrCmp(key, "commandline"))
AsciiStrCpyS(s->commandLine, 64, val);
code/build/build.c
+1 -1
View File
@@ -28,7 +28,7 @@ static const Driver_Target STANDALONE_DRIVERS[] = {
DRIVER_TARGET("ide", "obj\\drivers\\ide", "code\\drivers\\storage\\ide\\ide.c code\\drivers\\storage\\ide\\ide_bus.c"),
DRIVER_TARGET("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c"),
DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c"),
DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_bus.c code\\drivers\\bus\\usb\\usb_composite.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c code\\drivers\\bus\\usb\\usb_hub.c"),
DRIVER_TARGET("rtl816x", "obj\\drivers\\rtl816x", "code\\drivers\\net\\rtl816x\\rtl816x.c code\\drivers\\net\\rtl816x\\rtl816x_pci.c"),
DRIVER_TARGET("rtl8139", "obj\\drivers\\rtl8139", "code\\drivers\\net\\rtl8139\\rtl8139.c code\\drivers\\net\\rtl8139\\rtl8139_pci.c"),
};
code/drivers/bus/usb/usb.c
+26 -2
View File
@@ -75,9 +75,33 @@ KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal) return STATUS_INVALID_PARAMETER;
DbgPrint("usb: discovery starting\n");
KSTATUS st = UsbRegisterClassMatchers();
KSTATUS st = UsbInitializeBusModel();
if (!KSUCCESS(st)) {
DbgPrint("usb: class matcher registration failed st=0x%x\n", st);
DbgPrint("usb: bus model initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeCompositeDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: composite driver initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeHidDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: HID driver initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeStorageDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: storage driver initialisation failed st=0x%x\n", st);
return st;
}
st = UsbInitializeHubDriver();
if (!KSUCCESS(st)) {
DbgPrint("usb: hub driver initialisation failed st=0x%x\n", st);
return st;
}
code/drivers/bus/usb/usb.h
+440 -16
View File
@@ -16,6 +16,8 @@
#include <kernel/ke_dpc.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_virt.h>
@@ -114,6 +116,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define XHCI_TRB_TYPE_DATA 3u
#define XHCI_TRB_TYPE_STATUS 4u
#define XHCI_TRB_TYPE_ENABLE_SLOT 9u
#define XHCI_TRB_TYPE_DISABLE_SLOT 10u
#define XHCI_TRB_TYPE_ADDRESS_DEVICE 11u
#define XHCI_TRB_TYPE_CONFIGURE_ENDPOINT 12u
@@ -144,10 +147,15 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define USB_REQ_SET_CONFIGURATION 0x09u
#define USB_REQ_SET_IDLE 0x0Au
#define USB_REQ_SET_PROTOCOL 0x0Bu
#define USB_REQ_GET_STATUS 0x00u
#define USB_REQ_CLEAR_FEATURE 0x01u
#define USB_REQ_SET_FEATURE 0x03u
#define USB_DESC_DEVICE 0x01u
#define USB_DESC_CONFIGURATION 0x02u
#define USB_DESC_INTERFACE 0x04u
#define USB_DESC_ENDPOINT 0x05u
#define USB_DESC_HUB 0x29u
#define USB_DESC_SUPERSPEED_HUB 0x2Au
#define USB_DESC_HID 0x21u
#define USB_DESC_CS_INTERFACE 0x24u
#define USB_DIR_OUT 0x00u
@@ -155,6 +163,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define USB_TYPE_STANDARD 0x00u
#define USB_TYPE_CLASS 0x20u
#define USB_RECIP_DEVICE 0x00u
#define USB_RECIP_OTHER 0x03u
#define USB_RECIP_INTERFACE 0x01u
#define USB_ENDPOINT_DIR_IN 0x80u
#define USB_ENDPOINT_NUMBER_MASK 0x0Fu
@@ -165,6 +174,23 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define USB_ENDPOINT_TYPE_INTERRUPT 0x03u
#define USB_CLASS_HID 0x03u
#define USB_CLASS_MASS_STORAGE 0x08u
#define USB_CLASS_HUB 0x09u
#define USB_HUB_PORT_FEAT_CONNECTION 0u
#define USB_HUB_PORT_FEAT_ENABLE 1u
#define USB_HUB_PORT_FEAT_SUSPEND 2u
#define USB_HUB_PORT_FEAT_OVER_CURRENT 3u
#define USB_HUB_PORT_FEAT_RESET 4u
#define USB_HUB_PORT_FEAT_POWER 8u
#define USB_HUB_PORT_FEAT_C_CONNECTION 16u
#define USB_HUB_PORT_FEAT_C_ENABLE 17u
#define USB_HUB_PORT_FEAT_C_RESET 20u
#define USB_HUB_PORT_STATUS_CONNECTION (1u << 0)
#define USB_HUB_PORT_STATUS_ENABLE (1u << 1)
#define USB_HUB_PORT_STATUS_LOW_SPEED (1u << 9)
#define USB_HUB_PORT_STATUS_HIGH_SPEED (1u << 10)
#define USB_HUB_PORT_CHANGE_CONNECTION (1u << 0)
#define USB_HUB_PORT_CHANGE_ENABLE (1u << 1)
#define USB_HUB_PORT_CHANGE_RESET (1u << 4)
#define USB_MASS_SUBCLASS_SCSI 0x06u
#define USB_MASS_PROTOCOL_BOT 0x50u
#define USB_HID_SUBCLASS_BOOT 0x01u
@@ -193,12 +219,22 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define XHCI_EP0_RING_TRBS 256u
#define XHCI_HID_RING_TRBS 256u
#define XHCI_BULK_RING_TRBS 256u
#define XHCI_PIPE_RING_TRBS 256u
#define XHCI_MAX_HID_REPORT 16u
#define USB_STORAGE_DMA_PAGES 1u
#define USB_STORAGE_MAX_BYTES (USB_STORAGE_DMA_PAGES * PAGE_SIZE)
#define USB_STORAGE_SECTOR_SIZE 512u
#define USB_MAX_TRACKED_INTERFACES 8u
#define USB_MAX_TRACKED_ENDPOINTS 8u
#define USB_MAX_DEVICE_ID_TEXT 96u
#define USB_MAX_PIPES 32u
#define USB_IOCTL_SUBMIT_URB 0x55534201u /* USB1 */
#define USB_ROOT_HUB_MAX_PORTS XHCI_MAX_ROOT_PORTS
#define USB_EXTERNAL_HUB_MAX_PORTS 31u
#define USB_ROOT_PORT_ENUM_RETRIES 3u
#define USB_XHCI_CONTROL_TIMEOUT_US 1000000u
#define USB_XHCI_ENUM_CONTROL_TIMEOUT_US 120000u
#define USB_HUB_CHANGE_BITMAP_BYTES 8u
#pragma pack(push, 1)
typedef struct XHCI_TRB {
@@ -271,6 +307,21 @@ typedef struct USB_ENDPOINT_DESCRIPTOR {
uint8_t bInterval;
} USB_ENDPOINT_DESCRIPTOR;
typedef struct USB_HUB_DESCRIPTOR {
uint8_t bLength;
uint8_t bDescriptorType;
uint8_t bNbrPorts;
uint16_t wHubCharacteristics;
uint8_t bPwrOn2PwrGood;
uint8_t bHubContrCurrent;
uint8_t DeviceRemovable[1];
} USB_HUB_DESCRIPTOR;
typedef struct USB_HUB_PORT_STATUS {
uint16_t Status;
uint16_t Change;
} USB_HUB_PORT_STATUS;
typedef struct USB_MASS_CBW {
uint32_t Signature;
uint32_t Tag;
@@ -308,6 +359,309 @@ typedef struct USB_BAR {
uint8_t Index;
} USB_BAR;
typedef enum USB_DEVICE_STATE {
UsbDeviceAttached = 0,
UsbDevicePowered,
UsbDeviceDefault,
UsbDeviceAddressed,
UsbDeviceConfigured,
UsbDeviceSuspended,
UsbDeviceSurpriseRemoved,
UsbDeviceRemoved,
} USB_DEVICE_STATE;
typedef struct USB_PORT_PATH {
uint8_t RootPort;
uint8_t Depth;
uint32_t RouteString;
uint8_t ParentHubSlot;
uint8_t ParentHubPort;
bool BehindHub;
} USB_PORT_PATH;
typedef enum USB_PIPE_TYPE {
UsbPipeControl = 0,
UsbPipeBulk,
UsbPipeInterrupt,
UsbPipeIsochronous,
} USB_PIPE_TYPE;
typedef enum USB_TRANSFER_DIRECTION {
UsbTransferNoData = 0,
UsbTransferOut,
UsbTransferIn,
} USB_TRANSFER_DIRECTION;
typedef enum USB_URB_FUNCTION {
UrbFunctionControlTransfer = 0,
UrbFunctionBulkTransfer,
UrbFunctionInterruptTransfer,
UrbFunctionSelectConfiguration,
UrbFunctionSelectInterface,
UrbFunctionResetPipe,
UrbFunctionAbortPipe,
} USB_URB_FUNCTION;
typedef struct USB_HCD USB_HCD;
typedef struct USB_DEVICE USB_DEVICE;
typedef struct USB_PIPE USB_PIPE;
typedef struct USB_URB USB_URB;
typedef struct XHCI_DEVICE XHCI_DEVICE;
typedef void (*USB_URB_COMPLETION)(USB_URB* urb, void* context);
typedef struct USB_HCD_OPERATIONS {
KSTATUS (*SubmitUrb)(USB_HCD* hcd, USB_URB* urb);
KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb);
KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed);
KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port);
void* (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys);
} USB_HCD_OPERATIONS;
struct USB_HCD {
const USB_HCD_OPERATIONS* Operations;
void* ControllerContext;
DRIVER_OBJECT* DriverObject;
DEVICE_OBJECT* ControllerPdo;
DEVICE_OBJECT* RootHubPdo;
void* RootHubContext;
uint8_t ControllerIndex;
uint8_t MaxRootPorts;
};
struct USB_DEVICE {
OBJECT Header;
USB_HCD* Hcd;
DEVICE_OBJECT* Pdo;
void* HcdDeviceContext;
USB_DEVICE_STATE State;
USB_PORT_PATH Path;
uint8_t Address;
uint8_t SlotId;
uint8_t Speed;
uint16_t MaxPacketSize0;
uint16_t VendorId;
uint16_t ProductId;
uint16_t BcdUsb;
uint16_t BcdDevice;
uint8_t DeviceClass;
uint8_t DeviceSubclass;
uint8_t DeviceProtocol;
uint8_t ConfigurationValue;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
};
struct USB_PIPE {
OBJECT Header;
USB_DEVICE* Device;
USB_PIPE_TYPE Type;
uint8_t EndpointAddress;
uint8_t EndpointId;
uint16_t MaxPacketSize;
uint8_t Interval;
bool Open;
bool Halted;
void* HcdPipeContext;
};
struct USB_URB {
LIST_ENTRY Link;
USB_URB_FUNCTION Function;
USB_TRANSFER_DIRECTION Direction;
USB_DEVICE* Device;
USB_PIPE* Pipe;
USB_SETUP_PACKET Setup;
void* Buffer;
PHYSICAL_ADDRESS BufferPhys;
uint32_t Length;
uint32_t ActualLength;
KSTATUS Status;
IRP* Irp;
USB_URB_COMPLETION Complete;
void* CompleteContext;
bool Pending;
bool Cancelled;
};
typedef enum USB_PDO_KIND {
UsbPdoController = 1,
UsbPdoDevice = 2,
UsbPdoInterface = 3,
UsbPdoRootHub = 4,
} USB_PDO_KIND;
typedef struct USB_PDO_EXTENSION {
USB_PDO_KIND Kind;
USB_HCD* Hcd;
USB_DEVICE* Device;
uint8_t InterfaceIndex;
USB_INTERFACE_DESCRIPTOR InterfaceDescriptor;
uint8_t EndpointCount;
USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS];
} USB_PDO_EXTENSION;
typedef enum USB_HUB_KIND {
UsbHubKindRoot = 1,
UsbHubKindExternal = 2,
} USB_HUB_KIND;
typedef enum USB_HUB_PORT_STATE {
UsbHubPortDisconnected = 0,
UsbHubPortPowered,
UsbHubPortConnectedDebounce,
UsbHubPortResetting,
UsbHubPortEnabled,
UsbHubPortEnumerating,
UsbHubPortChildPresent,
UsbHubPortFailedConnected,
UsbHubPortRemoving,
} USB_HUB_PORT_STATE;
typedef struct USB_HUB_EXTENSION {
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* HubPdo;
USB_HUB_KIND Kind;
USB_HCD* Hcd;
USB_DEVICE* Device;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
bool Started;
bool SurpriseRemoved;
bool Removed;
uint8_t PortCount;
uint8_t RootPortSlot[USB_ROOT_HUB_MAX_PORTS + 1u];
uint8_t RootPortFailCount[USB_ROOT_HUB_MAX_PORTS + 1u];
USB_HUB_PORT_STATE RootPortState[USB_ROOT_HUB_MAX_PORTS + 1u];
uint8_t ChildSlot[USB_EXTERNAL_HUB_MAX_PORTS + 1u];
USB_HUB_PORT_STATE ChildPortState[USB_EXTERNAL_HUB_MAX_PORTS + 1u];
uint32_t PendingRootPorts[8];
uint32_t PendingHubPorts[1];
SPINLOCK PendingLock;
SEMAPHORE* WorkerSemaphore;
THREAD* WorkerThread;
bool WorkerRunning;
bool InitialScanPending;
bool InitialScanComplete;
bool InitialScanInProgress;
uint8_t InterfaceNumber;
uint8_t InterruptEndpointAddress;
uint16_t InterruptMaxPacket;
uint8_t InterruptInterval;
USB_PIPE* InterruptPipe;
uint8_t* ChangeBitmap;
PHYSICAL_ADDRESS ChangeBitmapPhys;
USB_URB ChangeUrb;
bool ChangeUrbPending;
uint8_t PowerOnToPowerGoodMs;
} USB_HUB_EXTENSION;
typedef struct USB_COMPOSITE_EXTENSION {
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* DevicePdo;
USB_DEVICE* Device;
bool Started;
bool InterfacesCreated;
} USB_COMPOSITE_EXTENSION;
typedef struct USB_HID_EXTENSION {
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* InterfacePdo;
USB_DEVICE* Device;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
bool Started;
bool SurpriseRemoved;
bool Removed;
bool Keyboard;
bool Mouse;
uint8_t InterfaceNumber;
uint8_t Protocol;
uint8_t EndpointAddress;
uint16_t MaxPacketSize;
uint8_t Interval;
USB_PIPE* InterruptPipe;
uint8_t* ReportBuffer;
PHYSICAL_ADDRESS ReportBufferPhys;
uint16_t ReportLength;
USB_URB InterruptUrb;
bool InterruptPending;
uint8_t LastModifiers;
uint8_t LastKeys[6];
uint8_t LastButtons;
uint32_t InputDeviceId;
} USB_HID_EXTENSION;
typedef enum USB_STORAGE_EXTENSION_KIND {
UsbStorageExtensionInterface = 0x55534946u,
UsbStorageExtensionDisk = 0x5553444Bu,
} USB_STORAGE_EXTENSION_KIND;
typedef struct USB_STORAGE_EXTENSION USB_STORAGE_EXTENSION;
typedef struct USB_STORAGE_DISK_EXTENSION USB_STORAGE_DISK_EXTENSION;
struct USB_STORAGE_EXTENSION {
USB_STORAGE_EXTENSION_KIND Kind;
DEVICE_OBJECT* Self;
DEVICE_OBJECT* LowerDevice;
DEVICE_OBJECT* InterfacePdo;
DEVICE_OBJECT* DiskDevice;
USB_DEVICE* Device;
XHCI_DEVICE* XhciDevice;
USB_PIPE* BulkInPipe;
USB_PIPE* BulkOutPipe;
IO_REMOVE_LOCK RemoveLock;
bool RemoveLockReady;
bool Started;
bool SurpriseRemoved;
bool Removed;
uint8_t InterfaceNumber;
uint8_t BulkInAddress;
uint8_t BulkOutAddress;
uint16_t BulkInMaxPacket;
uint16_t BulkOutMaxPacket;
uint32_t Tag;
uint32_t BlockSize;
uint64_t BlockCount;
volatile long IoBusy;
};
struct USB_STORAGE_DISK_EXTENSION {
USB_STORAGE_EXTENSION_KIND Kind;
USB_STORAGE_EXTENSION* Storage;
};
typedef struct XHCI_PIPE_CONTEXT {
XHCI_TRB* Ring;
PHYSICAL_ADDRESS RingPhys;
uint32_t Enqueue;
uint8_t Cycle;
uint8_t EndpointId;
USB_PIPE_TYPE Type;
bool Configured;
bool ActiveTransfer;
uint64_t ActiveTrbPhys;
USB_URB* ActiveUrb;
} XHCI_PIPE_CONTEXT;
typedef struct XHCI_DEVICE {
bool Allocated;
bool Addressed;
@@ -324,6 +678,11 @@ typedef struct XHCI_DEVICE {
uint8_t DeviceProtocol;
uint8_t ConfigurationValue;
bool Configured;
USB_DEVICE* UsbDevice;
DEVICE_OBJECT* UsbPdo;
DEVICE_OBJECT* ParentDeviceObject;
USB_PORT_PATH Path;
USB_PIPE* EndpointPipes[32];
uint8_t InterfaceCount;
USB_TRACKED_INTERFACE Interfaces[USB_MAX_TRACKED_INTERFACES];
@@ -364,6 +723,10 @@ typedef struct XHCI_DEVICE {
uint32_t MassBlockSize;
uint64_t MassBlockCount;
DEVICE_OBJECT* MassDeviceObject;
IO_REMOVE_LOCK MassRemoveLock;
bool MassRemoveLockReady;
bool MassSurpriseRemoved;
bool MassRemoved;
volatile long MassIoBusy;
PHYSICAL_ADDRESS InputContextPhys;
@@ -403,12 +766,14 @@ typedef struct XHCI_TRANSFER_RESULT {
} XHCI_TRANSFER_RESULT;
typedef struct XHCI_WAIT_TARGET {
struct XHCI_WAIT_TARGET* Next;
uint64_t CommandPhys;
uint64_t TransferPhys;
uint8_t TransferSlotId;
uint8_t TransferEndpointId;
bool CommandMatched;
bool TransferMatched;
volatile bool CommandMatched;
volatile bool TransferMatched;
volatile bool Registered;
XHCI_COMMAND_RESULT Command;
XHCI_TRANSFER_RESULT Transfer;
} XHCI_WAIT_TARGET;
@@ -470,15 +835,58 @@ typedef struct XHCI_CONTROLLER {
volatile uint32_t InterruptCount;
volatile uint32_t EventDpcCount;
SPINLOCK EventLock;
SPINLOCK WaitLock;
SPINLOCK CommandLock;
XHCI_WAIT_TARGET* WaiterList;
volatile XHCI_WAIT_TARGET* ActiveWait;
volatile bool Running;
USB_HCD Hcd;
} XHCI_CONTROLLER;
extern XHCI_CONTROLLER gXhci;
extern bool gHaveXhci;
extern DRIVER_OBJECT gUsbStorageDriver;
extern DRIVER_OBJECT gUsbBusDriver;
extern DRIVER_OBJECT gUsbCompositeDriver;
extern DRIVER_OBJECT gUsbHidDriver;
extern DRIVER_OBJECT gUsbHubDriver;
KSTATUS UsbInitializeBusModel(void);
KSTATUS UsbInitializeCompositeDriver(void);
KSTATUS UsbInitializeHidDriver(void);
KSTATUS UsbInitializeStorageDriver(void);
KSTATUS UsbInitializeHubDriver(void);
KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
void UsbInitializeHcd(USB_HCD* hcd,
const USB_HCD_OPERATIONS* ops,
void* controllerContext,
uint8_t controllerIndex,
uint8_t maxRootPorts);
KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device);
KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd);
KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise);
KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise);
KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent);
KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xhciDev, DEVICE_OBJECT* parent);
KSTATUS UsbCreatePipe(USB_DEVICE* dev,
USB_PIPE_TYPE type,
uint8_t endpointAddress,
uint16_t maxPacketSize,
uint8_t interval,
USB_PIPE** outPipe);
void UsbReferenceDevice(USB_DEVICE* dev);
void UsbDereferenceDevice(USB_DEVICE* dev);
void UsbReferencePipe(USB_PIPE* pipe);
void UsbDereferencePipe(USB_PIPE* pipe);
KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbCancelUrb(USB_URB* urb);
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys);
KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp);
uint32_t UsbAlignUp32(uint32_t value, uint32_t align);
void XhciBarrier(void);
@@ -512,11 +920,15 @@ const char* UsbEndpointTypeName(uint8_t type);
const char* UsbDirectionName(uint8_t endpointAddress);
const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto);
void XhciLogPortStatus(XHCI_CONTROLLER* xhci);
void XhciPowerRootPorts(XHCI_CONTROLLER* xhci);
void XhciAckInterrupter0(XHCI_CONTROLLER* xhci);
void XhciUpdateErdp(XHCI_CONTROLLER* xhci);
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target);
void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs);
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs);
void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2);
void XhciIsr(void);
@@ -535,12 +947,15 @@ void XhciEnableInterrupts(XHCI_CONTROLLER* xhci);
uint32_t XhciPortReg(uint8_t port);
void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port);
KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed);
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId);
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index);
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs);
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length);
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup);
KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs);
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc);
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength);
uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress);
@@ -561,33 +976,42 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub, uint8_t endpointType, bool wantIn);
bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle);
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe);
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
uint32_t UsbBe32(const uint8_t* p);
void UsbStoreBe32(uint8_t* p, uint32_t v);
void UsbStoreBe16(uint8_t* p, uint16_t v);
KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, bool write, uint64_t lba, uint32_t sectors, void* buffer);
void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void UsbStorageReleaseIo(XHCI_DEVICE* dev);
KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer);
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage);
void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
uint8_t UsbFindFreeDiskIndex(void);
KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev);
KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage);
bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context);
KSTATUS UsbRegisterClassMatchers(void);
KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationValue);
void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_CONFIGURATION_DESCRIPTOR* config, uint16_t totalLength);
void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_DEVICE_DESCRIPTOR* devDesc);
KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId);
KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci, USB_HUB_EXTENSION* hub, uint8_t hubPort, uint8_t speed, uint8_t* outSlotId);
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port);
void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci);
KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd);
void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port);
void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port);
extern const USB_HCD_OPERATIONS gXhciHcdOperations;
KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device);
code/drivers/bus/usb/usb_bus.c
+656
View File
@@ -0,0 +1,656 @@
// usb_bus.c - USB bus/HCD/PDO/URB core model.
//
// This is the structural USB bus layer used by the upcoming composite, hub,
// and hotplug stages. The legacy direct xHCI class path is still present for
// boot HID/storage during migration, but newly enumerated xHCI devices now also
// get first-class USB PDOs and can accept internal URB IRPs.
#include "usb.h"
DRIVER_OBJECT gUsbBusDriver;
static bool gUsbBusModelReady;
static uint8_t gUsbNextControllerIndex;
static void UsbDeviceDestroy(OBJECT* object) {
USB_DEVICE* dev = (USB_DEVICE*)object;
if (dev->RemoveLockReady) {
IoDestroyRemoveLock(&dev->RemoveLock);
dev->RemoveLockReady = false;
}
if (dev->Pdo) {
IoDereferenceDeviceObject(dev->Pdo);
dev->Pdo = (DEVICE_OBJECT*)0;
}
}
static void UsbPipeDestroy(OBJECT* object) {
USB_PIPE* pipe = (USB_PIPE*)object;
if (pipe->Device) {
UsbDereferenceDevice(pipe->Device);
pipe->Device = (USB_DEVICE*)0;
}
}
void UsbReferenceDevice(USB_DEVICE* dev) {
if (dev) ObReferenceObject(&dev->Header);
}
void UsbDereferenceDevice(USB_DEVICE* dev) {
if (dev) ObDereferenceObject(&dev->Header);
}
void UsbReferencePipe(USB_PIPE* pipe) {
if (pipe) ObReferenceObject(&pipe->Header);
}
void UsbDereferencePipe(USB_PIPE* pipe) {
if (pipe) ObDereferenceObject(&pipe->Header);
}
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma)
return (void*)0;
return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys);
}
static USB_DEVICE* UsbAllocateDevice(void) {
USB_DEVICE* dev = (USB_DEVICE*)ObAllocateObject(ObjectTypeUsbDevice, sizeof(USB_DEVICE));
if (!dev) return (USB_DEVICE*)0;
dev->Header.Destroy = UsbDeviceDestroy;
KSTATUS st = IoInitializeRemoveLock(&dev->RemoveLock);
if (!KSUCCESS(st)) {
ObDereferenceObject(&dev->Header);
return (USB_DEVICE*)0;
}
dev->RemoveLockReady = true;
dev->State = UsbDeviceAttached;
return dev;
}
KSTATUS UsbCreatePipe(USB_DEVICE* dev,
USB_PIPE_TYPE type,
uint8_t endpointAddress,
uint16_t maxPacketSize,
uint8_t interval,
USB_PIPE** outPipe) {
if (!dev || !outPipe) return STATUS_INVALID_PARAMETER;
USB_PIPE* pipe = (USB_PIPE*)ObAllocateObject(ObjectTypeUsbPipe, sizeof(USB_PIPE));
if (!pipe) return STATUS_NO_MEMORY;
pipe->Header.Destroy = UsbPipeDestroy;
pipe->Device = dev;
UsbReferenceDevice(dev);
pipe->Type = type;
pipe->EndpointAddress = endpointAddress;
pipe->EndpointId = XhciEndpointIdFromAddress(endpointAddress);
pipe->MaxPacketSize = maxPacketSize;
pipe->Interval = interval;
pipe->Open = true;
*outPipe = pipe;
return STATUS_SUCCESS;
}
static SYS_STRING_VIEW UsbIdFromBuffer(char* buf) {
return SysSvFromCstr(buf);
}
static void UsbIdPutChar(char* buf, size_t cap, size_t* pos, char ch) {
if (!buf || !pos || cap == 0)
return;
if (*pos + 1 < cap)
buf[*pos] = ch;
(*pos)++;
if (*pos < cap)
buf[*pos] = '\0';
else
buf[cap - 1] = '\0';
}
static void UsbIdAppendLiteral(char* buf, size_t cap, size_t* pos, const char* text) {
if (!text)
return;
while (*text)
UsbIdPutChar(buf, cap, pos, *text++);
}
static void UsbIdAppendHex(char* buf, size_t cap, size_t* pos, uint32_t value, uint8_t digits) {
static const char kHex[] = "0123456789ABCDEF";
while (digits) {
digits--;
UsbIdPutChar(buf, cap, pos, kHex[(value >> (digits * 4u)) & 0xFu]);
}
}
static void UsbFormatVidPidRev(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint16_t rev) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
UsbIdAppendHex(buf, cap, &pos, vid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
UsbIdAppendHex(buf, cap, &pos, pid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&REV_");
UsbIdAppendHex(buf, cap, &pos, rev, 4);
}
static void UsbFormatVidPid(char* buf, size_t cap, uint16_t vid, uint16_t pid) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
UsbIdAppendHex(buf, cap, &pos, vid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
UsbIdAppendHex(buf, cap, &pos, pid, 4);
}
static void UsbFormatVidPidMi(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint8_t mi) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\VID_");
UsbIdAppendHex(buf, cap, &pos, vid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&PID_");
UsbIdAppendHex(buf, cap, &pos, pid, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&MI_");
UsbIdAppendHex(buf, cap, &pos, mi, 2);
}
static void UsbFormatClassSubProt(char* buf, size_t cap, uint8_t cls, uint8_t sub, uint8_t proto) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_");
UsbIdAppendHex(buf, cap, &pos, cls, 2);
UsbIdAppendLiteral(buf, cap, &pos, "&SUBCLASS_");
UsbIdAppendHex(buf, cap, &pos, sub, 2);
UsbIdAppendLiteral(buf, cap, &pos, "&PROT_");
UsbIdAppendHex(buf, cap, &pos, proto, 2);
}
static void UsbFormatClass(char* buf, size_t cap, uint8_t cls) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\CLASS_");
UsbIdAppendHex(buf, cap, &pos, cls, 2);
}
static void UsbFormatComposite(char* buf, size_t cap) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\COMPOSITE");
}
static void UsbFormatPciXhci(char* buf, size_t cap, uint16_t vendor, uint16_t device) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "PCI\\VEN_");
UsbIdAppendHex(buf, cap, &pos, vendor, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&DEV_");
UsbIdAppendHex(buf, cap, &pos, device, 4);
UsbIdAppendLiteral(buf, cap, &pos, "&CC_0C0330");
}
static void UsbFormatRootHub(char* buf, size_t cap) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\ROOT_HUB");
}
static void UsbMakeDeviceIds(const XHCI_DEVICE* xdev, IO_DEVICE_ID* ids, char idbuf[][USB_MAX_DEVICE_ID_TEXT], size_t* outCount) {
size_t n = 0;
UsbFormatVidPidRev(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, xdev->BcdDevice);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatVidPid(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass, xdev->DeviceSubclass, xdev->DeviceProtocol);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->DeviceClass);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
if (xdev->DeviceClass == 0 || xdev->InterfaceCount > 1) {
UsbFormatComposite(idbuf[n], USB_MAX_DEVICE_ID_TEXT);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
}
*outCount = n;
}
static void UsbMakeInterfaceIds(const XHCI_DEVICE* xdev,
const USB_TRACKED_INTERFACE* iface,
IO_DEVICE_ID* ids,
char idbuf[][USB_MAX_DEVICE_ID_TEXT],
size_t* outCount) {
size_t n = 0;
const USB_INTERFACE_DESCRIPTOR* d = &iface->Descriptor;
UsbFormatVidPidMi(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, d->bInterfaceNumber);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClassSubProt(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass, d->bInterfaceSubClass, d->bInterfaceProtocol);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
UsbFormatClass(idbuf[n], USB_MAX_DEVICE_ID_TEXT, d->bInterfaceClass);
ids[n] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[n]) };
n++;
*outCount = n;
}
KSTATUS UsbInitializeBusModel(void) {
if (gUsbBusModelReady)
return STATUS_SUCCESS;
memset(&gUsbBusDriver, 0, sizeof(gUsbBusDriver));
IoInitializeDriverObject(&gUsbBusDriver, SYS_SV_LITERAL("usb-bus"));
gUsbBusDriver.MajorFunction[IRP_MJ_PNP] = UsbDispatchPdoPnp;
gUsbBusDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl;
KSTATUS st = IoRegisterDriverObject(&gUsbBusDriver);
if (!KSUCCESS(st)) return st;
gUsbBusModelReady = true;
return STATUS_SUCCESS;
}
void UsbInitializeHcd(USB_HCD* hcd,
const USB_HCD_OPERATIONS* ops,
void* controllerContext,
uint8_t controllerIndex,
uint8_t maxRootPorts) {
if (!hcd) return;
memset(hcd, 0, sizeof(*hcd));
hcd->Operations = ops;
hcd->ControllerContext = controllerContext;
hcd->DriverObject = &gUsbBusDriver;
hcd->ControllerIndex = controllerIndex ? controllerIndex : ++gUsbNextControllerIndex;
hcd->MaxRootPorts = maxRootPorts;
}
KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device) {
if (!hcd) return STATUS_INVALID_PARAMETER;
if (hcd->ControllerPdo) return STATUS_ALREADY_EXISTS;
IO_DEVICE_ID ids[2];
char idbuf[2][USB_MAX_DEVICE_ID_TEXT];
UsbFormatPciXhci(idbuf[0], USB_MAX_DEVICE_ID_TEXT, vendor, device);
{
size_t pos = 0;
idbuf[1][0] = '\0';
UsbIdAppendLiteral(idbuf[1], USB_MAX_DEVICE_ID_TEXT, &pos, "PCI\\CC_0C0330");
}
ids[0] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[0]) };
ids[1] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[1]) };
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver, (DEVICE_OBJECT*)0, ids, 2, sizeof(USB_PDO_EXTENSION), &pdo);
if (!KSUCCESS(st)) return st;
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("xhci-controller"));
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoController;
ext->Hcd = hcd;
hcd->ControllerPdo = pdo;
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
return STATUS_SUCCESS;
}
KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) {
if (!hcd) return STATUS_INVALID_PARAMETER;
if (hcd->RootHubPdo) return STATUS_ALREADY_EXISTS;
IO_DEVICE_ID ids[2];
char idbuf[2][USB_MAX_DEVICE_ID_TEXT];
UsbFormatRootHub(idbuf[0], USB_MAX_DEVICE_ID_TEXT);
UsbFormatClass(idbuf[1], USB_MAX_DEVICE_ID_TEXT, USB_CLASS_HUB);
ids[0] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[0]) };
ids[1] = (IO_DEVICE_ID){ IoBusUsb, UsbIdFromBuffer(idbuf[1]) };
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
hcd->ControllerPdo,
ids,
2,
sizeof(USB_PDO_EXTENSION),
&pdo);
if (!KSUCCESS(st)) return st;
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-root-hub"));
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoRootHub;
ext->Hcd = hcd;
ext->Device = (USB_DEVICE*)0;
hcd->RootHubPdo = pdo;
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
DbgPrint("usb: created root hub PDO ports=%u\n", hcd->MaxRootPorts);
return STATUS_SUCCESS;
}
KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
if (!hcd || !xdev) return STATUS_INVALID_PARAMETER;
if (xdev->UsbPdo) return STATUS_ALREADY_EXISTS;
USB_DEVICE* udev = UsbAllocateDevice();
if (!udev) return STATUS_NO_MEMORY;
udev->Hcd = hcd;
udev->HcdDeviceContext = xdev;
udev->State = xdev->Configured ? UsbDeviceConfigured : (xdev->Addressed ? UsbDeviceAddressed : UsbDeviceAttached);
udev->SlotId = xdev->SlotId;
udev->Speed = xdev->Speed;
udev->Address = xdev->SlotId;
udev->MaxPacketSize0 = xdev->MaxPacketSize0;
udev->VendorId = xdev->VendorId;
udev->ProductId = xdev->ProductId;
udev->BcdUsb = xdev->BcdUsb;
udev->BcdDevice = xdev->BcdDevice;
udev->DeviceClass = xdev->DeviceClass;
udev->DeviceSubclass = xdev->DeviceSubclass;
udev->DeviceProtocol = xdev->DeviceProtocol;
udev->ConfigurationValue = xdev->ConfigurationValue;
udev->Path = xdev->Path;
if (udev->Path.RootPort == 0) {
udev->Path.RootPort = xdev->Port;
udev->Path.Depth = 0;
udev->Path.RouteString = 0;
}
IO_DEVICE_ID ids[5];
char idbuf[5][USB_MAX_DEVICE_ID_TEXT];
size_t idCount = 0;
UsbMakeDeviceIds(xdev, ids, idbuf, &idCount);
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
parent ? parent : (hcd->RootHubPdo ? hcd->RootHubPdo : hcd->ControllerPdo),
ids,
idCount,
sizeof(USB_PDO_EXTENSION),
&pdo);
if (!KSUCCESS(st)) {
UsbDereferenceDevice(udev);
return st;
}
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-device"));
udev->Pdo = pdo;
IoReferenceDeviceObject(pdo);
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoDevice;
ext->Hcd = hcd;
ext->Device = udev;
xdev->UsbDevice = udev;
xdev->UsbPdo = pdo;
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
DbgPrint("usb: created device PDO slot=%u vid=%04x pid=%04x ids=%llu\n",
xdev->SlotId, xdev->VendorId, xdev->ProductId, (unsigned long long)idCount);
return STATUS_SUCCESS;
}
KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
if (!hcd || !xdev || !xdev->UsbDevice) return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* deviceParent = parent ? parent : xdev->UsbPdo;
if (!deviceParent) return STATUS_INVALID_PARAMETER;
for (uint8_t i = 0; i < xdev->InterfaceCount; i++) {
USB_TRACKED_INTERFACE* iface = &xdev->Interfaces[i];
if (!iface->Present)
continue;
IO_DEVICE_ID ids[3];
char idbuf[3][USB_MAX_DEVICE_ID_TEXT];
size_t idCount = 0;
UsbMakeInterfaceIds(xdev, iface, ids, idbuf, &idCount);
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreatePhysicalDevice(&gUsbCompositeDriver,
deviceParent,
ids,
idCount,
sizeof(USB_PDO_EXTENSION),
&pdo);
if (!KSUCCESS(st)) {
DbgPrint("usb: create interface PDO slot=%u if=%u failed st=0x%x\n",
xdev->SlotId, iface->Descriptor.bInterfaceNumber, st);
return st;
}
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("usb-interface"));
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
ext->Kind = UsbPdoInterface;
ext->Hcd = hcd;
ext->Device = xdev->UsbDevice;
ext->InterfaceIndex = i;
memcpy(&ext->InterfaceDescriptor, &iface->Descriptor, sizeof(iface->Descriptor));
ext->EndpointCount = iface->EndpointCount;
for (uint8_t ep = 0; ep < iface->EndpointCount && ep < USB_MAX_TRACKED_ENDPOINTS; ep++)
memcpy(&ext->Endpoints[ep], &iface->Endpoints[ep], sizeof(USB_ENDPOINT_DESCRIPTOR));
st = IoRegisterPhysicalDevice(pdo);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
return st;
DbgPrint("usb: created interface PDO slot=%u if=%u class=%02x subclass=%02x proto=%02x\n",
xdev->SlotId,
iface->Descriptor.bInterfaceNumber,
iface->Descriptor.bInterfaceClass,
iface->Descriptor.bInterfaceSubClass,
iface->Descriptor.bInterfaceProtocol);
}
return STATUS_SUCCESS;
}
static DEVICE_OBJECT* UsbReferenceFirstChild(DEVICE_OBJECT* parent) {
if (!parent) return (DEVICE_OBJECT*)0;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
DEVICE_OBJECT* child = parent->FirstChild;
if (child)
IoReferenceDeviceObject(child);
KeReleaseSpinlock(&parent->Lock, flags);
return child;
}
static void UsbSendPnpIfAlive(DEVICE_OBJECT* pdo, IO_PNP_MINOR_FUNCTION minor) {
if (!pdo || pdo->DeletePending)
return;
(void)IoSendPnpIrpSync(pdo, minor);
}
KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) {
if (!parent)
return STATUS_INVALID_PARAMETER;
for (;;) {
DEVICE_OBJECT* child = UsbReferenceFirstChild(parent);
if (!child)
break;
if (child->Type == DeviceObjectPhysical)
(void)UsbRemovePdoSubtree(child, surprise);
else
(void)UsbRemoveDeviceObjectChildren(child, surprise);
IoDereferenceDeviceObject(child);
}
return STATUS_SUCCESS;
}
KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise) {
if (!pdo || pdo->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
// Child PDOs may be owned either directly by the PDO or by an attached FDO
// such as usb-composite or usb-hub. The attached driver's PnP handler is
// therefore responsible for removing its own child PDOs before forwarding
// REMOVE_DEVICE to the lower PDO.
(void)UsbRemoveDeviceObjectChildren(pdo, surprise);
if (surprise)
UsbSendPnpIfAlive(pdo, IRP_MN_SURPRISE_REMOVAL);
UsbSendPnpIfAlive(pdo, IRP_MN_REMOVE_DEVICE);
return STATUS_SUCCESS;
}
static KSTATUS UsbSubmitUrbToHcd(USB_PDO_EXTENSION* ext, USB_URB* urb) {
if (!ext || !urb) return STATUS_INVALID_PARAMETER;
USB_HCD* hcd = ext->Hcd;
if (!hcd || !hcd->Operations || !hcd->Operations->SubmitUrb)
return STATUS_NOT_SUPPORTED;
if (!urb->Device)
urb->Device = ext->Device;
if (!urb->Device)
return STATUS_INVALID_PARAMETER;
return hcd->Operations->SubmitUrb(hcd, urb);
}
KSTATUS UsbCancelUrb(USB_URB* urb) {
if (!urb || !urb->Device || !urb->Device->Hcd)
return STATUS_INVALID_PARAMETER;
USB_HCD* hcd = urb->Device->Hcd;
if (!hcd->Operations || !hcd->Operations->CancelUrb)
return STATUS_NOT_SUPPORTED;
return hcd->Operations->CancelUrb(hcd, urb);
}
KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb) {
if (!target || !urb) return STATUS_INVALID_PARAMETER;
IRP irp;
IoInitializeIrp(&irp, 1);
IoBuildDeviceIoControlIrp(&irp,
IRP_MJ_INTERNAL_DEVICE_CONTROL,
USB_IOCTL_SUBMIT_URB,
urb,
sizeof(*urb),
urb,
sizeof(*urb));
KSTATUS st = IoSubmitIrpSync(target, &irp);
return KSUCCESS(st) ? urb->Status : st;
}
static void UsbAsyncUrbIrpComplete(IRP* irp, void* context) {
(void)context;
IoFreeIrp(irp);
}
KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb) {
if (!target || !urb) return STATUS_INVALID_PARAMETER;
IRP* irp = (IRP*)0;
KSTATUS st = IoAllocateIrp(1, &irp);
if (!KSUCCESS(st)) return st;
IoBuildDeviceIoControlIrp(irp,
IRP_MJ_INTERNAL_DEVICE_CONTROL,
USB_IOCTL_SUBMIT_URB,
urb,
sizeof(*urb),
urb,
sizeof(*urb));
IoSetCompletionRoutine(irp, UsbAsyncUrbIrpComplete, (void*)0);
st = IoSubmitIrp(target, irp);
if (st != STATUS_PENDING && !(irp->Flags & IRP_FLAG_COMPLETED)) {
IoFreeIrp(irp);
return st;
}
return st;
}
KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return STATUS_INVALID_PARAMETER;
if (ios->Parameters.DeviceIoControl.IoControlCode != USB_IOCTL_SUBMIT_URB) {
IoCompleteRequest(irp, STATUS_NOT_SUPPORTED, 0);
return STATUS_PENDING;
}
if (ios->Parameters.DeviceIoControl.InputLength < sizeof(USB_URB) ||
!ios->Parameters.DeviceIoControl.InputBuffer) {
IoCompleteRequest(irp, STATUS_INVALID_PARAMETER, 0);
return STATUS_PENDING;
}
USB_URB* urb = (USB_URB*)ios->Parameters.DeviceIoControl.InputBuffer;
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension;
urb->Irp = irp;
KSTATUS st = UsbSubmitUrbToHcd(ext, urb);
urb->Status = st;
if (st == STATUS_PENDING) {
urb->Pending = true;
return STATUS_PENDING;
}
urb->Pending = false;
IoCompleteRequest(irp, st, KSUCCESS(st) ? urb->ActualLength : 0);
return STATUS_PENDING;
}
KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)device->DeviceExtension;
KSTATUS st = STATUS_SUCCESS;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
if (ext && ext->Device && ext->Device->State < UsbDeviceConfigured)
ext->Device->State = UsbDeviceConfigured;
break;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
break;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
break;
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
if (ext && ext->Device)
ext->Device->State = UsbDeviceSurpriseRemoved;
break;
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
if (ext && ext->Device)
ext->Device->State = UsbDeviceRemoved;
IoDeleteDevice(device);
break;
default:
st = STATUS_NOT_SUPPORTED;
break;
}
if (st == STATUS_NOT_SUPPORTED)
return st;
IoCompleteRequest(irp, st, 0);
return STATUS_PENDING;
}
code/drivers/bus/usb/usb_composite.c
+150
View File
@@ -0,0 +1,150 @@
// usb_composite.c - USB composite function driver and interface-PDO bus.
#include "usb.h"
DRIVER_OBJECT gUsbCompositeDriver;
static const IO_DEVICE_ID gUsbCompositeIds[] = {
{ IoBusUsb, { "USB\\COMPOSITE", sizeof("USB\\COMPOSITE") - 1 } },
};
static const IO_DRIVER_ID_TABLE gUsbCompositeIdTable = {
gUsbCompositeIds,
sizeof(gUsbCompositeIds) / sizeof(gUsbCompositeIds[0]),
};
static KSTATUS UsbCompositeCreateInterfaces(USB_COMPOSITE_EXTENSION* ext) {
if (!ext || !ext->Device || ext->InterfacesCreated)
return STATUS_SUCCESS;
XHCI_DEVICE* xdev = (XHCI_DEVICE*)ext->Device->HcdDeviceContext;
if (!xdev)
return STATUS_INVALID_PARAMETER;
KSTATUS st = UsbCreateInterfacePdos(ext->Device->Hcd, xdev, ext->Self);
if (KSUCCESS(st))
ext->InterfacesCreated = true;
return st;
}
static KSTATUS UsbCompositeForwardPnp(DEVICE_OBJECT* device, IRP* irp, KSTATUS localStatus) {
if (!KSUCCESS(localStatus)) {
IoCompleteRequest(irp, localStatus, 0);
return STATUS_PENDING;
}
return IoForwardIrpToLower(device, irp);
}
KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
// The composite driver owns two different device-object roles:
// * an FDO attached above a USB device PDO, and
// * interface PDOs that it creates for class drivers.
// Forwarded PnP IRPs from HID/storage land on the interface PDO, so the
// PDO must use the generic USB PDO PnP handler. Treating its
// USB_PDO_EXTENSION as USB_COMPOSITE_EXTENSION corrupts the dispatch path.
if (device->Type == DeviceObjectPhysical)
return UsbDispatchPdoPnp(device, irp);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)device->DeviceExtension;
KSTATUS st = STATUS_SUCCESS;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
if (ext) {
if (ext->DevicePdo)
ext->DevicePdo->PnpState = DevicePnpStarted;
ext->Started = true;
st = UsbCompositeCreateInterfaces(ext);
}
return UsbCompositeForwardPnp(device, irp, st);
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return IoForwardIrpToLower(device, irp);
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
return IoForwardIrpToLower(device, irp);
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
if (ext)
ext->Started = false;
(void)UsbRemoveDeviceObjectChildren(device, true);
return IoForwardIrpToLower(device, irp);
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
if (ext) {
ext->Started = false;
(void)UsbRemoveDeviceObjectChildren(device, false);
if (ext->LowerDevice) {
KSTATUS lower = IoForwardIrpToLower(device, irp);
IoDetachDevice(device);
ext->LowerDevice = (DEVICE_OBJECT*)0;
IoDeleteDevice(device);
if (lower == STATUS_NOT_SUPPORTED) {
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
}
if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
IoCompleteRequest(irp, lower, 0);
return STATUS_PENDING;
}
return lower;
}
}
IoDeleteDevice(device);
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
default:
return IoForwardIrpToLower(device, irp);
}
}
KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoDevice || !pdoExt->Device)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(driver, sizeof(USB_COMPOSITE_EXTENSION), &fdo);
if (!KSUCCESS(st))
return st;
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-composite"));
USB_COMPOSITE_EXTENSION* ext = (USB_COMPOSITE_EXTENSION*)fdo->DeviceExtension;
ext->Self = fdo;
ext->DevicePdo = physicalDeviceObject;
ext->Device = pdoExt->Device;
ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!ext->LowerDevice) {
IoDeleteDevice(fdo);
return STATUS_FAILURE;
}
DbgPrint("usb: composite attached slot=%u\n", ext->Device->SlotId);
return STATUS_SUCCESS;
}
KSTATUS UsbInitializeCompositeDriver(void) {
memset(&gUsbCompositeDriver, 0, sizeof(gUsbCompositeDriver));
IoInitializeDriverObject(&gUsbCompositeDriver, SYS_SV_LITERAL("usb-composite"));
gUsbCompositeDriver.IdTable = &gUsbCompositeIdTable;
gUsbCompositeDriver.DriverAddDevice = UsbCompositeAddDevice;
gUsbCompositeDriver.MajorFunction[IRP_MJ_PNP] = UsbCompositeDispatchPnp;
gUsbCompositeDriver.MajorFunction[IRP_MJ_INTERNAL_DEVICE_CONTROL] = UsbDispatchInternalDeviceControl;
return IoRegisterDriverObject(&gUsbCompositeDriver);
}
code/drivers/bus/usb/usb_core.c
+190 -99
View File
@@ -25,6 +25,16 @@ KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
static USB_TRACKED_INTERFACE* UsbFindTrackedInterfaceByNumber(XHCI_DEVICE* dev, uint8_t interfaceNumber) {
if (!dev) return (USB_TRACKED_INTERFACE*)0;
for (uint8_t i = 0; i < dev->InterfaceCount && i < USB_MAX_TRACKED_INTERFACES; i++) {
if (dev->Interfaces[i].Present &&
dev->Interfaces[i].Descriptor.bInterfaceNumber == interfaceNumber)
return &dev->Interfaces[i];
}
return (USB_TRACKED_INTERFACE*)0;
}
void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_CONFIGURATION_DESCRIPTOR* config,
uint16_t totalLength) {
@@ -79,17 +89,38 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
massBulkOut = 0;
tracked = (USB_TRACKED_INTERFACE*)0;
if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) {
tracked = &dev->Interfaces[dev->InterfaceCount++];
memset(tracked, 0, sizeof(*tracked));
tracked->Present = true;
memcpy(&tracked->Descriptor, iface, sizeof(*iface));
if (iface->bAlternateSetting == 0) {
USB_TRACKED_INTERFACE* existing = UsbFindTrackedInterfaceByNumber(dev, iface->bInterfaceNumber);
if (existing) {
tracked = existing;
memset(tracked->Endpoints, 0, sizeof(tracked->Endpoints));
tracked->EndpointCount = 0;
memcpy(&tracked->Descriptor, iface, sizeof(*iface));
UsbTrace("usb: slot=%u duplicate alt0 interface %u replaced existing PDO candidate\n",
dev->SlotId, iface->bInterfaceNumber);
} else if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) {
tracked = &dev->Interfaces[dev->InterfaceCount++];
memset(tracked, 0, sizeof(*tracked));
tracked->Present = true;
memcpy(&tracked->Descriptor, iface, sizeof(*iface));
} else {
DbgPrint("usb: slot=%u too many interfaces, interface %u alt=0 not published\n",
dev->SlotId, iface->bInterfaceNumber);
}
} else {
DbgPrint("usb: slot=%u too many interfaces, interface %u not published\n",
dev->SlotId, iface->bInterfaceNumber);
/*
* Alternate settings are not independent USB interfaces. Real
* UVC cameras often expose one VideoStreaming interface with
* many alternate settings, and publishing each alternate setting
* as a separate PDO causes duplicate interface devices and can
* exhaust USB_MAX_TRACKED_INTERFACES. Track only alternate
* setting zero until a function driver explicitly selects a
* different setting.
*/
tracked = (USB_TRACKED_INTERFACE*)0;
}
UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)\n",
UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)%s\n",
dev->SlotId,
iface->bInterfaceNumber,
iface->bAlternateSetting,
@@ -97,7 +128,8 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
currentClass,
currentSubClass,
currentProtocol,
UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol));
UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol),
iface->bAlternateSetting ? " alt-not-published" : "");
} else if (type == USB_DESC_ENDPOINT && len >= sizeof(USB_ENDPOINT_DESCRIPTOR)) {
const USB_ENDPOINT_DESCRIPTOR* ep = (const USB_ENDPOINT_DESCRIPTOR*)desc;
uint8_t epType = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
@@ -178,54 +210,6 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
for (uint8_t i = 0; i < dev->InterfaceCount; i++) {
USB_TRACKED_INTERFACE* iface = &dev->Interfaces[i];
if (!iface->Present)
continue;
IO_PUBLISHED_DEVICE pub;
memset(&pub, 0, sizeof(pub));
pub.BusType = IoBusUsb;
pub.Name = "usb-interface";
pub.BusContext = xhci;
pub.DeviceContext = dev;
pub.u.Usb.VendorId = dev->VendorId;
pub.u.Usb.ProductId = dev->ProductId;
pub.u.Usb.BcdUsb = dev->BcdUsb;
pub.u.Usb.BcdDevice = dev->BcdDevice;
pub.u.Usb.DeviceClass = dev->DeviceClass;
pub.u.Usb.DeviceSubclass = dev->DeviceSubclass;
pub.u.Usb.DeviceProtocol = dev->DeviceProtocol;
pub.u.Usb.SlotId = dev->SlotId;
pub.u.Usb.Port = dev->Port;
pub.u.Usb.Speed = dev->Speed;
pub.u.Usb.ConfigurationValue = dev->ConfigurationValue;
pub.u.Usb.InterfaceNumber = iface->Descriptor.bInterfaceNumber;
pub.u.Usb.AlternateSetting = iface->Descriptor.bAlternateSetting;
pub.u.Usb.InterfaceClass = iface->Descriptor.bInterfaceClass;
pub.u.Usb.InterfaceSubclass = iface->Descriptor.bInterfaceSubClass;
pub.u.Usb.InterfaceProtocol = iface->Descriptor.bInterfaceProtocol;
pub.u.Usb.EndpointCount = iface->EndpointCount;
if (pub.u.Usb.EndpointCount > IO_PUBLISHED_USB_MAX_ENDPOINTS)
pub.u.Usb.EndpointCount = IO_PUBLISHED_USB_MAX_ENDPOINTS;
for (uint8_t ep = 0; ep < pub.u.Usb.EndpointCount; ep++) {
pub.u.Usb.Endpoints[ep].Address = iface->Endpoints[ep].bEndpointAddress;
pub.u.Usb.Endpoints[ep].Attributes = iface->Endpoints[ep].bmAttributes;
pub.u.Usb.Endpoints[ep].MaxPacketSize = iface->Endpoints[ep].wMaxPacketSize;
pub.u.Usb.Endpoints[ep].Interval = iface->Endpoints[ep].bInterval;
}
KSTATUS st = IoPublishDevice(&pub);
if (!KSUCCESS(st)) {
DbgPrint("usb: publish interface slot=%u if=%u failed st=0x%x\n",
dev->SlotId, iface->Descriptor.bInterfaceNumber, st);
}
}
}
KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_DEVICE_DESCRIPTOR* devDesc) {
@@ -255,43 +239,89 @@ KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
st = XhciSetConfiguration(xhci, dev, config.bConfigurationValue);
if (!KSUCCESS(st)) return st;
XhciPublishUsbInterfaces(xhci, dev);
if (!dev->UsbPdo) {
st = UsbCreateDevicePdo(&xhci->Hcd, dev,
dev->ParentDeviceObject ? dev->ParentDeviceObject :
(xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : xhci->Hcd.ControllerPdo));
if (!KSUCCESS(st)) {
DbgPrint("usb: create device PDO slot=%u failed st=0x%x\n", dev->SlotId, st);
return st;
}
}
return STATUS_SUCCESS;
}
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
uint8_t speed = 0;
KSTATUS st = XhciResetPort(xhci, port, &speed);
if (!KSUCCESS(st)) return st;
static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS)
return;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
if (dev->UsbPdo) {
(void)UsbRemovePdoSubtree(dev->UsbPdo, true);
dev->UsbPdo = (DEVICE_OBJECT*)0;
}
if (dev->UsbDevice) {
dev->UsbDevice->State = UsbDeviceRemoved;
UsbDereferenceDevice(dev->UsbDevice);
dev->UsbDevice = (USB_DEVICE*)0;
}
if (dev->Allocated)
(void)XhciDisableSlot(xhci, slotId);
memset(dev, 0, sizeof(*dev));
}
static KSTATUS XhciEnumeratePathEx(XHCI_CONTROLLER* xhci,
uint8_t logicalPort,
uint8_t speed,
const USB_PORT_PATH* path,
DEVICE_OBJECT* parentDevice,
uint8_t* outSlotId) {
if (outSlotId) *outSlotId = 0;
if (!xhci || !path || path->RootPort == 0)
return STATUS_INVALID_PARAMETER;
uint8_t slotId = 0;
st = XhciEnableSlot(xhci, &slotId);
KSTATUS st = XhciEnableSlot(xhci, &slotId);
if (!KSUCCESS(st)) return st;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
memset(dev, 0, sizeof(*dev));
dev->Allocated = true;
dev->SlotId = slotId;
dev->Port = port;
dev->Port = path->RootPort;
dev->Speed = speed;
dev->Path = *path;
dev->ParentDeviceObject = parentDevice;
st = XhciAllocateDeviceMemory(xhci, dev);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
XhciBuildDefaultControlContext(xhci, dev);
st = XhciAddressDevice(xhci, dev);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
dev->Addressed = true;
USB_DEVICE_DESCRIPTOR desc;
memset(&desc, 0, sizeof(desc));
st = XhciGetDeviceDescriptor(xhci, dev, &desc);
st = XhciGetDeviceDescriptorTimeout(xhci, dev, &desc, USB_XHCI_ENUM_CONTROL_TIMEOUT_US);
if (!KSUCCESS(st)) {
DbgPrint("usb: port %u slot %u descriptor read failed st=0x%x\n",
port, slotId, st);
logicalPort, slotId, st);
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
@@ -304,24 +334,100 @@ KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
dev->DeviceSubclass = desc.bDeviceSubClass;
dev->DeviceProtocol = desc.bDeviceProtocol;
DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n",
port,
slotId,
XhciPortSpeedName(speed),
speed,
desc.idVendor,
desc.idProduct,
desc.bDeviceClass,
desc.bDeviceSubClass,
desc.bDeviceProtocol,
desc.bMaxPacketSize0,
desc.bNumConfigurations,
desc.bcdUSB);
if (path->BehindHub) {
DbgPrint("usb: device root-port=%u hub-slot=%u hub-port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x route=%05x\n",
path->RootPort,
path->ParentHubSlot,
path->ParentHubPort,
slotId,
XhciPortSpeedName(speed),
speed,
desc.idVendor,
desc.idProduct,
desc.bDeviceClass,
desc.bDeviceSubClass,
desc.bDeviceProtocol,
desc.bMaxPacketSize0,
desc.bNumConfigurations,
desc.bcdUSB,
path->RouteString);
} else {
DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n",
logicalPort,
slotId,
XhciPortSpeedName(speed),
speed,
desc.idVendor,
desc.idProduct,
desc.bDeviceClass,
desc.bDeviceSubClass,
desc.bDeviceProtocol,
desc.bMaxPacketSize0,
desc.bNumConfigurations,
desc.bcdUSB);
}
st = XhciDiscoverAndSetConfiguration(xhci, dev, &desc);
if (!KSUCCESS(st)) {
XhciRollbackEnumeratedSlot(xhci, slotId);
return st;
}
if (outSlotId) *outSlotId = slotId;
return STATUS_SUCCESS;
}
KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId) {
if (outSlotId) *outSlotId = 0;
uint8_t speed = 0;
KSTATUS st = XhciResetPort(xhci, port, &speed);
if (!KSUCCESS(st)) return st;
return STATUS_SUCCESS;
USB_PORT_PATH path;
memset(&path, 0, sizeof(path));
path.RootPort = port;
path.Depth = 0;
path.RouteString = 0;
path.BehindHub = false;
DEVICE_OBJECT* parent = xhci && xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : (DEVICE_OBJECT*)0;
return XhciEnumeratePathEx(xhci, port, speed, &path, parent, outSlotId);
}
KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci,
USB_HUB_EXTENSION* hub,
uint8_t hubPort,
uint8_t speed,
uint8_t* outSlotId) {
if (outSlotId) *outSlotId = 0;
if (!xhci || !hub || !hub->Device || !hub->Self || hubPort == 0)
return STATUS_INVALID_PARAMETER;
USB_DEVICE* parent = hub->Device;
if (parent->Path.Depth >= 5u) {
DbgPrint("usb-hub: hub slot=%u port=%u exceeds xHCI route depth\n",
parent->SlotId, hubPort);
return STATUS_NOT_SUPPORTED;
}
if (hubPort > 15u) {
DbgPrint("usb-hub: hub slot=%u port=%u cannot be represented in xHCI route string yet\n",
parent->SlotId, hubPort);
return STATUS_NOT_SUPPORTED;
}
USB_PORT_PATH path = parent->Path;
path.BehindHub = true;
path.ParentHubSlot = parent->SlotId;
path.ParentHubPort = hubPort;
path.RouteString |= ((uint32_t)hubPort << (path.Depth * 4u));
path.Depth++;
return XhciEnumeratePathEx(xhci, hubPort, speed, &path, hub->Self, outSlotId);
}
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
return XhciEnumeratePortEx(xhci, port, (uint8_t*)0);
}
@@ -335,7 +441,8 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
continue;
attempted++;
KSTATUS st = XhciEnumeratePort(xhci, port);
uint8_t slot = 0;
KSTATUS st = XhciEnumeratePortEx(xhci, port, &slot);
if (KSUCCESS(st)) configured++;
else DbgPrint("usb: root-port %u enumeration failed st=0x%x\n", port, st);
}
@@ -343,19 +450,3 @@ void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
DbgPrint("usb: xHCI root-port enumeration complete; attempted=%u configured=%u\n",
attempted, configured);
}
KSTATUS UsbRegisterClassMatchers(void) {
memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
gUsbStorageDriver.Name = "usbstor";
gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
KSTATUS st = IoRegisterDriverMatcher("usbhid", IoBusUsb, UsbHidMatch, UsbHidBind, NULL);
if (!KSUCCESS(st)) return st;
st = IoRegisterDriverMatcher("usbstor", IoBusUsb, UsbStorageMatch, UsbStorageBind, NULL);
if (!KSUCCESS(st)) return st;
return STATUS_SUCCESS;
}
code/drivers/bus/usb/usb_hid.c
+412 -4
View File
@@ -125,7 +125,7 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return;
@@ -158,11 +158,10 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
if (xhci->HidDeviceCount == 0)
if (!xhci || xhci->HidDeviceCount == 0)
return STATUS_SUCCESS;
XhciArmAllHid(xhci);
DbgPrint("usb: HID interrupt input armed devices=%u\n", xhci->HidDeviceCount);
DbgPrint("usb: HID interrupt URBs active devices=%u\n", xhci->HidDeviceCount);
return STATUS_SUCCESS;
}
@@ -365,3 +364,412 @@ KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context) {
}
return st;
}
// -------------------------------------------------------------------------
// PDO/FDO boot HID function driver path.
// -------------------------------------------------------------------------
DRIVER_OBJECT gUsbHidDriver;
static const IO_DEVICE_ID gUsbHidIds[] = {
{ IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_01", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_01") - 1 } },
{ IoBusUsb, { "USB\\CLASS_03&SUBCLASS_01&PROT_02", sizeof("USB\\CLASS_03&SUBCLASS_01&PROT_02") - 1 } },
};
static const IO_DRIVER_ID_TABLE gUsbHidIdTable = {
gUsbHidIds,
sizeof(gUsbHidIds) / sizeof(gUsbHidIds[0]),
};
static const USB_ENDPOINT_DESCRIPTOR* UsbHidFindInterruptInPdo(const USB_PDO_EXTENSION* ext) {
if (!ext) return (const USB_ENDPOINT_DESCRIPTOR*)0;
for (uint8_t i = 0; i < ext->EndpointCount; i++) {
const USB_ENDPOINT_DESCRIPTOR* ep = &ext->Endpoints[i];
uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0;
if (type == USB_ENDPOINT_TYPE_INTERRUPT && in)
return ep;
}
return (const USB_ENDPOINT_DESCRIPTOR*)0;
}
static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 8) return;
uint8_t modifiers = report[0];
for (uint8_t bit = 0; bit < 8; bit++) {
uint8_t mask = (uint8_t)(1u << bit);
bool wasDown = (ext->LastModifiers & mask) != 0;
bool isDown = (modifiers & mask) != 0;
if (wasDown != isDown)
XhciSubmitHidKey(XhciHidModifierKey(bit), isDown);
}
const uint8_t* keys = &report[2];
for (uint8_t i = 0; i < 6; i++) {
uint8_t oldKey = ext->LastKeys[i];
if (oldKey > 1 && !XhciHidKeyInReport(keys, oldKey))
XhciSubmitHidKey((INPUT_KEY)oldKey, false);
}
for (uint8_t i = 0; i < 6; i++) {
uint8_t newKey = keys[i];
if (newKey > 1 && !XhciHidKeyInReport(ext->LastKeys, newKey))
XhciSubmitHidKey((INPUT_KEY)newKey, true);
}
ext->LastModifiers = modifiers;
memcpy(ext->LastKeys, keys, sizeof(ext->LastKeys));
}
static void UsbHidProcessBootMouseReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 3) return;
int dx = (int)(int8_t)report[1];
int dy = (int)(int8_t)report[2];
int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0;
uint8_t buttons = report[0] & 0x07u;
if (dx || dy || wheel || buttons != ext->LastButtons) {
INPUT_MOUSE_EVENT ev;
ev.dx = dx;
ev.dy = dy;
ev.wheel = wheel;
ev.buttons = buttons;
KeInputSubmitMouse(&ev);
}
ext->LastButtons = buttons;
}
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext);
static void UsbHidInterruptComplete(USB_URB* urb, void* context) {
USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)context;
if (!ext || urb != &ext->InterruptUrb)
return;
ext->InterruptPending = false;
if (ext->SurpriseRemoved || ext->Removed || !ext->Started)
return;
if (KSUCCESS(urb->Status)) {
uint16_t length = (uint16_t)urb->ActualLength;
if (length > ext->ReportLength) length = ext->ReportLength;
if (ext->Keyboard)
UsbHidProcessBootKeyboardReport(ext, ext->ReportBuffer, length);
else if (ext->Mouse)
UsbHidProcessBootMouseReport(ext, ext->ReportBuffer, length);
} else if (urb->Status != STATUS_CANCELLED) {
DbgPrint("usb: HID interrupt URB failed slot=%u if=%u ep=%02x st=0x%x\n",
ext->Device ? ext->Device->SlotId : 0,
ext->InterfaceNumber,
ext->EndpointAddress,
urb->Status);
}
if (!ext->SurpriseRemoved && !ext->Removed)
(void)UsbHidArmInterrupt(ext);
}
static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) {
if (!ext || !ext->LowerDevice || !ext->Device)
return STATUS_INVALID_PARAMETER;
USB_URB urb;
memset(&urb, 0, sizeof(urb));
urb.Function = UrbFunctionControlTransfer;
urb.Direction = UsbTransferNoData;
urb.Device = ext->Device;
urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
urb.Setup.bRequest = USB_REQ_SET_PROTOCOL;
urb.Setup.wValue = 0;
urb.Setup.wIndex = ext->InterfaceNumber;
urb.Setup.wLength = 0;
KSTATUS st = UsbSubmitUrbSync(ext->LowerDevice, &urb);
if (!KSUCCESS(st)) {
DbgPrint("usb: HID SET_PROTOCOL boot failed slot=%u if=%u st=0x%x\n",
ext->Device->SlotId, ext->InterfaceNumber, st);
return st;
}
memset(&urb, 0, sizeof(urb));
urb.Function = UrbFunctionControlTransfer;
urb.Direction = UsbTransferNoData;
urb.Device = ext->Device;
urb.Setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
urb.Setup.bRequest = USB_REQ_SET_IDLE;
urb.Setup.wValue = 0;
urb.Setup.wIndex = ext->InterfaceNumber;
urb.Setup.wLength = 0;
st = UsbSubmitUrbSync(ext->LowerDevice, &urb);
if (!KSUCCESS(st)) {
DbgPrint("usb: HID SET_IDLE failed slot=%u if=%u st=0x%x; continuing\n",
ext->Device->SlotId, ext->InterfaceNumber, st);
}
DbgPrint("usb: HID boot protocol selected slot=%u if=%u proto=%u\n",
ext->Device->SlotId, ext->InterfaceNumber, ext->Protocol);
return STATUS_SUCCESS;
}
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed)
return STATUS_OBJECT_DEAD;
if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys)
return STATUS_INVALID_PARAMETER;
if (ext->InterruptPending)
return STATUS_SUCCESS;
memset(ext->ReportBuffer, 0, ext->ReportLength);
memset(&ext->InterruptUrb, 0, sizeof(ext->InterruptUrb));
ext->InterruptUrb.Function = UrbFunctionInterruptTransfer;
ext->InterruptUrb.Direction = UsbTransferIn;
ext->InterruptUrb.Device = ext->Device;
ext->InterruptUrb.Pipe = ext->InterruptPipe;
ext->InterruptUrb.Buffer = ext->ReportBuffer;
ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys;
ext->InterruptUrb.Length = ext->ReportLength;
ext->InterruptUrb.Complete = UsbHidInterruptComplete;
ext->InterruptUrb.CompleteContext = ext;
KSTATUS st = UsbSubmitUrbAsync(ext->LowerDevice, &ext->InterruptUrb);
if (st != STATUS_PENDING && !KSUCCESS(st))
return st;
if (ext->InterruptUrb.Status != STATUS_PENDING && ext->InterruptUrb.Status != STATUS_SUCCESS)
return ext->InterruptUrb.Status;
ext->InterruptPending = true;
return STATUS_SUCCESS;
}
static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) {
if (!device || !ext || !ext->Device || !ext->InterfacePdo)
return STATUS_INVALID_PARAMETER;
if (ext->Started)
return STATUS_SUCCESS;
if (ext->SurpriseRemoved || ext->Removed)
return STATUS_OBJECT_DEAD;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)ext->InterfacePdo->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface)
return STATUS_INVALID_PARAMETER;
const USB_ENDPOINT_DESCRIPTOR* ep = UsbHidFindInterruptInPdo(pdoExt);
if (!ep)
return STATUS_NOT_FOUND;
ext->Keyboard = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD);
ext->Mouse = (pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_MOUSE);
ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
ext->Protocol = pdoExt->InterfaceDescriptor.bInterfaceProtocol;
ext->EndpointAddress = ep->bEndpointAddress;
ext->MaxPacketSize = ep->wMaxPacketSize;
ext->Interval = ep->bInterval;
uint16_t reportLength = ext->MaxPacketSize & 0x07FFu;
if (reportLength == 0) reportLength = ext->Keyboard ? 8u : 4u;
if (ext->Keyboard && reportLength < 8) reportLength = 8;
if (ext->Mouse && reportLength < 3) reportLength = 3;
if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT;
ext->ReportLength = reportLength;
ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys);
if (!ext->ReportBuffer)
return STATUS_NO_MEMORY;
KSTATUS st = UsbCreatePipe(ext->Device,
UsbPipeInterrupt,
ext->EndpointAddress,
ext->MaxPacketSize,
ext->Interval,
&ext->InterruptPipe);
if (!KSUCCESS(st))
return st;
st = UsbHidSendBootProtocol(ext);
if (!KSUCCESS(st))
return st;
memset(ext->LastKeys, 0, sizeof(ext->LastKeys));
ext->LastModifiers = 0;
ext->LastButtons = 0;
KeInputRegisterDevice(ext->Keyboard ? "usb-hid-keyboard" : "usb-hid-mouse", 0, &ext->InputDeviceId);
ext->Started = true;
device->PnpState = DevicePnpStarted;
if (ext->InterfacePdo)
ext->InterfacePdo->PnpState = DevicePnpStarted;
XHCI_CONTROLLER* xhci = ext->Device && ext->Device->Hcd ?
(XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
st = UsbHidArmInterrupt(ext);
if (!KSUCCESS(st))
return st;
if (xhci)
xhci->HidDeviceCount++;
DbgPrint("usb: HID boot %s bound slot=%u if=%u ep=%02x maxpkt=%u interval=%u input=%u urb\n",
ext->Keyboard ? "keyboard" : "mouse",
ext->Device->SlotId,
ext->InterfaceNumber,
ext->EndpointAddress,
ext->MaxPacketSize,
ext->Interval,
ext->InputDeviceId);
return STATUS_SUCCESS;
}
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)device->DeviceExtension;
KSTATUS st = STATUS_SUCCESS;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
st = UsbHidStartDevice(device, ext);
if (!KSUCCESS(st)) {
IoCompleteRequest(irp, st, 0);
return STATUS_PENDING;
}
return IoForwardIrpToLower(device, irp);
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return IoForwardIrpToLower(device, irp);
case IRP_MN_CANCEL_REMOVE_DEVICE:
if (!ext || !ext->SurpriseRemoved)
device->PnpState = DevicePnpStarted;
return IoForwardIrpToLower(device, irp);
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
if (ext) {
ext->SurpriseRemoved = true;
ext->Started = false;
if (ext->InterruptPending)
(void)UsbCancelUrb(&ext->InterruptUrb);
}
return IoForwardIrpToLower(device, irp);
case IRP_MN_REMOVE_DEVICE: {
device->PnpState = DevicePnpDeleted;
if (ext) {
ext->Removed = true;
ext->Started = false;
if (ext->InterruptPending)
(void)UsbCancelUrb(&ext->InterruptUrb);
if (ext->InterruptPipe) {
ext->InterruptPipe->Open = false;
UsbDereferencePipe(ext->InterruptPipe);
ext->InterruptPipe = (USB_PIPE*)0;
}
if (ext->Device && ext->Device->Hcd) {
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)ext->Device->Hcd->ControllerContext;
if (xhci && xhci->HidDeviceCount)
xhci->HidDeviceCount--;
}
if (ext->RemoveLockReady) {
IoReleaseRemoveLockAndWait(&ext->RemoveLock, irp);
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
}
}
KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS;
if (ext && ext->LowerDevice) {
IoDetachDevice(device);
ext->LowerDevice = (DEVICE_OBJECT*)0;
}
IoDeleteDevice(device);
if (lower == STATUS_NOT_SUPPORTED) {
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
}
if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
IoCompleteRequest(irp, lower, 0);
return STATUS_PENDING;
}
return lower;
}
default:
return IoForwardIrpToLower(device, irp);
}
}
KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device)
return STATUS_INVALID_PARAMETER;
if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_HID ||
pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_HID_SUBCLASS_BOOT)
return STATUS_INVALID_PARAMETER;
if (pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_KEYBOARD &&
pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_HID_PROTOCOL_MOUSE)
return STATUS_INVALID_PARAMETER;
if (!UsbHidFindInterruptInPdo(pdoExt))
return STATUS_NOT_FOUND;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(driver, sizeof(USB_HID_EXTENSION), &fdo);
if (!KSUCCESS(st))
return st;
(void)IoSetDeviceName(fdo,
pdoExt->InterfaceDescriptor.bInterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD
? SYS_SV_LITERAL("usb-hid-keyboard")
: SYS_SV_LITERAL("usb-hid-mouse"));
USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)fdo->DeviceExtension;
ext->Self = fdo;
ext->InterfacePdo = physicalDeviceObject;
ext->Device = pdoExt->Device;
st = IoInitializeRemoveLock(&ext->RemoveLock);
if (!KSUCCESS(st)) {
IoDeleteDevice(fdo);
return st;
}
ext->RemoveLockReady = true;
ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!ext->LowerDevice) {
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
IoDeleteDevice(fdo);
return STATUS_FAILURE;
}
DbgPrint("usb: HID FDO attached slot=%u if=%u proto=%u\n",
ext->Device->SlotId,
pdoExt->InterfaceDescriptor.bInterfaceNumber,
pdoExt->InterfaceDescriptor.bInterfaceProtocol);
return STATUS_SUCCESS;
}
KSTATUS UsbInitializeHidDriver(void) {
memset(&gUsbHidDriver, 0, sizeof(gUsbHidDriver));
IoInitializeDriverObject(&gUsbHidDriver, SYS_SV_LITERAL("usb-hid"));
gUsbHidDriver.IdTable = &gUsbHidIdTable;
gUsbHidDriver.DriverAddDevice = UsbHidAddDevice;
gUsbHidDriver.MajorFunction[IRP_MJ_PNP] = UsbHidDispatchPnp;
return IoRegisterDriverObject(&gUsbHidDriver);
}
code/drivers/bus/usb/usb_hub.c
+1027
View File
File diff suppressed because it is too large. Load diff
code/drivers/bus/usb/usb_storage.c
+528 -153
View File
@@ -1,53 +1,115 @@
// usb_storage.c - USB mass-storage BOT/SCSI class binding and disk bridge.
// usb_storage.c - USB mass-storage BOT/SCSI class driver.
//
// The storage driver now binds as a USB function driver to an interface PDO
// (USB\CLASS_08&SUBCLASS_06&PROT_50). It creates a disk FDO above that
// interface stack and issues BOT transfers through USB URBs. The xHCI HCD
// still backs the bulk URBs with the existing per-device rings; later stages
// will make endpoint rings fully pipe-owned and asynchronous.
#include "usb.h"
DRIVER_OBJECT gUsbStorageDriver;
KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
PHYSICAL_ADDRESS phys, uint32_t length,
uint32_t* outActual) {
return XhciBulkTransfer(xhci,
dev,
dev->MassBulkInEndpointId,
dev->MassInRing,
dev->MassInRingPhys,
&dev->MassInEnqueue,
&dev->MassInCycle,
phys,
length,
outActual);
static const IO_DEVICE_ID gUsbStorageIds[] = {
{ IoBusUsb, { "USB\\CLASS_08&SUBCLASS_06&PROT_50", sizeof("USB\\CLASS_08&SUBCLASS_06&PROT_50") - 1 } },
};
static const IO_DRIVER_ID_TABLE gUsbStorageIdTable = {
gUsbStorageIds,
sizeof(gUsbStorageIds) / sizeof(gUsbStorageIds[0]),
};
static USB_STORAGE_EXTENSION* UsbStorageFromDiskDevice(DEVICE_OBJECT* device) {
if (!device || !device->DeviceExtension)
return (USB_STORAGE_EXTENSION*)0;
USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)device->DeviceExtension;
if (diskExt->Kind != UsbStorageExtensionDisk)
return (USB_STORAGE_EXTENSION*)0;
return diskExt->Storage;
}
static USB_STORAGE_EXTENSION* UsbStorageFromInterfaceDevice(DEVICE_OBJECT* device) {
if (!device || !device->DeviceExtension)
return (USB_STORAGE_EXTENSION*)0;
KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
PHYSICAL_ADDRESS phys, uint32_t length,
uint32_t* outActual) {
return XhciBulkTransfer(xhci,
dev,
dev->MassBulkOutEndpointId,
dev->MassOutRing,
dev->MassOutRingPhys,
&dev->MassOutEnqueue,
&dev->MassOutCycle,
phys,
length,
outActual);
USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)device->DeviceExtension;
if (ext->Kind != UsbStorageExtensionInterface)
return (USB_STORAGE_EXTENSION*)0;
return ext;
}
static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_EXTENSION* pdoExt, bool wantIn) {
if (!pdoExt) return (const USB_ENDPOINT_DESCRIPTOR*)0;
KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const uint8_t* cdb, uint8_t cdbLen,
void* data, PHYSICAL_ADDRESS dataPhys,
uint32_t dataLength, bool dataIn) {
if (!xhci || !dev || !cdb || cdbLen == 0 || cdbLen > 16)
for (uint8_t i = 0; i < pdoExt->EndpointCount; i++) {
const USB_ENDPOINT_DESCRIPTOR* ep = &pdoExt->Endpoints[i];
uint8_t type = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0;
if (type == USB_ENDPOINT_TYPE_BULK && in == wantIn)
return ep;
}
return (const USB_ENDPOINT_DESCRIPTOR*)0;
}
static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
bool dataIn,
PHYSICAL_ADDRESS phys,
uint32_t length,
uint32_t* outActual) {
if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0)
return STATUS_INVALID_PARAMETER;
USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe;
if (!pipe || !pipe->Open)
return STATUS_OBJECT_DEAD;
USB_URB urb;
memset(&urb, 0, sizeof(urb));
urb.Function = UrbFunctionBulkTransfer;
urb.Direction = dataIn ? UsbTransferIn : UsbTransferOut;
urb.Device = storage->Device;
urb.Pipe = pipe;
urb.Buffer = (void*)0;
urb.BufferPhys = phys;
urb.Length = length;
KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb);
if (outActual)
*outActual = KSUCCESS(st) ? urb.ActualLength : 0;
return st;
}
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, true, phys, length, outActual);
}
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, false, phys, length, outActual);
}
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
PHYSICAL_ADDRESS dataPhys,
uint32_t dataLength,
bool dataIn) {
if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16)
return STATUS_INVALID_PARAMETER;
if (dataLength && (!data || !dataPhys))
return STATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
USB_MASS_CBW* cbw = (USB_MASS_CBW*)dev->DescriptorBuffer;
memset(cbw, 0, sizeof(*cbw));
cbw->Signature = USB_MASS_BOT_CBW_SIGNATURE;
cbw->Tag = ++dev->MassTag;
cbw->Tag = ++storage->Tag;
cbw->DataTransferLength = dataLength;
cbw->Flags = dataIn ? USB_DIR_IN : USB_DIR_OUT;
cbw->Lun = 0;
@@ -55,16 +117,14 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
memcpy(cbw->CommandBlock, cdb, cdbLen);
uint32_t actual = 0;
KSTATUS st = UsbStorageBulkOut(xhci, dev, dev->DescriptorPhys, sizeof(*cbw), &actual);
KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*cbw))
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
if (dataLength) {
actual = 0;
if (dataIn)
st = UsbStorageBulkIn(xhci, dev, dataPhys, dataLength, &actual);
else
st = UsbStorageBulkOut(xhci, dev, dataPhys, dataLength, &actual);
st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual)
: UsbStorageBulkOut(storage, dataPhys, dataLength, &actual);
if (!KSUCCESS(st))
return st;
}
@@ -72,13 +132,13 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer;
memset(csw, 0, sizeof(*csw));
actual = 0;
st = UsbStorageBulkIn(xhci, dev, dev->DescriptorPhys, sizeof(*csw), &actual);
st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*csw))
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != dev->MassTag) {
if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != storage->Tag) {
DbgPrint("usb-storage: invalid CSW slot=%u sig=%08x tag=%08x expected=%08x\n",
dev->SlotId, csw->Signature, csw->Tag, dev->MassTag);
dev->SlotId, csw->Signature, csw->Tag, storage->Tag);
return STATUS_DEVICE_ERROR;
}
if (csw->Status != USB_MASS_BOT_CSW_PASSED) {
@@ -90,13 +150,11 @@ KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
uint32_t UsbBe32(const uint8_t* p) {
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
((uint32_t)p[2] << 8) | (uint32_t)p[3];
}
void UsbStoreBe32(uint8_t* p, uint32_t v) {
p[0] = (uint8_t)(v >> 24);
p[1] = (uint8_t)(v >> 16);
@@ -104,17 +162,19 @@ void UsbStoreBe32(uint8_t* p, uint32_t v) {
p[3] = (uint8_t)v;
}
void UsbStoreBe16(uint8_t* p, uint16_t v) {
p[0] = (uint8_t)(v >> 8);
p[1] = (uint8_t)v;
}
KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 };
memset(dev->MassBuffer, 0, 36);
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
36, true);
if (!KSUCCESS(st)) return st;
@@ -128,37 +188,46 @@ KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
memset(dev->MassBuffer, 0, 18);
return UsbStorageBotCommand(xhci, dev, rs, sizeof(rs),
return UsbStorageBotCommand(storage, rs, sizeof(rs),
dev->MassBuffer, dev->MassBufferPhys, 18, true);
}
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
uint8_t cdb[6] = { SCSI_TEST_UNIT_READY, 0, 0, 0, 0, 0 };
KSTATUS last = STATUS_DEVICE_ERROR;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext;
for (uint32_t attempt = 0; attempt < 10; attempt++) {
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
(void*)0, 0, 0, false);
if (KSUCCESS(st)) return STATUS_SUCCESS;
last = st;
(void)UsbStorageRequestSense(xhci, dev);
(void)UsbStorageRequestSense(storage);
UsbDelayUs(xhci->Hal, 10000u);
}
return last;
}
KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
memset(dev->MassBuffer, 0, 8);
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
8, true);
if (!KSUCCESS(st)) return st;
@@ -168,27 +237,33 @@ KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (blockSize == 0)
return STATUS_DEVICE_ERROR;
storage->BlockSize = blockSize;
storage->BlockCount = (uint64_t)lastLba + 1ull;
dev->MassBlockSize = blockSize;
dev->MassBlockCount = (uint64_t)lastLba + 1ull;
dev->MassBlockCount = storage->BlockCount;
DbgPrint("usb-storage: capacity slot=%u blocks=%llu block_size=%u (%llu MB)\n",
dev->SlotId,
(unsigned long long)dev->MassBlockCount,
dev->MassBlockSize,
(unsigned long long)((dev->MassBlockCount * (uint64_t)dev->MassBlockSize) / (1024ull * 1024ull)));
(unsigned long long)storage->BlockCount,
storage->BlockSize,
(unsigned long long)((storage->BlockCount * (uint64_t)storage->BlockSize) / (1024ull * 1024ull)));
return STATUS_SUCCESS;
}
KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
bool write, uint64_t lba,
uint32_t sectors, void* buffer) {
if (!xhci || !dev || !buffer || sectors == 0)
KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
bool write,
uint64_t lba,
uint32_t sectors,
void* buffer) {
if (!storage || !storage->XhciDevice || !buffer || sectors == 0)
return STATUS_INVALID_PARAMETER;
if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE)
if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE)
return STATUS_NOT_SUPPORTED;
if (lba > 0xFFFFFFFFull || sectors > 0xFFFFu)
return STATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
uint32_t bytes = sectors * USB_STORAGE_SECTOR_SIZE;
if (bytes > USB_STORAGE_MAX_BYTES)
return STATUS_INVALID_PARAMETER;
@@ -202,7 +277,7 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
if (write)
memcpy(dev->MassBuffer, buffer, bytes);
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
bytes, !write);
if (!KSUCCESS(st)) return st;
@@ -212,42 +287,67 @@ KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
while (_InterlockedCompareExchange(&dev->MassIoBusy, 1, 0) != 0) {
UsbDelayUs(xhci->Hal, 10);
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
if (!storage) return;
XHCI_CONTROLLER* xhci = storage->Device ? (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
while (_InterlockedCompareExchange(&storage->IoBusy, 1, 0) != 0) {
if (xhci) UsbDelayUs(xhci->Hal, 10);
}
}
void UsbStorageReleaseIo(XHCI_DEVICE* dev) {
_InterlockedExchange(&dev->MassIoBusy, 0);
void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage) {
if (storage) _InterlockedExchange(&storage->IoBusy, 0);
}
KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = (XHCI_DEVICE*)device->DeviceExtension;
if (!dev->MassActive || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
IoCompleteRequest(irp, STATUS_DEVICE_ERROR, 0);
USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device);
if (!storage || !storage->RemoveLockReady)
return STATUS_OBJECT_DEAD;
KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return STATUS_INVALID_PARAMETER;
}
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return STATUS_INVALID_PARAMETER;
}
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
(ios->Parameters.ReadWrite.Length & 511u) ||
ios->Parameters.ReadWrite.Length == 0 ||
!irp->UserBuffer) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return STATUS_INVALID_PARAMETER;
}
if (!storage->Started || storage->SurpriseRemoved || storage->Removed ||
storage->BlockSize != USB_STORAGE_SECTOR_SIZE) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, STATUS_OBJECT_DEAD, 0);
return STATUS_PENDING;
}
uint8_t* buffer = (uint8_t*)irp->Buffer;
uint64_t lba = irp->Lba;
uint32_t remaining = irp->SectorCount;
uint8_t* buffer = (uint8_t*)irp->UserBuffer;
uint64_t lba = ios->Parameters.ReadWrite.ByteOffset >> 9;
uint32_t remaining = ios->Parameters.ReadWrite.Length >> 9;
uint32_t done = 0;
bool write = (irp->Flags & IRP_FLAG_WRITE) != 0;
bool write = (ios->MajorFunction == IRP_MJ_WRITE);
uint32_t maxSectors = USB_STORAGE_MAX_BYTES / USB_STORAGE_SECTOR_SIZE;
KSTATUS st = STATUS_SUCCESS;
UsbStorageAcquireIo(&gXhci, dev);
st = STATUS_SUCCESS;
UsbStorageAcquireIo(storage);
while (remaining) {
uint32_t chunk = remaining;
if (chunk > maxSectors) chunk = maxSectors;
st = UsbStorageReadWriteSectors(&gXhci, dev, write, lba, chunk,
st = UsbStorageReadWriteSectors(storage, write, lba, chunk,
buffer + (done * USB_STORAGE_SECTOR_SIZE));
if (!KSUCCESS(st))
break;
@@ -255,13 +355,13 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
done += chunk;
remaining -= chunk;
}
UsbStorageReleaseIo(dev);
UsbStorageReleaseIo(storage);
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? (size_t)done * USB_STORAGE_SECTOR_SIZE : 0);
return STATUS_PENDING;
}
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer)
return STATUS_SUCCESS;
@@ -282,7 +382,6 @@ KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (!dev->MassCandidate)
return STATUS_SUCCESS;
@@ -352,7 +451,6 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de
return STATUS_SUCCESS;
}
uint8_t UsbFindFreeDiskIndex(void) {
for (uint8_t i = 0; i < 8; i++) {
if (!IoDiskDevicePresent(i))
@@ -361,17 +459,23 @@ uint8_t UsbFindFreeDiskIndex(void) {
return 0xFFu;
}
KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) {
if (!dev || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE || dev->MassBlockCount == 0)
KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage) {
if (!storage || storage->BlockSize != USB_STORAGE_SECTOR_SIZE || storage->BlockCount == 0)
return STATUS_NOT_SUPPORTED;
if (storage->DiskDevice)
return STATUS_ALREADY_EXISTS;
DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(&gUsbStorageDriver, 0, &disk);
KSTATUS st = IoCreateDevice(&gUsbStorageDriver, sizeof(USB_STORAGE_DISK_EXTENSION), &disk);
if (!KSUCCESS(st)) return st;
(void)IoSetDeviceName(disk, SYS_SV_LITERAL("usb-storage-disk"));
disk->DeviceExtension = dev;
disk->SectorCount = dev->MassBlockCount;
USB_STORAGE_DISK_EXTENSION* diskExt = (USB_STORAGE_DISK_EXTENSION*)disk->DeviceExtension;
diskExt->Kind = UsbStorageExtensionDisk;
diskExt->Storage = storage;
disk->u.Disk.SectorCount = storage->BlockCount;
disk->u.Disk.SectorSize = USB_STORAGE_SECTOR_SIZE;
disk->AlignmentRequirement = USB_STORAGE_SECTOR_SIZE;
disk->Flags |= DEVICE_OBJECT_REMOVABLE;
@@ -387,12 +491,278 @@ KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) {
return st;
}
dev->MassDeviceObject = disk;
storage->DiskDevice = disk;
if (storage->XhciDevice)
storage->XhciDevice->MassDeviceObject = disk;
DbgPrint("usb-storage: registered disk %u slot=%u sectors=%llu\n",
index, dev->SlotId, (unsigned long long)dev->MassBlockCount);
index,
storage->Device ? storage->Device->SlotId : 0,
(unsigned long long)storage->BlockCount);
return STATUS_SUCCESS;
}
static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSION* storage) {
if (!device || !storage || !storage->InterfacePdo || !storage->Device)
return STATUS_INVALID_PARAMETER;
if (storage->Started)
return STATUS_SUCCESS;
if (storage->SurpriseRemoved || storage->Removed)
return STATUS_OBJECT_DEAD;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)storage->InterfacePdo->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface)
return STATUS_INVALID_PARAMETER;
const USB_ENDPOINT_DESCRIPTOR* bulkIn = UsbStorageFindBulkEndpoint(pdoExt, true);
const USB_ENDPOINT_DESCRIPTOR* bulkOut = UsbStorageFindBulkEndpoint(pdoExt, false);
if (!bulkIn || !bulkOut)
return STATUS_NOT_FOUND;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext;
XHCI_DEVICE* xdev = (XHCI_DEVICE*)storage->Device->HcdDeviceContext;
if (!xhci || !xdev)
return STATUS_INVALID_PARAMETER;
storage->XhciDevice = xdev;
storage->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
storage->BulkInAddress = bulkIn->bEndpointAddress;
storage->BulkOutAddress = bulkOut->bEndpointAddress;
storage->BulkInMaxPacket = bulkIn->wMaxPacketSize;
storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize;
storage->Tag = 0x1000u + storage->Device->SlotId;
xdev->MassCandidate = true;
xdev->MassInterface = storage->InterfaceNumber;
xdev->MassBulkInAddress = storage->BulkInAddress;
xdev->MassBulkOutAddress = storage->BulkOutAddress;
xdev->MassBulkInEndpointId = XhciEndpointIdFromAddress(storage->BulkInAddress);
xdev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(storage->BulkOutAddress);
xdev->MassBulkInMaxPacket = storage->BulkInMaxPacket;
xdev->MassBulkOutMaxPacket = storage->BulkOutMaxPacket;
KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, xdev);
if (!KSUCCESS(st)) goto fail;
st = UsbCreatePipe(storage->Device,
UsbPipeBulk,
storage->BulkInAddress,
storage->BulkInMaxPacket,
bulkIn->bInterval,
&storage->BulkInPipe);
if (!KSUCCESS(st)) goto fail;
st = UsbCreatePipe(storage->Device,
UsbPipeBulk,
storage->BulkOutAddress,
storage->BulkOutMaxPacket,
bulkOut->bInterval,
&storage->BulkOutPipe);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageInquiry(storage);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageTestUnitReady(storage);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", storage->Device->SlotId, st);
goto fail;
}
st = UsbStorageReadCapacity(storage);
if (!KSUCCESS(st)) goto fail;
if (storage->BlockSize != USB_STORAGE_SECTOR_SIZE) {
DbgPrint("usb-storage: unsupported block size slot=%u size=%u\n",
storage->Device->SlotId, storage->BlockSize);
st = STATUS_NOT_SUPPORTED;
goto fail;
}
// The disk namespace publishes an arrival notification synchronously from
// IoRegisterDiskDevice(). Mark the storage FDO usable before publishing the
// disk so filesystem hotplug callbacks can read sector 0 immediately.
xdev->MassActive = true;
storage->Started = true;
device->PnpState = DevicePnpStarted;
if (storage->InterfacePdo)
storage->InterfacePdo->PnpState = DevicePnpStarted;
st = UsbStorageRegisterDisk(storage);
if (!KSUCCESS(st)) {
storage->Started = false;
device->PnpState = DevicePnpNotStarted;
if (storage->InterfacePdo)
storage->InterfacePdo->PnpState = DevicePnpNotStarted;
goto fail;
}
DbgPrint("usb-storage: FDO bound slot=%u if=%u disk-ready\n",
storage->Device->SlotId, storage->InterfaceNumber);
return STATUS_SUCCESS;
fail:
xdev->MassCandidate = false;
xdev->MassActive = false;
if (storage->BulkInPipe) {
UsbDereferencePipe(storage->BulkInPipe);
storage->BulkInPipe = (USB_PIPE*)0;
}
if (storage->BulkOutPipe) {
UsbDereferencePipe(storage->BulkOutPipe);
storage->BulkOutPipe = (USB_PIPE*)0;
}
DbgPrint("usb-storage: start failed slot=%u if=%u st=0x%x\n",
storage->Device ? storage->Device->SlotId : 0,
storage->InterfaceNumber,
st);
return st;
}
static KSTATUS UsbStorageDiskPnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
if (!device || !irp || !storage)
return STATUS_INVALID_PARAMETER;
switch (minor) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_SURPRISE_REMOVAL:
storage->SurpriseRemoved = true;
if (storage->XhciDevice) {
storage->XhciDevice->MassSurpriseRemoved = true;
storage->XhciDevice->MassActive = false;
}
device->PnpState = DevicePnpSurpriseRemoved;
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
case IRP_MN_REMOVE_DEVICE:
storage->Removed = true;
if (storage->XhciDevice) {
storage->XhciDevice->MassRemoved = true;
storage->XhciDevice->MassActive = false;
if (storage->XhciDevice->MassDeviceObject == device)
storage->XhciDevice->MassDeviceObject = (DEVICE_OBJECT*)0;
}
device->PnpState = DevicePnpDeleted;
if (storage->DiskDevice == device)
storage->DiskDevice = (DEVICE_OBJECT*)0;
IoDeleteDevice(device);
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
default:
return STATUS_NOT_SUPPORTED;
}
}
static KSTATUS UsbStorageInterfacePnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
if (!device || !irp || !storage)
return STATUS_INVALID_PARAMETER;
KSTATUS st = STATUS_SUCCESS;
switch (minor) {
case IRP_MN_START_DEVICE:
st = UsbStorageStartDevice(device, storage);
if (!KSUCCESS(st)) {
IoCompleteRequest(irp, st, 0);
return STATUS_PENDING;
}
return IoForwardIrpToLower(device, irp);
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return IoForwardIrpToLower(device, irp);
case IRP_MN_CANCEL_REMOVE_DEVICE:
if (!storage->SurpriseRemoved)
device->PnpState = DevicePnpStarted;
return IoForwardIrpToLower(device, irp);
case IRP_MN_SURPRISE_REMOVAL:
storage->SurpriseRemoved = true;
storage->Started = false;
if (storage->XhciDevice) {
storage->XhciDevice->MassSurpriseRemoved = true;
storage->XhciDevice->MassActive = false;
}
if (storage->DiskDevice)
(void)IoNotifyDiskSurpriseRemoved(storage->DiskDevice->u.Disk.DiskIndex);
device->PnpState = DevicePnpSurpriseRemoved;
return IoForwardIrpToLower(device, irp);
case IRP_MN_REMOVE_DEVICE: {
storage->Removed = true;
storage->Started = false;
if (storage->DiskDevice)
(void)IoUnregisterDiskDevice(storage->DiskDevice->u.Disk.DiskIndex);
if (storage->BulkInPipe) {
UsbDereferencePipe(storage->BulkInPipe);
storage->BulkInPipe = (USB_PIPE*)0;
}
if (storage->BulkOutPipe) {
UsbDereferencePipe(storage->BulkOutPipe);
storage->BulkOutPipe = (USB_PIPE*)0;
}
if (storage->RemoveLockReady) {
IoReleaseRemoveLockAndWait(&storage->RemoveLock, irp);
IoDestroyRemoveLock(&storage->RemoveLock);
storage->RemoveLockReady = false;
}
device->PnpState = DevicePnpDeleted;
KSTATUS lower = storage->LowerDevice ? IoForwardIrpToLower(device, irp) : STATUS_SUCCESS;
if (storage->LowerDevice) {
IoDetachDevice(device);
storage->LowerDevice = (DEVICE_OBJECT*)0;
}
IoDeleteDevice(device);
if (lower == STATUS_NOT_SUPPORTED) {
IoCompleteRequest(irp, STATUS_SUCCESS, 0);
return STATUS_PENDING;
}
if (lower != STATUS_PENDING && !KSUCCESS(lower)) {
IoCompleteRequest(irp, lower, 0);
return STATUS_PENDING;
}
return lower;
}
default:
return IoForwardIrpToLower(device, irp);
}
}
KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
USB_STORAGE_EXTENSION* iface = UsbStorageFromInterfaceDevice(device);
if (iface)
return UsbStorageInterfacePnp(device, irp, iface, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction);
USB_STORAGE_EXTENSION* disk = UsbStorageFromDiskDevice(device);
if (disk)
return UsbStorageDiskPnp(device, irp, disk, (IO_PNP_MINOR_FUNCTION)ios->MinorFunction);
return STATUS_INVALID_PARAMETER;
}
bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
(void)context;
@@ -409,64 +779,69 @@ bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
}
KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context) {
(void)pub;
(void)context;
if (!pub || !pub->BusContext || !pub->DeviceContext)
return STATUS_INVALID_PARAMETER;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext;
XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext;
if (dev->MassActive || dev->MassCandidate)
return STATUS_ALREADY_EXISTS;
const IO_PUBLISHED_USB_ENDPOINT* bulkIn = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true);
const IO_PUBLISHED_USB_ENDPOINT* bulkOut = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
if (!bulkIn || !bulkOut)
return STATUS_NOT_FOUND;
dev->MassCandidate = true;
dev->MassInterface = pub->u.Usb.InterfaceNumber;
dev->MassBulkInAddress = bulkIn->Address;
dev->MassBulkOutAddress = bulkOut->Address;
dev->MassBulkInEndpointId = XhciEndpointIdFromAddress(bulkIn->Address);
dev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(bulkOut->Address);
dev->MassBulkInMaxPacket = bulkIn->MaxPacketSize;
dev->MassBulkOutMaxPacket = bulkOut->MaxPacketSize;
KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, dev);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageInquiry(xhci, dev);
if (!KSUCCESS(st)) goto fail;
st = UsbStorageTestUnitReady(xhci, dev);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", dev->SlotId, st);
goto fail;
}
st = UsbStorageReadCapacity(xhci, dev);
if (!KSUCCESS(st)) goto fail;
if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
DbgPrint("usb-storage: unsupported block size slot=%u block_size=%u\n",
dev->SlotId, dev->MassBlockSize);
st = STATUS_NOT_SUPPORTED;
goto fail;
}
dev->MassActive = true;
st = UsbStorageRegisterDisk(dev);
if (!KSUCCESS(st)) goto fail;
DbgPrint("usb-storage: bound slot=%u if=%u\n", dev->SlotId, dev->MassInterface);
return STATUS_SUCCESS;
fail:
dev->MassCandidate = false;
dev->MassActive = false;
DbgPrint("usb-storage: bind failed slot=%u if=%u st=0x%x\n",
dev->SlotId, pub->u.Usb.InterfaceNumber, st);
return st;
return STATUS_NOT_SUPPORTED;
}
KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
USB_PDO_EXTENSION* pdoExt = (USB_PDO_EXTENSION*)physicalDeviceObject->DeviceExtension;
if (!pdoExt || pdoExt->Kind != UsbPdoInterface || !pdoExt->Device)
return STATUS_INVALID_PARAMETER;
if (pdoExt->InterfaceDescriptor.bInterfaceClass != USB_CLASS_MASS_STORAGE ||
pdoExt->InterfaceDescriptor.bInterfaceSubClass != USB_MASS_SUBCLASS_SCSI ||
pdoExt->InterfaceDescriptor.bInterfaceProtocol != USB_MASS_PROTOCOL_BOT)
return STATUS_INVALID_PARAMETER;
if (!UsbStorageFindBulkEndpoint(pdoExt, true) || !UsbStorageFindBulkEndpoint(pdoExt, false))
return STATUS_NOT_FOUND;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(driver, sizeof(USB_STORAGE_EXTENSION), &fdo);
if (!KSUCCESS(st))
return st;
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("usb-storage"));
USB_STORAGE_EXTENSION* ext = (USB_STORAGE_EXTENSION*)fdo->DeviceExtension;
ext->Kind = UsbStorageExtensionInterface;
ext->Self = fdo;
ext->InterfacePdo = physicalDeviceObject;
ext->Device = pdoExt->Device;
ext->XhciDevice = (XHCI_DEVICE*)pdoExt->Device->HcdDeviceContext;
ext->InterfaceNumber = pdoExt->InterfaceDescriptor.bInterfaceNumber;
ext->IoBusy = 0;
st = IoInitializeRemoveLock(&ext->RemoveLock);
if (!KSUCCESS(st)) {
IoDeleteDevice(fdo);
return st;
}
ext->RemoveLockReady = true;
ext->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!ext->LowerDevice) {
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
IoDeleteDevice(fdo);
return STATUS_FAILURE;
}
DbgPrint("usb-storage: FDO attached slot=%u if=%u\n",
ext->Device->SlotId,
pdoExt->InterfaceDescriptor.bInterfaceNumber);
return STATUS_SUCCESS;
}
KSTATUS UsbInitializeStorageDriver(void) {
memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
IoInitializeDriverObject(&gUsbStorageDriver, SYS_SV_LITERAL("usb-storage"));
gUsbStorageDriver.IdTable = &gUsbStorageIdTable;
gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice;
gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp;
return IoRegisterDriverObject(&gUsbStorageDriver);
}
code/drivers/bus/usb/usb_xhci.c
+596 -66
View File
@@ -353,6 +353,31 @@ void XhciLogPortStatus(XHCI_CONTROLLER* xhci) {
}
}
void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->Op)
return;
uint32_t powered = 0;
for (uint8_t port = 1; port <= xhci->MaxPorts; port++) {
uint32_t off = XhciPortReg(port);
uint32_t portsc = XhciRead32(xhci->Op, off);
uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK;
if (!(portsc & XHCI_PORTSC_PP)) {
XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
powered++;
UsbTrace("usb: xHCI powered root port %u old-portsc=%08x\n", port, portsc);
} else if (changes) {
XhciWrite32(xhci->Op, off, XHCI_PORTSC_PP | changes);
}
}
if (powered) {
UsbDelayUs(xhci->Hal, 20000u);
DbgPrint("usb: xHCI powered %u previously-off root port(s)\n", powered);
}
}
void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
if (!xhci->Runtime) return;
@@ -369,7 +394,71 @@ void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
}
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->CommandPhys && w->CommandPhys == commandPhys)
return w;
}
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint8_t slotId,
uint8_t endpointId) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->TransferPhys &&
w->TransferPhys == transferPhys &&
w->TransferSlotId == slotId &&
w->TransferEndpointId == endpointId) {
return w;
}
}
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
uint64_t commandPhys,
uint32_t completionCode,
uint8_t slotId) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys);
if (w) {
w->Command.CompletionCode = completionCode;
w->Command.SlotId = slotId;
XhciBarrier();
w->CommandMatched = true;
}
KeReleaseSpinlock(&xhci->WaitLock, flags);
return w;
}
static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint8_t slotId,
uint8_t endpointId,
uint32_t completionCode,
uint32_t residue) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId);
if (w) {
w->Transfer.CompletionCode = completionCode;
w->Transfer.SlotId = slotId;
w->Transfer.EndpointId = endpointId;
w->Transfer.Residue = residue;
XhciBarrier();
w->TransferMatched = true;
}
KeReleaseSpinlock(&xhci->WaitLock, flags);
return w;
}
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
bool processed = false;
for (;;) {
@@ -384,41 +473,28 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
uint32_t cc = XhciCompletionCode(ev);
uint64_t param = ((uint64_t)ev->ParameterHi << 32) | ev->ParameterLo;
XHCI_WAIT_TARGET* waiter = target ? target : (XHCI_WAIT_TARGET*)xhci->ActiveWait;
if (type == XHCI_TRB_TYPE_CMD_DONE) {
uint8_t slotId = XhciEventSlotId(ev);
UsbTrace("usb: xHCI command completion ptr=0x%016llX cc=%u slot=%u\n",
(unsigned long long)param, cc, slotId);
if (waiter && waiter->CommandPhys && param == waiter->CommandPhys) {
waiter->CommandMatched = true;
waiter->Command.CompletionCode = cc;
waiter->Command.SlotId = slotId;
}
(void)XhciClaimCommandWaiter(xhci, param, cc, slotId);
} else if (type == XHCI_TRB_TYPE_TRANSFER) {
uint8_t slotId = XhciEventSlotId(ev);
uint8_t epId = XhciEventEndpointId(ev);
uint32_t residue = XhciTransferResidue(ev);
bool claimedByWaiter = false;
bool targetMatches = waiter && waiter->TransferPhys && param == waiter->TransferPhys &&
slotId == waiter->TransferSlotId && epId == waiter->TransferEndpointId;
if (cc != XHCI_CC_SUCCESS) {
UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n",
(unsigned long long)param, cc, slotId, epId, residue);
}
if (targetMatches) {
waiter->TransferMatched = true;
waiter->Transfer.CompletionCode = cc;
waiter->Transfer.SlotId = slotId;
waiter->Transfer.EndpointId = epId;
waiter->Transfer.Residue = residue;
claimedByWaiter = true;
}
if (!claimedByWaiter)
XHCI_WAIT_TARGET* waiter = XhciClaimTransferWaiter(xhci, param, slotId, epId, cc, residue);
if (!waiter)
XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue);
} else if (type == XHCI_TRB_TYPE_PORT_SC) {
uint32_t portId = ev->ParameterLo >> 24;
UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc);
if (portId && portId <= xhci->MaxPorts)
UsbHubQueueRootPortChange(&xhci->Hcd, (uint8_t)portId);
} else {
DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n",
type, cc, (unsigned long long)param, ev->Status, ev->Control);
@@ -454,22 +530,53 @@ bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
}
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || target->Registered)
return;
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
target->Next = xhci->WaiterList;
target->Registered = true;
xhci->WaiterList = target;
KeReleaseSpinlock(&xhci->WaitLock, flags);
}
void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || !target->Registered)
return;
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET** link = &xhci->WaiterList;
while (*link) {
if (*link == target) {
*link = target->Next;
target->Next = (XHCI_WAIT_TARGET*)0;
target->Registered = false;
break;
}
link = &(*link)->Next;
}
KeReleaseSpinlock(&xhci->WaitLock, flags);
}
KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
if (!xhci || !target)
return STATUS_INVALID_PARAMETER;
xhci->ActiveWait = target;
for (uint32_t i = 0; i < timeoutUs; i++) {
if (XhciWaitTargetComplete(target)) {
xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
XhciUnregisterWaitTarget(xhci, target);
return STATUS_SUCCESS;
}
// The IRQ/DPC path normally owns the event ring after enumeration, but
// synchronous storage/control waits must not depend exclusively on an
// interrupt being delivered. Opportunistically drain the ring from the
// waiter too; EventLock serializes this with the DPC.
// interrupt being delivered. Opportunistically drain the ring too;
// EventLock serializes this with the DPC while WaitLock protects waiter
// registration/matching.
if (!xhci->InterruptsEnabled || (i & 0x7u) == 0) {
XhciDrainEventRingLocked(xhci, target);
} else if ((XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_EINT) && (i & 0x3Fu) == 0) {
@@ -479,11 +586,20 @@ KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
UsbDelayUs(xhci->Hal, 1);
}
xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0;
XhciUnregisterWaitTarget(xhci, target);
return STATUS_TIMEOUT;
}
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
if (!xhci || !target)
return STATUS_INVALID_PARAMETER;
XhciRegisterWaitTarget(xhci, target);
return XhciWaitForArmedTarget(xhci, target, timeoutUs);
}
void XhciEventDpcRoutine(KDPC* dpc, void* context,
void* systemArgument1, void* systemArgument2) {
(void)dpc;
@@ -695,13 +811,26 @@ uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
}
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
XHCI_COMMAND_RESULT* outResult) {
if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u)
return STATUS_QUEUE_FULL;
static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
if (!xhci || !xhci->CommandRing)
return STATUS_INVALID_PARAMETER;
if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) {
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
xhci->CommandCycle;
xhci->CommandEnqueue = 0;
xhci->CommandCycle ^= 1u;
}
uint32_t index = xhci->CommandEnqueue++;
XHCI_TRB* trb = &xhci->CommandRing[index];
@@ -713,15 +842,38 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
XhciBarrier();
trb->Control = controlWithoutCycle | xhci->CommandCycle;
if (outPhys) *outPhys = trbPhys;
return STATUS_SUCCESS;
}
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
XHCI_COMMAND_RESULT* outResult) {
if (!xhci)
return STATUS_INVALID_PARAMETER;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock);
uint64_t trbPhys = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys);
if (!KSUCCESS(st)) {
KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
return st;
}
target.CommandPhys = trbPhys;
xhci->ActiveWait = &target;
XhciRegisterWaitTarget(xhci, &target);
XhciBarrier();
XhciWrite32(xhci->Doorbells, 0, 0); // host-controller doorbell, command ring
KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
KSTATUS waitStatus = XhciWaitForTarget(xhci, &target, 500000u);
KSTATUS waitStatus = XhciWaitForArmedTarget(xhci, &target, 500000u);
if (KSUCCESS(waitStatus) && target.CommandMatched) {
if (outResult) *outResult = target.Command;
return (target.Command.CompletionCode == XHCI_CC_SUCCESS)
@@ -781,6 +933,28 @@ KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
}
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0)
return STATUS_INVALID_PARAMETER;
XHCI_COMMAND_RESULT res;
memset(&res, 0, sizeof(res));
KSTATUS st = XhciSubmitCommand(xhci,
0,
0,
(XHCI_TRB_TYPE_DISABLE_SLOT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)slotId << 24),
&res);
if (!KSUCCESS(st))
return st;
if (res.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Disable Slot slot=%u failed cc=%u\n", slotId, res.CompletionCode);
return STATUS_DEVICE_ERROR;
}
return STATUS_SUCCESS;
}
KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
@@ -970,12 +1144,22 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
uint32_t maxPacket = XhciEpMaxPacket(dev->Speed);
dev->MaxPacketSize0 = (uint16_t)maxPacket;
slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed);
uint8_t rootPort = dev->Path.RootPort ? dev->Path.RootPort : dev->Port;
uint32_t routeString = dev->Path.RouteString & 0x000fffffu;
slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed) | routeString;
// Slot context dword 1 owns Root Hub Port Number bits 23:16.
// Putting the port number in dword 2 makes Address Device fail with a
// context/TRB error on QEMU xHCI.
slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
slot[1] = XHCI_SLOT_ROOT_PORT(rootPort);
slot[2] = 0;
if (dev->Path.BehindHub && dev->Speed <= 2u && dev->Path.ParentHubSlot) {
// TT Hub Slot ID bits 7:0 and TT Port Number bits 15:8.
// This is required for low/full-speed devices behind high-speed hubs;
// harmless for older full-speed hub paths on the controllers we target.
slot[2] = (uint32_t)dev->Path.ParentHubSlot |
((uint32_t)dev->Path.ParentHubPort << 8);
}
slot[3] = 0;
ep0[0] = 0;
@@ -999,8 +1183,21 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u)
return STATUS_QUEUE_FULL;
if (!dev || !dev->Ep0Ring)
return STATUS_INVALID_PARAMETER;
if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) {
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
dev->Ep0Cycle;
dev->Ep0Enqueue = 0;
dev->Ep0Cycle ^= 1u;
}
uint32_t index = dev->Ep0Enqueue++;
XHCI_TRB* trb = &dev->Ep0Ring[index];
@@ -1017,11 +1214,12 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
}
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint16_t length) {
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint16_t length,
uint32_t timeoutUs) {
if (!buffer || length == 0 || length > PAGE_SIZE)
return STATUS_INVALID_PARAMETER;
@@ -1057,18 +1255,21 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
&statusTrbPhys);
if (!KSUCCESS(st)) return st;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
st = XhciWaitForTarget(xhci, &target, 1000000u);
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0
if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
st = XhciWaitForArmedTarget(xhci, &target, timeoutUs);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI control IN timeout slot=%u len=%u\n", dev->SlotId, length);
DbgPrint("usb: xHCI control IN timeout slot=%u len=%u timeout=%uus\n",
dev->SlotId, length, timeoutUs);
return st;
}
if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS)
@@ -1082,6 +1283,16 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint16_t length) {
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length,
USB_XHCI_CONTROL_TIMEOUT_US);
}
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup) {
uint64_t setupValue = UsbSetupPacketToU64(setup);
@@ -1106,16 +1317,17 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
&statusTrbPhys);
if (!KSUCCESS(st)) return st;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
st = XhciWaitForTarget(xhci, &target, 1000000u);
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u);
st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI control no-data timeout slot=%u req=%u value=%04x\n",
dev->SlotId, setup->bRequest, setup->wValue);
@@ -1132,8 +1344,9 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc) {
KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc,
uint32_t timeoutUs) {
USB_SETUP_PACKET setup;
setup.bmRequestType = USB_DIR_IN;
setup.bRequest = USB_REQ_GET_DESCRIPTOR;
@@ -1141,10 +1354,12 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
setup.wIndex = 0;
setup.wLength = sizeof(USB_DEVICE_DESCRIPTOR);
KSTATUS st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
sizeof(USB_DEVICE_DESCRIPTOR));
if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
KSTATUS st = XhciControlInTimeout(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
sizeof(USB_DEVICE_DESCRIPTOR),
timeoutUs);
if (!KSUCCESS(st)) return st;
memcpy(outDesc, dev->DescriptorBuffer, sizeof(*outDesc));
@@ -1159,6 +1374,13 @@ KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc) {
return XhciGetDeviceDescriptorTimeout(xhci, dev, outDesc,
USB_XHCI_CONTROL_TIMEOUT_US);
}
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t configurationIndex,
USB_CONFIGURATION_DESCRIPTOR* outConfig,
@@ -1331,11 +1553,11 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = endpointId;
xhci->ActiveWait = &target;
XhciRegisterWaitTarget(xhci, &target);
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, endpointId);
st = XhciWaitForTarget(xhci, &target, 1000000u);
st = XhciWaitForArmedTarget(xhci, &target, 1000000u);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: bulk transfer timeout slot=%u ep=%u len=%u\n",
dev->SlotId, endpointId, length);
@@ -1356,6 +1578,300 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) {
if (!xhci || !dev || !pipe || !pipe->Open)
return STATUS_INVALID_PARAMETER;
uint8_t dci = pipe->EndpointId;
if (dci < 2 || dci > 31)
return STATUS_INVALID_PARAMETER;
if (pipe->HcdPipeContext) {
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
return ctx->Configured ? STATUS_SUCCESS : STATUS_DEVICE_ERROR;
}
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)MmAllocatePool(sizeof(XHCI_PIPE_CONTEXT));
if (!ctx) return STATUS_NO_MEMORY;
memset(ctx, 0, sizeof(*ctx));
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys);
if (!ctx->Ring) {
MmFreePool(ctx);
return STATUS_NO_MEMORY;
}
ctx->Enqueue = 0;
ctx->Cycle = 1;
ctx->EndpointId = dci;
ctx->Type = pipe->Type;
XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle);
memset(dev->InputContext, 0, PAGE_SIZE);
uint32_t* icc = XhciInputControlContext(xhci, dev);
uint32_t* slot = XhciInputSlotContext(xhci, dev);
uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci);
icc[0] = 0;
icc[1] = (1u << 0) | (1u << dci);
slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed);
slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
slot[2] = 0;
slot[3] = 0;
uint16_t maxPacket = pipe->MaxPacketSize & 0x07FFu;
if (maxPacket == 0) maxPacket = 8u;
uint32_t endpointType = 0;
uint8_t interval = 0;
if (pipe->Type == UsbPipeInterrupt && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
endpointType = XHCI_EP_TYPE_INT_IN;
interval = XhciInterruptInterval(dev->Speed, pipe->Interval);
} else if (pipe->Type == UsbPipeBulk && (pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) {
endpointType = XHCI_EP_TYPE_BULK_IN;
} else if (pipe->Type == UsbPipeBulk) {
endpointType = XHCI_EP_TYPE_BULK_OUT;
} else {
MmFreePool(ctx);
return STATUS_NOT_SUPPORTED;
}
ep[0] = (uint32_t)interval << 16;
ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16);
ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32);
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
&result);
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
MmFreePool(ctx);
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
}
ctx->Configured = true;
pipe->HcdPipeContext = ctx;
dev->EndpointPipes[dci] = pipe;
UsbReferencePipe(pipe);
return STATUS_SUCCESS;
}
static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
return STATUS_INVALID_PARAMETER;
if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN))
return STATUS_NOT_SUPPORTED;
KSTATUS st = XhciConfigurePipeEndpoint(xhci, dev, urb->Pipe);
if (!KSUCCESS(st)) return st;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)urb->Pipe->HcdPipeContext;
if (!ctx || !ctx->Configured)
return STATUS_DEVICE_ERROR;
if (ctx->ActiveTransfer)
return STATUS_QUEUE_FULL;
uint64_t trbPhys = 0;
st = XhciRingAppend(ctx->Ring,
ctx->RingPhys,
XHCI_PIPE_RING_TRBS,
&ctx->Enqueue,
&ctx->Cycle,
urb->BufferPhys,
urb->Length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
if (!KSUCCESS(st)) return st;
ctx->ActiveTransfer = true;
ctx->ActiveTrbPhys = trbPhys;
ctx->ActiveUrb = urb;
urb->Pending = true;
urb->Cancelled = false;
urb->Status = STATUS_PENDING;
urb->ActualLength = 0;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, ctx->EndpointId);
return STATUS_PENDING;
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
return;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
USB_PIPE* pipe = dev->EndpointPipes[endpointId];
if (!pipe || !pipe->HcdPipeContext)
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys)
return;
USB_URB* urb = ctx->ActiveUrb;
ctx->ActiveTransfer = false;
ctx->ActiveTrbPhys = 0;
ctx->ActiveUrb = (USB_URB*)0;
if (!urb)
return;
KSTATUS st = STATUS_SUCCESS;
if (urb->Cancelled) {
st = STATUS_CANCELLED;
} else if (completionCode != XHCI_CC_SUCCESS) {
st = STATUS_DEVICE_ERROR;
}
uint32_t actual = 0;
if (KSUCCESS(st)) {
actual = (residue <= urb->Length) ? (urb->Length - residue) : 0;
}
urb->Pending = false;
urb->Status = st;
urb->ActualLength = actual;
IRP* completeIrp = urb->Irp;
urb->Irp = (IRP*)0;
if (urb->Complete)
urb->Complete(urb, urb->CompleteContext);
if (completeIrp)
IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0);
}
static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
return STATUS_INVALID_PARAMETER;
uint8_t endpointId = urb->Pipe->EndpointId;
XHCI_TRB* ring = (XHCI_TRB*)0;
PHYSICAL_ADDRESS ringPhys = 0;
uint32_t* enqueue = (uint32_t*)0;
uint8_t* cycle = (uint8_t*)0;
if (endpointId == dev->MassBulkInEndpointId) {
ring = dev->MassInRing;
ringPhys = dev->MassInRingPhys;
enqueue = &dev->MassInEnqueue;
cycle = &dev->MassInCycle;
} else if (endpointId == dev->MassBulkOutEndpointId) {
ring = dev->MassOutRing;
ringPhys = dev->MassOutRingPhys;
enqueue = &dev->MassOutEnqueue;
cycle = &dev->MassOutCycle;
} else {
return STATUS_NOT_SUPPORTED;
}
uint32_t actual = 0;
KSTATUS st = XhciBulkTransfer(xhci,
dev,
endpointId,
ring,
ringPhys,
enqueue,
cycle,
urb->BufferPhys,
urb->Length,
&actual);
urb->ActualLength = KSUCCESS(st) ? actual : 0;
return st;
}
static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
if (!hcd || !urb || !urb->Device)
return STATUS_INVALID_PARAMETER;
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)hcd->ControllerContext;
XHCI_DEVICE* dev = (XHCI_DEVICE*)urb->Device->HcdDeviceContext;
if (!xhci || !dev || !dev->Allocated)
return STATUS_INVALID_PARAMETER;
KSTATUS st;
switch (urb->Function) {
case UrbFunctionControlTransfer:
if (urb->Length == 0) {
st = XhciControlNoData(xhci, dev, &urb->Setup);
urb->ActualLength = 0;
} else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) {
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length);
urb->ActualLength = KSUCCESS(st) ? urb->Length : 0;
} else {
st = STATUS_NOT_SUPPORTED;
urb->ActualLength = 0;
}
break;
case UrbFunctionBulkTransfer:
st = XhciHcdSubmitBulkUrb(xhci, dev, urb);
break;
case UrbFunctionInterruptTransfer:
st = XhciHcdSubmitInterruptUrb(xhci, dev, urb);
break;
default:
st = STATUS_NOT_SUPPORTED;
urb->ActualLength = 0;
break;
}
urb->Status = st;
return st;
}
static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
(void)hcd;
if (!urb) return STATUS_INVALID_PARAMETER;
urb->Cancelled = true;
if (!urb->Pending)
urb->Status = STATUS_CANCELLED;
return STATUS_SUCCESS;
}
static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!hcd) return (void*)0;
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
}
static KSTATUS XhciHcdResetRootPort(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed) {
if (!hcd) return STATUS_INVALID_PARAMETER;
return XhciResetPort((XHCI_CONTROLLER*)hcd->ControllerContext, port, outSpeed);
}
static KSTATUS XhciHcdClearRootPortChanges(USB_HCD* hcd, uint8_t port) {
if (!hcd) return STATUS_INVALID_PARAMETER;
XhciClearPortChanges((XHCI_CONTROLLER*)hcd->ControllerContext, port);
return STATUS_SUCCESS;
}
const USB_HCD_OPERATIONS gXhciHcdOperations = {
XhciHcdSubmitUrb,
XhciHcdCancelUrb,
XhciHcdResetRootPort,
XhciHcdClearRootPortChanges,
XhciHcdAllocateDma,
};
KSTATUS XhciInitialise(const HAL_API* hal,
uint8_t bus, uint8_t dev, uint8_t fn,
uint16_t vendor, uint16_t device) {
@@ -1438,6 +1954,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
return STATUS_DEVICE_ERROR;
}
UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts);
KSTATUS hcdPdoStatus = UsbCreateControllerPdo(&xhci->Hcd, vendor, device);
if (!KSUCCESS(hcdPdoStatus)) {
DbgPrint("usb: xHCI controller PDO creation failed st=0x%x\n", hcdPdoStatus);
return hcdPdoStatus;
}
uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE +
(uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE;
uint32_t needBytes = portEnd;
@@ -1489,12 +2012,13 @@ KSTATUS XhciInitialise(const HAL_API* hal,
st = XhciSubmitNoopCommand(xhci);
if (!KSUCCESS(st)) return st;
XhciPowerRootPorts(xhci);
XhciLogPortStatus(xhci);
XhciEnumerateRootPorts(xhci);
if (xhci->HidDeviceCount > 0) {
st = XhciStartHidInterrupts(xhci);
if (!KSUCCESS(st)) return st;
st = UsbCreateRootHubPdo(&xhci->Hcd);
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS) {
DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st);
return st;
}
KeIrqRegister(xhci->Irq, XhciIsr);
@@ -1504,8 +2028,14 @@ KSTATUS XhciInitialise(const HAL_API* hal,
DbgPrint("usb: xHCI ISR registered on IRQ %u%s; event ring handled by DPC\n",
xhci->Irq, xhci->UsingMsi ? "(msi)" : "");
st = UsbHubScheduleRootScan(&xhci->Hcd);
if (!KSUCCESS(st))
DbgPrint("usb: root hub async scan scheduling failed st=0x%x\n", st);
else
DbgPrint("usb: root hub initial scan queued asynchronously\n");
gHaveXhci = true;
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage is next\n",
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
xhci->HidDeviceCount);
return STATUS_SUCCESS;
}
code/drivers/fs/fat32/fat32.c
+140 -23
View File
@@ -1612,7 +1612,10 @@ static void Fat32AddCandidate(FAT32_VOL* probe,
static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
FAT32_PARTITION_CANDIDATE* out,
uint32_t maxCount) {
uint32_t maxCount,
bool* outMediaReadable) {
if (outMediaReadable)
*outMediaReadable = false;
if (!out || maxCount == 0) return 0;
FAT32_VOL probe;
@@ -1623,6 +1626,8 @@ static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
uint8_t mbr[512];
if (!KSUCCESS(ReadSectors(&probe, 0, 1, mbr)))
return 0;
if (outMediaReadable)
*outMediaReadable = true;
// A disk image may be a raw FAT32 volume with no partition table.
Fat32AddCandidate(&probe, out, &count, maxCount, 0, IoDiskSectorCount(diskIndex));
@@ -1734,6 +1739,7 @@ typedef struct FAT32_ASYNC_RW {
size_t CurrentCopy;
IRP Irp;
DEVICE_OBJECT* CurrentDevice;
IO_FS_ASYNC_COMPLETE Complete;
void* CompleteContext;
} FAT32_ASYNC_RW;
@@ -1811,6 +1817,10 @@ static void Fat32AsyncQueueContinuation(FAT32_ASYNC_RW* ctx) {
static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t information) {
if (!ctx) return;
if (ctx->CurrentDevice) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
}
if (ctx->RunBuf) MmFreePool(ctx->RunBuf);
IO_FS_ASYNC_COMPLETE complete = ctx->Complete;
void* completeContext = ctx->CompleteContext;
@@ -1821,6 +1831,10 @@ static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t informa
static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
if (!ctx) return;
if (ctx->CurrentDevice) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
}
if (!KSUCCESS(irp->Status)) {
Fat32AsyncFinish(ctx, irp->Status, ctx->Done);
@@ -1872,17 +1886,25 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
ctx->CurrentOffset = ctx->OffsetInCluster;
ctx->CurrentCopy = copy;
memset(&ctx->Irp, 0, sizeof(ctx->Irp));
ctx->Irp.DriveIndex = v->DriveIndex;
ctx->Irp.Lba = ClusterToLba(v, runStart);
ctx->Irp.SectorCount = runSectors;
ctx->Irp.Buffer = ctx->RunBuf;
ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex);
if (!ctx->CurrentDevice) {
Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done);
return;
}
IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_READ,
ClusterToLba(v, runStart) * 512ull,
runSectors * 512u,
ctx->RunBuf);
ctx->Irp.CompletionFn = Fat32AsyncReadIrpComplete;
ctx->Irp.CompletionCtx = ctx;
KSTATUS s = IoSubmitIrp(&ctx->Irp);
if (s != STATUS_PENDING)
KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp);
if (s != STATUS_PENDING) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
Fat32AsyncFinish(ctx, s, ctx->Done);
}
}
KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
@@ -1949,6 +1971,10 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) {
FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
if (!ctx) return;
if (ctx->CurrentDevice) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
}
if (!KSUCCESS(irp->Status)) {
Fat32AsyncFinish(ctx, irp->Status, ctx->Done);
@@ -2024,18 +2050,25 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
ctx->CurrentOffset = ctx->OffsetInCluster;
ctx->CurrentCopy = copy;
memset(&ctx->Irp, 0, sizeof(ctx->Irp));
ctx->Irp.DriveIndex = v->DriveIndex;
ctx->Irp.Flags = IRP_FLAG_WRITE;
ctx->Irp.Lba = ClusterToLba(v, runStart);
ctx->Irp.SectorCount = runSectors;
ctx->Irp.Buffer = ctx->RunBuf;
ctx->CurrentDevice = IoReferenceDiskDevice(v->DriveIndex);
if (!ctx->CurrentDevice) {
Fat32AsyncFinish(ctx, STATUS_NOT_FOUND, ctx->Done);
return;
}
IoBuildReadWriteIrp(&ctx->Irp, IRP_MJ_WRITE,
ClusterToLba(v, runStart) * 512ull,
runSectors * 512u,
ctx->RunBuf);
ctx->Irp.CompletionFn = Fat32AsyncWriteIrpComplete;
ctx->Irp.CompletionCtx = ctx;
KSTATUS s = IoSubmitIrp(&ctx->Irp);
if (s != STATUS_PENDING)
KSTATUS s = IoSubmitIrp(ctx->CurrentDevice, &ctx->Irp);
if (s != STATUS_PENDING) {
IoDereferenceDeviceObject(ctx->CurrentDevice);
ctx->CurrentDevice = (DEVICE_OBJECT*)0;
Fat32AsyncFinish(ctx, s, ctx->Done);
}
}
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
@@ -2124,31 +2157,55 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
static FAT32_VOL gVols[FAT32_MAX_VOLUMES];
static IO_FILESYSTEM gFs[FAT32_MAX_VOLUMES];
static bool gVolumeMounted[FAT32_MAX_VOLUMES];
static uint32_t gMountedCount;
static bool gDiskScanned[FAT32_MAX_VOLUMES];
static int Fat32FindFreeVolumeIndex(void) {
for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) {
if (!gVolumeMounted[i] && !IoFilesystemPresent(i))
return i;
}
return -1;
}
static uint32_t Fat32TryMountDisk(uint8_t disk) {
if (disk >= FAT32_MAX_VOLUMES)
return 0;
if (gDiskScanned[disk])
return 0;
gDiskScanned[disk] = true;
if (!IoDiskDevicePresent(disk))
return 0;
uint32_t mountedHere = 0;
FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK];
uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK);
bool mediaReadable = false;
uint32_t partCount = Fat32EnumerateDiskVolumes(disk,
parts,
FAT32_MAX_PARTITIONS_PER_DISK,
&mediaReadable);
if (!mediaReadable) {
DbgPrint("FAT32: disk %u: media probe deferred; sector 0 unreadable\n", disk);
return 0;
}
// Only remember a completed probe. A transient hotplug/readiness failure
// must not permanently suppress a later mount attempt for the same disk
// index after a device is unplugged and replugged.
gDiskScanned[disk] = true;
if (!partCount) {
DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk);
return 0;
}
for (uint32_t part = 0; part < partCount && gMountedCount < FAT32_MAX_VOLUMES; part++) {
uint8_t volumeIndex = (uint8_t)gMountedCount;
for (uint32_t part = 0; part < partCount; part++) {
int freeVolume = Fat32FindFreeVolumeIndex();
if (freeVolume < 0)
break;
uint8_t volumeIndex = (uint8_t)freeVolume;
FAT32_VOL* v = &gVols[volumeIndex];
memset(v, 0, sizeof(*v));
v->DriveIndex = disk;
@@ -2192,6 +2249,7 @@ static uint32_t Fat32TryMountDisk(uint8_t disk) {
KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams);
if (KSUCCESS(status)) {
gVolumeMounted[volumeIndex] = true;
gMountedCount++;
mountedHere++;
DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n",
@@ -2222,6 +2280,60 @@ static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* co
}
}
static void Fat32UnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) {
uint32_t removed = 0;
for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) {
if (!gVolumeMounted[i])
continue;
if (gVols[i].DriveIndex != diskIndex)
continue;
// Surprise removal means the backing device may already be gone; do not
// attempt writeback here. Drop cached FAT sectors instead.
FatCacheInvalidate();
(void)IoUnregisterFilesystem(i);
memset(&gVols[i], 0, sizeof(gVols[i]));
memset(&gFs[i], 0, sizeof(gFs[i]));
gVolumeMounted[i] = false;
if (gMountedCount)
gMountedCount--;
removed++;
}
if (diskIndex < FAT32_MAX_VOLUMES)
gDiskScanned[diskIndex] = false;
if (removed) {
DbgPrint("FAT32: disk %u %s; unmounted %u volume(s); total=%u\n",
diskIndex,
change == IoDeviceSurpriseRemoved ? "surprise removed" : "removed",
removed,
gMountedCount);
}
}
static void Fat32DiskChange(uint8_t driveIndex,
DEVICE_OBJECT* device,
IO_DEVICE_CHANGE change,
void* context) {
(void)device;
(void)context;
switch (change) {
case IoDeviceArrived:
Fat32DiskArrival(driveIndex, device, context);
break;
case IoDeviceSurpriseRemoved:
case IoDeviceRemoved:
Fat32UnmountDisk(driveIndex, change);
break;
case IoDeviceQueryRemove:
default:
break;
}
}
KSTATUS DriverEntry(const HAL_API *hal) {
DbgPrint("FAT32: Init\n");
if (!hal) return STATUS_INVALID_PARAMETER;
@@ -2231,9 +2343,14 @@ KSTATUS DriverEntry(const HAL_API *hal) {
gMountedCount = 0;
memset(gDiskScanned, 0, sizeof(gDiskScanned));
KSTATUS cbStatus = IoRegisterDiskArrivalCallback(Fat32DiskArrival, (void*)0);
KSTATUS cbStatus = IoRegisterDiskChangeCallback(Fat32DiskChange, (void*)0);
if (!KSUCCESS(cbStatus)) {
DbgPrint("FAT32: disk-arrival callback registration failed st=0x%x\n", cbStatus);
DbgPrint("FAT32: disk-change callback registration failed st=0x%x\n", cbStatus);
}
cbStatus = IoEnumerateDiskDevices(Fat32DiskArrival, (void*)0);
if (!KSUCCESS(cbStatus)) {
DbgPrint("FAT32: initial disk enumeration failed st=0x%x\n", cbStatus);
}
if (!gMountedCount) {
code/drivers/storage/ahci/ahci.c
+105 -12
View File
@@ -382,6 +382,28 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
// Async IRP dispatch
// ---------------------------------------------------------------------------
static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
uint32_t* outSectors, void** outBuffer) {
if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return STATUS_INVALID_PARAMETER;
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
return STATUS_INVALID_PARAMETER;
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
(ios->Parameters.ReadWrite.Length & 511u))
return STATUS_INVALID_PARAMETER;
if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
return STATUS_INVALID_PARAMETER;
*outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
*outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
*outSectors = ios->Parameters.ReadWrite.Length >> 9;
*outBuffer = irp->UserBuffer;
return STATUS_SUCCESS;
}
// AhciDispatchReadWrite - DEVICE_OBJECT dispatch entry for disk IRPs.
// Accepted IRPs are queued to the per-drive worker and completed later with
// IoCompleteRequest(). Rejected IRPs are returned synchronously and are not
@@ -390,8 +412,22 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
AHCI_DRIVE *drv = (AHCI_DRIVE *)device->DeviceExtension;
if (!drv) return STATUS_INVALID_PARAMETER;
if (irp->SectorCount == 0 || !irp->Buffer) return STATUS_INVALID_PARAMETER;
if (irp->Lba + irp->SectorCount > drv->sectorCount) return STATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp);
if (!KSUCCESS(st))
return st;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
(void)write;
(void)buffer;
if (!KSUCCESS(st) || lba + sectors > drv->sectorCount) {
IoReleaseRemoveLock(&drv->removeLock, irp);
return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
}
irp->Status = STATUS_FAILURE;
irp->Information = 0;
@@ -399,6 +435,49 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
return STATUS_PENDING;
}
static void AhciCancelReleaseRemoveLock(IRP* irp, void* context) {
AHCI_DRIVE* drv = (AHCI_DRIVE*)context;
if (drv)
IoReleaseRemoveLock(&drv->removeLock, irp);
}
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
if (!drv) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return STATUS_SUCCESS;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_SURPRISE_REMOVAL:
drv->surpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
return STATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
drv->removed = true;
device->PnpState = DevicePnpDeleted;
IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
IoDeleteDevice(device);
drv->deviceObject = (DEVICE_OBJECT*)0;
return STATUS_SUCCESS;
default:
return STATUS_NOT_SUPPORTED;
}
}
// AhciWorker - kernel thread, one per drive.
// Dequeues IRPs and executes them one at a time using the existing
// single-slot interrupt-driven path. Completion fires the IRP's event
@@ -420,17 +499,24 @@ static void AhciWorker(uintptr_t arg) {
KeWaitSemaphore(drv->irpQueue.Semaphore);
IRP *irp = IrpQueueDequeue(&drv->irpQueue);
if (!irp) KePanic(KE_PANIC_CODE_DRIVER, "AhciWorker: semaphore fired but queue is empty - IRP accounting bug\n");
if (!irp)
continue;
bool write = !!(irp->Flags & IRP_FLAG_WRITE);
KSTATUS st;
if (write)
st = do_write(drv, irp->Lba, irp->SectorCount, irp->Buffer);
else
st = do_read (drv, irp->Lba, irp->SectorCount, irp->Buffer);
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read (drv, lba, sectors, buffer);
}
// Publish result then fire completion hooks through the common I/O path.
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
IoReleaseRemoveLock(&drv->removeLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
}
}
@@ -467,9 +553,10 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
g_driveCount = 0;
memset(&g_driver, 0, sizeof(g_driver));
g_driver.Name = "ahci";
IoInitializeDriverObject(&g_driver, SYS_SV_LITERAL("ahci"));
g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
// -----------------------------------------------------------------------
// Locate AHCI controller via PCI class scan
@@ -588,6 +675,11 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
continue;
}
IrpQueueInit(&drv->irpQueue, irpSem);
st = IoInitializeRemoveLock(&drv->removeLock);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st);
continue;
}
// Phase 6: IDENTIFY (polled - scheduler not yet running)
st = port_identify(drv);
@@ -613,7 +705,8 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
continue;
}
drv->deviceObject->DeviceExtension = drv;
drv->deviceObject->SectorCount = drv->sectorCount;
drv->deviceObject->u.Disk.SectorCount = drv->sectorCount;
drv->deviceObject->u.Disk.SectorSize = 512;
drv->deviceObject->AlignmentRequirement = 512;
st = IoRegisterDiskDevice((uint8_t)driveIdx, drv->deviceObject);
code/drivers/storage/ahci/ahci.h
+3
View File
@@ -344,6 +344,9 @@ typedef struct {
// AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP
// and signals the semaphore to wake the worker.
IRP_QUEUE irpQueue;
IO_REMOVE_LOCK removeLock;
bool surpriseRemoved;
bool removed;
THREAD* workerThread;
DEVICE_OBJECT* deviceObject;
code/drivers/storage/ide/ide.c
+104 -17
View File
@@ -21,6 +21,9 @@ IRP_QUEUE gIrpQueue;
SEMAPHORE *gIrpSemaphore = NULL;
EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer
KDPC gDmaDpc; // bottom half for DMA completion
IO_REMOVE_LOCK gRemoveLock;
bool gSurpriseRemoved = false;
bool gRemoved = false;
// ---------------------------------------------------------------------------
// Bus Master I/O helpers
@@ -307,6 +310,28 @@ static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) {
return STATUS_SUCCESS;
}
static KSTATUS IdeGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
uint32_t* outSectors, void** outBuffer) {
if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return STATUS_INVALID_PARAMETER;
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
return STATUS_INVALID_PARAMETER;
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
(ios->Parameters.ReadWrite.Length & 511u))
return STATUS_INVALID_PARAMETER;
if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
return STATUS_INVALID_PARAMETER;
*outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
*outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
*outSectors = ios->Parameters.ReadWrite.Length >> 9;
*outBuffer = irp->UserBuffer;
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// IdeWorker — kernel thread, one per IDE controller.
//
@@ -320,17 +345,22 @@ void IdeWorker(uintptr_t arg) {
IRP *irp = IrpQueueDequeue(&gIrpQueue);
if (!irp)
KePanic(KE_PANIC_CODE_DRIVER, "IdeWorker: semaphore fired but IRP queue is empty — accounting bug\n");
continue;
KSTATUS st;
if (irp->Lba + irp->SectorCount > gSectorCount) {
st = STATUS_INVALID_PARAMETER;
} else {
bool write = !!(irp->Flags & IRP_FLAG_WRITE);
st = IdeDmaRw(irp->Lba, irp->SectorCount, irp->Buffer, write);
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = IdeGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (lba + sectors > gSectorCount)
st = STATUS_INVALID_PARAMETER;
else
st = IdeDmaRw(lba, sectors, buffer, write);
}
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
IoReleaseRemoveLock(&gRemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
}
}
@@ -344,8 +374,20 @@ void IdeWorker(uintptr_t arg) {
// ---------------------------------------------------------------------------
static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
(void)device;
if (!irp || !irp->Buffer || irp->SectorCount == 0) return STATUS_INVALID_PARAMETER;
if (irp->Lba + irp->SectorCount > gSectorCount) return STATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&gRemoveLock, irp);
if (!KSUCCESS(st))
return st;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = IdeGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (!KSUCCESS(st) || lba + sectors > gSectorCount) {
IoReleaseRemoveLock(&gRemoveLock, irp);
return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
}
irp->Status = STATUS_FAILURE;
irp->Information = 0;
@@ -355,13 +397,53 @@ static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
return STATUS_PENDING;
}
bool write = !!(irp->Flags & IRP_FLAG_WRITE);
KSTATUS st = write ? IdeWrite(NULL, irp->Lba, irp->SectorCount, irp->Buffer)
: IdeRead (NULL, irp->Lba, irp->SectorCount, irp->Buffer);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0);
st = write ? IdeWrite(NULL, lba, sectors, buffer)
: IdeRead (NULL, lba, sectors, buffer);
IoReleaseRemoveLock(&gRemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
return STATUS_PENDING;
}
static void IdeCancelReleaseRemoveLock(IRP* irp, void* context) {
(void)context;
IoReleaseRemoveLock(&gRemoveLock, irp);
}
static KSTATUS IdeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
return STATUS_INVALID_PARAMETER;
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
case IRP_MN_START_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_QUERY_REMOVE_DEVICE:
device->PnpState = DevicePnpRemovePending;
return STATUS_SUCCESS;
case IRP_MN_CANCEL_REMOVE_DEVICE:
device->PnpState = DevicePnpStarted;
return STATUS_SUCCESS;
case IRP_MN_SURPRISE_REMOVAL:
gSurpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
return STATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
gRemoved = true;
device->PnpState = DevicePnpDeleted;
IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
IoReleaseRemoveLockAndWait(&gRemoveLock, irp);
IoDeleteDevice(device);
gDevice = (DEVICE_OBJECT*)0;
return STATUS_SUCCESS;
default:
return STATUS_NOT_SUPPORTED;
}
}
// ---------------------------------------------------------------------------
// IdeDmaInit — locate the PCI IDE controller, grab BAR4, allocate the PRDT,
// create sync objects, and register IRQ 14.
@@ -427,13 +509,18 @@ KSTATUS DriverEntry(const HAL_API *hal) {
// ---- Register definitive DEVICE_OBJECT --------------------------------
memset(&gDriver, 0, sizeof(gDriver));
gDriver.Name = "ide";
IoInitializeDriverObject(&gDriver, SYS_SV_LITERAL("ide"));
gDriver.MajorFunction[IRP_MJ_READ] = IdeDispatchReadWrite;
gDriver.MajorFunction[IRP_MJ_WRITE] = IdeDispatchReadWrite;
gDriver.MajorFunction[IRP_MJ_PNP] = IdeDispatchPnp;
KSTATUS st = IoCreateDevice(&gDriver, 0, &gDevice);
KSTATUS st = IoInitializeRemoveLock(&gRemoveLock);
if (!KSUCCESS(st)) return st;
gDevice->SectorCount = gSectorCount;
st = IoCreateDevice(&gDriver, 0, &gDevice);
if (!KSUCCESS(st)) return st;
gDevice->u.Disk.SectorCount = gSectorCount;
gDevice->u.Disk.SectorSize = 512;
gDevice->AlignmentRequirement = 512;
// ---- Try to initialise Bus Master DMA ---------------------------------
code/kernel/hal/hal_main.c
+202 -70
View File
@@ -82,11 +82,13 @@ static uintptr_t gIoapicPhys = 0xFEC00000;
static uintptr_t gHpetPhys = 0xFED00000;
static HAL_CPU_TOPOLOGY gCpuTopology;
// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5)
// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5).
// Do not guess SLP_TYP on real hardware: these values are firmware-specific.
static uint16_t gPm1aCntBlk = 0;
static uint16_t gPm1bCntBlk = 0;
static uint8_t gSlpTypA = 5; // S5 SLP_TYP for PM1a; 5 is correct for QEMU/most VMs
static uint8_t gSlpTypB = 5; // S5 SLP_TYP for PM1b
static uint8_t gSlpTypA = 0;
static uint8_t gSlpTypB = 0;
static int gS5Found = 0;
int debugEnabled = 0;
int _fltused; // MSVC floating-point helper anchor
@@ -106,73 +108,183 @@ static void *HalPhysToVirt(uint64_t phys) {
return (void *)(phys + gPhysBase);
}
// FADT (Fixed ACPI Description Table) - partial layout up to PM1b_CNT_BLK.
// We only need offsets 40 (DSDT ptr), 64 (PM1a_CNT_BLK), 68 (PM1b_CNT_BLK).
// The full table is ~276 bytes (ACPI 2.0+); we never access beyond offset 68.
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
// systems may put the real DSDT and PM1 control registers in the extended
// X_* fields, so do not rely only on the legacy 32-bit fields.
#define FADT_DSDT_OFFSET 40u
#define FADT_PM1A_CNT_BLK_OFFSET 64u
#define FADT_PM1B_CNT_BLK_OFFSET 68u
#define FADT_X_DSDT_OFFSET 140u
#define FADT_X_PM1A_CNT_BLK_OFFSET 172u
#define FADT_X_PM1B_CNT_BLK_OFFSET 184u
#pragma pack(push, 1)
typedef struct {
ACPISDTHeader Header; // 0 (36 bytes)
uint32_t FirmwareCtrl; // 36
uint32_t Dsdt; // 40 - physical addr of DSDT
uint8_t Ignored44; // 44
uint8_t PreferredPmProfile; // 45
uint16_t SciInt; // 46
uint32_t SmiCmd; // 48
uint8_t AcpiEnable; // 52
uint8_t AcpiDisable; // 53
uint8_t S4BiosReq; // 54
uint8_t PstateCnt; // 55
uint32_t Pm1aEvtBlk; // 56
uint32_t Pm1bEvtBlk; // 60
uint32_t Pm1aCntBlk; // 64 - I/O port for PM1a control register
uint32_t Pm1bCntBlk; // 68 - I/O port for PM1b control register
} FADT;
uint8_t AddressSpaceId;
uint8_t RegisterBitWidth;
uint8_t RegisterBitOffset;
uint8_t AccessSize;
uint64_t Address;
} ACPI_GAS;
#pragma pack(pop)
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values.
// The encoding looks like:
// 0x08 '_' 'S' '5' '_' ; NameOp + "_S5_"
// 0x12 <PkgLen> ; PackageOp + encoded length
// <NumElements> ; usually 0x04
// 0x0A <slp_typ_a> ; BytePrefix + value (or 0x00 for zero)
// 0x0A <slp_typ_b>
static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
for (uint32_t i = 0; i + 9 < len; i++) {
// Match NameOp + "_S5_" + PackageOp
if (aml[i] != 0x08) continue;
if (aml[i+1] != 0x5F || aml[i+2] != 0x53 ||
aml[i+3] != 0x35 || aml[i+4] != 0x5F) continue;
if (aml[i+5] != 0x12) continue;
static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) {
uint32_t value = 0;
if (off + sizeof(value) <= tableLen)
memcpy(&value, (const uint8_t*)table + off, sizeof(value));
return value;
}
// PkgLength encoding: low 6 bits = byte count (0 → only this byte);
// bits [7:6] = number of following bytes in the length field.
uint8_t pkgByte = aml[i+6];
uint32_t pkgExtra = pkgByte >> 6; // 0..3 extra bytes
uint32_t j = i + 7 + pkgExtra; // skip PkgLen field
if (j + 2 >= len) break;
j++; // skip NumElements byte
static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off) {
uint64_t value = 0;
if (off + sizeof(value) <= tableLen)
memcpy(&value, (const uint8_t*)table + off, sizeof(value));
return value;
}
// SLP_TYP_A
if (aml[j] == 0x0A && j + 1 < len) {
gSlpTypA = aml[j + 1]; j += 2;
} else if (aml[j] == 0x00) {
gSlpTypA = 0; j++;
} else {
break; // unexpected encoding - keep defaults
static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t off) {
ACPI_GAS gas;
if (off + sizeof(gas) > tableLen) return 0;
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
// 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(),
// so ignore MMIO GAS entries until the HAL grows MMIO register access here.
if (gas.AddressSpaceId != 0x01 || gas.Address == 0 || gas.Address > 0xFFFF)
return 0;
return (uint16_t)gas.Address;
}
static int AmlParsePkgLength(const uint8_t* p, const uint8_t* end,
const uint8_t** body, const uint8_t** pkgEnd) {
if (p >= end) return 0;
const uint8_t* start = p;
uint8_t first = *p++;
uint32_t follow = first >> 6;
uint32_t length;
if (follow == 0) {
length = first & 0x3F;
} else {
length = first & 0x0F;
for (uint32_t i = 0; i < follow; i++) {
if (p >= end) return 0;
length |= ((uint32_t)*p++) << (4 + i * 8);
}
}
// SLP_TYP_B
if (j < len && aml[j] == 0x0A && j + 1 < len)
gSlpTypB = aml[j + 1];
else if (j < len && aml[j] == 0x00)
gSlpTypB = 0;
if (start + length < p || start + length > end) return 0;
*body = p;
*pkgEnd = start + length;
return 1;
}
static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
int* lastSegmentIsS5) {
uint32_t segments = 1;
*lastSegmentIsS5 = 0;
// Optional root or parent prefixes: '\\', '^', '^^', ...
while (p < end && (*p == 0x5C || *p == 0x5E)) p++;
if (p >= end) return NULL;
if (*p == 0x00) { // NullName
return p + 1;
} else if (*p == 0x2E) { // DualNamePrefix
p++;
segments = 2;
} else if (*p == 0x2F) { // MultiNamePrefix
p++;
if (p >= end) return NULL;
segments = *p++;
}
for (uint32_t s = 0; s < segments; s++) {
if (p + 4 > end) return NULL;
if (s == segments - 1 && memcmp(p, "_S5_", 4) == 0)
*lastSegmentIsS5 = 1;
p += 4;
}
return p;
}
static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
const uint8_t* p = *pp;
if (p >= end) return 0;
switch (*p++) {
case 0x00: // ZeroOp
*out = 0;
break;
case 0x01: // OneOp
*out = 1;
break;
case 0x0A: // BytePrefix
if (p >= end) return 0;
*out = *p++;
break;
case 0x0B: // WordPrefix
if (p + 2 > end) return 0;
*out = p[0];
p += 2;
break;
case 0x0C: // DWordPrefix
if (p + 4 > end) return 0;
*out = p[0];
p += 4;
break;
case 0x0E: // QWordPrefix
if (p + 8 > end) return 0;
*out = p[0];
p += 8;
break;
default:
return 0;
}
*pp = p;
return 1;
}
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and
// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants.
static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
const uint8_t* end = aml + len;
for (const uint8_t* cur = aml; cur < end; cur++) {
if (*cur != 0x08) continue; // NameOp
const uint8_t* p = cur + 1;
int isS5 = 0;
p = AmlParseNameString(p, end, &isS5);
if (!p || !isS5 || p >= end || *p != 0x12) continue; // PackageOp
const uint8_t* body;
const uint8_t* pkgEnd;
if (!AmlParsePkgLength(p + 1, end, &body, &pkgEnd)) continue;
if (body >= pkgEnd) continue;
uint8_t elements = *body++;
if (elements < 2) continue;
uint8_t typA, typB;
if (!AmlReadInteger(&body, pkgEnd, &typA)) continue;
if (!AmlReadInteger(&body, pkgEnd, &typB)) continue;
gSlpTypA = typA & 0x07;
gSlpTypB = typB & 0x07;
gS5Found = 1;
HalDbgPrint("HAL: _S5_ SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n",
gSlpTypA, gSlpTypB);
return;
}
HalDbgPrint("HAL: _S5_ not found in DSDT, using defaults (A=0x%02x B=0x%02x)\n",
gSlpTypA, gSlpTypB);
HalDbgPrint("HAL: _S5_ not found in DSDT; ACPI shutdown disabled\n");
}
static int validate_checksum(void* table, uint32_t length) {
@@ -184,19 +296,33 @@ static int validate_checksum(void* table, uint32_t length) {
return sum == 0;
}
static void ParseFadt(const FADT* fadt) {
gPm1aCntBlk = (uint16_t)fadt->Pm1aCntBlk;
gPm1bCntBlk = (uint16_t)fadt->Pm1bCntBlk;
HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%08x\n",
gPm1aCntBlk, gPm1bCntBlk, fadt->Dsdt);
static void ParseFadt(const ACPISDTHeader* header) {
const void* fadt = header;
uint32_t len = header->Length;
if (fadt->Dsdt) {
const ACPISDTHeader* dsdt =
(const ACPISDTHeader*)((uintptr_t)fadt->Dsdt + gPhysBase);
gPm1aCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1A_CNT_BLK_OFFSET);
gPm1bCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1B_CNT_BLK_OFFSET);
if (!gPm1aCntBlk)
gPm1aCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1A_CNT_BLK_OFFSET);
if (!gPm1bCntBlk)
gPm1bCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1B_CNT_BLK_OFFSET);
uint64_t dsdtPhys = AcpiReadU64(fadt, len, FADT_X_DSDT_OFFSET);
if (!dsdtPhys)
dsdtPhys = AcpiReadU32(fadt, len, FADT_DSDT_OFFSET);
HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%016llx\n",
gPm1aCntBlk, gPm1bCntBlk, dsdtPhys);
if (dsdtPhys) {
const ACPISDTHeader* dsdt = (const ACPISDTHeader*)(dsdtPhys + gPhysBase);
if (memcmp(dsdt->Signature, "DSDT", 4) == 0 &&
validate_checksum((void*)dsdt, dsdt->Length)) {
const uint8_t* aml = (const uint8_t*)(dsdt + 1);
ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader));
} else {
HalDbgPrint("HAL: DSDT invalid or checksum failed\n");
}
}
}
@@ -319,7 +445,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
}
// Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer
else if (memcmp(header->Signature, "FACP", 4) == 0) {
ParseFadt((const FADT*)header);
ParseFadt(header);
}
// Parse HPET
else if (memcmp(header->Signature, "HPET", 4) == 0) {
@@ -337,11 +463,17 @@ static void HalShutdown(void) {
// Silence! Stop the world!
_disable();
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
uint16_t valA = (uint16_t)((gSlpTypA << 10) | (1 << 13));
uint16_t valB = (uint16_t)((gSlpTypB << 10) | (1 << 13));
if (!gS5Found || !gPm1aCntBlk) {
// Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets
// put into S3/S4 sleep instead of S5 soft-off.
while (true) __halt();
}
if (gPm1aCntBlk) __outword(gPm1aCntBlk, valA);
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
uint16_t valA = (uint16_t)(((uint16_t)gSlpTypA << 10) | (1 << 13));
uint16_t valB = (uint16_t)(((uint16_t)gSlpTypB << 10) | (1 << 13));
__outword(gPm1aCntBlk, valA);
if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB);
// Die!
code/kernel/include/kernel/io_file.h
+4
View File
@@ -4,6 +4,7 @@
// Kernel and user code open files through IoOpenFile.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <ntlite/status.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
@@ -143,6 +144,9 @@ typedef struct IO_VOLUME_PARAMS {
// ---------------------------------------------------------------------------
// driveIndex matches the index passed to IoRegisterDiskDevice.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol);
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
__declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex);
// ---------------------------------------------------------------------------
// Kernel-side API
code/kernel/include/kernel/ke_cpu.h
+18
View File
@@ -30,6 +30,24 @@ typedef struct KCPU {
uint8_t Online;
uint8_t IsBsp;
// CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded
// from the APIC ID using CPUID leaf 0x1F/0x0B when available, with a
// conservative CPUID(1)/CPUID(4) fallback. PhysicalCoreIndex is a compact
// kernel-local core number used by the scheduler. CoreSiblingMask contains
// all logical CPUs that share this physical core. PackageMask contains all
// logical CPUs in the same package. IsPrimaryThread is the scheduler's
// preferred logical CPU for a physical core, currently the lowest CpuIndex
// in that core.
uint32_t PackageId;
uint32_t CoreId;
uint32_t ThreadId;
uint32_t PhysicalCoreIndex;
uint64_t CoreSiblingMask;
uint64_t PackageMask;
uint8_t IsPrimaryThread;
uint8_t SmtThreadCount;
uint16_t TopologyFlags;
struct THREAD* CurrentThread;
struct THREAD* IdleThread;
struct THREAD* OutgoingThread;
code/kernel/include/kernel/ke_debug.h
+9 -1
View File
@@ -3,10 +3,18 @@
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v);
#define KE_DEBUG_CONSOLE 0x00000001u
#define KE_DEBUG_SELFTEST 0x00000002u
#define KE_DEBUG_FILE_LOG 0x00000004u
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
uint32_t KeGetDebugFlags(void);
__declspec(dllexport) void DbgPrint(const char* fmt, ...);
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len);
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path);
typedef enum KE_PANIC_CODE {
KE_PANIC_CODE_GENERIC = 0,
code/kernel/include/kernel/ke_irp.h
+156 -50
View File
@@ -1,83 +1,157 @@
#pragma once
// ke_irp.h - I/O Request Packet
// ke_irp.h - I/O Request Packet and remove-lock primitives
//
// This header intentionally includes only ke_list.h, ke_spinlock.h, and
// ke_status.h - all of which are pure-inline / macro headers with no
// external symbol references. ke_event.h and ke_semaphore.h are forward-
// declared here so the HAL layer (hal_disk.c, io_driver.h) can include this
// header without dragging in kernel sync-object link dependencies.
//
// Files that actually call KeEventCreate / KeSignalEvent / KeWaitForEvent
// (ke_irp.c, ahci.c) include ke_event.h and ke_semaphore.h themselves.
// IRPs are stack-based. A caller builds an IRP with one or more
// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O
// manager/driver stack advances CurrentLocation as the IRP is forwarded.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <ntlite/status.h>
// Forward declarations - full types live in ke_event.h / ke_semaphore.h.
// We only ever hold pointers in IRP and IRP_QUEUE, so a forward decl suffices.
struct EVENT;
struct SEMAPHORE;
struct KAPC;
typedef struct DEVICE_OBJECT DEVICE_OBJECT;
// -------------------------------------------------------------------------
// IRP major/minor functions
// -------------------------------------------------------------------------
typedef enum IO_MAJOR_FUNCTION {
IRP_MJ_CREATE = 0,
IRP_MJ_CLOSE = 1,
IRP_MJ_READ = 2,
IRP_MJ_WRITE = 3,
IRP_MJ_DEVICE_CONTROL = 4,
IRP_MJ_INTERNAL_DEVICE_CONTROL = 5,
IRP_MJ_PNP = 6,
IRP_MJ_POWER = 7,
IRP_MJ_MAXIMUM_FUNCTION = 8,
} IO_MAJOR_FUNCTION;
typedef enum IO_PNP_MINOR_FUNCTION {
IRP_MN_START_DEVICE = 0,
IRP_MN_QUERY_REMOVE_DEVICE = 1,
IRP_MN_CANCEL_REMOVE_DEVICE = 2,
IRP_MN_SURPRISE_REMOVAL = 3,
IRP_MN_REMOVE_DEVICE = 4,
IRP_MN_QUERY_DEVICE_RELATIONS = 5,
IRP_MN_QUERY_ID = 6,
IRP_MN_QUERY_CAPABILITIES = 7,
} IO_PNP_MINOR_FUNCTION;
typedef enum IO_DEVICE_RELATION_TYPE {
IoBusRelations = 0,
IoRemovalRelations = 1,
} IO_DEVICE_RELATION_TYPE;
// -------------------------------------------------------------------------
// IRP flags
// -------------------------------------------------------------------------
#define IRP_FLAG_WRITE (1u << 0) // direction: write to device
#define IRP_FLAG_PRIORITY (1u << 1) // insert at head of queue (elevated)
#define IRP_FLAG_PRIORITY (1u << 0) // insert at head of a driver queue
#define IRP_FLAG_CANCELLED (1u << 1)
#define IRP_FLAG_COMPLETED (1u << 2)
// -------------------------------------------------------------------------
// Forward declaration (IRP is self-referential via CompletionFn)
// IRP stack parameters
// -------------------------------------------------------------------------
typedef struct IO_STACK_LOCATION {
IO_MAJOR_FUNCTION MajorFunction;
uint8_t MinorFunction;
DEVICE_OBJECT* DeviceObject;
union {
struct {
uint64_t ByteOffset;
uint32_t Length;
} ReadWrite;
struct {
uint32_t IoControlCode;
void* InputBuffer;
uint32_t InputLength;
void* OutputBuffer;
uint32_t OutputLength;
} DeviceIoControl;
struct {
IO_DEVICE_RELATION_TYPE RelationType;
} QueryDeviceRelations;
struct {
uint32_t IdType;
} QueryId;
} Parameters;
} IO_STACK_LOCATION;
// -------------------------------------------------------------------------
// Forward declaration / callbacks
// -------------------------------------------------------------------------
typedef struct IRP IRP;
// Completion callback: invoked by IoCompleteRequest() in the completing
// context. Driver worker, synchronous fallback, DPC, and APC completion paths
// all centralize policy here.
typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx);
typedef void (*IRP_COMPLETION_FN)(IRP* irp, void* ctx);
typedef void (*IRP_CANCEL_ROUTINE)(DEVICE_OBJECT* device, IRP* irp);
// -------------------------------------------------------------------------
// I/O Request Packet
// -------------------------------------------------------------------------
typedef struct IRP {
// Embedded list node - driver-internal; do not touch after submit.
LIST_ENTRY Link;
// ---- Request parameters (caller fills before IoSubmitIrp) ------------
uint8_t DriveIndex;
uint32_t Flags; // IRP_FLAG_*
uint64_t Lba;
uint32_t SectorCount;
void* Buffer; // kernel VA; valid until completion
// ---- Completion (caller fills before submit; driver writes Status) ----
volatile KSTATUS Status;
volatile size_t Information; // bytes transferred on success
volatile size_t Information;
// Forward-declared pointer: callers that create events include ke_event.h.
struct EVENT *CompletionEvent;
// Optional kernel APC. If set, IoCompleteRequest queues it to its
// target thread with arg1=irp and arg2=(void*)(uintptr_t)status.
struct KAPC *CompletionApc;
uint32_t Flags;
volatile bool Cancel;
SPINLOCK CancelLock;
KIRQL CancelIrql;
IRP_CANCEL_ROUTINE CancelRoutine;
struct EVENT* CompletionEvent;
struct KAPC* CompletionApc;
IRP_COMPLETION_FN CompletionFn;
void* CompletionCtx;
// ---- Driver-internal --------------------------------------------------
void* UserBuffer;
void* SystemBuffer;
uint8_t StackCount;
uint8_t CurrentLocation;
// Driver-owned scratch fields. They are deliberately not interpreted by
// the I/O manager and are preserved across queueing/completion.
uint8_t HwSlot;
void* DriverCtx;
IO_STACK_LOCATION Stack[1];
} IRP;
// -------------------------------------------------------------------------
// Remove lock
// -------------------------------------------------------------------------
typedef struct IO_REMOVE_LOCK {
SPINLOCK Lock;
volatile long IoCount;
bool Removing;
struct EVENT* RemovedEvent;
} IO_REMOVE_LOCK;
__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock);
__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock);
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp);
// -------------------------------------------------------------------------
// IRP_QUEUE (per-device, driver-internal)
// -------------------------------------------------------------------------
@@ -85,24 +159,56 @@ typedef struct IRP {
typedef struct IRP_QUEUE {
SPINLOCK Lock;
LIST_ENTRY Head;
struct SEMAPHORE *Semaphore; // forward-declared pointer
struct SEMAPHORE* Semaphore;
} IRP_QUEUE;
// Non-inline: defined in ke_irp.c, which can pull in ke_semaphore.h safely.
__declspec(dllexport) void IrpQueueInit (IRP_QUEUE *q, struct SEMAPHORE *sem);
__declspec(dllexport) void IrpQueueEnqueue (IRP_QUEUE *q, IRP *irp);
__declspec(dllexport) IRP *IrpQueueDequeue (IRP_QUEUE *q);
__declspec(dllexport) void IrpQueueInit (IRP_QUEUE* q, struct SEMAPHORE* sem);
__declspec(dllexport) void IrpQueueEnqueue (IRP_QUEUE* q, IRP* irp);
__declspec(dllexport) IRP* IrpQueueDequeue (IRP_QUEUE* q);
typedef void (*IRP_QUEUE_CANCEL_CALLBACK)(IRP* irp, void* context);
__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context);
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status);
// -------------------------------------------------------------------------
// Public dispatch API - implemented in kernel/ke_irp.c
// IRP allocation / stack helpers
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp);
__declspec(dllexport) void IoFreeIrp(IRP* irp);
__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount);
__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp);
__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp);
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp);
__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp);
__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx);
__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine);
__declspec(dllexport) void IoCancelIrp(IRP* irp);
__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint64_t byteOffset,
uint32_t length,
void* buffer);
__declspec(dllexport) void IoBuildPnpIrp(IRP* irp,
IO_PNP_MINOR_FUNCTION minor);
__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint32_t ioControlCode,
void* inputBuffer,
uint32_t inputLength,
void* outputBuffer,
uint32_t outputLength);
// Complete an IRP exactly once: publish status/information, signal the
// optional completion event, then invoke the optional completion callback.
__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information);
// optional completion event, queue the optional APC, then invoke the optional
// completion callback.
__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information);
// Submit asynchronously. Returns STATUS_PENDING on success.
__declspec(dllexport) KSTATUS IoSubmitIrp (IRP *irp);
// Submit and block until complete. Returns irp->Status.
__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp);
// Submit to a specific top-of-stack device object. The current stack location
// is stamped with the target device before dispatch.
__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp);
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp);
code/kernel/include/kernel/ke_sched.h
+2
View File
@@ -23,6 +23,8 @@ typedef struct KE_SCHED_STATS {
uint64_t PriorityPreemptions;
uint64_t VoluntaryBlocks;
uint64_t CpuBurstDemotions;
uint64_t SmtSiblingDeferrals;
uint64_t SmtPrimaryPokes;
} KE_SCHED_STATS;
// Called once by KernelMain before any threads are created.
code/kernel/include/kernel/ke_smp.h
+6
View File
@@ -13,6 +13,12 @@ KSTATUS KeStartApplicationProcessors(void);
// park loop into their idle threads.
KSTATUS KeAdmitApplicationProcessorsToScheduler(void);
// Decode CPUID topology and build per-KCPU package/core/SMT sibling masks.
// KeStartApplicationProcessors calls this after MADT enumeration; it is also
// safe to call again after failed AP startup disables a CPU record.
void KeBuildProcessorTopology(void);
// Run explicit AP-affine kernel workers that exercise the global scheduler,
// reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver
// threads remain BSP-affine unless their affinity is explicitly widened.
code/kernel/include/kernel/ob_object.h
+4
View File
@@ -17,6 +17,10 @@ typedef enum OBJECT_TYPE {
ObjectTypeDir = 8,
ObjectTypeAsyncIo = 9,
ObjectTypeSocket = 10,
ObjectTypeDevice = 11,
ObjectTypeDriver = 12,
ObjectTypeUsbDevice = 13,
ObjectTypeUsbPipe = 14,
} OBJECT_TYPE;
typedef struct OBJECT {
code/kernel/io_device.c
+559 -78
View File
@@ -1,13 +1,423 @@
// io_device.c - kernel driver/device dispatch and disk registry
// io_device.c - I/O manager: driver/device objects, stacks, PnP skeleton, disk namespace
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_irp.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
#define MAX_PUBLISHED_DEVICES 64
#define MAX_DRIVER_MATCHERS 32
#define MAX_DRIVER_MATCHERS 32
#define MAX_DRIVER_OBJECTS 64
#define MAX_PDO_OBJECTS 128
static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) {
return SysSvIsEmpty(name) ? SYS_SV_LITERAL("<unnamed>") : name;
}
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
switch (busType) {
case IoBusPci: return "pci";
case IoBusUsb: return "usb";
default: return "unknown";
}
}
// -------------------------------------------------------------------------
// Driver object registry / PDO AddDevice binding
// -------------------------------------------------------------------------
static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS];
static uint32_t gDriverObjectCount;
static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS];
static uint32_t gPhysicalDeviceCount;
static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical)
return false;
for (size_t d = 0; d < driver->IdTable->Count; d++) {
const IO_DEVICE_ID* want = &driver->IdTable->Ids[d];
if (want->BusType != IoBusUnknown && want->BusType != pdo->u.Pdo.BusType)
continue;
for (uint8_t p = 0; p < pdo->u.Pdo.IdCount; p++) {
if (SysSvEqual(want->Id, pdo->u.Pdo.Ids[p].Id))
return true;
}
}
return false;
}
static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) {
if (!driver || !pdo || !driver->DriverAddDevice)
return false;
if (!IoDriverMatchesPdo(driver, pdo))
return false;
KSTATUS st = driver->DriverAddDevice(driver, pdo);
if (KSUCCESS(st)) {
DbgPrint("io: AddDevice driver=%v pdo=%v bus=%s\n",
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), IoBusTypeName(pdo->u.Pdo.BusType));
KSTATUS start = IoSendPnpIrpSync(pdo, IRP_MN_START_DEVICE);
if (!KSUCCESS(start) && start != STATUS_NOT_SUPPORTED) {
DbgPrint("io: START_DEVICE failed driver=%v pdo=%v st=0x%x\n",
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), start);
}
return true;
} else {
DbgPrint("io: AddDevice failed driver=%v pdo=%v st=0x%x\n",
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), st);
return true;
}
}
__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
SYS_STRING_VIEW name) {
if (!driver) return;
memset(driver, 0, sizeof(*driver));
driver->Header.Type = ObjectTypeDriver;
driver->Header.RefCount = 1;
driver->Header.Destroy = (void*)0;
driver->Name = name;
}
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
if (!driver) return STATUS_INVALID_PARAMETER;
if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
return STATUS_NO_MEMORY;
if (driver->Header.Type != ObjectTypeDriver) {
driver->Header.Type = ObjectTypeDriver;
driver->Header.RefCount = 1;
}
gDriverObjects[gDriverObjectCount++] = driver;
DbgPrint("io: registered driver object %v\n", IoSvOrUnnamed(driver->Name));
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++)
(void)IoTryAddDeviceForDriver(driver, gPhysicalDevices[i]);
return STATUS_SUCCESS;
}
// -------------------------------------------------------------------------
// Device object lifetime
// -------------------------------------------------------------------------
static void IoDeviceDestroy(OBJECT* object) {
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == dev) {
gPhysicalDevices[i] = gPhysicalDevices[gPhysicalDeviceCount - 1u];
gPhysicalDevices[gPhysicalDeviceCount - 1u] = (DEVICE_OBJECT*)0;
gPhysicalDeviceCount--;
break;
}
}
if (dev->LowerDevice)
IoDetachDevice(dev);
if (dev->Parent) {
DEVICE_OBJECT* parent = dev->Parent;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
DEVICE_OBJECT** link = &parent->FirstChild;
while (*link) {
if (*link == dev) {
*link = dev->NextSibling;
break;
}
link = &(*link)->NextSibling;
}
KeReleaseSpinlock(&parent->Lock, flags);
dev->Parent = (DEVICE_OBJECT*)0;
dev->NextSibling = (DEVICE_OBJECT*)0;
IoDereferenceDeviceObject(parent);
}
}
static KSTATUS IoCreateDeviceInternal(DRIVER_OBJECT* driver,
DEVICE_OBJECT_TYPE type,
size_t extensionSize,
DEVICE_OBJECT** outDevice) {
if (!driver || !outDevice) return STATUS_INVALID_PARAMETER;
size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)ObAllocateObject(ObjectTypeDevice, total);
if (!dev) return STATUS_NO_MEMORY;
dev->Header.Destroy = IoDeviceDestroy;
dev->DriverObject = driver;
dev->Type = type;
dev->PnpState = DevicePnpNotStarted;
dev->AlignmentRequirement = 1;
dev->Lock = (SPINLOCK)SPINLOCK_INIT;
// The per-device union is interpreted differently for PDOs and disk FDOs.
// Do not write disk defaults into a PDO: u.Disk.DiskIndex aliases the low
// byte of u.Pdo.BusType, which prevents ID-table AddDevice matching.
if (type != DeviceObjectPhysical) {
dev->u.Disk.DiskIndex = 0xFF;
dev->u.Disk.SectorSize = 512;
}
if (extensionSize)
dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));
*outDevice = dev;
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice) {
return IoCreateDeviceInternal(driver, DeviceObjectFunction, extensionSize, outDevice);
}
__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
DEVICE_OBJECT* parent,
const IO_DEVICE_ID* ids,
size_t idCount,
size_t extensionSize,
DEVICE_OBJECT** outPdo) {
if (!busDriver || !outPdo) return STATUS_INVALID_PARAMETER;
if (idCount > IO_MAX_DEVICE_IDS) return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDeviceInternal(busDriver, DeviceObjectPhysical, extensionSize, &pdo);
if (!KSUCCESS(st)) return st;
if (idCount && ids) {
for (size_t i = 0; i < idCount; i++) {
if (!SysSvIsValid(ids[i].Id) || ids[i].Id.Length >= IO_MAX_DEVICE_ID_CHARS) {
IoDeleteDevice(pdo);
return STATUS_INVALID_PARAMETER;
}
char* dst = pdo->u.Pdo.IdStorage[i];
if (ids[i].Id.Length)
memcpy(dst, ids[i].Id.Data, ids[i].Id.Length);
dst[ids[i].Id.Length] = '\0';
pdo->u.Pdo.Ids[i].BusType = ids[i].BusType;
pdo->u.Pdo.Ids[i].Id = SysSvFromParts(dst, ids[i].Id.Length);
if (pdo->u.Pdo.BusType == IoBusUnknown)
pdo->u.Pdo.BusType = ids[i].BusType;
}
pdo->u.Pdo.IdCount = (uint8_t)idCount;
}
if (parent) {
IoReferenceDeviceObject(parent);
pdo->Parent = parent;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
pdo->NextSibling = parent->FirstChild;
parent->FirstChild = pdo;
KeReleaseSpinlock(&parent->Lock, flags);
}
*outPdo = pdo;
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
if (!pdo || pdo->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == pdo)
return STATUS_ALREADY_EXISTS;
}
if (gPhysicalDeviceCount >= MAX_PDO_OBJECTS)
return STATUS_NO_MEMORY;
gPhysicalDevices[gPhysicalDeviceCount++] = pdo;
bool anyMatched = false;
for (uint32_t i = 0; i < gDriverObjectCount; i++) {
if (IoTryAddDeviceForDriver(gDriverObjects[i], pdo))
anyMatched = true;
}
if (!anyMatched && pdo->u.Pdo.BusType == IoBusUsb) {
SYS_STRING_VIEW id = pdo->u.Pdo.IdCount ? pdo->u.Pdo.Ids[0].Id : SYS_SV_LITERAL("<no-id>");
DbgPrint("io: no function driver for pdo=%v id=%v bus=%s\n",
IoSvOrUnnamed(pdo->Name), id, IoBusTypeName(pdo->u.Pdo.BusType));
}
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
SYS_STRING_VIEW name) {
if (!device || !SysSvIsValid(name))
return STATUS_INVALID_PARAMETER;
if (name.Length >= IO_MAX_DEVICE_NAME_CHARS)
return STATUS_BUFFER_TOO_SMALL;
if (name.Length)
memcpy(device->NameStorage, name.Data, name.Length);
device->NameStorage[name.Length] = '\0';
device->Name = SysSvFromParts(device->NameStorage, name.Length);
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObReferenceObject(&device->Header);
}
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObDereferenceObject(&device->Header);
}
static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
if (!device || device->Type != DeviceObjectPhysical)
return;
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == device) {
for (uint32_t j = i + 1; j < gPhysicalDeviceCount; j++)
gPhysicalDevices[j - 1] = gPhysicalDevices[j];
gPhysicalDeviceCount--;
if (gPhysicalDeviceCount < MAX_PDO_OBJECTS)
gPhysicalDevices[gPhysicalDeviceCount] = (DEVICE_OBJECT*)0;
return;
}
}
}
static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
if (!device || !device->Parent)
return;
DEVICE_OBJECT* parent = device->Parent;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
DEVICE_OBJECT** link = &parent->FirstChild;
while (*link) {
if (*link == device) {
*link = device->NextSibling;
break;
}
link = &(*link)->NextSibling;
}
device->Parent = (DEVICE_OBJECT*)0;
device->NextSibling = (DEVICE_OBJECT*)0;
KeReleaseSpinlock(&parent->Lock, flags);
IoDereferenceDeviceObject(parent);
}
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
if (device->DeletePending)
return;
device->DeletePending = true;
device->PnpState = DevicePnpDeleted;
IoUnlinkDeviceFromParent(device);
IoRemovePhysicalDeviceRegistryEntry(device);
IoDereferenceDeviceObject(device);
}
// -------------------------------------------------------------------------
// Device stack management
// -------------------------------------------------------------------------
__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
DEVICE_OBJECT* target) {
if (!source || !target) return (DEVICE_OBJECT*)0;
DEVICE_OBJECT* top = target;
while (top->AttachedDevice)
top = top->AttachedDevice;
top->AttachedDevice = source;
source->LowerDevice = top;
IoReferenceDeviceObject(top);
return top;
}
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
if (!source || !source->LowerDevice) return;
DEVICE_OBJECT* lower = source->LowerDevice;
if (lower->AttachedDevice == source)
lower->AttachedDevice = (DEVICE_OBJECT*)0;
source->LowerDevice = (DEVICE_OBJECT*)0;
IoDereferenceDeviceObject(lower);
}
__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device) {
if (!device) return (DEVICE_OBJECT*)0;
DEVICE_OBJECT* top = device;
while (top->AttachedDevice)
top = top->AttachedDevice;
IoReferenceDeviceObject(top);
return top;
}
// -------------------------------------------------------------------------
// Device-stack helpers / PnP dispatch
// -------------------------------------------------------------------------
__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
uint8_t count = 0;
for (DEVICE_OBJECT* cur = device; cur && count < 0xFEu; cur = cur->LowerDevice)
count++;
return count ? count : 1;
}
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->LowerDevice)
return STATUS_NOT_SUPPORTED;
IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
if (!cur || !next)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* currentDevice = cur->DeviceObject;
memcpy(next, cur, sizeof(*next));
if (currentDevice == device) {
cur->DeviceObject = (DEVICE_OBJECT*)0;
_InterlockedDecrement(&device->OutstandingIrps);
IoDereferenceDeviceObject(device);
}
irp->CurrentLocation++;
return IoCallDriver(device->LowerDevice, irp);
}
__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
IO_PNP_MINOR_FUNCTION minor) {
if (!device) return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* top = IoGetAttachedDeviceReference(device);
if (!top) return STATUS_INVALID_PARAMETER;
uint8_t stackCount = IoGetDeviceStackSize(top);
IRP* irp = (IRP*)0;
KSTATUS st = IoAllocateIrp(stackCount, &irp);
if (!KSUCCESS(st)) {
IoDereferenceDeviceObject(top);
return st;
}
IoBuildPnpIrp(irp, minor);
st = IoSubmitIrpSync(top, irp);
IoFreeIrp(irp);
IoDereferenceDeviceObject(top);
return st;
}
// -------------------------------------------------------------------------
// Legacy publish/matcher path
// -------------------------------------------------------------------------
typedef struct IO_DRIVER_MATCHER_ENTRY {
bool Used;
@@ -23,14 +433,6 @@ static uint32_t gPublishedDeviceCount;
static IO_DRIVER_MATCHER_ENTRY gDriverMatchers[MAX_DRIVER_MATCHERS];
static uint32_t gDriverMatcherCount;
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
switch (busType) {
case IoBusPci: return "pci";
case IoBusUsb: return "usb";
default: return "unknown";
}
}
static bool IoTryBindDevice(IO_PUBLISHED_DEVICE* device,
IO_DRIVER_MATCHER_ENTRY* matcher) {
if (!device || !matcher || !matcher->Used || device->Bound)
@@ -80,12 +482,10 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
entry->Bind = bind;
entry->Context = context;
DbgPrint("io: registered %s matcher %s\n",
IoBusTypeName(busType), driverName);
DbgPrint("io: registered %s matcher %s\n", IoBusTypeName(busType), driverName);
for (uint32_t i = 0; i < gPublishedDeviceCount; i++) {
for (uint32_t i = 0; i < gPublishedDeviceCount; i++)
IoTryBindDevice(&gPublishedDevices[i], entry);
}
return STATUS_SUCCESS;
}
@@ -101,7 +501,6 @@ __declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device)
stored->Bound = false;
stored->BoundDriverName = (const char*)0;
for (uint32_t i = 0; i < gDriverMatcherCount; i++) {
if (IoTryBindDevice(stored, &gDriverMatchers[i]))
break;
@@ -114,50 +513,44 @@ __declspec(dllexport) uint32_t IoPublishedDeviceCount(void) {
return gPublishedDeviceCount;
}
#define MAX_DISK_DEVICES 8
#define MAX_DISK_ARRIVAL_CALLBACKS 16
// -------------------------------------------------------------------------
// Disk namespace
// -------------------------------------------------------------------------
typedef struct IO_DISK_CALLBACK_ENTRY {
#define MAX_DISK_DEVICES 8
#define MAX_DISK_CALLBACKS 16
typedef struct IO_DISK_CHANGE_CALLBACK_ENTRY {
bool Used;
IO_DISK_CHANGE_CALLBACK Callback;
void* Context;
} IO_DISK_CHANGE_CALLBACK_ENTRY;
typedef struct IO_DISK_ARRIVAL_CALLBACK_ENTRY {
bool Used;
IO_DISK_ARRIVAL_CALLBACK Callback;
void* Context;
} IO_DISK_CALLBACK_ENTRY;
} IO_DISK_ARRIVAL_CALLBACK_ENTRY;
static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES];
static IO_DISK_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_ARRIVAL_CALLBACKS];
static IO_DISK_CHANGE_CALLBACK_ENTRY gDiskChangeCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
static void IoNotifyDiskArrived(uint8_t driveIndex, DEVICE_OBJECT* device) {
for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i];
if (entry->Used && entry->Callback)
entry->Callback(driveIndex, device, entry->Context);
entry->Callback(driveIndex, device, change, entry->Context);
}
if (change == IoDeviceArrived) {
for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
if (entry->Used && entry->Callback)
entry->Callback(driveIndex, device, entry->Context);
}
}
}
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice) {
if (!driver || !outDevice) return STATUS_INVALID_PARAMETER;
size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)MmAllocatePool(total);
if (!dev) return STATUS_NO_MEMORY;
memset(dev, 0, total);
dev->DriverObject = driver;
if (extensionSize)
dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));
dev->AlignmentRequirement = 1;
dev->DiskIndex = 0xFF;
*outDevice = dev;
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
MmFreePool(device);
}
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
@@ -168,9 +561,57 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
return STATUS_ALREADY_EXISTS;
device->Flags |= DEVICE_OBJECT_DISK;
device->DiskIndex = driveIndex;
device->u.Disk.DiskIndex = driveIndex;
if (device->u.Disk.SectorSize == 0)
device->u.Disk.SectorSize = 512;
KSTATUS pnpStatus = IoSendPnpIrpSync(device, IRP_MN_START_DEVICE);
if (!KSUCCESS(pnpStatus) && pnpStatus != STATUS_NOT_SUPPORTED)
return pnpStatus;
if (pnpStatus == STATUS_NOT_SUPPORTED)
device->PnpState = DevicePnpStarted;
IoReferenceDeviceObject(device);
gDiskDevices[driveIndex] = device;
IoNotifyDiskArrived(driveIndex, device);
IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return STATUS_NOT_FOUND;
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return STATUS_NOT_FOUND;
if (dev->PnpState != DevicePnpSurpriseRemoved) {
IoNotifyDiskChange(driveIndex, dev, IoDeviceQueryRemove);
KSTATUS queryStatus = IoSendPnpIrpSync(dev, IRP_MN_QUERY_REMOVE_DEVICE);
if (!KSUCCESS(queryStatus) && queryStatus != STATUS_NOT_SUPPORTED)
return queryStatus;
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
}
dev->PnpState = DevicePnpRemovePending;
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
}
@@ -178,10 +619,10 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL
void* context) {
if (!callback)
return STATUS_INVALID_PARAMETER;
if (gDiskArrivalCallbackCount >= MAX_DISK_ARRIVAL_CALLBACKS)
if (gDiskArrivalCallbackCount >= MAX_DISK_CALLBACKS)
return STATUS_NO_MEMORY;
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
IO_DISK_ARRIVAL_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
memset(entry, 0, sizeof(*entry));
entry->Used = true;
entry->Callback = callback;
@@ -191,7 +632,21 @@ __declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALL
if (gDiskDevices[i])
callback(i, gDiskDevices[i], context);
}
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback,
void* context) {
if (!callback)
return STATUS_INVALID_PARAMETER;
if (gDiskChangeCallbackCount >= MAX_DISK_CALLBACKS)
return STATUS_NO_MEMORY;
IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[gDiskChangeCallbackCount++];
memset(entry, 0, sizeof(*entry));
entry->Used = true;
entry->Callback = callback;
entry->Context = context;
return STATUS_SUCCESS;
}
@@ -204,7 +659,6 @@ __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK ca
if (gDiskDevices[i])
callback(i, gDiskDevices[i], context);
}
return STATUS_SUCCESS;
}
@@ -213,28 +667,59 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
return gDiskDevices[driveIndex];
}
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
if (dev) IoReferenceDeviceObject(dev);
return dev;
}
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
}
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
return dev ? dev->SectorCount : 0;
return dev ? dev->u.Disk.SectorCount : 0;
}
// -------------------------------------------------------------------------
// Dispatch
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DriverObject || !irp)
return STATUS_INVALID_PARAMETER;
IO_MAJOR_FUNCTION major = (irp->Flags & IRP_FLAG_WRITE) ? IRP_MJ_WRITE : IRP_MJ_READ;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios)
return STATUS_INVALID_PARAMETER;
IO_MAJOR_FUNCTION major = ios->MajorFunction;
if (major >= IRP_MJ_MAXIMUM_FUNCTION)
return STATUS_INVALID_PARAMETER;
if (major != IRP_MJ_PNP &&
(device->DeletePending || device->PnpState == DevicePnpSurpriseRemoved ||
device->PnpState == DevicePnpRemovePending || device->PnpState == DevicePnpDeleted))
return STATUS_OBJECT_DEAD;
PDRIVER_DISPATCH dispatch = device->DriverObject->MajorFunction[major];
if (!dispatch)
return STATUS_NOT_SUPPORTED;
return dispatch(device, irp);
ios->DeviceObject = device;
IoReferenceDeviceObject(device);
_InterlockedIncrement(&device->OutstandingIrps);
KSTATUS st = dispatch(device, irp);
if (st != STATUS_PENDING) {
if (ios->DeviceObject == device) {
ios->DeviceObject = (DEVICE_OBJECT*)0;
_InterlockedDecrement(&device->OutstandingIrps);
IoDereferenceDeviceObject(device);
}
}
return st;
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
@@ -243,17 +728,15 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
void* buffer) {
if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
memset(&irp, 0, sizeof(irp));
irp.DriveIndex = driveIndex;
irp.Flags = 0;
irp.Lba = lba;
irp.SectorCount = count;
irp.Buffer = buffer;
irp.Status = STATUS_FAILURE;
irp.Information = 0;
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
return IoSubmitIrpSync(&irp);
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
IoDereferenceDeviceObject(dev);
return st;
}
__declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
@@ -262,15 +745,13 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
const void* buffer) {
if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
memset(&irp, 0, sizeof(irp));
irp.DriveIndex = driveIndex;
irp.Flags = IRP_FLAG_WRITE;
irp.Lba = lba;
irp.SectorCount = count;
irp.Buffer = (void*)buffer;
irp.Status = STATUS_FAILURE;
irp.Information = 0;
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
return IoSubmitIrpSync(&irp);
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
IoDereferenceDeviceObject(dev);
return st;
}
code/kernel/io_file.c
+27
View File
@@ -586,6 +586,33 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return STATUS_INVALID_PARAMETER;
if (!gVolumes[volumeIndex].Fs)
return STATUS_NOT_FOUND;
memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex]));
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
KSTATUS last = STATUS_NOT_FOUND;
for (uint8_t i = 0; i < MAX_DRIVES; i++) {
if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) {
memset(&gVolumes[i], 0, sizeof(gVolumes[i]));
last = STATUS_SUCCESS;
}
}
return last;
}
__declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return false;
return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
}
// ---------------------------------------------------------------------------
// Destructor
// ---------------------------------------------------------------------------
code/kernel/ke_cpu.c
+9
View File
@@ -48,6 +48,15 @@ void KeInitializeBootstrapCpu(void) {
cpu->Parked = 0;
cpu->Online = 1;
cpu->IsBsp = 1;
cpu->PackageId = 0;
cpu->CoreId = 0;
cpu->ThreadId = 0;
cpu->PhysicalCoreIndex = 0;
cpu->CoreSiblingMask = 1ULL;
cpu->PackageMask = 1ULL;
cpu->IsPrimaryThread = 1;
cpu->SmtThreadCount = 1;
cpu->TopologyFlags = 0;
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
code/kernel/ke_debug.c
+124 -7
View File
@@ -13,12 +13,53 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/mm_virt.h>
#include <kernel/io_file.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor
int gDebugEnabled = 0;
static uint32_t gDebugFlags = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile long gDebugLogFlushing;
static void KiDebugLogAppendLocked(const char* text, size_t len) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
return;
if (len >= gDebugLogSize) {
text += len - (gDebugLogSize - 1u);
len = gDebugLogSize - 1u;
}
size_t first = gDebugLogSize - gDebugLogWrite;
if (first > len) first = len;
memcpy(gDebugLogBuffer + gDebugLogWrite, text, first);
if (len > first)
memcpy(gDebugLogBuffer, text + first, len - first);
gDebugLogWrite = (gDebugLogWrite + len) % gDebugLogSize;
if (gDebugLogUsed + len >= gDebugLogSize)
gDebugLogUsed = gDebugLogSize;
else
gDebugLogUsed += len;
}
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
return;
for (size_t i = 0; i < len; i++)
PrintChar(text[i], 0x00FF00, 0x000000);
}
static volatile long gPanicInProgress = 0;
static volatile long gPanicStoppedCpuCount = 0;
@@ -50,22 +91,98 @@ void KiKernelPutChar(char c) {
PrintChar(c, 0x00FF00, 0x000000);
}
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) {
gDebugEnabled = v;
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
gDebugFlags = flags;
gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE);
if (gDebugLogBuffer) {
gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
gDebugLogWrite = 0;
gDebugLogUsed = 0;
memset(gDebugLogBuffer, 0, gDebugLogSize);
}
}
}
uint32_t KeGetDebugFlags(void) {
return gDebugFlags;
}
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled) return;
if (!gDebugEnabled || !fmt) return;
static SPINLOCK sl = {0};
uintptr_t flags = KeAcquireSpinlock(&sl);
char line[KE_DEBUG_LOG_LINE_SIZE];
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
int n = vsnprintf_(line, sizeof(line), fmt, va);
va_end(va);
KeReleaseSpinlock(&sl, flags);
size_t len = 0;
if (n > 0) {
len = (size_t)n;
if (len >= sizeof(line))
len = sizeof(line) - 1u;
}
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE)
KiDebugConsoleWrite(line, len);
KeReleaseSpinlock(&gDebugPrintLock, flags);
}
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
if (!KeCanWait())
return STATUS_INVALID_IRQL;
if (_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 1, 0) != 0)
return STATUS_PENDING;
size_t used;
size_t start;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
used = gDebugLogUsed;
start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (used == 0) {
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return STATUS_SUCCESS;
}
(void)IoDeleteFile(path);
FILE_OBJECT* file = (FILE_OBJECT*)0;
KSTATUS st = IoCreateFile(path, &file);
if (!KSUCCESS(st)) {
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return st;
}
size_t written = 0;
size_t totalWritten = 0;
size_t first = gDebugLogSize - start;
if (first > used) first = used;
st = IoWriteFile(file, gDebugLogBuffer + start, first, &written);
totalWritten += written;
if (KSUCCESS(st) && used > first) {
written = 0;
st = IoWriteFile(file, gDebugLogBuffer, used - first, &written);
totalWritten += written;
}
IoCloseFile(file);
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// KePanic
code/kernel/ke_irp.c
+259 -68
View File
@@ -1,130 +1,321 @@
// ke_irp.c - I/O Request Packet dispatch and queue helpers
//
// This file lives in the kernel layer and is the ONLY place that bridges the
// IRP abstraction to kernel sync objects. All calls to KeEventCreate,
// KeSignalEvent, KeWaitForEvent, KeSignalSemaphore, and ObDereferenceObject
// are confined here; hal_disk.c and io_driver.h remain free of those symbols.
//
// Dependency graph (→ = includes / calls):
// ke_irp.c → ke_irp.h (IRP type, IRP_QUEUE, forward decls)
// → ke_event.h (KeEventCreate, KeSignalEvent, KeWaitForEvent)
// → ke_semaphore.h (KeSignalSemaphore)
// → ob_object.h (ObDereferenceObject)
// → io_driver.h (DEVICE_OBJECT / IoCallDriver)
// ke_irp.c - I/O Request Packet dispatch, queue, cancellation, remove-locks
#include <string.h>
#include <intrin.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_event.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_apc.h>
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
#include <ntlite/ddi/io_driver.h>
// -------------------------------------------------------------------------
// IRP_QUEUE operations
// -------------------------------------------------------------------------
// These are defined here (not as static inlines in ke_irp.h) so that
// ke_irp.h can avoid including ke_semaphore.h, keeping the header safe
// for inclusion from the HAL layer.
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE *q, struct SEMAPHORE *sem) {
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
q->Lock = (SPINLOCK)SPINLOCK_INIT;
q->Semaphore = sem;
LIST_INIT(&q->Head);
}
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE *q, IRP *irp) {
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
if (irp->Flags & IRP_FLAG_PRIORITY)
ListInsertHead(&q->Head, &irp->Link);
else
ListInsertTail(&q->Head, &irp->Link);
KeReleaseSpinlock(&q->Lock, flags);
// Signal outside the spinlock - safe, and avoids priority-inversion risk.
KeSignalSemaphore((SEMAPHORE *)q->Semaphore);
KeSignalSemaphore((SEMAPHORE*)q->Semaphore);
}
__declspec(dllexport) IRP *IrpQueueDequeue(IRP_QUEUE *q) {
__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY *node = ListPopHead(&q->Head);
LIST_ENTRY* node = ListPopHead(&q->Head);
KeReleaseSpinlock(&q->Lock, flags);
if (!node) return (IRP *)0;
if (!node) return (IRP*)0;
return CONTAINING_RECORD(node, IRP, Link);
}
// -------------------------------------------------------------------------
// IoCompleteRequest
// -------------------------------------------------------------------------
__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context) {
if (!q) return;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY* node = ListPopHead(&q->Head);
KeReleaseSpinlock(&q->Lock, flags);
if (!node) break;
static size_t IoIrpTransferBytes(const IRP *irp, KSTATUS status) {
if (!irp || !KSUCCESS(status)) return 0;
return (size_t)irp->SectorCount * 512u;
IRP* irp = CONTAINING_RECORD(node, IRP, Link);
irp->Cancel = true;
irp->Flags |= IRP_FLAG_CANCELLED;
if (callback)
callback(irp, context);
IoCompleteRequest(irp, status, 0);
}
}
__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information) {
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
}
// -------------------------------------------------------------------------
// Remove locks
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return STATUS_INVALID_PARAMETER;
memset(lock, 0, sizeof(*lock));
lock->Lock = (SPINLOCK)SPINLOCK_INIT;
lock->IoCount = 1; // sentinel reference released by IoReleaseRemoveLockAndWait
KSTATUS st = KeEventCreate(EventManualReset, 0, &lock->RemovedEvent);
if (!KSUCCESS(st)) return st;
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return;
if (lock->RemovedEvent) {
ObDereferenceObject(&lock->RemovedEvent->Header);
lock->RemovedEvent = (EVENT*)0;
}
}
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
if (lock->Removing) {
KeReleaseSpinlock(&lock->Lock, flags);
return STATUS_OBJECT_DEAD;
}
_InterlockedIncrement(&lock->IoCount);
KeReleaseSpinlock(&lock->Lock, flags);
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return;
long left = _InterlockedDecrement(&lock->IoCount);
if (left == 0 && lock->Removing && lock->RemovedEvent)
KeSignalEvent(lock->RemovedEvent);
}
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp) {
(void)removeIrp;
if (!lock) return;
uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
lock->Removing = true;
long left = _InterlockedDecrement(&lock->IoCount); // release sentinel
bool done = (left == 0);
KeReleaseSpinlock(&lock->Lock, flags);
if (done) {
if (lock->RemovedEvent) KeSignalEvent(lock->RemovedEvent);
return;
}
if (lock->RemovedEvent)
KeWaitForEvent(lock->RemovedEvent, 0);
}
// -------------------------------------------------------------------------
// IRP helpers
// -------------------------------------------------------------------------
static size_t IoIrpSize(uint8_t stackCount) {
if (stackCount == 0) stackCount = 1;
return sizeof(IRP) + ((size_t)stackCount - 1u) * sizeof(IO_STACK_LOCATION);
}
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
if (!outIrp) return STATUS_INVALID_PARAMETER;
if (stackCount == 0) stackCount = 1;
IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount));
if (!irp) return STATUS_NO_MEMORY;
IoInitializeIrp(irp, stackCount);
*outIrp = irp;
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoFreeIrp(IRP* irp) {
if (irp) MmFreePool(irp);
}
__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
if (!irp) return;
if (stackCount == 0) stackCount = 1;
memset(irp, 0, IoIrpSize(stackCount));
LIST_INIT(&irp->Link);
irp->Status = STATUS_FAILURE;
irp->Information = 0;
irp->CancelLock = (SPINLOCK)SPINLOCK_INIT;
irp->StackCount = stackCount;
irp->CurrentLocation = 0;
}
__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation];
}
__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation + 1u >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation + 1u];
}
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) {
if (!irp) return;
if (irp->CurrentLocation + 1u < irp->StackCount)
irp->CurrentLocation++;
}
__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
if (!cur || !next) return;
memcpy(next, cur, sizeof(*next));
irp->CurrentLocation++;
}
__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx) {
if (!irp) return;
irp->CompletionFn = fn;
irp->CompletionCtx = ctx;
}
__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine) {
if (!irp) return (IRP_CANCEL_ROUTINE)0;
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
IRP_CANCEL_ROUTINE old = irp->CancelRoutine;
irp->CancelRoutine = routine;
KeReleaseSpinlock(&irp->CancelLock, flags);
return old;
}
__declspec(dllexport) void IoCancelIrp(IRP* irp) {
if (!irp) return;
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
irp->Cancel = true;
irp->Flags |= IRP_FLAG_CANCELLED;
IRP_CANCEL_ROUTINE routine = irp->CancelRoutine;
irp->CancelRoutine = (IRP_CANCEL_ROUTINE)0;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
KeReleaseSpinlock(&irp->CancelLock, flags);
if (routine)
routine(dev, irp);
}
__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint64_t byteOffset,
uint32_t length,
void* buffer) {
if (!irp) return;
IoInitializeIrp(irp, 1);
irp->UserBuffer = buffer;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return;
ios->MajorFunction = major;
ios->MinorFunction = 0;
ios->Parameters.ReadWrite.ByteOffset = byteOffset;
ios->Parameters.ReadWrite.Length = length;
}
__declspec(dllexport) void IoBuildPnpIrp(IRP* irp,
IO_PNP_MINOR_FUNCTION minor) {
if (!irp) return;
uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
IoInitializeIrp(irp, stackCount);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return;
ios->MajorFunction = IRP_MJ_PNP;
ios->MinorFunction = (uint8_t)minor;
}
__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
IO_MAJOR_FUNCTION major,
uint32_t ioControlCode,
void* inputBuffer,
uint32_t inputLength,
void* outputBuffer,
uint32_t outputLength) {
if (!irp) return;
uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
IoInitializeIrp(irp, stackCount);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios) return;
ios->MajorFunction = major;
ios->MinorFunction = 0;
ios->Parameters.DeviceIoControl.IoControlCode = ioControlCode;
ios->Parameters.DeviceIoControl.InputBuffer = inputBuffer;
ios->Parameters.DeviceIoControl.InputLength = inputLength;
ios->Parameters.DeviceIoControl.OutputBuffer = outputBuffer;
ios->Parameters.DeviceIoControl.OutputLength = outputLength;
}
// -------------------------------------------------------------------------
// Completion/submission
// -------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information) {
if (!irp) return STATUS_INVALID_PARAMETER;
if (irp->Flags & IRP_FLAG_COMPLETED)
return STATUS_ALREADY_EXISTS;
irp->Flags |= IRP_FLAG_COMPLETED;
irp->Status = status;
irp->Information = information;
// Queue the APC before waking event waiters so a thread that waits on the
// completion event will see and deliver the APC on its wait-return path.
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
if (dev) {
_InterlockedDecrement(&dev->OutstandingIrps);
IoDereferenceDeviceObject(dev);
ios->DeviceObject = (DEVICE_OBJECT*)0;
}
if (irp->CompletionApc)
KeInsertQueueApc((KAPC *)irp->CompletionApc, irp, (void *)(uintptr_t)status);
KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status);
if (irp->CompletionEvent)
KeSignalEvent((EVENT *)irp->CompletionEvent);
KeSignalEvent((EVENT*)irp->CompletionEvent);
if (irp->CompletionFn)
irp->CompletionFn(irp, irp->CompletionCtx);
return STATUS_SUCCESS;
}
// -------------------------------------------------------------------------
// IoSubmitIrp
// -------------------------------------------------------------------------
// Dispatches through the registered DEVICE_OBJECT.
// Accepted IRPs are always completed by the target driver via IoCompleteRequest().
__declspec(dllexport) KSTATUS IoSubmitIrp(IRP *irp) {
if (!irp) return STATUS_INVALID_PARAMETER;
__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
irp->Status = STATUS_FAILURE;
irp->Information = 0;
DEVICE_OBJECT *device = IoGetDiskDevice(irp->DriveIndex);
if (!device) return STATUS_NOT_FOUND;
KSTATUS st = IoCallDriver(device, irp);
// Definitive driver contract:
// STATUS_PENDING means the IRP was accepted and will be completed via
// IoCompleteRequest(), possibly inline before IoCallDriver returns.
// Any other status means the IRP was rejected and no completion occurs.
return st;
irp->Flags &= ~IRP_FLAG_COMPLETED;
return IoCallDriver(device, irp);
}
// -------------------------------------------------------------------------
// IoSubmitIrpSync
// -------------------------------------------------------------------------
// Creates a stack-allocated auto-reset event, attaches it to the IRP,
// submits, and blocks. Callers that already provided a CompletionEvent
// should use IoSubmitIrp() directly and wait on their own event instead.
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp) {
if (!irp) return STATUS_INVALID_PARAMETER;
EVENT *ev = (EVENT *)0;
EVENT* ev = (EVENT*)0;
KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev);
if (!KSUCCESS(st)) return st;
// Attach our event; preserve any APC/callback the caller may have set.
irp->CompletionEvent = ev;
st = IoSubmitIrp(irp);
st = IoSubmitIrp(device, irp);
if (st == STATUS_PENDING) {
KeWaitForEvent(ev, 0);
st = irp->Status;
}
// Release our reference to the temporary event object.
ObDereferenceObject(&ev->Header);
return st;
}
code/kernel/ke_sched.c
+73 -2
View File
@@ -36,6 +36,44 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
}
static int KiCpuIsIdleLocked(const KCPU* cpu) {
if (!cpu || !cpu->Online)
return 0;
THREAD* cur = cpu->CurrentThread;
return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
}
static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
if (!thread || thread->IsIdleThread)
return 0;
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
if (!cpu->IsPrimaryThread)
continue;
if (!KiCpuIsIdleLocked(cpu))
continue;
if (KiThreadCanRunOnCpuLocked(thread, cpu))
return 1;
}
return 0;
}
static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
if (!cpu || !thread || thread->IsIdleThread)
return 0;
if (cpu->IsPrimaryThread || cpu->SmtThreadCount <= 1)
return 0;
// Do not fill a secondary SMT sibling while any primary hardware thread
// that can run this thread is idle. KiPokeRunnableCpusLocked targets those
// primary CPUs first, so this CPU can safely remain idle and wait for more
// work or for physical cores to fill.
return KiAnyIdlePrimaryForThreadLocked(thread);
}
static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || !target->Online)
return;
@@ -52,6 +90,9 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
if (!thread || thread->IsIdleThread)
return;
int haveIdlePrimary = KiAnyIdlePrimaryForThreadLocked(thread);
int requestedIdle = 0;
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
@@ -61,12 +102,35 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
THREAD* cur = cpu->CurrentThread;
if (!cur || cur == cpu->IdleThread) {
if (haveIdlePrimary && !cpu->IsPrimaryThread)
continue;
if (cpu->IsPrimaryThread)
gSchedStats.SmtPrimaryPokes++;
KiRequestRescheduleCpuLocked(cpu);
requestedIdle = 1;
} else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) {
gSchedStats.PriorityPreemptions++;
KiRequestRescheduleCpuLocked(cpu);
}
}
// If no preferred idle primary was actually targetable, allow an idle SMT
// sibling to take the work. This covers explicit affinity to a secondary
// logical CPU and partially-online systems.
if (!requestedIdle && haveIdlePrimary == 0) {
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
THREAD* cur = cpu->CurrentThread;
if (!cur || cur == cpu->IdleThread) {
KiRequestRescheduleCpuLocked(cpu);
break;
}
}
}
}
static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
@@ -369,8 +433,11 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
LIST_ENTRY* head = &gRunQueue[p];
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (KiThreadCanRunOnCpuLocked(thread, cpu))
return 1;
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
if (KiShouldDeferToIdlePrimaryLocked(cpu, thread))
continue;
return 1;
}
}
return 0;
@@ -385,6 +452,10 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) {
gSchedStats.SmtSiblingDeferrals++;
continue;
}
ListRemove(node);
thread->Queued = 0;
code/kernel/ke_selftest.c
+8 -11
View File
@@ -352,6 +352,7 @@ static void KiRunKernelApcSelfTest(void) {
typedef struct KI_IRP_APC_SELFTEST_STATE {
EVENT* CompletionEvent;
DEVICE_OBJECT* DiskDevice;
IRP Irp;
KAPC Apc;
uint8_t* Buffer;
@@ -430,23 +431,18 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
KiIrpApcSelfTestNormalRoutine,
(void*)&st);
LIST_INIT(&st.Irp.Link);
st.Irp.DriveIndex = 0;
st.Irp.Flags = 0;
st.Irp.Lba = 0;
st.Irp.SectorCount = 1;
st.Irp.Buffer = st.Buffer;
st.Irp.Status = STATUS_FAILURE;
st.Irp.Information = 0;
st.DiskDevice = IoReferenceDiskDevice(0);
if (!st.DiskDevice)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: drive 0 disappeared before submit\n");
IoBuildReadWriteIrp(&st.Irp, IRP_MJ_READ, 0, 512, st.Buffer);
st.Irp.CompletionEvent = st.CompletionEvent;
st.Irp.CompletionApc = &st.Apc;
st.Irp.CompletionFn = (IRP_COMPLETION_FN)0;
st.Irp.CompletionCtx = (void*)0;
st.Irp.HwSlot = 0;
st.Irp.DriverCtx = (void*)0;
DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n");
s = IoSubmitIrp(&st.Irp);
s = IoSubmitIrp(st.DiskDevice, &st.Irp);
if (s != STATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: IoSubmitIrp returned %u\n", s);
@@ -472,6 +468,7 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
st.Buffer[510], st.Buffer[511]);
MmFreePool(st.Buffer);
if (st.DiskDevice) IoDereferenceDeviceObject(st.DiskDevice);
ObDereferenceObject(&st.CompletionEvent->Header);
DbgPrint("IRP/APC selftest: PASS\n");
code/kernel/ke_smp.c
+227
View File
@@ -321,6 +321,15 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->Parked = 0;
cpu->Online = 0;
cpu->IsBsp = info->IsBsp ? 1 : 0;
cpu->PackageId = 0;
cpu->CoreId = cpuIndex;
cpu->ThreadId = 0;
cpu->PhysicalCoreIndex = cpuIndex;
cpu->CoreSiblingMask = (cpuIndex < 64) ? (1ULL << cpuIndex) : 0ULL;
cpu->PackageMask = cpu->CoreSiblingMask;
cpu->IsPrimaryThread = 1;
cpu->SmtThreadCount = 1;
cpu->TopologyFlags = 0;
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
@@ -328,6 +337,223 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->SchedulerActive = 0;
}
static uint32_t KiCeilLog2U32(uint32_t value) {
if (value <= 1u)
return 0u;
uint32_t shift = 0;
uint32_t n = value - 1u;
while (n) {
shift++;
n >>= 1;
}
return shift;
}
static uint32_t KiMaskWidth(uint32_t shift) {
if (shift >= 32u)
return 0xFFFFFFFFu;
if (shift == 0u)
return 0u;
return (1u << shift) - 1u;
}
typedef struct KI_CPUID_TOPOLOGY_PARAMS {
uint32_t SmtShift;
uint32_t CoreShift;
uint32_t ThreadsPerCore;
uint32_t LogicalPerPackage;
const char* Source;
} KI_CPUID_TOPOLOGY_PARAMS;
static void KiReadCpuidTopologyLeaf(uint32_t leaf, KI_CPUID_TOPOLOGY_PARAMS* out) {
if (!out)
return;
for (uint32_t sub = 0; sub < 32u; sub++) {
int r[4] = {0, 0, 0, 0};
__cpuidex(r, (int)leaf, (int)sub);
uint32_t levelShift = (uint32_t)r[0] & 0x1Fu;
uint32_t logicals = (uint32_t)r[1] & 0xFFFFu;
uint32_t levelType = ((uint32_t)r[2] >> 8) & 0xFFu;
if (levelType == 0u || logicals == 0u)
break;
if (levelType == 1u) { // SMT level
out->SmtShift = levelShift;
out->ThreadsPerCore = logicals ? logicals : 1u;
} else if (levelType == 2u) { // Core/package level
out->CoreShift = levelShift;
out->LogicalPerPackage = logicals ? logicals : 1u;
break;
}
}
}
static KI_CPUID_TOPOLOGY_PARAMS KiDetectCpuidTopologyParams(void) {
KI_CPUID_TOPOLOGY_PARAMS topo;
memset(&topo, 0, sizeof(topo));
topo.ThreadsPerCore = 1;
topo.LogicalPerPackage = 1;
topo.Source = "flat";
int maxRegs[4] = {0, 0, 0, 0};
__cpuid(maxRegs, 0);
uint32_t maxLeaf = (uint32_t)maxRegs[0];
if (maxLeaf >= 0x1Fu) {
KiReadCpuidTopologyLeaf(0x1Fu, &topo);
if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) {
topo.Source = "cpuid.1f";
return topo;
}
memset(&topo, 0, sizeof(topo));
topo.ThreadsPerCore = 1;
topo.LogicalPerPackage = 1;
topo.Source = "flat";
}
if (maxLeaf >= 0x0Bu) {
KiReadCpuidTopologyLeaf(0x0Bu, &topo);
if (topo.CoreShift >= topo.SmtShift && topo.CoreShift != 0u) {
topo.Source = "cpuid.0b";
return topo;
}
memset(&topo, 0, sizeof(topo));
topo.ThreadsPerCore = 1;
topo.LogicalPerPackage = 1;
topo.Source = "flat";
}
int r1[4] = {0, 0, 0, 0};
__cpuid(r1, 1);
uint32_t logicalPerPackage = ((uint32_t)r1[1] >> 16) & 0xFFu;
if (logicalPerPackage == 0u)
logicalPerPackage = 1u;
uint32_t coresPerPackage = 1u;
if (maxLeaf >= 4u) {
int r4[4] = {0, 0, 0, 0};
__cpuidex(r4, 4, 0);
coresPerPackage = (((uint32_t)r4[0] >> 26) & 0x3Fu) + 1u;
if (coresPerPackage == 0u)
coresPerPackage = 1u;
}
uint32_t threadsPerCore = logicalPerPackage / coresPerPackage;
if (threadsPerCore == 0u)
threadsPerCore = 1u;
topo.ThreadsPerCore = threadsPerCore;
topo.LogicalPerPackage = logicalPerPackage;
topo.SmtShift = KiCeilLog2U32(threadsPerCore);
topo.CoreShift = KiCeilLog2U32(logicalPerPackage);
if (topo.CoreShift < topo.SmtShift)
topo.CoreShift = topo.SmtShift;
topo.Source = "cpuid.1/4";
return topo;
}
void KeBuildProcessorTopology(void) {
KI_CPUID_TOPOLOGY_PARAMS topo = KiDetectCpuidTopologyParams();
uint32_t smtMask = KiMaskWidth(topo.SmtShift);
uint32_t coreBits = (topo.CoreShift > topo.SmtShift) ?
(topo.CoreShift - topo.SmtShift) : 0u;
uint32_t coreMask = KiMaskWidth(coreBits);
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
uint32_t apic = cpu->LapicId;
cpu->ThreadId = smtMask ? (apic & smtMask) : 0u;
cpu->CoreId = coreBits ? ((apic >> topo.SmtShift) & coreMask) :
(apic >> topo.SmtShift);
cpu->PackageId = (topo.CoreShift < 32u) ? (apic >> topo.CoreShift) : 0u;
cpu->PhysicalCoreIndex = KE_INVALID_CPU;
cpu->CoreSiblingMask = 0;
cpu->PackageMask = 0;
cpu->IsPrimaryThread = 0;
cpu->SmtThreadCount = 1;
cpu->TopologyFlags = (topo.ThreadsPerCore > 1u) ? 1u : 0u;
}
uint32_t nextCoreIndex = 0;
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
if (cpu->PhysicalCoreIndex != KE_INVALID_CPU)
continue;
uint32_t coreIndex = nextCoreIndex++;
uint64_t coreMaskBits = 0;
uint64_t packageMaskBits = 0;
uint32_t primaryCpuIndex = i;
uint32_t siblingCount = 0;
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
continue;
if (peer->PackageId == cpu->PackageId) {
if (j < 64)
packageMaskBits |= (1ULL << j);
}
if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
peer->PhysicalCoreIndex = coreIndex;
if (j < 64)
coreMaskBits |= (1ULL << j);
if (j < primaryCpuIndex)
primaryCpuIndex = j;
siblingCount++;
}
}
if (siblingCount == 0)
siblingCount = 1;
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
continue;
if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
peer->CoreSiblingMask = coreMaskBits;
peer->PackageMask = packageMaskBits;
peer->SmtThreadCount = (uint8_t)((siblingCount > 255u) ? 255u : siblingCount);
peer->IsPrimaryThread = (j == primaryCpuIndex) ? 1u : 0u;
}
}
}
DbgPrint("SMP: SMT topology source=%s smtShift=%u coreShift=%u threads/core=%u logical/package=%u physicalCores=%u\n",
topo.Source ? topo.Source : "?",
topo.SmtShift,
topo.CoreShift,
topo.ThreadsPerCore,
topo.LogicalPerPackage,
nextCoreIndex);
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
i,
cpu->LapicId,
cpu->PackageId,
cpu->CoreId,
cpu->ThreadId,
cpu->IsPrimaryThread,
(unsigned long long)cpu->CoreSiblingMask,
(unsigned long long)cpu->PackageMask);
}
}
static KSTATUS KiAllocateApStack(KCPU* cpu) {
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
@@ -510,6 +736,7 @@ KSTATUS KeStartApplicationProcessors(void) {
if (claimed == 0 && gCpuCount > 1)
DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n");
KeBuildProcessorTopology();
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
return STATUS_SUCCESS;
}
code/kernel/kmain.c
+7 -2
View File
@@ -341,7 +341,7 @@ void SystemInitThread(void* arg) {
KiResolveSystemImports(args->bootInfo);
KiReclaimBootloaderMemory(args->bootInfo);
if (args->bootInfo->Debug & 2)
if (args->bootInfo->Debug & KE_DEBUG_SELFTEST)
KiRunKernelSelfTests(args->hal);
// Launch User-Mode init.exe and restart it until it exits cleanly.
@@ -363,6 +363,11 @@ void SystemInitThread(void* arg) {
}
DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n");
if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log"));
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus);
}
gKmainHal->Power.Shutdown();
MmFreePool(args);
}
@@ -416,7 +421,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
const HAL_API* hal = HalGetApi();
gKmainHal = hal;
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1);
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug);
// CPU startup intentionally lives after HAL initialization but before IRQs
// and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI;
code/public/ntlite/ddi/io_driver.h
+173 -43
View File
@@ -1,39 +1,35 @@
#pragma once
// io_driver.h - definitive kernel driver/device I/O model
// io_driver.h - kernel driver/device I/O model
//
// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through
// IoCallDriver()/IoSubmitIrp(); drivers complete accepted IRPs by calling
// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table.
// Device lifetime is owned by the I/O manager. Bus drivers create PDOs,
// function/filter drivers attach FDOs, and PnP/remove IRPs tear stacks down.
// Disk indexes are only a compatibility namespace above DEVICE_OBJECTs.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <kernel/ob_object.h>
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irp.h>
typedef struct HAL_API HAL_API;
typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API* hal);
typedef void (*DRIVER_UNLOAD_FN)(void);
struct IRP;
typedef struct DRIVER_OBJECT DRIVER_OBJECT;
typedef struct DEVICE_OBJECT DEVICE_OBJECT;
typedef struct IRP IRP;
typedef enum IO_MAJOR_FUNCTION {
IRP_MJ_READ = 0,
IRP_MJ_WRITE = 1,
IRP_MJ_DEVICE_CONTROL = 2,
IRP_MJ_MAXIMUM_FUNCTION = 3,
} IO_MAJOR_FUNCTION;
typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, struct IRP* irp);
struct DRIVER_OBJECT {
const char* Name;
PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION];
DRIVER_UNLOAD_FN DriverUnload;
};
typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, IRP* irp);
typedef KSTATUS (*PDRIVER_ADD_DEVICE)(DRIVER_OBJECT* driver,
DEVICE_OBJECT* physicalDeviceObject);
// -------------------------------------------------------------------------
// Bus / identity
// -------------------------------------------------------------------------
typedef enum IO_BUS_TYPE {
IoBusUnknown = 0,
@@ -41,6 +37,139 @@ typedef enum IO_BUS_TYPE {
IoBusUsb = 2,
} IO_BUS_TYPE;
typedef struct IO_DEVICE_ID {
IO_BUS_TYPE BusType;
SYS_STRING_VIEW Id;
} IO_DEVICE_ID;
typedef struct IO_DRIVER_ID_TABLE {
const IO_DEVICE_ID* Ids;
size_t Count;
} IO_DRIVER_ID_TABLE;
#define IO_MAX_DEVICE_IDS 8u
#define IO_MAX_DEVICE_ID_CHARS 96u
#define IO_MAX_DEVICE_NAME_CHARS 64u
// -------------------------------------------------------------------------
// Driver/device objects
// -------------------------------------------------------------------------
typedef enum DEVICE_OBJECT_TYPE {
DeviceObjectPhysical = 1,
DeviceObjectFunction = 2,
DeviceObjectFilter = 3,
} DEVICE_OBJECT_TYPE;
typedef enum DEVICE_PNP_STATE {
DevicePnpNotStarted = 0,
DevicePnpStarted,
DevicePnpStopPending,
DevicePnpStopped,
DevicePnpRemovePending,
DevicePnpSurpriseRemoved,
DevicePnpDeleted,
} DEVICE_PNP_STATE;
struct DRIVER_OBJECT {
OBJECT Header;
SYS_STRING_VIEW Name;
PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION];
PDRIVER_ADD_DEVICE DriverAddDevice;
DRIVER_UNLOAD_FN DriverUnload;
const IO_DRIVER_ID_TABLE* IdTable;
};
struct DEVICE_OBJECT {
OBJECT Header;
DRIVER_OBJECT* DriverObject;
DEVICE_OBJECT_TYPE Type;
DEVICE_PNP_STATE PnpState;
DEVICE_OBJECT* Parent;
DEVICE_OBJECT* FirstChild;
DEVICE_OBJECT* NextSibling;
DEVICE_OBJECT* AttachedDevice; // device directly above this object
DEVICE_OBJECT* LowerDevice; // device directly below this FDO/filter
void* DeviceExtension;
uint32_t Flags;
uint32_t Characteristics;
uint32_t AlignmentRequirement;
SPINLOCK Lock;
volatile long OutstandingIrps;
bool DeletePending;
SYS_STRING_VIEW Name;
char NameStorage[IO_MAX_DEVICE_NAME_CHARS];
union {
struct {
IO_BUS_TYPE BusType;
void* BusContext;
void* DeviceContext;
IO_DEVICE_ID Ids[IO_MAX_DEVICE_IDS];
char IdStorage[IO_MAX_DEVICE_IDS][IO_MAX_DEVICE_ID_CHARS];
uint8_t IdCount;
} Pdo;
struct {
uint8_t DiskIndex;
uint64_t SectorCount;
uint32_t SectorSize;
} Disk;
} u;
};
#define DEVICE_OBJECT_DISK 0x00000001u
#define DEVICE_OBJECT_REMOVABLE 0x00000002u
// -------------------------------------------------------------------------
// Object/device creation and stack management
// -------------------------------------------------------------------------
__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
SYS_STRING_VIEW name);
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver);
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice);
__declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
DEVICE_OBJECT* parent,
const IO_DEVICE_ID* ids,
size_t idCount,
size_t extensionSize,
DEVICE_OBJECT** outPdo);
__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo);
__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
SYS_STRING_VIEW name);
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device);
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device);
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device);
__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
DEVICE_OBJECT* target);
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source);
__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp);
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp);
__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
IO_PNP_MINOR_FUNCTION minor);
// -------------------------------------------------------------------------
// Legacy USB publish/matcher path. This remains only until USB is migrated to
// PDO/interface-PDO enumeration in the later USB stages.
// -------------------------------------------------------------------------
#define IO_PUBLISHED_USB_MAX_ENDPOINTS 8u
typedef struct IO_PUBLISHED_USB_ENDPOINT {
@@ -102,41 +231,42 @@ __declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
__declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device);
__declspec(dllexport) uint32_t IoPublishedDeviceCount(void);
struct DEVICE_OBJECT {
DRIVER_OBJECT* DriverObject;
void* DeviceExtension;
uint32_t Flags;
uint32_t AlignmentRequirement;
uint64_t SectorCount;
uint8_t DiskIndex;
};
// -------------------------------------------------------------------------
// Disk namespace above storage DEVICE_OBJECTs
// -------------------------------------------------------------------------
#define DEVICE_OBJECT_DISK 0x00000001u
#define DEVICE_OBJECT_REMOVABLE 0x00000002u
typedef enum IO_DEVICE_CHANGE {
IoDeviceArrived = 0,
IoDeviceQueryRemove,
IoDeviceSurpriseRemoved,
IoDeviceRemoved,
} IO_DEVICE_CHANGE;
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice);
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
DEVICE_OBJECT* device);
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex);
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex);
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex);
typedef void (*IO_DISK_CHANGE_CALLBACK)(uint8_t driveIndex,
DEVICE_OBJECT* device,
IO_DEVICE_CHANGE change,
void* context);
typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex,
DEVICE_OBJECT* device,
void* context);
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
DEVICE_OBJECT* device);
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex);
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex);
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex);
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex);
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback,
void* context);
__declspec(dllexport) KSTATUS IoRegisterDiskChangeCallback(IO_DISK_CHANGE_CALLBACK callback,
void* context);
__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback,
void* context);
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device,
struct IRP* irp);
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
code/public/ntlite/syscall.h
-1
View File
@@ -106,7 +106,6 @@ typedef struct SYS_USER_APC_FRAME {
void SysUserApcDispatcher(void);
// ---------------------------------------------------------------------------
// Trap-frame query ABI
// ---------------------------------------------------------------------------
readme.md
+1 -1
View File
@@ -10,7 +10,7 @@ NT-Lite is a 64-bit operating system kernel written from scratch. It has a UEFI
It boots into a shell, launches user processes, survives application crashes, and can run things it probably should not be able to run yet.
The codebase is still small enough to understand, but large enough to hurt your feelings. The entire project is about 40k lines.
The codebase is still small enough to understand, but large enough to hurt your feelings.
## features
runa.bat
+18 -1
View File
@@ -1 +1,18 @@
@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=vpc,file=disk.vhd,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -serial stdio -M q35 -m 256M -smp cpus=4,cores=4,threads=1,sockets=1 -netdev user,id=net0,hostfwd=tcp::8080-:8080 -device rtl8139,netdev=net0 -device qemu-xhci,id=usb -device usb-kbd,bus=usb.0 -device usb-mouse,bus=usb.0 -drive if=none,id=usbflash,format=vpc,file=usb.vhd -device usb-storage,drive=usbflash,bus=usb.0
@start qemu-system-x86_64 ^
-drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd ^
-drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd ^
-device ahci,id=ahci ^
-drive format=vpc,file=disk.vhd,if=none,id=disk0 ^
-device ide-hd,bus=ahci.0,drive=disk0 ^
-serial stdio ^
-M q35 ^
-m 256M ^
-smp cpus=4,cores=4,threads=1,sockets=1 ^
-netdev user,id=net0,hostfwd=tcp::8080-:8080 ^
-device rtl8139,netdev=net0 ^
-device qemu-xhci,id=usb ^
-device usb-kbd,bus=usb.0 ^
-device usb-mouse,bus=usb.0 ^
-device usb-hub,id=myhub,bus=usb.0,port=3 ^
-drive if=none,id=usbflash,format=vpc,file=usb.vhd ^
-device usb-storage,drive=usbflash,bus=usb.0,port=3.1,id=myusbflash