add system time, clean up pci into pci.sys, clean up syscall names, fix USERAPC1 typo
This commit is contained in:
maksw committed
1 parent
c1a291c4c9
commit
e41bf6e2c0
64 files changed
+4514
-2220
No files matched your search
@@ -0,0 +1,603 @@
|
||||
// pci.c - PCI bus driver, PDO enumerator, and exported PCI DDI
|
||||
#include "pci.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <common/printf.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/hal/hal_apic.h>
|
||||
|
||||
int _fltused; // MSVC floating-point helper anchor required by printf.obj
|
||||
|
||||
static const HAL_API* gHal;
|
||||
static DRIVER_OBJECT gPciDriver;
|
||||
static uint32_t gPciDeviceCount;
|
||||
|
||||
static bool PciIsPdo(DEVICE_OBJECT* pdo, PCI_PDO_EXTENSION** outExt) {
|
||||
if (!pdo || pdo->Type != DeviceObjectPhysical || pdo->u.Pdo.BusType != IoBusPci)
|
||||
return false;
|
||||
PCI_PDO_EXTENSION* ext = (PCI_PDO_EXTENSION*)pdo->DeviceExtension;
|
||||
if (!ext) return false;
|
||||
if (outExt) *outExt = ext;
|
||||
return true;
|
||||
}
|
||||
|
||||
static uint32_t PciRawRead32(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
|
||||
if (!gHal || offset > 0xFCu) return 0xFFFFFFFFu;
|
||||
return gHal->Pci.Read32(0, bus, dev, fn, offset & 0xFCu);
|
||||
}
|
||||
|
||||
static void PciRawWrite32(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset, uint32_t value) {
|
||||
if (!gHal || offset > 0xFCu) return;
|
||||
gHal->Pci.Write32(0, bus, dev, fn, offset & 0xFCu, value);
|
||||
}
|
||||
|
||||
static uint8_t PciRawRead8(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
|
||||
uint32_t v = PciRawRead32(bus, dev, fn, offset);
|
||||
return (uint8_t)(v >> ((offset & 3u) * 8u));
|
||||
}
|
||||
|
||||
static uint16_t PciRawRead16(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
|
||||
uint32_t v = PciRawRead32(bus, dev, fn, offset);
|
||||
return (uint16_t)(v >> ((offset & 2u) * 8u));
|
||||
}
|
||||
|
||||
static void PciRawWrite8(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset, uint8_t value) {
|
||||
uint16_t aligned = offset & ~3u;
|
||||
uint32_t shift = (offset & 3u) * 8u;
|
||||
uint32_t old = PciRawRead32(bus, dev, fn, aligned);
|
||||
old &= ~(0xFFu << shift);
|
||||
old |= ((uint32_t)value << shift);
|
||||
PciRawWrite32(bus, dev, fn, aligned, old);
|
||||
}
|
||||
|
||||
static void PciRawWrite16(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset, uint16_t value) {
|
||||
uint16_t aligned = offset & ~3u;
|
||||
uint32_t shift = (offset & 2u) * 8u;
|
||||
uint32_t old = PciRawRead32(bus, dev, fn, aligned);
|
||||
old &= ~(0xFFFFu << shift);
|
||||
old |= ((uint32_t)value << shift);
|
||||
PciRawWrite32(bus, dev, fn, aligned, old);
|
||||
}
|
||||
|
||||
static uint64_t PciBarSizeFromMask(uint64_t mask, uint64_t baseMask) {
|
||||
uint64_t sizeMask = mask & baseMask;
|
||||
if (!sizeMask || sizeMask == baseMask)
|
||||
return 0;
|
||||
return (~sizeMask + 1u) & baseMask;
|
||||
}
|
||||
|
||||
static uint8_t PciHeaderBarCount(uint8_t headerType) {
|
||||
switch (headerType & PCI_HEADER_TYPE_MASK) {
|
||||
case 0x00: return 6;
|
||||
case 0x01: return 2;
|
||||
case 0x02: return 1;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void PciDecodeBars(PCI_PDO_EXTENSION* ext) {
|
||||
uint8_t barCount = PciHeaderBarCount(ext->Info.HeaderType);
|
||||
for (uint8_t i = 0; i < PCI_MAX_BARS; i++) {
|
||||
ext->Bars[i].Type = PciBarUnused;
|
||||
ext->Bars[i].Index = i;
|
||||
}
|
||||
|
||||
// Size BARs with I/O and memory decoding disabled so all-ones probes do
|
||||
// not accidentally alias a live resource. Restore policy immediately;
|
||||
// PciEnableDevice() is the only exported path that changes command bits.
|
||||
uint16_t savedCommand = PciRawRead16(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
PCI_REG_COMMAND_STATUS);
|
||||
uint16_t disabledCommand = (uint16_t)(savedCommand &
|
||||
(uint16_t)~(PCI_COMMAND_IO_SPACE | PCI_COMMAND_MEMORY_SPACE));
|
||||
if (disabledCommand != savedCommand) {
|
||||
PciRawWrite16(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
PCI_REG_COMMAND_STATUS,
|
||||
disabledCommand);
|
||||
}
|
||||
|
||||
for (uint8_t i = 0; i < barCount && i < PCI_MAX_BARS; i++) {
|
||||
uint16_t reg = (uint16_t)(PCI_REG_BAR0 + i * 4u);
|
||||
uint32_t original = PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg);
|
||||
if (original == 0 || original == 0xFFFFFFFFu)
|
||||
continue;
|
||||
|
||||
PCI_BAR* bar = &ext->Bars[i];
|
||||
bar->Index = i;
|
||||
bar->Length = 0;
|
||||
bar->Prefetchable = false;
|
||||
|
||||
if (original & 0x1u) {
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg,
|
||||
0xFFFFFFFFu);
|
||||
uint32_t maskLo = PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg);
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg,
|
||||
original);
|
||||
|
||||
bar->Type = PciBarIo;
|
||||
bar->Base = (uint64_t)(original & ~0x3u);
|
||||
bar->Length = PciBarSizeFromMask((uint64_t)maskLo, 0xFFFFFFFCull);
|
||||
continue;
|
||||
}
|
||||
|
||||
uint32_t memType = (original >> 1) & 0x3u;
|
||||
bar->Prefetchable = (original & 0x8u) != 0;
|
||||
|
||||
if (memType == 0x2u && i + 1u < barCount && i + 1u < PCI_MAX_BARS) {
|
||||
uint16_t regHi = (uint16_t)(reg + 4u);
|
||||
uint32_t originalHi = PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
regHi);
|
||||
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg,
|
||||
0xFFFFFFFFu);
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
regHi,
|
||||
0xFFFFFFFFu);
|
||||
uint32_t maskLo = PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg);
|
||||
uint32_t maskHi = PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
regHi);
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg,
|
||||
original);
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
regHi,
|
||||
originalHi);
|
||||
|
||||
bar->Type = PciBarMemory64;
|
||||
bar->Base = ((uint64_t)originalHi << 32) | (uint64_t)(original & ~0xFu);
|
||||
uint64_t mask = ((uint64_t)maskHi << 32) | (uint64_t)(maskLo & ~0xFu);
|
||||
bar->Length = PciBarSizeFromMask(mask, 0xFFFFFFFFFFFFFFF0ull);
|
||||
|
||||
i++;
|
||||
ext->Bars[i].Type = PciBarUnused;
|
||||
ext->Bars[i].Index = i;
|
||||
} else {
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg,
|
||||
0xFFFFFFFFu);
|
||||
uint32_t maskLo = PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg);
|
||||
PciRawWrite32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
reg,
|
||||
original);
|
||||
|
||||
bar->Type = PciBarMemory32;
|
||||
bar->Base = (uint64_t)(original & ~0xFu);
|
||||
bar->Length = PciBarSizeFromMask((uint64_t)maskLo, 0xFFFFFFF0ull);
|
||||
}
|
||||
}
|
||||
|
||||
if (disabledCommand != savedCommand) {
|
||||
PciRawWrite16(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
PCI_REG_COMMAND_STATUS,
|
||||
savedCommand);
|
||||
}
|
||||
}
|
||||
|
||||
static uint8_t PciFindCapabilityOffset(PCI_PDO_EXTENSION* ext) {
|
||||
uint16_t status = (uint16_t)(PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
PCI_REG_COMMAND_STATUS) >> 16);
|
||||
if ((status & PCI_STATUS_CAP_LIST) == 0)
|
||||
return 0;
|
||||
|
||||
uint8_t cap = PciRawRead8(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
PCI_REG_CAP_PTR) & 0xFCu;
|
||||
return cap;
|
||||
}
|
||||
|
||||
static void PciMakeIds(const PCI_DEVICE_INFO* info, IO_DEVICE_ID* ids, char storage[][IO_MAX_DEVICE_ID_CHARS], size_t* outCount) {
|
||||
size_t count = 0;
|
||||
|
||||
snprintf(storage[count], IO_MAX_DEVICE_ID_CHARS,
|
||||
"PCI\\VEN_%04X&DEV_%04X&REV_%02X",
|
||||
info->VendorId, info->DeviceId, info->RevisionId);
|
||||
ids[count].BusType = IoBusPci;
|
||||
ids[count].Id = SysSvFromCstr(storage[count]);
|
||||
count++;
|
||||
|
||||
snprintf(storage[count], IO_MAX_DEVICE_ID_CHARS,
|
||||
"PCI\\VEN_%04X&DEV_%04X",
|
||||
info->VendorId, info->DeviceId);
|
||||
ids[count].BusType = IoBusPci;
|
||||
ids[count].Id = SysSvFromCstr(storage[count]);
|
||||
count++;
|
||||
|
||||
snprintf(storage[count], IO_MAX_DEVICE_ID_CHARS,
|
||||
"PCI\\CC_%02X%02X%02X",
|
||||
info->ClassCode, info->Subclass, info->ProgIf);
|
||||
ids[count].BusType = IoBusPci;
|
||||
ids[count].Id = SysSvFromCstr(storage[count]);
|
||||
count++;
|
||||
|
||||
snprintf(storage[count], IO_MAX_DEVICE_ID_CHARS,
|
||||
"PCI\\CC_%02X%02X",
|
||||
info->ClassCode, info->Subclass);
|
||||
ids[count].BusType = IoBusPci;
|
||||
ids[count].Id = SysSvFromCstr(storage[count]);
|
||||
count++;
|
||||
|
||||
snprintf(storage[count], IO_MAX_DEVICE_ID_CHARS,
|
||||
"PCI\\CC_%02X",
|
||||
info->ClassCode);
|
||||
ids[count].BusType = IoBusPci;
|
||||
ids[count].Id = SysSvFromCstr(storage[count]);
|
||||
count++;
|
||||
|
||||
*outCount = count;
|
||||
}
|
||||
|
||||
static KSTATUS PciCreatePdo(uint8_t bus, uint8_t dev, uint8_t fn) {
|
||||
uint32_t id = PciRawRead32(bus, dev, fn, PCI_REG_VENDOR_DEVICE);
|
||||
uint16_t vendor = (uint16_t)(id & 0xFFFFu);
|
||||
if (vendor == PCI_VENDOR_INVALID)
|
||||
return STATUS_NOT_FOUND;
|
||||
|
||||
uint32_t classReg = PciRawRead32(bus, dev, fn, PCI_REG_REVISION_CLASS);
|
||||
uint32_t hdrReg = PciRawRead32(bus, dev, fn, PCI_REG_HEADER_TYPE);
|
||||
uint32_t irqReg = PciRawRead32(bus, dev, fn, PCI_REG_INTERRUPT_LINE_PIN);
|
||||
uint32_t subsysReg = PciRawRead32(bus, dev, fn, PCI_REG_SUBSYSTEM);
|
||||
|
||||
PCI_DEVICE_INFO info;
|
||||
memset(&info, 0, sizeof(info));
|
||||
info.Address.Segment = 0;
|
||||
info.Address.Bus = bus;
|
||||
info.Address.Device = dev;
|
||||
info.Address.Function = fn;
|
||||
info.VendorId = vendor;
|
||||
info.DeviceId = (uint16_t)(id >> 16);
|
||||
info.SubsystemVendorId = (uint16_t)(subsysReg & 0xFFFFu);
|
||||
info.SubsystemId = (uint16_t)(subsysReg >> 16);
|
||||
info.RevisionId = (uint8_t)(classReg & 0xFFu);
|
||||
info.ProgIf = (uint8_t)((classReg >> 8) & 0xFFu);
|
||||
info.Subclass = (uint8_t)((classReg >> 16) & 0xFFu);
|
||||
info.ClassCode = (uint8_t)((classReg >> 24) & 0xFFu);
|
||||
info.HeaderType = (uint8_t)((hdrReg >> 16) & 0xFFu);
|
||||
info.InterruptLine = (uint8_t)(irqReg & 0xFFu);
|
||||
info.InterruptPin = (uint8_t)((irqReg >> 8) & 0xFFu);
|
||||
|
||||
IO_DEVICE_ID ids[5];
|
||||
char idStorage[5][IO_MAX_DEVICE_ID_CHARS];
|
||||
size_t idCount = 0;
|
||||
PciMakeIds(&info, ids, idStorage, &idCount);
|
||||
|
||||
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreatePhysicalDevice(&gPciDriver,
|
||||
(DEVICE_OBJECT*)0,
|
||||
ids,
|
||||
idCount,
|
||||
sizeof(PCI_PDO_EXTENSION),
|
||||
&pdo);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
PCI_PDO_EXTENSION* ext = (PCI_PDO_EXTENSION*)pdo->DeviceExtension;
|
||||
memset(ext, 0, sizeof(*ext));
|
||||
ext->Info = info;
|
||||
PciDecodeBars(ext);
|
||||
ext->CapabilityOffset = PciFindCapabilityOffset(ext);
|
||||
pdo->u.Pdo.DeviceContext = ext;
|
||||
|
||||
char name[IO_MAX_DEVICE_NAME_CHARS];
|
||||
snprintf(name, sizeof(name), "pci-%04X:%02X:%02X.%u",
|
||||
info.Address.Segment, info.Address.Bus, info.Address.Device, info.Address.Function);
|
||||
IoSetDeviceName(pdo, SysSvFromCstr(name));
|
||||
|
||||
st = IoRegisterPhysicalDevice(pdo);
|
||||
if (!KSUCCESS(st)) {
|
||||
IoDeleteDevice(pdo);
|
||||
return st;
|
||||
}
|
||||
|
||||
gPciDeviceCount++;
|
||||
DbgPrint("pci: %v ven=%04x dev=%04x class=%02x%02x%02x irq=%u\n",
|
||||
pdo->Name,
|
||||
info.VendorId,
|
||||
info.DeviceId,
|
||||
info.ClassCode,
|
||||
info.Subclass,
|
||||
info.ProgIf,
|
||||
info.InterruptLine);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static void PciEnumerateBus(void) {
|
||||
for (uint32_t bus = 0; bus < 256u; bus++) {
|
||||
for (uint32_t dev = 0; dev < 32u; dev++) {
|
||||
uint32_t id0 = PciRawRead32((uint8_t)bus, (uint8_t)dev, 0, PCI_REG_VENDOR_DEVICE);
|
||||
if ((uint16_t)(id0 & 0xFFFFu) == PCI_VENDOR_INVALID)
|
||||
continue;
|
||||
|
||||
uint8_t headerType = PciRawRead8((uint8_t)bus, (uint8_t)dev, 0, PCI_REG_HEADER_TYPE + 2u);
|
||||
uint8_t fnLimit = (headerType & PCI_HEADER_TYPE_MULTI) ? 8u : 1u;
|
||||
|
||||
for (uint8_t fn = 0; fn < fnLimit; fn++) {
|
||||
KSTATUS st = PciCreatePdo((uint8_t)bus, (uint8_t)dev, fn);
|
||||
if (!KSUCCESS(st) && st != STATUS_NOT_FOUND) {
|
||||
DbgPrint("pci: failed to create PDO for %02x:%02x.%u st=0x%x\n",
|
||||
bus, dev, fn, st);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static KSTATUS PciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp) return STATUS_INVALID_PARAMETER;
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios || ios->MajorFunction != IRP_MJ_PNP) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
device->PnpState = DevicePnpStarted;
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
default:
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) uint32_t PciDriverAbiVersion(void) {
|
||||
return 1u;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciGetDeviceInfo(DEVICE_OBJECT* pdo, PCI_DEVICE_INFO* outInfo) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!outInfo || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
*outInfo = ext->Info;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciReadConfig8(DEVICE_OBJECT* pdo, uint16_t offset, uint8_t* outValue) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!outValue || offset > 0xFFu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
*outValue = PciRawRead8(ext->Info.Address.Bus, ext->Info.Address.Device, ext->Info.Address.Function, offset);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciReadConfig16(DEVICE_OBJECT* pdo, uint16_t offset, uint16_t* outValue) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!outValue || offset > 0xFEu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
*outValue = PciRawRead16(ext->Info.Address.Bus, ext->Info.Address.Device, ext->Info.Address.Function, offset);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciReadConfig32(DEVICE_OBJECT* pdo, uint16_t offset, uint32_t* outValue) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!outValue || offset > 0xFCu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
*outValue = PciRawRead32(ext->Info.Address.Bus, ext->Info.Address.Device, ext->Info.Address.Function, offset);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciWriteConfig8(DEVICE_OBJECT* pdo, uint16_t offset, uint8_t value) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (offset > 0xFFu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
PciRawWrite8(ext->Info.Address.Bus, ext->Info.Address.Device, ext->Info.Address.Function, offset, value);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciWriteConfig16(DEVICE_OBJECT* pdo, uint16_t offset, uint16_t value) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (offset > 0xFEu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
PciRawWrite16(ext->Info.Address.Bus, ext->Info.Address.Device, ext->Info.Address.Function, offset, value);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciWriteConfig32(DEVICE_OBJECT* pdo, uint16_t offset, uint32_t value) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (offset > 0xFCu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
PciRawWrite32(ext->Info.Address.Bus, ext->Info.Address.Device, ext->Info.Address.Function, offset, value);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciGetBar(DEVICE_OBJECT* pdo, uint8_t index, PCI_BAR* outBar) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!outBar || index >= PCI_MAX_BARS || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
if (ext->Bars[index].Type == PciBarUnused) return STATUS_NOT_FOUND;
|
||||
*outBar = ext->Bars[index];
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciMapBar(DEVICE_OBJECT* pdo, uint8_t index, uint64_t pageFlags, void** outVirtual, uint64_t* outLength) {
|
||||
PCI_BAR bar;
|
||||
if (!outVirtual) return STATUS_INVALID_PARAMETER;
|
||||
KSTATUS st = PciGetBar(pdo, index, &bar);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
if (bar.Type != PciBarMemory32 && bar.Type != PciBarMemory64)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
if (!bar.Length)
|
||||
return STATUS_DEVICE_ERROR;
|
||||
|
||||
uint64_t flags = pageFlags ? pageFlags : (PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
uint64_t pageBase = bar.Base & ~(uint64_t)(PAGE_SIZE - 1u);
|
||||
uint64_t pageEnd = (bar.Base + bar.Length + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
|
||||
|
||||
for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE) {
|
||||
uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys);
|
||||
st = MmMapPage(virt, phys, flags, MmMapFailIfPresent);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, flags);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
*outVirtual = (void*)((uintptr_t)gHal->PhysToVirt(pageBase) + (uintptr_t)(bar.Base - pageBase));
|
||||
if (outLength) *outLength = bar.Length;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciEnableDevice(DEVICE_OBJECT* pdo, uint32_t flags) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uint16_t cmd = PciRawRead16(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
PCI_REG_COMMAND_STATUS);
|
||||
if (flags & PCI_ENABLE_IO) cmd |= PCI_COMMAND_IO_SPACE;
|
||||
if (flags & PCI_ENABLE_MEMORY) cmd |= PCI_COMMAND_MEMORY_SPACE;
|
||||
if (flags & PCI_ENABLE_BUSMASTER) cmd |= PCI_COMMAND_BUS_MASTER;
|
||||
if (flags & PCI_ENABLE_INTX) cmd &= (uint16_t)~PCI_COMMAND_INTX_DISABLE;
|
||||
if (flags & PCI_DISABLE_INTX) cmd |= PCI_COMMAND_INTX_DISABLE;
|
||||
|
||||
PciRawWrite16(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
PCI_REG_COMMAND_STATUS,
|
||||
cmd);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciGetInterrupt(DEVICE_OBJECT* pdo, uint8_t* outIrq) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!outIrq || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
|
||||
if (ext->Info.InterruptLine == 0xFFu) return STATUS_NOT_FOUND;
|
||||
*outIrq = ext->Info.InterruptLine;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static uint8_t PciFindCapability(PCI_PDO_EXTENSION* ext, uint8_t capId) {
|
||||
if (!ext || !ext->CapabilityOffset)
|
||||
return 0;
|
||||
|
||||
uint8_t cap = ext->CapabilityOffset & 0xFCu;
|
||||
for (uint32_t guard = 0; cap && guard < 48u; guard++) {
|
||||
uint32_t hdr = PciRawRead32(ext->Info.Address.Bus,
|
||||
ext->Info.Address.Device,
|
||||
ext->Info.Address.Function,
|
||||
cap);
|
||||
uint8_t id = (uint8_t)(hdr & 0xFFu);
|
||||
uint8_t next = (uint8_t)((hdr >> 8) & 0xFCu);
|
||||
if (id == capId)
|
||||
return cap;
|
||||
cap = next;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot) {
|
||||
PCI_PDO_EXTENSION* ext;
|
||||
if (!PciIsPdo(pdo, &ext))
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (irqSlot >= 24u)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uint8_t cap = PciFindCapability(ext, PCI_CAP_ID_MSI);
|
||||
if (!cap)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
|
||||
uint32_t bspLapic = HalGetBspLapicId();
|
||||
if (!HalApicUsingX2Apic() && bspLapic >= 256u)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
if (HalApicUsingX2Apic() && bspLapic >= 256u) {
|
||||
// Extended x2APIC MSI destinations need interrupt-remapping awareness.
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
uint8_t bus = ext->Info.Address.Bus;
|
||||
uint8_t dev = ext->Info.Address.Device;
|
||||
uint8_t fn = ext->Info.Address.Function;
|
||||
|
||||
uint16_t control = PciRawRead16(bus, dev, fn, (uint16_t)(cap + 2u));
|
||||
bool is64 = (control & PCI_MSI_CTRL_64BIT) != 0;
|
||||
bool pvm = (control & PCI_MSI_CTRL_PVM) != 0;
|
||||
uint8_t vector = (uint8_t)(PCI_IRQ_VECTOR_BASE + irqSlot);
|
||||
uint32_t address = PCI_MSI_ADDRESS_BASE | ((bspLapic & 0xFFu) << 12);
|
||||
uint16_t data = vector;
|
||||
|
||||
PciRawWrite32(bus, dev, fn, (uint16_t)(cap + 4u), address);
|
||||
if (is64) {
|
||||
PciRawWrite32(bus, dev, fn, (uint16_t)(cap + 8u), 0);
|
||||
PciRawWrite16(bus, dev, fn, (uint16_t)(cap + 12u), data);
|
||||
if (pvm)
|
||||
PciRawWrite32(bus, dev, fn, (uint16_t)(cap + 16u), 0);
|
||||
} else {
|
||||
PciRawWrite16(bus, dev, fn, (uint16_t)(cap + 8u), data);
|
||||
if (pvm)
|
||||
PciRawWrite32(bus, dev, fn, (uint16_t)(cap + 12u), 0);
|
||||
}
|
||||
|
||||
control &= (uint16_t)~PCI_MSI_CTRL_MME_MASK;
|
||||
control |= PCI_MSI_CTRL_ENABLE;
|
||||
PciRawWrite16(bus, dev, fn, (uint16_t)(cap + 2u), control);
|
||||
|
||||
uint16_t cmd = PciRawRead16(bus, dev, fn, PCI_REG_COMMAND_STATUS);
|
||||
cmd |= PCI_COMMAND_INTX_DISABLE;
|
||||
PciRawWrite16(bus, dev, fn, PCI_REG_COMMAND_STATUS, cmd);
|
||||
|
||||
DbgPrint("pci: MSI enabled %02x:%02x.%u cap=0x%02x vector=0x%02x irqslot=%u dest=%u %s\n",
|
||||
bus, dev, fn, cap, vector, irqSlot, bspLapic, is64 ? "64-bit" : "32-bit");
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
if (!hal) return STATUS_INVALID_PARAMETER;
|
||||
gHal = hal;
|
||||
gPciDeviceCount = 0;
|
||||
|
||||
IoInitializeDriverObject(&gPciDriver, SYS_SV_LITERAL("pci"));
|
||||
gPciDriver.MajorFunction[IRP_MJ_PNP] = PciDispatchPnp;
|
||||
|
||||
KSTATUS st = IoRegisterDriverObject(&gPciDriver);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
DbgPrint("pci: DriverEntry ABI=%u\n", PciDriverAbiVersion());
|
||||
PciEnumerateBus();
|
||||
DbgPrint("pci: enumeration complete, devices=%u\n", gPciDeviceCount);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <ntlite/ddi/pci.h>
|
||||
|
||||
#define PCI_MAX_BARS 6u
|
||||
|
||||
#define PCI_VENDOR_INVALID 0xFFFFu
|
||||
|
||||
#define PCI_REG_VENDOR_DEVICE 0x00u
|
||||
#define PCI_REG_COMMAND_STATUS 0x04u
|
||||
#define PCI_REG_REVISION_CLASS 0x08u
|
||||
#define PCI_REG_HEADER_TYPE 0x0Cu
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_REG_SUBSYSTEM 0x2Cu
|
||||
#define PCI_REG_CAP_PTR 0x34u
|
||||
#define PCI_REG_INTERRUPT_LINE_PIN 0x3Cu
|
||||
|
||||
#define PCI_COMMAND_IO_SPACE 0x0001u
|
||||
#define PCI_COMMAND_MEMORY_SPACE 0x0002u
|
||||
#define PCI_COMMAND_BUS_MASTER 0x0004u
|
||||
#define PCI_COMMAND_INTX_DISABLE 0x0400u
|
||||
|
||||
#define PCI_HEADER_TYPE_MASK 0x7Fu
|
||||
#define PCI_HEADER_TYPE_MULTI 0x80u
|
||||
|
||||
#define PCI_STATUS_CAP_LIST 0x0010u
|
||||
|
||||
#define PCI_CAP_ID_MSI 0x05u
|
||||
#define PCI_CAP_ID_MSIX 0x11u
|
||||
|
||||
#define PCI_MSI_CTRL_ENABLE 0x0001u
|
||||
#define PCI_MSI_CTRL_MME_MASK 0x0070u
|
||||
#define PCI_MSI_CTRL_64BIT 0x0080u
|
||||
#define PCI_MSI_CTRL_PVM 0x0100u
|
||||
|
||||
#define PCI_MSI_ADDRESS_BASE 0xFEE00000u
|
||||
#define PCI_IRQ_VECTOR_BASE 0x20u
|
||||
|
||||
typedef struct PCI_PDO_EXTENSION {
|
||||
PCI_DEVICE_INFO Info;
|
||||
PCI_BAR Bars[PCI_MAX_BARS];
|
||||
uint8_t CapabilityOffset;
|
||||
} PCI_PDO_EXTENSION;
|
||||
+28
-60
@@ -1,8 +1,17 @@
|
||||
// usb.c - USB bus driver entry point and PCI controller discovery.
|
||||
// usb.c - USB bus driver entry point and host-controller driver registration.
|
||||
#include "usb.h"
|
||||
|
||||
XHCI_CONTROLLER gXhci;
|
||||
bool gHaveXhci = false;
|
||||
DRIVER_OBJECT gXhciDriver;
|
||||
const HAL_API* gUsbHal;
|
||||
|
||||
static const IO_DEVICE_ID gXhciIds[] = {
|
||||
{ IoBusPci, { "PCI\\CC_0C0330", sizeof("PCI\\CC_0C0330") - 1 } },
|
||||
};
|
||||
|
||||
static const IO_DRIVER_ID_TABLE gXhciIdTable = {
|
||||
gXhciIds,
|
||||
sizeof(gXhciIds) / sizeof(gXhciIds[0]),
|
||||
};
|
||||
|
||||
const char* UsbControllerName(uint8_t progIf) {
|
||||
switch (progIf) {
|
||||
@@ -14,66 +23,19 @@ const char* UsbControllerName(uint8_t progIf) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void UsbDiscoverControllers(const HAL_API* hal) {
|
||||
uint32_t found = 0;
|
||||
uint32_t xhciOk = 0;
|
||||
|
||||
for (uint16_t bus = 0; bus < 256; bus++) {
|
||||
for (uint8_t dev = 0; dev < 32; dev++) {
|
||||
for (uint8_t fn = 0; fn < 8; fn++) {
|
||||
uint32_t vd = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_VENDOR_DEVICE);
|
||||
if ((vd & 0xFFFFu) == PCI_VENDOR_INVALID)
|
||||
continue;
|
||||
|
||||
uint32_t classReg = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_CLASS);
|
||||
uint8_t classCode = (uint8_t)(classReg >> 24);
|
||||
uint8_t subclass = (uint8_t)(classReg >> 16);
|
||||
uint8_t progIf = (uint8_t)(classReg >> 8);
|
||||
if (classCode != PCI_CLASS_SERIAL_BUS || subclass != PCI_SUBCLASS_USB)
|
||||
continue;
|
||||
|
||||
USB_BAR bar0;
|
||||
bool hasBar0 = UsbReadBar(hal, (uint8_t)bus, dev, fn, 0, &bar0);
|
||||
uint32_t rawBar0 = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_BAR0);
|
||||
uint32_t rawBar1 = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_BAR0 + 4u);
|
||||
uint8_t irq = hal->Pci.GetIrqLine((uint8_t)bus, dev, fn);
|
||||
|
||||
DbgPrint("usb: found %s controller at %u:%u.%u vendor=%04x device=%04x irq=%u bar0raw=%08x bar1raw=%08x",
|
||||
UsbControllerName(progIf),
|
||||
bus, dev, fn,
|
||||
vd & 0xFFFFu, vd >> 16,
|
||||
irq, rawBar0, rawBar1);
|
||||
if (hasBar0 && bar0.Kind == UsbBarMmio) {
|
||||
DbgPrint(" mmio=0x%016llX\n", (unsigned long long)bar0.Base);
|
||||
} else if (hasBar0 && bar0.Kind == UsbBarIo) {
|
||||
DbgPrint(" io=0x%016llX\n", (unsigned long long)bar0.Base);
|
||||
} else {
|
||||
DbgPrint(" no-bar0\n");
|
||||
}
|
||||
|
||||
if (progIf == PCI_PROGIF_XHCI) {
|
||||
KSTATUS st = XhciInitialise(hal, (uint8_t)bus, dev, fn,
|
||||
(uint16_t)(vd & 0xFFFFu),
|
||||
(uint16_t)(vd >> 16));
|
||||
if (KSUCCESS(st) || st == STATUS_ALREADY_EXISTS) xhciOk++;
|
||||
else DbgPrint("usb: xHCI init failed st=0x%x\n", st);
|
||||
}
|
||||
found++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!found)
|
||||
DbgPrint("usb: no PCI USB controllers discovered\n");
|
||||
else
|
||||
DbgPrint("usb: discovery complete; controllers=%u xhci_initialised=%u\n", found, xhciOk);
|
||||
static KSTATUS UsbInitializeXhciDriver(void) {
|
||||
memset(&gXhciDriver, 0, sizeof(gXhciDriver));
|
||||
IoInitializeDriverObject(&gXhciDriver, SYS_SV_LITERAL("xhci"));
|
||||
gXhciDriver.IdTable = &gXhciIdTable;
|
||||
gXhciDriver.DriverAddDevice = XhciAddDevice;
|
||||
gXhciDriver.MajorFunction[IRP_MJ_PNP] = XhciDispatchPnp;
|
||||
return IoRegisterDriverObject(&gXhciDriver);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
if (!hal) return STATUS_INVALID_PARAMETER;
|
||||
DbgPrint("usb: discovery starting\n");
|
||||
DbgPrint("usb: driver loading\n");
|
||||
gUsbHal = hal;
|
||||
|
||||
KSTATUS st = UsbInitializeBusModel();
|
||||
if (!KSUCCESS(st)) {
|
||||
@@ -105,6 +67,12 @@ KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
return st;
|
||||
}
|
||||
|
||||
UsbDiscoverControllers(hal);
|
||||
st = UsbInitializeXhciDriver();
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI driver initialisation failed st=0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
DbgPrint("usb: driver initialized\n");
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
+14
-31
@@ -10,6 +10,7 @@
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/ddi/input_device.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/pci.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
@@ -32,17 +33,9 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
|
||||
#endif
|
||||
|
||||
|
||||
#define PCI_VENDOR_INVALID 0xFFFFu
|
||||
#define PCI_CLASS_SERIAL_BUS 0x0Cu
|
||||
#define PCI_SUBCLASS_USB 0x03u
|
||||
#define PCI_PROGIF_XHCI 0x30u
|
||||
#define PCI_REG_VENDOR_DEVICE 0x00u
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_CLASS 0x08u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
|
||||
#define XHCI_CAP_CAPLENGTH 0x00u
|
||||
#define XHCI_CAP_HCSPARAMS1 0x04u
|
||||
@@ -212,7 +205,6 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
|
||||
#define XHCI_RING_TRBS 256u
|
||||
#define XHCI_EVENT_TRBS 256u
|
||||
#define XHCI_DCBAA_ENTRIES 256u
|
||||
#define XHCI_MMIO_FLOOR_BYTES 0x10000u
|
||||
#define XHCI_MAX_SCRATCHPADS 1024u
|
||||
#define XHCI_MAX_TRACKED_SLOTS 64u
|
||||
#define XHCI_MAX_ROOT_PORTS 255u
|
||||
@@ -347,19 +339,6 @@ typedef struct USB_TRACKED_INTERFACE {
|
||||
USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS];
|
||||
} USB_TRACKED_INTERFACE;
|
||||
|
||||
typedef enum USB_BAR_KIND {
|
||||
UsbBarNone,
|
||||
UsbBarIo,
|
||||
UsbBarMmio
|
||||
} USB_BAR_KIND;
|
||||
|
||||
typedef struct USB_BAR {
|
||||
USB_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} USB_BAR;
|
||||
|
||||
|
||||
typedef enum USB_DEVICE_STATE {
|
||||
UsbDeviceAttached = 0,
|
||||
UsbDevicePowered,
|
||||
@@ -422,7 +401,7 @@ struct USB_HCD {
|
||||
const USB_HCD_OPERATIONS* Operations;
|
||||
void* ControllerContext;
|
||||
DRIVER_OBJECT* DriverObject;
|
||||
DEVICE_OBJECT* ControllerPdo;
|
||||
DEVICE_OBJECT* ControllerDevice;
|
||||
DEVICE_OBJECT* RootHubPdo;
|
||||
void* RootHubContext;
|
||||
uint8_t ControllerIndex;
|
||||
@@ -488,10 +467,9 @@ struct USB_URB {
|
||||
};
|
||||
|
||||
typedef enum USB_PDO_KIND {
|
||||
UsbPdoController = 1,
|
||||
UsbPdoRootHub = 1,
|
||||
UsbPdoDevice = 2,
|
||||
UsbPdoInterface = 3,
|
||||
UsbPdoRootHub = 4,
|
||||
} USB_PDO_KIND;
|
||||
|
||||
typedef struct USB_PDO_EXTENSION {
|
||||
@@ -779,7 +757,13 @@ typedef struct XHCI_WAIT_TARGET {
|
||||
} XHCI_WAIT_TARGET;
|
||||
|
||||
typedef struct XHCI_CONTROLLER {
|
||||
uint32_t Magic;
|
||||
const HAL_API* Hal;
|
||||
DEVICE_OBJECT* Fdo;
|
||||
DEVICE_OBJECT* Pdo;
|
||||
DEVICE_OBJECT* LowerDevice;
|
||||
PCI_DEVICE_INFO PciInfo;
|
||||
PCI_BAR MmioBar;
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
@@ -845,13 +829,13 @@ typedef struct XHCI_CONTROLLER {
|
||||
} XHCI_CONTROLLER;
|
||||
|
||||
|
||||
extern XHCI_CONTROLLER gXhci;
|
||||
extern bool gHaveXhci;
|
||||
extern DRIVER_OBJECT gUsbStorageDriver;
|
||||
extern DRIVER_OBJECT gUsbBusDriver;
|
||||
extern DRIVER_OBJECT gUsbCompositeDriver;
|
||||
extern DRIVER_OBJECT gUsbHidDriver;
|
||||
extern DRIVER_OBJECT gUsbHubDriver;
|
||||
extern DRIVER_OBJECT gXhciDriver;
|
||||
extern const HAL_API* gUsbHal;
|
||||
|
||||
KSTATUS UsbInitializeBusModel(void);
|
||||
KSTATUS UsbInitializeCompositeDriver(void);
|
||||
@@ -865,7 +849,6 @@ void UsbInitializeHcd(USB_HCD* hcd,
|
||||
void* controllerContext,
|
||||
uint8_t controllerIndex,
|
||||
uint8_t maxRootPorts);
|
||||
KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device);
|
||||
KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd);
|
||||
KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise);
|
||||
KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise);
|
||||
@@ -899,8 +882,6 @@ void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value);
|
||||
void UsbDelayUs(const HAL_API* hal, uint32_t us);
|
||||
bool UsbWaitSet32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
|
||||
bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
|
||||
bool UsbReadBar(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, USB_BAR* out);
|
||||
KSTATUS UsbMapMmio(const HAL_API* hal, uint64_t physBase, uint32_t bytes, volatile uint8_t** outMmio);
|
||||
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys);
|
||||
uint16_t XhciScratchpadCount(uint32_t hcs2);
|
||||
uint32_t XhciTrbType(const XHCI_TRB* trb);
|
||||
@@ -1014,4 +995,6 @@ KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd);
|
||||
void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port);
|
||||
void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port);
|
||||
extern const USB_HCD_OPERATIONS gXhciHcdOperations;
|
||||
KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device);
|
||||
KSTATUS XhciStartController(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
|
||||
KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
|
||||
@@ -177,16 +177,6 @@ static void UsbFormatComposite(char* buf, size_t cap) {
|
||||
UsbIdAppendLiteral(buf, cap, &pos, "USB\\COMPOSITE");
|
||||
}
|
||||
|
||||
static void UsbFormatPciXhci(char* buf, size_t cap, uint16_t vendor, uint16_t device) {
|
||||
size_t pos = 0;
|
||||
if (cap) buf[0] = '\0';
|
||||
UsbIdAppendLiteral(buf, cap, &pos, "PCI\\VEN_");
|
||||
UsbIdAppendHex(buf, cap, &pos, vendor, 4);
|
||||
UsbIdAppendLiteral(buf, cap, &pos, "&DEV_");
|
||||
UsbIdAppendHex(buf, cap, &pos, device, 4);
|
||||
UsbIdAppendLiteral(buf, cap, &pos, "&CC_0C0330");
|
||||
}
|
||||
|
||||
static void UsbFormatRootHub(char* buf, size_t cap) {
|
||||
size_t pos = 0;
|
||||
if (cap) buf[0] = '\0';
|
||||
@@ -273,38 +263,6 @@ void UsbInitializeHcd(USB_HCD* hcd,
|
||||
hcd->MaxRootPorts = maxRootPorts;
|
||||
}
|
||||
|
||||
KSTATUS UsbCreateControllerPdo(USB_HCD* hcd, uint16_t vendor, uint16_t device) {
|
||||
if (!hcd) return STATUS_INVALID_PARAMETER;
|
||||
if (hcd->ControllerPdo) return STATUS_ALREADY_EXISTS;
|
||||
|
||||
IO_DEVICE_ID ids[2];
|
||||
char idbuf[2][USB_MAX_DEVICE_ID_TEXT];
|
||||
UsbFormatPciXhci(idbuf[0], USB_MAX_DEVICE_ID_TEXT, vendor, device);
|
||||
{
|
||||
size_t pos = 0;
|
||||
idbuf[1][0] = '\0';
|
||||
UsbIdAppendLiteral(idbuf[1], USB_MAX_DEVICE_ID_TEXT, &pos, "PCI\\CC_0C0330");
|
||||
}
|
||||
ids[0] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[0]) };
|
||||
ids[1] = (IO_DEVICE_ID){ IoBusPci, UsbIdFromBuffer(idbuf[1]) };
|
||||
|
||||
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver, (DEVICE_OBJECT*)0, ids, 2, sizeof(USB_PDO_EXTENSION), &pdo);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
(void)IoSetDeviceName(pdo, SYS_SV_LITERAL("xhci-controller"));
|
||||
|
||||
USB_PDO_EXTENSION* ext = (USB_PDO_EXTENSION*)pdo->DeviceExtension;
|
||||
ext->Kind = UsbPdoController;
|
||||
ext->Hcd = hcd;
|
||||
hcd->ControllerPdo = pdo;
|
||||
|
||||
st = IoRegisterPhysicalDevice(pdo);
|
||||
if (!KSUCCESS(st) && st != STATUS_ALREADY_EXISTS)
|
||||
return st;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) {
|
||||
if (!hcd) return STATUS_INVALID_PARAMETER;
|
||||
if (hcd->RootHubPdo) return STATUS_ALREADY_EXISTS;
|
||||
@@ -318,7 +276,7 @@ KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) {
|
||||
|
||||
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
|
||||
hcd->ControllerPdo,
|
||||
hcd->ControllerDevice,
|
||||
ids,
|
||||
2,
|
||||
sizeof(USB_PDO_EXTENSION),
|
||||
@@ -376,7 +334,7 @@ KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* paren
|
||||
|
||||
DEVICE_OBJECT* pdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreatePhysicalDevice(&gUsbBusDriver,
|
||||
parent ? parent : (hcd->RootHubPdo ? hcd->RootHubPdo : hcd->ControllerPdo),
|
||||
parent ? parent : (hcd->RootHubPdo ? hcd->RootHubPdo : hcd->ControllerDevice),
|
||||
ids,
|
||||
idCount,
|
||||
sizeof(USB_PDO_EXTENSION),
|
||||
|
||||
@@ -242,7 +242,7 @@ KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
if (!dev->UsbPdo) {
|
||||
st = UsbCreateDevicePdo(&xhci->Hcd, dev,
|
||||
dev->ParentDeviceObject ? dev->ParentDeviceObject :
|
||||
(xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : xhci->Hcd.ControllerPdo));
|
||||
(xhci->Hcd.RootHubPdo ? xhci->Hcd.RootHubPdo : xhci->Hcd.ControllerDevice));
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: create device PDO slot=%u failed st=0x%x\n", dev->SlotId, st);
|
||||
return st;
|
||||
|
||||
+183
-136
@@ -74,62 +74,6 @@ bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg,
|
||||
}
|
||||
|
||||
|
||||
bool UsbReadBar(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn,
|
||||
uint8_t index, USB_BAR* out) {
|
||||
uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u);
|
||||
uint32_t lo = hal->Pci.Read32(bus, dev, fn, reg);
|
||||
|
||||
out->Kind = UsbBarNone;
|
||||
out->Base = 0;
|
||||
out->Index = index;
|
||||
|
||||
if (lo == 0 || lo == 0xFFFFFFFFu)
|
||||
return false;
|
||||
|
||||
if (lo & 1u) {
|
||||
out->Kind = UsbBarIo;
|
||||
out->Base = (uint64_t)(lo & ~0x3u);
|
||||
return out->Base != 0;
|
||||
}
|
||||
|
||||
uint64_t base = (uint64_t)(lo & ~0xFu);
|
||||
uint32_t type = (lo >> 1) & 0x3u;
|
||||
if (type == 0x2u && index < 5) {
|
||||
uint32_t hi = hal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u));
|
||||
base |= ((uint64_t)hi << 32);
|
||||
}
|
||||
|
||||
out->Kind = UsbBarMmio;
|
||||
out->Base = base;
|
||||
return base != 0;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbMapMmio(const HAL_API* hal, uint64_t physBase,
|
||||
uint32_t bytes, volatile uint8_t** outMmio) {
|
||||
uint64_t pageBase = physBase & PAGE_MASK;
|
||||
uint64_t pageOff = physBase - pageBase;
|
||||
uint32_t total = UsbAlignUp32((uint32_t)pageOff + bytes, (uint32_t)PAGE_SIZE);
|
||||
uint32_t pages = total / (uint32_t)PAGE_SIZE;
|
||||
|
||||
for (uint32_t i = 0; i < pages; i++) {
|
||||
PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE;
|
||||
uintptr_t virt = (uintptr_t)hal->PhysToVirt(phys);
|
||||
KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, MmMapFailIfPresent);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI MmMapPage failed phys=0x%016llX st=0x%x\n",
|
||||
(unsigned long long)phys, st);
|
||||
return st;
|
||||
}
|
||||
}
|
||||
|
||||
*outMmio = (volatile uint8_t*)((uintptr_t)hal->PhysToVirt(pageBase) + pageOff);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
|
||||
PHYSICAL_ADDRESS* outPhys) {
|
||||
if (!outPhys || pages == 0)
|
||||
@@ -617,19 +561,39 @@ void XhciEventDpcRoutine(KDPC* dpc, void* context,
|
||||
}
|
||||
|
||||
|
||||
#define XHCI_CONTROLLER_MAGIC 0x58484349u /* XHCI */
|
||||
#define XHCI_MAX_CONTROLLERS 8u
|
||||
|
||||
static XHCI_CONTROLLER* gXhciControllers[XHCI_MAX_CONTROLLERS];
|
||||
static uint32_t gXhciControllerCount;
|
||||
|
||||
static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) {
|
||||
if (!xhci) return STATUS_INVALID_PARAMETER;
|
||||
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
|
||||
if (gXhciControllers[i] == xhci)
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
if (gXhciControllerCount >= XHCI_MAX_CONTROLLERS)
|
||||
return STATUS_NO_MEMORY;
|
||||
gXhciControllers[gXhciControllerCount++] = xhci;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void XhciIsr(void) {
|
||||
XHCI_CONTROLLER* xhci = &gXhci;
|
||||
if (!xhci->Running || !xhci->Op || !xhci->Runtime)
|
||||
return;
|
||||
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
|
||||
XHCI_CONTROLLER* xhci = gXhciControllers[i];
|
||||
if (!xhci || !xhci->Running || !xhci->Op || !xhci->Runtime)
|
||||
continue;
|
||||
|
||||
uint32_t status = XhciRead32(xhci->Op, XHCI_OP_USBSTS);
|
||||
if (!(status & (XHCI_USBSTS_EINT | XHCI_USBSTS_PCD | XHCI_USBSTS_HSE | XHCI_USBSTS_HCE)))
|
||||
return;
|
||||
uint32_t status = XhciRead32(xhci->Op, XHCI_OP_USBSTS);
|
||||
if (!(status & (XHCI_USBSTS_EINT | XHCI_USBSTS_PCD | XHCI_USBSTS_HSE | XHCI_USBSTS_HCE)))
|
||||
continue;
|
||||
|
||||
xhci->InterruptCount++;
|
||||
XhciWrite32(xhci->Op, XHCI_OP_USBSTS, status & XHCI_USBSTS_CLEAR_MASK);
|
||||
XhciAckInterrupter0(xhci);
|
||||
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
|
||||
xhci->InterruptCount++;
|
||||
XhciWrite32(xhci->Op, XHCI_OP_USBSTS, status & XHCI_USBSTS_CLEAR_MASK);
|
||||
XhciAckInterrupter0(xhci);
|
||||
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1872,65 +1836,67 @@ const USB_HCD_OPERATIONS gXhciHcdOperations = {
|
||||
XhciHcdAllocateDma,
|
||||
};
|
||||
|
||||
KSTATUS XhciInitialise(const HAL_API* hal,
|
||||
uint8_t bus, uint8_t dev, uint8_t fn,
|
||||
uint16_t vendor, uint16_t device) {
|
||||
if (gHaveXhci) {
|
||||
DbgPrint("usb: additional xHCI controller at %u:%u.%u ignored for now\n",
|
||||
bus, dev, fn);
|
||||
return STATUS_ALREADY_EXISTS;
|
||||
KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
|
||||
if (!xhci || xhci->Magic != XHCI_CONTROLLER_MAGIC || !xhci->Pdo || !xhci->Fdo)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (xhci->Running)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
KSTATUS st = PciGetDeviceInfo(xhci->Pdo, &xhci->PciInfo);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI PciGetDeviceInfo failed st=0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
USB_BAR bar;
|
||||
if (!UsbReadBar(hal, bus, dev, fn, 0, &bar) || bar.Kind != UsbBarMmio) {
|
||||
DbgPrint("usb: xHCI controller has no usable MMIO BAR0\n");
|
||||
xhci->Bus = xhci->PciInfo.Address.Bus;
|
||||
xhci->Dev = xhci->PciInfo.Address.Device;
|
||||
xhci->Fn = xhci->PciInfo.Address.Function;
|
||||
|
||||
st = PciGetBar(xhci->Pdo, 0, &xhci->MmioBar);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI PCI %02x:%02x.%x BAR0 missing st=0x%x\n",
|
||||
xhci->Bus, xhci->Dev, xhci->Fn, st);
|
||||
return st;
|
||||
}
|
||||
if (xhci->MmioBar.Type != PciBarMemory32 && xhci->MmioBar.Type != PciBarMemory64) {
|
||||
DbgPrint("usb: xHCI PCI %02x:%02x.%x BAR0 is not MMIO type=%u\n",
|
||||
xhci->Bus, xhci->Dev, xhci->Fn, xhci->MmioBar.Type);
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
|
||||
st = PciGetInterrupt(xhci->Pdo, &xhci->Irq);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI PciGetInterrupt failed st=0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
if (xhci->Irq == 0xFFu) {
|
||||
DbgPrint("usb: xHCI PCI %02x:%02x.%x has no INTx IRQ; MSI is not in pci.sys yet\n",
|
||||
xhci->Bus, xhci->Dev, xhci->Fn);
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
memset(&gXhci, 0, sizeof(gXhci));
|
||||
XHCI_CONTROLLER* xhci = &gXhci;
|
||||
KeInitializeDpc(&xhci->EventDpc, XhciEventDpcRoutine, xhci);
|
||||
xhci->Hal = hal;
|
||||
xhci->Bus = bus;
|
||||
xhci->Dev = dev;
|
||||
xhci->Fn = fn;
|
||||
xhci->Irq = hal->Pci.GetIrqLine(bus, dev, fn);
|
||||
xhci->MmioPhys = bar.Base;
|
||||
|
||||
uint32_t pciCmd = hal->Pci.Read32(bus, dev, fn, PCI_REG_COMMAND);
|
||||
pciCmd |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER;
|
||||
hal->Pci.Write32(bus, dev, fn, PCI_REG_COMMAND, pciCmd);
|
||||
|
||||
if (xhci->Irq == 0xFFu) {
|
||||
uint8_t msiIrq = 0xFFu;
|
||||
KSTATUS irqSt = KeIrqAllocateMsiVector(&msiIrq);
|
||||
if (!KSUCCESS(irqSt)) {
|
||||
DbgPrint("usb: xHCI has no PCI INTx IRQ line and MSI vector allocation failed st=0x%x\n",
|
||||
irqSt);
|
||||
return irqSt;
|
||||
}
|
||||
if (!hal->Pci.EnableMsi) {
|
||||
DbgPrint("usb: xHCI has no PCI INTx IRQ line and HAL MSI support is unavailable\n");
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
irqSt = hal->Pci.EnableMsi(bus, dev, fn, msiIrq);
|
||||
if (!KSUCCESS(irqSt)) {
|
||||
DbgPrint("usb: xHCI has no PCI INTx IRQ line and MSI enable failed st=0x%x\n",
|
||||
irqSt);
|
||||
return irqSt;
|
||||
}
|
||||
xhci->Irq = msiIrq;
|
||||
xhci->UsingMsi = true;
|
||||
DbgPrint("usb: xHCI using MSI irqslot=%u vector=0x%02x\n",
|
||||
xhci->Irq, 0x20u + xhci->Irq);
|
||||
} else {
|
||||
pciCmd = hal->Pci.Read32(bus, dev, fn, PCI_REG_COMMAND);
|
||||
pciCmd &= ~PCI_COMMAND_INTX_DISABLE;
|
||||
hal->Pci.Write32(bus, dev, fn, PCI_REG_COMMAND, pciCmd);
|
||||
st = PciEnableDevice(xhci->Pdo,
|
||||
PCI_ENABLE_MEMORY |
|
||||
PCI_ENABLE_BUSMASTER |
|
||||
PCI_ENABLE_INTX);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI PciEnableDevice failed st=0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
KSTATUS st = UsbMapMmio(hal, xhci->MmioPhys, PAGE_SIZE, &xhci->Mmio);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
xhci->MmioPhys = xhci->MmioBar.Base;
|
||||
|
||||
uint64_t mappedLength = 0;
|
||||
st = PciMapBar(xhci->Pdo,
|
||||
0,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
|
||||
(void**)&xhci->Mmio,
|
||||
&mappedLength);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI PciMapBar(BAR0) failed st=0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
xhci->MmioBytes = (mappedLength > 0xFFFFFFFFull) ? 0xFFFFFFFFu : (uint32_t)mappedLength;
|
||||
|
||||
xhci->Cap = xhci->Mmio;
|
||||
uint32_t cap0 = XhciRead32(xhci->Cap, XHCI_CAP_CAPLENGTH);
|
||||
@@ -1954,32 +1920,29 @@ KSTATUS XhciInitialise(const HAL_API* hal,
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
|
||||
UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts);
|
||||
KSTATUS hcdPdoStatus = UsbCreateControllerPdo(&xhci->Hcd, vendor, device);
|
||||
if (!KSUCCESS(hcdPdoStatus)) {
|
||||
DbgPrint("usb: xHCI controller PDO creation failed st=0x%x\n", hcdPdoStatus);
|
||||
return hcdPdoStatus;
|
||||
}
|
||||
|
||||
uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE +
|
||||
(uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE;
|
||||
uint32_t needBytes = portEnd;
|
||||
if (xhci->DbOff + 0x100u > needBytes) needBytes = xhci->DbOff + 0x100u;
|
||||
if (xhci->RtsOff + 0x1000u > needBytes) needBytes = xhci->RtsOff + 0x1000u;
|
||||
if (needBytes < XHCI_MMIO_FLOOR_BYTES) needBytes = XHCI_MMIO_FLOOR_BYTES;
|
||||
needBytes = UsbAlignUp32(needBytes, (uint32_t)PAGE_SIZE);
|
||||
if (xhci->MmioBytes < needBytes) {
|
||||
DbgPrint("usb: xHCI BAR0 too small length=0x%x need=0x%x dboff=0x%x rtsoff=0x%x ports=%u\n",
|
||||
xhci->MmioBytes, needBytes, xhci->DbOff, xhci->RtsOff, xhci->MaxPorts);
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
|
||||
st = UsbMapMmio(hal, xhci->MmioPhys, needBytes, &xhci->Mmio);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
xhci->MmioBytes = needBytes;
|
||||
xhci->Cap = xhci->Mmio;
|
||||
xhci->Op = xhci->Mmio + xhci->CapLength;
|
||||
xhci->Runtime = xhci->Mmio + xhci->RtsOff;
|
||||
xhci->Doorbells = xhci->Mmio + xhci->DbOff;
|
||||
|
||||
DbgPrint("usb: xHCI init %u:%u.%u vendor=%04x device=%04x irq=%u%s mmio=0x%016llX caplen=%u version=%04x\n",
|
||||
bus, dev, fn, vendor, device, xhci->Irq,
|
||||
xhci->UsingMsi ? "(msi)" : "",
|
||||
UsbInitializeHcd(&xhci->Hcd, &gXhciHcdOperations, xhci, 0, xhci->MaxPorts);
|
||||
xhci->Hcd.ControllerDevice = xhci->Fdo;
|
||||
|
||||
DbgPrint("usb: xHCI init %02x:%02x.%x vendor=%04x device=%04x irq=%u mmio=0x%016llX caplen=%u version=%04x\n",
|
||||
xhci->Bus, xhci->Dev, xhci->Fn,
|
||||
xhci->PciInfo.VendorId, xhci->PciInfo.DeviceId,
|
||||
xhci->Irq,
|
||||
(unsigned long long)xhci->MmioPhys, xhci->CapLength, xhci->Version);
|
||||
DbgPrint("usb: xHCI caps slots=%u intrs=%u ports=%u ctx=%u scratchpads=%u hcs2=%08x hcc1=%08x dboff=0x%x rtsoff=0x%x pagesize=%08x\n",
|
||||
xhci->MaxSlots, xhci->MaxIntrs, xhci->MaxPorts, xhci->ContextBytes,
|
||||
@@ -2021,12 +1984,18 @@ KSTATUS XhciInitialise(const HAL_API* hal,
|
||||
return st;
|
||||
}
|
||||
|
||||
st = XhciRegisterController(xhci);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: xHCI controller registry full st=0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
KeIrqRegister(xhci->Irq, XhciIsr);
|
||||
XhciEnableInterrupts(xhci);
|
||||
xhci->InterruptsEnabled = true;
|
||||
xhci->Fdo->PnpState = DevicePnpStarted;
|
||||
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
|
||||
DbgPrint("usb: xHCI ISR registered on IRQ %u%s; event ring handled by DPC\n",
|
||||
xhci->Irq, xhci->UsingMsi ? "(msi)" : "");
|
||||
DbgPrint("usb: xHCI ISR registered on IRQ %u; event ring handled by DPC\n", xhci->Irq);
|
||||
|
||||
st = UsbHubScheduleRootScan(&xhci->Hcd);
|
||||
if (!KSUCCESS(st))
|
||||
@@ -2034,8 +2003,86 @@ KSTATUS XhciInitialise(const HAL_API* hal,
|
||||
else
|
||||
DbgPrint("usb: root hub initial scan queued asynchronously\n");
|
||||
|
||||
gHaveXhci = true;
|
||||
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
|
||||
xhci->HidDeviceCount);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp || !device->DeviceExtension)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)device->DeviceExtension;
|
||||
if (xhci->Magic != XHCI_CONTROLLER_MAGIC)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
return XhciStartController(xhci);
|
||||
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
device->PnpState = DevicePnpRemovePending;
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
device->PnpState = xhci->Running ? DevicePnpStarted : DevicePnpNotStarted;
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
device->PnpState = DevicePnpSurpriseRemoved;
|
||||
xhci->Running = false;
|
||||
xhci->InterruptsEnabled = false;
|
||||
if (xhci->Op) {
|
||||
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
|
||||
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS);
|
||||
}
|
||||
if (xhci->Hcd.RootHubPdo)
|
||||
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
xhci->Running = false;
|
||||
xhci->InterruptsEnabled = false;
|
||||
if (xhci->Hcd.RootHubPdo)
|
||||
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
|
||||
IoDetachDevice(device);
|
||||
IoDeleteDevice(device);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
default:
|
||||
return IoForwardIrpToLower(device, irp);
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS XhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
|
||||
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreateDevice(driver, sizeof(XHCI_CONTROLLER), &fdo);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("xhci-controller"));
|
||||
|
||||
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)fdo->DeviceExtension;
|
||||
memset(xhci, 0, sizeof(*xhci));
|
||||
xhci->Magic = XHCI_CONTROLLER_MAGIC;
|
||||
xhci->Hal = gUsbHal;
|
||||
xhci->Fdo = fdo;
|
||||
xhci->Pdo = physicalDeviceObject;
|
||||
xhci->Irq = 0xFFu;
|
||||
KeInitializeDpc(&xhci->EventDpc, XhciEventDpcRoutine, xhci);
|
||||
|
||||
xhci->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
|
||||
if (!xhci->LowerDevice) {
|
||||
IoDeleteDevice(fdo);
|
||||
return STATUS_FAILURE;
|
||||
}
|
||||
|
||||
DbgPrint("usb: xHCI FDO attached to %v\n", physicalDeviceObject->Name);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
[boot]
|
||||
pci.sys
|
||||
ps2.sys
|
||||
ide.sys
|
||||
ahci.sys
|
||||
ata.sys
|
||||
usb.sys
|
||||
fat32.sys
|
||||
|
||||
[runtime]
|
||||
lwip.sys
|
||||
rtl8139.sys
|
||||
rtl816x.sys
|
||||
+274
-28
@@ -101,6 +101,8 @@ typedef struct {
|
||||
#define ATTR_DIRECTORY 0x10
|
||||
#define ATTR_ARCHIVE 0x20
|
||||
#define ATTR_LFN 0x0F
|
||||
#define FAT_NTRES_BASE_LOWER 0x08u
|
||||
#define FAT_NTRES_EXT_LOWER 0x10u
|
||||
#define FAT32_EOC 0x0FFFFFF8u
|
||||
#define FAT32_EOC_MARK 0x0FFFFFFFu
|
||||
#define FAT32_FREE 0x00000000u
|
||||
@@ -343,34 +345,186 @@ static void FreeClusterChain(FAT32_VOL *v, uint32_t start) {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DIRENT update helpers
|
||||
// FAT timestamp and DIRENT metadata helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS DirentUpdateSize(FAT32_VOL *v, uint64_t lba,
|
||||
uint32_t offset, uint32_t size) {
|
||||
|
||||
typedef struct FAT32_ENCODED_TIME {
|
||||
uint16_t Date;
|
||||
uint16_t Time;
|
||||
uint8_t CreateTimeTenth;
|
||||
} FAT32_ENCODED_TIME;
|
||||
|
||||
typedef struct FAT32_DIRENT_UPDATE {
|
||||
uint32_t Flags;
|
||||
uint32_t Size;
|
||||
uint32_t FirstCluster;
|
||||
FAT32_ENCODED_TIME Timestamp;
|
||||
} FAT32_DIRENT_UPDATE;
|
||||
|
||||
#define FAT32_DIRENT_UPDATE_SIZE 0x00000001u
|
||||
#define FAT32_DIRENT_UPDATE_FIRST_CLUSTER 0x00000002u
|
||||
#define FAT32_DIRENT_UPDATE_CREATE_TIME 0x00000004u
|
||||
#define FAT32_DIRENT_UPDATE_WRITE_TIME 0x00000008u
|
||||
#define FAT32_DIRENT_UPDATE_ACCESS_DATE 0x00000010u
|
||||
|
||||
static int FatIsLeapYear(uint32_t year) {
|
||||
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
|
||||
}
|
||||
|
||||
static uint32_t FatDaysInMonth(uint32_t year, uint32_t month) {
|
||||
static const uint8_t days[12] = { 31,28,31,30,31,30,31,31,30,31,30,31 };
|
||||
if (month < 1u || month > 12u) return 31u;
|
||||
if (month == 2u && FatIsLeapYear(year)) return 29u;
|
||||
return days[month - 1u];
|
||||
}
|
||||
|
||||
static uint64_t FatCalendarToEpochSeconds(uint32_t year, uint32_t month, uint32_t day,
|
||||
uint32_t hour, uint32_t minute, uint32_t second) {
|
||||
uint64_t days = 0;
|
||||
for (uint32_t y = 1970u; y < year; y++)
|
||||
days += FatIsLeapYear(y) ? 366u : 365u;
|
||||
for (uint32_t m = 1u; m < month; m++)
|
||||
days += FatDaysInMonth(year, m);
|
||||
days += day - 1u;
|
||||
return (((days * 24u) + hour) * 60u + minute) * 60u + second;
|
||||
}
|
||||
|
||||
static uint8_t FatDayOfWeek(uint32_t year, uint32_t month, uint32_t day) {
|
||||
uint64_t epoch = FatCalendarToEpochSeconds(year, month, day, 0, 0, 0);
|
||||
return (uint8_t)(((epoch / 86400u) + 4u) % 7u);
|
||||
}
|
||||
|
||||
static uint16_t FatEncodeDate(const SYS_SYSTEM_TIME* t) {
|
||||
uint32_t year = t && t->Year ? t->Year : 1980u;
|
||||
uint32_t month = t && t->Month ? t->Month : 1u;
|
||||
uint32_t day = t && t->Day ? t->Day : 1u;
|
||||
if (year < 1980u) year = 1980u;
|
||||
if (year > 2107u) year = 2107u;
|
||||
if (month < 1u) month = 1u;
|
||||
if (month > 12u) month = 12u;
|
||||
uint32_t dim = FatDaysInMonth(year, month);
|
||||
if (day < 1u) day = 1u;
|
||||
if (day > dim) day = dim;
|
||||
return (uint16_t)(((year - 1980u) << 9) | (month << 5) | day);
|
||||
}
|
||||
|
||||
static uint16_t FatEncodeTime(const SYS_SYSTEM_TIME* t) {
|
||||
uint32_t hour = t ? t->Hour : 0u;
|
||||
uint32_t minute = t ? t->Minute : 0u;
|
||||
uint32_t second = t ? t->Second : 0u;
|
||||
if (hour > 23u) hour = 23u;
|
||||
if (minute > 59u) minute = 59u;
|
||||
if (second > 59u) second = 59u;
|
||||
return (uint16_t)((hour << 11) | (minute << 5) | (second / 2u));
|
||||
}
|
||||
|
||||
static uint8_t FatEncodeCreateTimeTenth(const SYS_SYSTEM_TIME* t) {
|
||||
uint32_t second = t ? t->Second : 0u;
|
||||
uint32_t ms = t ? t->Millisecond : 0u;
|
||||
if (second > 59u) second = 59u;
|
||||
if (ms > 999u) ms = 999u;
|
||||
uint32_t v = ((second & 1u) * 100u) + (ms / 10u);
|
||||
if (v > 199u) v = 199u;
|
||||
return (uint8_t)v;
|
||||
}
|
||||
|
||||
static void FatQueryEncodedTime(FAT32_ENCODED_TIME* out) {
|
||||
SYS_SYSTEM_TIME now;
|
||||
memset(&now, 0, sizeof(now));
|
||||
if (KeQuerySystemTime(&now) != STATUS_SUCCESS) {
|
||||
now.Year = 1980;
|
||||
now.Month = 1;
|
||||
now.Day = 1;
|
||||
}
|
||||
out->Date = FatEncodeDate(&now);
|
||||
out->Time = FatEncodeTime(&now);
|
||||
out->CreateTimeTenth = FatEncodeCreateTimeTenth(&now);
|
||||
}
|
||||
|
||||
static void FatDecodeDateTime(uint16_t date, uint16_t time, uint8_t createTenth,
|
||||
SYS_SYSTEM_TIME* out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
uint32_t year = 1980u + ((date >> 9) & 0x7Fu);
|
||||
uint32_t month = (date >> 5) & 0x0Fu;
|
||||
uint32_t day = date & 0x1Fu;
|
||||
uint32_t hour = (time >> 11) & 0x1Fu;
|
||||
uint32_t minute = (time >> 5) & 0x3Fu;
|
||||
uint32_t second = (time & 0x1Fu) * 2u;
|
||||
|
||||
if (date == 0 || month < 1u || month > 12u) { year = 1980u; month = 1u; day = 1u; }
|
||||
uint32_t dim = FatDaysInMonth(year, month);
|
||||
if (day < 1u) day = 1u;
|
||||
if (day > dim) day = dim;
|
||||
if (hour > 23u) hour = 23u;
|
||||
if (minute > 59u) minute = 59u;
|
||||
if (second > 58u) second = 58u;
|
||||
if (createTenth >= 100u && second < 59u) second++;
|
||||
|
||||
out->Year = (uint16_t)year;
|
||||
out->Month = (uint8_t)month;
|
||||
out->Day = (uint8_t)day;
|
||||
out->Hour = (uint8_t)hour;
|
||||
out->Minute = (uint8_t)minute;
|
||||
out->Second = (uint8_t)second;
|
||||
out->Millisecond = (uint16_t)((createTenth % 100u) * 10u);
|
||||
out->DayOfWeek = FatDayOfWeek(year, month, day);
|
||||
out->Flags = SYS_TIME_FLAG_VALID;
|
||||
out->EpochSeconds = FatCalendarToEpochSeconds(year, month, day, hour, minute, second);
|
||||
}
|
||||
|
||||
static void FatDecodeAccessDate(uint16_t date, SYS_SYSTEM_TIME* out) {
|
||||
FatDecodeDateTime(date, 0, 0, out);
|
||||
}
|
||||
|
||||
static KSTATUS DirentUpdateMetadata(FAT32_VOL *v, uint64_t lba,
|
||||
uint32_t offset,
|
||||
const FAT32_DIRENT_UPDATE* update) {
|
||||
if (!v || !update || !update->Flags) return STATUS_INVALID_PARAMETER;
|
||||
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
|
||||
if (!sec) return STATUS_NO_MEMORY;
|
||||
KSTATUS s = ReadSectors(v, lba, 1, sec);
|
||||
if (KSUCCESS(s)) {
|
||||
((DIRENT *)(sec + offset))->FileSize = size;
|
||||
DIRENT *d = (DIRENT *)(sec + offset);
|
||||
if (update->Flags & FAT32_DIRENT_UPDATE_SIZE)
|
||||
d->FileSize = update->Size;
|
||||
if (update->Flags & FAT32_DIRENT_UPDATE_FIRST_CLUSTER) {
|
||||
d->FirstClusterHi = (uint16_t)(update->FirstCluster >> 16);
|
||||
d->FirstClusterLo = (uint16_t)(update->FirstCluster & 0xFFFF);
|
||||
}
|
||||
if (update->Flags & FAT32_DIRENT_UPDATE_CREATE_TIME) {
|
||||
d->CrtTimeTenth = update->Timestamp.CreateTimeTenth;
|
||||
d->CrtTime = update->Timestamp.Time;
|
||||
d->CrtDate = update->Timestamp.Date;
|
||||
}
|
||||
if (update->Flags & FAT32_DIRENT_UPDATE_WRITE_TIME) {
|
||||
d->WrtTime = update->Timestamp.Time;
|
||||
d->WrtDate = update->Timestamp.Date;
|
||||
}
|
||||
if (update->Flags & FAT32_DIRENT_UPDATE_ACCESS_DATE)
|
||||
d->LastAccDate = update->Timestamp.Date;
|
||||
s = WriteSectors(v, lba, 1, sec);
|
||||
}
|
||||
MmFreePool(sec);
|
||||
return s;
|
||||
}
|
||||
|
||||
static KSTATUS DirentUpdateSize(FAT32_VOL *v, uint64_t lba,
|
||||
uint32_t offset, uint32_t size) {
|
||||
FAT32_DIRENT_UPDATE u;
|
||||
memset(&u, 0, sizeof(u));
|
||||
FatQueryEncodedTime(&u.Timestamp);
|
||||
u.Flags = FAT32_DIRENT_UPDATE_SIZE | FAT32_DIRENT_UPDATE_WRITE_TIME;
|
||||
u.Size = size;
|
||||
return DirentUpdateMetadata(v, lba, offset, &u);
|
||||
}
|
||||
|
||||
static KSTATUS DirentUpdateFirstCluster(FAT32_VOL *v, uint64_t lba,
|
||||
uint32_t offset, uint32_t cluster) {
|
||||
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
|
||||
if (!sec) return STATUS_NO_MEMORY;
|
||||
KSTATUS s = ReadSectors(v, lba, 1, sec);
|
||||
if (KSUCCESS(s)) {
|
||||
DIRENT *d = (DIRENT *)(sec + offset);
|
||||
d->FirstClusterHi = (uint16_t)(cluster >> 16);
|
||||
d->FirstClusterLo = (uint16_t)(cluster & 0xFFFF);
|
||||
s = WriteSectors(v, lba, 1, sec);
|
||||
}
|
||||
MmFreePool(sec);
|
||||
return s;
|
||||
FAT32_DIRENT_UPDATE u;
|
||||
memset(&u, 0, sizeof(u));
|
||||
u.Flags = FAT32_DIRENT_UPDATE_FIRST_CLUSTER;
|
||||
u.FirstCluster = cluster;
|
||||
return DirentUpdateMetadata(v, lba, offset, &u);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -467,14 +621,67 @@ static int FatNameIsStructurallyShort83V(SYS_STRING_VIEW name) {
|
||||
return baseLen >= 1 && baseLen <= 8 && extLen <= 3;
|
||||
}
|
||||
|
||||
// Creation uses LFN when needed for structure, or when lowercase would be lost.
|
||||
static int FatNameNeedsLfnForCreateV(SYS_STRING_VIEW name) {
|
||||
if (!FatNameIsStructurallyShort83V(name)) return 1;
|
||||
for (size_t i = 0; i < name.Length; i++)
|
||||
static int FatComponentHasLowerV(SYS_STRING_VIEW name, size_t start, size_t end) {
|
||||
for (size_t i = start; i < end; i++)
|
||||
if (FatIsLowerChar(name.Data[i])) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int FatComponentHasUpperV(SYS_STRING_VIEW name, size_t start, size_t end) {
|
||||
for (size_t i = start; i < end; i++)
|
||||
if (name.Data[i] >= 'A' && name.Data[i] <= 'Z') return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void FatSplitShortNameV(SYS_STRING_VIEW name, size_t* baseEnd,
|
||||
size_t* extStart, size_t* extEnd) {
|
||||
size_t dot = (size_t)-1;
|
||||
for (size_t i = 0; i < name.Length; i++)
|
||||
if (name.Data[i] == '.') dot = i;
|
||||
if (dot == (size_t)-1) {
|
||||
*baseEnd = name.Length;
|
||||
*extStart = name.Length;
|
||||
*extEnd = name.Length;
|
||||
} else {
|
||||
*baseEnd = dot;
|
||||
*extStart = dot + 1u;
|
||||
*extEnd = name.Length;
|
||||
}
|
||||
}
|
||||
|
||||
// Creation uses LFN only when the spelling cannot be represented by a short
|
||||
// entry, or when mixed-case 8.3 spelling would be lost. Plain lowercase
|
||||
// 8.3 names are represented with FAT's NT case bits, not VFAT LFN entries.
|
||||
static int FatNameNeedsLfnForCreateV(SYS_STRING_VIEW name) {
|
||||
if (!FatNameIsStructurallyShort83V(name)) return 1;
|
||||
|
||||
size_t baseEnd, extStart, extEnd;
|
||||
FatSplitShortNameV(name, &baseEnd, &extStart, &extEnd);
|
||||
|
||||
int baseLower = FatComponentHasLowerV(name, 0, baseEnd);
|
||||
int baseUpper = FatComponentHasUpperV(name, 0, baseEnd);
|
||||
int extLower = FatComponentHasLowerV(name, extStart, extEnd);
|
||||
int extUpper = FatComponentHasUpperV(name, extStart, extEnd);
|
||||
|
||||
if ((baseLower && baseUpper) || (extLower && extUpper)) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint8_t FatShortNameCaseFlagsV(SYS_STRING_VIEW name) {
|
||||
if (!FatNameIsStructurallyShort83V(name)) return 0;
|
||||
size_t baseEnd, extStart, extEnd;
|
||||
FatSplitShortNameV(name, &baseEnd, &extStart, &extEnd);
|
||||
uint8_t flags = 0;
|
||||
if (FatComponentHasLowerV(name, 0, baseEnd) &&
|
||||
!FatComponentHasUpperV(name, 0, baseEnd))
|
||||
flags |= FAT_NTRES_BASE_LOWER;
|
||||
if (extStart < extEnd &&
|
||||
FatComponentHasLowerV(name, extStart, extEnd) &&
|
||||
!FatComponentHasUpperV(name, extStart, extEnd))
|
||||
flags |= FAT_NTRES_EXT_LOWER;
|
||||
return flags;
|
||||
}
|
||||
|
||||
static uint8_t LfnChecksum(const char name8[8], const char ext3[3]) {
|
||||
uint8_t sum = 0;
|
||||
for (int i = 0; i < 11; i++) {
|
||||
@@ -1218,9 +1425,17 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
MmFreePool(runBuf);
|
||||
|
||||
uint64_t newEnd = offset + written;
|
||||
if (newEnd > f->FileSize) {
|
||||
f->FileSize = (uint32_t)newEnd;
|
||||
DirentUpdateSize(v, f->DirentLba, f->DirentOffset, f->FileSize);
|
||||
if (written > 0) {
|
||||
FAT32_DIRENT_UPDATE u;
|
||||
memset(&u, 0, sizeof(u));
|
||||
FatQueryEncodedTime(&u.Timestamp);
|
||||
u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME;
|
||||
if (newEnd > f->FileSize) {
|
||||
f->FileSize = (uint32_t)newEnd;
|
||||
u.Flags |= FAT32_DIRENT_UPDATE_SIZE;
|
||||
u.Size = f->FileSize;
|
||||
}
|
||||
DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
|
||||
}
|
||||
|
||||
*bytesWritten = written;
|
||||
@@ -1316,6 +1531,17 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
|
||||
memcpy(d->Name, shortName, 8);
|
||||
memcpy(d->Ext, shortExt, 3);
|
||||
d->Attr = ATTR_ARCHIVE;
|
||||
d->Reserved = FatShortNameCaseFlagsV(filenameView);
|
||||
{
|
||||
FAT32_ENCODED_TIME now;
|
||||
FatQueryEncodedTime(&now);
|
||||
d->CrtTimeTenth = now.CreateTimeTenth;
|
||||
d->CrtTime = now.Time;
|
||||
d->CrtDate = now.Date;
|
||||
d->LastAccDate = now.Date;
|
||||
d->WrtTime = now.Time;
|
||||
d->WrtDate = now.Date;
|
||||
}
|
||||
|
||||
s = WriteSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf);
|
||||
MmFreePool(clusterBuf);
|
||||
@@ -1340,7 +1566,13 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
|
||||
(void)fs;
|
||||
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
|
||||
FAT32_VOL *v = f->Vol;
|
||||
if ((uint32_t)newSize == f->FileSize) return STATUS_SUCCESS;
|
||||
if ((uint32_t)newSize == f->FileSize) {
|
||||
FAT32_DIRENT_UPDATE u;
|
||||
memset(&u, 0, sizeof(u));
|
||||
FatQueryEncodedTime(&u.Timestamp);
|
||||
u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME;
|
||||
return DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
|
||||
}
|
||||
|
||||
if (newSize == 0) {
|
||||
if (f->StartCluster >= 2) {
|
||||
@@ -1495,6 +1727,8 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
|
||||
if (e->Attr == ATTR_LFN) { LfnAppend(ctx->LfnBuf, (LFN_ENTRY *)e); ctx->HasLfn = 1; continue; }
|
||||
if (e->Attr & 0x08 && !(e->Attr & ATTR_DIRECTORY)) { ctx->HasLfn = 0; memset(ctx->LfnBuf, 0, sizeof(ctx->LfnBuf)); continue; }
|
||||
|
||||
memset(out, 0, sizeof(*out));
|
||||
|
||||
if (ctx->HasLfn && ctx->LfnBuf[0]) {
|
||||
size_t len = strlen(ctx->LfnBuf);
|
||||
if (len >= KDIR_NAME_MAX) len = KDIR_NAME_MAX - 1;
|
||||
@@ -1513,7 +1747,11 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
|
||||
memcpy(out->Name, name83, len); out->Name[len] = '\0';
|
||||
}
|
||||
|
||||
out->Attr = e->Attr; out->FileSize = e->FileSize;
|
||||
out->Attr = e->Attr;
|
||||
out->FileSize = e->FileSize;
|
||||
FatDecodeDateTime(e->CrtDate, e->CrtTime, e->CrtTimeTenth, &out->CreateTime);
|
||||
FatDecodeAccessDate(e->LastAccDate, &out->LastAccessTime);
|
||||
FatDecodeDateTime(e->WrtDate, e->WrtTime, 0, &out->LastWriteTime);
|
||||
ctx->HasLfn = 0; memset(ctx->LfnBuf, 0, sizeof(ctx->LfnBuf));
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -1993,10 +2231,18 @@ static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) {
|
||||
static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
|
||||
if (ctx->Remaining == 0) {
|
||||
uint64_t newEnd = ctx->Offset + ctx->Done;
|
||||
if (newEnd > ctx->File->FileSize) {
|
||||
ctx->File->FileSize = (uint32_t)newEnd;
|
||||
DirentUpdateSize(ctx->Vol, ctx->File->DirentLba,
|
||||
ctx->File->DirentOffset, ctx->File->FileSize);
|
||||
if (ctx->Done > 0) {
|
||||
FAT32_DIRENT_UPDATE u;
|
||||
memset(&u, 0, sizeof(u));
|
||||
FatQueryEncodedTime(&u.Timestamp);
|
||||
u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME;
|
||||
if (newEnd > ctx->File->FileSize) {
|
||||
ctx->File->FileSize = (uint32_t)newEnd;
|
||||
u.Flags |= FAT32_DIRENT_UPDATE_SIZE;
|
||||
u.Size = ctx->File->FileSize;
|
||||
}
|
||||
DirentUpdateMetadata(ctx->Vol, ctx->File->DirentLba,
|
||||
ctx->File->DirentOffset, &u);
|
||||
}
|
||||
Fat32AsyncFinish(ctx, STATUS_SUCCESS, ctx->Done);
|
||||
return;
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_time.h>
|
||||
|
||||
typedef struct FAT32_VOL_S FAT32_VOL;
|
||||
|
||||
|
||||
@@ -2,8 +2,19 @@
|
||||
#include "rtl8139.h"
|
||||
|
||||
const HAL_API* gHal;
|
||||
RTL8139_DEVICE gDevice;
|
||||
bool gDevicePresent = false;
|
||||
static DRIVER_OBJECT gRtl8139Driver;
|
||||
static RTL8139_DEVICE* gRtl8139Devices[RTL8139_MAX_DEVICES];
|
||||
static uint32_t gRtl8139DeviceCount;
|
||||
static SPINLOCK gRtl8139RegistryLock = SPINLOCK_INIT;
|
||||
|
||||
static const IO_DEVICE_ID gRtl8139Ids[] = {
|
||||
{ IoBusPci, { "PCI\\VEN_10EC&DEV_8139", sizeof("PCI\\VEN_10EC&DEV_8139") - 1 } },
|
||||
};
|
||||
|
||||
static const IO_DRIVER_ID_TABLE gRtl8139IdTable = {
|
||||
gRtl8139Ids,
|
||||
sizeof(gRtl8139Ids) / sizeof(gRtl8139Ids[0]),
|
||||
};
|
||||
|
||||
static __forceinline void Rtl8139MemoryBarrier(void) {
|
||||
_ReadWriteBarrier();
|
||||
@@ -201,19 +212,68 @@ static void Rtl8139DpcRoutine(KDPC* dpc, void* context, void* systemArgument1, v
|
||||
DbgPrint("RTL8139: interrupt error bits=0x%04x\n", status);
|
||||
}
|
||||
|
||||
static KSTATUS Rtl8139RegisterDevice(RTL8139_DEVICE* r) {
|
||||
if (!r) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gRtl8139RegistryLock);
|
||||
for (uint32_t i = 0; i < gRtl8139DeviceCount; i++) {
|
||||
if (gRtl8139Devices[i] == r) {
|
||||
KeReleaseSpinlock(&gRtl8139RegistryLock, flags);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
if (gRtl8139DeviceCount >= RTL8139_MAX_DEVICES) {
|
||||
KeReleaseSpinlock(&gRtl8139RegistryLock, flags);
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
gRtl8139Devices[gRtl8139DeviceCount++] = r;
|
||||
KeReleaseSpinlock(&gRtl8139RegistryLock, flags);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static void Rtl8139UnregisterDevice(RTL8139_DEVICE* r) {
|
||||
if (!r) return;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gRtl8139RegistryLock);
|
||||
for (uint32_t i = 0; i < gRtl8139DeviceCount; i++) {
|
||||
if (gRtl8139Devices[i] == r) {
|
||||
gRtl8139Devices[i] = gRtl8139Devices[gRtl8139DeviceCount - 1u];
|
||||
gRtl8139Devices[gRtl8139DeviceCount - 1u] = (RTL8139_DEVICE*)0;
|
||||
gRtl8139DeviceCount--;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gRtl8139RegistryLock, flags);
|
||||
}
|
||||
|
||||
static void Rtl8139Isr(void) {
|
||||
if (!gDevicePresent)
|
||||
return;
|
||||
RTL8139_DEVICE* snapshot[RTL8139_MAX_DEVICES];
|
||||
uint32_t count = 0;
|
||||
|
||||
RTL8139_DEVICE* r = &gDevice;
|
||||
uint16_t status = Rtl8139Read16(r, RTL8139_REG_ISR);
|
||||
status &= RTL8139_INTR_ENABLES;
|
||||
if (!status || status == 0xFFFFu)
|
||||
return;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gRtl8139RegistryLock);
|
||||
count = gRtl8139DeviceCount;
|
||||
if (count > RTL8139_MAX_DEVICES)
|
||||
count = RTL8139_MAX_DEVICES;
|
||||
for (uint32_t i = 0; i < count; i++)
|
||||
snapshot[i] = gRtl8139Devices[i];
|
||||
KeReleaseSpinlock(&gRtl8139RegistryLock, flags);
|
||||
|
||||
Rtl8139Write16(r, RTL8139_REG_ISR, status);
|
||||
r->PendingIsr |= status;
|
||||
KeInsertQueueDpc(&r->Dpc, (void*)0, (void*)0);
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
RTL8139_DEVICE* r = snapshot[i];
|
||||
if (!r || !r->Started || r->Removed || r->SurpriseRemoved)
|
||||
continue;
|
||||
|
||||
uint16_t status = Rtl8139Read16(r, RTL8139_REG_ISR);
|
||||
status &= RTL8139_INTR_ENABLES;
|
||||
if (!status || status == 0xFFFFu)
|
||||
continue;
|
||||
|
||||
Rtl8139Write16(r, RTL8139_REG_ISR, status);
|
||||
r->PendingIsr |= status;
|
||||
KeInsertQueueDpc(&r->Dpc, (void*)0, (void*)0);
|
||||
}
|
||||
}
|
||||
|
||||
static KSTATUS Rtl8139SendFrame(NET_DEVICE* device, const void* frame, size_t length) {
|
||||
@@ -252,76 +312,242 @@ static KSTATUS Rtl8139SendFrame(NET_DEVICE* device, const void* frame, size_t le
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS Rtl8139Initialize(const HAL_API* hal) {
|
||||
gHal = hal;
|
||||
memset(&gDevice, 0, sizeof(gDevice));
|
||||
gDevice.TxLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
|
||||
if (!r || !r->Pdo)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
if (!gHal->Pci.FindDevice(RTL8139_VENDOR_REALTEK, RTL8139_DEVICE_ID,
|
||||
&gDevice.Bus, &gDevice.Dev, &gDevice.Fn)) {
|
||||
DbgPrint("RTL8139: no PCI device 10ec:8139 found; skipping\n");
|
||||
PCI_BAR best;
|
||||
PCI_BAR fallback;
|
||||
memset(&best, 0, sizeof(best));
|
||||
memset(&fallback, 0, sizeof(fallback));
|
||||
best.Type = PciBarUnused;
|
||||
fallback.Type = PciBarUnused;
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
PCI_BAR bar;
|
||||
KSTATUS st = PciGetBar(r->Pdo, i, &bar);
|
||||
if (!KSUCCESS(st))
|
||||
continue;
|
||||
|
||||
if (bar.Type == PciBarMemory32 || bar.Type == PciBarMemory64) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Type == PciBarIo && fallback.Type == PciBarUnused)
|
||||
fallback = bar;
|
||||
}
|
||||
|
||||
if (best.Type == PciBarUnused)
|
||||
best = fallback;
|
||||
|
||||
if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) {
|
||||
KSTATUS st = PciMapBar(r->Pdo,
|
||||
best.Index,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
|
||||
(void**)&r->Mmio,
|
||||
&r->RegisterBytes);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
r->RegisterBar = best;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL8139: BAR%u MMIO base=0x%016llX len=0x%llx virt=0x%016llX\n",
|
||||
best.Index,
|
||||
(unsigned long long)best.Base,
|
||||
(unsigned long long)r->RegisterBytes,
|
||||
r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
DbgPrint("RTL8139: found Realtek PCI device 10ec:8139 at %u:%u.%u\n",
|
||||
gDevice.Bus, gDevice.Dev, gDevice.Fn);
|
||||
if (best.Type == PciBarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
|
||||
KSTATUS st = Rtl8139SelectRegisterBar(&gDevice);
|
||||
r->RegisterBar = best;
|
||||
r->RegisterBytes = best.Length;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL8139: BAR%u I/O base=0x%04x len=0x%llx\n",
|
||||
best.Index,
|
||||
r->IoBase,
|
||||
(unsigned long long)r->RegisterBytes);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) {
|
||||
if (!r || r->Magic != RTL8139_DEVICE_MAGIC || !r->Pdo || !r->Fdo)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (r->Started)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
KSTATUS st = PciGetDeviceInfo(r->Pdo, &r->PciInfo);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL8139: PciGetDeviceInfo failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
DbgPrint("RTL8139: controller at PCI %02x:%02x.%x vendor=%04x device=%04x\n",
|
||||
r->PciInfo.Address.Bus,
|
||||
r->PciInfo.Address.Device,
|
||||
r->PciInfo.Address.Function,
|
||||
r->PciInfo.VendorId,
|
||||
r->PciInfo.DeviceId);
|
||||
|
||||
st = Rtl8139SelectRegisterBar(r);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL8139: no usable register BAR (0x%x)\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
Rtl8139EnablePci(&gDevice);
|
||||
|
||||
gDevice.Irq = hal->Pci.GetIrqLine(gDevice.Bus, gDevice.Dev, gDevice.Fn);
|
||||
if (gDevice.Irq == 0xFFu) {
|
||||
DbgPrint("RTL8139: no legacy PCI IRQ line wired\n");
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
st = PciEnableDevice(r->Pdo,
|
||||
PCI_ENABLE_BUSMASTER |
|
||||
PCI_ENABLE_INTX |
|
||||
(r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL8139: PciEnableDevice failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
// Wake, reset, then read the station address.
|
||||
Rtl8139Write8(&gDevice, RTL8139_REG_CONFIG1, 0x00);
|
||||
st = Rtl8139Reset(&gDevice);
|
||||
st = PciGetInterrupt(r->Pdo, &r->Irq);
|
||||
if (!KSUCCESS(st) || r->Irq == 0xFFu) {
|
||||
DbgPrint("RTL8139: no legacy PCI IRQ line wired\n");
|
||||
return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
|
||||
}
|
||||
|
||||
Rtl8139Write8(r, RTL8139_REG_CONFIG1, 0x00);
|
||||
st = Rtl8139Reset(r);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL8139: reset timed out\n");
|
||||
return st;
|
||||
}
|
||||
|
||||
Rtl8139ReadMac(&gDevice);
|
||||
Rtl8139ReadMac(r);
|
||||
DbgPrint("RTL8139: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u\n",
|
||||
gDevice.NetDevice.HardwareAddress[0], gDevice.NetDevice.HardwareAddress[1],
|
||||
gDevice.NetDevice.HardwareAddress[2], gDevice.NetDevice.HardwareAddress[3],
|
||||
gDevice.NetDevice.HardwareAddress[4], gDevice.NetDevice.HardwareAddress[5],
|
||||
gDevice.Irq);
|
||||
r->NetDevice.HardwareAddress[0], r->NetDevice.HardwareAddress[1],
|
||||
r->NetDevice.HardwareAddress[2], r->NetDevice.HardwareAddress[3],
|
||||
r->NetDevice.HardwareAddress[4], r->NetDevice.HardwareAddress[5],
|
||||
r->Irq);
|
||||
|
||||
st = Rtl8139AllocateDma(&gDevice);
|
||||
st = Rtl8139AllocateDma(r);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
KeInitializeDpc(&gDevice.Dpc, Rtl8139DpcRoutine, &gDevice);
|
||||
KeInitializeDpc(&r->Dpc, Rtl8139DpcRoutine, r);
|
||||
|
||||
gDevice.NetDevice.Mtu = NET_MTU_ETHERNET;
|
||||
gDevice.NetDevice.DriverContext = &gDevice;
|
||||
gDevice.NetDevice.SendFrame = Rtl8139SendFrame;
|
||||
r->NetDevice.Mtu = NET_MTU_ETHERNET;
|
||||
r->NetDevice.DriverContext = r;
|
||||
r->NetDevice.SendFrame = Rtl8139SendFrame;
|
||||
|
||||
Rtl8139ProgramHardware(&gDevice);
|
||||
|
||||
gDevicePresent = true;
|
||||
KeIrqRegister(gDevice.Irq, Rtl8139Isr);
|
||||
|
||||
st = NetRegisterDevice(&gDevice.NetDevice);
|
||||
st = Rtl8139RegisterDevice(r);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
DbgPrint("RTL8139: initialized; RX/TX enabled on IRQ %u\n", gDevice.Irq);
|
||||
KeIrqRegister(r->Irq, Rtl8139Isr);
|
||||
r->Started = true;
|
||||
Rtl8139ProgramHardware(r);
|
||||
r->Fdo->PnpState = DevicePnpStarted;
|
||||
|
||||
st = NetRegisterDevice(&r->NetDevice);
|
||||
if (!KSUCCESS(st)) {
|
||||
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
|
||||
r->Started = false;
|
||||
Rtl8139UnregisterDevice(r);
|
||||
return st;
|
||||
}
|
||||
|
||||
DbgPrint("RTL8139: initialized; RX/TX enabled on IRQ %u\n", r->Irq);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp || !device->DeviceExtension)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
RTL8139_DEVICE* r = (RTL8139_DEVICE*)device->DeviceExtension;
|
||||
if (r->Magic != RTL8139_DEVICE_MAGIC)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
return Rtl8139StartDevice(r);
|
||||
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
r->SurpriseRemoved = true;
|
||||
device->PnpState = DevicePnpSurpriseRemoved;
|
||||
if (r->Started)
|
||||
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
r->Removed = true;
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
if (r->Started)
|
||||
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
|
||||
Rtl8139UnregisterDevice(r);
|
||||
IoDetachDevice(device);
|
||||
IoDeleteDevice(device);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
default:
|
||||
return IoForwardIrpToLower(device, irp);
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
|
||||
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreateDevice(driver, sizeof(RTL8139_DEVICE), &fdo);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("rtl8139"));
|
||||
|
||||
RTL8139_DEVICE* r = (RTL8139_DEVICE*)fdo->DeviceExtension;
|
||||
memset(r, 0, sizeof(*r));
|
||||
r->Magic = RTL8139_DEVICE_MAGIC;
|
||||
r->Fdo = fdo;
|
||||
r->Pdo = physicalDeviceObject;
|
||||
r->Irq = 0xFFu;
|
||||
r->TxLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
|
||||
r->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
|
||||
if (!r->LowerDevice) {
|
||||
IoDeleteDevice(fdo);
|
||||
return STATUS_FAILURE;
|
||||
}
|
||||
|
||||
DbgPrint("RTL8139: FDO attached to %v\n", physicalDeviceObject->Name);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
if (!hal)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DbgPrint("RTL8139: driver loading...\n");
|
||||
KSTATUS st = Rtl8139Initialize(hal);
|
||||
gHal = hal;
|
||||
gRtl8139DeviceCount = 0;
|
||||
|
||||
memset(&gRtl8139Driver, 0, sizeof(gRtl8139Driver));
|
||||
IoInitializeDriverObject(&gRtl8139Driver, SYS_SV_LITERAL("rtl8139"));
|
||||
gRtl8139Driver.IdTable = &gRtl8139IdTable;
|
||||
gRtl8139Driver.DriverAddDevice = Rtl8139AddDevice;
|
||||
gRtl8139Driver.MajorFunction[IRP_MJ_PNP] = Rtl8139DispatchPnp;
|
||||
|
||||
KSTATUS st = IoRegisterDriverObject(&gRtl8139Driver);
|
||||
if (!KSUCCESS(st))
|
||||
DbgPrint("RTL8139: init failed: 0x%x\n", st);
|
||||
DbgPrint("RTL8139: IoRegisterDriverObject failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
@@ -8,6 +8,8 @@
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/pci.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
@@ -20,12 +22,7 @@
|
||||
#define RTL8139_VENDOR_REALTEK 0x10ECu
|
||||
#define RTL8139_DEVICE_ID 0x8139u
|
||||
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_COMMAND_IO (1u << 0)
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
#define RTL8139_DEVICE_MAGIC 0x81390001u
|
||||
|
||||
#define RTL8139_REG_IDR0 0x00u
|
||||
#define RTL8139_REG_MAR0 0x08u
|
||||
@@ -91,30 +88,29 @@
|
||||
#define RTL8139_RX_ALLOC_SIZE (RTL8139_RX_RING_SIZE + RTL8139_RX_PAD + RTL8139_RX_GUARD)
|
||||
#define RTL8139_RX_PAGES ((RTL8139_RX_ALLOC_SIZE + 4095u) / 4096u)
|
||||
#define RTL8139_TX_PAGES (((RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE) + 4095u) / 4096u)
|
||||
#define RTL8139_MMIO_PAGES 1u
|
||||
#define RTL8139_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u)
|
||||
#define RTL8139_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u)
|
||||
|
||||
#define RTL8139_REG_TSD(slot) (RTL8139_REG_TSD0 + ((slot) * 4u))
|
||||
#define RTL8139_REG_TSAD(slot) (RTL8139_REG_TSAD0 + ((slot) * 4u))
|
||||
|
||||
typedef enum RTL8139_BAR_KIND {
|
||||
Rtl8139BarNone = 0,
|
||||
Rtl8139BarIo,
|
||||
Rtl8139BarMmio,
|
||||
} RTL8139_BAR_KIND;
|
||||
|
||||
typedef struct RTL8139_BAR {
|
||||
RTL8139_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} RTL8139_BAR;
|
||||
#define RTL8139_MAX_DEVICES 8u
|
||||
|
||||
typedef struct RTL8139_DEVICE {
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
uint32_t Magic;
|
||||
|
||||
DEVICE_OBJECT* Fdo;
|
||||
DEVICE_OBJECT* Pdo;
|
||||
DEVICE_OBJECT* LowerDevice;
|
||||
|
||||
PCI_DEVICE_INFO PciInfo;
|
||||
PCI_BAR RegisterBar;
|
||||
uint64_t RegisterBytes;
|
||||
uint8_t Irq;
|
||||
|
||||
bool Started;
|
||||
bool Removed;
|
||||
bool SurpriseRemoved;
|
||||
|
||||
bool UseMmio;
|
||||
volatile uint8_t* Mmio;
|
||||
uint16_t IoBase;
|
||||
@@ -134,12 +130,7 @@ typedef struct RTL8139_DEVICE {
|
||||
NET_DEVICE NetDevice;
|
||||
} RTL8139_DEVICE;
|
||||
|
||||
|
||||
extern const HAL_API* gHal;
|
||||
extern RTL8139_DEVICE gDevice;
|
||||
extern bool gDevicePresent;
|
||||
|
||||
bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out);
|
||||
KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio);
|
||||
KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r);
|
||||
void Rtl8139EnablePci(RTL8139_DEVICE* r);
|
||||
KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
|
||||
KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp);
|
||||
@@ -1,112 +1,2 @@
|
||||
// rtl8139_pci.c - PCI BAR/MMIO front-end for rtl8139.
|
||||
#include "rtl8139.h"
|
||||
|
||||
bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out) {
|
||||
uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u);
|
||||
uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg);
|
||||
|
||||
out->Kind = Rtl8139BarNone;
|
||||
out->Base = 0;
|
||||
out->Index = index;
|
||||
|
||||
if (lo == 0 || lo == 0xFFFFFFFFu)
|
||||
return false;
|
||||
|
||||
if (lo & 1u) {
|
||||
uint32_t base = lo & ~0x3u;
|
||||
if (!base) return false;
|
||||
out->Kind = Rtl8139BarIo;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t base = (uint64_t)(lo & ~0xFu);
|
||||
uint32_t type = (lo >> 1) & 0x3u;
|
||||
if (type == 0x2u && index < 5) {
|
||||
uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u));
|
||||
base |= ((uint64_t)hi << 32);
|
||||
}
|
||||
|
||||
if (!base) return false;
|
||||
out->Kind = Rtl8139BarMmio;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio) {
|
||||
uint64_t pageBase = physBase & PAGE_MASK;
|
||||
uint64_t pageOff = physBase - pageBase;
|
||||
|
||||
for (uint32_t i = 0; i < RTL8139_MMIO_PAGES; i++) {
|
||||
PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE;
|
||||
uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys);
|
||||
KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, MmMapFailIfPresent);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
*outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
|
||||
RTL8139_BAR best = { Rtl8139BarNone, 0, 0 };
|
||||
RTL8139_BAR fallback = { Rtl8139BarNone, 0, 0 };
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
RTL8139_BAR bar;
|
||||
if (!Rtl8139ReadBar(r->Bus, r->Dev, r->Fn, i, &bar))
|
||||
continue;
|
||||
|
||||
if (bar.Kind == Rtl8139BarMmio) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Kind == Rtl8139BarIo && fallback.Kind == Rtl8139BarNone)
|
||||
fallback = bar;
|
||||
|
||||
if (bar.Kind == Rtl8139BarMmio) {
|
||||
uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u));
|
||||
if (((lo >> 1) & 0x3u) == 0x2u)
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (best.Kind == Rtl8139BarNone)
|
||||
best = fallback;
|
||||
|
||||
if (best.Kind == Rtl8139BarMmio) {
|
||||
KSTATUS st = Rtl8139MapMmio(best.Base, &r->Mmio);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL8139: BAR%u MMIO base=0x%016llX virt=0x%016llX\n",
|
||||
best.Index, (unsigned long long)best.Base, r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (best.Kind == Rtl8139BarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL8139: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
|
||||
void Rtl8139EnablePci(RTL8139_DEVICE* r) {
|
||||
uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND);
|
||||
cmd |= PCI_COMMAND_MASTER;
|
||||
if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY;
|
||||
else cmd |= PCI_COMMAND_IO;
|
||||
cmd &= ~PCI_COMMAND_INTX_DISABLE;
|
||||
gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd);
|
||||
}
|
||||
// rtl8139_pci.c intentionally empty.
|
||||
// PCI enumeration, BAR decoding, mapping, and command-register policy now live in pci.sys.
|
||||
@@ -1,9 +1,21 @@
|
||||
// rtl816x.c - packet I/O, interrupt, and DriverEntry path.
|
||||
// rtl816x.c - rtl8168/rtl8169 packet I/O, interrupt, and PCI function-driver path.
|
||||
#include "rtl816x.h"
|
||||
|
||||
const HAL_API* gHal;
|
||||
RTL816X_DEVICE gDevice;
|
||||
bool gDevicePresent = false;
|
||||
static DRIVER_OBJECT gRtl816xDriver;
|
||||
static RTL816X_DEVICE* gRtl816xDevices[RTL816X_MAX_DEVICES];
|
||||
static uint32_t gRtl816xDeviceCount;
|
||||
static SPINLOCK gRtl816xRegistryLock = SPINLOCK_INIT;
|
||||
|
||||
static const IO_DEVICE_ID gRtl816xIds[] = {
|
||||
{ IoBusPci, { "PCI\\VEN_10EC&DEV_8168", sizeof("PCI\\VEN_10EC&DEV_8168") - 1 } },
|
||||
{ IoBusPci, { "PCI\\VEN_10EC&DEV_8169", sizeof("PCI\\VEN_10EC&DEV_8169") - 1 } },
|
||||
};
|
||||
|
||||
static const IO_DRIVER_ID_TABLE gRtl816xIdTable = {
|
||||
gRtl816xIds,
|
||||
sizeof(gRtl816xIds) / sizeof(gRtl816xIds[0]),
|
||||
};
|
||||
|
||||
static __forceinline void RtlMemoryBarrier(void) {
|
||||
_ReadWriteBarrier();
|
||||
@@ -44,11 +56,6 @@ static void RtlWrite32(RTL816X_DEVICE* r, uint32_t reg, uint32_t value) {
|
||||
else __outdword((unsigned short)(r->IoBase + reg), value);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static KSTATUS RtlReset(RTL816X_DEVICE* r) {
|
||||
RtlWrite8(r, RTL_REG_CHIP_CMD, RTL_CMD_RESET);
|
||||
for (uint32_t i = 0; i < 1000; i++) {
|
||||
@@ -116,14 +123,11 @@ static void RtlInitDescriptors(RTL816X_DEVICE* r) {
|
||||
}
|
||||
|
||||
static void RtlProgramHardware(RTL816X_DEVICE* r) {
|
||||
// Unlock the 9346-style config register window while programming.
|
||||
RtlWrite8(r, RTL_REG_CFG9346, 0xC0);
|
||||
|
||||
// Disable interrupts while rings are being installed.
|
||||
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
|
||||
RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
|
||||
|
||||
// DMA ring base addresses.
|
||||
RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingPhys);
|
||||
RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingPhys >> 32));
|
||||
RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingPhys);
|
||||
@@ -131,7 +135,6 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) {
|
||||
RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingPhys);
|
||||
RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingPhys >> 32));
|
||||
|
||||
// Plain packet mode. Keep C+ features mostly disabled for bring-up.
|
||||
RtlWrite16(r, RTL_REG_CPLUS_CMD, RtlRead16(r, RTL_REG_CPLUS_CMD) & ~(uint16_t)0x00E0u);
|
||||
RtlWrite16(r, RTL_REG_RX_MAXSIZE, RTL_BUF_SIZE);
|
||||
|
||||
@@ -141,10 +144,8 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) {
|
||||
|
||||
RtlMemoryBarrier();
|
||||
|
||||
// Enable receiver and transmitter.
|
||||
RtlWrite8(r, RTL_REG_CHIP_CMD, RTL_CMD_RX_ENABLE | RTL_CMD_TX_ENABLE);
|
||||
|
||||
// Lock config space again and enable selected interrupts.
|
||||
RtlWrite8(r, RTL_REG_CFG9346, 0x00);
|
||||
RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
|
||||
RtlWrite16(r, RTL_REG_INTR_MASK, RTL_INTR_ENABLES);
|
||||
@@ -164,8 +165,6 @@ static void RtlHandleRx(RTL816X_DEVICE* r) {
|
||||
len >= NET_ETHERNET_HEADER_LENGTH && len <= RTL_RX_WIRE_MAX;
|
||||
|
||||
if (good) {
|
||||
// Realtek reports the Ethernet FCS in the descriptor length on many
|
||||
// chips. Remove it when present so upper layers see a normal frame.
|
||||
if (len > 4)
|
||||
len -= 4;
|
||||
if (len > NET_ETHERNET_MAX_FRAME) {
|
||||
@@ -195,6 +194,9 @@ static void RtlDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void*
|
||||
(void)systemArgument2;
|
||||
|
||||
RTL816X_DEVICE* r = (RTL816X_DEVICE*)context;
|
||||
if (!r || r->Removed || r->SurpriseRemoved)
|
||||
return;
|
||||
|
||||
uint16_t status = (uint16_t)r->PendingIsr;
|
||||
r->PendingIsr = 0;
|
||||
|
||||
@@ -210,18 +212,68 @@ static void RtlDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void*
|
||||
DbgPrint("RTL816X: interrupt status error bits=0x%04x\n", status);
|
||||
}
|
||||
|
||||
static KSTATUS RtlRegisterDevice(RTL816X_DEVICE* r) {
|
||||
if (!r) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gRtl816xRegistryLock);
|
||||
for (uint32_t i = 0; i < gRtl816xDeviceCount; i++) {
|
||||
if (gRtl816xDevices[i] == r) {
|
||||
KeReleaseSpinlock(&gRtl816xRegistryLock, flags);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
if (gRtl816xDeviceCount >= RTL816X_MAX_DEVICES) {
|
||||
KeReleaseSpinlock(&gRtl816xRegistryLock, flags);
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
gRtl816xDevices[gRtl816xDeviceCount++] = r;
|
||||
KeReleaseSpinlock(&gRtl816xRegistryLock, flags);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static void RtlUnregisterDevice(RTL816X_DEVICE* r) {
|
||||
if (!r) return;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gRtl816xRegistryLock);
|
||||
for (uint32_t i = 0; i < gRtl816xDeviceCount; i++) {
|
||||
if (gRtl816xDevices[i] == r) {
|
||||
gRtl816xDevices[i] = gRtl816xDevices[gRtl816xDeviceCount - 1u];
|
||||
gRtl816xDevices[gRtl816xDeviceCount - 1u] = (RTL816X_DEVICE*)0;
|
||||
gRtl816xDeviceCount--;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gRtl816xRegistryLock, flags);
|
||||
}
|
||||
|
||||
static void RtlIsr(void) {
|
||||
if (!gDevicePresent)
|
||||
return;
|
||||
RTL816X_DEVICE* snapshot[RTL816X_MAX_DEVICES];
|
||||
uint32_t count = 0;
|
||||
|
||||
RTL816X_DEVICE* r = &gDevice;
|
||||
uint16_t status = RtlRead16(r, RTL_REG_INTR_STAT);
|
||||
if (!status || status == 0xFFFFu)
|
||||
return;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gRtl816xRegistryLock);
|
||||
count = gRtl816xDeviceCount;
|
||||
if (count > RTL816X_MAX_DEVICES)
|
||||
count = RTL816X_MAX_DEVICES;
|
||||
for (uint32_t i = 0; i < count; i++)
|
||||
snapshot[i] = gRtl816xDevices[i];
|
||||
KeReleaseSpinlock(&gRtl816xRegistryLock, flags);
|
||||
|
||||
RtlWrite16(r, RTL_REG_INTR_STAT, status);
|
||||
r->PendingIsr |= status;
|
||||
KeInsertQueueDpc(&r->Dpc, (void*)0, (void*)0);
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
RTL816X_DEVICE* r = snapshot[i];
|
||||
if (!r || !r->Started || r->Removed || r->SurpriseRemoved)
|
||||
continue;
|
||||
|
||||
uint16_t status = RtlRead16(r, RTL_REG_INTR_STAT);
|
||||
status &= RTL_INTR_ENABLES;
|
||||
if (!status || status == 0xFFFFu)
|
||||
continue;
|
||||
|
||||
RtlWrite16(r, RTL_REG_INTR_STAT, status);
|
||||
r->PendingIsr |= status;
|
||||
KeInsertQueueDpc(&r->Dpc, (void*)0, (void*)0);
|
||||
}
|
||||
}
|
||||
|
||||
static KSTATUS RtlSendFrame(NET_DEVICE* device, const void* frame, size_t length) {
|
||||
@@ -233,8 +285,8 @@ static KSTATUS RtlSendFrame(NET_DEVICE* device, const void* frame, size_t length
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
RTL816X_DEVICE* r = (RTL816X_DEVICE*)device->DriverContext;
|
||||
if (!r)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (!r || !r->Started || r->Removed || r->SurpriseRemoved)
|
||||
return STATUS_OBJECT_DEAD;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&r->TxLock);
|
||||
uint32_t index = r->TxIndex;
|
||||
@@ -263,75 +315,242 @@ static KSTATUS RtlSendFrame(NET_DEVICE* device, const void* frame, size_t length
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS RtlInitialize(const HAL_API* hal) {
|
||||
gHal = hal;
|
||||
memset(&gDevice, 0, sizeof(gDevice));
|
||||
gDevice.TxLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
|
||||
if (!r || !r->Pdo)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uint16_t deviceId;
|
||||
if (!RtlFindDevice(&gDevice.Bus, &gDevice.Dev, &gDevice.Fn, &deviceId)) {
|
||||
DbgPrint("RTL816X: no RTL8168/RTL8169 PCI device found; skipping\n");
|
||||
PCI_BAR best;
|
||||
PCI_BAR fallback;
|
||||
memset(&best, 0, sizeof(best));
|
||||
memset(&fallback, 0, sizeof(fallback));
|
||||
best.Type = PciBarUnused;
|
||||
fallback.Type = PciBarUnused;
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
PCI_BAR bar;
|
||||
KSTATUS st = PciGetBar(r->Pdo, i, &bar);
|
||||
if (!KSUCCESS(st))
|
||||
continue;
|
||||
|
||||
if (bar.Type == PciBarMemory32 || bar.Type == PciBarMemory64) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Type == PciBarIo && fallback.Type == PciBarUnused)
|
||||
fallback = bar;
|
||||
}
|
||||
|
||||
if (best.Type == PciBarUnused)
|
||||
best = fallback;
|
||||
|
||||
if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) {
|
||||
KSTATUS st = PciMapBar(r->Pdo,
|
||||
best.Index,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
|
||||
(void**)&r->Mmio,
|
||||
&r->RegisterBytes);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
r->RegisterBar = best;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL816X: BAR%u MMIO base=0x%016llX len=0x%llx virt=0x%016llX\n",
|
||||
best.Index,
|
||||
(unsigned long long)best.Base,
|
||||
(unsigned long long)r->RegisterBytes,
|
||||
r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
gDevice.DeviceId = deviceId;
|
||||
|
||||
DbgPrint("RTL816X: found Realtek PCI device 10ec:%04x at %u:%u.%u\n",
|
||||
deviceId, gDevice.Bus, gDevice.Dev, gDevice.Fn);
|
||||
if (best.Type == PciBarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
|
||||
KSTATUS st = RtlSelectRegisterBar(&gDevice);
|
||||
r->RegisterBar = best;
|
||||
r->RegisterBytes = best.Length;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL816X: BAR%u I/O base=0x%04x len=0x%llx\n",
|
||||
best.Index,
|
||||
r->IoBase,
|
||||
(unsigned long long)r->RegisterBytes);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
|
||||
if (!r || r->Magic != RTL816X_DEVICE_MAGIC || !r->Pdo || !r->Fdo)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (r->Started)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
KSTATUS st = PciGetDeviceInfo(r->Pdo, &r->PciInfo);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL816X: PciGetDeviceInfo failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
DbgPrint("RTL816X: controller at PCI %02x:%02x.%x vendor=%04x device=%04x\n",
|
||||
r->PciInfo.Address.Bus,
|
||||
r->PciInfo.Address.Device,
|
||||
r->PciInfo.Address.Function,
|
||||
r->PciInfo.VendorId,
|
||||
r->PciInfo.DeviceId);
|
||||
|
||||
st = RtlSelectRegisterBar(r);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL816X: no usable register BAR (0x%x)\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
RtlEnablePci(&gDevice);
|
||||
|
||||
gDevice.Irq = hal->Pci.GetIrqLine(gDevice.Bus, gDevice.Dev, gDevice.Fn);
|
||||
if (gDevice.Irq == 0xFFu) {
|
||||
DbgPrint("RTL816X: no legacy PCI IRQ line wired\n");
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
st = PciEnableDevice(r->Pdo,
|
||||
PCI_ENABLE_BUSMASTER |
|
||||
PCI_ENABLE_INTX |
|
||||
(r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL816X: PciEnableDevice failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
st = RtlReset(&gDevice);
|
||||
st = PciGetInterrupt(r->Pdo, &r->Irq);
|
||||
if (!KSUCCESS(st) || r->Irq == 0xFFu) {
|
||||
DbgPrint("RTL816X: no legacy PCI IRQ line wired\n");
|
||||
return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
|
||||
}
|
||||
|
||||
st = RtlReset(r);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("RTL816X: reset timed out\n");
|
||||
return st;
|
||||
}
|
||||
|
||||
RtlReadMac(&gDevice);
|
||||
RtlReadMac(r);
|
||||
DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u\n",
|
||||
gDevice.NetDevice.HardwareAddress[0], gDevice.NetDevice.HardwareAddress[1],
|
||||
gDevice.NetDevice.HardwareAddress[2], gDevice.NetDevice.HardwareAddress[3],
|
||||
gDevice.NetDevice.HardwareAddress[4], gDevice.NetDevice.HardwareAddress[5],
|
||||
gDevice.Irq);
|
||||
r->NetDevice.HardwareAddress[0], r->NetDevice.HardwareAddress[1],
|
||||
r->NetDevice.HardwareAddress[2], r->NetDevice.HardwareAddress[3],
|
||||
r->NetDevice.HardwareAddress[4], r->NetDevice.HardwareAddress[5],
|
||||
r->Irq);
|
||||
|
||||
st = RtlAllocateDma(&gDevice);
|
||||
st = RtlAllocateDma(r);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
RtlInitDescriptors(&gDevice);
|
||||
KeInitializeDpc(&gDevice.Dpc, RtlDpcRoutine, &gDevice);
|
||||
RtlInitDescriptors(r);
|
||||
KeInitializeDpc(&r->Dpc, RtlDpcRoutine, r);
|
||||
|
||||
gDevice.NetDevice.Mtu = NET_MTU_ETHERNET;
|
||||
gDevice.NetDevice.DriverContext = &gDevice;
|
||||
gDevice.NetDevice.SendFrame = RtlSendFrame;
|
||||
RtlProgramHardware(&gDevice);
|
||||
r->NetDevice.Mtu = NET_MTU_ETHERNET;
|
||||
r->NetDevice.DriverContext = r;
|
||||
r->NetDevice.SendFrame = RtlSendFrame;
|
||||
|
||||
gDevicePresent = true;
|
||||
KeIrqRegister(gDevice.Irq, RtlIsr);
|
||||
|
||||
st = NetRegisterDevice(&gDevice.NetDevice);
|
||||
st = RtlRegisterDevice(r);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
DbgPrint("RTL816X: initialized; RX/TX enabled on IRQ %u\n", gDevice.Irq);
|
||||
KeIrqRegister(r->Irq, RtlIsr);
|
||||
r->Started = true;
|
||||
RtlProgramHardware(r);
|
||||
r->Fdo->PnpState = DevicePnpStarted;
|
||||
|
||||
st = NetRegisterDevice(&r->NetDevice);
|
||||
if (!KSUCCESS(st)) {
|
||||
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
|
||||
r->Started = false;
|
||||
RtlUnregisterDevice(r);
|
||||
return st;
|
||||
}
|
||||
|
||||
DbgPrint("RTL816X: initialized; RX/TX enabled on IRQ %u\n", r->Irq);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp || !device->DeviceExtension)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
RTL816X_DEVICE* r = (RTL816X_DEVICE*)device->DeviceExtension;
|
||||
if (r->Magic != RTL816X_DEVICE_MAGIC)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
return RtlStartDevice(r);
|
||||
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
r->SurpriseRemoved = true;
|
||||
device->PnpState = DevicePnpSurpriseRemoved;
|
||||
if (r->Started)
|
||||
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
r->Removed = true;
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
if (r->Started)
|
||||
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
|
||||
RtlUnregisterDevice(r);
|
||||
IoDetachDevice(device);
|
||||
IoDeleteDevice(device);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
default:
|
||||
return IoForwardIrpToLower(device, irp);
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
|
||||
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreateDevice(driver, sizeof(RTL816X_DEVICE), &fdo);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("rtl816x"));
|
||||
|
||||
RTL816X_DEVICE* r = (RTL816X_DEVICE*)fdo->DeviceExtension;
|
||||
memset(r, 0, sizeof(*r));
|
||||
r->Magic = RTL816X_DEVICE_MAGIC;
|
||||
r->Fdo = fdo;
|
||||
r->Pdo = physicalDeviceObject;
|
||||
r->Irq = 0xFFu;
|
||||
r->TxLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
|
||||
r->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
|
||||
if (!r->LowerDevice) {
|
||||
IoDeleteDevice(fdo);
|
||||
return STATUS_FAILURE;
|
||||
}
|
||||
|
||||
DbgPrint("RTL816X: FDO attached to %v\n", physicalDeviceObject->Name);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
if (!hal)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DbgPrint("RTL816X: driver loading...\n");
|
||||
KSTATUS st = RtlInitialize(hal);
|
||||
gHal = hal;
|
||||
gRtl816xDeviceCount = 0;
|
||||
|
||||
memset(&gRtl816xDriver, 0, sizeof(gRtl816xDriver));
|
||||
IoInitializeDriverObject(&gRtl816xDriver, SYS_SV_LITERAL("rtl816x"));
|
||||
gRtl816xDriver.IdTable = &gRtl816xIdTable;
|
||||
gRtl816xDriver.DriverAddDevice = Rtl816xAddDevice;
|
||||
gRtl816xDriver.MajorFunction[IRP_MJ_PNP] = Rtl816xDispatchPnp;
|
||||
|
||||
KSTATUS st = IoRegisterDriverObject(&gRtl816xDriver);
|
||||
if (!KSUCCESS(st))
|
||||
DbgPrint("RTL816X: init failed: 0x%x\n", st);
|
||||
DbgPrint("RTL816X: IoRegisterDriverObject failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
// rtl816x.h - shared rtl816x driver internals.
|
||||
// rtl816x.h - shared rtl8168/rtl8169 driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
@@ -8,6 +8,8 @@
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/pci.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
@@ -21,13 +23,8 @@
|
||||
#define RTL_DEVICE_8168 0x8168u
|
||||
#define RTL_DEVICE_8169 0x8169u
|
||||
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_REG_IRQ 0x3Cu
|
||||
#define PCI_COMMAND_IO (1u << 0)
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
#define RTL816X_DEVICE_MAGIC 0x81690001u
|
||||
#define RTL816X_MAX_DEVICES 8u
|
||||
|
||||
#define RTL_REG_MAC0 0x00u
|
||||
#define RTL_REG_MAR0 0x08u
|
||||
@@ -99,7 +96,6 @@
|
||||
#define RTL_RING_PAGES 1u
|
||||
#define RTL_RXBUF_PAGES ((RTL_RX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u)
|
||||
#define RTL_TXBUF_PAGES ((RTL_TX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u)
|
||||
#define RTL_MMIO_PAGES 1u
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef struct RTL_DESC {
|
||||
@@ -110,24 +106,21 @@ typedef struct RTL_DESC {
|
||||
} RTL_DESC;
|
||||
#pragma pack(pop)
|
||||
|
||||
typedef enum RTL_BAR_KIND {
|
||||
RtlBarNone = 0,
|
||||
RtlBarIo,
|
||||
RtlBarMmio,
|
||||
} RTL_BAR_KIND;
|
||||
|
||||
typedef struct RTL_BAR {
|
||||
RTL_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} RTL_BAR;
|
||||
|
||||
typedef struct RTL816X_DEVICE {
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
uint32_t Magic;
|
||||
|
||||
DEVICE_OBJECT* Fdo;
|
||||
DEVICE_OBJECT* Pdo;
|
||||
DEVICE_OBJECT* LowerDevice;
|
||||
|
||||
PCI_DEVICE_INFO PciInfo;
|
||||
PCI_BAR RegisterBar;
|
||||
uint64_t RegisterBytes;
|
||||
uint8_t Irq;
|
||||
uint16_t DeviceId;
|
||||
|
||||
bool Started;
|
||||
bool Removed;
|
||||
bool SurpriseRemoved;
|
||||
|
||||
bool UseMmio;
|
||||
volatile uint8_t* Mmio;
|
||||
@@ -152,13 +145,7 @@ typedef struct RTL816X_DEVICE {
|
||||
NET_DEVICE NetDevice;
|
||||
} RTL816X_DEVICE;
|
||||
|
||||
|
||||
extern const HAL_API* gHal;
|
||||
extern RTL816X_DEVICE gDevice;
|
||||
extern bool gDevicePresent;
|
||||
|
||||
bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId);
|
||||
bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out);
|
||||
KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio);
|
||||
KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r);
|
||||
void RtlEnablePci(RTL816X_DEVICE* r);
|
||||
KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
|
||||
KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
|
||||
@@ -1,126 +0,0 @@
|
||||
// rtl816x_pci.c - PCI BAR/MMIO front-end for rtl816x.
|
||||
#include "rtl816x.h"
|
||||
|
||||
bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId) {
|
||||
if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8168, outBus, outDev, outFn)) {
|
||||
*outDeviceId = RTL_DEVICE_8168;
|
||||
return true;
|
||||
}
|
||||
if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8169, outBus, outDev, outFn)) {
|
||||
*outDeviceId = RTL_DEVICE_8169;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out) {
|
||||
uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u);
|
||||
uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg);
|
||||
|
||||
out->Kind = RtlBarNone;
|
||||
out->Base = 0;
|
||||
out->Index = index;
|
||||
|
||||
if (lo == 0 || lo == 0xFFFFFFFFu)
|
||||
return false;
|
||||
|
||||
if (lo & 1u) {
|
||||
uint32_t base = lo & ~0x3u;
|
||||
if (!base) return false;
|
||||
out->Kind = RtlBarIo;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t base = (uint64_t)(lo & ~0xFu);
|
||||
uint32_t type = (lo >> 1) & 0x3u;
|
||||
if (type == 0x2u && index < 5) {
|
||||
uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u));
|
||||
base |= ((uint64_t)hi << 32);
|
||||
}
|
||||
|
||||
if (!base) return false;
|
||||
out->Kind = RtlBarMmio;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio) {
|
||||
uint64_t pageBase = physBase & PAGE_MASK;
|
||||
uint64_t pageOff = physBase - pageBase;
|
||||
|
||||
for (uint32_t i = 0; i < RTL_MMIO_PAGES; i++) {
|
||||
PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE;
|
||||
uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys);
|
||||
KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, MmMapFailIfPresent);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
*outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
|
||||
RTL_BAR best = { RtlBarNone, 0, 0 };
|
||||
RTL_BAR fallback = { RtlBarNone, 0, 0 };
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
RTL_BAR bar;
|
||||
if (!RtlReadBar(r->Bus, r->Dev, r->Fn, i, &bar))
|
||||
continue;
|
||||
|
||||
if (bar.Kind == RtlBarMmio) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Kind == RtlBarIo && fallback.Kind == RtlBarNone)
|
||||
fallback = bar;
|
||||
|
||||
// Skip the high dword of 64-bit memory BARs.
|
||||
if (bar.Kind == RtlBarMmio) {
|
||||
uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u));
|
||||
if (((lo >> 1) & 0x3u) == 0x2u)
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (best.Kind == RtlBarNone)
|
||||
best = fallback;
|
||||
|
||||
if (best.Kind == RtlBarMmio) {
|
||||
KSTATUS st = RtlMapMmio(best.Base, &r->Mmio);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL816X: BAR%u MMIO base=0x%016llX virt=0x%016llX\n",
|
||||
best.Index, (unsigned long long)best.Base, r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (best.Kind == RtlBarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL816X: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
|
||||
void RtlEnablePci(RTL816X_DEVICE* r) {
|
||||
uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND);
|
||||
cmd |= PCI_COMMAND_MASTER;
|
||||
if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY;
|
||||
else cmd |= PCI_COMMAND_IO;
|
||||
cmd &= ~PCI_COMMAND_INTX_DISABLE;
|
||||
gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd);
|
||||
}
|
||||
+246
-104
@@ -2,17 +2,26 @@
|
||||
#include "ahci.h"
|
||||
|
||||
const HAL_API *g_hal;
|
||||
HBA_MEM *g_hba;
|
||||
uint8_t g_irq;
|
||||
DRIVER_OBJECT g_driver;
|
||||
AHCI_DRIVE g_drives[MAX_AHCI_DRIVES];
|
||||
int g_driveCount;
|
||||
|
||||
static AHCI_CONTROLLER* g_controllers[MAX_AHCI_CONTROLLERS];
|
||||
static int g_controllerCount;
|
||||
static uint8_t g_nextDiskIndex;
|
||||
|
||||
static const IO_DEVICE_ID g_ahciIds[] = {
|
||||
{ IoBusPci, { "PCI\\CC_010601", sizeof("PCI\\CC_010601") - 1 } },
|
||||
};
|
||||
|
||||
static const IO_DRIVER_ID_TABLE g_ahciIdTable = {
|
||||
g_ahciIds,
|
||||
sizeof(g_ahciIds) / sizeof(g_ahciIds[0]),
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl);
|
||||
|
||||
// Accessors for port structure memory at known offsets from structVirt/Phys
|
||||
static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) {
|
||||
@@ -166,29 +175,34 @@ static void AhciCompletionDpc(KDPC *dpc, void *context,
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static void AhciIsr(void) {
|
||||
if (!g_hba) return;
|
||||
for (int c = 0; c < g_controllerCount; c++) {
|
||||
AHCI_CONTROLLER* ctl = g_controllers[c];
|
||||
if (!ctl || !ctl->hba || ctl->removed)
|
||||
continue;
|
||||
|
||||
// GHC.IS tells us which ports are pending.
|
||||
uint32_t pending = g_hba->is;
|
||||
if (!pending) return;
|
||||
// GHC.IS tells us which ports are pending for this HBA.
|
||||
uint32_t pending = ctl->hba->is;
|
||||
if (!pending)
|
||||
continue;
|
||||
|
||||
for (int i = 0; i < g_driveCount; i++) {
|
||||
AHCI_DRIVE *drv = &g_drives[i];
|
||||
uint32_t portBit = (1u << drv->portNo);
|
||||
if (!(pending & portBit)) continue;
|
||||
for (int i = 0; i < ctl->driveCount; i++) {
|
||||
AHCI_DRIVE *drv = &ctl->drives[i];
|
||||
uint32_t portBit = (1u << drv->portNo);
|
||||
if (!(pending & portBit)) continue;
|
||||
|
||||
// Capture and clear port IS before queueing the bottom half. The
|
||||
// waiter reads lastIs to check for errors without racing the next
|
||||
// command. OR in case hardware reports multiple causes before the
|
||||
// DPC releases the worker.
|
||||
uint32_t portIs = drv->port->is;
|
||||
drv->port->is = portIs; // W1C - write back to clear
|
||||
drv->lastIs |= portIs;
|
||||
// Capture and clear port IS before queueing the bottom half. The
|
||||
// waiter reads lastIs to check for errors without racing the next
|
||||
// command. OR in case hardware reports multiple causes before the
|
||||
// DPC releases the worker.
|
||||
uint32_t portIs = drv->port->is;
|
||||
drv->port->is = portIs; // W1C - write back to clear
|
||||
drv->lastIs |= portIs;
|
||||
|
||||
// Clear the HBA-level bit.
|
||||
g_hba->is = portBit;
|
||||
// Clear the HBA-level bit.
|
||||
ctl->hba->is = portBit;
|
||||
|
||||
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
|
||||
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -441,7 +455,7 @@ static void AhciCancelReleaseRemoveLock(IRP* irp, void* context) {
|
||||
IoReleaseRemoveLock(&drv->removeLock, irp);
|
||||
}
|
||||
|
||||
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp) return STATUS_INVALID_PARAMETER;
|
||||
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
|
||||
if (!drv) return STATUS_INVALID_PARAMETER;
|
||||
@@ -478,6 +492,71 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
}
|
||||
}
|
||||
|
||||
static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp) return STATUS_INVALID_PARAMETER;
|
||||
AHCI_CONTROLLER* ctl = (AHCI_CONTROLLER*)device->DeviceExtension;
|
||||
if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
if (ctl->started) {
|
||||
device->PnpState = DevicePnpStarted;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
return AhciStartController(ctl);
|
||||
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
device->PnpState = DevicePnpRemovePending;
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
device->PnpState = DevicePnpStarted;
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
ctl->surpriseRemoved = true;
|
||||
device->PnpState = DevicePnpSurpriseRemoved;
|
||||
for (int i = 0; i < ctl->driveCount; i++) {
|
||||
AHCI_DRIVE* drv = &ctl->drives[i];
|
||||
drv->surpriseRemoved = true;
|
||||
IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
|
||||
}
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
ctl->removed = true;
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
for (int i = 0; i < ctl->driveCount; i++) {
|
||||
AHCI_DRIVE* drv = &ctl->drives[i];
|
||||
drv->removed = true;
|
||||
IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
|
||||
}
|
||||
IoDetachDevice(device);
|
||||
IoDeleteDevice(device);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
default:
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !device->DeviceExtension)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uint32_t magic = *(uint32_t*)device->DeviceExtension;
|
||||
if (magic == AHCI_CONTROLLER_MAGIC)
|
||||
return AhciDispatchControllerPnp(device, irp);
|
||||
if (magic == AHCI_DRIVE_MAGIC)
|
||||
return AhciDispatchDrivePnp(device, irp);
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
}
|
||||
|
||||
// AhciWorker - kernel thread, one per drive.
|
||||
// Dequeues IRPs and executes them one at a time using the existing
|
||||
// single-slot interrupt-driven path. Completion fires the IRP's event
|
||||
@@ -548,80 +627,101 @@ static void AhciWorker(uintptr_t arg) {
|
||||
#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
|
||||
#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS)
|
||||
|
||||
KSTATUS AhciInitialize(const HAL_API *hal) {
|
||||
g_hal = hal;
|
||||
g_driveCount = 0;
|
||||
|
||||
memset(&g_driver, 0, sizeof(g_driver));
|
||||
IoInitializeDriverObject(&g_driver, SYS_SV_LITERAL("ahci"));
|
||||
g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
|
||||
g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
|
||||
g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Locate AHCI controller via PCI class scan
|
||||
// -----------------------------------------------------------------------
|
||||
uint8_t pciBus, pciDev, pciFn;
|
||||
if (!hal->Pci.FindClass(PCI_AHCI_CLASS, PCI_AHCI_SUBCLASS,
|
||||
&pciBus, &pciDev, &pciFn)) {
|
||||
DbgPrint("AHCI: no AHCI controller found\n");
|
||||
return STATUS_NOT_FOUND;
|
||||
static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
|
||||
if (!ctl) return STATUS_INVALID_PARAMETER;
|
||||
for (int i = 0; i < g_controllerCount; i++) {
|
||||
if (g_controllers[i] == ctl)
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
DbgPrint("AHCI: controller at PCI %02x:%02x.%x\n", pciBus, pciDev, pciFn);
|
||||
if (g_controllerCount >= MAX_AHCI_CONTROLLERS)
|
||||
return STATUS_NO_MEMORY;
|
||||
g_controllers[g_controllerCount++] = ctl;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Enable Bus Master + Memory Space in PCI command register
|
||||
// -----------------------------------------------------------------------
|
||||
uint32_t cmd = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_CMD_REG);
|
||||
cmd |= (PCI_CMD_BUSMASTER | PCI_CMD_MEMSPACE); cmd &= ~(1 << 10); cmd |= (1 << 2);
|
||||
hal->Pci.Write32(pciBus, pciDev, pciFn, PCI_CMD_REG, cmd);
|
||||
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
|
||||
if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC || !ctl->pdo || !ctl->fdo)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Read PCI IRQ line
|
||||
// -----------------------------------------------------------------------
|
||||
g_irq = hal->Pci.GetIrqLine(pciBus, pciDev, pciFn);
|
||||
if (g_irq == 0xFF) {
|
||||
if (ctl->started)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
KSTATUS st = PciGetDeviceInfo(ctl->pdo, &ctl->pciInfo);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: PciGetDeviceInfo failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
DbgPrint("AHCI: controller at PCI %02x:%02x.%x vendor=%04x device=%04x\n",
|
||||
ctl->pciInfo.Address.Bus,
|
||||
ctl->pciInfo.Address.Device,
|
||||
ctl->pciInfo.Address.Function,
|
||||
ctl->pciInfo.VendorId,
|
||||
ctl->pciInfo.DeviceId);
|
||||
|
||||
st = PciGetInterrupt(ctl->pdo, &ctl->irq);
|
||||
if (!KSUCCESS(st) || ctl->irq == 0xFF) {
|
||||
DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n");
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
|
||||
}
|
||||
DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)g_irq);
|
||||
DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)ctl->irq);
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Read ABAR (BAR5) and map as uncacheable
|
||||
// -----------------------------------------------------------------------
|
||||
uint32_t bar5 = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_BAR5_REG);
|
||||
uint64_t abarPhys = (uint64_t)(bar5 & ~0xFFFu);
|
||||
KSTATUS st = ahci_map_mmio(abarPhys);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
st = PciGetBar(ctl->pdo, AHCI_PCI_BAR_ABAR, &ctl->abar);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: BAR5/ABAR not present (0x%x)\n", st);
|
||||
return st;
|
||||
}
|
||||
if (ctl->abar.Type != PciBarMemory32 && ctl->abar.Type != PciBarMemory64) {
|
||||
DbgPrint("AHCI: BAR5/ABAR is not MMIO (type=%u)\n", ctl->abar.Type);
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
|
||||
g_hba = (HBA_MEM *)hal->PhysToVirt(abarPhys);
|
||||
DbgPrint("AHCI: ABAR phys=0x%016llX version=0x%08x\n",
|
||||
(unsigned long long)abarPhys, g_hba->vs);
|
||||
st = PciMapBar(ctl->pdo,
|
||||
AHCI_PCI_BAR_ABAR,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
|
||||
(void**)&ctl->hba,
|
||||
&ctl->abarLength);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: PciMapBar(BAR5) failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// HBA reset + BIOS handoff (also enables GHC.IE)
|
||||
// -----------------------------------------------------------------------
|
||||
st = hba_reset(g_hba);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
st = PciEnableDevice(ctl->pdo,
|
||||
PCI_ENABLE_MEMORY |
|
||||
PCI_ENABLE_BUSMASTER |
|
||||
PCI_ENABLE_INTX);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: PciEnableDevice failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Log capability summary
|
||||
// -----------------------------------------------------------------------
|
||||
uint32_t cap = g_hba->cap;
|
||||
DbgPrint("AHCI: ABAR phys=0x%016llX len=0x%llx version=0x%08x\n",
|
||||
(unsigned long long)ctl->abar.Base,
|
||||
(unsigned long long)ctl->abarLength,
|
||||
ctl->hba->vs);
|
||||
|
||||
st = hba_reset(ctl->hba);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
uint32_t cap = ctl->hba->cap;
|
||||
uint32_t nPorts = (cap & 0x1F) + 1;
|
||||
uint32_t nSlots = ((cap >> 8) & 0x1F) + 1;
|
||||
bool s64a = !!(cap & (1u << 31));
|
||||
DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n",
|
||||
cap, nPorts, nSlots, (int)s64a);
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Enumerate implemented ports
|
||||
// -----------------------------------------------------------------------
|
||||
uint32_t pi = g_hba->pi;
|
||||
for (uint32_t portno = 0; portno < 32 && g_driveCount < MAX_AHCI_DRIVES; portno++) {
|
||||
st = AhciRegisterController(ctl);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: controller registry full (0x%x)\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
uint32_t pi = ctl->hba->pi;
|
||||
for (uint32_t portno = 0; portno < 32 && ctl->driveCount < MAX_AHCI_DRIVES; portno++) {
|
||||
if (!(pi & (1u << portno))) continue;
|
||||
|
||||
HBA_PORT *port = &g_hba->ports[portno];
|
||||
HBA_PORT *port = &ctl->hba->ports[portno];
|
||||
|
||||
// Phase 1: physical link check
|
||||
if (!port_link_up(port)) {
|
||||
@@ -631,9 +731,11 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
|
||||
|
||||
DbgPrint("AHCI: port %u: link up, initializing...\n", portno);
|
||||
|
||||
int driveIdx = g_driveCount;
|
||||
AHCI_DRIVE *drv = &g_drives[driveIdx];
|
||||
int driveIdx = ctl->driveCount;
|
||||
AHCI_DRIVE *drv = &ctl->drives[driveIdx];
|
||||
memset(drv, 0, sizeof(*drv));
|
||||
drv->magic = AHCI_DRIVE_MAGIC;
|
||||
drv->controller = ctl;
|
||||
drv->port = port;
|
||||
drv->portNo = (uint8_t)portno;
|
||||
|
||||
@@ -689,7 +791,7 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
|
||||
}
|
||||
|
||||
DbgPrint("AHCI: drive %d: %llu sectors (%llu MB)\n",
|
||||
driveIdx,
|
||||
g_nextDiskIndex,
|
||||
(unsigned long long)drv->sectorCount,
|
||||
(unsigned long long)(drv->sectorCount / 2048));
|
||||
|
||||
@@ -701,7 +803,7 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
|
||||
// Phase 8: create and register a disk DEVICE_OBJECT.
|
||||
st = IoCreateDevice(&g_driver, 0, &drv->deviceObject);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", driveIdx, st);
|
||||
DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", g_nextDiskIndex, st);
|
||||
continue;
|
||||
}
|
||||
drv->deviceObject->DeviceExtension = drv;
|
||||
@@ -709,12 +811,14 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
|
||||
drv->deviceObject->u.Disk.SectorSize = 512;
|
||||
drv->deviceObject->AlignmentRequirement = 512;
|
||||
|
||||
st = IoRegisterDiskDevice((uint8_t)driveIdx, drv->deviceObject);
|
||||
uint8_t diskIndex = g_nextDiskIndex;
|
||||
st = IoRegisterDiskDevice(diskIndex, drv->deviceObject);
|
||||
if (KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: drive %d registered\n", driveIdx);
|
||||
g_driveCount++;
|
||||
DbgPrint("AHCI: drive %u registered\n", diskIndex);
|
||||
ctl->driveCount++;
|
||||
g_nextDiskIndex++;
|
||||
} else {
|
||||
DbgPrint("AHCI: IoRegisterDiskDevice(%d) failed (0x%x)\n", driveIdx, st);
|
||||
DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st);
|
||||
IoDeleteDevice(drv->deviceObject);
|
||||
drv->deviceObject = (DEVICE_OBJECT*)0;
|
||||
continue;
|
||||
@@ -726,33 +830,71 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
|
||||
st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv,
|
||||
PriorityHigh, &drv->workerThread);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: drive %d: worker thread create failed (0x%x)\n",
|
||||
driveIdx, st);
|
||||
DbgPrint("AHCI: drive %u: worker thread create failed (0x%x)\n",
|
||||
diskIndex, st);
|
||||
// Fatal for this device: all disk access is IRP-based now, so
|
||||
// leave the device registered but clearly fail during testing.
|
||||
}
|
||||
}
|
||||
|
||||
if (g_driveCount == 0) {
|
||||
DbgPrint("AHCI: no drives found\n");
|
||||
return STATUS_NOT_FOUND;
|
||||
if (ctl->driveCount == 0)
|
||||
DbgPrint("AHCI: no drives found on PCI %02x:%02x.%x\n",
|
||||
ctl->pciInfo.Address.Bus,
|
||||
ctl->pciInfo.Address.Device,
|
||||
ctl->pciInfo.Address.Function);
|
||||
|
||||
KeIrqRegister(ctl->irq, AhciIsr);
|
||||
ctl->started = true;
|
||||
ctl->fdo->PnpState = DevicePnpStarted;
|
||||
DbgPrint("AHCI: ISR registered on IRQ %u; controller drives=%d\n",
|
||||
(unsigned)ctl->irq, ctl->driveCount);
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
|
||||
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreateDevice(driver, sizeof(AHCI_CONTROLLER), &fdo);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
AHCI_CONTROLLER* ctl = (AHCI_CONTROLLER*)fdo->DeviceExtension;
|
||||
memset(ctl, 0, sizeof(*ctl));
|
||||
ctl->magic = AHCI_CONTROLLER_MAGIC;
|
||||
ctl->fdo = fdo;
|
||||
ctl->pdo = physicalDeviceObject;
|
||||
ctl->irq = 0xFF;
|
||||
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("ahci-controller"));
|
||||
|
||||
ctl->lowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
|
||||
if (!ctl->lowerDevice) {
|
||||
IoDeleteDevice(fdo);
|
||||
return STATUS_FAILURE;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Register the ISR. All ports on this HBA share one PCI IRQ line.
|
||||
// Do this after all drives are initialised so the ISR sees valid state.
|
||||
// -----------------------------------------------------------------------
|
||||
KeIrqRegister(g_irq, AhciIsr);
|
||||
DbgPrint("AHCI: ISR registered on IRQ %u; initialised %d drive(s)\n",
|
||||
(unsigned)g_irq, g_driveCount);
|
||||
|
||||
DbgPrint("AHCI: FDO attached to %v\n", physicalDeviceObject->Name);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API *hal) {
|
||||
DbgPrint("AHCI: driver loading...\n");
|
||||
KSTATUS st = AhciInitialize(hal);
|
||||
g_hal = hal;
|
||||
g_controllerCount = 0;
|
||||
g_nextDiskIndex = 0;
|
||||
|
||||
memset(&g_driver, 0, sizeof(g_driver));
|
||||
IoInitializeDriverObject(&g_driver, SYS_SV_LITERAL("ahci"));
|
||||
g_driver.IdTable = &g_ahciIdTable;
|
||||
g_driver.DriverAddDevice = AhciAddDevice;
|
||||
g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
|
||||
g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
|
||||
g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
|
||||
|
||||
KSTATUS st = IoRegisterDriverObject(&g_driver);
|
||||
if (!KSUCCESS(st))
|
||||
DbgPrint("AHCI: init failed: 0x%x\n", st);
|
||||
DbgPrint("AHCI: IoRegisterDriverObject failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/pci.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
@@ -223,13 +224,12 @@ typedef volatile struct {
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// PCI
|
||||
#define PCI_AHCI_CLASS 0x01
|
||||
#define PCI_AHCI_SUBCLASS 0x06
|
||||
#define PCI_CMD_REG 0x04
|
||||
#define PCI_BAR5_REG 0x24
|
||||
#define PCI_CMD_BUSMASTER (1 << 2)
|
||||
#define PCI_CMD_MEMSPACE (1 << 1)
|
||||
// PCI/AHCI binding. PCI enumeration, BAR decoding, interrupt line
|
||||
// discovery, and command-register policy are owned by pci.sys.
|
||||
#define AHCI_PCI_BAR_ABAR 5u
|
||||
#define AHCI_CONTROLLER_MAGIC 0x41484349u // 'AHCI'
|
||||
#define AHCI_DRIVE_MAGIC 0x41484452u // 'AHDR'
|
||||
#define MAX_AHCI_CONTROLLERS 8
|
||||
|
||||
// GHC register bits
|
||||
#define GHC_AE (1u << 31) // AHCI Enable
|
||||
@@ -314,8 +314,12 @@ typedef volatile struct {
|
||||
// We map 8 pages (32KB) without probing the BAR size.
|
||||
#define AHCI_MMIO_PAGES 8
|
||||
|
||||
typedef struct AHCI_CONTROLLER AHCI_CONTROLLER;
|
||||
|
||||
// Per-drive context
|
||||
typedef struct {
|
||||
uint32_t magic; // AHCI_DRIVE_MAGIC
|
||||
AHCI_CONTROLLER* controller; // owning AHCI controller FDO extension
|
||||
HBA_PORT *port; // MMIO port registers (virtual)
|
||||
uint64_t sectorCount; // Total LBA sectors on device
|
||||
uint8_t portNo; // Physical port index (for ISR dispatch)
|
||||
@@ -358,15 +362,30 @@ typedef struct {
|
||||
// volatile uint32_t prevSact; // previous port->sact snapshot
|
||||
} AHCI_DRIVE;
|
||||
|
||||
struct AHCI_CONTROLLER {
|
||||
uint32_t magic; // AHCI_CONTROLLER_MAGIC
|
||||
DEVICE_OBJECT* fdo;
|
||||
DEVICE_OBJECT* pdo;
|
||||
DEVICE_OBJECT* lowerDevice;
|
||||
|
||||
PCI_DEVICE_INFO pciInfo;
|
||||
PCI_BAR abar;
|
||||
HBA_MEM* hba;
|
||||
uint64_t abarLength;
|
||||
uint8_t irq;
|
||||
|
||||
AHCI_DRIVE drives[MAX_AHCI_DRIVES];
|
||||
int driveCount;
|
||||
|
||||
bool started;
|
||||
bool surpriseRemoved;
|
||||
bool removed;
|
||||
};
|
||||
|
||||
extern const HAL_API *g_hal;
|
||||
extern HBA_MEM *g_hba;
|
||||
extern uint8_t g_irq;
|
||||
extern DRIVER_OBJECT g_driver;
|
||||
extern AHCI_DRIVE g_drives[MAX_AHCI_DRIVES];
|
||||
extern int g_driveCount;
|
||||
|
||||
KSTATUS ahci_map_mmio(uint64_t physBase);
|
||||
KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length);
|
||||
void port_stop(HBA_PORT *port);
|
||||
void port_start(HBA_PORT *port);
|
||||
KSTATUS hba_reset(HBA_MEM *hba);
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
// ahci_port.c - AHCI MMIO mapping, HBA reset, and port link bring-up helpers.
|
||||
#include "ahci.h"
|
||||
|
||||
KSTATUS ahci_map_mmio(uint64_t physBase) {
|
||||
for (int i = 0; i < AHCI_MMIO_PAGES; i++) {
|
||||
uint64_t phys = physBase + (uint64_t)i * 4096;
|
||||
KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
|
||||
if (length == 0)
|
||||
length = AHCI_MMIO_PAGES * 4096ull;
|
||||
uint64_t pageBase = physBase & ~(uint64_t)(PAGE_SIZE - 1u);
|
||||
uint64_t pageEnd = (physBase + length + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
|
||||
int pageIndex = 0;
|
||||
for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) {
|
||||
uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys);
|
||||
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
|
||||
KSTATUS st = MmMapPage(virt, phys, flags, MmMapFailIfPresent);
|
||||
@@ -15,7 +19,7 @@ KSTATUS ahci_map_mmio(uint64_t physBase) {
|
||||
}
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
|
||||
i, (unsigned long long)phys, st);
|
||||
pageIndex, (unsigned long long)phys, st);
|
||||
return st;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,686 @@
|
||||
// ata.c - PCI IDE / ATA disk driver.
|
||||
#include "ata.h"
|
||||
|
||||
const HAL_API* gAtaHal;
|
||||
static DRIVER_OBJECT gAtaDriver;
|
||||
static ATA_CONTROLLER* gAtaControllers[ATA_MAX_CONTROLLERS];
|
||||
static uint32_t gAtaControllerCount;
|
||||
|
||||
static const IO_DEVICE_ID gAtaIds[] = {
|
||||
{ IoBusPci, { "PCI\\CC_0101", sizeof("PCI\\CC_0101") - 1 } },
|
||||
};
|
||||
|
||||
static const IO_DRIVER_ID_TABLE gAtaIdTable = {
|
||||
gAtaIds,
|
||||
sizeof(gAtaIds) / sizeof(gAtaIds[0]),
|
||||
};
|
||||
|
||||
static uint8_t AtaIn8(ATA_CHANNEL* ch, uint8_t reg) {
|
||||
return __inbyte((unsigned short)(ch->CommandBase + reg));
|
||||
}
|
||||
|
||||
static uint16_t AtaIn16(ATA_CHANNEL* ch, uint8_t reg) {
|
||||
return __inword((unsigned short)(ch->CommandBase + reg));
|
||||
}
|
||||
|
||||
static void AtaOut8(ATA_CHANNEL* ch, uint8_t reg, uint8_t value) {
|
||||
__outbyte((unsigned short)(ch->CommandBase + reg), value);
|
||||
}
|
||||
|
||||
static void AtaOut16(ATA_CHANNEL* ch, uint8_t reg, uint16_t value) {
|
||||
__outword((unsigned short)(ch->CommandBase + reg), value);
|
||||
}
|
||||
|
||||
static uint8_t AtaAltStatus(ATA_CHANNEL* ch) {
|
||||
return __inbyte((unsigned short)(ch->ControlBase + ATA_REG_ALT_STATUS));
|
||||
}
|
||||
|
||||
static void AtaDeviceControl(ATA_CHANNEL* ch, uint8_t value) {
|
||||
__outbyte((unsigned short)(ch->ControlBase + ATA_REG_DEVICE_CONTROL), value);
|
||||
}
|
||||
|
||||
static void AtaDelay400ns(ATA_CHANNEL* ch) {
|
||||
(void)AtaAltStatus(ch);
|
||||
(void)AtaAltStatus(ch);
|
||||
(void)AtaAltStatus(ch);
|
||||
(void)AtaAltStatus(ch);
|
||||
}
|
||||
|
||||
static KSTATUS AtaWaitNotBusy(ATA_CHANNEL* ch) {
|
||||
for (uint32_t i = 0; i < ATA_POLL_LIMIT; i++) {
|
||||
if (!(AtaIn8(ch, ATA_REG_STATUS) & ATA_STATUS_BSY))
|
||||
return STATUS_SUCCESS;
|
||||
KeYield();
|
||||
}
|
||||
return STATUS_TIMEOUT;
|
||||
}
|
||||
|
||||
static KSTATUS AtaWaitDrq(ATA_CHANNEL* ch) {
|
||||
for (uint32_t i = 0; i < ATA_POLL_LIMIT; i++) {
|
||||
uint8_t st = AtaIn8(ch, ATA_REG_STATUS);
|
||||
if (st & ATA_STATUS_ERR)
|
||||
return STATUS_DEVICE_ERROR;
|
||||
if (st & ATA_STATUS_DF)
|
||||
return STATUS_DEVICE_ERROR;
|
||||
if (!(st & ATA_STATUS_BSY) && (st & ATA_STATUS_DRQ))
|
||||
return STATUS_SUCCESS;
|
||||
KeYield();
|
||||
}
|
||||
return STATUS_TIMEOUT;
|
||||
}
|
||||
|
||||
static void AtaSelectDrive(ATA_DEVICE* dev) {
|
||||
ATA_CHANNEL* ch = dev->Channel;
|
||||
AtaOut8(ch, ATA_REG_DRIVE_HEAD, (uint8_t)(0xA0u | (dev->Drive ? 0x10u : 0u)));
|
||||
AtaDelay400ns(ch);
|
||||
}
|
||||
|
||||
static KSTATUS AtaIdentify(ATA_DEVICE* dev) {
|
||||
ATA_CHANNEL* ch = dev->Channel;
|
||||
uint16_t ident[256];
|
||||
memset(ident, 0, sizeof(ident));
|
||||
|
||||
KSTATUS st = KeWaitSemaphore(ch->Lock);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
AtaSelectDrive(dev);
|
||||
AtaOut8(ch, ATA_REG_SECTOR_COUNT, 0);
|
||||
AtaOut8(ch, ATA_REG_LBA0, 0);
|
||||
AtaOut8(ch, ATA_REG_LBA1, 0);
|
||||
AtaOut8(ch, ATA_REG_LBA2, 0);
|
||||
AtaOut8(ch, ATA_REG_COMMAND, ATA_CMD_IDENTIFY_DEVICE);
|
||||
AtaDelay400ns(ch);
|
||||
|
||||
uint8_t status = AtaIn8(ch, ATA_REG_STATUS);
|
||||
if (status == 0) {
|
||||
KeSignalSemaphore(ch->Lock);
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
st = AtaWaitNotBusy(ch);
|
||||
if (!KSUCCESS(st)) {
|
||||
KeSignalSemaphore(ch->Lock);
|
||||
return st;
|
||||
}
|
||||
|
||||
if (AtaIn8(ch, ATA_REG_LBA1) != 0 || AtaIn8(ch, ATA_REG_LBA2) != 0) {
|
||||
KeSignalSemaphore(ch->Lock);
|
||||
return STATUS_NOT_SUPPORTED; // ATAPI or non-ATA signature.
|
||||
}
|
||||
|
||||
st = AtaWaitDrq(ch);
|
||||
if (!KSUCCESS(st)) {
|
||||
KeSignalSemaphore(ch->Lock);
|
||||
return st;
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < 256; i++)
|
||||
ident[i] = AtaIn16(ch, ATA_REG_DATA);
|
||||
|
||||
KeSignalSemaphore(ch->Lock);
|
||||
|
||||
if (!(ident[ATA_IDENT_CAPABILITIES] & ATA_IDENT_LBA_SUPPORTED))
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
|
||||
dev->SupportsLba48 = (ident[ATA_IDENT_COMMAND_SETS] & ATA_IDENT_LBA48_SUPPORTED) != 0;
|
||||
if (dev->SupportsLba48) {
|
||||
uint64_t sectors = 0;
|
||||
sectors |= (uint64_t)ident[100];
|
||||
sectors |= (uint64_t)ident[101] << 16;
|
||||
sectors |= (uint64_t)ident[102] << 32;
|
||||
sectors |= (uint64_t)ident[103] << 48;
|
||||
dev->SectorCount = sectors;
|
||||
} else {
|
||||
dev->SectorCount = ((uint32_t)ident[ATA_IDENT_LBA28_SECTORS_HI] << 16) |
|
||||
ident[ATA_IDENT_LBA28_SECTORS_LO];
|
||||
}
|
||||
|
||||
return dev->SectorCount ? STATUS_SUCCESS : STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
static KSTATUS AtaProgramLba(ATA_DEVICE* dev, uint64_t lba, uint16_t count, bool write) {
|
||||
ATA_CHANNEL* ch = dev->Channel;
|
||||
|
||||
if (dev->SupportsLba48) {
|
||||
AtaOut8(ch, ATA_REG_DRIVE_HEAD, (uint8_t)(0x40u | (dev->Drive ? 0x10u : 0u)));
|
||||
AtaOut8(ch, ATA_REG_SECTOR_COUNT, (uint8_t)(count >> 8));
|
||||
AtaOut8(ch, ATA_REG_LBA0, (uint8_t)(lba >> 24));
|
||||
AtaOut8(ch, ATA_REG_LBA1, (uint8_t)(lba >> 32));
|
||||
AtaOut8(ch, ATA_REG_LBA2, (uint8_t)(lba >> 40));
|
||||
AtaOut8(ch, ATA_REG_SECTOR_COUNT, (uint8_t)count);
|
||||
AtaOut8(ch, ATA_REG_LBA0, (uint8_t)lba);
|
||||
AtaOut8(ch, ATA_REG_LBA1, (uint8_t)(lba >> 8));
|
||||
AtaOut8(ch, ATA_REG_LBA2, (uint8_t)(lba >> 16));
|
||||
AtaOut8(ch, ATA_REG_COMMAND, write ? ATA_CMD_WRITE_SECTORS_EXT : ATA_CMD_READ_SECTORS_EXT);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (lba > 0x0FFFFFFFULL || count > 256)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
AtaOut8(ch, ATA_REG_DRIVE_HEAD, (uint8_t)(0xE0u | (dev->Drive ? 0x10u : 0u) | ((lba >> 24) & 0x0Fu)));
|
||||
AtaOut8(ch, ATA_REG_SECTOR_COUNT, (uint8_t)count);
|
||||
AtaOut8(ch, ATA_REG_LBA0, (uint8_t)lba);
|
||||
AtaOut8(ch, ATA_REG_LBA1, (uint8_t)(lba >> 8));
|
||||
AtaOut8(ch, ATA_REG_LBA2, (uint8_t)(lba >> 16));
|
||||
AtaOut8(ch, ATA_REG_COMMAND, write ? ATA_CMD_WRITE_SECTORS : ATA_CMD_READ_SECTORS);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS AtaReadPio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, void* buffer) {
|
||||
if (!dev || !buffer)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (count == 0)
|
||||
return STATUS_SUCCESS;
|
||||
if (lba + count > dev->SectorCount)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
ATA_CHANNEL* ch = dev->Channel;
|
||||
uint16_t* out = (uint16_t*)buffer;
|
||||
|
||||
KSTATUS st = KeWaitSemaphore(ch->Lock);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
while (count > 0) {
|
||||
uint16_t batch = (count > 256u) ? 256u : (uint16_t)count;
|
||||
uint16_t encodedCount = (!dev->SupportsLba48 && batch == 256u) ? 0u : batch;
|
||||
|
||||
AtaSelectDrive(dev);
|
||||
st = AtaWaitNotBusy(ch);
|
||||
if (!KSUCCESS(st)) break;
|
||||
|
||||
st = AtaProgramLba(dev, lba, encodedCount, false);
|
||||
if (!KSUCCESS(st)) break;
|
||||
|
||||
for (uint32_t s = 0; s < batch; s++) {
|
||||
st = AtaWaitDrq(ch);
|
||||
if (!KSUCCESS(st)) break;
|
||||
for (uint32_t w = 0; w < ATA_SECTOR_SIZE / 2u; w++)
|
||||
*out++ = AtaIn16(ch, ATA_REG_DATA);
|
||||
}
|
||||
if (!KSUCCESS(st)) break;
|
||||
|
||||
lba += batch;
|
||||
count -= batch;
|
||||
}
|
||||
|
||||
KeSignalSemaphore(ch->Lock);
|
||||
return st;
|
||||
}
|
||||
|
||||
static KSTATUS AtaFlush(ATA_DEVICE* dev) {
|
||||
ATA_CHANNEL* ch = dev->Channel;
|
||||
KSTATUS st = AtaWaitNotBusy(ch);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
AtaOut8(ch, ATA_REG_COMMAND, dev->SupportsLba48 ? ATA_CMD_CACHE_FLUSH_EXT : ATA_CMD_CACHE_FLUSH);
|
||||
return AtaWaitNotBusy(ch);
|
||||
}
|
||||
|
||||
static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const void* buffer) {
|
||||
if (!dev || !buffer)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (count == 0)
|
||||
return STATUS_SUCCESS;
|
||||
if (lba + count > dev->SectorCount)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
ATA_CHANNEL* ch = dev->Channel;
|
||||
const uint16_t* in = (const uint16_t*)buffer;
|
||||
|
||||
KSTATUS st = KeWaitSemaphore(ch->Lock);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
while (count > 0) {
|
||||
uint16_t batch = (count > 256u) ? 256u : (uint16_t)count;
|
||||
uint16_t encodedCount = (!dev->SupportsLba48 && batch == 256u) ? 0u : batch;
|
||||
|
||||
AtaSelectDrive(dev);
|
||||
st = AtaWaitNotBusy(ch);
|
||||
if (!KSUCCESS(st)) break;
|
||||
|
||||
st = AtaProgramLba(dev, lba, encodedCount, true);
|
||||
if (!KSUCCESS(st)) break;
|
||||
|
||||
for (uint32_t s = 0; s < batch; s++) {
|
||||
st = AtaWaitDrq(ch);
|
||||
if (!KSUCCESS(st)) break;
|
||||
for (uint32_t w = 0; w < ATA_SECTOR_SIZE / 2u; w++)
|
||||
AtaOut16(ch, ATA_REG_DATA, *in++);
|
||||
}
|
||||
if (!KSUCCESS(st)) break;
|
||||
|
||||
st = AtaFlush(dev);
|
||||
if (!KSUCCESS(st)) break;
|
||||
|
||||
lba += batch;
|
||||
count -= batch;
|
||||
}
|
||||
|
||||
KeSignalSemaphore(ch->Lock);
|
||||
return st;
|
||||
}
|
||||
|
||||
static KSTATUS AtaGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
|
||||
uint32_t* outSectors, void** outBuffer) {
|
||||
if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios) return STATUS_INVALID_PARAMETER;
|
||||
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
|
||||
(ios->Parameters.ReadWrite.Length & 511u))
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
*outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
|
||||
*outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
|
||||
*outSectors = ios->Parameters.ReadWrite.Length >> 9;
|
||||
*outBuffer = irp->UserBuffer;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static void AtaWorker(uintptr_t arg) {
|
||||
ATA_DEVICE* dev = (ATA_DEVICE*)arg;
|
||||
if (!dev)
|
||||
return;
|
||||
|
||||
while (true) {
|
||||
KeWaitSemaphore(dev->IrpQueue.Semaphore);
|
||||
|
||||
IRP* irp = IrpQueueDequeue(&dev->IrpQueue);
|
||||
if (!irp)
|
||||
continue;
|
||||
|
||||
bool write = false;
|
||||
uint64_t lba = 0;
|
||||
uint32_t sectors = 0;
|
||||
void* buffer = (void*)0;
|
||||
KSTATUS st = AtaGetIrpReadWrite(irp, &write, &lba, §ors, &buffer);
|
||||
if (KSUCCESS(st)) {
|
||||
if (lba + sectors > dev->SectorCount)
|
||||
st = STATUS_INVALID_PARAMETER;
|
||||
else
|
||||
st = write ? AtaWritePio(dev, lba, sectors, buffer)
|
||||
: AtaReadPio(dev, lba, sectors, buffer);
|
||||
}
|
||||
|
||||
IoReleaseRemoveLock(&dev->RemoveLock, irp);
|
||||
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * ATA_SECTOR_SIZE) : 0);
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp || !device->DeviceExtension)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
ATA_DEVICE* dev = (ATA_DEVICE*)device->DeviceExtension;
|
||||
if (dev->Magic != ATA_DEVICE_MAGIC || !dev->Present)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
KSTATUS st = IoAcquireRemoveLock(&dev->RemoveLock, irp);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
bool write = false;
|
||||
uint64_t lba = 0;
|
||||
uint32_t sectors = 0;
|
||||
void* buffer = (void*)0;
|
||||
st = AtaGetIrpReadWrite(irp, &write, &lba, §ors, &buffer);
|
||||
(void)write;
|
||||
(void)buffer;
|
||||
if (!KSUCCESS(st) || lba + sectors > dev->SectorCount) {
|
||||
IoReleaseRemoveLock(&dev->RemoveLock, irp);
|
||||
return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
|
||||
}
|
||||
|
||||
irp->Status = STATUS_FAILURE;
|
||||
irp->Information = 0;
|
||||
IrpQueueEnqueue(&dev->IrpQueue, irp);
|
||||
return STATUS_PENDING;
|
||||
}
|
||||
|
||||
static uint8_t AtaFindFreeDiskIndex(void) {
|
||||
for (uint8_t i = 0; i < 8; i++) {
|
||||
if (!IoDiskDevicePresent(i))
|
||||
return i;
|
||||
}
|
||||
return 0xFFu;
|
||||
}
|
||||
|
||||
static uint16_t AtaBarIoBase(ATA_CONTROLLER* ctl, uint8_t barIndex, uint16_t fallback) {
|
||||
PCI_BAR bar;
|
||||
KSTATUS st = PciGetBar(ctl->Pdo, barIndex, &bar);
|
||||
if (KSUCCESS(st) && bar.Type == PciBarIo && bar.Base && bar.Base <= 0xFFFFu)
|
||||
return (uint16_t)bar.Base;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
static KSTATUS AtaInitChannel(ATA_CONTROLLER* ctl, uint8_t index) {
|
||||
ATA_CHANNEL* ch = &ctl->Channels[index];
|
||||
memset(ch, 0, sizeof(*ch));
|
||||
ch->Controller = ctl;
|
||||
ch->Index = index;
|
||||
|
||||
if (index == 0) {
|
||||
ch->CommandBase = AtaBarIoBase(ctl, 0, ATA_PRIMARY_LEGACY_CMD_BASE);
|
||||
ch->ControlBase = AtaBarIoBase(ctl, 1, ATA_PRIMARY_LEGACY_CTL_BASE);
|
||||
ch->Irq = ATA_PRIMARY_LEGACY_IRQ;
|
||||
} else {
|
||||
ch->CommandBase = AtaBarIoBase(ctl, 2, ATA_SECONDARY_LEGACY_CMD_BASE);
|
||||
ch->ControlBase = AtaBarIoBase(ctl, 3, ATA_SECONDARY_LEGACY_CTL_BASE);
|
||||
ch->Irq = ATA_SECONDARY_LEGACY_IRQ;
|
||||
}
|
||||
|
||||
if (ctl->PciInfo.ProgIf & (index == 0 ? 0x01u : 0x04u)) {
|
||||
uint8_t irq = 0xFFu;
|
||||
if (KSUCCESS(PciGetInterrupt(ctl->Pdo, &irq)) && irq != 0xFFu)
|
||||
ch->Irq = irq;
|
||||
ch->NativeMode = true;
|
||||
}
|
||||
|
||||
KSTATUS st = KeSemaphoreCreate(1, 1, &ch->Lock);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
AtaDeviceControl(ch, 0x02u); // nIEN: polled I/O for now.
|
||||
DbgPrint("ATA: channel %u cmd=0x%04x ctl=0x%04x irq=%u mode=%s\n",
|
||||
index,
|
||||
ch->CommandBase,
|
||||
ch->ControlBase,
|
||||
ch->Irq,
|
||||
ch->NativeMode ? "native" : "compat");
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS AtaRegisterDisk(ATA_DEVICE* dev) {
|
||||
DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreateDevice(&gAtaDriver, 0, &disk);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
disk->DeviceExtension = dev;
|
||||
disk->u.Disk.SectorCount = dev->SectorCount;
|
||||
disk->u.Disk.SectorSize = ATA_SECTOR_SIZE;
|
||||
disk->AlignmentRequirement = ATA_SECTOR_SIZE;
|
||||
(void)IoSetDeviceName(disk, SYS_SV_LITERAL("ata-disk"));
|
||||
|
||||
uint8_t index = AtaFindFreeDiskIndex();
|
||||
if (index == 0xFFu) {
|
||||
IoDeleteDevice(disk);
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
st = IoRegisterDiskDevice(index, disk);
|
||||
if (!KSUCCESS(st)) {
|
||||
IoDeleteDevice(disk);
|
||||
return st;
|
||||
}
|
||||
|
||||
dev->DeviceObject = disk;
|
||||
dev->DiskIndex = index;
|
||||
DbgPrint("ATA: drive %u registered channel=%u %s sectors=%llu (%llu MB)\n",
|
||||
index,
|
||||
dev->Channel->Index,
|
||||
dev->Drive ? "slave" : "master",
|
||||
(unsigned long long)dev->SectorCount,
|
||||
(unsigned long long)(dev->SectorCount / 2048));
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS AtaProbeDrive(ATA_CONTROLLER* ctl, ATA_CHANNEL* ch, uint8_t drive) {
|
||||
ATA_DEVICE* dev = &ch->Devices[drive];
|
||||
memset(dev, 0, sizeof(*dev));
|
||||
dev->Magic = ATA_DEVICE_MAGIC;
|
||||
dev->Controller = ctl;
|
||||
dev->Channel = ch;
|
||||
dev->Drive = drive;
|
||||
dev->DiskIndex = 0xFFu;
|
||||
|
||||
KSTATUS st = AtaIdentify(dev);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
st = IoInitializeRemoveLock(&dev->RemoveLock);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
SEMAPHORE* sem = (SEMAPHORE*)0;
|
||||
st = KeSemaphoreCreate(0, 0x7FFFFFFF, &sem);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
IrpQueueInit(&dev->IrpQueue, sem);
|
||||
|
||||
dev->Present = true;
|
||||
st = AtaRegisterDisk(dev);
|
||||
if (!KSUCCESS(st)) {
|
||||
dev->Present = false;
|
||||
return st;
|
||||
}
|
||||
|
||||
st = KeCreateKernelThread((uintptr_t)AtaWorker, (uintptr_t)dev,
|
||||
PriorityHigh, &dev->WorkerThread);
|
||||
if (!KSUCCESS(st))
|
||||
DbgPrint("ATA: drive %u worker thread create failed (0x%x)\n", dev->DiskIndex, st);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS AtaRegisterController(ATA_CONTROLLER* ctl) {
|
||||
if (!ctl) return STATUS_INVALID_PARAMETER;
|
||||
for (uint32_t i = 0; i < gAtaControllerCount; i++) {
|
||||
if (gAtaControllers[i] == ctl)
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
if (gAtaControllerCount >= ATA_MAX_CONTROLLERS)
|
||||
return STATUS_NO_MEMORY;
|
||||
gAtaControllers[gAtaControllerCount++] = ctl;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS AtaStartController(ATA_CONTROLLER* ctl) {
|
||||
if (!ctl || ctl->Magic != ATA_CONTROLLER_MAGIC || !ctl->Pdo || !ctl->Fdo)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (ctl->Started)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
KSTATUS st = PciGetDeviceInfo(ctl->Pdo, &ctl->PciInfo);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("ATA: PciGetDeviceInfo failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
DbgPrint("ATA: controller at PCI %02x:%02x.%x vendor=%04x device=%04x progif=%02x\n",
|
||||
ctl->PciInfo.Address.Bus,
|
||||
ctl->PciInfo.Address.Device,
|
||||
ctl->PciInfo.Address.Function,
|
||||
ctl->PciInfo.VendorId,
|
||||
ctl->PciInfo.DeviceId,
|
||||
ctl->PciInfo.ProgIf);
|
||||
|
||||
st = PciEnableDevice(ctl->Pdo, PCI_ENABLE_IO | PCI_ENABLE_INTX);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("ATA: PciEnableDevice failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
st = AtaRegisterController(ctl);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
for (uint8_t chIndex = 0; chIndex < ATA_CHANNEL_COUNT; chIndex++) {
|
||||
ATA_CHANNEL* ch = &ctl->Channels[chIndex];
|
||||
st = AtaInitChannel(ctl, chIndex);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("ATA: channel %u init failed (0x%x)\n", chIndex, st);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (uint8_t drive = 0; drive < ATA_DRIVES_PER_CHANNEL; drive++) {
|
||||
st = AtaProbeDrive(ctl, ch, drive);
|
||||
if (KSUCCESS(st))
|
||||
ctl->DeviceCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (ctl->DeviceCount == 0)
|
||||
DbgPrint("ATA: no ATA disks found on PCI %02x:%02x.%x\n",
|
||||
ctl->PciInfo.Address.Bus,
|
||||
ctl->PciInfo.Address.Device,
|
||||
ctl->PciInfo.Address.Function);
|
||||
|
||||
ctl->Started = true;
|
||||
ctl->Fdo->PnpState = DevicePnpStarted;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static void AtaCancelReleaseRemoveLock(IRP* irp, void* context) {
|
||||
ATA_DEVICE* dev = (ATA_DEVICE*)context;
|
||||
if (dev)
|
||||
IoReleaseRemoveLock(&dev->RemoveLock, irp);
|
||||
}
|
||||
|
||||
static void AtaRemoveControllerDevices(ATA_CONTROLLER* ctl, IO_DEVICE_CHANGE change) {
|
||||
if (!ctl) return;
|
||||
for (uint8_t c = 0; c < ATA_CHANNEL_COUNT; c++) {
|
||||
ATA_CHANNEL* ch = &ctl->Channels[c];
|
||||
for (uint8_t d = 0; d < ATA_DRIVES_PER_CHANNEL; d++) {
|
||||
ATA_DEVICE* dev = &ch->Devices[d];
|
||||
if (!dev->Present)
|
||||
continue;
|
||||
|
||||
IrpQueueCancelAllEx(&dev->IrpQueue, STATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
|
||||
if (dev->DiskIndex != 0xFFu) {
|
||||
if (change == IoDeviceSurpriseRemoved)
|
||||
IoNotifyDiskSurpriseRemoved(dev->DiskIndex);
|
||||
else
|
||||
IoUnregisterDiskDevice(dev->DiskIndex);
|
||||
dev->DiskIndex = 0xFFu;
|
||||
}
|
||||
dev->Present = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp || !device->DeviceExtension)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
ATA_CONTROLLER* ctl = (ATA_CONTROLLER*)device->DeviceExtension;
|
||||
if (ctl->Magic == ATA_CONTROLLER_MAGIC) {
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
return AtaStartController(ctl);
|
||||
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
ctl->SurpriseRemoved = true;
|
||||
device->PnpState = DevicePnpSurpriseRemoved;
|
||||
AtaRemoveControllerDevices(ctl, IoDeviceSurpriseRemoved);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
ctl->Removed = true;
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
AtaRemoveControllerDevices(ctl, IoDeviceRemoved);
|
||||
IoDetachDevice(device);
|
||||
IoDeleteDevice(device);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
default:
|
||||
return IoForwardIrpToLower(device, irp);
|
||||
}
|
||||
}
|
||||
|
||||
ATA_DEVICE* dev = (ATA_DEVICE*)device->DeviceExtension;
|
||||
if (dev->Magic == ATA_DEVICE_MAGIC) {
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
device->PnpState = DevicePnpStarted;
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
device->PnpState = DevicePnpSurpriseRemoved;
|
||||
dev->Present = false;
|
||||
IrpQueueCancelAllEx(&dev->IrpQueue, STATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
dev->Present = false;
|
||||
IrpQueueCancelAllEx(&dev->IrpQueue, STATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
|
||||
IoReleaseRemoveLockAndWait(&dev->RemoveLock, irp);
|
||||
IoDeleteDevice(device);
|
||||
dev->DeviceObject = (DEVICE_OBJECT*)0;
|
||||
return STATUS_SUCCESS;
|
||||
default:
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
}
|
||||
|
||||
KSTATUS AtaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
|
||||
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreateDevice(driver, sizeof(ATA_CONTROLLER), &fdo);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("ata-controller"));
|
||||
|
||||
ATA_CONTROLLER* ctl = (ATA_CONTROLLER*)fdo->DeviceExtension;
|
||||
memset(ctl, 0, sizeof(*ctl));
|
||||
ctl->Magic = ATA_CONTROLLER_MAGIC;
|
||||
ctl->Fdo = fdo;
|
||||
ctl->Pdo = physicalDeviceObject;
|
||||
|
||||
ctl->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
|
||||
if (!ctl->LowerDevice) {
|
||||
IoDeleteDevice(fdo);
|
||||
return STATUS_FAILURE;
|
||||
}
|
||||
|
||||
DbgPrint("ATA: FDO attached to %v\n", physicalDeviceObject->Name);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
if (!hal)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
DbgPrint("ATA: driver loading...\n");
|
||||
gAtaHal = hal;
|
||||
gAtaControllerCount = 0;
|
||||
|
||||
memset(&gAtaDriver, 0, sizeof(gAtaDriver));
|
||||
IoInitializeDriverObject(&gAtaDriver, SYS_SV_LITERAL("ata"));
|
||||
gAtaDriver.IdTable = &gAtaIdTable;
|
||||
gAtaDriver.DriverAddDevice = AtaAddDevice;
|
||||
gAtaDriver.MajorFunction[IRP_MJ_READ] = AtaDispatchReadWrite;
|
||||
gAtaDriver.MajorFunction[IRP_MJ_WRITE] = AtaDispatchReadWrite;
|
||||
gAtaDriver.MajorFunction[IRP_MJ_PNP] = AtaDispatchPnp;
|
||||
|
||||
KSTATUS st = IoRegisterDriverObject(&gAtaDriver);
|
||||
if (!KSUCCESS(st))
|
||||
DbgPrint("ATA: IoRegisterDriverObject failed: 0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// ata.h - PCI IDE / ATA driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/pci.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
|
||||
#define ATA_CONTROLLER_MAGIC 0x41544143u // ATAC
|
||||
#define ATA_DEVICE_MAGIC 0x41544144u // ATAD
|
||||
|
||||
#define ATA_MAX_CONTROLLERS 8u
|
||||
#define ATA_CHANNEL_COUNT 2u
|
||||
#define ATA_DRIVES_PER_CHANNEL 2u
|
||||
#define ATA_MAX_DEVICES_PER_CTL (ATA_CHANNEL_COUNT * ATA_DRIVES_PER_CHANNEL)
|
||||
|
||||
#define ATA_PRIMARY_LEGACY_CMD_BASE 0x1F0u
|
||||
#define ATA_PRIMARY_LEGACY_CTL_BASE 0x3F6u
|
||||
#define ATA_PRIMARY_LEGACY_IRQ 14u
|
||||
#define ATA_SECONDARY_LEGACY_CMD_BASE 0x170u
|
||||
#define ATA_SECONDARY_LEGACY_CTL_BASE 0x376u
|
||||
#define ATA_SECONDARY_LEGACY_IRQ 15u
|
||||
|
||||
#define ATA_REG_DATA 0x00u
|
||||
#define ATA_REG_ERROR 0x01u
|
||||
#define ATA_REG_FEATURES 0x01u
|
||||
#define ATA_REG_SECTOR_COUNT 0x02u
|
||||
#define ATA_REG_LBA0 0x03u
|
||||
#define ATA_REG_LBA1 0x04u
|
||||
#define ATA_REG_LBA2 0x05u
|
||||
#define ATA_REG_DRIVE_HEAD 0x06u
|
||||
#define ATA_REG_STATUS 0x07u
|
||||
#define ATA_REG_COMMAND 0x07u
|
||||
#define ATA_REG_ALT_STATUS 0x00u
|
||||
#define ATA_REG_DEVICE_CONTROL 0x00u
|
||||
|
||||
#define ATA_STATUS_ERR 0x01u
|
||||
#define ATA_STATUS_DRQ 0x08u
|
||||
#define ATA_STATUS_DF 0x20u
|
||||
#define ATA_STATUS_DRDY 0x40u
|
||||
#define ATA_STATUS_BSY 0x80u
|
||||
|
||||
#define ATA_CMD_READ_SECTORS 0x20u
|
||||
#define ATA_CMD_WRITE_SECTORS 0x30u
|
||||
#define ATA_CMD_CACHE_FLUSH 0xE7u
|
||||
#define ATA_CMD_IDENTIFY_DEVICE 0xECu
|
||||
#define ATA_CMD_READ_SECTORS_EXT 0x24u
|
||||
#define ATA_CMD_WRITE_SECTORS_EXT 0x34u
|
||||
#define ATA_CMD_CACHE_FLUSH_EXT 0xEAu
|
||||
|
||||
#define ATA_IDENT_CAPABILITIES 49u
|
||||
#define ATA_IDENT_COMMAND_SETS 83u
|
||||
#define ATA_IDENT_LBA28_SECTORS_LO 60u
|
||||
#define ATA_IDENT_LBA28_SECTORS_HI 61u
|
||||
#define ATA_IDENT_LBA48_SECTORS 100u
|
||||
#define ATA_IDENT_LBA_SUPPORTED (1u << 9)
|
||||
#define ATA_IDENT_LBA48_SUPPORTED (1u << 10)
|
||||
|
||||
#define ATA_SECTOR_SIZE 512u
|
||||
#define ATA_POLL_LIMIT 100000u
|
||||
|
||||
struct ATA_CONTROLLER;
|
||||
struct ATA_CHANNEL;
|
||||
|
||||
typedef struct ATA_DEVICE {
|
||||
uint32_t Magic;
|
||||
struct ATA_CONTROLLER* Controller;
|
||||
struct ATA_CHANNEL* Channel;
|
||||
|
||||
DEVICE_OBJECT* DeviceObject;
|
||||
uint8_t Drive; // 0=master, 1=slave
|
||||
uint8_t DiskIndex;
|
||||
|
||||
bool Present;
|
||||
bool SupportsLba48;
|
||||
uint64_t SectorCount;
|
||||
|
||||
IRP_QUEUE IrpQueue;
|
||||
SEMAPHORE* IrpSemaphore;
|
||||
IO_REMOVE_LOCK RemoveLock;
|
||||
THREAD* WorkerThread;
|
||||
} ATA_DEVICE;
|
||||
|
||||
typedef struct ATA_CHANNEL {
|
||||
struct ATA_CONTROLLER* Controller;
|
||||
uint8_t Index;
|
||||
|
||||
uint16_t CommandBase;
|
||||
uint16_t ControlBase;
|
||||
uint8_t Irq;
|
||||
bool NativeMode;
|
||||
|
||||
SEMAPHORE* Lock;
|
||||
ATA_DEVICE Devices[ATA_DRIVES_PER_CHANNEL];
|
||||
} ATA_CHANNEL;
|
||||
|
||||
typedef struct ATA_CONTROLLER {
|
||||
uint32_t Magic;
|
||||
|
||||
DEVICE_OBJECT* Fdo;
|
||||
DEVICE_OBJECT* Pdo;
|
||||
DEVICE_OBJECT* LowerDevice;
|
||||
|
||||
PCI_DEVICE_INFO PciInfo;
|
||||
bool Started;
|
||||
bool Removed;
|
||||
bool SurpriseRemoved;
|
||||
|
||||
ATA_CHANNEL Channels[ATA_CHANNEL_COUNT];
|
||||
uint8_t DeviceCount;
|
||||
} ATA_CONTROLLER;
|
||||
|
||||
extern const HAL_API* gAtaHal;
|
||||
|
||||
KSTATUS AtaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
|
||||
KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
|
||||
KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp);
|
||||
@@ -1,531 +0,0 @@
|
||||
// ide.c - IDE/ATA disk I/O path and DriverEntry.
|
||||
#include "ide.h"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Module state
|
||||
// ---------------------------------------------------------------------------
|
||||
const HAL_API *gHal;
|
||||
DRIVER_OBJECT gDriver;
|
||||
DEVICE_OBJECT *gDevice = NULL;
|
||||
bool gDmaEnabled = false;
|
||||
uint64_t gSectorCount = 0;
|
||||
bool gSupportsLba48 = false;
|
||||
|
||||
// DMA hardware state
|
||||
uint16_t gBmBase = 0; // Bus Master I/O base port (BAR4 & ~3)
|
||||
PRD *gPrdtVirt = NULL; // kernel VA of PRDT page
|
||||
uint32_t gPrdtPhys = 0; // 32-bit physical address of PRDT
|
||||
|
||||
// Async worker synchronisation
|
||||
IRP_QUEUE gIrpQueue;
|
||||
SEMAPHORE *gIrpSemaphore = NULL;
|
||||
EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer
|
||||
KDPC gDmaDpc; // bottom half for DMA completion
|
||||
IO_REMOVE_LOCK gRemoveLock;
|
||||
bool gSurpriseRemoved = false;
|
||||
bool gRemoved = false;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bus Master I/O helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
static inline uint8_t BmRead8 (uint8_t reg) { return __inbyte (gBmBase + reg); }
|
||||
static inline void BmWrite8 (uint8_t reg, uint8_t v) { __outbyte (gBmBase + reg, v); }
|
||||
static inline void BmWrite32(uint8_t reg, uint32_t v) { __outdword(gBmBase + reg, v); }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PIO polling helpers (unchanged)
|
||||
// ---------------------------------------------------------------------------
|
||||
static void IdeWait(void) {
|
||||
while (__inbyte(IDE_STATUS) & IDE_BUSY)
|
||||
KeYield();
|
||||
}
|
||||
|
||||
static KSTATUS IdeWaitData(void) {
|
||||
for (int i = 0; i < 4; i++) __inbyte(IDE_ALT); // 400 ns delay
|
||||
while (true) {
|
||||
uint8_t st = __inbyte(IDE_STATUS);
|
||||
if (st & IDE_ERR_F) return STATUS_DEVICE_ERROR;
|
||||
if (!(st & IDE_BUSY) && (st & IDE_DRQ)) return STATUS_SUCCESS;
|
||||
KeYield();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PIO read/write — sync fallback when DMA is unavailable
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS IdeRead(void *ctx, uint64_t lba, uint32_t count, void *buf) {
|
||||
(void)ctx;
|
||||
if (lba + count > gSectorCount) return STATUS_INVALID_PARAMETER;
|
||||
uint16_t *p = (uint16_t *)buf;
|
||||
|
||||
for (uint32_t i = 0; i < count; i++, lba++) {
|
||||
IdeWait();
|
||||
if (gSupportsLba48) {
|
||||
__outbyte(IDE_DRIVE, 0x40);
|
||||
__outbyte(IDE_SECCNT, 0);
|
||||
__outbyte(IDE_LBA0, (lba >> 24) & 0xFF);
|
||||
__outbyte(IDE_LBA1, (lba >> 32) & 0xFF);
|
||||
__outbyte(IDE_LBA2, (lba >> 40) & 0xFF);
|
||||
__outbyte(IDE_SECCNT, 1);
|
||||
__outbyte(IDE_LBA0, (uint8_t)lba);
|
||||
__outbyte(IDE_LBA1, (uint8_t)(lba >> 8));
|
||||
__outbyte(IDE_LBA2, (uint8_t)(lba >> 16));
|
||||
__outbyte(IDE_CMD, CMD_READ_EXT);
|
||||
} else {
|
||||
__outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF));
|
||||
__outbyte(IDE_SECCNT, 1);
|
||||
__outbyte(IDE_LBA0, (uint8_t)lba);
|
||||
__outbyte(IDE_LBA1, (uint8_t)(lba >> 8));
|
||||
__outbyte(IDE_LBA2, (uint8_t)(lba >> 16));
|
||||
__outbyte(IDE_CMD, 0x20);
|
||||
}
|
||||
if (IdeWaitData() != STATUS_SUCCESS) return STATUS_DEVICE_ERROR;
|
||||
for (int w = 0; w < 256; w++) *p++ = __inword(IDE_DATA);
|
||||
}
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS IdeWrite(void *ctx, uint64_t lba, uint32_t count, const void *buf) {
|
||||
(void)ctx;
|
||||
if (lba + count > gSectorCount) return STATUS_INVALID_PARAMETER;
|
||||
const uint16_t *p = (const uint16_t *)buf;
|
||||
|
||||
for (uint32_t i = 0; i < count; i++, lba++) {
|
||||
IdeWait();
|
||||
if (gSupportsLba48) {
|
||||
__outbyte(IDE_DRIVE, 0x40);
|
||||
__outbyte(IDE_SECCNT, 0);
|
||||
__outbyte(IDE_LBA0, (lba >> 24) & 0xFF);
|
||||
__outbyte(IDE_LBA1, (lba >> 32) & 0xFF);
|
||||
__outbyte(IDE_LBA2, (lba >> 40) & 0xFF);
|
||||
__outbyte(IDE_SECCNT, 1);
|
||||
__outbyte(IDE_LBA0, (uint8_t)lba);
|
||||
__outbyte(IDE_LBA1, (uint8_t)(lba >> 8));
|
||||
__outbyte(IDE_LBA2, (uint8_t)(lba >> 16));
|
||||
__outbyte(IDE_CMD, CMD_WRITE_EXT);
|
||||
} else {
|
||||
__outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF));
|
||||
__outbyte(IDE_SECCNT, 1);
|
||||
__outbyte(IDE_LBA0, (uint8_t)lba);
|
||||
__outbyte(IDE_LBA1, (uint8_t)(lba >> 8));
|
||||
__outbyte(IDE_LBA2, (uint8_t)(lba >> 16));
|
||||
__outbyte(IDE_CMD, 0x30);
|
||||
}
|
||||
IdeWait();
|
||||
for (int w = 0; w < 256; w++) __outword(IDE_DATA, *p++);
|
||||
__outbyte(IDE_ALT, 0); // flush write cache
|
||||
if (IdeWaitData() != STATUS_SUCCESS) return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BuildPrdt — fill gPrdtVirt[] for a transfer.
|
||||
//
|
||||
// Walks the virtual buffer in page-aligned chunks, looks up each chunk's
|
||||
// physical address via MmGetPhysicalAddress, and coalesces adjacent physical
|
||||
// pages into a single PRD entry where possible. This is correct regardless of
|
||||
// whether the pool gave us physically contiguous pages.
|
||||
//
|
||||
// Constraints enforced:
|
||||
// - Each PRD entry covers at most one 4 KB physical page (never crosses
|
||||
// a page boundary), so the physical address is valid for the whole chunk.
|
||||
// - Total byte count per entry is ≤ 64 KB (the PRD limit).
|
||||
// - Physical address must fit in 32 bits (BM IDE hardware limit).
|
||||
//
|
||||
// Returns the number of PRD entries written, or 0 on error.
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint32_t BuildPrdt(void *virtBuf, uint32_t byteLen) {
|
||||
uint32_t nPrd = 0;
|
||||
uintptr_t va = (uintptr_t)virtBuf;
|
||||
uint32_t remain = byteLen;
|
||||
|
||||
while (remain > 0) {
|
||||
if (nPrd >= PRDT_MAX_ENTRIES) {
|
||||
DbgPrint("IDE DMA: transfer too large for PRDT (%u entries)\n", nPrd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Bytes from va to the end of its 4 KB page (never straddles a page).
|
||||
uint32_t pageOff = (uint32_t)(va & 0xFFF);
|
||||
uint32_t inPage = 0x1000 - pageOff;
|
||||
uint32_t chunk = (remain < inPage) ? remain : inPage;
|
||||
|
||||
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va);
|
||||
if (phys >> 32) {
|
||||
// Physical address above 4 GB — BM IDE's 32-bit PRDT can't reach it.
|
||||
DbgPrint("IDE DMA: buffer page at VA 0x%016llX has phys 0x%016llX > 4 GB\n",
|
||||
(uint64_t)va, (uint64_t)phys);
|
||||
return 0;
|
||||
}
|
||||
uint32_t phys32 = (uint32_t)phys;
|
||||
|
||||
// Try to coalesce with the previous PRD entry if the physical regions
|
||||
// are adjacent and the combined size stays within the 64 KB limit.
|
||||
if (nPrd > 0) {
|
||||
PRD *prev = &gPrdtVirt[nPrd - 1];
|
||||
uint32_t prevSz = prev->byteCount ? prev->byteCount : 0x10000u;
|
||||
uint32_t prevEnd = prev->physAddr + prevSz;
|
||||
uint32_t combined = prevSz + chunk;
|
||||
if (prevEnd == phys32 && combined <= 0x10000u) {
|
||||
prev->byteCount = (uint16_t)(combined == 0x10000u ? 0u : combined);
|
||||
va += chunk;
|
||||
remain -= chunk;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
gPrdtVirt[nPrd].physAddr = phys32;
|
||||
gPrdtVirt[nPrd].byteCount = (uint16_t)(chunk == 0x10000u ? 0u : chunk);
|
||||
gPrdtVirt[nPrd].flags = 0;
|
||||
nPrd++;
|
||||
va += chunk;
|
||||
remain -= chunk;
|
||||
}
|
||||
|
||||
if (nPrd > 0)
|
||||
gPrdtVirt[nPrd - 1].flags |= PRD_EOT;
|
||||
|
||||
return nPrd;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IDE DMA completion DPC. The ISR owns hardware acknowledgement; this bottom
|
||||
// half wakes the worker outside hard IRQ/DIRQL context.
|
||||
// ---------------------------------------------------------------------------
|
||||
void IdeDmaCompletionDpc(KDPC *dpc, void *context,
|
||||
void *systemArgument1, void *systemArgument2) {
|
||||
(void)dpc;
|
||||
(void)context;
|
||||
(void)systemArgument1;
|
||||
(void)systemArgument2;
|
||||
|
||||
if (gDmaEvent)
|
||||
KeSignalEvent(gDmaEvent);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IDE ISR (IRQ 14 — primary channel)
|
||||
//
|
||||
// Must be fast — runs with interrupts disabled. We only clear the hardware
|
||||
// interrupt bits here and queue a DPC to wake the worker thread.
|
||||
// The worker reads the final status and fires IRP completion hooks.
|
||||
// ---------------------------------------------------------------------------
|
||||
void IdeIsr(void) {
|
||||
uint8_t bmSt = BmRead8(BM_STATUS_REG);
|
||||
if (!(bmSt & BM_STATUS_IRQ))
|
||||
return; // not our interrupt (spurious or secondary channel)
|
||||
|
||||
// Stop the Bus Master engine.
|
||||
BmWrite8(BM_CMD_REG, 0x00);
|
||||
|
||||
// Clear interrupt and error sticky bits (write-1-to-clear).
|
||||
BmWrite8(BM_STATUS_REG, BM_STATUS_IRQ | BM_STATUS_ERROR);
|
||||
|
||||
// Read IDE status and alt-status to fully acknowledge the device interrupt.
|
||||
(void)__inbyte(IDE_ALT);
|
||||
(void)__inbyte(IDE_STATUS);
|
||||
|
||||
// Wake happens in the DPC. gDmaEvent is auto-reset so exactly one waiter
|
||||
// is released when the bottom half runs.
|
||||
KeInsertQueueDpc(&gDmaDpc, (void *)0, (void *)0);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IdeDmaRw — program the hardware and block until the ISR completes it.
|
||||
//
|
||||
// Called exclusively from IdeWorker. Has full control of the IDE/BM
|
||||
// registers; no other thread touches them while the worker is running.
|
||||
// Falls back to PIO inline if BuildPrdt fails (address space issue).
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) {
|
||||
uint32_t byteLen = count * 512u;
|
||||
|
||||
uint32_t nPrd = BuildPrdt(buf, byteLen);
|
||||
if (nPrd == 0) {
|
||||
// Graceful inline PIO fallback for awkward buffers.
|
||||
DbgPrint("IDE DMA: BuildPrdt failed — falling back to PIO\n");
|
||||
return write ? IdeWrite(NULL, lba, count, buf)
|
||||
: IdeRead (NULL, lba, count, buf);
|
||||
}
|
||||
|
||||
// ---- Program Bus Master -----------------------------------------------
|
||||
|
||||
// Load PRDT physical address.
|
||||
BmWrite32(BM_PRDT_REG, gPrdtPhys);
|
||||
|
||||
// Clear any leftover interrupt/error bits from the previous transfer.
|
||||
BmWrite8(BM_STATUS_REG, BM_STATUS_IRQ | BM_STATUS_ERROR);
|
||||
|
||||
// Set direction: BM_CMD_READ = disk→memory; 0 = memory→disk.
|
||||
// Do NOT set BM_CMD_START yet — spec says program ATA regs first.
|
||||
BmWrite8(BM_CMD_REG, write ? 0x00 : BM_CMD_READ);
|
||||
|
||||
// ---- Program ATA registers -------------------------------------------
|
||||
IdeWait();
|
||||
|
||||
if (gSupportsLba48) {
|
||||
__outbyte(IDE_DRIVE, 0x40);
|
||||
__outbyte(IDE_SECCNT, (uint8_t)(count >> 8)); // sector count high byte
|
||||
__outbyte(IDE_LBA0, (uint8_t)(lba >> 24));
|
||||
__outbyte(IDE_LBA1, (uint8_t)(lba >> 32));
|
||||
__outbyte(IDE_LBA2, (uint8_t)(lba >> 40));
|
||||
__outbyte(IDE_SECCNT, (uint8_t)count); // sector count low byte
|
||||
__outbyte(IDE_LBA0, (uint8_t)lba);
|
||||
__outbyte(IDE_LBA1, (uint8_t)(lba >> 8));
|
||||
__outbyte(IDE_LBA2, (uint8_t)(lba >> 16));
|
||||
__outbyte(IDE_CMD, write ? CMD_WRITE_DMA_EXT : CMD_READ_DMA_EXT);
|
||||
} else {
|
||||
__outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF));
|
||||
__outbyte(IDE_SECCNT, (uint8_t)count);
|
||||
__outbyte(IDE_LBA0, (uint8_t)lba);
|
||||
__outbyte(IDE_LBA1, (uint8_t)(lba >> 8));
|
||||
__outbyte(IDE_LBA2, (uint8_t)(lba >> 16));
|
||||
__outbyte(IDE_CMD, write ? CMD_WRITE_DMA : CMD_READ_DMA);
|
||||
}
|
||||
|
||||
// ---- Start DMA --------------------------------------------------------
|
||||
// Set Start bit. Direction bit is already set from above.
|
||||
BmWrite8(BM_CMD_REG, (write ? 0x00 : BM_CMD_READ) | BM_CMD_START);
|
||||
|
||||
// Block the worker thread until the ISR fires.
|
||||
KeWaitForEvent(gDmaEvent, 0);
|
||||
|
||||
// ---- Read result -------------------------------------------------------
|
||||
// ISR already stopped the BM engine and cleared BM_STATUS_IRQ.
|
||||
// We still need to check BM_STATUS_ERROR and IDE_ERR_F.
|
||||
uint8_t bmSt = BmRead8(BM_STATUS_REG);
|
||||
uint8_t ideSt = __inbyte(IDE_STATUS);
|
||||
|
||||
if (bmSt & BM_STATUS_ERROR) {
|
||||
DbgPrint("IDE DMA: BM error (bmStatus=0x%02x, ideStatus=0x%02x, ideErr=0x%02x)\n",
|
||||
bmSt, ideSt, __inbyte(IDE_ERR));
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
if (ideSt & IDE_ERR_F) {
|
||||
DbgPrint("IDE DMA: ATA error (ideStatus=0x%02x, ideErr=0x%02x)\n",
|
||||
ideSt, __inbyte(IDE_ERR));
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static KSTATUS IdeGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
|
||||
uint32_t* outSectors, void** outBuffer) {
|
||||
if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios) return STATUS_INVALID_PARAMETER;
|
||||
if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) ||
|
||||
(ios->Parameters.ReadWrite.Length & 511u))
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
*outWrite = (ios->MajorFunction == IRP_MJ_WRITE);
|
||||
*outLba = ios->Parameters.ReadWrite.ByteOffset >> 9;
|
||||
*outSectors = ios->Parameters.ReadWrite.Length >> 9;
|
||||
*outBuffer = irp->UserBuffer;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IdeWorker — kernel thread, one per IDE controller.
|
||||
//
|
||||
// Blocks on gIrpSemaphore. Each IrpQueueEnqueue increments it by one, so
|
||||
// the worker wakes once per submitted IRP. Mirrors AhciWorker exactly.
|
||||
// ---------------------------------------------------------------------------
|
||||
void IdeWorker(uintptr_t arg) {
|
||||
(void)arg;
|
||||
while (true) {
|
||||
KeWaitSemaphore(gIrpSemaphore);
|
||||
|
||||
IRP *irp = IrpQueueDequeue(&gIrpQueue);
|
||||
if (!irp)
|
||||
continue;
|
||||
|
||||
bool write = false;
|
||||
uint64_t lba = 0;
|
||||
uint32_t sectors = 0;
|
||||
void* buffer = (void*)0;
|
||||
KSTATUS st = IdeGetIrpReadWrite(irp, &write, &lba, §ors, &buffer);
|
||||
if (KSUCCESS(st)) {
|
||||
if (lba + sectors > gSectorCount)
|
||||
st = STATUS_INVALID_PARAMETER;
|
||||
else
|
||||
st = IdeDmaRw(lba, sectors, buffer, write);
|
||||
}
|
||||
|
||||
IoReleaseRemoveLock(&gRemoveLock, irp);
|
||||
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IdeDispatchReadWrite — DEVICE_OBJECT dispatch entry for disk IRPs.
|
||||
//
|
||||
// If DMA is online, the IRP is queued to the worker and completed later.
|
||||
// If DMA is unavailable, the IRP is completed inline through the PIO path.
|
||||
// Either way, accepted IRPs return STATUS_PENDING and complete through
|
||||
// IoCompleteRequest().
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
|
||||
(void)device;
|
||||
|
||||
KSTATUS st = IoAcquireRemoveLock(&gRemoveLock, irp);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
|
||||
bool write = false;
|
||||
uint64_t lba = 0;
|
||||
uint32_t sectors = 0;
|
||||
void* buffer = (void*)0;
|
||||
st = IdeGetIrpReadWrite(irp, &write, &lba, §ors, &buffer);
|
||||
if (!KSUCCESS(st) || lba + sectors > gSectorCount) {
|
||||
IoReleaseRemoveLock(&gRemoveLock, irp);
|
||||
return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st;
|
||||
}
|
||||
|
||||
irp->Status = STATUS_FAILURE;
|
||||
irp->Information = 0;
|
||||
|
||||
if (gDmaEnabled) {
|
||||
IrpQueueEnqueue(&gIrpQueue, irp);
|
||||
return STATUS_PENDING;
|
||||
}
|
||||
|
||||
st = write ? IdeWrite(NULL, lba, sectors, buffer)
|
||||
: IdeRead (NULL, lba, sectors, buffer);
|
||||
IoReleaseRemoveLock(&gRemoveLock, irp);
|
||||
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
|
||||
return STATUS_PENDING;
|
||||
}
|
||||
|
||||
static void IdeCancelReleaseRemoveLock(IRP* irp, void* context) {
|
||||
(void)context;
|
||||
IoReleaseRemoveLock(&gRemoveLock, irp);
|
||||
}
|
||||
|
||||
static KSTATUS IdeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios || ios->MajorFunction != IRP_MJ_PNP)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) {
|
||||
case IRP_MN_START_DEVICE:
|
||||
device->PnpState = DevicePnpStarted;
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_QUERY_REMOVE_DEVICE:
|
||||
device->PnpState = DevicePnpRemovePending;
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_CANCEL_REMOVE_DEVICE:
|
||||
device->PnpState = DevicePnpStarted;
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_SURPRISE_REMOVAL:
|
||||
gSurpriseRemoved = true;
|
||||
device->PnpState = DevicePnpSurpriseRemoved;
|
||||
IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
|
||||
return STATUS_SUCCESS;
|
||||
case IRP_MN_REMOVE_DEVICE:
|
||||
gRemoved = true;
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
IrpQueueCancelAllEx(&gIrpQueue, STATUS_OBJECT_DEAD, IdeCancelReleaseRemoveLock, (void*)0);
|
||||
IoReleaseRemoveLockAndWait(&gRemoveLock, irp);
|
||||
IoDeleteDevice(device);
|
||||
gDevice = (DEVICE_OBJECT*)0;
|
||||
return STATUS_SUCCESS;
|
||||
default:
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IdeDmaInit — locate the PCI IDE controller, grab BAR4, allocate the PRDT,
|
||||
// create sync objects, and register IRQ 14.
|
||||
//
|
||||
// Returns true on success. On any failure, logs a message and returns false;
|
||||
// dispatch will complete IRPs inline through PIO.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DriverEntry
|
||||
// ---------------------------------------------------------------------------
|
||||
KSTATUS DriverEntry(const HAL_API *hal) {
|
||||
DbgPrint("IDE: Init\n");
|
||||
gHal = hal;
|
||||
|
||||
// ---- IDENTIFY drive (same as before) ----------------------------------
|
||||
__outbyte(IDE_DRIVE, 0xA0);
|
||||
for (int i = 0; i < 4; i++) __inbyte(IDE_ALT);
|
||||
|
||||
__outbyte(IDE_SECCNT, 0);
|
||||
__outbyte(IDE_LBA0, 0);
|
||||
__outbyte(IDE_LBA1, 0);
|
||||
__outbyte(IDE_LBA2, 0);
|
||||
__outbyte(IDE_CMD, CMD_IDENT);
|
||||
|
||||
if (__inbyte(IDE_STATUS) == 0) {
|
||||
DbgPrint("IDE: No drive on primary channel\n");
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
int timeout = 10000;
|
||||
while (__inbyte(IDE_STATUS) & IDE_BUSY) {
|
||||
if (--timeout == 0) return STATUS_DEVICE_ERROR;
|
||||
KeYield();
|
||||
}
|
||||
|
||||
if (__inbyte(IDE_LBA1) != 0 || __inbyte(IDE_LBA2) != 0) {
|
||||
DbgPrint("IDE: Not a standard ATA drive (ATAPI?)\n");
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
timeout = 10000;
|
||||
while (true) {
|
||||
uint8_t st = __inbyte(IDE_STATUS);
|
||||
if (st & IDE_ERR_F) return STATUS_DEVICE_ERROR;
|
||||
if (st & IDE_DRQ) break;
|
||||
if (--timeout == 0) return STATUS_DEVICE_ERROR;
|
||||
KeYield();
|
||||
}
|
||||
|
||||
uint16_t ident[256];
|
||||
for (int i = 0; i < 256; i++) ident[i] = __inword(IDE_DATA);
|
||||
|
||||
gSupportsLba48 = (ident[83] & (1 << 10)) != 0;
|
||||
if (gSupportsLba48) {
|
||||
gSectorCount = *(uint64_t *)&ident[100];
|
||||
DbgPrint("IDE: LBA48, %llu sectors\n", gSectorCount);
|
||||
} else {
|
||||
gSectorCount = ((uint32_t)ident[61] << 16) | ident[60];
|
||||
DbgPrint("IDE: LBA28, %u sectors\n", (uint32_t)gSectorCount);
|
||||
}
|
||||
if (gSectorCount == 0) return STATUS_NOT_FOUND;
|
||||
|
||||
// ---- Register definitive DEVICE_OBJECT --------------------------------
|
||||
memset(&gDriver, 0, sizeof(gDriver));
|
||||
IoInitializeDriverObject(&gDriver, SYS_SV_LITERAL("ide"));
|
||||
gDriver.MajorFunction[IRP_MJ_READ] = IdeDispatchReadWrite;
|
||||
gDriver.MajorFunction[IRP_MJ_WRITE] = IdeDispatchReadWrite;
|
||||
gDriver.MajorFunction[IRP_MJ_PNP] = IdeDispatchPnp;
|
||||
|
||||
KSTATUS st = IoInitializeRemoveLock(&gRemoveLock);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
st = IoCreateDevice(&gDriver, 0, &gDevice);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
gDevice->u.Disk.SectorCount = gSectorCount;
|
||||
gDevice->u.Disk.SectorSize = 512;
|
||||
gDevice->AlignmentRequirement = 512;
|
||||
|
||||
// ---- Try to initialise Bus Master DMA ---------------------------------
|
||||
gDmaEnabled = IdeDmaInit();
|
||||
|
||||
DbgPrint("IDE: Registered (%s)\n", gDmaEnabled ? "DMA" : "PIO fallback");
|
||||
return IoRegisterDiskDevice(0, gDevice);
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
// ide.h - shared IDE/ATA driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IDE register map (primary channel, base 0x1F0)
|
||||
// ---------------------------------------------------------------------------
|
||||
#define IDE_DATA 0x1F0
|
||||
#define IDE_ERR 0x1F1
|
||||
#define IDE_SECCNT 0x1F2
|
||||
#define IDE_LBA0 0x1F3
|
||||
#define IDE_LBA1 0x1F4
|
||||
#define IDE_LBA2 0x1F5
|
||||
#define IDE_DRIVE 0x1F6
|
||||
#define IDE_STATUS 0x1F7
|
||||
#define IDE_CMD 0x1F7
|
||||
#define IDE_ALT 0x3F6
|
||||
|
||||
#define IDE_BUSY 0x80
|
||||
#define IDE_READY 0x40
|
||||
#define IDE_ERR_F 0x01
|
||||
#define IDE_DRQ 0x08
|
||||
|
||||
// ATA commands
|
||||
#define CMD_READ_EXT 0x24 // PIO 48-bit read
|
||||
#define CMD_WRITE_EXT 0x34 // PIO 48-bit write
|
||||
#define CMD_READ_DMA_EXT 0x25 // DMA 48-bit read
|
||||
#define CMD_WRITE_DMA_EXT 0x35 // DMA 48-bit write
|
||||
#define CMD_READ_DMA 0xC8 // DMA 28-bit read
|
||||
#define CMD_WRITE_DMA 0xCA // DMA 28-bit write
|
||||
#define CMD_IDENT 0xEC
|
||||
|
||||
// IDE primary IRQ
|
||||
#define IDE_IRQ_PRIMARY 14
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bus Master IDE register offsets (relative to gBmBase, primary channel)
|
||||
// ---------------------------------------------------------------------------
|
||||
#define BM_CMD_REG 0x00 // command
|
||||
#define BM_STATUS_REG 0x02 // status
|
||||
#define BM_PRDT_REG 0x04 // PRDT physical address (32-bit)
|
||||
|
||||
// BM_CMD_REG bits
|
||||
#define BM_CMD_START 0x01 // start DMA transfer
|
||||
#define BM_CMD_READ 0x08 // 1 = disk→memory (read), 0 = memory→disk (write)
|
||||
|
||||
// BM_STATUS_REG bits
|
||||
#define BM_STATUS_ACTIVE 0x01 // DMA active
|
||||
#define BM_STATUS_ERROR 0x02 // DMA error (write 1 to clear)
|
||||
#define BM_STATUS_IRQ 0x04 // interrupt pending (write 1 to clear)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Physical Region Descriptor (PRDT entry, 8 bytes)
|
||||
// The PRDT itself must not cross a 64 KB boundary and must be 4-byte aligned.
|
||||
// We allocate a full page so both constraints are trivially satisfied.
|
||||
// ---------------------------------------------------------------------------
|
||||
#pragma pack(push, 1)
|
||||
typedef struct {
|
||||
uint32_t physAddr; // physical base of this memory region
|
||||
uint16_t byteCount; // byte count; 0 means 64 KB
|
||||
uint16_t flags; // bit 15: End Of Table
|
||||
} PRD;
|
||||
#pragma pack(pop)
|
||||
|
||||
#define PRD_EOT 0x8000u
|
||||
// 512 entries × 8 bytes = 4096 bytes = exactly one page; supports 512×64 KB = 32 MB
|
||||
#define PRDT_MAX_ENTRIES 512
|
||||
|
||||
|
||||
extern const HAL_API *gHal;
|
||||
extern DRIVER_OBJECT gDriver;
|
||||
extern DEVICE_OBJECT *gDevice;
|
||||
extern bool gDmaEnabled;
|
||||
extern uint64_t gSectorCount;
|
||||
extern bool gSupportsLba48;
|
||||
extern uint16_t gBmBase;
|
||||
extern PRD *gPrdtVirt;
|
||||
extern uint32_t gPrdtPhys;
|
||||
extern IRP_QUEUE gIrpQueue;
|
||||
extern SEMAPHORE *gIrpSemaphore;
|
||||
extern EVENT *gDmaEvent;
|
||||
extern KDPC gDmaDpc;
|
||||
|
||||
void IdeDmaCompletionDpc(KDPC *dpc, void *context, void *systemArgument1, void *systemArgument2);
|
||||
void IdeIsr(void);
|
||||
void IdeWorker(uintptr_t arg);
|
||||
bool IdeDmaInit(void);
|
||||
@@ -1,87 +0,0 @@
|
||||
// ide_bus.c - Bus-master IDE PCI/DMA initialisation.
|
||||
#include "ide.h"
|
||||
|
||||
bool IdeDmaInit(void) {
|
||||
// ---- Find PCI IDE controller (class 0x01, subclass 0x01) --------------
|
||||
uint8_t bus, dev, fn;
|
||||
if (!gHal->Pci.FindClass(0x01, 0x01, &bus, &dev, &fn)) {
|
||||
DbgPrint("IDE DMA: No PCI IDE controller found — using PIO\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Read BAR4 (Bus Master base, I/O space) ---------------------------
|
||||
// PCI config offset 0x20. Bit 0 must be 1 for I/O BAR.
|
||||
uint32_t bar4 = gHal->Pci.Read32(bus, dev, fn, 0x20);
|
||||
if (!(bar4 & 0x1)) {
|
||||
DbgPrint("IDE DMA: BAR4 is a memory BAR (0x%08x) — unexpected, using PIO\n", bar4);
|
||||
return false;
|
||||
}
|
||||
uint16_t bmBase = (uint16_t)(bar4 & 0xFFFC);
|
||||
if (bmBase == 0) {
|
||||
DbgPrint("IDE DMA: BAR4 is zero (firmware didn't assign it) — using PIO\n");
|
||||
return false;
|
||||
}
|
||||
DbgPrint("IDE DMA: Bus Master base = 0x%04X (bus %u, dev %u, fn %u)\n",
|
||||
bmBase, bus, dev, fn);
|
||||
|
||||
// ---- Enable Bus Mastering in PCI Command register (bit 2) ------------
|
||||
uint32_t pciCmd = gHal->Pci.Read32(bus, dev, fn, 0x04);
|
||||
gHal->Pci.Write32(bus, dev, fn, 0x04, pciCmd | (1u << 2));
|
||||
|
||||
// ---- Allocate PRDT page (must be 32-bit physical, 4-byte aligned) ----
|
||||
PHYSICAL_ADDRESS prdtPhys = MmAllocateContiguous(1);
|
||||
if (!prdtPhys) {
|
||||
DbgPrint("IDE DMA: Failed to allocate PRDT page\n");
|
||||
return false;
|
||||
}
|
||||
if (prdtPhys >> 32) {
|
||||
// MmAllocateContiguous returned memory above 4 GB — BM IDE can't
|
||||
// address it. This shouldn't happen on normal x86 systems with early
|
||||
// boot allocations but we check defensively.
|
||||
DbgPrint("IDE DMA: PRDT page is above 4 GB (0x%016llX) — using PIO\n",
|
||||
(uint64_t)prdtPhys);
|
||||
return false;
|
||||
}
|
||||
void *prdtVirt = gHal->PhysToVirt(prdtPhys);
|
||||
DbgPrint("IDE DMA: PRDT at phys 0x%08x, virt 0x%016llX\n", (uint32_t)prdtPhys, prdtVirt);
|
||||
|
||||
// ---- Create synchronisation objects ------------------------------------
|
||||
SEMAPHORE *sem = NULL;
|
||||
if (!KSUCCESS(KeSemaphoreCreate(0, 0x7FFFFFFF, &sem))) {
|
||||
DbgPrint("IDE DMA: Failed to create IRP semaphore\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
EVENT *dmaEv = NULL;
|
||||
if (!KSUCCESS(KeEventCreate(EventAutoReset, 0, &dmaEv))) {
|
||||
DbgPrint("IDE DMA: Failed to create DMA completion event\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Commit module state -----------------------------------------------
|
||||
gBmBase = bmBase;
|
||||
gPrdtPhys = (uint32_t)prdtPhys;
|
||||
gPrdtVirt = (PRD *)prdtVirt;
|
||||
gIrpSemaphore = sem;
|
||||
gDmaEvent = dmaEv;
|
||||
KeInitializeDpc(&gDmaDpc, IdeDmaCompletionDpc, (void *)0);
|
||||
|
||||
IrpQueueInit(&gIrpQueue, sem);
|
||||
|
||||
// ---- Register IRQ 14 --------------------------------------------------
|
||||
KeIrqRegister(IDE_IRQ_PRIMARY, IdeIsr);
|
||||
DbgPrint("IDE DMA: IRQ %u registered\n", IDE_IRQ_PRIMARY);
|
||||
|
||||
// ---- Spawn worker thread -----------------------------------------------
|
||||
KSTATUS st = KeCreateKernelThread((uintptr_t)IdeWorker, 0,
|
||||
PriorityHigh, (THREAD **)NULL);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("IDE DMA: Worker thread creation failed (0x%x)\n", st);
|
||||
// Not fatal for init — the PRDT/IRQ are live but async callers will
|
||||
// never get completions. Better to surface the failure clearly.
|
||||
return false;
|
||||
}
|
||||
|
||||
DbgPrint("IDE DMA: Init complete\n");
|
||||
return true;
|
||||
}
|
||||
Reference in new issue
Block a user