maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

Browse Source
This commit is contained in:
maksw committed 2026-08-11 18:38:36 +02:00
1 parent 8b254f84f4
commit 4add6789ba
177 files changed
+22565 -3070
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/drivers/storage/ahci/ahci.c
+366 -104
View File
@@ -9,6 +9,11 @@ static AHCI_CONTROLLER* g_controllers[MAX_AHCI_CONTROLLERS];
static int g_controllerCount;
static uint8_t g_nextDiskIndex;
#define AHCI_COMMAND_TIMEOUT_MS 10000ull
#define AHCI_PORT_ERROR_MASK \
(PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define AHCI_PORT_TERMINAL_MASK (PORT_IS_DHRS | AHCI_PORT_ERROR_MASK)
static const IO_DEVICE_ID g_ahciIds[] = {
{ IoBusPci, { "PCI\\CC_010601", sizeof("PCI\\CC_010601") - 1 } },
};
@@ -21,20 +26,21 @@ static const IO_DRIVER_ID_TABLE g_ahciIdTable = {
// Helpers.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl);
static void AhciUnregisterController(AHCI_CONTROLLER* ctl);
// Accessors for port structure memory at known offsets from structVirt/Phys.
// Accessors for port structure memory at known CPU and device-address offsets.
static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) {
return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST);
}
// Implements the cmd list phys operation.
static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_CMDLIST;
// Returns the command-list device address.
static inline uint64_t cmd_list_device_address(AHCI_DRIVE *drv) {
return drv->structDeviceAddress + PORT_OFF_CMDLIST;
}
// Implements the fis phys operation.
static inline uint64_t fis_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_FIS;
// Returns the received-FIS device address.
static inline uint64_t fis_device_address(AHCI_DRIVE *drv) {
return drv->structDeviceAddress + PORT_OFF_FIS;
}
static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
@@ -43,9 +49,9 @@ static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
+ (uint32_t)slot * PORT_CMDTBL_SIZE);
}
// Implements the cmd tbl phys operation.
static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
return drv->structPhys + PORT_OFF_CMDTBLS
// Returns one command-table device address.
static inline uint64_t cmd_tbl_device_address(AHCI_DRIVE *drv, int slot) {
return drv->structDeviceAddress + PORT_OFF_CMDTBLS
+ (uint64_t)slot * PORT_CMDTBL_SIZE;
}
@@ -53,53 +59,108 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
// Port memory initialisation.
// Stops one drive worker before its queue and DMA storage are released.
static void AhciStopWorker(AHCI_DRIVE* drv) {
if (!drv)
return;
drv->workerRunning = false;
if (drv->irpQueue.Semaphore)
KeSignalSemaphore(drv->irpQueue.Semaphore);
if (drv->workerThread && drv->workerFinished)
(void)KeWaitForEvent(drv->workerFinished, 0);
if (drv->workerThread) {
ObDereferenceObject((OBJECT*)drv->workerThread);
drv->workerThread = (THREAD*)0;
}
if (drv->workerFinished) {
ObDereferenceObject(&drv->workerFinished->Header);
drv->workerFinished = (EVENT*)0;
}
}
// Releases the non-DMA synchronization objects owned by one AHCI drive.
static void AhciReleaseDriveRuntime(AHCI_DRIVE* drv) {
if (!drv)
return;
AhciStopWorker(drv);
if (drv->removeLockReady) {
IoDestroyRemoveLock(&drv->removeLock);
drv->removeLockReady = false;
}
if (drv->irpQueue.Semaphore) {
ObDereferenceObject(&drv->irpQueue.Semaphore->Header);
drv->irpQueue.Semaphore = (SEMAPHORE*)0;
}
if (drv->completionEvent) {
ObDereferenceObject(&drv->completionEvent->Header);
drv->completionEvent = (EVENT*)0;
}
}
// Releases the DMA buffers owned by one AHCI drive.
static void AhciReleaseDriveDma(AHCI_DRIVE* drv) {
if (!drv || !gAhciDdk || !gAhciDdk->Dma)
return;
if (drv->port && drv->controller &&
!drv->surpriseRemoved && !drv->controller->surpriseRemoved)
port_stop(drv->port);
gAhciDdk->Dma->FreeCommonBuffer(drv->bounceDma);
gAhciDdk->Dma->FreeCommonBuffer(drv->structDma);
drv->bounceDma = (IO_DMA_BUFFER*)0;
drv->structDma = (IO_DMA_BUFFER*)0;
drv->bounceDeviceAddress = 0;
drv->structDeviceAddress = 0;
drv->bounceVirt = (void*)0;
drv->structVirt = (void*)0;
}
// Allocates and programs one port's command, FIS, and bounce buffers.
static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
if (!drv || !drv->controller || !drv->controller->dmaAdapter)
return NTLSTATUS_INVALID_PARAMETER;
drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma);
if (!drv->structPhys) return NTLSTATUS_NO_MEMORY;
drv->structVirt = gAhciDdk->Memory->PhysToVirt(drv->structPhys);
// Force NOCACHE on DMA descriptor memory.
MmPteProtectRange((uintptr_t)drv->structVirt,
PORT_PAGES_STRUCT * 4096, flags);
memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096);
KSTATUS status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter,
PORT_PAGES_STRUCT * PAGE_SIZE,
PAGE_SIZE,
&drv->structDma);
if (!KSUCCESS(status))
return status;
drv->structVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->structDma);
drv->structDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->structDma);
drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma);
if (!drv->bouncePhys) return NTLSTATUS_NO_MEMORY;
drv->bounceVirt = gAhciDdk->Memory->PhysToVirt(drv->bouncePhys);
// Bounce buffer: NOCACHE to ensure CPU writes reach physical memory
// before the HBA's DMA engine reads them.
MmPteProtectRange((uintptr_t)drv->bounceVirt,
BOUNCE_PAGES * 4096, flags);
memset(drv->bounceVirt, 0, BOUNCE_BYTES);
status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter,
BOUNCE_BYTES,
PAGE_SIZE,
&drv->bounceDma);
if (!KSUCCESS(status)) {
AhciReleaseDriveDma(drv);
return status;
}
drv->bounceVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->bounceDma);
drv->bounceDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->bounceDma);
HBA_PORT *port = drv->port;
port_stop(port);
// Set command list base.
port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF);
port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32);
port->clb = (uint32_t)(cmd_list_device_address(drv) & 0xFFFFFFFF);
port->clbu = (uint32_t)(cmd_list_device_address(drv) >> 32);
port->fb = (uint32_t)(fis_device_address(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_device_address(drv) >> 32);
// Set FIS base.
port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_phys(drv) >> 32);
// Initialise all 32 command headers, pointing each to its command table.
HBA_CMD_HEADER *hdrs = cmd_list(drv);
for (int i = 0; i < NUM_CMD_SLOTS; i++) {
hdrs[i].prdtl = PORT_PRDT_COUNT;
uint64_t tbl = cmd_tbl_phys(drv, i);
hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF);
hdrs[i].ctbau = (uint32_t)(tbl >> 32);
uint64_t table = cmd_tbl_device_address(drv, i);
hdrs[i].ctba = (uint32_t)(table & 0xFFFFFFFF);
hdrs[i].ctbau = (uint32_t)(table >> 32);
}
// Clear any stale interrupts then start the port engine.
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
port_start(port);
return NTLSTATUS_SUCCESS;
}
@@ -125,16 +186,22 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
// Issue slot 0.
port->ci = (1u << 0);
// Poll until slot 0 clears.
while (true) {
if (!(port->ci & (1u << 0))) break;
if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) {
// Poll until slot 0 clears, but never let early-init I/O hang forever.
spin = 0;
while ((port->ci & (1u << 0)) && spin < POLL_TIMEOUT) {
if (port->is & AHCI_PORT_ERROR_MASK) {
DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n",
port->is, port->tfd, port->serr);
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
return NTLSTATUS_DEVICE_ERROR;
}
spin++;
}
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: command timed out while polling CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, port->is, port->serr);
return NTLSTATUS_TIMEOUT;
}
if (port->is & PORT_IS_TFES) {
@@ -146,6 +213,30 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
return NTLSTATUS_SUCCESS;
}
// Recovers a port after a command that cannot be trusted to have retired.
static void AhciRecoverPort(AHCI_DRIVE *drv) {
if (!drv || !drv->port)
return;
HBA_PORT *port = drv->port;
uint32_t savedIe = port->ie;
// Keep reset-generated FIS traffic from satisfying a command waiter.
port->ie = 0;
KSTATUS status = port_comreset(drv);
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
drv->lastIs = 0;
if (drv->completionEvent)
KeClearEvent(drv->completionEvent);
port->ie = savedIe;
if (!KSUCCESS(status)) {
DbgPrint("AHCI: port recovery failed st=0x%x CI=0x%x TFD=0x%x SSTS=0x%x\n",
status, port->ci, port->tfd, port->ssts);
}
}
// Interrupt bottom half. The ISR owns hardware acknowledgement and status
// capture; this DPC wakes the worker thread after the hard IRQ has unwound.
@@ -191,7 +282,11 @@ static void AhciIsr(void) {
// Clear the HBA-level bit.
ctl->hba->is = portBit;
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
// DMA Setup, PIO Setup, and other progress interrupts are not
// command completion. Only wake the worker for D2H completion or
// an error that terminates the command.
if (portIs & AHCI_PORT_TERMINAL_MASK)
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
}
}
}
@@ -206,8 +301,19 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
// Clear stale IS and snapshot before issuing so we don't mis-read a
// leftover value from a prior command.
// A serialized slot must be idle before reuse. A stale active bit means a
// prior command escaped normal completion and must be recovered first.
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: slot 0 still active before issue CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, port->is, port->serr);
AhciRecoverPort(drv);
if (port->ci & (1u << 0))
return NTLSTATUS_DEVICE_ERROR;
}
// Clear stale software and hardware completion state before submission.
// The event can otherwise retain a late DPC signal from an older command.
KeClearEvent(drv->completionEvent);
port->is = (uint32_t)-1;
drv->lastIs = 0;
@@ -216,20 +322,54 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT)
spin++;
if (spin == POLL_TIMEOUT) {
DbgPrint("AHCI: port hung (pre-issue)\n");
DbgPrint("AHCI: port hung before issue TFD=0x%x IS=0x%x SERR=0x%x\n",
port->tfd, port->is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_TIMEOUT;
}
// Issue slot 0, then block on the event.
// Issue slot 0, then block with a finite deadline.
port->ci = (1u << 0);
KeWaitForEvent(drv->completionEvent, 0);
EVENT *events[1] = { drv->completionEvent };
KSTATUS waitStatus = KeWaitForMultipleEvents(events, 1, 0,
AHCI_COMMAND_TIMEOUT_MS,
(uint32_t *)0);
// The ISR has cleared IS - check for errors in the snapshot.
uint32_t is = drv->lastIs;
if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) {
DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n",
is, port->tfd, port->serr);
port->serr = (uint32_t)-1;
// Merge any live status that was not consumed by the ISR. This lets a
// missed IRQ still report the hardware error that accompanied it.
uint32_t liveIs = port->is;
uint32_t is = drv->lastIs | liveIs;
if (liveIs)
port->is = liveIs;
if (is & AHCI_PORT_ERROR_MASK) {
DbgPrint("AHCI: command error IS=0x%x CI=0x%x TFD=0x%x SERR=0x%x\n",
is, port->ci, port->tfd, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_DEVICE_ERROR;
}
if (waitStatus == NTLSTATUS_TIMEOUT) {
if (!(port->ci & (1u << 0))) {
DbgPrint("AHCI: completion IRQ missed; command retired IS=0x%x TFD=0x%x SERR=0x%x\n",
is, port->tfd, port->serr);
return NTLSTATUS_SUCCESS;
}
DbgPrint("AHCI: command timeout CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_TIMEOUT;
}
if (waitStatus != NTLSTATUS_SUCCESS)
return waitStatus;
// An interrupt is only completion if hardware has also retired slot 0.
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: premature completion IRQ CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_DEVICE_ERROR;
}
@@ -239,7 +379,7 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
// Build a standard DMA read or write command in slot 0.
static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
uint64_t bufPhys, bool write) {
uint64_t bufferDeviceAddress, bool write) {
HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0
HBA_CMD_TBL *tbl = cmd_tbl(drv, 0);
@@ -256,7 +396,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
// Fill PRDT entries.
uint32_t remaining = sectors;
uint64_t buf = bufPhys;
uint64_t buf = bufferDeviceAddress;
for (uint32_t i = 0; i < prdtl; i++) {
uint32_t sects = (remaining > 16) ? 16 : remaining;
uint32_t bytes = sects * 512;
@@ -289,6 +429,31 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
fis->counth = (uint8_t)(sectors >> 8);
}
// Builds a non-data ATA FLUSH CACHE EXT command in slot 0.
static void build_flush_cmd(AHCI_DRIVE* drv) {
HBA_CMD_HEADER* hdr = cmd_list(drv);
HBA_CMD_TBL* tbl = cmd_tbl(drv, 0);
memset(tbl, 0, PORT_CMDTBL_SIZE);
hdr->cfl = sizeof(FIS_REG_H2D) / 4;
hdr->w = 0;
hdr->prdtl = 0;
hdr->prdbc = 0;
FIS_REG_H2D* fis = (FIS_REG_H2D*)tbl->cfis;
memset(fis, 0, sizeof(*fis));
fis->fis_type = FIS_TYPE_REG_H2D;
fis->c = 1;
fis->command = ATA_CMD_FLUSH_EXT;
}
// Flushes the drive write cache through the serialized command worker.
static KSTATUS do_flush(AHCI_DRIVE* drv) {
if (!drv) return NTLSTATUS_INVALID_PARAMETER;
build_flush_cmd(drv);
return port_issue_cmd(drv);
}
// IDENTIFY command - fills a 512-byte buffer and returns it.
static KSTATUS port_identify(AHCI_DRIVE *drv) {
@@ -305,8 +470,8 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
hdr->prdtl = 1;
hdr->prdbc = 0;
tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF);
tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32);
tbl->prdt_entry[0].dba = (uint32_t)(drv->bounceDeviceAddress & 0xFFFFFFFF);
tbl->prdt_entry[0].dbau = (uint32_t)(drv->bounceDeviceAddress >> 32);
tbl->prdt_entry[0].dbc = 512 - 1;
tbl->prdt_entry[0].i = 1;
@@ -347,7 +512,7 @@ static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba,
while (count > 0) {
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
build_rw_cmd(drv, lba, batch, drv->bouncePhys, false);
build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, false);
KSTATUS st = port_issue_cmd(drv);
if (!KSUCCESS(st)) return st;
@@ -368,7 +533,7 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
memcpy(drv->bounceVirt, in, (size_t)batch * 512);
build_rw_cmd(drv, lba, batch, drv->bouncePhys, true);
build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, true);
KSTATUS st = port_issue_cmd(drv);
if (!KSUCCESS(st)) return st;
@@ -463,13 +628,16 @@ static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
drv->surpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
drv->removed = true;
device->PnpState = DevicePnpDeleted;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->removeLockReady)
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
IoDeleteDevice(device);
drv->deviceObject = (DEVICE_OBJECT*)0;
return NTLSTATUS_SUCCESS;
@@ -511,7 +679,10 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
for (int i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
drv->surpriseRemoved = true;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->deviceObject)
(void)IoNotifyDiskSurpriseRemoved(drv->deviceObject->u.Disk.DiskIndex);
else
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
}
return NTLSTATUS_SUCCESS;
@@ -520,9 +691,18 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
device->PnpState = DevicePnpDeleted;
for (int i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
drv->removed = true;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->deviceObject) {
(void)IoUnregisterDiskDevice(drv->deviceObject->u.Disk.DiskIndex);
} else {
drv->removed = true;
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
}
}
AhciUnregisterController(ctl);
gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter);
ctl->dmaAdapter = (IO_DMA_ADAPTER*)0;
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
@@ -532,6 +712,29 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Queues one write-cache flush on the per-drive worker.
static KSTATUS AhciDispatchFlush(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER;
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
if (!drv || drv->magic != AHCI_DRIVE_MAGIC)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) {
IoReleaseRemoveLock(&drv->removeLock, irp);
return NTLSTATUS_INVALID_PARAMETER;
}
irp->Status = NTLSTATUS_FAILURE;
irp->Information = 0;
IrpQueueEnqueue(&drv->irpQueue, irp);
return NTLSTATUS_PENDING;
}
// Dispatches pnp.
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DeviceExtension)
@@ -552,7 +755,7 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
// ever having blocked in a driver call.
// NCQ remains serialized until the driver grows a multi-slot dispatcher:
// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP.
// 1. Wait for a free slot and a pending IRP through the queue semaphore.
// 2. Pick the oldest IRP from the queue.
// 3. Grab a free bit in drv->slotMask (under the slot spinlock).
// 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA),
@@ -561,30 +764,45 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
static void AhciWorker(uintptr_t arg) {
AHCI_DRIVE *drv = (AHCI_DRIVE *)arg;
while (true) {
while (drv && drv->workerRunning) {
// Block until AHCI dispatch signals a new IRP.
KeWaitSemaphore(drv->irpQueue.Semaphore);
KeWaitSemaphore(drv->irpQueue.Semaphore, KE_WAIT_INFINITE);
if (!drv->workerRunning)
break;
IRP *irp = IrpQueueDequeue(&drv->irpQueue);
if (!irp)
continue;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read (drv, lba, sectors, buffer);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
KSTATUS st = NTLSTATUS_INVALID_PARAMETER;
size_t information = 0;
if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) {
st = do_flush(drv);
} else {
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read(drv, lba, sectors, buffer);
if (KSUCCESS(st))
information = (size_t)sectors * 512u;
}
}
// Publish result then fire completion hooks through the common I/O path.
IoReleaseRemoveLock(&drv->removeLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
IoCompleteRequest(irp, st, information);
}
if (drv && drv->workerFinished)
KeSignalEvent(drv->workerFinished);
}
// AHCI controller initialisation.
@@ -597,18 +815,8 @@ static void AhciWorker(uintptr_t arg) {
// FRE must already be set (port_init_memory does this) so the HBA can
// capture the signature FIS the device sends after the reset.
// PORT_IE bits we care about:
// DHRS (bit 0) - D2H Register FIS received - normal completion
// PSS (bit 1) - PIO Setup FIS received
// DSS (bit 2) - DMA Setup FIS received
// SDBS (bit 3) - Set Device Bits FIS received
// TFES (bit 30) - Task File Error Status - error
// HBFS (bit 29) - Host Bus Fatal error
// HBDS (bit 28) - Host Bus Data error
// IFS (bit 27) - Interface Fatal error.
#define PORT_IE_NORMAL (0x0000000Fu)
#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS)
// Runtime commands are non-NCQ DMA commands. DHRS is their completion signal;
// setup/progress FIS interrupts must not wake the serialized command waiter.
// Registers controller.
static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
@@ -623,6 +831,20 @@ static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the interrupt registry.
static void AhciUnregisterController(AHCI_CONTROLLER* ctl) {
if (!ctl)
return;
for (int i = 0; i < g_controllerCount; i++) {
if (g_controllers[i] != ctl)
continue;
g_controllerCount--;
g_controllers[i] = g_controllers[g_controllerCount];
g_controllers[g_controllerCount] = (AHCI_CONTROLLER*)0;
return;
}
}
// Starts controller.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC || !ctl->pdo || !ctl->fdo)
@@ -676,7 +898,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = PciMapBar(ctl->pdo,
AHCI_PCI_BAR_ABAR,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&ctl->hba,
&ctl->abarLength);
if (!KSUCCESS(st)) {
@@ -712,9 +934,23 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n",
cap, nPorts, nSlots, (int)s64a);
IO_DMA_REQUIREMENTS dmaRequirements;
memset(&dmaRequirements, 0, sizeof(dmaRequirements));
dmaRequirements.MaximumDeviceAddress = s64a ? (uint64_t)-1 : 0xFFFFFFFFULL;
dmaRequirements.Alignment = 2;
dmaRequirements.MaximumSegmentLength = 1u << 22;
dmaRequirements.MaximumSegments = PORT_PRDT_COUNT;
st = gAhciDdk->Dma->GetAdapter(ctl->fdo, &dmaRequirements, &ctl->dmaAdapter);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: DMA adapter creation failed (0x%x)\n", st);
return st;
}
st = AhciRegisterController(ctl);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: controller registry full (0x%x)\n", st);
gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter);
ctl->dmaAdapter = (IO_DMA_ADAPTER*)0;
return st;
}
@@ -751,6 +987,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = port_comreset(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: COMRESET failed\n", portno);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -758,6 +996,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
if (!port_is_sata_disk(port)) {
DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n",
portno, port->sig);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -765,6 +1005,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
KeInitializeDpc(&drv->completionDpc, AhciCompletionDpc, drv);
@@ -775,19 +1017,26 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
IrpQueueInit(&drv->irpQueue, irpSem);
st = IoInitializeRemoveLock(&drv->removeLock);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
drv->removeLockReady = true;
// Phase 6: IDENTIFY (polled - scheduler not yet running).
st = port_identify(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -799,12 +1048,14 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
// Phase 7: enable per-port interrupts.
// Clear any stale IS first so we don't get a spurious signal immediately.
port->is = (uint32_t)-1;
port->ie = PORT_IE_ALL;
port->ie = AHCI_PORT_TERMINAL_MASK;
// Phase 8: create and register a disk DEVICE_OBJECT.
st = IoCreateDevice(&g_driver, 0, &drv->deviceObject);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", g_nextDiskIndex, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
drv->deviceObject->DeviceExtension = drv;
@@ -822,19 +1073,29 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st);
IoDeleteDevice(drv->deviceObject);
drv->deviceObject = (DEVICE_OBJECT*)0;
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
// Phase 9: spawn the per-drive IRP worker thread.
// The worker owns do_read / do_write exclusively once the scheduler
// is live; it is the only thread that may call port_issue_cmd().
st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv,
PriorityHigh, &drv->workerThread);
st = KeEventCreate(EventManualReset, 0, &drv->workerFinished);
if (KSUCCESS(st)) {
drv->workerRunning = true;
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh);
st = KeCreateKernelThread((uintptr_t)AhciWorker,
(uintptr_t)drv,
&workerParameters,
&drv->workerThread);
}
if (!KSUCCESS(st)) {
DbgPrint("AHCI: drive %u: worker thread create failed (0x%x)\n",
diskIndex, st);
// Fatal for this device: all disk access is IRP-based now, so
// leave the device registered but clearly fail during testing.
drv->workerRunning = false;
DbgPrint("AHCI: drive %u: worker creation failed (0x%x)\n", diskIndex, st);
(void)IoUnregisterDiskDevice(diskIndex);
ctl->driveCount--;
continue;
}
}
@@ -889,7 +1150,7 @@ static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevic
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt)
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Dma)
return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("AHCI: driver loading...\n");
@@ -903,6 +1164,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
g_driver.DriverAddDevice = AhciAddDevice;
g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AhciDispatchFlush;
g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
KSTATUS st = IoRegisterDriverObject(&g_driver);
code/drivers/storage/ahci/ahci.h
+11 -4
View File
@@ -250,6 +250,7 @@ typedef volatile struct {
#define PORT_CMD_ICC_ACTIVE (0x1u << 28)
// Port IS register bits (interrupt status).
#define PORT_IS_DHRS (1u << 0) // D2H Register FIS Received.
#define PORT_IS_TFES (1u << 30) // Task File Error Status
#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error
#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error
@@ -321,11 +322,13 @@ typedef struct {
uint8_t portNo; // Physical port index (for ISR dispatch)
// Structures block (command list + FIS + cmd tables).
uint64_t structPhys; // Physical base
void* structVirt; // Virtual base
IO_DMA_BUFFER* structDma;
uint64_t structDeviceAddress; // Device base.
void* structVirt; // CPU base.
// Bounce buffer for DMA data.
uint64_t bouncePhys;
IO_DMA_BUFFER* bounceDma;
uint64_t bounceDeviceAddress;
void* bounceVirt;
// Signaled by the completion DPC when the port raises an interrupt.
@@ -345,13 +348,16 @@ typedef struct {
IO_REMOVE_LOCK removeLock;
bool surpriseRemoved;
bool removed;
bool removeLockReady;
volatile bool workerRunning;
THREAD* workerThread;
EVENT* workerFinished;
DEVICE_OBJECT* deviceObject;
// Multi-slot NCQ needs per-slot ownership state in addition to the serialized queue:
// uint32_t slotMask; // bitmask of in-flight slots
// IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP
// uint64_t slotBouncePhys[NUM_CMD_SLOTS];
// uint64_t slotBounceDeviceAddress[NUM_CMD_SLOTS];
// void* slotBounceVirt[NUM_CMD_SLOTS];
// volatile uint32_t prevSact; // previous port->sact snapshot.
} AHCI_DRIVE;
@@ -361,6 +367,7 @@ struct AHCI_CONTROLLER {
DEVICE_OBJECT* fdo;
DEVICE_OBJECT* pdo;
DEVICE_OBJECT* lowerDevice;
IO_DMA_ADAPTER* dmaAdapter;
PCI_DEVICE_INFO pciInfo;
PCI_BAR abar;
code/drivers/storage/ahci/ahci_port.c
+6 -9
View File
@@ -11,16 +11,13 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
int pageIndex = 0;
for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) {
uintptr_t virt = (uintptr_t)gAhciDdk->Memory->PhysToVirt(phys);
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent);
// Ignore NTLSTATUS_ALREADY_EXISTS - a prior mapping with different flags
// is possible for the first page if the bootloader was overly generous.
// Use MmPteProtectRange to force NOCACHE in that case.
if (st == NTLSTATUS_ALREADY_EXISTS) {
st = MmPteProtectRange(virt, 4096, flags);
}
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC;
KSTATUS st = gAhciDdk->Memory->MapPage(virt,
phys,
flags,
NtlDdkMapReplaceExisting);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
DbgPrint("AHCI: MMIO map failed for page %d (phys=0x%016llX, st=0x%x)\n",
pageIndex, (unsigned long long)phys, st);
return st;
}