// ahci.c - AHCI disk command path and DriverEntry. #include "ahci.h" const HAL_API *g_hal; HBA_MEM *g_hba; uint8_t g_irq; DRIVER_OBJECT g_driver; AHCI_DRIVE g_drives[MAX_AHCI_DRIVES]; int g_driveCount; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- // Accessors for port structure memory at known offsets from structVirt/Phys static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) { return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST); } static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) { return drv->structPhys + PORT_OFF_CMDLIST; } static inline uint64_t fis_phys(AHCI_DRIVE *drv) { return drv->structPhys + PORT_OFF_FIS; } static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) { return (HBA_CMD_TBL *)((uint8_t *)drv->structVirt + PORT_OFF_CMDTBLS + (uint32_t)slot * PORT_CMDTBL_SIZE); } static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { return drv->structPhys + PORT_OFF_CMDTBLS + (uint64_t)slot * PORT_CMDTBL_SIZE; } // --------------------------------------------------------------------------- // Port engine start / stop (polling safe) // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // Port memory initialisation // --------------------------------------------------------------------------- static KSTATUS port_init_memory(AHCI_DRIVE *drv) { uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT); if (!drv->structPhys) return STATUS_NO_MEMORY; drv->structVirt = g_hal->PhysToVirt(drv->structPhys); // Force NOCACHE on DMA descriptor memory MmSetPageProtect((uintptr_t)drv->structVirt, PORT_PAGES_STRUCT * 4096, flags); memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096); drv->bouncePhys = MmAllocateContiguous(BOUNCE_PAGES); if (!drv->bouncePhys) return STATUS_NO_MEMORY; drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys); // Bounce buffer: NOCACHE to ensure CPU writes reach physical memory // before the HBA's DMA engine reads them MmSetPageProtect((uintptr_t)drv->bounceVirt, BOUNCE_PAGES * 4096, flags); memset(drv->bounceVirt, 0, BOUNCE_BYTES); HBA_PORT *port = drv->port; port_stop(port); // Set command list base port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF); port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32); // Set FIS base port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF); port->fbu = (uint32_t)(fis_phys(drv) >> 32); // Initialise all 32 command headers, pointing each to its command table HBA_CMD_HEADER *hdrs = cmd_list(drv); for (int i = 0; i < NUM_CMD_SLOTS; i++) { hdrs[i].prdtl = PORT_PRDT_COUNT; uint64_t tbl = cmd_tbl_phys(drv, i); hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF); hdrs[i].ctbau = (uint32_t)(tbl >> 32); } // Clear any stale interrupts then start the port engine port->serr = (uint32_t)-1; port->is = (uint32_t)-1; port_start(port); return STATUS_SUCCESS; } // --------------------------------------------------------------------------- // Issue a command - polling fallback (used during early init / IDENTIFY only, // before the scheduler and interrupt handler are live). // Returns STATUS_SUCCESS, STATUS_DEVICE_ERROR, or STATUS_TIMEOUT. // --------------------------------------------------------------------------- static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) { HBA_PORT *port = drv->port; // Clear pending interrupts before issuing port->is = (uint32_t)-1; // Wait for port to be idle (not BSY or DRQ) int spin = 0; while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) spin++; if (spin == POLL_TIMEOUT) { DbgPrint("AHCI: port hung (pre-issue)\n"); return STATUS_TIMEOUT; } // Issue slot 0 port->ci = (1u << 0); // Poll until slot 0 clears while (true) { if (!(port->ci & (1u << 0))) break; if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", port->is, port->tfd, port->serr); port->serr = (uint32_t)-1; port->is = (uint32_t)-1; return STATUS_DEVICE_ERROR; } } if (port->is & PORT_IS_TFES) { DbgPrint("AHCI: TFES after completion TFD=0x%x\n", port->tfd); port->is = (uint32_t)-1; return STATUS_DEVICE_ERROR; } return STATUS_SUCCESS; } // --------------------------------------------------------------------------- // Interrupt bottom half. The ISR owns hardware acknowledgement and status // capture; this DPC wakes the worker thread after the hard IRQ has unwound. // --------------------------------------------------------------------------- static void AhciCompletionDpc(KDPC *dpc, void *context, void *systemArgument1, void *systemArgument2) { (void)dpc; (void)systemArgument1; (void)systemArgument2; AHCI_DRIVE *drv = (AHCI_DRIVE *)context; if (drv && drv->completionEvent) KeSignalEvent(drv->completionEvent); } // --------------------------------------------------------------------------- // ISR - called from KeIrqDispatch on the AHCI PCI IRQ line. // Walks all registered drives, checks if their port raised an interrupt, // captures IS, clears it, and queues the per-drive completion DPC. // --------------------------------------------------------------------------- static void AhciIsr(void) { if (!g_hba) return; // GHC.IS tells us which ports are pending. uint32_t pending = g_hba->is; if (!pending) return; for (int i = 0; i < g_driveCount; i++) { AHCI_DRIVE *drv = &g_drives[i]; uint32_t portBit = (1u << drv->portNo); if (!(pending & portBit)) continue; // Capture and clear port IS before queueing the bottom half. The // waiter reads lastIs to check for errors without racing the next // command. OR in case hardware reports multiple causes before the // DPC releases the worker. uint32_t portIs = drv->port->is; drv->port->is = portIs; // W1C - write back to clear drv->lastIs |= portIs; // Clear the HBA-level bit. g_hba->is = portBit; KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0); } } // --------------------------------------------------------------------------- // Issue a command - interrupt-driven (runtime path). // Caller has already built slot 0 via build_rw_cmd / port_identify. // Blocks the current thread until the ISR signals completionEvent. // --------------------------------------------------------------------------- static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { HBA_PORT *port = drv->port; // Clear stale IS and snapshot before issuing so we don't mis-read a // leftover value from a prior command. port->is = (uint32_t)-1; drv->lastIs = 0; // Wait for port idle int spin = 0; while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) spin++; if (spin == POLL_TIMEOUT) { DbgPrint("AHCI: port hung (pre-issue)\n"); return STATUS_TIMEOUT; } // Issue slot 0, then block on the event. port->ci = (1u << 0); KeWaitForEvent(drv->completionEvent, 0); // The ISR has cleared IS - check for errors in the snapshot. uint32_t is = drv->lastIs; if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", is, port->tfd, port->serr); port->serr = (uint32_t)-1; return STATUS_DEVICE_ERROR; } return STATUS_SUCCESS; } // --------------------------------------------------------------------------- // Build a standard DMA read or write command in slot 0 // --------------------------------------------------------------------------- static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, uint64_t bufPhys, bool write) { HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0 HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); memset(tbl, 0, PORT_CMDTBL_SIZE); // How many PRDT entries we need (each covers up to 8K = 16 sectors) uint32_t prdtl = (sectors + 15) / 16; if (prdtl > PORT_PRDT_COUNT) prdtl = PORT_PRDT_COUNT; // capped (see caller) hdr->cfl = sizeof(FIS_REG_H2D) / 4; hdr->w = write ? 1 : 0; hdr->prdtl = (uint16_t)prdtl; hdr->prdbc = 0; // Fill PRDT entries uint32_t remaining = sectors; uint64_t buf = bufPhys; for (uint32_t i = 0; i < prdtl; i++) { uint32_t sects = (remaining > 16) ? 16 : remaining; uint32_t bytes = sects * 512; tbl->prdt_entry[i].dba = (uint32_t)(buf & 0xFFFFFFFF); tbl->prdt_entry[i].dbau = (uint32_t)(buf >> 32); tbl->prdt_entry[i].dbc = bytes - 1; // hardware adds 1 tbl->prdt_entry[i].i = (i == prdtl - 1) ? 1 : 0; buf += bytes; remaining -= sects; } // Build H2D FIS FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis; memset(fis, 0, sizeof(*fis)); fis->fis_type = FIS_TYPE_REG_H2D; fis->c = 1; fis->command = write ? ATA_CMD_WRITE_DMA_EX : ATA_CMD_READ_DMA_EX; fis->device = 1u << 6; // LBA mode fis->lba0 = (uint8_t)(lba); fis->lba1 = (uint8_t)(lba >> 8); fis->lba2 = (uint8_t)(lba >> 16); fis->lba3 = (uint8_t)(lba >> 24); fis->lba4 = (uint8_t)(lba >> 32); fis->lba5 = (uint8_t)(lba >> 40); fis->countl = (uint8_t)(sectors & 0xFF); fis->counth = (uint8_t)(sectors >> 8); } // --------------------------------------------------------------------------- // IDENTIFY command - fills a 512-byte buffer and returns it // --------------------------------------------------------------------------- static KSTATUS port_identify(AHCI_DRIVE *drv) { // Reuse bounce buffer for IDENTIFY data (512 bytes) memset(drv->bounceVirt, 0, 512); HBA_CMD_HEADER *hdr = cmd_list(drv); HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); memset(tbl, 0, PORT_CMDTBL_SIZE); hdr->cfl = sizeof(FIS_REG_H2D) / 4; hdr->w = 0; hdr->prdtl = 1; hdr->prdbc = 0; tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF); tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32); tbl->prdt_entry[0].dbc = 512 - 1; tbl->prdt_entry[0].i = 1; FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis; memset(fis, 0, sizeof(*fis)); fis->fis_type = FIS_TYPE_REG_H2D; fis->c = 1; fis->command = ATA_CMD_IDENTIFY; fis->device = 0; // Poll: IDENTIFY runs before the scheduler and IRQ handler are live. KSTATUS st = port_issue_cmd_poll(drv); if (!KSUCCESS(st)) return st; // Parse sector count from IDENTIFY data const uint16_t *id = (const uint16_t *)drv->bounceVirt; if (id[ATA_IDENT_W83] & (1u << 10)) { // 48-bit LBA supported drv->sectorCount = (uint64_t)id[ATA_IDENT_W100] | ((uint64_t)id[ATA_IDENT_W100 + 1] << 16) | ((uint64_t)id[ATA_IDENT_W100 + 2] << 32) | ((uint64_t)id[ATA_IDENT_W100 + 3] << 48); } else { // 28-bit LBA drv->sectorCount = (uint64_t)id[ATA_IDENT_W60] | ((uint64_t)id[ATA_IDENT_W60 + 1] << 16); } return STATUS_SUCCESS; } // Core read logic static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba, uint32_t count, void *buf) { uint8_t *out = (uint8_t *)buf; while (count > 0) { uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; build_rw_cmd(drv, lba, batch, drv->bouncePhys, false); KSTATUS st = port_issue_cmd(drv); if (!KSUCCESS(st)) return st; memcpy(out, drv->bounceVirt, (size_t)batch * 512); out += (size_t)batch * 512; lba += batch; count -= batch; } return STATUS_SUCCESS; } // Core write logic static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba, uint32_t count, const void *buf) { const uint8_t *in = (const uint8_t *)buf; while (count > 0) { uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; memcpy(drv->bounceVirt, in, (size_t)batch * 512); build_rw_cmd(drv, lba, batch, drv->bouncePhys, true); KSTATUS st = port_issue_cmd(drv); if (!KSUCCESS(st)) return st; in += (size_t)batch * 512; lba += batch; count -= batch; } return STATUS_SUCCESS; } // --------------------------------------------------------------------------- // Async IRP dispatch // --------------------------------------------------------------------------- static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba, uint32_t* outSectors, void** outBuffer) { if (!irp || !outWrite || !outLba || !outSectors || !outBuffer) return STATUS_INVALID_PARAMETER; IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); if (!ios) return STATUS_INVALID_PARAMETER; if (ios->MajorFunction != IRP_MJ_READ && ios->MajorFunction != IRP_MJ_WRITE) return STATUS_INVALID_PARAMETER; if ((ios->Parameters.ReadWrite.ByteOffset & 511ull) || (ios->Parameters.ReadWrite.Length & 511u)) return STATUS_INVALID_PARAMETER; if (!irp->UserBuffer || ios->Parameters.ReadWrite.Length == 0) return STATUS_INVALID_PARAMETER; *outWrite = (ios->MajorFunction == IRP_MJ_WRITE); *outLba = ios->Parameters.ReadWrite.ByteOffset >> 9; *outSectors = ios->Parameters.ReadWrite.Length >> 9; *outBuffer = irp->UserBuffer; return STATUS_SUCCESS; } // AhciDispatchReadWrite - DEVICE_OBJECT dispatch entry for disk IRPs. // Accepted IRPs are queued to the per-drive worker and completed later with // IoCompleteRequest(). Rejected IRPs are returned synchronously and are not // completed. static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { if (!device || !irp) return STATUS_INVALID_PARAMETER; AHCI_DRIVE *drv = (AHCI_DRIVE *)device->DeviceExtension; if (!drv) return STATUS_INVALID_PARAMETER; KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp); if (!KSUCCESS(st)) return st; bool write = false; uint64_t lba = 0; uint32_t sectors = 0; void* buffer = (void*)0; st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); (void)write; (void)buffer; if (!KSUCCESS(st) || lba + sectors > drv->sectorCount) { IoReleaseRemoveLock(&drv->removeLock, irp); return KSUCCESS(st) ? STATUS_INVALID_PARAMETER : st; } irp->Status = STATUS_FAILURE; irp->Information = 0; IrpQueueEnqueue(&drv->irpQueue, irp); return STATUS_PENDING; } static void AhciCancelReleaseRemoveLock(IRP* irp, void* context) { AHCI_DRIVE* drv = (AHCI_DRIVE*)context; if (drv) IoReleaseRemoveLock(&drv->removeLock, irp); } static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { if (!device || !irp) return STATUS_INVALID_PARAMETER; AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension; if (!drv) return STATUS_INVALID_PARAMETER; IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); if (!ios || ios->MajorFunction != IRP_MJ_PNP) return STATUS_INVALID_PARAMETER; switch ((IO_PNP_MINOR_FUNCTION)ios->MinorFunction) { case IRP_MN_START_DEVICE: device->PnpState = DevicePnpStarted; return STATUS_SUCCESS; case IRP_MN_QUERY_REMOVE_DEVICE: device->PnpState = DevicePnpRemovePending; return STATUS_SUCCESS; case IRP_MN_CANCEL_REMOVE_DEVICE: device->PnpState = DevicePnpStarted; return STATUS_SUCCESS; case IRP_MN_SURPRISE_REMOVAL: drv->surpriseRemoved = true; device->PnpState = DevicePnpSurpriseRemoved; IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); return STATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: drv->removed = true; device->PnpState = DevicePnpDeleted; IrpQueueCancelAllEx(&drv->irpQueue, STATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); IoReleaseRemoveLockAndWait(&drv->removeLock, irp); IoDeleteDevice(device); drv->deviceObject = (DEVICE_OBJECT*)0; return STATUS_SUCCESS; default: return STATUS_NOT_SUPPORTED; } } // AhciWorker - kernel thread, one per drive. // Dequeues IRPs and executes them one at a time using the existing // single-slot interrupt-driven path. Completion fires the IRP's event // and/or callback so the original caller can observe the result without // ever having blocked in a driver call. // // NCQ TODO: replace the serialised loop with a multi-slot dispatcher: // 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP. // 2. Pick the oldest IRP from the queue. // 3. Grab a free bit in drv->slotMask (under the slot spinlock). // 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA), // record slotIrp[slot] = irp, and issue via port->sact + port->ci. // 5. Return immediately - the ISR drives all completions. static void AhciWorker(uintptr_t arg) { AHCI_DRIVE *drv = (AHCI_DRIVE *)arg; while (true) { // Block until AHCI dispatch signals a new IRP. KeWaitSemaphore(drv->irpQueue.Semaphore); IRP *irp = IrpQueueDequeue(&drv->irpQueue); if (!irp) continue; bool write = false; uint64_t lba = 0; uint32_t sectors = 0; void* buffer = (void*)0; KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); if (KSUCCESS(st)) { if (write) st = do_write(drv, lba, sectors, buffer); else st = do_read (drv, lba, sectors, buffer); } // Publish result then fire completion hooks through the common I/O path. IoReleaseRemoveLock(&drv->removeLock, irp); IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0); } } // --------------------------------------------------------------------------- // AHCI controller initialisation // --------------------------------------------------------------------------- // Before memory init: only check physical link. Signature is not yet valid. // After COMRESET + FIS receive: check signature for SATA disk. // Issue a COMRESET on the port, wait for link re-establishment and D2H FIS. // FRE must already be set (port_init_memory does this) so the HBA can // capture the signature FIS the device sends after the reset. // PORT_IE bits we care about: // DHRS (bit 0) - D2H Register FIS received - normal completion // PSS (bit 1) - PIO Setup FIS received // DSS (bit 2) - DMA Setup FIS received // SDBS (bit 3) - Set Device Bits FIS received // TFES (bit 30) - Task File Error Status - error // HBFS (bit 29) - Host Bus Fatal error // HBDS (bit 28) - Host Bus Data error // IFS (bit 27) - Interface Fatal error #define PORT_IE_NORMAL (0x0000000Fu) #define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS) #define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS) KSTATUS AhciInitialize(const HAL_API *hal) { g_hal = hal; g_driveCount = 0; memset(&g_driver, 0, sizeof(g_driver)); IoInitializeDriverObject(&g_driver, SYS_SV_LITERAL("ahci")); g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite; g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite; g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp; // ----------------------------------------------------------------------- // Locate AHCI controller via PCI class scan // ----------------------------------------------------------------------- uint8_t pciBus, pciDev, pciFn; if (!hal->Pci.FindClass(PCI_AHCI_CLASS, PCI_AHCI_SUBCLASS, &pciBus, &pciDev, &pciFn)) { DbgPrint("AHCI: no AHCI controller found\n"); return STATUS_NOT_FOUND; } DbgPrint("AHCI: controller at PCI %02x:%02x.%x\n", pciBus, pciDev, pciFn); // ----------------------------------------------------------------------- // Enable Bus Master + Memory Space in PCI command register // ----------------------------------------------------------------------- uint32_t cmd = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_CMD_REG); cmd |= (PCI_CMD_BUSMASTER | PCI_CMD_MEMSPACE); cmd &= ~(1 << 10); cmd |= (1 << 2); hal->Pci.Write32(pciBus, pciDev, pciFn, PCI_CMD_REG, cmd); // ----------------------------------------------------------------------- // Read PCI IRQ line // ----------------------------------------------------------------------- g_irq = hal->Pci.GetIrqLine(pciBus, pciDev, pciFn); if (g_irq == 0xFF) { DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n"); return STATUS_NOT_SUPPORTED; } DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)g_irq); // ----------------------------------------------------------------------- // Read ABAR (BAR5) and map as uncacheable // ----------------------------------------------------------------------- uint32_t bar5 = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_BAR5_REG); uint64_t abarPhys = (uint64_t)(bar5 & ~0xFFFu); KSTATUS st = ahci_map_mmio(abarPhys); if (!KSUCCESS(st)) return st; g_hba = (HBA_MEM *)hal->PhysToVirt(abarPhys); DbgPrint("AHCI: ABAR phys=0x%016llX version=0x%08x\n", (unsigned long long)abarPhys, g_hba->vs); // ----------------------------------------------------------------------- // HBA reset + BIOS handoff (also enables GHC.IE) // ----------------------------------------------------------------------- st = hba_reset(g_hba); if (!KSUCCESS(st)) return st; // ----------------------------------------------------------------------- // Log capability summary // ----------------------------------------------------------------------- uint32_t cap = g_hba->cap; uint32_t nPorts = (cap & 0x1F) + 1; uint32_t nSlots = ((cap >> 8) & 0x1F) + 1; bool s64a = !!(cap & (1u << 31)); DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n", cap, nPorts, nSlots, (int)s64a); // ----------------------------------------------------------------------- // Enumerate implemented ports // ----------------------------------------------------------------------- uint32_t pi = g_hba->pi; for (uint32_t portno = 0; portno < 32 && g_driveCount < MAX_AHCI_DRIVES; portno++) { if (!(pi & (1u << portno))) continue; HBA_PORT *port = &g_hba->ports[portno]; // Phase 1: physical link check if (!port_link_up(port)) { DbgPrint("AHCI: port %u: no link (ssts=0x%08x)\n", portno, port->ssts); continue; } DbgPrint("AHCI: port %u: link up, initializing...\n", portno); int driveIdx = g_driveCount; AHCI_DRIVE *drv = &g_drives[driveIdx]; memset(drv, 0, sizeof(*drv)); drv->port = port; drv->portNo = (uint8_t)portno; // Phase 2: allocate command list, FIS buffer, bounce buffer st = port_init_memory(drv); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: memory init failed (0x%x)\n", portno, st); continue; } // Phase 3: COMRESET - forces device to re-send D2H FIS for valid SIG st = port_comreset(drv); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: COMRESET failed\n", portno); continue; } // Phase 4: filter to SATA disks if (!port_is_sata_disk(port)) { DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n", portno, port->sig); continue; } // Phase 5: create per-drive completion event (auto-reset, initially clear) st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st); continue; } KeInitializeDpc(&drv->completionDpc, AhciCompletionDpc, drv); // Phase 5b: create the IRP queue semaphore and initialise the queue. // The semaphore starts at 0; each submitted IRP increments it by one. SEMAPHORE *irpSem = (SEMAPHORE *)0; st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st); continue; } IrpQueueInit(&drv->irpQueue, irpSem); st = IoInitializeRemoveLock(&drv->removeLock); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st); continue; } // Phase 6: IDENTIFY (polled - scheduler not yet running) st = port_identify(drv); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st); continue; } DbgPrint("AHCI: drive %d: %llu sectors (%llu MB)\n", driveIdx, (unsigned long long)drv->sectorCount, (unsigned long long)(drv->sectorCount / 2048)); // Phase 7: enable per-port interrupts. // Clear any stale IS first so we don't get a spurious signal immediately. port->is = (uint32_t)-1; port->ie = PORT_IE_ALL; // Phase 8: create and register a disk DEVICE_OBJECT. st = IoCreateDevice(&g_driver, 0, &drv->deviceObject); if (!KSUCCESS(st)) { DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", driveIdx, st); continue; } drv->deviceObject->DeviceExtension = drv; drv->deviceObject->u.Disk.SectorCount = drv->sectorCount; drv->deviceObject->u.Disk.SectorSize = 512; drv->deviceObject->AlignmentRequirement = 512; st = IoRegisterDiskDevice((uint8_t)driveIdx, drv->deviceObject); if (KSUCCESS(st)) { DbgPrint("AHCI: drive %d registered\n", driveIdx); g_driveCount++; } else { DbgPrint("AHCI: IoRegisterDiskDevice(%d) failed (0x%x)\n", driveIdx, st); IoDeleteDevice(drv->deviceObject); drv->deviceObject = (DEVICE_OBJECT*)0; continue; } // Phase 9: spawn the per-drive IRP worker thread. // The worker owns do_read / do_write exclusively once the scheduler // is live; it is the only thread that may call port_issue_cmd(). st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv, PriorityHigh, &drv->workerThread); if (!KSUCCESS(st)) { DbgPrint("AHCI: drive %d: worker thread create failed (0x%x)\n", driveIdx, st); // Fatal for this device: all disk access is IRP-based now, so // leave the device registered but clearly fail during testing. } } if (g_driveCount == 0) { DbgPrint("AHCI: no drives found\n"); return STATUS_NOT_FOUND; } // ----------------------------------------------------------------------- // Register the ISR. All ports on this HBA share one PCI IRQ line. // Do this after all drives are initialised so the ISR sees valid state. // ----------------------------------------------------------------------- KeIrqRegister(g_irq, AhciIsr); DbgPrint("AHCI: ISR registered on IRQ %u; initialised %d drive(s)\n", (unsigned)g_irq, g_driveCount); return STATUS_SUCCESS; } KSTATUS DriverEntry(const HAL_API *hal) { DbgPrint("AHCI: driver loading...\n"); KSTATUS st = AhciInitialize(hal); if (!KSUCCESS(st)) DbgPrint("AHCI: init failed: 0x%x\n", st); return st; }