maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

redo boot around an explicit system volume and fixed kernel/HAL images, redo volumes and filesystem paths around capabilities instead of drive numbers and kernel CWD state, add a capability/security model with derived and revocable handles and security contexts, add port IPC and rssrv to manage the userspace resource namespace and publish the system volume, move process, dll and path loading to userspace and add process builder syscalls, init now supervises rssrv and the session process, redo the syscall ABI around request structs and four arguments, add a volume manager and proper partition arrival/removal handling so FAT and NTFS mount dynamically, clean up the HAL/DDK boundary so drivers don't depend on HAL internals, save fpu state properly now, the serial driver doesn't hang on a missing port, fix AHCI startup races, fix timeout accounting, fix shared mmap writeback, more smoke tests

Browse Source
This commit is contained in:
maksw committed 2026-08-20 12:33:17 +02:00
1 parent 4add6789ba
commit f882d339fb
232 files changed
+20934 -8898
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/drivers/storage/ahci/ahci.c
+84 -73
View File
@@ -29,28 +29,28 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl);
static void AhciUnregisterController(AHCI_CONTROLLER* ctl);
// Accessors for port structure memory at known CPU and device-address offsets.
static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) {
return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST);
static inline HBA_CMD_HEADER* cmd_list(AHCI_DRIVE* drv) {
return (HBA_CMD_HEADER*)((uint8_t*)drv->structVirt + PORT_OFF_CMDLIST);
}
// Returns the command-list device address.
static inline uint64_t cmd_list_device_address(AHCI_DRIVE *drv) {
static inline uint64_t cmd_list_device_address(AHCI_DRIVE* drv) {
return drv->structDeviceAddress + PORT_OFF_CMDLIST;
}
// Returns the received-FIS device address.
static inline uint64_t fis_device_address(AHCI_DRIVE *drv) {
static inline uint64_t fis_device_address(AHCI_DRIVE* drv) {
return drv->structDeviceAddress + PORT_OFF_FIS;
}
static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
return (HBA_CMD_TBL *)((uint8_t *)drv->structVirt
static inline HBA_CMD_TBL* cmd_tbl(AHCI_DRIVE* drv, int slot) {
return (HBA_CMD_TBL*)((uint8_t*)drv->structVirt
+ PORT_OFF_CMDTBLS
+ (uint32_t)slot * PORT_CMDTBL_SIZE);
}
// Returns one command-table device address.
static inline uint64_t cmd_tbl_device_address(AHCI_DRIVE *drv, int slot) {
static inline uint64_t cmd_tbl_device_address(AHCI_DRIVE* drv, int slot) {
return drv->structDeviceAddress + PORT_OFF_CMDTBLS
+ (uint64_t)slot * PORT_CMDTBL_SIZE;
}
@@ -118,7 +118,7 @@ static void AhciReleaseDriveDma(AHCI_DRIVE* drv) {
}
// Allocates and programs one port's command, FIS, and bounce buffers.
static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
static KSTATUS port_init_memory(AHCI_DRIVE* drv) {
if (!drv || !drv->controller || !drv->controller->dmaAdapter)
return NTLSTATUS_INVALID_PARAMETER;
@@ -142,7 +142,7 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
drv->bounceVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->bounceDma);
drv->bounceDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->bounceDma);
HBA_PORT *port = drv->port;
HBA_PORT* port = drv->port;
port_stop(port);
port->clb = (uint32_t)(cmd_list_device_address(drv) & 0xFFFFFFFF);
@@ -150,7 +150,7 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
port->fb = (uint32_t)(fis_device_address(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_device_address(drv) >> 32);
HBA_CMD_HEADER *hdrs = cmd_list(drv);
HBA_CMD_HEADER* hdrs = cmd_list(drv);
for (int i = 0; i < NUM_CMD_SLOTS; i++) {
hdrs[i].prdtl = PORT_PRDT_COUNT;
uint64_t table = cmd_tbl_device_address(drv, i);
@@ -168,8 +168,8 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
// before the scheduler and interrupt handler are live).
// Returns NTLSTATUS_SUCCESS, NTLSTATUS_DEVICE_ERROR, or NTLSTATUS_TIMEOUT.
static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
static KSTATUS port_issue_cmd_poll(AHCI_DRIVE* drv) {
HBA_PORT* port = drv->port;
// Clear pending interrupts before issuing.
port->is = (uint32_t)-1;
@@ -214,11 +214,11 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
}
// Recovers a port after a command that cannot be trusted to have retired.
static void AhciRecoverPort(AHCI_DRIVE *drv) {
static void AhciRecoverPort(AHCI_DRIVE* drv) {
if (!drv || !drv->port)
return;
HBA_PORT *port = drv->port;
HBA_PORT* port = drv->port;
uint32_t savedIe = port->ie;
// Keep reset-generated FIS traffic from satisfying a command waiter.
@@ -240,13 +240,13 @@ static void AhciRecoverPort(AHCI_DRIVE *drv) {
// Interrupt bottom half. The ISR owns hardware acknowledgement and status
// capture; this DPC wakes the worker thread after the hard IRQ has unwound.
static void AhciCompletionDpc(KDPC *dpc, void *context,
void *systemArgument1, void *systemArgument2) {
static void AhciCompletionDpc(KDPC* dpc, void* context,
void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)systemArgument1;
(void)systemArgument2;
AHCI_DRIVE *drv = (AHCI_DRIVE *)context;
AHCI_DRIVE* drv = (AHCI_DRIVE*)context;
if (drv && drv->completionEvent)
KeSignalEvent(drv->completionEvent);
}
@@ -267,7 +267,7 @@ static void AhciIsr(void) {
continue;
for (int i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE *drv = &ctl->drives[i];
AHCI_DRIVE* drv = &ctl->drives[i];
uint32_t portBit = (1u << drv->portNo);
if (!(pending & portBit)) continue;
@@ -286,7 +286,7 @@ static void AhciIsr(void) {
// command completion. Only wake the worker for D2H completion or
// an error that terminates the command.
if (portIs & AHCI_PORT_TERMINAL_MASK)
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
KeInsertQueueDpc(&drv->completionDpc, (void*)0, (void*)0);
}
}
}
@@ -295,11 +295,11 @@ static void AhciIsr(void) {
// Caller has already built slot 0 via build_rw_cmd / port_identify.
// Blocks the current thread until the ISR signals completionEvent.
static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
static KSTATUS port_issue_cmd(AHCI_DRIVE* drv) {
if (drv->controller && drv->controller->pollingMode)
return port_issue_cmd_poll(drv);
HBA_PORT *port = drv->port;
HBA_PORT* port = drv->port;
// A serialized slot must be idle before reuse. A stale active bit means a
// prior command escaped normal completion and must be recovered first.
@@ -330,10 +330,10 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
// Issue slot 0, then block with a finite deadline.
port->ci = (1u << 0);
EVENT *events[1] = { drv->completionEvent };
EVENT* events[1] = { drv->completionEvent };
KSTATUS waitStatus = KeWaitForMultipleEvents(events, 1, 0,
AHCI_COMMAND_TIMEOUT_MS,
(uint32_t *)0);
(uint32_t*)0);
// Merge any live status that was not consumed by the ISR. This lets a
// missed IRQ still report the hardware error that accompanied it.
@@ -378,10 +378,10 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
// Build a standard DMA read or write command in slot 0.
static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
static void build_rw_cmd(AHCI_DRIVE* drv, uint64_t lba, uint32_t sectors,
uint64_t bufferDeviceAddress, bool write) {
HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0
HBA_CMD_TBL *tbl = cmd_tbl(drv, 0);
HBA_CMD_HEADER* hdr = cmd_list(drv); // slot 0
HBA_CMD_TBL* tbl = cmd_tbl(drv, 0);
memset(tbl, 0, PORT_CMDTBL_SIZE);
@@ -411,7 +411,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
}
// Build H2D FIS.
FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis;
FIS_REG_H2D* fis = (FIS_REG_H2D*)tbl->cfis;
memset(fis, 0, sizeof(*fis));
fis->fis_type = FIS_TYPE_REG_H2D;
fis->c = 1;
@@ -456,12 +456,12 @@ static KSTATUS do_flush(AHCI_DRIVE* drv) {
// IDENTIFY command - fills a 512-byte buffer and returns it.
static KSTATUS port_identify(AHCI_DRIVE *drv) {
static KSTATUS port_identify(AHCI_DRIVE* drv) {
// Reuse bounce buffer for IDENTIFY data (512 bytes).
memset(drv->bounceVirt, 0, 512);
HBA_CMD_HEADER *hdr = cmd_list(drv);
HBA_CMD_TBL *tbl = cmd_tbl(drv, 0);
HBA_CMD_HEADER* hdr = cmd_list(drv);
HBA_CMD_TBL* tbl = cmd_tbl(drv, 0);
memset(tbl, 0, PORT_CMDTBL_SIZE);
@@ -475,7 +475,7 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
tbl->prdt_entry[0].dbc = 512 - 1;
tbl->prdt_entry[0].i = 1;
FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis;
FIS_REG_H2D* fis = (FIS_REG_H2D*)tbl->cfis;
memset(fis, 0, sizeof(*fis));
fis->fis_type = FIS_TYPE_REG_H2D;
fis->c = 1;
@@ -487,7 +487,7 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
if (!KSUCCESS(st)) return st;
// Parse sector count from IDENTIFY data.
const uint16_t *id = (const uint16_t *)drv->bounceVirt;
const uint16_t* id = (const uint16_t*)drv->bounceVirt;
if (id[ATA_IDENT_W83] & (1u << 10)) {
// 48-bit LBA supported.
@@ -505,9 +505,9 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
}
// Core read logic.
static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba,
uint32_t count, void *buf) {
uint8_t *out = (uint8_t *)buf;
static KSTATUS do_read(AHCI_DRIVE* drv, uint64_t lba,
uint32_t count, void* buf) {
uint8_t* out = (uint8_t*)buf;
while (count > 0) {
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
@@ -525,9 +525,9 @@ static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba,
}
// Core write logic.
static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
uint32_t count, const void *buf) {
const uint8_t *in = (const uint8_t *)buf;
static KSTATUS do_write(AHCI_DRIVE* drv, uint64_t lba,
uint32_t count, const void* buf) {
const uint8_t* in = (const uint8_t*)buf;
while (count > 0) {
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
@@ -571,9 +571,9 @@ static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
// Accepted IRPs are queued to the per-drive worker and completed later with
// IoCompleteRequest(). Rejected IRPs are returned synchronously and are not
// completed.
static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER;
AHCI_DRIVE *drv = (AHCI_DRIVE *)device->DeviceExtension;
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
if (!drv) return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp);
@@ -762,7 +762,7 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
// record slotIrp[slot] = irp, and issue via port->sact + port->ci.
// 5. Return immediately so the ISR can drive all completions.
static void AhciWorker(uintptr_t arg) {
AHCI_DRIVE *drv = (AHCI_DRIVE *)arg;
AHCI_DRIVE* drv = (AHCI_DRIVE*)arg;
while (drv && drv->workerRunning) {
// Block until AHCI dispatch signals a new IRP.
@@ -770,7 +770,7 @@ static void AhciWorker(uintptr_t arg) {
if (!drv->workerRunning)
break;
IRP *irp = IrpQueueDequeue(&drv->irpQueue);
IRP* irp = IrpQueueDequeue(&drv->irpQueue);
if (!irp)
continue;
@@ -954,11 +954,17 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
return st;
}
// IoRegisterDiskDevice publishes arrival synchronously, and partition/filesystem
// consumers may issue reads from that callback. Install the controller ISR before
// any disk is published so an interrupt-driven drive can complete those reads.
if (!ctl->pollingMode)
KeIrqRegister(ctl->irq, AhciIsr);
uint32_t pi = ctl->hba->pi;
for (uint32_t portno = 0; portno < 32 && ctl->driveCount < MAX_AHCI_DRIVES; portno++) {
if (!(pi & (1u << portno))) continue;
HBA_PORT *port = &ctl->hba->ports[portno];
HBA_PORT* port = &ctl->hba->ports[portno];
// Phase 1: physical link check.
if (!port_link_up(port)) {
@@ -969,7 +975,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: port %u: link up, initializing...\n", portno);
int driveIdx = ctl->driveCount;
AHCI_DRIVE *drv = &ctl->drives[driveIdx];
AHCI_DRIVE* drv = &ctl->drives[driveIdx];
memset(drv, 0, sizeof(*drv));
drv->magic = AHCI_DRIVE_MAGIC;
drv->controller = ctl;
@@ -1013,7 +1019,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
// Phase 5b: create the IRP queue semaphore and initialise the queue.
// The semaphore starts at 0; each submitted IRP increments it by one.
SEMAPHORE *irpSem = (SEMAPHORE *)0;
SEMAPHORE* irpSem = (SEMAPHORE*)0;
st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st);
@@ -1031,7 +1037,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
}
drv->removeLockReady = true;
// Phase 6: IDENTIFY (polled - scheduler not yet running).
// Phase 6: IDENTIFY remains polled until the disk runtime path is ready.
st = port_identify(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st);
@@ -1045,12 +1051,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
(unsigned long long)drv->sectorCount,
(unsigned long long)(drv->sectorCount / 2048));
// Phase 7: enable per-port interrupts.
// Clear any stale IS first so we don't get a spurious signal immediately.
port->is = (uint32_t)-1;
port->ie = AHCI_PORT_TERMINAL_MASK;
// Phase 8: create and register a disk DEVICE_OBJECT.
// Phase 7: create the disk DEVICE_OBJECT, but do not publish it until the
// asynchronous runtime path is fully operational.
st = IoCreateDevice(&g_driver, 0, &drv->deviceObject);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", g_nextDiskIndex, st);
@@ -1063,24 +1065,9 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
drv->deviceObject->u.Disk.SectorSize = 512;
drv->deviceObject->AlignmentRequirement = 512;
uint8_t diskIndex = g_nextDiskIndex;
st = IoRegisterDiskDevice(diskIndex, drv->deviceObject);
if (KSUCCESS(st)) {
DbgPrint("AHCI: drive %u registered\n", diskIndex);
ctl->driveCount++;
g_nextDiskIndex++;
} else {
DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st);
IoDeleteDevice(drv->deviceObject);
drv->deviceObject = (DEVICE_OBJECT*)0;
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
// Phase 9: spawn the per-drive IRP worker thread.
// The worker owns do_read / do_write exclusively once the scheduler
// is live; it is the only thread that may call port_issue_cmd().
// Phase 8: start the per-drive IRP worker before publishing the disk.
// IoRegisterDiskDevice synchronously notifies partition consumers, which may
// immediately submit reads that must have a live worker and completion path.
st = KeEventCreate(EventManualReset, 0, &drv->workerFinished);
if (KSUCCESS(st)) {
drv->workerRunning = true;
@@ -1092,9 +1079,34 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
}
if (!KSUCCESS(st)) {
drv->workerRunning = false;
DbgPrint("AHCI: drive %u: worker creation failed (0x%x)\n", diskIndex, st);
(void)IoUnregisterDiskDevice(diskIndex);
DbgPrint("AHCI: drive %u: worker creation failed (0x%x)\n", g_nextDiskIndex, st);
IoDeleteDevice(drv->deviceObject);
drv->deviceObject = (DEVICE_OBJECT*)0;
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
// Make the initialized drive visible to the controller ISR, then arm its
// interrupt source. No consumer can submit I/O until publication below.
ctl->driveCount++;
port->is = (uint32_t)-1;
port->ie = AHCI_PORT_TERMINAL_MASK;
// Phase 9: publish only after reads and writes can complete synchronously.
uint8_t diskIndex = g_nextDiskIndex;
st = IoRegisterDiskDevice(diskIndex, drv->deviceObject);
if (KSUCCESS(st)) {
DbgPrint("AHCI: drive %u registered\n", diskIndex);
g_nextDiskIndex++;
} else {
DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st);
port->ie = 0;
ctl->driveCount--;
IoDeleteDevice(drv->deviceObject);
drv->deviceObject = (DEVICE_OBJECT*)0;
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
}
@@ -1106,7 +1118,6 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
ctl->pciInfo.Address.Function);
if (!ctl->pollingMode) {
KeIrqRegister(ctl->irq, AhciIsr);
DbgPrint("AHCI: ISR registered on IRQ %u (%s); controller drives=%d\n",
(unsigned)ctl->irq, ctl->usingMsi ? "MSI" : "INTx", ctl->driveCount);
} else {