diff --git a/OVMF/OVMF_VARS_4M_d.fd b/OVMF/OVMF_VARS_4M_d.fd index 2bf364d..830a905 100644 Binary files a/OVMF/OVMF_VARS_4M_d.fd and b/OVMF/OVMF_VARS_4M_d.fd differ diff --git a/build.bat b/build.bat index 3d58cc2..c6cda6b 100644 --- a/build.bat +++ b/build.bat @@ -23,6 +23,8 @@ mkdir "obj\init" 2>nul mkdir "obj\shell_win32" 2>nul mkdir "obj\shell_posix" 2>nul mkdir "obj\shell_native" 2>nul +mkdir "obj\async_apc" 2>nul +mkdir "obj\async_cancel" 2>nul mkdir "obj\doom" 2>nul mkdir "obj\video" 2>nul mkdir "bin" 2>nul @@ -120,8 +122,6 @@ echo Compiling init... for %%f in (%SRC%\init\*.c) do ( cl %CFLAGS% /c /Fo"obj\init\%%~nf.obj" "%%f" || exit /b 1 ) - -:: ---- Link init.exe ---- link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:_start ^ /OUT:"bin\init.exe" /PDB:"bin\init.pdb" bin\syscall.lib obj\libc.obj obj\init\init.obj || exit /b 1 @@ -130,42 +130,47 @@ echo Compiling shell_win32... for %%f in (%SRC%\shell_win32\*.c) do ( cl %CFLAGS% /c /Fo"obj\shell_win32\%%~nf.obj" "%%f" || exit /b 1 ) +link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ + /OUT:"bin\shell_win32.exe" /PDB:"bin\shell_win32.pdb" bin\win32.lib obj\libc.obj obj\shell_win32\*.obj || exit /b 1 :: ---- Compile shell_posix ---- echo Compiling shell_posix... for %%f in (%SRC%\shell_posix\*.c) do ( cl %CFLAGS% /c /Fo"obj\shell_posix\%%~nf.obj" "%%f" || exit /b 1 ) +link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ + /OUT:"bin\shell_posix.exe" /PDB:"bin\shell_posix.pdb" bin\posix.lib obj\libc.obj obj\shell_posix\*.obj || exit /b 1 :: ---- Compile shell_native ---- echo Compiling shell_native... for %%f in (%SRC%\shell_native\*.c) do ( cl %CFLAGS% /c /Fo"obj\shell_native\%%~nf.obj" "%%f" || exit /b 1 ) - -:: ---- Link shell_win32.exe ---- -echo Linking shell_win32.exe... -link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ - /OUT:"bin\shell_win32.exe" /PDB:"bin\shell_win32.pdb" bin\win32.lib obj\libc.obj obj\shell_win32\*.obj || exit /b 1 - -:: ---- Link shell_posix.exe ---- -echo Linking shell_posix.exe... -link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ - /OUT:"bin\shell_posix.exe" /PDB:"bin\shell_posix.pdb" bin\posix.lib obj\libc.obj obj\shell_posix\*.obj || exit /b 1 - -:: ---- Link shell_native.exe ---- -echo Linking shell_native.exe... link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\shell_native.exe" /PDB:"bin\shell_native.pdb" bin\syscall.lib obj\libc.obj obj\shell_native\*.obj || exit /b 1 +:: ---- Compile async_apc ---- +echo Compiling async_apc... +for %%f in (%SRC%\async_apc\*.c) do ( + cl %CFLAGS% /c /Fo"obj\async_apc\%%~nf.obj" "%%f" || exit /b 1 +) +link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ + /OUT:"bin\async_apc.exe" /PDB:"bin\async_apc.pdb" bin\syscall.lib obj\libc.obj obj\async_apc\*.obj || exit /b 1 + + +:: ---- Compile async_cancel ---- +echo Compiling async_cancel... +for %%f in (%SRC%\async_cancel\*.c) do ( + cl %CFLAGS% /c /Fo"obj\async_cancel\%%~nf.obj" "%%f" || exit /b 1 +) +link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ + /OUT:"bin\async_cancel.exe" /PDB:"bin\async_cancel.pdb" bin\syscall.lib obj\libc.obj obj\async_cancel\*.obj || exit /b 1 + :: ---- Compile doom ---- echo Compiling doom... for %%f in (%SRC%\doom\*.c) do ( cl %CFLAGS% /c /Fo"obj\doom\%%~nf.obj" "%%f" || exit /b 1 ) - -:: ---- Link doom.exe ---- -echo Linking doom.exe... link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\doom.exe" /PDB:"bin\doom.pdb" bin\syscall.lib obj\libc.obj obj\doom\*.obj || exit /b 1 @@ -174,9 +179,6 @@ echo Compiling video... for %%f in (%SRC%\video\*.c) do ( cl %CFLAGS% /c /Fo"obj\video\%%~nf.obj" "%%f" || exit /b 1 ) - -:: ---- Link video.exe ---- -echo Linking video.exe... link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\video.exe" /PDB:"bin\video.pdb" bin\win32.lib obj\libc.obj %SRC%\video\bink2w64.lib obj\video\*.obj || exit /b 1 @@ -187,6 +189,7 @@ set DPS=%TEMP%\ntlite_mnt_%RANDOM%.txt echo select vdisk file="%VHD%" echo attach vdisk echo select partition 1 + echo format fs=fat32 quick echo assign letter=Z echo exit ) > "%DPS%" @@ -220,6 +223,8 @@ copy /y "bin\init.exe" "%DEST%\system\" copy /y "bin\shell_win32.exe" "%DEST%\apps\" copy /y "bin\shell_posix.exe" "%DEST%\apps\" copy /y "bin\shell_native.exe" "%DEST%\apps\" +copy /y "bin\async_apc.exe" "%DEST%\apps\" +copy /y "bin\async_cancel.exe" "%DEST%\apps\" copy /y "bin\doom.exe" "%DEST%\apps\" copy /y "bin\video.exe" "%DEST%\apps\" copy /y "%SRC%\boot.ini" "%DEST%\" diff --git a/code/async_apc/async_apc.c b/code/async_apc/async_apc.c new file mode 100644 index 0000000..79e6264 --- /dev/null +++ b/code/async_apc/async_apc.c @@ -0,0 +1,148 @@ +#include +#include +#include +#include +#include + +#define EVENT_AUTO_RESET 0u + +static void print(const char* s) { + SysWriteConsole(s, strlen(s)); +} + +static void print_hex8(uint8_t v) { + static const char h[] = "0123456789ABCDEF"; + char b[3]; + b[0] = h[(v >> 4) & 0xF]; + b[1] = h[v & 0xF]; + b[2] = 0; + print(b); +} + +static void print_u64(uint64_t n) { + char buf[32]; + int i = 0; + if (!n) { print("0"); return; } + while (n && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (n % 10)); + n /= 10; + } + while (i) { + char c[2] = { buf[--i], 0 }; + print(c); + } +} + +typedef struct APC_TEST_STATE { + volatile uint32_t Delivered; + volatile uint32_t Status; + volatile uint64_t Information; +} APC_TEST_STATE; + +static void file_apc(void* context, SYS_IO_STATUS_BLOCK* iosb, uint64_t information) { + APC_TEST_STATE* st = (APC_TEST_STATE*)context; + st->Status = iosb ? iosb->Status : 0xFFFFFFFFu; + st->Information = information; + st->Delivered++; + SysDebugPrint("async_apc: user APC callback ran\n"); +} + +int main(void) { + print("async_apc: start\n"); + + uint32_t s = SysSetUserApcDispatcher((void*)SysUserApcDispatcher); + if (s != SYS_STATUS_SUCCESS) { + print("async_apc: dispatcher registration failed status="); + print_u64(s); + print("\n"); + SysExitProcess(1); + } + + uint32_t file = 0; + s = SysOpenFile("system/init.exe", &file); + if (s != SYS_STATUS_SUCCESS) { + print("async_apc: open failed status="); + print_u64(s); + print("\n"); + SysExitProcess(1); + } + + uint32_t event = 0; + s = SysEventCreate(EVENT_AUTO_RESET, 0, &event); + if (s != SYS_STATUS_SUCCESS) { + print("async_apc: event create failed status="); + print_u64(s); + print("\n"); + SysExitProcess(1); + } + + uint8_t* buffer = (uint8_t*)SysAlloc(512); + if (!buffer) { + print("async_apc: allocation failed\n"); + SysExitProcess(1); + } + memset(buffer, 0, 512); + + SYS_IO_STATUS_BLOCK iosb; + iosb.Status = 0; + iosb.Information = 0; + + APC_TEST_STATE state; + memset(&state, 0, sizeof(state)); + + SYS_ASYNC_FILE_REQUEST req; + memset(&req, 0, sizeof(req)); + req.FileHandle = file; + req.EventHandle = event; + req.Offset = 0; + req.Buffer = buffer; + req.Size = 512; + req.Iosb = &iosb; + req.ApcRoutine = file_apc; + req.ApcContext = &state; + + s = SysReadFileAsync(&req); + if (s != SYS_STATUS_PENDING) { + print("async_apc: submit failed status="); + print_u64(s); + print("\n"); + SysExitProcess(1); + } + + while (!state.Delivered) { + s = SysEventWait(event, 1); + print("async_apc: alertable wait returned "); + print_u64(s); + print("\n"); + if (s != SYS_STATUS_USER_APC && s != SYS_STATUS_SUCCESS && s != SYS_STATUS_PENDING) + break; + } + + print("async_apc: delivered="); + print_u64(state.Delivered); + print(" status="); + print_u64(state.Status); + print(" info="); + print_u64(state.Information); + print(" bytes="); + print_hex8(buffer[0]); print(" "); + print_hex8(buffer[1]); print(" "); + print_hex8(buffer[2]); print(" "); + print_hex8(buffer[3]); print("\n"); + + uint32_t ok = (state.Delivered == 1 && state.Status == SYS_STATUS_SUCCESS && + state.Information == 512 && buffer[0] == 'M' && buffer[1] == 'Z'); + + SysCloseHandle(event); + SysCloseHandle(file); + SysFree(buffer); + + if (!ok) { + print("async_apc: FAIL\n"); + SysExitProcess(1); + } + + print("async_apc: PASS\n"); + SysExitProcess(0); + return 0; +} diff --git a/code/async_cancel/async_cancel.c b/code/async_cancel/async_cancel.c new file mode 100644 index 0000000..92e4d52 --- /dev/null +++ b/code/async_cancel/async_cancel.c @@ -0,0 +1,108 @@ +#include +#include +#include +#include + +#define EVENT_AUTO_RESET 0u + +static void print(const char* s) { + SysWriteConsole(s, strlen(s)); +} + +static void print_u64(uint64_t v) { + char buf[32]; + int i = 0; + if (v == 0) { + print("0"); + return; + } + while (v && i < (int)sizeof(buf)) { + buf[i++] = (char)('0' + (v % 10)); + v /= 10; + } + while (i--) { + char c = buf[i]; + SysWriteConsole(&c, 1); + } +} + +int main(void) { + print("async_cancel: start\n"); + + uint32_t file = 0; + uint32_t event = 0; + uint32_t request = 0; + + uint32_t s = SysOpenFile("system/init.exe", &file); + if (s != SYS_STATUS_SUCCESS) { + print("async_cancel: open failed status="); print_u64(s); print("\n"); + SysExitProcess(1); + } + + s = SysEventCreate(EVENT_AUTO_RESET, 0, &event); + if (s != SYS_STATUS_SUCCESS) { + print("async_cancel: event create failed status="); print_u64(s); print("\n"); + SysExitProcess(1); + } + + uint8_t* buffer = (uint8_t*)SysAlloc(512); + if (!buffer) { + print("async_cancel: alloc failed\n"); + SysExitProcess(1); + } + memset(buffer, 0, 512); + + SYS_IO_STATUS_BLOCK iosb; + iosb.Status = 0; + iosb.Information = 0; + + SYS_ASYNC_FILE_REQUEST req; + memset(&req, 0, sizeof(req)); + req.FileHandle = file; + req.EventHandle = event; + req.Offset = 0; + req.Buffer = buffer; + req.Size = 512; + req.Iosb = &iosb; + req.RequestHandleOut = &request; + + s = SysReadFileAsync(&req); + if (s != SYS_STATUS_PENDING || request == 0) { + print("async_cancel: submit failed status="); print_u64(s); + print(" request="); print_u64(request); print("\n"); + SysExitProcess(1); + } + + uint32_t cs = SysCancelIo(request); + print("async_cancel: cancel returned "); print_u64(cs); print("\n"); + + while (iosb.Status == SYS_STATUS_PENDING) { + s = SysEventWait(event, 1); + print("async_cancel: wait returned "); print_u64(s); print("\n"); + if (s != SYS_STATUS_SUCCESS && s != SYS_STATUS_KERNEL_APC && s != SYS_STATUS_USER_APC && s != SYS_STATUS_ALERTED) + break; + } + + print("async_cancel: iosb status="); print_u64(iosb.Status); + print(" info="); print_u64(iosb.Information); print("\n"); + + // If the worker had already started, cancellation is best-effort and the + // request may complete successfully. If it was still queued, it must report + // SYS_STATUS_CANCELLED. Both outcomes are valid; SYS_STATUS_PENDING is not. + uint32_t ok = (iosb.Status == SYS_STATUS_CANCELLED) || + (iosb.Status == SYS_STATUS_SUCCESS && iosb.Information == 512); + + SysCloseHandle(request); + SysCloseHandle(event); + SysCloseHandle(file); + SysFree(buffer); + + if (!ok) { + print("async_cancel: FAIL\n"); + SysExitProcess(1); + } + + print("async_cancel: PASS\n"); + SysExitProcess(0); + return 0; +} diff --git a/code/boot.ini b/code/boot.ini index b6e55ca..8c0cd3c 100644 --- a/code/boot.ini +++ b/code/boot.ini @@ -1,4 +1,5 @@ [kernel] +; bitmask: 1 = debug prints, 2 = self check debug = 1 [init] commandline = apps/shell_native.exe diff --git a/code/boot/main.c b/code/boot/main.c index 9012b33..caa946a 100644 --- a/code/boot/main.c +++ b/code/boot/main.c @@ -859,7 +859,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable Print(L"Found RSDP at: 0x%llx\n", bi->RsdpPhysical); bi->PhysicalMapBase = PHYS_MAP_BASE; - bi->DebugEnabled = (UINT32)ps.debug; + bi->Debug = (UINT32)ps.debug; Print(L"9\n"); diff --git a/code/doom/doom.c b/code/doom/doom.c index 69f4388..0c847b2 100644 --- a/code/doom/doom.c +++ b/code/doom/doom.c @@ -24,28 +24,19 @@ static void doom_print_wrapper(const char* msg) // --------------------------------------------------------------------------- // doom_malloc_fn / doom_free_fn -// Prepend size tracking metadata to satisfy SysFree's size requirement. // --------------------------------------------------------------------------- static void* doom_malloc_wrapper(int size) { - size_t total_size = size + sizeof(size_t); - void* p = SysAlloc(total_size); + void* p = SysAlloc(size); if (!p) SysExitProcess(1); - - // Store total allocated size at the start of the block - *(size_t*)p = total_size; - - // Return pointer past the size prefix - return (void*)((size_t*)p + 1); + return p; } static void doom_free_wrapper(void* ptr) { if (!ptr) return; - - size_t* real_p = (size_t*)ptr - 1; - size_t total_size = *real_p; - SysFree(real_p, total_size); + + SysFree(ptr); } // --------------------------------------------------------------------------- @@ -314,7 +305,7 @@ static int build_argv_from_command_line(const char* cmdline, char* storage, size // --------------------------------------------------------------------------- int main(void) { - const char* cmdline = SysGetCommandLine(); + const char* cmdline = SysGetPeb()->CommandLine; SysWriteConsole("doom: starting\n", 15); diff --git a/code/drivers/ahci/ahci.c b/code/drivers/ahci/ahci.c index e9a02a7..b626f7d 100644 --- a/code/drivers/ahci/ahci.c +++ b/code/drivers/ahci/ahci.c @@ -1,1186 +1,1189 @@ -// ahci.c - AHCI driver, interrupt-driven with per-drive async IRP queue. -// Supports SATA drives only (no ATAPI). Registers each found drive with the -// HAL disk subsystem via IoRegisterDisk(). -// -// Memory model: -// MmAllocateContiguous(n) returns a physical address for n*4K pages. -// hal->PhysToVirt(phys) converts to kernel virtual address. -// DMA buffers are bounce-buffered (allocated via MmAllocateContiguous) to -// avoid needing a VirtToPhys conversion for arbitrary kernel pointers. -// -// Async I/O model (IRP path): -// Each drive has a dedicated kernel worker thread (AhciWorker) that blocks -// on a per-drive counting semaphore. Callers submit IRPs via AhciSubmitIrp, -// which enqueues the IRP and signals the semaphore. The worker dequeues -// one IRP at a time, runs the synchronous hardware transfer (blocking the -// worker, NOT the submitting thread), then writes irp->Status and fires -// irp->CompletionEvent / irp->CompletionFn. -// -// This gives true async I/O from the caller's perspective: IoSubmitIrp() -// returns STATUS_PENDING immediately. Multiple callers can have outstanding -// requests against different drives concurrently without stalling each other. -// -// NCQ extension (future work - see "NCQ TODO" comments): -// To saturate modern SSDs, replace the serialised single-slot model with -// Native Command Queuing: -// • Allocate per-slot bounce buffers (NUM_NCQ_SLOTS × SLOT_BOUNCE_BYTES). -// • Use ATA_CMD_READ_FPDMA_QUEUED / WRITE_FPDMA_QUEUED. -// • Encode the slot tag in FIS.countl bits 7:3. -// • Issue via port->sact (NCQ active) rather than port->ci. -// • In AhciIsr: completed slots = prev_sact ^ port->sact; look up -// slot_irp[slot] and fire each completion from within the ISR. -// • The worker becomes a pure dispatcher: grab a free slot, build the -// command, issue it, return immediately. No per-slot KeWaitForEvent. -// -// Interrupt model (unchanged): -// GHC.IE and per-port PORT_IE are enabled after each port is initialised. -// AhciIsr fires on the PCI IRQ line and signals the per-drive completionEvent. -// port_issue_cmd blocks the worker thread on KeWaitForEvent instead of -// spinning on CI. port_issue_cmd_poll is kept for early init (IDENTIFY) -// before the scheduler is live. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// --------------------------------------------------------------------------- -typedef enum { - FIS_TYPE_REG_H2D = 0x27, - FIS_TYPE_REG_D2H = 0x34, - FIS_TYPE_DMA_ACT = 0x39, - FIS_TYPE_DMA_SETUP = 0x41, - FIS_TYPE_DATA = 0x46, - FIS_TYPE_BIST = 0x58, - FIS_TYPE_PIO_SETUP = 0x5F, - FIS_TYPE_DEV_BITS = 0xA1, -} FIS_TYPE; - -#pragma pack(push, 1) -// Register FIS - Host to Device (H2D) -typedef struct { - uint8_t fis_type; // FIS_TYPE_REG_H2D - uint8_t pmport : 4; - uint8_t rsv0 : 3; - uint8_t c : 1; // 1 = Command, 0 = Control - uint8_t command; - uint8_t featurel; - // DW1 - uint8_t lba0, lba1, lba2; - uint8_t device; - // DW2 - uint8_t lba3, lba4, lba5; - uint8_t featureh; - // DW3 - uint8_t countl, counth; - uint8_t icc; - uint8_t control; - // DW4 - uint8_t rsv1[4]; -} FIS_REG_H2D; - -// Register FIS - Device to Host (D2H) -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 2; - uint8_t i : 1; - uint8_t rsv1 : 1; - uint8_t status; - uint8_t error; - uint8_t lba0, lba1, lba2; - uint8_t device; - uint8_t lba3, lba4, lba5; - uint8_t rsv2; - uint8_t countl, counth; - uint8_t rsv3[2]; - uint8_t rsv4[4]; -} FIS_REG_D2H; - -// DMA Setup FIS -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 1; - uint8_t d : 1; - uint8_t i : 1; - uint8_t a : 1; - uint8_t rsved[2]; - uint64_t DMAbufferID; - uint32_t rsvd; - uint32_t DMAbufOffset; - uint32_t TransferCount; - uint32_t resvd; -} FIS_DMA_SETUP; - -// PIO Setup FIS -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 1; - uint8_t d : 1; - uint8_t i : 1; - uint8_t rsv1 : 1; - uint8_t status; - uint8_t error; - uint8_t lba0, lba1, lba2; - uint8_t device; - uint8_t lba3, lba4, lba5; - uint8_t rsv2; - uint8_t countl, counth; - uint8_t rsv3; - uint8_t e_status; - uint16_t tc; - uint8_t rsv4[2]; -} FIS_PIO_SETUP; - -// Set Device Bits FIS -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 2; - uint8_t i : 1; - uint8_t n : 1; - uint8_t statusl : 4; - uint8_t rsv1 : 4; - uint8_t statush : 4; - uint8_t rsv2 : 4; - uint8_t error; - uint32_t protocol; -} FIS_DEV_BITS; - -// --------------------------------------------------------------------------- -// HBA structures (volatile - MMIO) -// --------------------------------------------------------------------------- - -// Received FIS structure in host memory -typedef volatile struct { - FIS_DMA_SETUP dsfis; // 0x00 DMA Setup FIS - uint8_t pad0[4]; - FIS_PIO_SETUP psfis; // 0x20 PIO Setup FIS - uint8_t pad1[12]; - FIS_REG_D2H rfis; // 0x40 Register D2H FIS - uint8_t pad2[4]; - FIS_DEV_BITS sdbfis; // 0x58 Set Device Bits FIS - uint8_t ufis[64]; // 0x60 Unknown FIS - uint8_t rsv[0x100-0xA0]; -} HBA_FIS; - -// Command List header (one entry = one slot) -typedef struct { - // DW0 - uint8_t cfl : 5; // Command FIS length in DWORDs (2..16) - uint8_t a : 1; // ATAPI - uint8_t w : 1; // Write (H2D=1, D2H=0) - uint8_t p : 1; // Prefetchable - uint8_t r : 1; // Reset - uint8_t b : 1; // BIST - uint8_t c : 1; // Clear busy on R_OK - uint8_t rsv0 : 1; - uint8_t pmp : 4; // Port multiplier port - uint16_t prdtl; // PRDT length in entries - // DW1 - volatile uint32_t prdbc; // PRDT byte count transferred - // DW2,3 - uint32_t ctba; // Command table base address (128-byte aligned) - uint32_t ctbau; // Upper 32 bits - // DW4-7 - uint32_t rsv1[4]; -} HBA_CMD_HEADER; - -// Physical Region Descriptor Table entry -typedef struct { - uint32_t dba; // Data base address - uint32_t dbau; // Upper 32 bits - uint32_t rsv0; - uint32_t dbc : 22; // Byte count (0-based, so actual = dbc+1) - uint32_t rsv1 : 9; - uint32_t i : 1; // Interrupt on completion -} HBA_PRDT_ENTRY; - -// Command Table -typedef struct { - uint8_t cfis[64]; // Command FIS (up to 64 bytes) - uint8_t acmd[16]; // ATAPI command (12 or 16 bytes) - uint8_t rsv[48]; // Reserved - HBA_PRDT_ENTRY prdt_entry[1]; // PRDT entries (variable length) -} HBA_CMD_TBL; -#pragma pack(pop) - -// Port registers -typedef volatile struct { - uint32_t clb; // 0x00 Command list base (1K-aligned) - uint32_t clbu; // 0x04 Command list base upper - uint32_t fb; // 0x08 FIS base (256-byte aligned) - uint32_t fbu; // 0x0C FIS base upper - uint32_t is; // 0x10 Interrupt status - uint32_t ie; // 0x14 Interrupt enable - uint32_t cmd; // 0x18 Command and status - uint32_t rsv0; // 0x1C Reserved - uint32_t tfd; // 0x20 Task file data - uint32_t sig; // 0x24 Signature - uint32_t ssts; // 0x28 SATA status - uint32_t sctl; // 0x2C SATA control - uint32_t serr; // 0x30 SATA error - uint32_t sact; // 0x34 SATA active - uint32_t ci; // 0x38 Command issue - uint32_t sntf; // 0x3C SATA notification - uint32_t fbs; // 0x40 FIS-based switch control - uint32_t rsv1[11]; - uint32_t vendor[4]; -} HBA_PORT; - -// Generic Host Control registers (starts at ABAR) -typedef volatile struct { - uint32_t cap; // 0x00 Host capability - uint32_t ghc; // 0x04 Global host control - uint32_t is; // 0x08 Interrupt status - uint32_t pi; // 0x0C Ports implemented - uint32_t vs; // 0x10 Version - uint32_t ccc_ctl; // 0x14 CCC control - uint32_t ccc_pts; // 0x18 CCC ports - uint32_t em_loc; // 0x1C Enclosure mgmt location - uint32_t em_ctl; // 0x20 Enclosure mgmt control - uint32_t cap2; // 0x24 Extended capabilities - uint32_t bohc; // 0x28 BIOS/OS handoff control - uint8_t rsv[0xA0-0x2C]; - uint8_t vendor[0x100-0xA0]; - HBA_PORT ports[32]; // 0x100 Port control registers -} HBA_MEM; - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -// PCI -#define PCI_AHCI_CLASS 0x01 -#define PCI_AHCI_SUBCLASS 0x06 -#define PCI_CMD_REG 0x04 -#define PCI_BAR5_REG 0x24 -#define PCI_CMD_BUSMASTER (1 << 2) -#define PCI_CMD_MEMSPACE (1 << 1) - -// GHC register bits -#define GHC_AE (1u << 31) // AHCI Enable -#define GHC_IE (1u << 1) // Interrupt Enable -#define GHC_HR (1u << 0) // HBA Reset - -// BOHC register bits -#define BOHC_BOS (1u << 0) // BIOS-owned semaphore -#define BOHC_OOS (1u << 1) // OS-owned semaphore -#define BOHC_BB (1u << 4) // BIOS Busy - -// CAP2 bits -#define CAP2_BOH (1u << 0) // BIOS/OS handoff capable - -// Port CMD register bits -#define PORT_CMD_ST (1u << 0) // Start -#define PORT_CMD_FRE (1u << 4) // FIS Receive Enable -#define PORT_CMD_FR (1u << 14) // FIS Receive Running -#define PORT_CMD_CR (1u << 15) // Command List Running -#define PORT_CMD_POD (1u << 2) // Power On Device -#define PORT_CMD_SUD (1u << 1) // Spin-Up Device -#define PORT_CMD_ICC_ACTIVE (0x1u << 28) - -// Port IS register bits (interrupt status) -#define PORT_IS_TFES (1u << 30) // Task File Error Status -#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error -#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error -#define PORT_IS_IFS (1u << 27) // Interface Fatal Error - -// Port SSTS register -#define PORT_DET_PRESENT 0x3 -#define PORT_IPM_ACTIVE 0x1 - -// Port signatures -#define SATA_SIG_ATA 0x00000101 -#define SATA_SIG_ATAPI 0xEB140101 -#define SATA_SIG_SEMB 0xC33C0101 -#define SATA_SIG_PM 0x96690101 - -// ATA commands -#define ATA_CMD_READ_DMA_EX 0x25 -#define ATA_CMD_WRITE_DMA_EX 0x35 -#define ATA_CMD_IDENTIFY 0xEC -#define ATA_CMD_FLUSH_EXT 0xEA - -// ATA status bits -#define ATA_SR_BUSY 0x80 -#define ATA_SR_DRQ 0x08 -#define ATA_SR_ERR 0x01 - -// IDENTIFY word offsets -#define ATA_IDENT_W83 83 // bit 10 = 48-bit LBA supported -#define ATA_IDENT_W100 100 // 48-bit LBA sector count (4 words) -#define ATA_IDENT_W60 60 // 28-bit LBA sector count (2 words) - -// Layout constants (per port, single contiguous physical allocation) -// Offset 0: Command List 1024 bytes (32 × 32-byte headers) -// Offset 1024: Received FIS 256 bytes -// Offset 2048: Command Tables 32 × 256 bytes = 8192 bytes -// Total: 10240 bytes → 3 pages (12288 bytes) -#define PORT_PAGES_STRUCT 3 -#define PORT_OFF_CMDLIST 0 -#define PORT_OFF_FIS 1024 -#define PORT_OFF_CMDTBLS 2048 -#define PORT_CMDTBL_SIZE 256 // 64 CFIS + 16 ACMD + 48 rsv + 8×16 PRDT -#define PORT_PRDT_COUNT 8 // PRDT entries per command table - -// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages -#define BOUNCE_SECTORS 128 -#define BOUNCE_BYTES (BOUNCE_SECTORS * 512) -#define BOUNCE_PAGES (BOUNCE_BYTES / 4096) - -// Polling timeout (spin iterations) -#define POLL_TIMEOUT 1000000 - -#define MAX_AHCI_DRIVES 8 -#define NUM_CMD_SLOTS 32 - -// Map MMIO physical range into the physical window as uncacheable. -// Must be called before any pointer dereference via PhysToVirt(). -// AHCI MMIO layout: 0x100 GHC + 32*0x80 ports = 0x1100 bytes → 2 pages minimum. -// We map 8 pages (32KB) without probing the BAR size. -#define AHCI_MMIO_PAGES 8 - -// --------------------------------------------------------------------------- -// Driver-global state -// --------------------------------------------------------------------------- - -static const HAL_API *g_hal; -static HBA_MEM *g_hba; // HBA MMIO base (set during init) -static uint8_t g_irq; // PCI IRQ line for this HBA - -// Per-drive context -typedef struct { - HBA_PORT *port; // MMIO port registers (virtual) - uint64_t sectorCount; // Total LBA sectors on device - uint8_t portNo; // Physical port index (for ISR dispatch) - - // Structures block (command list + FIS + cmd tables) - uint64_t structPhys; // Physical base - void *structVirt; // Virtual base - - // Bounce buffer for DMA data - uint64_t bouncePhys; - void *bounceVirt; - - // Signaled by the ISR when the port raises an interrupt. - // Auto-reset: one Wait consumes one completion. - EVENT *completionEvent; - - // Last IS snapshot captured by the ISR (for error checking post-wait). - volatile uint32_t lastIs; - - // ----------------------------------------------------------------------- - // IRP queue - async dispatch - // ----------------------------------------------------------------------- - // AhciWorker blocks here waiting for work. AhciSubmitIrp enqueues an IRP - // and signals the semaphore to wake the worker. - IRP_QUEUE irpQueue; - THREAD *workerThread; - - // NCQ TODO: for multi-slot NCQ, add: - // uint32_t slotMask; // bitmask of in-flight slots - // IRP *slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP - // uint64_t slotBouncePhys[NUM_CMD_SLOTS]; - // void *slotBounceVirt[NUM_CMD_SLOTS]; - // volatile uint32_t prevSact; // previous port->sact snapshot -} AHCI_DRIVE; - -static AHCI_DRIVE g_drives[MAX_AHCI_DRIVES]; -static int g_driveCount; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -static inline void *phys_to_virt(uint64_t phys) { - return g_hal->PhysToVirt(phys); -} - -static KSTATUS ahci_map_mmio(uint64_t physBase) { - for (int i = 0; i < AHCI_MMIO_PAGES; i++) { - uint64_t phys = physBase + (uint64_t)i * 4096; - uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys); - uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; - KSTATUS st = MmMapPage(virt, phys, flags); - // Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags - // is possible for the first page if the bootloader was overly generous. - // Use MmSetPageProtect to force NOCACHE in that case. - if (st == STATUS_ALREADY_EXISTS) { - st = MmSetPageProtect(virt, 4096, flags); - } - if (!KSUCCESS(st)) { - DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%llx, st=0x%x)\n", - i, (unsigned long long)phys, st); - return st; - } - } - return STATUS_SUCCESS; -} - -// Accessors for port structure memory at known offsets from structVirt/Phys -static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) { - return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST); -} - -static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) { - return drv->structPhys + PORT_OFF_CMDLIST; -} - -static inline uint64_t fis_phys(AHCI_DRIVE *drv) { - return drv->structPhys + PORT_OFF_FIS; -} - -static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) { - return (HBA_CMD_TBL *)((uint8_t *)drv->structVirt - + PORT_OFF_CMDTBLS - + (uint32_t)slot * PORT_CMDTBL_SIZE); -} - -static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { - return drv->structPhys + PORT_OFF_CMDTBLS - + (uint64_t)slot * PORT_CMDTBL_SIZE; -} - -// --------------------------------------------------------------------------- -// Port engine start / stop (polling safe) -// --------------------------------------------------------------------------- - -static void port_stop(HBA_PORT *port) { - // Clear ST (bit 0) - port->cmd &= ~PORT_CMD_ST; - // Wait for CR (bit 15) to clear (command list DMA stops) - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(port->cmd & PORT_CMD_CR)) break; - } - // Clear FRE (bit 4) - port->cmd &= ~PORT_CMD_FRE; - // Wait for FR (bit 14) to clear - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(port->cmd & PORT_CMD_FR)) break; - } -} - -static void port_start(HBA_PORT *port) { - // Wait until CR is clear before setting ST - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(port->cmd & PORT_CMD_CR)) break; - } - port->cmd |= PORT_CMD_FRE; - port->cmd |= PORT_CMD_ST; -} - -// --------------------------------------------------------------------------- -// Port memory initialisation -// --------------------------------------------------------------------------- - -static KSTATUS port_init_memory(AHCI_DRIVE *drv) { - uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; - - drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT); - if (!drv->structPhys) return STATUS_NO_MEMORY; - drv->structVirt = g_hal->PhysToVirt(drv->structPhys); - // Force NOCACHE on DMA descriptor memory - MmSetPageProtect((uintptr_t)drv->structVirt, - PORT_PAGES_STRUCT * 4096, flags); - memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096); - - drv->bouncePhys = MmAllocateContiguous(BOUNCE_PAGES); - if (!drv->bouncePhys) return STATUS_NO_MEMORY; - drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys); - // Bounce buffer: NOCACHE to ensure CPU writes reach physical memory - // before the HBA's DMA engine reads them - MmSetPageProtect((uintptr_t)drv->bounceVirt, - BOUNCE_PAGES * 4096, flags); - memset(drv->bounceVirt, 0, BOUNCE_BYTES); - - HBA_PORT *port = drv->port; - - port_stop(port); - - // Set command list base - port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF); - port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32); - - // Set FIS base - port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF); - port->fbu = (uint32_t)(fis_phys(drv) >> 32); - - // Initialise all 32 command headers, pointing each to its command table - HBA_CMD_HEADER *hdrs = cmd_list(drv); - for (int i = 0; i < NUM_CMD_SLOTS; i++) { - hdrs[i].prdtl = PORT_PRDT_COUNT; - uint64_t tbl = cmd_tbl_phys(drv, i); - hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF); - hdrs[i].ctbau = (uint32_t)(tbl >> 32); - } - - // Clear any stale interrupts then start the port engine - port->serr = (uint32_t)-1; - port->is = (uint32_t)-1; - - port_start(port); - - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// Issue a command - polling fallback (used during early init / IDENTIFY only, -// before the scheduler and interrupt handler are live). -// Returns STATUS_SUCCESS, STATUS_DEVICE_ERROR, or STATUS_TIMEOUT. -// --------------------------------------------------------------------------- - -static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) { - HBA_PORT *port = drv->port; - - // Clear pending interrupts before issuing - port->is = (uint32_t)-1; - - // Wait for port to be idle (not BSY or DRQ) - int spin = 0; - while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) - spin++; - if (spin == POLL_TIMEOUT) { - DbgPrint("AHCI: port hung (pre-issue)\n"); - return STATUS_TIMEOUT; - } - - // Issue slot 0 - port->ci = (1u << 0); - - // Poll until slot 0 clears - while (true) { - if (!(port->ci & (1u << 0))) break; - if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { - DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", - port->is, port->tfd, port->serr); - port->serr = (uint32_t)-1; - port->is = (uint32_t)-1; - return STATUS_DEVICE_ERROR; - } - } - - if (port->is & PORT_IS_TFES) { - DbgPrint("AHCI: TFES after completion TFD=0x%x\n", port->tfd); - port->is = (uint32_t)-1; - return STATUS_DEVICE_ERROR; - } - - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// ISR - called from KeIrqDispatch on the AHCI PCI IRQ line. -// Walks all registered drives, checks if their port raised an interrupt, -// captures IS, clears it, and signals the per-drive completion event. -// --------------------------------------------------------------------------- - -static void AhciIsr(void) { - if (!g_hba) return; - - // GHC.IS tells us which ports are pending. - uint32_t pending = g_hba->is; - if (!pending) return; - - for (int i = 0; i < g_driveCount; i++) { - AHCI_DRIVE *drv = &g_drives[i]; - uint32_t portBit = (1u << drv->portNo); - if (!(pending & portBit)) continue; - - // Capture and clear port IS before signalling - the waiter reads - // lastIs to check for errors without racing the next command. - uint32_t portIs = drv->port->is; - drv->port->is = portIs; // W1C - write back to clear - drv->lastIs = portIs; - - // Clear the HBA-level bit. - g_hba->is = portBit; - - if (drv->completionEvent) - KeSignalEvent(drv->completionEvent); - } -} - -// --------------------------------------------------------------------------- -// Issue a command - interrupt-driven (runtime path). -// Caller has already built slot 0 via build_rw_cmd / port_identify. -// Blocks the current thread until the ISR signals completionEvent. -// --------------------------------------------------------------------------- - -static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { - HBA_PORT *port = drv->port; - - // Clear stale IS and snapshot before issuing so we don't mis-read a - // leftover value from a prior command. - port->is = (uint32_t)-1; - drv->lastIs = 0; - - // Wait for port idle - int spin = 0; - while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) - spin++; - if (spin == POLL_TIMEOUT) { - DbgPrint("AHCI: port hung (pre-issue)\n"); - return STATUS_TIMEOUT; - } - - // Issue slot 0, then block on the event. - port->ci = (1u << 0); - KeWaitForEvent(drv->completionEvent); - - // The ISR has cleared IS - check for errors in the snapshot. - uint32_t is = drv->lastIs; - if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { - DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", - is, port->tfd, port->serr); - port->serr = (uint32_t)-1; - return STATUS_DEVICE_ERROR; - } - - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// Build a standard DMA read or write command in slot 0 -// --------------------------------------------------------------------------- - -static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, - uint64_t bufPhys, bool write) { - HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0 - HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); - - memset(tbl, 0, PORT_CMDTBL_SIZE); - - // How many PRDT entries we need (each covers up to 8K = 16 sectors) - uint32_t prdtl = (sectors + 15) / 16; - if (prdtl > PORT_PRDT_COUNT) prdtl = PORT_PRDT_COUNT; // capped (see caller) - - hdr->cfl = sizeof(FIS_REG_H2D) / 4; - hdr->w = write ? 1 : 0; - hdr->prdtl = (uint16_t)prdtl; - hdr->prdbc = 0; - - // Fill PRDT entries - uint32_t remaining = sectors; - uint64_t buf = bufPhys; - for (uint32_t i = 0; i < prdtl; i++) { - uint32_t sects = (remaining > 16) ? 16 : remaining; - uint32_t bytes = sects * 512; - - tbl->prdt_entry[i].dba = (uint32_t)(buf & 0xFFFFFFFF); - tbl->prdt_entry[i].dbau = (uint32_t)(buf >> 32); - tbl->prdt_entry[i].dbc = bytes - 1; // hardware adds 1 - tbl->prdt_entry[i].i = (i == prdtl - 1) ? 1 : 0; - - buf += bytes; - remaining -= sects; - } - - // Build H2D FIS - FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis; - memset(fis, 0, sizeof(*fis)); - fis->fis_type = FIS_TYPE_REG_H2D; - fis->c = 1; - fis->command = write ? ATA_CMD_WRITE_DMA_EX : ATA_CMD_READ_DMA_EX; - fis->device = 1u << 6; // LBA mode - - fis->lba0 = (uint8_t)(lba); - fis->lba1 = (uint8_t)(lba >> 8); - fis->lba2 = (uint8_t)(lba >> 16); - fis->lba3 = (uint8_t)(lba >> 24); - fis->lba4 = (uint8_t)(lba >> 32); - fis->lba5 = (uint8_t)(lba >> 40); - - fis->countl = (uint8_t)(sectors & 0xFF); - fis->counth = (uint8_t)(sectors >> 8); -} - -// --------------------------------------------------------------------------- -// IDENTIFY command - fills a 512-byte buffer and returns it -// --------------------------------------------------------------------------- - -static KSTATUS port_identify(AHCI_DRIVE *drv) { - // Reuse bounce buffer for IDENTIFY data (512 bytes) - memset(drv->bounceVirt, 0, 512); - - HBA_CMD_HEADER *hdr = cmd_list(drv); - HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); - - memset(tbl, 0, PORT_CMDTBL_SIZE); - - hdr->cfl = sizeof(FIS_REG_H2D) / 4; - hdr->w = 0; - hdr->prdtl = 1; - hdr->prdbc = 0; - - tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF); - tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32); - tbl->prdt_entry[0].dbc = 512 - 1; - tbl->prdt_entry[0].i = 1; - - FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis; - memset(fis, 0, sizeof(*fis)); - fis->fis_type = FIS_TYPE_REG_H2D; - fis->c = 1; - fis->command = ATA_CMD_IDENTIFY; - fis->device = 0; - - // Poll: IDENTIFY runs before the scheduler and IRQ handler are live. - KSTATUS st = port_issue_cmd_poll(drv); - if (!KSUCCESS(st)) return st; - - // Parse sector count from IDENTIFY data - const uint16_t *id = (const uint16_t *)drv->bounceVirt; - - if (id[ATA_IDENT_W83] & (1u << 10)) { - // 48-bit LBA supported - drv->sectorCount = (uint64_t)id[ATA_IDENT_W100] - | ((uint64_t)id[ATA_IDENT_W100 + 1] << 16) - | ((uint64_t)id[ATA_IDENT_W100 + 2] << 32) - | ((uint64_t)id[ATA_IDENT_W100 + 3] << 48); - } else { - // 28-bit LBA - drv->sectorCount = (uint64_t)id[ATA_IDENT_W60] - | ((uint64_t)id[ATA_IDENT_W60 + 1] << 16); - } - - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// HAL_DISK callbacks (forwarded through per-drive context) -// We need one closure per drive. Since C has no closures, we use a small -// trampoline table indexed by drive number. -// --------------------------------------------------------------------------- - -// Thin wrappers; the drive index is baked in at registration time by using -// separately named static functions. We generate MAX_AHCI_DRIVES of them with -// a macro to keep the code DRY without requiring heap-allocated closures. - -// Core read logic -static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba, - uint32_t count, void *buf) { - uint8_t *out = (uint8_t *)buf; - - while (count > 0) { - uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; - - build_rw_cmd(drv, lba, batch, drv->bouncePhys, false); - KSTATUS st = port_issue_cmd(drv); - if (!KSUCCESS(st)) return st; - - memcpy(out, drv->bounceVirt, (size_t)batch * 512); - out += (size_t)batch * 512; - lba += batch; - count -= batch; - } - return STATUS_SUCCESS; -} - -// Core write logic -static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba, - uint32_t count, const void *buf) { - const uint8_t *in = (const uint8_t *)buf; - - while (count > 0) { - uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; - - memcpy(drv->bounceVirt, in, (size_t)batch * 512); - build_rw_cmd(drv, lba, batch, drv->bouncePhys, true); - KSTATUS st = port_issue_cmd(drv); - if (!KSUCCESS(st)) return st; - - in += (size_t)batch * 512; - lba += batch; - count -= batch; - } - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// HAL_DISK callbacks - context pointer replaces trampolines. -// Ctx is the AHCI_DRIVE* stored in DISK_DRIVER.Ctx; the dispatch layer in -// hal_disk.c passes it as the first argument to every call. -// --------------------------------------------------------------------------- - -static KSTATUS AhciRead(void *ctx, uint64_t lba, uint32_t count, void *buf) { - return do_read((AHCI_DRIVE *)ctx, lba, count, buf); -} - -static KSTATUS AhciWrite(void *ctx, uint64_t lba, uint32_t count, const void *buf) { - return do_write((AHCI_DRIVE *)ctx, lba, count, buf); -} - -// --------------------------------------------------------------------------- -// Async IRP dispatch -// --------------------------------------------------------------------------- - -// AhciSubmitIrp - called by IoSubmitIrp(). -// Enqueues the IRP and wakes the per-drive worker thread. -// Returns STATUS_PENDING immediately; the caller waits on irp->CompletionEvent -// or irp->CompletionFn. -static KSTATUS AhciSubmitIrp(void *ctx, IRP *irp) { - AHCI_DRIVE *drv = (AHCI_DRIVE *)ctx; - irp->Status = STATUS_FAILURE; // sentinel: I/O has not completed yet - IrpQueueEnqueue(&drv->irpQueue, irp); - return STATUS_PENDING; -} - -// AhciWorker - kernel thread, one per drive. -// Dequeues IRPs and executes them one at a time using the existing -// single-slot interrupt-driven path. Completion fires the IRP's event -// and/or callback so the original caller can observe the result without -// ever having blocked in a driver call. -// -// NCQ TODO: replace the serialised loop with a multi-slot dispatcher: -// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP. -// 2. Pick the oldest IRP from the queue. -// 3. Grab a free bit in drv->slotMask (under the slot spinlock). -// 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA), -// record slotIrp[slot] = irp, and issue via port->sact + port->ci. -// 5. Return immediately - the ISR drives all completions. -static void AhciWorker(uintptr_t arg) { - AHCI_DRIVE *drv = (AHCI_DRIVE *)arg; - - while (true) { - // Block until AhciSubmitIrp signals a new IRP. - KeWaitSemaphore(drv->irpQueue.Semaphore); - - IRP *irp = IrpQueueDequeue(&drv->irpQueue); - if (!irp) KePanic("AhciWorker: semaphore fired but queue is empty - IRP accounting bug\n"); - - bool write = !!(irp->Flags & IRP_FLAG_WRITE); - KSTATUS st; - if (write) - st = do_write(drv, irp->Lba, irp->SectorCount, irp->Buffer); - else - st = do_read (drv, irp->Lba, irp->SectorCount, irp->Buffer); - - // Publish result then fire completion hooks. - irp->Status = st; - if (irp->CompletionEvent) KeSignalEvent(irp->CompletionEvent); - if (irp->CompletionFn) irp->CompletionFn(irp, irp->CompletionCtx); - } -} - -// --------------------------------------------------------------------------- -// AHCI controller initialisation -// --------------------------------------------------------------------------- - -static KSTATUS hba_reset(HBA_MEM *hba) { - // BIOS/OS handoff (spec §10.6.3) - if (hba->cap2 & CAP2_BOH) { - hba->bohc |= BOHC_OOS; - // Wait for BIOS to clear BOS - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(hba->bohc & BOHC_BOS)) break; - } - // Give the BIOS another moment to clear Busy - for (int i = 0; i < 800000; i++) { - if (!(hba->bohc & BOHC_BB)) break; - } - } - - // Enable AHCI mode before reset so the reset actually takes effect - hba->ghc |= GHC_AE; - - // HBA reset - hba->ghc |= GHC_HR; - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(hba->ghc & GHC_HR)) break; - } - if (hba->ghc & GHC_HR) { - DbgPrint("AHCI: HBA reset timed out\n"); - return STATUS_TIMEOUT; - } - - // Re-enable AHCI mode (reset cleared it), then enable global interrupts. - // GHC.IE lets the HBA assert its PCI interrupt line when a port fires. - hba->ghc |= GHC_AE | GHC_IE; - return STATUS_SUCCESS; -} - -static bool port_has_sata_drive(HBA_PORT *port) { - uint32_t det = port->ssts & 0x0F; - uint32_t ipm = (port->ssts >> 8) & 0x0F; - if (det != PORT_DET_PRESENT) return false; - if (ipm != PORT_IPM_ACTIVE) return false; - return port->sig == SATA_SIG_ATA; // SATA disk only (no ATAPI for now) -} - -// Before memory init: only check physical link. Signature is not yet valid. -static bool port_link_up(HBA_PORT *port) { - uint32_t det = port->ssts & 0x0F; - uint32_t ipm = (port->ssts >> 8) & 0x0F; - return (det == PORT_DET_PRESENT && ipm == PORT_IPM_ACTIVE); -} - -// After COMRESET + FIS receive: check signature for SATA disk. -static bool port_is_sata_disk(HBA_PORT *port) { - return port->sig == SATA_SIG_ATA; -} - -static void ahci_delay_ms(uint32_t ms) { - for (uint32_t i = 0; i < ms * 1000; i++) - __outbyte(0x80, 0); -} - -// Issue a COMRESET on the port, wait for link re-establishment and D2H FIS. -// FRE must already be set (port_init_memory does this) so the HBA can -// capture the signature FIS the device sends after the reset. -static KSTATUS port_comreset(AHCI_DRIVE *drv) { - HBA_PORT *port = drv->port; - - port->cmd &= ~PORT_CMD_ST; - for (int i = 0; i < POLL_TIMEOUT; i++) - if (!(port->cmd & PORT_CMD_CR)) break; - - // Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms - port->sctl = (port->sctl & ~0x0Fu) | 0x1u; - ahci_delay_ms(1); - - // Deassert: DET = 0 - port->sctl = port->sctl & ~0x0Fu; - - // Poll for DET = 3, up to 1 second - bool link_ok = false; - for (int i = 0; i < 1000; i++) { - if ((port->ssts & 0x0Fu) == PORT_DET_PRESENT) { link_ok = true; break; } - ahci_delay_ms(1); - } - if (!link_ok) { - DbgPrint("AHCI: COMRESET: link did not come back (ssts=0x%x)\n", port->ssts); - return STATUS_TIMEOUT; - } - - port->serr = (uint32_t)-1; - port->is = (uint32_t)-1; - port->cmd |= PORT_CMD_ST; - - // Wait up to 500ms for a valid signature (D2H FIS from device) - for (int i = 0; i < 500; i++) { - uint32_t sig = port->sig; - if (sig != 0xFFFFFFFFu && sig != 0x00000000u) break; - ahci_delay_ms(1); - } - - DbgPrint("AHCI: port COMRESET done: sig=0x%08x ssts=0x%08x\n", - port->sig, port->ssts); - return STATUS_SUCCESS; -} - -// PORT_IE bits we care about: -// DHRS (bit 0) - D2H Register FIS received - normal completion -// PSS (bit 1) - PIO Setup FIS received -// DSS (bit 2) - DMA Setup FIS received -// SDBS (bit 3) - Set Device Bits FIS received -// TFES (bit 30) - Task File Error Status - error -// HBFS (bit 29) - Host Bus Fatal error -// HBDS (bit 28) - Host Bus Data error -// IFS (bit 27) - Interface Fatal error -#define PORT_IE_NORMAL (0x0000000Fu) -#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS) -#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS) - -KSTATUS AhciInitialize(const HAL_API *hal) { - g_hal = hal; - g_driveCount = 0; - - // ----------------------------------------------------------------------- - // Locate AHCI controller via PCI class scan - // ----------------------------------------------------------------------- - uint8_t pciBus, pciDev, pciFn; - if (!hal->Pci.FindClass(PCI_AHCI_CLASS, PCI_AHCI_SUBCLASS, - &pciBus, &pciDev, &pciFn)) { - DbgPrint("AHCI: no AHCI controller found\n"); - return STATUS_NOT_FOUND; - } - DbgPrint("AHCI: controller at PCI %02x:%02x.%x\n", pciBus, pciDev, pciFn); - - // ----------------------------------------------------------------------- - // Enable Bus Master + Memory Space in PCI command register - // ----------------------------------------------------------------------- - uint32_t cmd = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_CMD_REG); - cmd |= (PCI_CMD_BUSMASTER | PCI_CMD_MEMSPACE); cmd &= ~(1 << 10); cmd |= (1 << 2); - hal->Pci.Write32(pciBus, pciDev, pciFn, PCI_CMD_REG, cmd); - - // ----------------------------------------------------------------------- - // Read PCI IRQ line - // ----------------------------------------------------------------------- - g_irq = hal->Pci.GetIrqLine(pciBus, pciDev, pciFn); - if (g_irq == 0xFF) { - DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n"); - return STATUS_NOT_SUPPORTED; - } - DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)g_irq); - - // ----------------------------------------------------------------------- - // Read ABAR (BAR5) and map as uncacheable - // ----------------------------------------------------------------------- - uint32_t bar5 = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_BAR5_REG); - uint64_t abarPhys = (uint64_t)(bar5 & ~0xFFFu); - KSTATUS st = ahci_map_mmio(abarPhys); - if (!KSUCCESS(st)) return st; - - g_hba = (HBA_MEM *)hal->PhysToVirt(abarPhys); - DbgPrint("AHCI: ABAR phys=0x%llx version=0x%08x\n", - (unsigned long long)abarPhys, g_hba->vs); - - // ----------------------------------------------------------------------- - // HBA reset + BIOS handoff (also enables GHC.IE) - // ----------------------------------------------------------------------- - st = hba_reset(g_hba); - if (!KSUCCESS(st)) return st; - - // ----------------------------------------------------------------------- - // Log capability summary - // ----------------------------------------------------------------------- - uint32_t cap = g_hba->cap; - uint32_t nPorts = (cap & 0x1F) + 1; - uint32_t nSlots = ((cap >> 8) & 0x1F) + 1; - bool s64a = !!(cap & (1u << 31)); - DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n", - cap, nPorts, nSlots, (int)s64a); - - // ----------------------------------------------------------------------- - // Enumerate implemented ports - // ----------------------------------------------------------------------- - uint32_t pi = g_hba->pi; - for (uint32_t portno = 0; portno < 32 && g_driveCount < MAX_AHCI_DRIVES; portno++) { - if (!(pi & (1u << portno))) continue; - - HBA_PORT *port = &g_hba->ports[portno]; - - // Phase 1: physical link check - if (!port_link_up(port)) { - DbgPrint("AHCI: port %u: no link (ssts=0x%08x)\n", portno, port->ssts); - continue; - } - - DbgPrint("AHCI: port %u: link up, initializing...\n", portno); - - int driveIdx = g_driveCount; - AHCI_DRIVE *drv = &g_drives[driveIdx]; - memset(drv, 0, sizeof(*drv)); - drv->port = port; - drv->portNo = (uint8_t)portno; - - // Phase 2: allocate command list, FIS buffer, bounce buffer - st = port_init_memory(drv); - if (!KSUCCESS(st)) { - DbgPrint("AHCI: port %u: memory init failed (0x%x)\n", portno, st); - continue; - } - - // Phase 3: COMRESET - forces device to re-send D2H FIS for valid SIG - st = port_comreset(drv); - if (!KSUCCESS(st)) { - DbgPrint("AHCI: port %u: COMRESET failed\n", portno); - continue; - } - - // Phase 4: filter to SATA disks - if (!port_is_sata_disk(port)) { - DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n", - portno, port->sig); - continue; - } - - // Phase 5: create per-drive completion event (auto-reset, initially clear) - st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent); - if (!KSUCCESS(st)) { - DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st); - continue; - } - - // Phase 5b: create the IRP queue semaphore and initialise the queue. - // The semaphore starts at 0; each submitted IRP increments it by one. - SEMAPHORE *irpSem = (SEMAPHORE *)0; - st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem); - if (!KSUCCESS(st)) { - DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st); - continue; - } - IrpQueueInit(&drv->irpQueue, irpSem); - - // Phase 6: IDENTIFY (polled - scheduler not yet running) - st = port_identify(drv); - if (!KSUCCESS(st)) { - DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st); - continue; - } - - DbgPrint("AHCI: drive %d: %llu sectors (%llu MB)\n", - driveIdx, - (unsigned long long)drv->sectorCount, - (unsigned long long)(drv->sectorCount / 2048)); - - // Phase 7: enable per-port interrupts. - // Clear any stale IS first so we don't get a spurious signal immediately. - port->is = (uint32_t)-1; - port->ie = PORT_IE_ALL; - - // Phase 8: register with disk subsystem using Ctx pointer - DISK_DRIVER dd = { - .Ctx = drv, - .ReadSectors = AhciRead, - .WriteSectors = AhciWrite, - .SubmitIrp = AhciSubmitIrp, // async IRP path - .SectorCount = drv->sectorCount, - }; - st = IoRegisterDisk((uint8_t)driveIdx, &dd); - if (KSUCCESS(st)) { - DbgPrint("AHCI: drive %d registered\n", driveIdx); - g_driveCount++; - } else { - DbgPrint("AHCI: IoRegisterDisk(%d) failed (0x%x)\n", driveIdx, st); - continue; - } - - // Phase 9: spawn the per-drive IRP worker thread. - // The worker owns do_read / do_write exclusively once the scheduler - // is live; it is the only thread that may call port_issue_cmd(). - st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv, - PriorityHigh, &drv->workerThread); - if (!KSUCCESS(st)) { - DbgPrint("AHCI: drive %d: worker thread create failed (0x%x)\n", - driveIdx, st); - // Non-fatal: sync ReadSectors / WriteSectors still work via the - // IoSubmitIrpSync() fallback in hal_disk.c, but async callers - // will hang waiting on irp->CompletionEvent forever. - } - } - - if (g_driveCount == 0) { - DbgPrint("AHCI: no drives found\n"); - return STATUS_NOT_FOUND; - } - - // ----------------------------------------------------------------------- - // Register the ISR. All ports on this HBA share one PCI IRQ line. - // Do this after all drives are initialised so the ISR sees valid state. - // ----------------------------------------------------------------------- - KeIrqRegister(g_irq, AhciIsr); - DbgPrint("AHCI: ISR registered on IRQ %u; initialised %d drive(s)\n", - (unsigned)g_irq, g_driveCount); - - return STATUS_SUCCESS; -} - -KSTATUS DriverEntry(const HAL_API *hal) { - DbgPrint("AHCI: driver loading...\n"); - KSTATUS st = AhciInitialize(hal); - if (!KSUCCESS(st)) - DbgPrint("AHCI: init failed: 0x%x\n", st); - return st; -} +// ahci.c - AHCI driver, interrupt-driven with per-drive async IRP queue. +// Supports SATA drives only (no ATAPI). Registers each found drive with the +// kernel disk device registry via IoRegisterDiskDevice(). +// +// Memory model: +// MmAllocateContiguous(n) returns a physical address for n*4K pages. +// hal->PhysToVirt(phys) converts to kernel virtual address. +// DMA buffers are bounce-buffered (allocated via MmAllocateContiguous) to +// avoid needing a VirtToPhys conversion for arbitrary kernel pointers. +// +// Async I/O model (IRP path): +// Each drive has a dedicated kernel worker thread (AhciWorker) that blocks +// on a per-drive counting semaphore. Callers submit IRPs through IoCallDriver, +// which dispatches to AHCI and enqueues the IRP. The worker dequeues +// one IRP at a time, runs the synchronous hardware transfer (blocking the +// worker, NOT the submitting thread), then writes irp->Status and fires +// irp->CompletionEvent / irp->CompletionFn. +// +// This gives true async I/O from the caller's perspective: IoSubmitIrp() +// returns STATUS_PENDING immediately. Multiple callers can have outstanding +// requests against different drives concurrently without stalling each other. +// +// NCQ extension (future work - see "NCQ TODO" comments): +// To saturate modern SSDs, replace the serialised single-slot model with +// Native Command Queuing: +// • Allocate per-slot bounce buffers (NUM_NCQ_SLOTS × SLOT_BOUNCE_BYTES). +// • Use ATA_CMD_READ_FPDMA_QUEUED / WRITE_FPDMA_QUEUED. +// • Encode the slot tag in FIS.countl bits 7:3. +// • Issue via port->sact (NCQ active) rather than port->ci. +// • In AhciIsr: completed slots = prev_sact ^ port->sact; look up +// slot_irp[slot] and fire each completion from within the ISR. +// • The worker becomes a pure dispatcher: grab a free slot, build the +// command, issue it, return immediately. No per-slot KeWaitForEvent. +// +// Interrupt model (unchanged): +// GHC.IE and per-port PORT_IE are enabled after each port is initialised. +// AhciIsr fires on the PCI IRQ line, captures/clears port status, and queues +// a per-drive DPC. The DPC signals completionEvent outside hard IRQ context. +// port_issue_cmd blocks the worker thread on KeWaitForEvent instead of +// spinning on CI. port_issue_cmd_poll is kept for early init (IDENTIFY) +// before the scheduler is live. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +typedef enum { + FIS_TYPE_REG_H2D = 0x27, + FIS_TYPE_REG_D2H = 0x34, + FIS_TYPE_DMA_ACT = 0x39, + FIS_TYPE_DMA_SETUP = 0x41, + FIS_TYPE_DATA = 0x46, + FIS_TYPE_BIST = 0x58, + FIS_TYPE_PIO_SETUP = 0x5F, + FIS_TYPE_DEV_BITS = 0xA1, +} FIS_TYPE; + +#pragma pack(push, 1) +// Register FIS - Host to Device (H2D) +typedef struct { + uint8_t fis_type; // FIS_TYPE_REG_H2D + uint8_t pmport : 4; + uint8_t rsv0 : 3; + uint8_t c : 1; // 1 = Command, 0 = Control + uint8_t command; + uint8_t featurel; + // DW1 + uint8_t lba0, lba1, lba2; + uint8_t device; + // DW2 + uint8_t lba3, lba4, lba5; + uint8_t featureh; + // DW3 + uint8_t countl, counth; + uint8_t icc; + uint8_t control; + // DW4 + uint8_t rsv1[4]; +} FIS_REG_H2D; + +// Register FIS - Device to Host (D2H) +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 2; + uint8_t i : 1; + uint8_t rsv1 : 1; + uint8_t status; + uint8_t error; + uint8_t lba0, lba1, lba2; + uint8_t device; + uint8_t lba3, lba4, lba5; + uint8_t rsv2; + uint8_t countl, counth; + uint8_t rsv3[2]; + uint8_t rsv4[4]; +} FIS_REG_D2H; + +// DMA Setup FIS +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 1; + uint8_t d : 1; + uint8_t i : 1; + uint8_t a : 1; + uint8_t rsved[2]; + uint64_t DMAbufferID; + uint32_t rsvd; + uint32_t DMAbufOffset; + uint32_t TransferCount; + uint32_t resvd; +} FIS_DMA_SETUP; + +// PIO Setup FIS +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 1; + uint8_t d : 1; + uint8_t i : 1; + uint8_t rsv1 : 1; + uint8_t status; + uint8_t error; + uint8_t lba0, lba1, lba2; + uint8_t device; + uint8_t lba3, lba4, lba5; + uint8_t rsv2; + uint8_t countl, counth; + uint8_t rsv3; + uint8_t e_status; + uint16_t tc; + uint8_t rsv4[2]; +} FIS_PIO_SETUP; + +// Set Device Bits FIS +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 2; + uint8_t i : 1; + uint8_t n : 1; + uint8_t statusl : 4; + uint8_t rsv1 : 4; + uint8_t statush : 4; + uint8_t rsv2 : 4; + uint8_t error; + uint32_t protocol; +} FIS_DEV_BITS; + +// --------------------------------------------------------------------------- +// HBA structures (volatile - MMIO) +// --------------------------------------------------------------------------- + +// Received FIS structure in host memory +typedef volatile struct { + FIS_DMA_SETUP dsfis; // 0x00 DMA Setup FIS + uint8_t pad0[4]; + FIS_PIO_SETUP psfis; // 0x20 PIO Setup FIS + uint8_t pad1[12]; + FIS_REG_D2H rfis; // 0x40 Register D2H FIS + uint8_t pad2[4]; + FIS_DEV_BITS sdbfis; // 0x58 Set Device Bits FIS + uint8_t ufis[64]; // 0x60 Unknown FIS + uint8_t rsv[0x100-0xA0]; +} HBA_FIS; + +// Command List header (one entry = one slot) +typedef struct { + // DW0 + uint8_t cfl : 5; // Command FIS length in DWORDs (2..16) + uint8_t a : 1; // ATAPI + uint8_t w : 1; // Write (H2D=1, D2H=0) + uint8_t p : 1; // Prefetchable + uint8_t r : 1; // Reset + uint8_t b : 1; // BIST + uint8_t c : 1; // Clear busy on R_OK + uint8_t rsv0 : 1; + uint8_t pmp : 4; // Port multiplier port + uint16_t prdtl; // PRDT length in entries + // DW1 + volatile uint32_t prdbc; // PRDT byte count transferred + // DW2,3 + uint32_t ctba; // Command table base address (128-byte aligned) + uint32_t ctbau; // Upper 32 bits + // DW4-7 + uint32_t rsv1[4]; +} HBA_CMD_HEADER; + +// Physical Region Descriptor Table entry +typedef struct { + uint32_t dba; // Data base address + uint32_t dbau; // Upper 32 bits + uint32_t rsv0; + uint32_t dbc : 22; // Byte count (0-based, so actual = dbc+1) + uint32_t rsv1 : 9; + uint32_t i : 1; // Interrupt on completion +} HBA_PRDT_ENTRY; + +// Command Table +typedef struct { + uint8_t cfis[64]; // Command FIS (up to 64 bytes) + uint8_t acmd[16]; // ATAPI command (12 or 16 bytes) + uint8_t rsv[48]; // Reserved + HBA_PRDT_ENTRY prdt_entry[1]; // PRDT entries (variable length) +} HBA_CMD_TBL; +#pragma pack(pop) + +// Port registers +typedef volatile struct { + uint32_t clb; // 0x00 Command list base (1K-aligned) + uint32_t clbu; // 0x04 Command list base upper + uint32_t fb; // 0x08 FIS base (256-byte aligned) + uint32_t fbu; // 0x0C FIS base upper + uint32_t is; // 0x10 Interrupt status + uint32_t ie; // 0x14 Interrupt enable + uint32_t cmd; // 0x18 Command and status + uint32_t rsv0; // 0x1C Reserved + uint32_t tfd; // 0x20 Task file data + uint32_t sig; // 0x24 Signature + uint32_t ssts; // 0x28 SATA status + uint32_t sctl; // 0x2C SATA control + uint32_t serr; // 0x30 SATA error + uint32_t sact; // 0x34 SATA active + uint32_t ci; // 0x38 Command issue + uint32_t sntf; // 0x3C SATA notification + uint32_t fbs; // 0x40 FIS-based switch control + uint32_t rsv1[11]; + uint32_t vendor[4]; +} HBA_PORT; + +// Generic Host Control registers (starts at ABAR) +typedef volatile struct { + uint32_t cap; // 0x00 Host capability + uint32_t ghc; // 0x04 Global host control + uint32_t is; // 0x08 Interrupt status + uint32_t pi; // 0x0C Ports implemented + uint32_t vs; // 0x10 Version + uint32_t ccc_ctl; // 0x14 CCC control + uint32_t ccc_pts; // 0x18 CCC ports + uint32_t em_loc; // 0x1C Enclosure mgmt location + uint32_t em_ctl; // 0x20 Enclosure mgmt control + uint32_t cap2; // 0x24 Extended capabilities + uint32_t bohc; // 0x28 BIOS/OS handoff control + uint8_t rsv[0xA0-0x2C]; + uint8_t vendor[0x100-0xA0]; + HBA_PORT ports[32]; // 0x100 Port control registers +} HBA_MEM; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +// PCI +#define PCI_AHCI_CLASS 0x01 +#define PCI_AHCI_SUBCLASS 0x06 +#define PCI_CMD_REG 0x04 +#define PCI_BAR5_REG 0x24 +#define PCI_CMD_BUSMASTER (1 << 2) +#define PCI_CMD_MEMSPACE (1 << 1) + +// GHC register bits +#define GHC_AE (1u << 31) // AHCI Enable +#define GHC_IE (1u << 1) // Interrupt Enable +#define GHC_HR (1u << 0) // HBA Reset + +// BOHC register bits +#define BOHC_BOS (1u << 0) // BIOS-owned semaphore +#define BOHC_OOS (1u << 1) // OS-owned semaphore +#define BOHC_BB (1u << 4) // BIOS Busy + +// CAP2 bits +#define CAP2_BOH (1u << 0) // BIOS/OS handoff capable + +// Port CMD register bits +#define PORT_CMD_ST (1u << 0) // Start +#define PORT_CMD_FRE (1u << 4) // FIS Receive Enable +#define PORT_CMD_FR (1u << 14) // FIS Receive Running +#define PORT_CMD_CR (1u << 15) // Command List Running +#define PORT_CMD_POD (1u << 2) // Power On Device +#define PORT_CMD_SUD (1u << 1) // Spin-Up Device +#define PORT_CMD_ICC_ACTIVE (0x1u << 28) + +// Port IS register bits (interrupt status) +#define PORT_IS_TFES (1u << 30) // Task File Error Status +#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error +#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error +#define PORT_IS_IFS (1u << 27) // Interface Fatal Error + +// Port SSTS register +#define PORT_DET_PRESENT 0x3 +#define PORT_IPM_ACTIVE 0x1 + +// Port signatures +#define SATA_SIG_ATA 0x00000101 +#define SATA_SIG_ATAPI 0xEB140101 +#define SATA_SIG_SEMB 0xC33C0101 +#define SATA_SIG_PM 0x96690101 + +// ATA commands +#define ATA_CMD_READ_DMA_EX 0x25 +#define ATA_CMD_WRITE_DMA_EX 0x35 +#define ATA_CMD_IDENTIFY 0xEC +#define ATA_CMD_FLUSH_EXT 0xEA + +// ATA status bits +#define ATA_SR_BUSY 0x80 +#define ATA_SR_DRQ 0x08 +#define ATA_SR_ERR 0x01 + +// IDENTIFY word offsets +#define ATA_IDENT_W83 83 // bit 10 = 48-bit LBA supported +#define ATA_IDENT_W100 100 // 48-bit LBA sector count (4 words) +#define ATA_IDENT_W60 60 // 28-bit LBA sector count (2 words) + +// Layout constants (per port, single contiguous physical allocation) +// Offset 0: Command List 1024 bytes (32 × 32-byte headers) +// Offset 1024: Received FIS 256 bytes +// Offset 2048: Command Tables 32 × 256 bytes = 8192 bytes +// Total: 10240 bytes → 3 pages (12288 bytes) +#define PORT_PAGES_STRUCT 3 +#define PORT_OFF_CMDLIST 0 +#define PORT_OFF_FIS 1024 +#define PORT_OFF_CMDTBLS 2048 +#define PORT_CMDTBL_SIZE 256 // 64 CFIS + 16 ACMD + 48 rsv + 8×16 PRDT +#define PORT_PRDT_COUNT 8 // PRDT entries per command table + +// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages +#define BOUNCE_SECTORS 128 +#define BOUNCE_BYTES (BOUNCE_SECTORS * 512) +#define BOUNCE_PAGES (BOUNCE_BYTES / 4096) + +// Polling timeout (spin iterations) +#define POLL_TIMEOUT 1000000 + +#define MAX_AHCI_DRIVES 8 +#define NUM_CMD_SLOTS 32 + +// Map MMIO physical range into the physical window as uncacheable. +// Must be called before any pointer dereference via PhysToVirt(). +// AHCI MMIO layout: 0x100 GHC + 32*0x80 ports = 0x1100 bytes → 2 pages minimum. +// We map 8 pages (32KB) without probing the BAR size. +#define AHCI_MMIO_PAGES 8 + +// --------------------------------------------------------------------------- +// Driver-global state +// --------------------------------------------------------------------------- + +static const HAL_API *g_hal; +static HBA_MEM *g_hba; // HBA MMIO base (set during init) +static uint8_t g_irq; // PCI IRQ line for this HBA +static DRIVER_OBJECT g_driver; // AHCI disk driver object + +// Per-drive context +typedef struct { + HBA_PORT *port; // MMIO port registers (virtual) + uint64_t sectorCount; // Total LBA sectors on device + uint8_t portNo; // Physical port index (for ISR dispatch) + + // Structures block (command list + FIS + cmd tables) + uint64_t structPhys; // Physical base + void *structVirt; // Virtual base + + // Bounce buffer for DMA data + uint64_t bouncePhys; + void *bounceVirt; + + // Signaled by the completion DPC when the port raises an interrupt. + // Auto-reset: one Wait consumes one completion. + EVENT *completionEvent; + + // Last IS snapshot captured by the ISR (for error checking post-wait). + volatile uint32_t lastIs; + + // Bottom-half used to wake the worker outside hard IRQ/DIRQL context. + KDPC completionDpc; + + // ----------------------------------------------------------------------- + // IRP queue - async dispatch + // ----------------------------------------------------------------------- + // AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP + // and signals the semaphore to wake the worker. + IRP_QUEUE irpQueue; + THREAD *workerThread; + DEVICE_OBJECT *deviceObject; + + // NCQ TODO: for multi-slot NCQ, add: + // uint32_t slotMask; // bitmask of in-flight slots + // IRP *slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP + // uint64_t slotBouncePhys[NUM_CMD_SLOTS]; + // void *slotBounceVirt[NUM_CMD_SLOTS]; + // volatile uint32_t prevSact; // previous port->sact snapshot +} AHCI_DRIVE; + +static AHCI_DRIVE g_drives[MAX_AHCI_DRIVES]; +static int g_driveCount; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + + +static KSTATUS ahci_map_mmio(uint64_t physBase) { + for (int i = 0; i < AHCI_MMIO_PAGES; i++) { + uint64_t phys = physBase + (uint64_t)i * 4096; + uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys); + uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; + KSTATUS st = MmMapPage(virt, phys, flags); + // Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags + // is possible for the first page if the bootloader was overly generous. + // Use MmSetPageProtect to force NOCACHE in that case. + if (st == STATUS_ALREADY_EXISTS) { + st = MmSetPageProtect(virt, 4096, flags); + } + if (!KSUCCESS(st)) { + DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%llx, st=0x%x)\n", + i, (unsigned long long)phys, st); + return st; + } + } + return STATUS_SUCCESS; +} + +// Accessors for port structure memory at known offsets from structVirt/Phys +static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) { + return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST); +} + +static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) { + return drv->structPhys + PORT_OFF_CMDLIST; +} + +static inline uint64_t fis_phys(AHCI_DRIVE *drv) { + return drv->structPhys + PORT_OFF_FIS; +} + +static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) { + return (HBA_CMD_TBL *)((uint8_t *)drv->structVirt + + PORT_OFF_CMDTBLS + + (uint32_t)slot * PORT_CMDTBL_SIZE); +} + +static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { + return drv->structPhys + PORT_OFF_CMDTBLS + + (uint64_t)slot * PORT_CMDTBL_SIZE; +} + +// --------------------------------------------------------------------------- +// Port engine start / stop (polling safe) +// --------------------------------------------------------------------------- + +static void port_stop(HBA_PORT *port) { + // Clear ST (bit 0) + port->cmd &= ~PORT_CMD_ST; + // Wait for CR (bit 15) to clear (command list DMA stops) + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(port->cmd & PORT_CMD_CR)) break; + } + // Clear FRE (bit 4) + port->cmd &= ~PORT_CMD_FRE; + // Wait for FR (bit 14) to clear + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(port->cmd & PORT_CMD_FR)) break; + } +} + +static void port_start(HBA_PORT *port) { + // Wait until CR is clear before setting ST + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(port->cmd & PORT_CMD_CR)) break; + } + port->cmd |= PORT_CMD_FRE; + port->cmd |= PORT_CMD_ST; +} + +// --------------------------------------------------------------------------- +// Port memory initialisation +// --------------------------------------------------------------------------- + +static KSTATUS port_init_memory(AHCI_DRIVE *drv) { + uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; + + drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT); + if (!drv->structPhys) return STATUS_NO_MEMORY; + drv->structVirt = g_hal->PhysToVirt(drv->structPhys); + // Force NOCACHE on DMA descriptor memory + MmSetPageProtect((uintptr_t)drv->structVirt, + PORT_PAGES_STRUCT * 4096, flags); + memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096); + + drv->bouncePhys = MmAllocateContiguous(BOUNCE_PAGES); + if (!drv->bouncePhys) return STATUS_NO_MEMORY; + drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys); + // Bounce buffer: NOCACHE to ensure CPU writes reach physical memory + // before the HBA's DMA engine reads them + MmSetPageProtect((uintptr_t)drv->bounceVirt, + BOUNCE_PAGES * 4096, flags); + memset(drv->bounceVirt, 0, BOUNCE_BYTES); + + HBA_PORT *port = drv->port; + + port_stop(port); + + // Set command list base + port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF); + port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32); + + // Set FIS base + port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF); + port->fbu = (uint32_t)(fis_phys(drv) >> 32); + + // Initialise all 32 command headers, pointing each to its command table + HBA_CMD_HEADER *hdrs = cmd_list(drv); + for (int i = 0; i < NUM_CMD_SLOTS; i++) { + hdrs[i].prdtl = PORT_PRDT_COUNT; + uint64_t tbl = cmd_tbl_phys(drv, i); + hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF); + hdrs[i].ctbau = (uint32_t)(tbl >> 32); + } + + // Clear any stale interrupts then start the port engine + port->serr = (uint32_t)-1; + port->is = (uint32_t)-1; + + port_start(port); + + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// Issue a command - polling fallback (used during early init / IDENTIFY only, +// before the scheduler and interrupt handler are live). +// Returns STATUS_SUCCESS, STATUS_DEVICE_ERROR, or STATUS_TIMEOUT. +// --------------------------------------------------------------------------- + +static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) { + HBA_PORT *port = drv->port; + + // Clear pending interrupts before issuing + port->is = (uint32_t)-1; + + // Wait for port to be idle (not BSY or DRQ) + int spin = 0; + while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) + spin++; + if (spin == POLL_TIMEOUT) { + DbgPrint("AHCI: port hung (pre-issue)\n"); + return STATUS_TIMEOUT; + } + + // Issue slot 0 + port->ci = (1u << 0); + + // Poll until slot 0 clears + while (true) { + if (!(port->ci & (1u << 0))) break; + if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { + DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", + port->is, port->tfd, port->serr); + port->serr = (uint32_t)-1; + port->is = (uint32_t)-1; + return STATUS_DEVICE_ERROR; + } + } + + if (port->is & PORT_IS_TFES) { + DbgPrint("AHCI: TFES after completion TFD=0x%x\n", port->tfd); + port->is = (uint32_t)-1; + return STATUS_DEVICE_ERROR; + } + + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// Interrupt bottom half. The ISR owns hardware acknowledgement and status +// capture; this DPC wakes the worker thread after the hard IRQ has unwound. +// --------------------------------------------------------------------------- + +static void AhciCompletionDpc(KDPC *dpc, void *context, + void *systemArgument1, void *systemArgument2) { + (void)dpc; + (void)systemArgument1; + (void)systemArgument2; + + AHCI_DRIVE *drv = (AHCI_DRIVE *)context; + if (drv && drv->completionEvent) + KeSignalEvent(drv->completionEvent); +} + +// --------------------------------------------------------------------------- +// ISR - called from KeIrqDispatch on the AHCI PCI IRQ line. +// Walks all registered drives, checks if their port raised an interrupt, +// captures IS, clears it, and queues the per-drive completion DPC. +// --------------------------------------------------------------------------- + +static void AhciIsr(void) { + if (!g_hba) return; + + // GHC.IS tells us which ports are pending. + uint32_t pending = g_hba->is; + if (!pending) return; + + for (int i = 0; i < g_driveCount; i++) { + AHCI_DRIVE *drv = &g_drives[i]; + uint32_t portBit = (1u << drv->portNo); + if (!(pending & portBit)) continue; + + // Capture and clear port IS before queueing the bottom half. The + // waiter reads lastIs to check for errors without racing the next + // command. OR in case hardware reports multiple causes before the + // DPC releases the worker. + uint32_t portIs = drv->port->is; + drv->port->is = portIs; // W1C - write back to clear + drv->lastIs |= portIs; + + // Clear the HBA-level bit. + g_hba->is = portBit; + + KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0); + } +} + +// --------------------------------------------------------------------------- +// Issue a command - interrupt-driven (runtime path). +// Caller has already built slot 0 via build_rw_cmd / port_identify. +// Blocks the current thread until the ISR signals completionEvent. +// --------------------------------------------------------------------------- + +static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { + HBA_PORT *port = drv->port; + + // Clear stale IS and snapshot before issuing so we don't mis-read a + // leftover value from a prior command. + port->is = (uint32_t)-1; + drv->lastIs = 0; + + // Wait for port idle + int spin = 0; + while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) + spin++; + if (spin == POLL_TIMEOUT) { + DbgPrint("AHCI: port hung (pre-issue)\n"); + return STATUS_TIMEOUT; + } + + // Issue slot 0, then block on the event. + port->ci = (1u << 0); + KeWaitForEvent(drv->completionEvent, 0); + + // The ISR has cleared IS - check for errors in the snapshot. + uint32_t is = drv->lastIs; + if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { + DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", + is, port->tfd, port->serr); + port->serr = (uint32_t)-1; + return STATUS_DEVICE_ERROR; + } + + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// Build a standard DMA read or write command in slot 0 +// --------------------------------------------------------------------------- + +static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, + uint64_t bufPhys, bool write) { + HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0 + HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); + + memset(tbl, 0, PORT_CMDTBL_SIZE); + + // How many PRDT entries we need (each covers up to 8K = 16 sectors) + uint32_t prdtl = (sectors + 15) / 16; + if (prdtl > PORT_PRDT_COUNT) prdtl = PORT_PRDT_COUNT; // capped (see caller) + + hdr->cfl = sizeof(FIS_REG_H2D) / 4; + hdr->w = write ? 1 : 0; + hdr->prdtl = (uint16_t)prdtl; + hdr->prdbc = 0; + + // Fill PRDT entries + uint32_t remaining = sectors; + uint64_t buf = bufPhys; + for (uint32_t i = 0; i < prdtl; i++) { + uint32_t sects = (remaining > 16) ? 16 : remaining; + uint32_t bytes = sects * 512; + + tbl->prdt_entry[i].dba = (uint32_t)(buf & 0xFFFFFFFF); + tbl->prdt_entry[i].dbau = (uint32_t)(buf >> 32); + tbl->prdt_entry[i].dbc = bytes - 1; // hardware adds 1 + tbl->prdt_entry[i].i = (i == prdtl - 1) ? 1 : 0; + + buf += bytes; + remaining -= sects; + } + + // Build H2D FIS + FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis; + memset(fis, 0, sizeof(*fis)); + fis->fis_type = FIS_TYPE_REG_H2D; + fis->c = 1; + fis->command = write ? ATA_CMD_WRITE_DMA_EX : ATA_CMD_READ_DMA_EX; + fis->device = 1u << 6; // LBA mode + + fis->lba0 = (uint8_t)(lba); + fis->lba1 = (uint8_t)(lba >> 8); + fis->lba2 = (uint8_t)(lba >> 16); + fis->lba3 = (uint8_t)(lba >> 24); + fis->lba4 = (uint8_t)(lba >> 32); + fis->lba5 = (uint8_t)(lba >> 40); + + fis->countl = (uint8_t)(sectors & 0xFF); + fis->counth = (uint8_t)(sectors >> 8); +} + +// --------------------------------------------------------------------------- +// IDENTIFY command - fills a 512-byte buffer and returns it +// --------------------------------------------------------------------------- + +static KSTATUS port_identify(AHCI_DRIVE *drv) { + // Reuse bounce buffer for IDENTIFY data (512 bytes) + memset(drv->bounceVirt, 0, 512); + + HBA_CMD_HEADER *hdr = cmd_list(drv); + HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); + + memset(tbl, 0, PORT_CMDTBL_SIZE); + + hdr->cfl = sizeof(FIS_REG_H2D) / 4; + hdr->w = 0; + hdr->prdtl = 1; + hdr->prdbc = 0; + + tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF); + tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32); + tbl->prdt_entry[0].dbc = 512 - 1; + tbl->prdt_entry[0].i = 1; + + FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis; + memset(fis, 0, sizeof(*fis)); + fis->fis_type = FIS_TYPE_REG_H2D; + fis->c = 1; + fis->command = ATA_CMD_IDENTIFY; + fis->device = 0; + + // Poll: IDENTIFY runs before the scheduler and IRQ handler are live. + KSTATUS st = port_issue_cmd_poll(drv); + if (!KSUCCESS(st)) return st; + + // Parse sector count from IDENTIFY data + const uint16_t *id = (const uint16_t *)drv->bounceVirt; + + if (id[ATA_IDENT_W83] & (1u << 10)) { + // 48-bit LBA supported + drv->sectorCount = (uint64_t)id[ATA_IDENT_W100] + | ((uint64_t)id[ATA_IDENT_W100 + 1] << 16) + | ((uint64_t)id[ATA_IDENT_W100 + 2] << 32) + | ((uint64_t)id[ATA_IDENT_W100 + 3] << 48); + } else { + // 28-bit LBA + drv->sectorCount = (uint64_t)id[ATA_IDENT_W60] + | ((uint64_t)id[ATA_IDENT_W60 + 1] << 16); + } + + return STATUS_SUCCESS; +} + +// Core read logic +static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba, + uint32_t count, void *buf) { + uint8_t *out = (uint8_t *)buf; + + while (count > 0) { + uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; + + build_rw_cmd(drv, lba, batch, drv->bouncePhys, false); + KSTATUS st = port_issue_cmd(drv); + if (!KSUCCESS(st)) return st; + + memcpy(out, drv->bounceVirt, (size_t)batch * 512); + out += (size_t)batch * 512; + lba += batch; + count -= batch; + } + return STATUS_SUCCESS; +} + +// Core write logic +static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba, + uint32_t count, const void *buf) { + const uint8_t *in = (const uint8_t *)buf; + + while (count > 0) { + uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; + + memcpy(drv->bounceVirt, in, (size_t)batch * 512); + build_rw_cmd(drv, lba, batch, drv->bouncePhys, true); + KSTATUS st = port_issue_cmd(drv); + if (!KSUCCESS(st)) return st; + + in += (size_t)batch * 512; + lba += batch; + count -= batch; + } + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// Async IRP dispatch +// --------------------------------------------------------------------------- + +// AhciDispatchReadWrite - DEVICE_OBJECT dispatch entry for disk IRPs. +// Accepted IRPs are queued to the per-drive worker and completed later with +// IoCompleteRequest(). Rejected IRPs are returned synchronously and are not +// completed. +static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { + if (!device || !irp) return STATUS_INVALID_PARAMETER; + AHCI_DRIVE *drv = (AHCI_DRIVE *)device->DeviceExtension; + if (!drv) return STATUS_INVALID_PARAMETER; + if (irp->SectorCount == 0 || !irp->Buffer) return STATUS_INVALID_PARAMETER; + if (irp->Lba + irp->SectorCount > drv->sectorCount) return STATUS_INVALID_PARAMETER; + + irp->Status = STATUS_FAILURE; + irp->Information = 0; + IrpQueueEnqueue(&drv->irpQueue, irp); + return STATUS_PENDING; +} + +// AhciWorker - kernel thread, one per drive. +// Dequeues IRPs and executes them one at a time using the existing +// single-slot interrupt-driven path. Completion fires the IRP's event +// and/or callback so the original caller can observe the result without +// ever having blocked in a driver call. +// +// NCQ TODO: replace the serialised loop with a multi-slot dispatcher: +// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP. +// 2. Pick the oldest IRP from the queue. +// 3. Grab a free bit in drv->slotMask (under the slot spinlock). +// 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA), +// record slotIrp[slot] = irp, and issue via port->sact + port->ci. +// 5. Return immediately - the ISR drives all completions. +static void AhciWorker(uintptr_t arg) { + AHCI_DRIVE *drv = (AHCI_DRIVE *)arg; + + while (true) { + // Block until AHCI dispatch signals a new IRP. + KeWaitSemaphore(drv->irpQueue.Semaphore); + + IRP *irp = IrpQueueDequeue(&drv->irpQueue); + if (!irp) KePanic("AhciWorker: semaphore fired but queue is empty - IRP accounting bug\n"); + + bool write = !!(irp->Flags & IRP_FLAG_WRITE); + KSTATUS st; + if (write) + st = do_write(drv, irp->Lba, irp->SectorCount, irp->Buffer); + else + st = do_read (drv, irp->Lba, irp->SectorCount, irp->Buffer); + + // Publish result then fire completion hooks through the common I/O path. + IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0); + } +} + +// --------------------------------------------------------------------------- +// AHCI controller initialisation +// --------------------------------------------------------------------------- + +static KSTATUS hba_reset(HBA_MEM *hba) { + // BIOS/OS handoff (spec §10.6.3) + if (hba->cap2 & CAP2_BOH) { + hba->bohc |= BOHC_OOS; + // Wait for BIOS to clear BOS + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(hba->bohc & BOHC_BOS)) break; + } + // Give the BIOS another moment to clear Busy + for (int i = 0; i < 800000; i++) { + if (!(hba->bohc & BOHC_BB)) break; + } + } + + // Enable AHCI mode before reset so the reset actually takes effect + hba->ghc |= GHC_AE; + + // HBA reset + hba->ghc |= GHC_HR; + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(hba->ghc & GHC_HR)) break; + } + if (hba->ghc & GHC_HR) { + DbgPrint("AHCI: HBA reset timed out\n"); + return STATUS_TIMEOUT; + } + + // Re-enable AHCI mode (reset cleared it), then enable global interrupts. + // GHC.IE lets the HBA assert its PCI interrupt line when a port fires. + hba->ghc |= GHC_AE | GHC_IE; + return STATUS_SUCCESS; +} + + +// Before memory init: only check physical link. Signature is not yet valid. +static bool port_link_up(HBA_PORT *port) { + uint32_t det = port->ssts & 0x0F; + uint32_t ipm = (port->ssts >> 8) & 0x0F; + return (det == PORT_DET_PRESENT && ipm == PORT_IPM_ACTIVE); +} + +// After COMRESET + FIS receive: check signature for SATA disk. +static bool port_is_sata_disk(HBA_PORT *port) { + return port->sig == SATA_SIG_ATA; +} + +static void ahci_delay_ms(uint32_t ms) { + for (uint32_t i = 0; i < ms * 1000; i++) + __outbyte(0x80, 0); +} + +// Issue a COMRESET on the port, wait for link re-establishment and D2H FIS. +// FRE must already be set (port_init_memory does this) so the HBA can +// capture the signature FIS the device sends after the reset. +static KSTATUS port_comreset(AHCI_DRIVE *drv) { + HBA_PORT *port = drv->port; + + port->cmd &= ~PORT_CMD_ST; + for (int i = 0; i < POLL_TIMEOUT; i++) + if (!(port->cmd & PORT_CMD_CR)) break; + + // Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms + port->sctl = (port->sctl & ~0x0Fu) | 0x1u; + ahci_delay_ms(1); + + // Deassert: DET = 0 + port->sctl = port->sctl & ~0x0Fu; + + // Poll for DET = 3, up to 1 second + bool link_ok = false; + for (int i = 0; i < 1000; i++) { + if ((port->ssts & 0x0Fu) == PORT_DET_PRESENT) { link_ok = true; break; } + ahci_delay_ms(1); + } + if (!link_ok) { + DbgPrint("AHCI: COMRESET: link did not come back (ssts=0x%x)\n", port->ssts); + return STATUS_TIMEOUT; + } + + port->serr = (uint32_t)-1; + port->is = (uint32_t)-1; + port->cmd |= PORT_CMD_ST; + + // Wait up to 500ms for a valid signature (D2H FIS from device) + for (int i = 0; i < 500; i++) { + uint32_t sig = port->sig; + if (sig != 0xFFFFFFFFu && sig != 0x00000000u) break; + ahci_delay_ms(1); + } + + DbgPrint("AHCI: port COMRESET done: sig=0x%08x ssts=0x%08x\n", + port->sig, port->ssts); + return STATUS_SUCCESS; +} + +// PORT_IE bits we care about: +// DHRS (bit 0) - D2H Register FIS received - normal completion +// PSS (bit 1) - PIO Setup FIS received +// DSS (bit 2) - DMA Setup FIS received +// SDBS (bit 3) - Set Device Bits FIS received +// TFES (bit 30) - Task File Error Status - error +// HBFS (bit 29) - Host Bus Fatal error +// HBDS (bit 28) - Host Bus Data error +// IFS (bit 27) - Interface Fatal error +#define PORT_IE_NORMAL (0x0000000Fu) +#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS) +#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS) + +KSTATUS AhciInitialize(const HAL_API *hal) { + g_hal = hal; + g_driveCount = 0; + + memset(&g_driver, 0, sizeof(g_driver)); + g_driver.Name = "ahci"; + g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite; + g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite; + + // ----------------------------------------------------------------------- + // Locate AHCI controller via PCI class scan + // ----------------------------------------------------------------------- + uint8_t pciBus, pciDev, pciFn; + if (!hal->Pci.FindClass(PCI_AHCI_CLASS, PCI_AHCI_SUBCLASS, + &pciBus, &pciDev, &pciFn)) { + DbgPrint("AHCI: no AHCI controller found\n"); + return STATUS_NOT_FOUND; + } + DbgPrint("AHCI: controller at PCI %02x:%02x.%x\n", pciBus, pciDev, pciFn); + + // ----------------------------------------------------------------------- + // Enable Bus Master + Memory Space in PCI command register + // ----------------------------------------------------------------------- + uint32_t cmd = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_CMD_REG); + cmd |= (PCI_CMD_BUSMASTER | PCI_CMD_MEMSPACE); cmd &= ~(1 << 10); cmd |= (1 << 2); + hal->Pci.Write32(pciBus, pciDev, pciFn, PCI_CMD_REG, cmd); + + // ----------------------------------------------------------------------- + // Read PCI IRQ line + // ----------------------------------------------------------------------- + g_irq = hal->Pci.GetIrqLine(pciBus, pciDev, pciFn); + if (g_irq == 0xFF) { + DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n"); + return STATUS_NOT_SUPPORTED; + } + DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)g_irq); + + // ----------------------------------------------------------------------- + // Read ABAR (BAR5) and map as uncacheable + // ----------------------------------------------------------------------- + uint32_t bar5 = hal->Pci.Read32(pciBus, pciDev, pciFn, PCI_BAR5_REG); + uint64_t abarPhys = (uint64_t)(bar5 & ~0xFFFu); + KSTATUS st = ahci_map_mmio(abarPhys); + if (!KSUCCESS(st)) return st; + + g_hba = (HBA_MEM *)hal->PhysToVirt(abarPhys); + DbgPrint("AHCI: ABAR phys=0x%llx version=0x%08x\n", + (unsigned long long)abarPhys, g_hba->vs); + + // ----------------------------------------------------------------------- + // HBA reset + BIOS handoff (also enables GHC.IE) + // ----------------------------------------------------------------------- + st = hba_reset(g_hba); + if (!KSUCCESS(st)) return st; + + // ----------------------------------------------------------------------- + // Log capability summary + // ----------------------------------------------------------------------- + uint32_t cap = g_hba->cap; + uint32_t nPorts = (cap & 0x1F) + 1; + uint32_t nSlots = ((cap >> 8) & 0x1F) + 1; + bool s64a = !!(cap & (1u << 31)); + DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n", + cap, nPorts, nSlots, (int)s64a); + + // ----------------------------------------------------------------------- + // Enumerate implemented ports + // ----------------------------------------------------------------------- + uint32_t pi = g_hba->pi; + for (uint32_t portno = 0; portno < 32 && g_driveCount < MAX_AHCI_DRIVES; portno++) { + if (!(pi & (1u << portno))) continue; + + HBA_PORT *port = &g_hba->ports[portno]; + + // Phase 1: physical link check + if (!port_link_up(port)) { + DbgPrint("AHCI: port %u: no link (ssts=0x%08x)\n", portno, port->ssts); + continue; + } + + DbgPrint("AHCI: port %u: link up, initializing...\n", portno); + + int driveIdx = g_driveCount; + AHCI_DRIVE *drv = &g_drives[driveIdx]; + memset(drv, 0, sizeof(*drv)); + drv->port = port; + drv->portNo = (uint8_t)portno; + + // Phase 2: allocate command list, FIS buffer, bounce buffer + st = port_init_memory(drv); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: port %u: memory init failed (0x%x)\n", portno, st); + continue; + } + + // Phase 3: COMRESET - forces device to re-send D2H FIS for valid SIG + st = port_comreset(drv); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: port %u: COMRESET failed\n", portno); + continue; + } + + // Phase 4: filter to SATA disks + if (!port_is_sata_disk(port)) { + DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n", + portno, port->sig); + continue; + } + + // Phase 5: create per-drive completion event (auto-reset, initially clear) + st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st); + continue; + } + KeInitializeDpc(&drv->completionDpc, AhciCompletionDpc, drv); + + // Phase 5b: create the IRP queue semaphore and initialise the queue. + // The semaphore starts at 0; each submitted IRP increments it by one. + SEMAPHORE *irpSem = (SEMAPHORE *)0; + st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st); + continue; + } + IrpQueueInit(&drv->irpQueue, irpSem); + + // Phase 6: IDENTIFY (polled - scheduler not yet running) + st = port_identify(drv); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st); + continue; + } + + DbgPrint("AHCI: drive %d: %llu sectors (%llu MB)\n", + driveIdx, + (unsigned long long)drv->sectorCount, + (unsigned long long)(drv->sectorCount / 2048)); + + // Phase 7: enable per-port interrupts. + // Clear any stale IS first so we don't get a spurious signal immediately. + port->is = (uint32_t)-1; + port->ie = PORT_IE_ALL; + + // Phase 8: create and register a disk DEVICE_OBJECT. + st = IoCreateDevice(&g_driver, 0, &drv->deviceObject); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", driveIdx, st); + continue; + } + drv->deviceObject->DeviceExtension = drv; + drv->deviceObject->SectorCount = drv->sectorCount; + drv->deviceObject->AlignmentRequirement = 512; + + st = IoRegisterDiskDevice((uint8_t)driveIdx, drv->deviceObject); + if (KSUCCESS(st)) { + DbgPrint("AHCI: drive %d registered\n", driveIdx); + g_driveCount++; + } else { + DbgPrint("AHCI: IoRegisterDiskDevice(%d) failed (0x%x)\n", driveIdx, st); + IoDeleteDevice(drv->deviceObject); + drv->deviceObject = (DEVICE_OBJECT*)0; + continue; + } + + // Phase 9: spawn the per-drive IRP worker thread. + // The worker owns do_read / do_write exclusively once the scheduler + // is live; it is the only thread that may call port_issue_cmd(). + st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv, + PriorityHigh, &drv->workerThread); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: drive %d: worker thread create failed (0x%x)\n", + driveIdx, st); + // Fatal for this device: all disk access is IRP-based now, so + // leave the device registered but clearly fail during testing. + } + } + + if (g_driveCount == 0) { + DbgPrint("AHCI: no drives found\n"); + return STATUS_NOT_FOUND; + } + + // ----------------------------------------------------------------------- + // Register the ISR. All ports on this HBA share one PCI IRQ line. + // Do this after all drives are initialised so the ISR sees valid state. + // ----------------------------------------------------------------------- + KeIrqRegister(g_irq, AhciIsr); + DbgPrint("AHCI: ISR registered on IRQ %u; initialised %d drive(s)\n", + (unsigned)g_irq, g_driveCount); + + return STATUS_SUCCESS; +} + +KSTATUS DriverEntry(const HAL_API *hal) { + DbgPrint("AHCI: driver loading...\n"); + KSTATUS st = AhciInitialize(hal); + if (!KSUCCESS(st)) + DbgPrint("AHCI: init failed: 0x%x\n", st); + return st; +} diff --git a/code/drivers/fat32/fat32.c b/code/drivers/fat32/fat32.c index ca8aee6..6c8f657 100644 --- a/code/drivers/fat32/fat32.c +++ b/code/drivers/fat32/fat32.c @@ -19,7 +19,7 @@ // // 3. AllocCluster() next-free hint. // The search now starts from the cluster after the last allocation -// (gVol.NextFreeHint) and wraps around once, so repeated allocations +// (volume NextFreeHint) and wraps around once, so repeated allocations // on a fragmented volume don't rescan from sector 0 every time. // // 4. Contiguous-cluster run coalescing in Fat32Read() and Fat32Write(). @@ -38,12 +38,17 @@ #include #include +#include #include #include #include #include #include #include +#include +#include +#include +#include // --------------------------------------------------------------------------- // On-disk structures @@ -149,6 +154,9 @@ typedef struct { #define FAT32_EOC_MARK 0x0FFFFFFFu #define FAT32_FREE 0x00000000u +// FAT32_VOL forward declaration (needed by cache helpers). +typedef struct FAT32_VOL_S FAT32_VOL; + // --------------------------------------------------------------------------- // FAT sector cache (fixes #1 and #5) // --------------------------------------------------------------------------- @@ -161,6 +169,7 @@ typedef struct { #define FAT_CACHE_SLOTS 32 typedef struct { + FAT32_VOL *vol; uint64_t lba; uint8_t data[512]; bool valid; @@ -169,9 +178,6 @@ typedef struct { static FAT_CACHE_ENTRY gFatCache[FAT_CACHE_SLOTS]; -// FAT32_VOL forward declaration (needed by cache helpers). -typedef struct FAT32_VOL_S FAT32_VOL; - // Raw (non-cached) sector I/O — forward declarations. static KSTATUS ReadSectorsRaw (FAT32_VOL *v, uint64_t lba, uint32_t count, void *buf); static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, const void *buf); @@ -182,21 +188,22 @@ static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) { uint32_t slot = (uint32_t)(lba % FAT_CACHE_SLOTS); FAT_CACHE_ENTRY *e = &gFatCache[slot]; - if (e->valid && e->lba == lba) { + if (e->valid && e->vol == v && e->lba == lba) { *out = e->data; return STATUS_SUCCESS; } // Evict current occupant: single-copy writeback on collision. // The full multi-copy flush happens in FlushFatCache(). - if (e->valid && e->dirty) { - WriteSectorsRaw(v, e->lba, 1, e->data); + if (e->valid && e->dirty && e->vol) { + WriteSectorsRaw(e->vol, e->lba, 1, e->data); e->dirty = false; } KSTATUS s = ReadSectorsRaw(v, lba, 1, e->data); - if (!KSUCCESS(s)) { e->valid = false; return s; } + if (!KSUCCESS(s)) { e->valid = false; e->vol = (FAT32_VOL*)0; return s; } + e->vol = v; e->lba = lba; e->valid = true; e->dirty = false; @@ -204,14 +211,15 @@ static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) { return STATUS_SUCCESS; } -static void FatCacheMarkDirty(uint64_t lba) { +static void FatCacheMarkDirty(FAT32_VOL *v, uint64_t lba) { uint32_t slot = (uint32_t)(lba % FAT_CACHE_SLOTS); - if (gFatCache[slot].valid && gFatCache[slot].lba == lba) + if (gFatCache[slot].valid && gFatCache[slot].vol == v && gFatCache[slot].lba == lba) gFatCache[slot].dirty = true; } static void FatCacheInvalidate(void) { for (int i = 0; i < FAT_CACHE_SLOTS; i++) { + gFatCache[i].vol = (FAT32_VOL*)0; gFatCache[i].valid = false; gFatCache[i].dirty = false; } @@ -237,7 +245,6 @@ struct FAT32_VOL_S { uint32_t DataStart; // absolute LBA of cluster 2 uint32_t TotalClusters; // usable data clusters uint32_t NextFreeHint; // fix #3: start cluster for next free-scan - const HAL_API *Hal; }; typedef struct { @@ -253,16 +260,14 @@ typedef struct { // --------------------------------------------------------------------------- static KSTATUS ReadSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, void *buf) { - if (!v || !v->Hal || !v->Hal->Disk.ReadSectors) - return STATUS_INVALID_PARAMETER; - return v->Hal->Disk.ReadSectors(v->DriveIndex, lba, count, buf); + if (!v) return STATUS_INVALID_PARAMETER; + return IoReadDiskSectors(v->DriveIndex, lba, count, buf); } static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, const void *buf) { - if (!v || !v->Hal || !v->Hal->Disk.WriteSectors) - return STATUS_INVALID_PARAMETER; - return v->Hal->Disk.WriteSectors(v->DriveIndex, lba, count, buf); + if (!v) return STATUS_INVALID_PARAMETER; + return IoWriteDiskSectors(v->DriveIndex, lba, count, buf); } // Convenience aliases for file-data I/O. @@ -281,7 +286,7 @@ static void FlushFatCache(FAT32_VOL *v) { for (int i = 0; i < FAT_CACHE_SLOTS; i++) { FAT_CACHE_ENTRY *e = &gFatCache[i]; - if (!e->valid || !e->dirty) continue; + if (!e->valid || !e->dirty || e->vol != v) continue; if (e->lba < fat0Start) continue; uint64_t sectorIdx = e->lba - fat0Start; @@ -323,7 +328,7 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) { uint32_t existing = *(uint32_t *)(sec + offInSec); *(uint32_t *)(sec + offInSec) = (existing & 0xF0000000u) | (value & 0x0FFFFFFFu); - FatCacheMarkDirty(sector); + FatCacheMarkDirty(v, sector); return STATUS_SUCCESS; } @@ -441,19 +446,41 @@ static int MatchName83(const char *dirName, const char *dirExt, return *want == '\0' && *p == '\0'; } +static int MatchRaw83(const char dirName[8], const char dirExt[3], + const char wantName[8], const char wantExt[3]) { + return memcmp(dirName, wantName, 8) == 0 && + memcmp(dirExt, wantExt, 3) == 0; +} + static void Make83Name(const char *src, char name8[8], char ext3[3]) { memset(name8, ' ', 8); memset(ext3, ' ', 3); + + // A leading dot is not a FAT 8.3 extension separator. The old version + // turned ".doomrc" into an invalid empty-base short name with extension + // "DOO", making .doomrc, .doom and .doo all alias the same entry. Treat + // leading-dot names as a basename here; the create path will usually emit + // a VFAT long-name entry for them anyway. + const char *base = src; const char *dot = NULL; - for (const char *p = src; *p; p++) if (*p == '.') dot = p; + if (src && src[0] == '.' && src[1]) { + base = src + 1; + } else { + for (const char *p = src; p && *p; p++) if (*p == '.') dot = p; + } + int ni = 0; - const char *end = dot ? dot : src + strlen(src); - for (const char *p = src; p < end && ni < 8; p++) { - char c = *p; if (c >= 'a' && c <= 'z') c -= 32; name8[ni++] = c; + const char *end = dot ? dot : base + strlen(base); + for (const char *p = base; p < end && ni < 8; p++) { + char c = *p; + if (c >= 'a' && c <= 'z') c -= 32; + name8[ni++] = c; } if (dot) { int ei = 0; - for (const char *p = dot+1; *p && ei < 3; p++) { - char c = *p; if (c >= 'a' && c <= 'z') c -= 32; ext3[ei++] = c; + for (const char *p = dot + 1; *p && ei < 3; p++) { + char c = *p; + if (c >= 'a' && c <= 'z') c -= 32; + ext3[ei++] = c; } } } @@ -463,6 +490,255 @@ static void Make83Name(const char *src, char name8[8], char ext3[3]) { // --------------------------------------------------------------------------- #define MAX_LFN 256 +static char FatUpperChar(char c) { + return (c >= 'a' && c <= 'z') ? (char)(c - 32) : c; +} + +static int FatIsLowerChar(char c) { + return c >= 'a' && c <= 'z'; +} + +static int FatIsValidShortChar(char c) { + c = FatUpperChar(c); + if (c >= 'A' && c <= 'Z') return 1; + if (c >= '0' && c <= '9') return 1; + switch (c) { + case '$': case '%': case '\'': case '-': case '_': case '@': + case '~': case '`': case '!': case '(': case ')': case '{': + case '}': case '^': case '#': case '&': + return 1; + default: + return 0; + } +} + +// True when the user spelling can refer to a normal FAT 8.3 entry. This is +// intentionally structural and case-insensitive: "doom1.wad" must still open +// an existing short entry named DOOM1.WAD. Leading-dot names are not 8.3. +static int FatNameIsStructurallyShort83(const char *name) { + if (!name || !name[0]) return 0; + if (name[0] == '.') return 0; + + const char *dot = NULL; + for (const char *p = name; *p; p++) { + if (*p == '.') { + if (dot) return 0; // multiple dots need LFN + dot = p; + } else if (!FatIsValidShortChar(*p)) { + return 0; + } + } + + size_t baseLen = dot ? (size_t)(dot - name) : strlen(name); + size_t extLen = dot ? strlen(dot + 1) : 0; + return baseLen >= 1 && baseLen <= 8 && extLen <= 3; +} + +// Creation uses LFN when needed for structure, or when lowercase would be lost. +static int FatNameNeedsLfnForCreate(const char *name) { + if (!FatNameIsStructurallyShort83(name)) return 1; + for (const char *p = name; *p; p++) + if (FatIsLowerChar(*p)) return 1; + return 0; +} + +static uint8_t LfnChecksum(const char name8[8], const char ext3[3]) { + uint8_t sum = 0; + for (int i = 0; i < 11; i++) { + uint8_t c = (uint8_t)(i < 8 ? name8[i] : ext3[i - 8]); + sum = (uint8_t)(((sum & 1) ? 0x80 : 0) + (sum >> 1) + c); + } + return sum; +} + +static void LfnSetChar(LFN_ENTRY *e, int idx, uint16_t value) { + if (idx < 5) e->Name1[idx] = value; + else if (idx < 11) e->Name2[idx - 5] = value; + else e->Name3[idx - 11] = value; +} + +static void BuildLfnEntry(LFN_ENTRY *e, const char *name, size_t nameLen, + uint32_t ordinal, uint32_t lfnCount, + uint8_t checksum) { + memset(e, 0, sizeof(*e)); + e->Order = (uint8_t)ordinal; + if (ordinal == lfnCount) e->Order |= 0x40; + e->Attr = ATTR_LFN; + e->Type = 0; + e->Checksum = checksum; + e->FirstCluster = 0; + + size_t start = (size_t)(ordinal - 1) * 13u; + for (int i = 0; i < 13; i++) { + size_t pos = start + (size_t)i; + uint16_t ch; + if (pos < nameLen) ch = (uint8_t)name[pos]; + else if (pos == nameLen) ch = 0x0000; + else ch = 0xFFFF; + LfnSetChar(e, i, ch); + } +} + +static void MakeLegacyBrokenDot83(const char *src, char name8[8], char ext3[3]) { + // This exactly models the pre-fix bug for ".doomrc": empty name + "DOO". + memset(name8, ' ', 8); + memset(ext3, ' ', 3); + const char *dot = NULL; + for (const char *p = src; *p; p++) if (*p == '.') dot = p; + if (!dot) return; + int ei = 0; + for (const char *p = dot + 1; *p && ei < 3; p++) + ext3[ei++] = FatUpperChar(*p); +} + +static int ShortNameExistsRaw(FAT32_VOL *v, uint32_t dirCluster, + const char name8[8], const char ext3[3]) { + uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster); + if (!clusterBuf) return 1; // fail closed; caller should pick another alias + + uint32_t cluster = dirCluster; + while (cluster >= 2 && cluster < FAT32_EOC) { + uint64_t clusterLba = ClusterToLba(v, cluster); + if (!KSUCCESS(ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf))) break; + + uint32_t entries = v->BytesPerCluster / sizeof(DIRENT); + DIRENT *dir = (DIRENT *)clusterBuf; + for (uint32_t i = 0; i < entries; i++) { + if (dir[i].Name[0] == 0x00) { MmFreePool(clusterBuf); return 0; } + if ((uint8_t)dir[i].Name[0] == 0xE5 || dir[i].Attr == ATTR_LFN) continue; + if (MatchRaw83(dir[i].Name, dir[i].Ext, name8, ext3)) { + MmFreePool(clusterBuf); + return 1; + } + } + cluster = FatEntry(v, cluster); + } + + MmFreePool(clusterBuf); + return 0; +} + +static void FatAliasAppendChar(char *dst, int *len, int maxLen, char c) { + c = FatUpperChar(c); + if (!FatIsValidShortChar(c)) return; + if (*len < maxLen) dst[(*len)++] = c; +} + +static void BuildAliasStemExt(const char *src, char stem[9], char ext3[3]) { + memset(stem, 0, 9); + memset(ext3, ' ', 3); + + const char *base = src; + const char *dot = NULL; + if (src[0] == '.' && src[1]) { + // Hidden/Unix-style names like .doomrc become alias stem DOOMRC. + base = src + 1; + } else { + for (const char *p = src; *p; p++) if (*p == '.') dot = p; + } + + int slen = 0; + const char *end = dot ? dot : src + strlen(src); + for (const char *p = base; p < end && *p; p++) + FatAliasAppendChar(stem, &slen, 8, *p); + if (slen == 0) { + memcpy(stem, "FILE", 4); + slen = 4; + } + stem[slen] = '\0'; + + if (dot && dot[1]) { + int ei = 0; + for (const char *p = dot + 1; *p && ei < 3; p++) { + char c = FatUpperChar(*p); + if (!FatIsValidShortChar(c)) continue; + ext3[ei++] = c; + } + } +} + +static void MakeAliasAttempt(const char stem[9], const char extIn[3], + uint32_t attempt, char name8[8], char ext3[3]) { + memset(name8, ' ', 8); + memcpy(ext3, extIn, 3); + + char suffix[8]; + int suffixLen = 0; + suffix[suffixLen++] = '~'; + char digits[7]; + int digitLen = 0; + uint32_t n = attempt; + do { + digits[digitLen++] = (char)('0' + (n % 10)); + n /= 10; + } while (n && digitLen < (int)sizeof(digits)); + for (int i = digitLen - 1; i >= 0; i--) + suffix[suffixLen++] = digits[i]; + + int stemLen = (int)strlen(stem); + int keep = 8 - suffixLen; + if (keep < 1) keep = 1; + if (stemLen < keep) keep = stemLen; + memcpy(name8, stem, keep); + memcpy(name8 + keep, suffix, suffixLen); +} + +static int MakeUniqueShortAlias(FAT32_VOL *v, uint32_t dirCluster, + const char *longName, + char name8[8], char ext3[3]) { + char stem[9], extCandidate[3]; + BuildAliasStemExt(longName, stem, extCandidate); + + for (uint32_t attempt = 1; attempt < 1000000; attempt++) { + MakeAliasAttempt(stem, extCandidate, attempt, name8, ext3); + if (!ShortNameExistsRaw(v, dirCluster, name8, ext3)) return 1; + } + return 0; +} + +static KSTATUS DeleteLegacyBrokenDotAlias(FAT32_VOL *v, uint32_t dirCluster, + const char *longName) { + if (!longName || longName[0] != '.' || !longName[1]) return STATUS_SUCCESS; + + char badName[8], badExt[3]; + MakeLegacyBrokenDot83(longName, badName, badExt); + + uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster); + if (!clusterBuf) return STATUS_NO_MEMORY; + + uint32_t cluster = dirCluster; + while (cluster >= 2 && cluster < FAT32_EOC) { + uint64_t clusterLba = ClusterToLba(v, cluster); + KSTATUS s = ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf); + if (!KSUCCESS(s)) { MmFreePool(clusterBuf); return s; } + + uint32_t entries = v->BytesPerCluster / sizeof(DIRENT); + DIRENT *dir = (DIRENT *)clusterBuf; + for (uint32_t i = 0; i < entries; i++) { + if (dir[i].Name[0] == 0x00) { MmFreePool(clusterBuf); return STATUS_SUCCESS; } + if ((uint8_t)dir[i].Name[0] == 0xE5 || dir[i].Attr == ATTR_LFN) continue; + if (MatchRaw83(dir[i].Name, dir[i].Ext, badName, badExt)) { + int emptyBase = 1; + for (int j = 0; j < 8; j++) if (dir[i].Name[j] != ' ') emptyBase = 0; + if (!emptyBase) continue; + + // Do not free the cluster chain here. The broken entry may have + // aliased existing data; deleting the directory entry is enough to + // hide it from hosts and avoids freeing clusters still owned by a + // valid file. + dir[i].Name[0] = (char)0xE5; + s = WriteSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf); + MmFreePool(clusterBuf); + return s; + } + } + cluster = FatEntry(v, cluster); + } + + MmFreePool(clusterBuf); + return STATUS_SUCCESS; +} + static void LfnAppend(char *lfn, const LFN_ENTRY *e) { int seq = (e->Order & 0x3F) - 1; if (seq < 0 || seq >= (MAX_LFN / 13)) return; @@ -498,6 +774,9 @@ static int SearchDir(FAT32_VOL *v, uint32_t dirCluster, const char *name, if (!clusterBuf) return 0; char lfn[MAX_LFN]; int hasLfn = 0; memset(lfn, 0, sizeof(lfn)); + char wantName83[8], wantExt83[3]; + int allowShortMatch = FatNameIsStructurallyShort83(name); + Make83Name(name, wantName83, wantExt83); uint32_t cluster = dirCluster; while (cluster >= 2 && cluster < FAT32_EOC) { @@ -514,7 +793,10 @@ static int SearchDir(FAT32_VOL *v, uint32_t dirCluster, const char *name, int match = 0; if (hasLfn && lfn[0]) match = LfnMatch(lfn, name); - if (!match) match = MatchName83(dir[i].Name, dir[i].Ext, name); + if (!match && allowShortMatch) + match = MatchName83(dir[i].Name, dir[i].Ext, name); + if (!match && allowShortMatch) + match = MatchRaw83(dir[i].Name, dir[i].Ext, wantName83, wantExt83); hasLfn = 0; memset(lfn, 0, sizeof(lfn)); if (match) { @@ -540,7 +822,7 @@ done: // --------------------------------------------------------------------------- // Path traversal // --------------------------------------------------------------------------- -static KSTATUS TraverseEx(FAT32_VOL *v, const char *path, +static KSTATUS Fat32LookupPath(FAT32_VOL *v, const char *path, uint32_t *outCluster, uint32_t *outSize, uint64_t *outDirentLba, uint32_t *outDirentOffset) { uint32_t cluster = v->RootCluster, size = 0; @@ -574,7 +856,7 @@ static KSTATUS TraverseEx(FAT32_VOL *v, const char *path, static KSTATUS Traverse(FAT32_VOL *v, const char *path, uint32_t *outCluster, uint32_t *outSize) { - return TraverseEx(v, path, outCluster, outSize, NULL, NULL); + return Fat32LookupPath(v, path, outCluster, outSize, NULL, NULL); } // --------------------------------------------------------------------------- @@ -629,13 +911,68 @@ static int FindFreeDirent(FAT32_VOL *v, uint32_t dirCluster, return 0; } + +// Find a contiguous run of free directory entries within one directory cluster. +// This is required for VFAT LFN creation because N long-name entries must +// immediately precede the final short entry. +static int FindFreeDirentRun(FAT32_VOL *v, uint32_t dirCluster, uint32_t needed, + uint32_t *outCluster, uint32_t *outFirstIndex) { + if (!needed) return 0; + + uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster); + if (!clusterBuf) return 0; + + uint32_t cluster = dirCluster, prevCluster = dirCluster; + while (cluster >= 2 && cluster < FAT32_EOC) { + uint64_t clusterLba = ClusterToLba(v, cluster); + if (!KSUCCESS(ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf))) { + MmFreePool(clusterBuf); + return 0; + } + + uint32_t entries = v->BytesPerCluster / sizeof(DIRENT); + DIRENT *dir = (DIRENT *)clusterBuf; + uint32_t runStart = 0, runLen = 0; + for (uint32_t i = 0; i < entries; i++) { + uint8_t first = (uint8_t)dir[i].Name[0]; + int freeEntry = (first == 0xE5 || first == 0x00); + if (freeEntry) { + if (runLen == 0) runStart = i; + runLen++; + if (runLen >= needed) { + *outCluster = cluster; + *outFirstIndex = runStart; + MmFreePool(clusterBuf); + return 1; + } + } else { + runLen = 0; + } + } + + prevCluster = cluster; + cluster = FatEntry(v, cluster); + } + + uint32_t newCluster = AppendCluster(v, prevCluster); + if (newCluster) { + *outCluster = newCluster; + *outFirstIndex = 0; + MmFreePool(clusterBuf); + return 1; + } + + MmFreePool(clusterBuf); + return 0; +} + // --------------------------------------------------------------------------- // Open / Close // --------------------------------------------------------------------------- static KSTATUS Fat32Open(IO_FILESYSTEM *fs, const char *path, void **fileCtx) { FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx; uint32_t cluster, size; uint64_t direntLba; uint32_t direntOffset; - KSTATUS s = TraverseEx(v, path, &cluster, &size, &direntLba, &direntOffset); + KSTATUS s = Fat32LookupPath(v, path, &cluster, &size, &direntLba, &direntOffset); if (!KSUCCESS(s)) return s; FAT32_FILE *f = (FAT32_FILE *)MmAllocatePool(sizeof(FAT32_FILE)); if (!f) return STATUS_NO_MEMORY; @@ -849,6 +1186,8 @@ static KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, // --------------------------------------------------------------------------- // Create // --------------------------------------------------------------------------- +static KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize); + static KSTATUS Fat32Create(IO_FILESYSTEM *fs, const char *path, void **fileCtx) { FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx; char parentPath[256], filename[256]; @@ -859,28 +1198,85 @@ static KSTATUS Fat32Create(IO_FILESYSTEM *fs, const char *path, void **fileCtx) KSTATUS s = Traverse(v, parentPath, &parentCluster, &parentSize); if (!KSUCCESS(s)) return s; - { uint32_t ec, es; uint8_t ea; - if (SearchDir(v, parentCluster, filename, &ec, &es, &ea, NULL, NULL)) - return STATUS_OBJECT_NAME_COLLISION; } + { + uint32_t ec, es; uint8_t ea; + uint64_t existingLba = 0; uint32_t existingOffset = 0; + if (SearchDir(v, parentCluster, filename, &ec, &es, &ea, + &existingLba, &existingOffset)) { + if (ea & ATTR_DIRECTORY) + return STATUS_OBJECT_NAME_COLLISION; - uint64_t direntLba; uint32_t direntOffset; - if (!FindFreeDirent(v, parentCluster, &direntLba, &direntOffset)) + FAT32_FILE *f = (FAT32_FILE *)MmAllocatePool(sizeof(FAT32_FILE)); + if (!f) return STATUS_NO_MEMORY; + f->StartCluster = ec; + f->FileSize = es; + f->Vol = v; + f->DirentLba = existingLba; + f->DirentOffset = existingOffset; + + // Create means overwrite by default for regular files. Reuse the + // existing directory entry, drop the old cluster chain, and return + // an open zero-length file instead of allocating a duplicate name. + KSTATUS ts = Fat32Truncate(fs, f, 0); + if (!KSUCCESS(ts)) { MmFreePool(f); return ts; } + *fileCtx = f; + return STATUS_SUCCESS; + } + } + + int useLfn = FatNameNeedsLfnForCreate(filename); + char shortName[8], shortExt[3]; + if (useLfn) { + s = DeleteLegacyBrokenDotAlias(v, parentCluster, filename); + if (!KSUCCESS(s)) return s; + if (!MakeUniqueShortAlias(v, parentCluster, filename, shortName, shortExt)) + return STATUS_OBJECT_NAME_COLLISION; + } else { + Make83Name(filename, shortName, shortExt); + } + + size_t nameLen = strlen(filename); + if (nameLen == 0 || nameLen >= MAX_LFN) return STATUS_INVALID_PARAMETER; + uint32_t lfnCount = useLfn ? (uint32_t)((nameLen + 12u) / 13u) : 0; + uint32_t neededEntries = lfnCount + 1u; + + uint32_t entryCluster, firstIndex; + if (!FindFreeDirentRun(v, parentCluster, neededEntries, &entryCluster, &firstIndex)) return STATUS_NO_MEMORY; - uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector); - if (!sec) return STATUS_NO_MEMORY; - s = ReadSectors(v, direntLba, 1, sec); - if (!KSUCCESS(s)) { MmFreePool(sec); return s; } + uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster); + if (!clusterBuf) return STATUS_NO_MEMORY; - DIRENT *d = (DIRENT *)(sec + direntOffset); + uint64_t clusterLba = ClusterToLba(v, entryCluster); + s = ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf); + if (!KSUCCESS(s)) { MmFreePool(clusterBuf); return s; } + + DIRENT *dir = (DIRENT *)clusterBuf; + uint32_t shortIndex = firstIndex + lfnCount; + + if (useLfn) { + uint8_t checksum = LfnChecksum(shortName, shortExt); + for (uint32_t slot = 0; slot < lfnCount; slot++) { + uint32_t ordinal = lfnCount - slot; + BuildLfnEntry((LFN_ENTRY *)&dir[firstIndex + slot], filename, + nameLen, ordinal, lfnCount, checksum); + } + } + + DIRENT *d = &dir[shortIndex]; memset(d, 0, sizeof(DIRENT)); - Make83Name(filename, d->Name, d->Ext); + memcpy(d->Name, shortName, 8); + memcpy(d->Ext, shortExt, 3); d->Attr = ATTR_ARCHIVE; - s = WriteSectors(v, direntLba, 1, sec); - MmFreePool(sec); + s = WriteSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf); + MmFreePool(clusterBuf); if (!KSUCCESS(s)) return s; + uint64_t shortEntryByte = (uint64_t)shortIndex * sizeof(DIRENT); + uint64_t direntLba = clusterLba + shortEntryByte / v->BytesPerSector; + uint32_t direntOffset = (uint32_t)(shortEntryByte % v->BytesPerSector); + FAT32_FILE *f = (FAT32_FILE *)MmAllocatePool(sizeof(FAT32_FILE)); if (!f) return STATUS_NO_MEMORY; f->StartCluster = 0; f->FileSize = 0; f->Vol = v; @@ -955,7 +1351,7 @@ update_size: static KSTATUS Fat32Delete(IO_FILESYSTEM *fs, const char *path) { FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx; uint32_t cluster, size; uint64_t direntLba; uint32_t direntOffset; - KSTATUS s = TraverseEx(v, path, &cluster, &size, &direntLba, &direntOffset); + KSTATUS s = Fat32LookupPath(v, path, &cluster, &size, &direntLba, &direntOffset); if (!KSUCCESS(s)) return s; if (!direntLba) return STATUS_ACCESS_DENIED; @@ -1143,64 +1539,522 @@ static uint32_t FindFat32Partition(FAT32_VOL *v) { return 0; } + +// --------------------------------------------------------------------------- +// Native async read/write +// --------------------------------------------------------------------------- +// Metadata work (FAT traversal/allocation and partial-cluster read-modify +// preparation) still runs synchronously in the caller/completion context, but +// file data transfers are submitted as disk IRPs and completion is chained from +// the driver's IRP completion callback instead of the generic VFS worker. + +typedef struct FAT32_ASYNC_RW { + LIST_ENTRY Link; + FAT32_VOL* Vol; + FAT32_FILE* File; + uint8_t IsWrite; + + uint64_t Offset; + uint32_t OffsetInCluster; + uint32_t Cluster; + size_t Remaining; + size_t Done; + size_t Requested; + + uint8_t* Dst; + const uint8_t* Src; + uint8_t* RunBuf; + + uint32_t CurrentNext; + uint32_t CurrentLast; + uint32_t CurrentOffset; + size_t CurrentCopy; + + IRP Irp; + IO_FS_ASYNC_COMPLETE Complete; + void* CompleteContext; +} FAT32_ASYNC_RW; + +static SPINLOCK gFat32AsyncLock = SPINLOCK_INIT; +static LIST_ENTRY gFat32AsyncQueue; +static SEMAPHORE* gFat32AsyncSemaphore; +static volatile int gFat32AsyncInitState; + +static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx); +static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx); + +static void Fat32AsyncWorker(void* arg) { + (void)arg; + + for (;;) { + KSTATUS s = KeWaitSemaphore(gFat32AsyncSemaphore); + if (!KSUCCESS(s)) { + KeYield(); + continue; + } + + uintptr_t flags = KeAcquireSpinlock(&gFat32AsyncLock); + LIST_ENTRY* node = ListPopHead(&gFat32AsyncQueue); + KeReleaseSpinlock(&gFat32AsyncLock, flags); + if (!node) continue; + + FAT32_ASYNC_RW* ctx = CONTAINING_RECORD(node, FAT32_ASYNC_RW, Link); + if (ctx->IsWrite) + Fat32AsyncWriteSubmitNext(ctx); + else + Fat32AsyncReadSubmitNext(ctx); + } +} + +static KSTATUS Fat32EnsureAsyncWorkerStarted(void) { + uintptr_t flags = KeAcquireSpinlock(&gFat32AsyncLock); + if (gFat32AsyncInitState == 2) { + KeReleaseSpinlock(&gFat32AsyncLock, flags); + return STATUS_SUCCESS; + } + if (gFat32AsyncInitState == 1) { + KeReleaseSpinlock(&gFat32AsyncLock, flags); + while (gFat32AsyncInitState == 1) + KeYield(); + return gFat32AsyncInitState == 2 ? STATUS_SUCCESS : STATUS_FAILURE; + } + + gFat32AsyncInitState = 1; + LIST_INIT(&gFat32AsyncQueue); + KeReleaseSpinlock(&gFat32AsyncLock, flags); + + KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gFat32AsyncSemaphore); + if (KSUCCESS(s)) { + THREAD* worker = (THREAD*)0; + s = KeCreateKernelThread((uintptr_t)Fat32AsyncWorker, 0, + PriorityNormal, &worker); + (void)worker; + } + + flags = KeAcquireSpinlock(&gFat32AsyncLock); + gFat32AsyncInitState = KSUCCESS(s) ? 2 : 0; + KeReleaseSpinlock(&gFat32AsyncLock, flags); + return s; +} + +static void Fat32AsyncQueueContinuation(FAT32_ASYNC_RW* ctx) { + if (!ctx) return; + + uintptr_t flags = KeAcquireSpinlock(&gFat32AsyncLock); + ListInsertTail(&gFat32AsyncQueue, &ctx->Link); + KeReleaseSpinlock(&gFat32AsyncLock, flags); + KeSignalSemaphore(gFat32AsyncSemaphore); +} + +static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t information) { + if (!ctx) return; + if (ctx->RunBuf) MmFreePool(ctx->RunBuf); + IO_FS_ASYNC_COMPLETE complete = ctx->Complete; + void* completeContext = ctx->CompleteContext; + MmFreePool(ctx); + if (complete) complete(completeContext, status, information); +} + +static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) { + FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx; + if (!ctx) return; + + if (!KSUCCESS(irp->Status)) { + Fat32AsyncFinish(ctx, irp->Status, ctx->Done); + return; + } + + memcpy(ctx->Dst, ctx->RunBuf + ctx->CurrentOffset, ctx->CurrentCopy); + ctx->Dst += ctx->CurrentCopy; + ctx->Done += ctx->CurrentCopy; + ctx->Remaining -= ctx->CurrentCopy; + ctx->OffsetInCluster = 0; + ctx->Cluster = ctx->CurrentNext; + + Fat32AsyncQueueContinuation(ctx); +} + +static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) { + if (ctx->Remaining == 0) { + Fat32AsyncFinish(ctx, STATUS_SUCCESS, ctx->Done); + return; + } + + if (ctx->Cluster < 2 || ctx->Cluster >= FAT32_EOC) { + if (ctx->Done < ctx->Requested) + memset(ctx->Dst, 0, ctx->Requested - ctx->Done); + Fat32AsyncFinish(ctx, STATUS_DEVICE_ERROR, ctx->Done); + return; + } + + FAT32_VOL* v = ctx->Vol; + uint32_t runStart = ctx->Cluster; + uint32_t runSectors = v->SectorsPerCluster; + uint32_t last = ctx->Cluster; + uint32_t next = FatEntry(v, last); + + while (next == last + 1 && next < FAT32_EOC && + runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) { + last = next; + next = FatEntry(v, last); + runSectors += v->SectorsPerCluster; + } + + uint32_t runBytes = runSectors * 512u; + size_t avail = runBytes - ctx->OffsetInCluster; + size_t copy = avail < ctx->Remaining ? avail : ctx->Remaining; + + ctx->CurrentNext = next; + ctx->CurrentLast = last; + ctx->CurrentOffset = ctx->OffsetInCluster; + ctx->CurrentCopy = copy; + + memset(&ctx->Irp, 0, sizeof(ctx->Irp)); + ctx->Irp.DriveIndex = v->DriveIndex; + ctx->Irp.Lba = ClusterToLba(v, runStart); + ctx->Irp.SectorCount = runSectors; + ctx->Irp.Buffer = ctx->RunBuf; + ctx->Irp.CompletionFn = Fat32AsyncReadIrpComplete; + ctx->Irp.CompletionCtx = ctx; + + KSTATUS s = IoSubmitIrp(&ctx->Irp); + if (s != STATUS_PENDING) + Fat32AsyncFinish(ctx, s, ctx->Done); +} + +static KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, + uint64_t offset, void *buf, size_t size, + IO_FS_ASYNC_COMPLETE complete, + void *completeContext) { + (void)fs; + if (!complete) return STATUS_INVALID_PARAMETER; + if (!fileCtx || (!buf && size)) { + complete(completeContext, STATUS_INVALID_PARAMETER, 0); + return STATUS_PENDING; + } + + FAT32_FILE* f = (FAT32_FILE*)fileCtx; + FAT32_VOL* v = f->Vol; + if (offset >= f->FileSize || size == 0) { + complete(completeContext, STATUS_SUCCESS, 0); + return STATUS_PENDING; + } + if (offset + size > f->FileSize) + size = (size_t)(f->FileSize - offset); + + KSTATUS ws = Fat32EnsureAsyncWorkerStarted(); + if (!KSUCCESS(ws)) { + complete(completeContext, ws, 0); + return STATUS_PENDING; + } + + FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)MmAllocatePool(sizeof(FAT32_ASYNC_RW)); + if (!ctx) { + complete(completeContext, STATUS_NO_MEMORY, 0); + return STATUS_PENDING; + } + memset(ctx, 0, sizeof(*ctx)); + LIST_INIT(&ctx->Link); + + ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u); + if (!ctx->RunBuf) { + MmFreePool(ctx); + complete(completeContext, STATUS_NO_MEMORY, 0); + return STATUS_PENDING; + } + + ctx->Vol = v; + ctx->File = f; + ctx->IsWrite = 0; + ctx->Offset = offset; + ctx->OffsetInCluster = (uint32_t)(offset % v->BytesPerCluster); + ctx->Cluster = f->StartCluster; + ctx->Remaining = size; + ctx->Requested = size; + ctx->Dst = (uint8_t*)buf; + ctx->Complete = complete; + ctx->CompleteContext = completeContext; + + uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster); + for (uint32_t i = 0; i < skipClusters && ctx->Cluster < FAT32_EOC; i++) + ctx->Cluster = FatEntry(v, ctx->Cluster); + + Fat32AsyncQueueContinuation(ctx); + return STATUS_PENDING; +} + +static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) { + FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx; + if (!ctx) return; + + if (!KSUCCESS(irp->Status)) { + Fat32AsyncFinish(ctx, irp->Status, ctx->Done); + return; + } + + ctx->Src += ctx->CurrentCopy; + ctx->Done += ctx->CurrentCopy; + ctx->Remaining -= ctx->CurrentCopy; + ctx->OffsetInCluster = 0; + ctx->Cluster = ctx->CurrentNext; + + Fat32AsyncQueueContinuation(ctx); +} + +static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) { + if (ctx->Remaining == 0) { + uint64_t newEnd = ctx->Offset + ctx->Done; + if (newEnd > ctx->File->FileSize) { + ctx->File->FileSize = (uint32_t)newEnd; + DirentUpdateSize(ctx->Vol, ctx->File->DirentLba, + ctx->File->DirentOffset, ctx->File->FileSize); + } + Fat32AsyncFinish(ctx, STATUS_SUCCESS, ctx->Done); + return; + } + + FAT32_VOL* v = ctx->Vol; + if (ctx->Cluster < 2 || ctx->Cluster >= FAT32_EOC) { + if (ctx->CurrentLast >= 2 && ctx->CurrentLast < FAT32_EOC) { + ctx->Cluster = AppendCluster(v, ctx->CurrentLast); + if (!ctx->Cluster) { + Fat32AsyncFinish(ctx, STATUS_NO_MEMORY, ctx->Done); + return; + } + } else { + Fat32AsyncFinish(ctx, STATUS_DEVICE_ERROR, ctx->Done); + return; + } + } + + uint32_t runStart = ctx->Cluster; + uint32_t runSectors = v->SectorsPerCluster; + uint32_t last = ctx->Cluster; + uint32_t next = FatEntry(v, last); + + if (ctx->OffsetInCluster == 0) { + while (next == last + 1 && next < FAT32_EOC && + runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) { + last = next; + next = FatEntry(v, last); + runSectors += v->SectorsPerCluster; + } + } + + uint32_t runBytes = runSectors * 512u; + size_t avail = runBytes - ctx->OffsetInCluster; + size_t copy = avail < ctx->Remaining ? avail : ctx->Remaining; + + bool partial = (ctx->OffsetInCluster > 0) || (copy < (size_t)runBytes); + if (partial) { + KSTATUS rs = ReadSectors(v, ClusterToLba(v, runStart), runSectors, ctx->RunBuf); + if (!KSUCCESS(rs)) { + Fat32AsyncFinish(ctx, rs, ctx->Done); + return; + } + } + + memcpy(ctx->RunBuf + ctx->OffsetInCluster, ctx->Src, copy); + + ctx->CurrentNext = next; + ctx->CurrentLast = last; + ctx->CurrentOffset = ctx->OffsetInCluster; + ctx->CurrentCopy = copy; + + memset(&ctx->Irp, 0, sizeof(ctx->Irp)); + ctx->Irp.DriveIndex = v->DriveIndex; + ctx->Irp.Flags = IRP_FLAG_WRITE; + ctx->Irp.Lba = ClusterToLba(v, runStart); + ctx->Irp.SectorCount = runSectors; + ctx->Irp.Buffer = ctx->RunBuf; + ctx->Irp.CompletionFn = Fat32AsyncWriteIrpComplete; + ctx->Irp.CompletionCtx = ctx; + + KSTATUS s = IoSubmitIrp(&ctx->Irp); + if (s != STATUS_PENDING) + Fat32AsyncFinish(ctx, s, ctx->Done); +} + +static KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, + uint64_t offset, const void *buf, size_t size, + IO_FS_ASYNC_COMPLETE complete, + void *completeContext) { + (void)fs; + if (!complete) return STATUS_INVALID_PARAMETER; + if (!fileCtx || (!buf && size)) { + complete(completeContext, STATUS_INVALID_PARAMETER, 0); + return STATUS_PENDING; + } + if (size == 0) { + complete(completeContext, STATUS_SUCCESS, 0); + return STATUS_PENDING; + } + + KSTATUS ws = Fat32EnsureAsyncWorkerStarted(); + if (!KSUCCESS(ws)) { + complete(completeContext, ws, 0); + return STATUS_PENDING; + } + + FAT32_FILE* f = (FAT32_FILE*)fileCtx; + FAT32_VOL* v = f->Vol; + + if (f->StartCluster < 2) { + uint32_t first = AllocCluster(v); + if (!first) { + complete(completeContext, STATUS_NO_MEMORY, 0); + return STATUS_PENDING; + } + f->StartCluster = first; + DirentUpdateFirstCluster(v, f->DirentLba, f->DirentOffset, first); + } + + uint32_t clusterIndex = (uint32_t)(offset / v->BytesPerCluster); + uint32_t cluster = f->StartCluster; + for (uint32_t i = 0; i < clusterIndex; i++) { + uint32_t next = FatEntry(v, cluster); + if (next < 2 || next >= FAT32_EOC) { + next = AppendCluster(v, cluster); + if (!next) { + complete(completeContext, STATUS_NO_MEMORY, 0); + return STATUS_PENDING; + } + } + cluster = next; + } + + FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)MmAllocatePool(sizeof(FAT32_ASYNC_RW)); + if (!ctx) { + complete(completeContext, STATUS_NO_MEMORY, 0); + return STATUS_PENDING; + } + memset(ctx, 0, sizeof(*ctx)); + LIST_INIT(&ctx->Link); + + ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u); + if (!ctx->RunBuf) { + MmFreePool(ctx); + complete(completeContext, STATUS_NO_MEMORY, 0); + return STATUS_PENDING; + } + + ctx->Vol = v; + ctx->File = f; + ctx->IsWrite = 1; + ctx->Offset = offset; + ctx->OffsetInCluster = (uint32_t)(offset % v->BytesPerCluster); + ctx->Cluster = cluster; + ctx->Remaining = size; + ctx->Requested = size; + ctx->Src = (const uint8_t*)buf; + ctx->Complete = complete; + ctx->CompleteContext = completeContext; + + Fat32AsyncQueueContinuation(ctx); + return STATUS_PENDING; +} + // --------------------------------------------------------------------------- // DriverEntry // --------------------------------------------------------------------------- -static FAT32_VOL gVol; -static IO_FILESYSTEM gFs; +#define FAT32_MAX_VOLUMES 8 + +static FAT32_VOL gVols[FAT32_MAX_VOLUMES]; +static IO_FILESYSTEM gFs[FAT32_MAX_VOLUMES]; + +static void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) { + memset(fs, 0, sizeof(*fs)); + fs->Open = Fat32Open; + fs->Read = Fat32Read; + fs->Write = Fat32Write; + fs->ReadAsync = Fat32ReadAsync; + fs->WriteAsync = Fat32WriteAsync; + fs->Create = Fat32Create; + fs->Delete = Fat32Delete; + fs->Truncate = Fat32Truncate; + fs->GetSize = Fat32GetSize; + fs->Close = Fat32Close; + fs->DriverCtx = vol; + fs->OpenDir = Fat32OpenDir; + fs->ReadDir = Fat32ReadDir; + fs->CloseDir = Fat32CloseDir; +} KSTATUS DriverEntry(const HAL_API *hal) { DbgPrint("FAT32: Init\n"); if (!hal) return STATUS_INVALID_PARAMETER; FatCacheInvalidate(); + (void)hal; - gVol.Hal = hal; - gVol.DriveIndex = 0; + uint32_t mounted = 0; + KSTATUS lastStatus = STATUS_NOT_SUPPORTED; - uint32_t partLba = FindFat32Partition(&gVol); - if (!partLba) { DbgPrint("FAT32: No FAT32 volume found.\n"); return STATUS_NOT_SUPPORTED; } - gVol.PartitionLba = partLba; + for (uint8_t drive = 0; drive < FAT32_MAX_VOLUMES; drive++) { + if (!IoDiskDevicePresent(drive)) + continue; - uint8_t sector[512]; - if (!KSUCCESS(ReadSectors(&gVol, partLba, 1, sector))) return STATUS_DEVICE_ERROR; + FAT32_VOL* v = &gVols[drive]; + memset(v, 0, sizeof(*v)); + v->DriveIndex = drive; - BPB *bpb = (BPB *)sector; - if (bpb->FatSize16 != 0 || bpb->RootEntryCount != 0) return STATUS_NOT_SUPPORTED; + uint32_t partLba = FindFat32Partition(v); + if (!partLba) { + DbgPrint("FAT32: drive %u: no FAT32 volume\n", drive); + continue; + } + v->PartitionLba = partLba; - gVol.BytesPerSector = bpb->BytesPerSector; - gVol.SectorsPerCluster = bpb->SectorsPerCluster; - gVol.BytesPerCluster = bpb->BytesPerSector * bpb->SectorsPerCluster; - gVol.ReservedSectors = bpb->ReservedSectors; - gVol.NumFats = bpb->NumFats; - gVol.FatSize = bpb->FatSize32; - gVol.RootCluster = bpb->RootCluster; - gVol.DataStart = partLba + bpb->ReservedSectors + bpb->NumFats * bpb->FatSize32; - gVol.NextFreeHint = 2; + uint8_t sector[512]; + if (!KSUCCESS(ReadSectors(v, partLba, 1, sector))) { + lastStatus = STATUS_DEVICE_ERROR; + DbgPrint("FAT32: drive %u: failed to read VBR\n", drive); + continue; + } - uint32_t totalSectors = bpb->TotalSectors16 ? bpb->TotalSectors16 : bpb->TotalSectors32; - uint32_t dataSectors = totalSectors - bpb->ReservedSectors - bpb->NumFats * bpb->FatSize32; - gVol.TotalClusters = dataSectors / bpb->SectorsPerCluster; + BPB *bpb = (BPB *)sector; + if (bpb->FatSize16 != 0 || bpb->RootEntryCount != 0) { + lastStatus = STATUS_NOT_SUPPORTED; + DbgPrint("FAT32: drive %u: unsupported FAT variant\n", drive); + continue; + } - gFs.Open = Fat32Open; - gFs.Read = Fat32Read; - gFs.Write = Fat32Write; - gFs.Create = Fat32Create; - gFs.Delete = Fat32Delete; - gFs.Truncate = Fat32Truncate; - gFs.GetSize = Fat32GetSize; - gFs.Close = Fat32Close; - gFs.DriverCtx = &gVol; - gFs.OpenDir = Fat32OpenDir; - gFs.ReadDir = Fat32ReadDir; - gFs.CloseDir = Fat32CloseDir; + v->BytesPerSector = bpb->BytesPerSector; + v->SectorsPerCluster = bpb->SectorsPerCluster; + v->BytesPerCluster = bpb->BytesPerSector * bpb->SectorsPerCluster; + v->ReservedSectors = bpb->ReservedSectors; + v->NumFats = bpb->NumFats; + v->FatSize = bpb->FatSize32; + v->RootCluster = bpb->RootCluster; + v->DataStart = partLba + bpb->ReservedSectors + bpb->NumFats * bpb->FatSize32; + v->NextFreeHint = 2; - IO_VOLUME_PARAMS volParams; - volParams.DiskIndex = gVol.DriveIndex; - volParams.PartitionLba = partLba; - volParams.PartitionSize = totalSectors; + uint32_t totalSectors = bpb->TotalSectors16 ? bpb->TotalSectors16 : bpb->TotalSectors32; + uint32_t dataSectors = totalSectors - bpb->ReservedSectors - bpb->NumFats * bpb->FatSize32; + v->TotalClusters = dataSectors / bpb->SectorsPerCluster; - KSTATUS status = IoRegisterFilesystem(gVol.DriveIndex, &gFs, &volParams); - DbgPrint("FAT32: Init complete (status=%d, clusters=%u).\n", status, gVol.TotalClusters); + Fat32InitVtable(&gFs[drive], v); + + IO_VOLUME_PARAMS volParams; + volParams.DiskIndex = v->DriveIndex; + volParams.PartitionLba = partLba; + volParams.PartitionSize = totalSectors; + + KSTATUS status = IoRegisterFilesystem(drive, &gFs[drive], &volParams); + if (KSUCCESS(status)) { + mounted++; + DbgPrint("FAT32: drive %u mounted (clusters=%u).\n", drive, v->TotalClusters); + } else { + lastStatus = status; + DbgPrint("FAT32: drive %u mount failed status=%d.\n", drive, status); + } + } + + if (!mounted) { + DbgPrint("FAT32: no mountable FAT32 volumes found.\n"); + return lastStatus; + } + + DbgPrint("FAT32: Init complete; mounted %u volume(s).\n", mounted); return STATUS_SUCCESS; } diff --git a/code/drivers/ide/ide.c b/code/drivers/ide/ide.c index 9265b31..bb4bae4 100644 --- a/code/drivers/ide/ide.c +++ b/code/drivers/ide/ide.c @@ -1,592 +1,632 @@ -// ide.c — IDE/ATA driver: PIO (sync fallback) + Bus Master DMA (async) -// -// Architecture -// ------------ -// The DMA path mirrors AHCI exactly: -// SubmitIrp → enqueues IRP + signals gIrpSemaphore → returns STATUS_PENDING -// IdeWorker → dequeues IRP, calls IdeDmaRw(), fires completion hooks -// IdeIsr → clears BM status, signals gDmaEvent to unblock the worker -// -// Graceful degradation: if Bus Master init fails (no PCI IDE controller, -// BAR4 not assigned, PRDT allocation above 4 GB, etc.) SubmitIrp is left -// NULL. HalDiskDispatch sees NULL and falls back to the synchronous PIO -// path with no changes at the call sites. -// -// PRDT layout -// ----------- -// One page allocated via MmAllocateContiguous. We build up to PRDT_MAX_ENTRIES -// entries by walking the transfer buffer in page-sized chunks and looking up -// each page's physical address via MmGetPhysicalAddress. Adjacent physical -// pages are coalesced into a single PRD entry where possible. -// -// LBA48 vs LBA28 -// -------------- -// DMA commands exist in both modes: CMD_READ/WRITE_DMA_EXT (0x25/0x35) for -// LBA48 and CMD_READ/WRITE_DMA (0xC8/0xCA) for LBA28. The existing -// gSupportsLba48 flag picks the right variant. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// --------------------------------------------------------------------------- -// IDE register map (primary channel, base 0x1F0) -// --------------------------------------------------------------------------- -#define IDE_DATA 0x1F0 -#define IDE_ERR 0x1F1 -#define IDE_SECCNT 0x1F2 -#define IDE_LBA0 0x1F3 -#define IDE_LBA1 0x1F4 -#define IDE_LBA2 0x1F5 -#define IDE_DRIVE 0x1F6 -#define IDE_STATUS 0x1F7 -#define IDE_CMD 0x1F7 -#define IDE_ALT 0x3F6 - -#define IDE_BUSY 0x80 -#define IDE_READY 0x40 -#define IDE_ERR_F 0x01 -#define IDE_DRQ 0x08 - -// ATA commands -#define CMD_READ_EXT 0x24 // PIO 48-bit read -#define CMD_WRITE_EXT 0x34 // PIO 48-bit write -#define CMD_READ_DMA_EXT 0x25 // DMA 48-bit read -#define CMD_WRITE_DMA_EXT 0x35 // DMA 48-bit write -#define CMD_READ_DMA 0xC8 // DMA 28-bit read -#define CMD_WRITE_DMA 0xCA // DMA 28-bit write -#define CMD_IDENT 0xEC - -// IDE primary IRQ -#define IDE_IRQ_PRIMARY 14 - -// --------------------------------------------------------------------------- -// Bus Master IDE register offsets (relative to gBmBase, primary channel) -// --------------------------------------------------------------------------- -#define BM_CMD_REG 0x00 // command -#define BM_STATUS_REG 0x02 // status -#define BM_PRDT_REG 0x04 // PRDT physical address (32-bit) - -// BM_CMD_REG bits -#define BM_CMD_START 0x01 // start DMA transfer -#define BM_CMD_READ 0x08 // 1 = disk→memory (read), 0 = memory→disk (write) - -// BM_STATUS_REG bits -#define BM_STATUS_ACTIVE 0x01 // DMA active -#define BM_STATUS_ERROR 0x02 // DMA error (write 1 to clear) -#define BM_STATUS_IRQ 0x04 // interrupt pending (write 1 to clear) - -// --------------------------------------------------------------------------- -// Physical Region Descriptor (PRDT entry, 8 bytes) -// The PRDT itself must not cross a 64 KB boundary and must be 4-byte aligned. -// We allocate a full page so both constraints are trivially satisfied. -// --------------------------------------------------------------------------- -#pragma pack(push, 1) -typedef struct { - uint32_t physAddr; // physical base of this memory region - uint16_t byteCount; // byte count; 0 means 64 KB - uint16_t flags; // bit 15: End Of Table -} PRD; -#pragma pack(pop) - -#define PRD_EOT 0x8000u -// 512 entries × 8 bytes = 4096 bytes = exactly one page; supports 512×64 KB = 32 MB -#define PRDT_MAX_ENTRIES 512 - -// --------------------------------------------------------------------------- -// Module state -// --------------------------------------------------------------------------- -static const HAL_API *gHal; -static DISK_DRIVER gDrv; -static uint64_t gSectorCount = 0; -static bool gSupportsLba48 = false; - -// DMA hardware state -static uint16_t gBmBase = 0; // Bus Master I/O base port (BAR4 & ~3) -static PRD *gPrdtVirt = NULL; // kernel VA of PRDT page -static uint32_t gPrdtPhys = 0; // 32-bit physical address of PRDT - -// Async worker synchronisation -static IRP_QUEUE gIrpQueue; -static SEMAPHORE *gIrpSemaphore = NULL; -static EVENT *gDmaEvent = NULL; // auto-reset; ISR signals after each xfer - -// --------------------------------------------------------------------------- -// Bus Master I/O helpers -// --------------------------------------------------------------------------- -static inline uint8_t BmRead8 (uint8_t reg) { return __inbyte (gBmBase + reg); } -static inline void BmWrite8 (uint8_t reg, uint8_t v) { __outbyte (gBmBase + reg, v); } -static inline void BmWrite32(uint8_t reg, uint32_t v) { __outdword(gBmBase + reg, v); } - -// --------------------------------------------------------------------------- -// PIO polling helpers (unchanged) -// --------------------------------------------------------------------------- -static void IdeWait(void) { - while (__inbyte(IDE_STATUS) & IDE_BUSY) - KeYield(); -} - -static KSTATUS IdeWaitData(void) { - for (int i = 0; i < 4; i++) __inbyte(IDE_ALT); // 400 ns delay - while (true) { - uint8_t st = __inbyte(IDE_STATUS); - if (st & IDE_ERR_F) return STATUS_DEVICE_ERROR; - if (!(st & IDE_BUSY) && (st & IDE_DRQ)) return STATUS_SUCCESS; - KeYield(); - } -} - -// --------------------------------------------------------------------------- -// PIO read/write — sync fallback when DMA is unavailable -// --------------------------------------------------------------------------- -static KSTATUS IdeRead(void *ctx, uint64_t lba, uint32_t count, void *buf) { - (void)ctx; - if (lba + count > gSectorCount) return STATUS_INVALID_PARAMETER; - uint16_t *p = (uint16_t *)buf; - - for (uint32_t i = 0; i < count; i++, lba++) { - IdeWait(); - if (gSupportsLba48) { - __outbyte(IDE_DRIVE, 0x40); - __outbyte(IDE_SECCNT, 0); - __outbyte(IDE_LBA0, (lba >> 24) & 0xFF); - __outbyte(IDE_LBA1, (lba >> 32) & 0xFF); - __outbyte(IDE_LBA2, (lba >> 40) & 0xFF); - __outbyte(IDE_SECCNT, 1); - __outbyte(IDE_LBA0, (uint8_t)lba); - __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); - __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); - __outbyte(IDE_CMD, CMD_READ_EXT); - } else { - __outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF)); - __outbyte(IDE_SECCNT, 1); - __outbyte(IDE_LBA0, (uint8_t)lba); - __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); - __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); - __outbyte(IDE_CMD, 0x20); - } - if (IdeWaitData() != STATUS_SUCCESS) return STATUS_DEVICE_ERROR; - for (int w = 0; w < 256; w++) *p++ = __inword(IDE_DATA); - } - return STATUS_SUCCESS; -} - -static KSTATUS IdeWrite(void *ctx, uint64_t lba, uint32_t count, const void *buf) { - (void)ctx; - if (lba + count > gSectorCount) return STATUS_INVALID_PARAMETER; - const uint16_t *p = (const uint16_t *)buf; - - for (uint32_t i = 0; i < count; i++, lba++) { - IdeWait(); - if (gSupportsLba48) { - __outbyte(IDE_DRIVE, 0x40); - __outbyte(IDE_SECCNT, 0); - __outbyte(IDE_LBA0, (lba >> 24) & 0xFF); - __outbyte(IDE_LBA1, (lba >> 32) & 0xFF); - __outbyte(IDE_LBA2, (lba >> 40) & 0xFF); - __outbyte(IDE_SECCNT, 1); - __outbyte(IDE_LBA0, (uint8_t)lba); - __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); - __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); - __outbyte(IDE_CMD, CMD_WRITE_EXT); - } else { - __outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF)); - __outbyte(IDE_SECCNT, 1); - __outbyte(IDE_LBA0, (uint8_t)lba); - __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); - __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); - __outbyte(IDE_CMD, 0x30); - } - IdeWait(); - for (int w = 0; w < 256; w++) __outword(IDE_DATA, *p++); - __outbyte(IDE_ALT, 0); // flush write cache - if (IdeWaitData() != STATUS_SUCCESS) return STATUS_DEVICE_ERROR; - } - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// BuildPrdt — fill gPrdtVirt[] for a transfer. -// -// Walks the virtual buffer in page-aligned chunks, looks up each chunk's -// physical address via MmGetPhysicalAddress, and coalesces adjacent physical -// pages into a single PRD entry where possible. This is correct regardless of -// whether the pool gave us physically contiguous pages. -// -// Constraints enforced: -// - Each PRD entry covers at most one 4 KB physical page (never crosses -// a page boundary), so the physical address is valid for the whole chunk. -// - Total byte count per entry is ≤ 64 KB (the PRD limit). -// - Physical address must fit in 32 bits (BM IDE hardware limit). -// -// Returns the number of PRD entries written, or 0 on error. -// --------------------------------------------------------------------------- -static uint32_t BuildPrdt(void *virtBuf, uint32_t byteLen) { - uint32_t nPrd = 0; - uintptr_t va = (uintptr_t)virtBuf; - uint32_t remain = byteLen; - - while (remain > 0) { - if (nPrd >= PRDT_MAX_ENTRIES) { - DbgPrint("IDE DMA: transfer too large for PRDT (%u entries)\n", nPrd); - return 0; - } - - // Bytes from va to the end of its 4 KB page (never straddles a page). - uint32_t pageOff = (uint32_t)(va & 0xFFF); - uint32_t inPage = 0x1000 - pageOff; - uint32_t chunk = (remain < inPage) ? remain : inPage; - - PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); - if (phys >> 32) { - // Physical address above 4 GB — BM IDE's 32-bit PRDT can't reach it. - DbgPrint("IDE DMA: buffer page at VA 0x%llx has phys 0x%llx > 4 GB\n", - (uint64_t)va, (uint64_t)phys); - return 0; - } - uint32_t phys32 = (uint32_t)phys; - - // Try to coalesce with the previous PRD entry if the physical regions - // are adjacent and the combined size stays within the 64 KB limit. - if (nPrd > 0) { - PRD *prev = &gPrdtVirt[nPrd - 1]; - uint32_t prevSz = prev->byteCount ? prev->byteCount : 0x10000u; - uint32_t prevEnd = prev->physAddr + prevSz; - uint32_t combined = prevSz + chunk; - if (prevEnd == phys32 && combined <= 0x10000u) { - prev->byteCount = (uint16_t)(combined == 0x10000u ? 0u : combined); - va += chunk; - remain -= chunk; - continue; - } - } - - gPrdtVirt[nPrd].physAddr = phys32; - gPrdtVirt[nPrd].byteCount = (uint16_t)(chunk == 0x10000u ? 0u : chunk); - gPrdtVirt[nPrd].flags = 0; - nPrd++; - va += chunk; - remain -= chunk; - } - - if (nPrd > 0) - gPrdtVirt[nPrd - 1].flags |= PRD_EOT; - - return nPrd; -} - -// --------------------------------------------------------------------------- -// IDE ISR (IRQ 14 — primary channel) -// -// Must be fast — runs with interrupts disabled. We only clear the hardware -// interrupt bits here and signal gDmaEvent to wake the worker thread. -// The worker reads the final status and fires IRP completion hooks. -// --------------------------------------------------------------------------- -static void IdeIsr(void) { - uint8_t bmSt = BmRead8(BM_STATUS_REG); - if (!(bmSt & BM_STATUS_IRQ)) - return; // not our interrupt (spurious or secondary channel) - - // Stop the Bus Master engine. - BmWrite8(BM_CMD_REG, 0x00); - - // Clear interrupt and error sticky bits (write-1-to-clear). - BmWrite8(BM_STATUS_REG, BM_STATUS_IRQ | BM_STATUS_ERROR); - - // Read IDE status and alt-status to fully acknowledge the device interrupt. - (void)__inbyte(IDE_ALT); - (void)__inbyte(IDE_STATUS); - - // Wake the worker thread; it will read the final BM status and complete - // the IRP. gDmaEvent is auto-reset so exactly one waiter is released. - KeSignalEvent(gDmaEvent); -} - -// --------------------------------------------------------------------------- -// IdeDmaRw — program the hardware and block until the ISR completes it. -// -// Called exclusively from IdeWorker. Has full control of the IDE/BM -// registers; no other thread touches them while the worker is running. -// Falls back to PIO inline if BuildPrdt fails (address space issue). -// --------------------------------------------------------------------------- -static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) { - uint32_t byteLen = count * 512u; - - uint32_t nPrd = BuildPrdt(buf, byteLen); - if (nPrd == 0) { - // Graceful inline PIO fallback for awkward buffers. - DbgPrint("IDE DMA: BuildPrdt failed — falling back to PIO\n"); - return write ? IdeWrite(NULL, lba, count, buf) - : IdeRead (NULL, lba, count, buf); - } - - // ---- Program Bus Master ----------------------------------------------- - - // Load PRDT physical address. - BmWrite32(BM_PRDT_REG, gPrdtPhys); - - // Clear any leftover interrupt/error bits from the previous transfer. - BmWrite8(BM_STATUS_REG, BM_STATUS_IRQ | BM_STATUS_ERROR); - - // Set direction: BM_CMD_READ = disk→memory; 0 = memory→disk. - // Do NOT set BM_CMD_START yet — spec says program ATA regs first. - BmWrite8(BM_CMD_REG, write ? 0x00 : BM_CMD_READ); - - // ---- Program ATA registers ------------------------------------------- - IdeWait(); - - if (gSupportsLba48) { - __outbyte(IDE_DRIVE, 0x40); - __outbyte(IDE_SECCNT, (uint8_t)(count >> 8)); // sector count high byte - __outbyte(IDE_LBA0, (uint8_t)(lba >> 24)); - __outbyte(IDE_LBA1, (uint8_t)(lba >> 32)); - __outbyte(IDE_LBA2, (uint8_t)(lba >> 40)); - __outbyte(IDE_SECCNT, (uint8_t)count); // sector count low byte - __outbyte(IDE_LBA0, (uint8_t)lba); - __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); - __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); - __outbyte(IDE_CMD, write ? CMD_WRITE_DMA_EXT : CMD_READ_DMA_EXT); - } else { - __outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF)); - __outbyte(IDE_SECCNT, (uint8_t)count); - __outbyte(IDE_LBA0, (uint8_t)lba); - __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); - __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); - __outbyte(IDE_CMD, write ? CMD_WRITE_DMA : CMD_READ_DMA); - } - - // ---- Start DMA -------------------------------------------------------- - // Set Start bit. Direction bit is already set from above. - BmWrite8(BM_CMD_REG, (write ? 0x00 : BM_CMD_READ) | BM_CMD_START); - - // Block the worker thread until the ISR fires. - KeWaitForEvent(gDmaEvent); - - // ---- Read result ------------------------------------------------------- - // ISR already stopped the BM engine and cleared BM_STATUS_IRQ. - // We still need to check BM_STATUS_ERROR and IDE_ERR_F. - uint8_t bmSt = BmRead8(BM_STATUS_REG); - uint8_t ideSt = __inbyte(IDE_STATUS); - - if (bmSt & BM_STATUS_ERROR) { - DbgPrint("IDE DMA: BM error (bmStatus=0x%02x, ideStatus=0x%02x, ideErr=0x%02x)\n", - bmSt, ideSt, __inbyte(IDE_ERR)); - return STATUS_DEVICE_ERROR; - } - if (ideSt & IDE_ERR_F) { - DbgPrint("IDE DMA: ATA error (ideStatus=0x%02x, ideErr=0x%02x)\n", - ideSt, __inbyte(IDE_ERR)); - return STATUS_DEVICE_ERROR; - } - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// IdeWorker — kernel thread, one per IDE controller. -// -// Blocks on gIrpSemaphore. Each IrpQueueEnqueue increments it by one, so -// the worker wakes once per submitted IRP. Mirrors AhciWorker exactly. -// --------------------------------------------------------------------------- -static void IdeWorker(uintptr_t arg) { - (void)arg; - while (true) { - KeWaitSemaphore(gIrpSemaphore); - - IRP *irp = IrpQueueDequeue(&gIrpQueue); - if (!irp) - KePanic("IdeWorker: semaphore fired but IRP queue is empty — accounting bug\n"); - - if (irp->Lba + irp->SectorCount > gSectorCount) { - irp->Status = STATUS_INVALID_PARAMETER; - } else { - bool write = !!(irp->Flags & IRP_FLAG_WRITE); - irp->Status = IdeDmaRw(irp->Lba, irp->SectorCount, irp->Buffer, write); - } - - if (irp->CompletionEvent) KeSignalEvent(irp->CompletionEvent); - if (irp->CompletionFn) irp->CompletionFn(irp, irp->CompletionCtx); - } -} - -// --------------------------------------------------------------------------- -// IdeSubmitIrp — async entry point wired into DISK_DRIVER.SubmitIrp. -// Enqueues the IRP and immediately returns STATUS_PENDING. -// --------------------------------------------------------------------------- -static KSTATUS IdeSubmitIrp(void *ctx, IRP *irp) { - (void)ctx; - IrpQueueEnqueue(&gIrpQueue, irp); - return STATUS_PENDING; -} - -// --------------------------------------------------------------------------- -// IdeDmaInit — locate the PCI IDE controller, grab BAR4, allocate the PRDT, -// create sync objects, and register IRQ 14. -// -// Returns true on success. On any failure, logs a message and returns false; -// DriverEntry then leaves DISK_DRIVER.SubmitIrp = NULL so the dispatch layer -// falls back to the synchronous PIO path automatically. -// --------------------------------------------------------------------------- -static bool IdeDmaInit(void) { - // ---- Find PCI IDE controller (class 0x01, subclass 0x01) -------------- - uint8_t bus, dev, fn; - if (!gHal->Pci.FindClass(0x01, 0x01, &bus, &dev, &fn)) { - DbgPrint("IDE DMA: No PCI IDE controller found — using PIO\n"); - return false; - } - - // ---- Read BAR4 (Bus Master base, I/O space) --------------------------- - // PCI config offset 0x20. Bit 0 must be 1 for I/O BAR. - uint32_t bar4 = gHal->Pci.Read32(bus, dev, fn, 0x20); - if (!(bar4 & 0x1)) { - DbgPrint("IDE DMA: BAR4 is a memory BAR (0x%08x) — unexpected, using PIO\n", bar4); - return false; - } - uint16_t bmBase = (uint16_t)(bar4 & 0xFFFC); - if (bmBase == 0) { - DbgPrint("IDE DMA: BAR4 is zero (firmware didn't assign it) — using PIO\n"); - return false; - } - DbgPrint("IDE DMA: Bus Master base = 0x%04X (bus %u, dev %u, fn %u)\n", - bmBase, bus, dev, fn); - - // ---- Enable Bus Mastering in PCI Command register (bit 2) ------------ - uint32_t pciCmd = gHal->Pci.Read32(bus, dev, fn, 0x04); - gHal->Pci.Write32(bus, dev, fn, 0x04, pciCmd | (1u << 2)); - - // ---- Allocate PRDT page (must be 32-bit physical, 4-byte aligned) ---- - PHYSICAL_ADDRESS prdtPhys = MmAllocateContiguous(1); - if (!prdtPhys) { - DbgPrint("IDE DMA: Failed to allocate PRDT page\n"); - return false; - } - if (prdtPhys >> 32) { - // MmAllocateContiguous returned memory above 4 GB — BM IDE can't - // address it. This shouldn't happen on normal x86 systems with early - // boot allocations but we check defensively. - DbgPrint("IDE DMA: PRDT page is above 4 GB (0x%llx) — using PIO\n", - (uint64_t)prdtPhys); - return false; - } - void *prdtVirt = gHal->PhysToVirt(prdtPhys); - DbgPrint("IDE DMA: PRDT at phys 0x%08x, virt %p\n", (uint32_t)prdtPhys, prdtVirt); - - // ---- Create synchronisation objects ------------------------------------ - SEMAPHORE *sem = NULL; - if (!KSUCCESS(KeSemaphoreCreate(0, 0x7FFFFFFF, &sem))) { - DbgPrint("IDE DMA: Failed to create IRP semaphore\n"); - return false; - } - - EVENT *dmaEv = NULL; - if (!KSUCCESS(KeEventCreate(EventAutoReset, 0, &dmaEv))) { - DbgPrint("IDE DMA: Failed to create DMA completion event\n"); - return false; - } - - // ---- Commit module state ----------------------------------------------- - gBmBase = bmBase; - gPrdtPhys = (uint32_t)prdtPhys; - gPrdtVirt = (PRD *)prdtVirt; - gIrpSemaphore = sem; - gDmaEvent = dmaEv; - - IrpQueueInit(&gIrpQueue, sem); - - // ---- Register IRQ 14 -------------------------------------------------- - KeIrqRegister(IDE_IRQ_PRIMARY, IdeIsr); - DbgPrint("IDE DMA: IRQ %u registered\n", IDE_IRQ_PRIMARY); - - // ---- Spawn worker thread ----------------------------------------------- - KSTATUS st = KeCreateKernelThread((uintptr_t)IdeWorker, 0, - PriorityHigh, (THREAD **)NULL); - if (!KSUCCESS(st)) { - DbgPrint("IDE DMA: Worker thread creation failed (0x%x)\n", st); - // Not fatal for init — the PRDT/IRQ are live but async callers will - // never get completions. Better to surface the failure clearly. - return false; - } - - DbgPrint("IDE DMA: Init complete\n"); - return true; -} - -// --------------------------------------------------------------------------- -// DriverEntry -// --------------------------------------------------------------------------- -KSTATUS DriverEntry(const HAL_API *hal) { - DbgPrint("IDE: Init\n"); - gHal = hal; - - // ---- IDENTIFY drive (same as before) ---------------------------------- - __outbyte(IDE_DRIVE, 0xA0); - for (int i = 0; i < 4; i++) __inbyte(IDE_ALT); - - __outbyte(IDE_SECCNT, 0); - __outbyte(IDE_LBA0, 0); - __outbyte(IDE_LBA1, 0); - __outbyte(IDE_LBA2, 0); - __outbyte(IDE_CMD, CMD_IDENT); - - if (__inbyte(IDE_STATUS) == 0) { - DbgPrint("IDE: No drive on primary channel\n"); - return STATUS_NOT_FOUND; - } - - int timeout = 10000; - while (__inbyte(IDE_STATUS) & IDE_BUSY) { - if (--timeout == 0) return STATUS_DEVICE_ERROR; - KeYield(); - } - - if (__inbyte(IDE_LBA1) != 0 || __inbyte(IDE_LBA2) != 0) { - DbgPrint("IDE: Not a standard ATA drive (ATAPI?)\n"); - return STATUS_NOT_SUPPORTED; - } - - timeout = 10000; - while (true) { - uint8_t st = __inbyte(IDE_STATUS); - if (st & IDE_ERR_F) return STATUS_DEVICE_ERROR; - if (st & IDE_DRQ) break; - if (--timeout == 0) return STATUS_DEVICE_ERROR; - KeYield(); - } - - uint16_t ident[256]; - for (int i = 0; i < 256; i++) ident[i] = __inword(IDE_DATA); - - gSupportsLba48 = (ident[83] & (1 << 10)) != 0; - if (gSupportsLba48) { - gSectorCount = *(uint64_t *)&ident[100]; - DbgPrint("IDE: LBA48, %llu sectors\n", gSectorCount); - } else { - gSectorCount = ((uint32_t)ident[61] << 16) | ident[60]; - DbgPrint("IDE: LBA28, %u sectors\n", (uint32_t)gSectorCount); - } - if (gSectorCount == 0) return STATUS_NOT_FOUND; - - // ---- Try to initialise Bus Master DMA --------------------------------- - // On success, wire in IdeSubmitIrp. On failure, leave it NULL so - // HalDiskDispatch falls back to the synchronous PIO callbacks. - bool dmaOk = IdeDmaInit(); - - gDrv.Ctx = NULL; - gDrv.ReadSectors = IdeRead; - gDrv.WriteSectors = IdeWrite; - gDrv.SubmitIrp = dmaOk ? IdeSubmitIrp : NULL; - gDrv.SectorCount = gSectorCount; - - DbgPrint("IDE: Registered (%s)\n", dmaOk ? "DMA" : "PIO fallback"); - return IoRegisterDisk(0, &gDrv); -} +// ide.c — IDE/ATA driver: PIO (sync fallback) + Bus Master DMA (async) +// +// Architecture +// ------------ +// The dispatch path mirrors AHCI: +// IoCallDriver → IdeDispatchReadWrite → IRP queue or inline PIO +// IdeWorker → dequeues IRP, calls IdeDmaRw(), completes the IRP +// IdeIsr → clears BM status, queues a DPC +// IdeDpc → signals gDmaEvent to unblock the worker +// +// Graceful degradation: if Bus Master init fails (no PCI IDE controller, +// BAR4 not assigned, PRDT allocation above 4 GB, etc.) the same device +// object remains registered and dispatch completes IRPs inline through PIO. +// +// PRDT layout +// ----------- +// One page allocated via MmAllocateContiguous. We build up to PRDT_MAX_ENTRIES +// entries by walking the transfer buffer in page-sized chunks and looking up +// each page's physical address via MmGetPhysicalAddress. Adjacent physical +// pages are coalesced into a single PRD entry where possible. +// +// LBA48 vs LBA28 +// -------------- +// DMA commands exist in both modes: CMD_READ/WRITE_DMA_EXT (0x25/0x35) for +// LBA48 and CMD_READ/WRITE_DMA (0xC8/0xCA) for LBA28. The existing +// gSupportsLba48 flag picks the right variant. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// IDE register map (primary channel, base 0x1F0) +// --------------------------------------------------------------------------- +#define IDE_DATA 0x1F0 +#define IDE_ERR 0x1F1 +#define IDE_SECCNT 0x1F2 +#define IDE_LBA0 0x1F3 +#define IDE_LBA1 0x1F4 +#define IDE_LBA2 0x1F5 +#define IDE_DRIVE 0x1F6 +#define IDE_STATUS 0x1F7 +#define IDE_CMD 0x1F7 +#define IDE_ALT 0x3F6 + +#define IDE_BUSY 0x80 +#define IDE_READY 0x40 +#define IDE_ERR_F 0x01 +#define IDE_DRQ 0x08 + +// ATA commands +#define CMD_READ_EXT 0x24 // PIO 48-bit read +#define CMD_WRITE_EXT 0x34 // PIO 48-bit write +#define CMD_READ_DMA_EXT 0x25 // DMA 48-bit read +#define CMD_WRITE_DMA_EXT 0x35 // DMA 48-bit write +#define CMD_READ_DMA 0xC8 // DMA 28-bit read +#define CMD_WRITE_DMA 0xCA // DMA 28-bit write +#define CMD_IDENT 0xEC + +// IDE primary IRQ +#define IDE_IRQ_PRIMARY 14 + +// --------------------------------------------------------------------------- +// Bus Master IDE register offsets (relative to gBmBase, primary channel) +// --------------------------------------------------------------------------- +#define BM_CMD_REG 0x00 // command +#define BM_STATUS_REG 0x02 // status +#define BM_PRDT_REG 0x04 // PRDT physical address (32-bit) + +// BM_CMD_REG bits +#define BM_CMD_START 0x01 // start DMA transfer +#define BM_CMD_READ 0x08 // 1 = disk→memory (read), 0 = memory→disk (write) + +// BM_STATUS_REG bits +#define BM_STATUS_ACTIVE 0x01 // DMA active +#define BM_STATUS_ERROR 0x02 // DMA error (write 1 to clear) +#define BM_STATUS_IRQ 0x04 // interrupt pending (write 1 to clear) + +// --------------------------------------------------------------------------- +// Physical Region Descriptor (PRDT entry, 8 bytes) +// The PRDT itself must not cross a 64 KB boundary and must be 4-byte aligned. +// We allocate a full page so both constraints are trivially satisfied. +// --------------------------------------------------------------------------- +#pragma pack(push, 1) +typedef struct { + uint32_t physAddr; // physical base of this memory region + uint16_t byteCount; // byte count; 0 means 64 KB + uint16_t flags; // bit 15: End Of Table +} PRD; +#pragma pack(pop) + +#define PRD_EOT 0x8000u +// 512 entries × 8 bytes = 4096 bytes = exactly one page; supports 512×64 KB = 32 MB +#define PRDT_MAX_ENTRIES 512 + +// --------------------------------------------------------------------------- +// Module state +// --------------------------------------------------------------------------- +static const HAL_API *gHal; +static DRIVER_OBJECT gDriver; +static DEVICE_OBJECT *gDevice = NULL; +static bool gDmaEnabled = false; +static uint64_t gSectorCount = 0; +static bool gSupportsLba48 = false; + +// DMA hardware state +static uint16_t gBmBase = 0; // Bus Master I/O base port (BAR4 & ~3) +static PRD *gPrdtVirt = NULL; // kernel VA of PRDT page +static uint32_t gPrdtPhys = 0; // 32-bit physical address of PRDT + +// Async worker synchronisation +static IRP_QUEUE gIrpQueue; +static SEMAPHORE *gIrpSemaphore = NULL; +static EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer +static KDPC gDmaDpc; // bottom half for DMA completion + +// --------------------------------------------------------------------------- +// Bus Master I/O helpers +// --------------------------------------------------------------------------- +static inline uint8_t BmRead8 (uint8_t reg) { return __inbyte (gBmBase + reg); } +static inline void BmWrite8 (uint8_t reg, uint8_t v) { __outbyte (gBmBase + reg, v); } +static inline void BmWrite32(uint8_t reg, uint32_t v) { __outdword(gBmBase + reg, v); } + +// --------------------------------------------------------------------------- +// PIO polling helpers (unchanged) +// --------------------------------------------------------------------------- +static void IdeWait(void) { + while (__inbyte(IDE_STATUS) & IDE_BUSY) + KeYield(); +} + +static KSTATUS IdeWaitData(void) { + for (int i = 0; i < 4; i++) __inbyte(IDE_ALT); // 400 ns delay + while (true) { + uint8_t st = __inbyte(IDE_STATUS); + if (st & IDE_ERR_F) return STATUS_DEVICE_ERROR; + if (!(st & IDE_BUSY) && (st & IDE_DRQ)) return STATUS_SUCCESS; + KeYield(); + } +} + +// --------------------------------------------------------------------------- +// PIO read/write — sync fallback when DMA is unavailable +// --------------------------------------------------------------------------- +static KSTATUS IdeRead(void *ctx, uint64_t lba, uint32_t count, void *buf) { + (void)ctx; + if (lba + count > gSectorCount) return STATUS_INVALID_PARAMETER; + uint16_t *p = (uint16_t *)buf; + + for (uint32_t i = 0; i < count; i++, lba++) { + IdeWait(); + if (gSupportsLba48) { + __outbyte(IDE_DRIVE, 0x40); + __outbyte(IDE_SECCNT, 0); + __outbyte(IDE_LBA0, (lba >> 24) & 0xFF); + __outbyte(IDE_LBA1, (lba >> 32) & 0xFF); + __outbyte(IDE_LBA2, (lba >> 40) & 0xFF); + __outbyte(IDE_SECCNT, 1); + __outbyte(IDE_LBA0, (uint8_t)lba); + __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); + __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); + __outbyte(IDE_CMD, CMD_READ_EXT); + } else { + __outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF)); + __outbyte(IDE_SECCNT, 1); + __outbyte(IDE_LBA0, (uint8_t)lba); + __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); + __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); + __outbyte(IDE_CMD, 0x20); + } + if (IdeWaitData() != STATUS_SUCCESS) return STATUS_DEVICE_ERROR; + for (int w = 0; w < 256; w++) *p++ = __inword(IDE_DATA); + } + return STATUS_SUCCESS; +} + +static KSTATUS IdeWrite(void *ctx, uint64_t lba, uint32_t count, const void *buf) { + (void)ctx; + if (lba + count > gSectorCount) return STATUS_INVALID_PARAMETER; + const uint16_t *p = (const uint16_t *)buf; + + for (uint32_t i = 0; i < count; i++, lba++) { + IdeWait(); + if (gSupportsLba48) { + __outbyte(IDE_DRIVE, 0x40); + __outbyte(IDE_SECCNT, 0); + __outbyte(IDE_LBA0, (lba >> 24) & 0xFF); + __outbyte(IDE_LBA1, (lba >> 32) & 0xFF); + __outbyte(IDE_LBA2, (lba >> 40) & 0xFF); + __outbyte(IDE_SECCNT, 1); + __outbyte(IDE_LBA0, (uint8_t)lba); + __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); + __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); + __outbyte(IDE_CMD, CMD_WRITE_EXT); + } else { + __outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF)); + __outbyte(IDE_SECCNT, 1); + __outbyte(IDE_LBA0, (uint8_t)lba); + __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); + __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); + __outbyte(IDE_CMD, 0x30); + } + IdeWait(); + for (int w = 0; w < 256; w++) __outword(IDE_DATA, *p++); + __outbyte(IDE_ALT, 0); // flush write cache + if (IdeWaitData() != STATUS_SUCCESS) return STATUS_DEVICE_ERROR; + } + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// BuildPrdt — fill gPrdtVirt[] for a transfer. +// +// Walks the virtual buffer in page-aligned chunks, looks up each chunk's +// physical address via MmGetPhysicalAddress, and coalesces adjacent physical +// pages into a single PRD entry where possible. This is correct regardless of +// whether the pool gave us physically contiguous pages. +// +// Constraints enforced: +// - Each PRD entry covers at most one 4 KB physical page (never crosses +// a page boundary), so the physical address is valid for the whole chunk. +// - Total byte count per entry is ≤ 64 KB (the PRD limit). +// - Physical address must fit in 32 bits (BM IDE hardware limit). +// +// Returns the number of PRD entries written, or 0 on error. +// --------------------------------------------------------------------------- +static uint32_t BuildPrdt(void *virtBuf, uint32_t byteLen) { + uint32_t nPrd = 0; + uintptr_t va = (uintptr_t)virtBuf; + uint32_t remain = byteLen; + + while (remain > 0) { + if (nPrd >= PRDT_MAX_ENTRIES) { + DbgPrint("IDE DMA: transfer too large for PRDT (%u entries)\n", nPrd); + return 0; + } + + // Bytes from va to the end of its 4 KB page (never straddles a page). + uint32_t pageOff = (uint32_t)(va & 0xFFF); + uint32_t inPage = 0x1000 - pageOff; + uint32_t chunk = (remain < inPage) ? remain : inPage; + + PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); + if (phys >> 32) { + // Physical address above 4 GB — BM IDE's 32-bit PRDT can't reach it. + DbgPrint("IDE DMA: buffer page at VA 0x%llx has phys 0x%llx > 4 GB\n", + (uint64_t)va, (uint64_t)phys); + return 0; + } + uint32_t phys32 = (uint32_t)phys; + + // Try to coalesce with the previous PRD entry if the physical regions + // are adjacent and the combined size stays within the 64 KB limit. + if (nPrd > 0) { + PRD *prev = &gPrdtVirt[nPrd - 1]; + uint32_t prevSz = prev->byteCount ? prev->byteCount : 0x10000u; + uint32_t prevEnd = prev->physAddr + prevSz; + uint32_t combined = prevSz + chunk; + if (prevEnd == phys32 && combined <= 0x10000u) { + prev->byteCount = (uint16_t)(combined == 0x10000u ? 0u : combined); + va += chunk; + remain -= chunk; + continue; + } + } + + gPrdtVirt[nPrd].physAddr = phys32; + gPrdtVirt[nPrd].byteCount = (uint16_t)(chunk == 0x10000u ? 0u : chunk); + gPrdtVirt[nPrd].flags = 0; + nPrd++; + va += chunk; + remain -= chunk; + } + + if (nPrd > 0) + gPrdtVirt[nPrd - 1].flags |= PRD_EOT; + + return nPrd; +} + +// --------------------------------------------------------------------------- +// IDE DMA completion DPC. The ISR owns hardware acknowledgement; this bottom +// half wakes the worker outside hard IRQ/DIRQL context. +// --------------------------------------------------------------------------- +static void IdeDmaCompletionDpc(KDPC *dpc, void *context, + void *systemArgument1, void *systemArgument2) { + (void)dpc; + (void)context; + (void)systemArgument1; + (void)systemArgument2; + + if (gDmaEvent) + KeSignalEvent(gDmaEvent); +} + +// --------------------------------------------------------------------------- +// IDE ISR (IRQ 14 — primary channel) +// +// Must be fast — runs with interrupts disabled. We only clear the hardware +// interrupt bits here and queue a DPC to wake the worker thread. +// The worker reads the final status and fires IRP completion hooks. +// --------------------------------------------------------------------------- +static void IdeIsr(void) { + uint8_t bmSt = BmRead8(BM_STATUS_REG); + if (!(bmSt & BM_STATUS_IRQ)) + return; // not our interrupt (spurious or secondary channel) + + // Stop the Bus Master engine. + BmWrite8(BM_CMD_REG, 0x00); + + // Clear interrupt and error sticky bits (write-1-to-clear). + BmWrite8(BM_STATUS_REG, BM_STATUS_IRQ | BM_STATUS_ERROR); + + // Read IDE status and alt-status to fully acknowledge the device interrupt. + (void)__inbyte(IDE_ALT); + (void)__inbyte(IDE_STATUS); + + // Wake happens in the DPC. gDmaEvent is auto-reset so exactly one waiter + // is released when the bottom half runs. + KeInsertQueueDpc(&gDmaDpc, (void *)0, (void *)0); +} + +// --------------------------------------------------------------------------- +// IdeDmaRw — program the hardware and block until the ISR completes it. +// +// Called exclusively from IdeWorker. Has full control of the IDE/BM +// registers; no other thread touches them while the worker is running. +// Falls back to PIO inline if BuildPrdt fails (address space issue). +// --------------------------------------------------------------------------- +static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) { + uint32_t byteLen = count * 512u; + + uint32_t nPrd = BuildPrdt(buf, byteLen); + if (nPrd == 0) { + // Graceful inline PIO fallback for awkward buffers. + DbgPrint("IDE DMA: BuildPrdt failed — falling back to PIO\n"); + return write ? IdeWrite(NULL, lba, count, buf) + : IdeRead (NULL, lba, count, buf); + } + + // ---- Program Bus Master ----------------------------------------------- + + // Load PRDT physical address. + BmWrite32(BM_PRDT_REG, gPrdtPhys); + + // Clear any leftover interrupt/error bits from the previous transfer. + BmWrite8(BM_STATUS_REG, BM_STATUS_IRQ | BM_STATUS_ERROR); + + // Set direction: BM_CMD_READ = disk→memory; 0 = memory→disk. + // Do NOT set BM_CMD_START yet — spec says program ATA regs first. + BmWrite8(BM_CMD_REG, write ? 0x00 : BM_CMD_READ); + + // ---- Program ATA registers ------------------------------------------- + IdeWait(); + + if (gSupportsLba48) { + __outbyte(IDE_DRIVE, 0x40); + __outbyte(IDE_SECCNT, (uint8_t)(count >> 8)); // sector count high byte + __outbyte(IDE_LBA0, (uint8_t)(lba >> 24)); + __outbyte(IDE_LBA1, (uint8_t)(lba >> 32)); + __outbyte(IDE_LBA2, (uint8_t)(lba >> 40)); + __outbyte(IDE_SECCNT, (uint8_t)count); // sector count low byte + __outbyte(IDE_LBA0, (uint8_t)lba); + __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); + __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); + __outbyte(IDE_CMD, write ? CMD_WRITE_DMA_EXT : CMD_READ_DMA_EXT); + } else { + __outbyte(IDE_DRIVE, 0xE0 | ((lba >> 24) & 0xF)); + __outbyte(IDE_SECCNT, (uint8_t)count); + __outbyte(IDE_LBA0, (uint8_t)lba); + __outbyte(IDE_LBA1, (uint8_t)(lba >> 8)); + __outbyte(IDE_LBA2, (uint8_t)(lba >> 16)); + __outbyte(IDE_CMD, write ? CMD_WRITE_DMA : CMD_READ_DMA); + } + + // ---- Start DMA -------------------------------------------------------- + // Set Start bit. Direction bit is already set from above. + BmWrite8(BM_CMD_REG, (write ? 0x00 : BM_CMD_READ) | BM_CMD_START); + + // Block the worker thread until the ISR fires. + KeWaitForEvent(gDmaEvent, 0); + + // ---- Read result ------------------------------------------------------- + // ISR already stopped the BM engine and cleared BM_STATUS_IRQ. + // We still need to check BM_STATUS_ERROR and IDE_ERR_F. + uint8_t bmSt = BmRead8(BM_STATUS_REG); + uint8_t ideSt = __inbyte(IDE_STATUS); + + if (bmSt & BM_STATUS_ERROR) { + DbgPrint("IDE DMA: BM error (bmStatus=0x%02x, ideStatus=0x%02x, ideErr=0x%02x)\n", + bmSt, ideSt, __inbyte(IDE_ERR)); + return STATUS_DEVICE_ERROR; + } + if (ideSt & IDE_ERR_F) { + DbgPrint("IDE DMA: ATA error (ideStatus=0x%02x, ideErr=0x%02x)\n", + ideSt, __inbyte(IDE_ERR)); + return STATUS_DEVICE_ERROR; + } + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// IdeWorker — kernel thread, one per IDE controller. +// +// Blocks on gIrpSemaphore. Each IrpQueueEnqueue increments it by one, so +// the worker wakes once per submitted IRP. Mirrors AhciWorker exactly. +// --------------------------------------------------------------------------- +static void IdeWorker(uintptr_t arg) { + (void)arg; + while (true) { + KeWaitSemaphore(gIrpSemaphore); + + IRP *irp = IrpQueueDequeue(&gIrpQueue); + if (!irp) + KePanic("IdeWorker: semaphore fired but IRP queue is empty — accounting bug\n"); + + KSTATUS st; + if (irp->Lba + irp->SectorCount > gSectorCount) { + st = STATUS_INVALID_PARAMETER; + } else { + bool write = !!(irp->Flags & IRP_FLAG_WRITE); + st = IdeDmaRw(irp->Lba, irp->SectorCount, irp->Buffer, write); + } + + IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0); + } +} + +// --------------------------------------------------------------------------- +// IdeDispatchReadWrite — DEVICE_OBJECT dispatch entry for disk IRPs. +// +// If DMA is online, the IRP is queued to the worker and completed later. +// If DMA is unavailable, the IRP is completed inline through the PIO path. +// Either way, accepted IRPs return STATUS_PENDING and complete through +// IoCompleteRequest(). +// --------------------------------------------------------------------------- +static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { + (void)device; + if (!irp || !irp->Buffer || irp->SectorCount == 0) return STATUS_INVALID_PARAMETER; + if (irp->Lba + irp->SectorCount > gSectorCount) return STATUS_INVALID_PARAMETER; + + irp->Status = STATUS_FAILURE; + irp->Information = 0; + + if (gDmaEnabled) { + IrpQueueEnqueue(&gIrpQueue, irp); + return STATUS_PENDING; + } + + bool write = !!(irp->Flags & IRP_FLAG_WRITE); + KSTATUS st = write ? IdeWrite(NULL, irp->Lba, irp->SectorCount, irp->Buffer) + : IdeRead (NULL, irp->Lba, irp->SectorCount, irp->Buffer); + IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)irp->SectorCount * 512u) : 0); + return STATUS_PENDING; +} + +// --------------------------------------------------------------------------- +// IdeDmaInit — locate the PCI IDE controller, grab BAR4, allocate the PRDT, +// create sync objects, and register IRQ 14. +// +// Returns true on success. On any failure, logs a message and returns false; +// dispatch will complete IRPs inline through PIO. +// --------------------------------------------------------------------------- +static bool IdeDmaInit(void) { + // ---- Find PCI IDE controller (class 0x01, subclass 0x01) -------------- + uint8_t bus, dev, fn; + if (!gHal->Pci.FindClass(0x01, 0x01, &bus, &dev, &fn)) { + DbgPrint("IDE DMA: No PCI IDE controller found — using PIO\n"); + return false; + } + + // ---- Read BAR4 (Bus Master base, I/O space) --------------------------- + // PCI config offset 0x20. Bit 0 must be 1 for I/O BAR. + uint32_t bar4 = gHal->Pci.Read32(bus, dev, fn, 0x20); + if (!(bar4 & 0x1)) { + DbgPrint("IDE DMA: BAR4 is a memory BAR (0x%08x) — unexpected, using PIO\n", bar4); + return false; + } + uint16_t bmBase = (uint16_t)(bar4 & 0xFFFC); + if (bmBase == 0) { + DbgPrint("IDE DMA: BAR4 is zero (firmware didn't assign it) — using PIO\n"); + return false; + } + DbgPrint("IDE DMA: Bus Master base = 0x%04X (bus %u, dev %u, fn %u)\n", + bmBase, bus, dev, fn); + + // ---- Enable Bus Mastering in PCI Command register (bit 2) ------------ + uint32_t pciCmd = gHal->Pci.Read32(bus, dev, fn, 0x04); + gHal->Pci.Write32(bus, dev, fn, 0x04, pciCmd | (1u << 2)); + + // ---- Allocate PRDT page (must be 32-bit physical, 4-byte aligned) ---- + PHYSICAL_ADDRESS prdtPhys = MmAllocateContiguous(1); + if (!prdtPhys) { + DbgPrint("IDE DMA: Failed to allocate PRDT page\n"); + return false; + } + if (prdtPhys >> 32) { + // MmAllocateContiguous returned memory above 4 GB — BM IDE can't + // address it. This shouldn't happen on normal x86 systems with early + // boot allocations but we check defensively. + DbgPrint("IDE DMA: PRDT page is above 4 GB (0x%llx) — using PIO\n", + (uint64_t)prdtPhys); + return false; + } + void *prdtVirt = gHal->PhysToVirt(prdtPhys); + DbgPrint("IDE DMA: PRDT at phys 0x%08x, virt %p\n", (uint32_t)prdtPhys, prdtVirt); + + // ---- Create synchronisation objects ------------------------------------ + SEMAPHORE *sem = NULL; + if (!KSUCCESS(KeSemaphoreCreate(0, 0x7FFFFFFF, &sem))) { + DbgPrint("IDE DMA: Failed to create IRP semaphore\n"); + return false; + } + + EVENT *dmaEv = NULL; + if (!KSUCCESS(KeEventCreate(EventAutoReset, 0, &dmaEv))) { + DbgPrint("IDE DMA: Failed to create DMA completion event\n"); + return false; + } + + // ---- Commit module state ----------------------------------------------- + gBmBase = bmBase; + gPrdtPhys = (uint32_t)prdtPhys; + gPrdtVirt = (PRD *)prdtVirt; + gIrpSemaphore = sem; + gDmaEvent = dmaEv; + KeInitializeDpc(&gDmaDpc, IdeDmaCompletionDpc, (void *)0); + + IrpQueueInit(&gIrpQueue, sem); + + // ---- Register IRQ 14 -------------------------------------------------- + KeIrqRegister(IDE_IRQ_PRIMARY, IdeIsr); + DbgPrint("IDE DMA: IRQ %u registered\n", IDE_IRQ_PRIMARY); + + // ---- Spawn worker thread ----------------------------------------------- + KSTATUS st = KeCreateKernelThread((uintptr_t)IdeWorker, 0, + PriorityHigh, (THREAD **)NULL); + if (!KSUCCESS(st)) { + DbgPrint("IDE DMA: Worker thread creation failed (0x%x)\n", st); + // Not fatal for init — the PRDT/IRQ are live but async callers will + // never get completions. Better to surface the failure clearly. + return false; + } + + DbgPrint("IDE DMA: Init complete\n"); + return true; +} + +// --------------------------------------------------------------------------- +// DriverEntry +// --------------------------------------------------------------------------- +KSTATUS DriverEntry(const HAL_API *hal) { + DbgPrint("IDE: Init\n"); + gHal = hal; + + // ---- IDENTIFY drive (same as before) ---------------------------------- + __outbyte(IDE_DRIVE, 0xA0); + for (int i = 0; i < 4; i++) __inbyte(IDE_ALT); + + __outbyte(IDE_SECCNT, 0); + __outbyte(IDE_LBA0, 0); + __outbyte(IDE_LBA1, 0); + __outbyte(IDE_LBA2, 0); + __outbyte(IDE_CMD, CMD_IDENT); + + if (__inbyte(IDE_STATUS) == 0) { + DbgPrint("IDE: No drive on primary channel\n"); + return STATUS_NOT_FOUND; + } + + int timeout = 10000; + while (__inbyte(IDE_STATUS) & IDE_BUSY) { + if (--timeout == 0) return STATUS_DEVICE_ERROR; + KeYield(); + } + + if (__inbyte(IDE_LBA1) != 0 || __inbyte(IDE_LBA2) != 0) { + DbgPrint("IDE: Not a standard ATA drive (ATAPI?)\n"); + return STATUS_NOT_SUPPORTED; + } + + timeout = 10000; + while (true) { + uint8_t st = __inbyte(IDE_STATUS); + if (st & IDE_ERR_F) return STATUS_DEVICE_ERROR; + if (st & IDE_DRQ) break; + if (--timeout == 0) return STATUS_DEVICE_ERROR; + KeYield(); + } + + uint16_t ident[256]; + for (int i = 0; i < 256; i++) ident[i] = __inword(IDE_DATA); + + gSupportsLba48 = (ident[83] & (1 << 10)) != 0; + if (gSupportsLba48) { + gSectorCount = *(uint64_t *)&ident[100]; + DbgPrint("IDE: LBA48, %llu sectors\n", gSectorCount); + } else { + gSectorCount = ((uint32_t)ident[61] << 16) | ident[60]; + DbgPrint("IDE: LBA28, %u sectors\n", (uint32_t)gSectorCount); + } + if (gSectorCount == 0) return STATUS_NOT_FOUND; + + // ---- Register definitive DEVICE_OBJECT -------------------------------- + memset(&gDriver, 0, sizeof(gDriver)); + gDriver.Name = "ide"; + gDriver.MajorFunction[IRP_MJ_READ] = IdeDispatchReadWrite; + gDriver.MajorFunction[IRP_MJ_WRITE] = IdeDispatchReadWrite; + + KSTATUS st = IoCreateDevice(&gDriver, 0, &gDevice); + if (!KSUCCESS(st)) return st; + gDevice->SectorCount = gSectorCount; + gDevice->AlignmentRequirement = 512; + + // ---- Try to initialise Bus Master DMA --------------------------------- + gDmaEnabled = IdeDmaInit(); + + DbgPrint("IDE: Registered (%s)\n", gDmaEnabled ? "DMA" : "PIO fallback"); + return IoRegisterDiskDevice(0, gDevice); +} diff --git a/code/hal/hal_api.h b/code/hal/hal_api.h index 818d1b1..2b59b15 100644 --- a/code/hal/hal_api.h +++ b/code/hal/hal_api.h @@ -1,132 +1,126 @@ -#pragma once -// hal_api.h - HAL interface table exported as HalGetApi() -#include -#include -#include -#include - -typedef struct BOOT_INFO BOOT_INFO; - -typedef struct HAL_SCREEN { - void (*Clear)(uint32_t color); - void (*PutPixel)(int x, int y, uint32_t color); - void (*Blit)(int x, int y, int w, int h, const uint32_t* pixels); - void (*GetSize)(int* width, int* height); - void (*Scroll)(int lines); - void (*PutChar)(char c, int x, int y, uint32_t fg, uint32_t bg); - void (*PrintChar)(char c, uint32_t fg, uint32_t bg); - // Hardware mouse cursor: save/restore background and blit sprite - void (*DrawCursor)(int x, int y); - void (*EraseCursor)(void); -} HAL_SCREEN; - -typedef struct HAL_KEYBOARD { - uint8_t (*Poll)(void); - uint8_t (*ReadScancode)(void); - char (*ReadChar)(void); - uint32_t (*GetModifiers)(void); - void (*SetCallback)(void (*cb)(uint8_t scancode)); - bool (*IsKeyDown)(uint8_t scancode, bool extended); - void (*Isr)(void); - uint8_t IrqNumber; -} HAL_KEYBOARD; - -// HAL_MOUSE – PS/2 auxiliary mouse on IRQ 12. -// Packet format: standard 3-byte PS/2 (buttons, deltaX, deltaY). -// Y is already screen-oriented (positive = down) after ISR inversion. -typedef struct HAL_MOUSE { - // Non-blocking: returns true if a packet is waiting in the ring buffer. - bool (*Poll)(void); - // Non-blocking dequeue. Returns false when buffer is empty. - // buttons: bit0=left, bit1=right, bit2=middle - bool (*Read)(int* dx, int* dy, uint8_t* buttons); - // Callback invoked from ISR context after every completed packet. - void (*SetCallback)(void (*cb)(int dx, int dy, uint8_t buttons)); - // Absolute cursor position, clamped to screen bounds. - void (*GetPosition)(int* x, int* y); - void (*SetPosition)(int x, int y); - // Update the clamping rectangle (call after mode change / resize). - void (*SetBounds)(int w, int h); - // Raw ISR – called by KeIrqDispatch for IRQ 12. - void (*Isr)(void); - uint8_t IrqNumber; // always 12 -} HAL_MOUSE; - -typedef struct HAL_DISK { - KSTATUS (*ReadSectors)(uint8_t drive, uint64_t lba, - uint32_t count, void* buf); - KSTATUS (*WriteSectors)(uint8_t drive, uint64_t lba, - uint32_t count, const void* buf); - bool (*DrivePresent)(uint8_t drive); - uint64_t (*DriveSectorCount)(uint8_t drive); -} HAL_DISK; - -typedef struct HAL_TIMER { - uint64_t (*QueryTicks)(void); - void (*Stall)(uint32_t us); - void (*Isr)(void); - uint8_t IrqNumber; -} HAL_TIMER; - -typedef struct HAL_PCI { - uint32_t (*Read32)(uint8_t bus, uint8_t dev, - uint8_t fn, uint8_t reg); - void (*Write32)(uint8_t bus, uint8_t dev, - uint8_t fn, uint8_t reg, uint32_t val); - bool (*FindDevice)(uint16_t vendor, uint16_t device, - uint8_t* bus, uint8_t* dev, uint8_t* fn); - bool (*FindClass)(uint8_t classCode, uint8_t subclass, - uint8_t* bus, uint8_t* dev, uint8_t* fn); - // Returns the IRQ line from PCI config space (register 0x3C, byte 0). - // Returns 0xFF if no interrupt is wired. - uint8_t (*GetIrqLine)(uint8_t bus, uint8_t dev, uint8_t fn); -} HAL_PCI; - -typedef struct HAL_INTC { - void (*RouteIrq)(uint8_t irq); - void (*MaskIrq)(uint8_t irq); - void (*UnmaskIrq)(uint8_t irq); - void (*SendEoi)(void); -} HAL_INTC; - -typedef struct HAL_POWER { - // ACPI S5 soft-off. Disables interrupts, writes SLP_EN to PM1a/b_CNT, - // and halts if firmware doesn't respond. Does not return. - void (*Shutdown)(void); -} HAL_POWER; - -typedef struct HAL_API { - uint32_t Version; - void *(*PhysToVirt)(uint64_t physAddr); - HAL_SCREEN Screen; - HAL_KEYBOARD Keyboard; - HAL_MOUSE Mouse; - HAL_DISK Disk; - HAL_TIMER Timer; - HAL_PCI Pci; - HAL_INTC Intc; - HAL_POWER Power; -} HAL_API; - -typedef struct HAL_INIT_PARAMS { - // Framebuffer - uintptr_t FramebufferBase; - uint32_t FramebufferWidth; - uint32_t FramebufferHeight; - uint32_t FramebufferStride; - - // ACPI - uintptr_t RsdpPhysical; - - // Memory - uint64_t PhysicalWindowBase; - - // Debug - int DebugEnabled; -} HAL_INIT_PARAMS; - -#define HAL_API_VERSION 1 - -__declspec(dllexport) const HAL_API* HalGetApi(void); -__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)); -__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* params); +#pragma once +// hal_api.h - HAL interface table exported as HalGetApi() +#include +#include +#include +#include + +typedef struct BOOT_INFO BOOT_INFO; + +typedef struct HAL_SCREEN { + void (*Clear)(uint32_t color); + void (*PutPixel)(int x, int y, uint32_t color); + void (*Blit)(int x, int y, int w, int h, const uint32_t* pixels); + void (*GetSize)(int* width, int* height); + void (*Scroll)(int lines); + void (*PutChar)(char c, int x, int y, uint32_t fg, uint32_t bg); + void (*PrintChar)(char c, uint32_t fg, uint32_t bg); + // Hardware mouse cursor: save/restore background and blit sprite + void (*DrawCursor)(int x, int y); + void (*EraseCursor)(void); +} HAL_SCREEN; + +typedef struct HAL_KEYBOARD { + uint8_t (*Poll)(void); + uint8_t (*ReadScancode)(void); + char (*ReadChar)(void); + uint32_t (*GetModifiers)(void); + void (*SetCallback)(void (*cb)(uint8_t scancode)); + bool (*IsKeyDown)(uint8_t scancode, bool extended); + void (*Isr)(void); + // Optional bottom half, queued by the kernel after IRQ dispatch. + void (*Dpc)(void); + uint8_t IrqNumber; +} HAL_KEYBOARD; + +// HAL_MOUSE – PS/2 auxiliary mouse on IRQ 12. +// Packet format: standard 3-byte PS/2 (buttons, deltaX, deltaY). +// Y is already screen-oriented (positive = down) after ISR inversion. +typedef struct HAL_MOUSE { + // Non-blocking: returns true if a packet is waiting in the ring buffer. + bool (*Poll)(void); + // Non-blocking dequeue. Returns false when buffer is empty. + // buttons: bit0=left, bit1=right, bit2=middle + bool (*Read)(int* dx, int* dy, uint8_t* buttons); + // Callback invoked from DPC context after every completed packet. + void (*SetCallback)(void (*cb)(int dx, int dy, uint8_t buttons)); + // Absolute cursor position, clamped to screen bounds. + void (*GetPosition)(int* x, int* y); + void (*SetPosition)(int x, int y); + // Update the clamping rectangle (call after mode change / resize). + void (*SetBounds)(int w, int h); + // Raw ISR – called by KeIrqDispatch for IRQ 12. + void (*Isr)(void); + // Optional bottom half, queued by the kernel after IRQ dispatch. + void (*Dpc)(void); + uint8_t IrqNumber; // always 12 +} HAL_MOUSE; + +typedef struct HAL_TIMER { + uint64_t (*QueryTicks)(void); + void (*Stall)(uint32_t us); + void (*Isr)(void); + uint8_t IrqNumber; +} HAL_TIMER; + +typedef struct HAL_PCI { + uint32_t (*Read32)(uint8_t bus, uint8_t dev, + uint8_t fn, uint8_t reg); + void (*Write32)(uint8_t bus, uint8_t dev, + uint8_t fn, uint8_t reg, uint32_t val); + bool (*FindDevice)(uint16_t vendor, uint16_t device, + uint8_t* bus, uint8_t* dev, uint8_t* fn); + bool (*FindClass)(uint8_t classCode, uint8_t subclass, + uint8_t* bus, uint8_t* dev, uint8_t* fn); + // Returns the IRQ line from PCI config space (register 0x3C, byte 0). + // Returns 0xFF if no interrupt is wired. + uint8_t (*GetIrqLine)(uint8_t bus, uint8_t dev, uint8_t fn); +} HAL_PCI; + +typedef struct HAL_INTC { + void (*RouteIrq)(uint8_t irq); + void (*MaskIrq)(uint8_t irq); + void (*UnmaskIrq)(uint8_t irq); + void (*SendEoi)(void); +} HAL_INTC; + +typedef struct HAL_POWER { + // ACPI S5 soft-off. Disables interrupts, writes SLP_EN to PM1a/b_CNT, + // and halts if firmware doesn't respond. Does not return. + void (*Shutdown)(void); +} HAL_POWER; + +typedef struct HAL_API { + uint32_t Version; + void *(*PhysToVirt)(uint64_t physAddr); + HAL_SCREEN Screen; + HAL_KEYBOARD Keyboard; + HAL_MOUSE Mouse; + HAL_TIMER Timer; + HAL_PCI Pci; + HAL_INTC Intc; + HAL_POWER Power; +} HAL_API; + +typedef struct HAL_INIT_PARAMS { + // Framebuffer + uintptr_t FramebufferBase; + uint32_t FramebufferWidth; + uint32_t FramebufferHeight; + uint32_t FramebufferStride; + + // ACPI + uintptr_t RsdpPhysical; + + // Memory + uint64_t PhysicalWindowBase; + + // Debug + int Debug; +} HAL_INIT_PARAMS; + +#define HAL_API_VERSION 2 + +__declspec(dllexport) const HAL_API* HalGetApi(void); +__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)); +__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* params); diff --git a/code/hal/hal_disk.c b/code/hal/hal_disk.c index 53e04a5..74b72df 100644 --- a/code/hal/hal_disk.c +++ b/code/hal/hal_disk.c @@ -1,93 +1,2 @@ -// hal_disk.c - drive registry; drivers register themselves via IoRegisterDisk. -// -// This file lives in the HAL layer and must NOT reference any kernel sync -// objects (KeEventCreate, KeSignalEvent, KeWaitForEvent, ObDereferenceObject). -// All IRP event signalling is done in kernel/ke_irp.c, which sits above us -// in the dependency graph. -// -// HalDiskDispatch is the one bridge: it calls the driver's SubmitIrp (async) -// or runs the sync callbacks inline and writes irp->Status. It does not -// touch irp->CompletionEvent - that is ke_irp.c's responsibility. - -#include -#include -#include -#include -#include - -#define MAX_DRIVES 8 - -static DISK_DRIVER gDrives[MAX_DRIVES]; -static bool gPresent[MAX_DRIVES]; - -__declspec(dllexport) KSTATUS IoRegisterDisk(uint8_t index, const DISK_DRIVER *driver) { - if (index >= MAX_DRIVES || !driver) return STATUS_INVALID_PARAMETER; - if (gPresent[index]) return STATUS_ALREADY_EXISTS; - gDrives[index] = *driver; - gPresent[index] = true; - return STATUS_SUCCESS; -} - -// ------------------------------------------------------------------------- -// HalDiskDispatch - called exclusively by ke_irp.c::IoSubmitIrp() -// ------------------------------------------------------------------------- -// Separating dispatch here keeps hal_disk.c free of event primitives while -// still allowing ke_irp.c (kernel layer) to fire completion hooks after we -// return. -// -// Returns STATUS_PENDING - driver accepted the IRP asynchronously. -// ke_irp.c must NOT touch irp->CompletionEvent -// yet; the driver will fire it. -// Returns STATUS_SUCCESS - sync fallback ran; irp->Status is already -// written. ke_irp.c should fire completion hooks. -// Returns STATUS_NOT_FOUND - drive not registered. - -__declspec(dllexport) KSTATUS HalDiskDispatch(uint8_t drive, IRP *irp) { - if (drive >= MAX_DRIVES || !gPresent[drive]) return STATUS_NOT_FOUND; - DISK_DRIVER *drv = &gDrives[drive]; - - if (drv->SubmitIrp) - return drv->SubmitIrp(drv->Ctx, irp); // STATUS_PENDING - - // Sync fallback: execute inline, write result, let caller fire hooks. - bool write = !!(irp->Flags & IRP_FLAG_WRITE); - if (write) - irp->Status = drv->WriteSectors(drv->Ctx, irp->Lba, irp->SectorCount, irp->Buffer); - else - irp->Status = drv->ReadSectors (drv->Ctx, irp->Lba, irp->SectorCount, irp->Buffer); - - return STATUS_SUCCESS; // ran synchronously -} - -// ------------------------------------------------------------------------- -// Legacy HAL_DISK vtable - direct sync calls, no IRP involvement. -// Existing callers (FAT32, shell) use this path unchanged. -// ------------------------------------------------------------------------- - -static KSTATUS DiskRead(uint8_t drive, uint64_t lba, - uint32_t count, void *buf) { - if (drive >= MAX_DRIVES || !gPresent[drive]) return STATUS_NOT_FOUND; - return gDrives[drive].ReadSectors(gDrives[drive].Ctx, lba, count, buf); -} - -static KSTATUS DiskWrite(uint8_t drive, uint64_t lba, - uint32_t count, const void *buf) { - if (drive >= MAX_DRIVES || !gPresent[drive]) return STATUS_NOT_FOUND; - return gDrives[drive].WriteSectors(gDrives[drive].Ctx, lba, count, buf); -} - -static bool DiskPresent(uint8_t drive) { - return drive < MAX_DRIVES && gPresent[drive]; -} - -static uint64_t DiskSectorCount(uint8_t drive) { - if (drive >= MAX_DRIVES || !gPresent[drive]) return 0; - return gDrives[drive].SectorCount; -} - -void HalDiskInitialize(HAL_DISK *out) { - out->ReadSectors = DiskRead; - out->WriteSectors = DiskWrite; - out->DrivePresent = DiskPresent; - out->DriveSectorCount = DiskSectorCount; -} +// hal_disk.c intentionally left empty. +// Disk registration and dispatch live in kernel/io_device.c. diff --git a/code/hal/hal_keyboard.c b/code/hal/hal_keyboard.c index 854b7f4..fe7af4d 100644 --- a/code/hal/hal_keyboard.c +++ b/code/hal/hal_keyboard.c @@ -17,6 +17,17 @@ static uint8_t gBuf[KBD_BUF_SIZE]; static volatile int gHead = 0; static volatile int gTail = 0; +// -- Deferred callback queue --------------------------------------------------- +// The hard IRQ path updates the key state and stores completed make codes. +// Optional client callbacks are delivered later from the kernel-queued DPC, +// never directly from DIRQL/ISR context. +#define KBD_DPC_BUF_SIZE 128u +#define KBD_DPC_BUF_MASK (KBD_DPC_BUF_SIZE - 1u) + +static uint8_t gDpcBuf[KBD_DPC_BUF_SIZE]; +static volatile int gDpcHead = 0; +static volatile int gDpcTail = 0; + // -- Live Key State (for GetAsyncKeyState) ------------------------------------ // These arrays track whether a key is currently held down. static volatile uint8_t gKeyState[128]; // Standard scan codes @@ -62,6 +73,21 @@ static const char gShifted[128] = { 0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.' }; +// -- Deferred callback delivery ------------------------------------------------ +static void KbdDpc(void) { + for (;;) { + if (gDpcHead == gDpcTail) + break; + + uint8_t make = gDpcBuf[gDpcHead]; + gDpcHead = (gDpcHead + 1) & (int)KBD_DPC_BUF_MASK; + + void (*cb)(uint8_t) = (void (*)(uint8_t))gCallback; + if (cb) + cb(make); + } +} + // -- ISR: The Engine ----------------------------------------------------------- void KbdIsr(void) { uint8_t sc = __inbyte(PS2_DATA); @@ -108,8 +134,15 @@ void KbdIsr(void) { gTail = next; } - void (*cb)(uint8_t) = (void (*)(uint8_t))gCallback; - if (cb) cb(make); + // Queue optional callback delivery to the DPC path. Keep a separate + // callback queue so the public ReadScancode ring remains independent. + if (gCallback) { + int dpcNext = (gDpcTail + 1) & (int)KBD_DPC_BUF_MASK; + if (dpcNext != gDpcHead) { + gDpcBuf[gDpcTail] = make; + gDpcTail = dpcNext; + } + } } } @@ -146,6 +179,14 @@ static char KbdReadChar(void) { return shifted ? gShifted[make] : gUnshifted[make]; } +static uint32_t KbdGetModifiers(void) { + return gModifiers; +} + +static void KbdSetCallback(void (*cb)(uint8_t scancode)) { + gCallback = cb; +} + void HalKeyboardInitialize(HAL_KEYBOARD *out) { while (__inbyte(PS2_STAT) & 2); __outbyte(PS2_CMD, 0xA8); @@ -154,7 +195,10 @@ void HalKeyboardInitialize(HAL_KEYBOARD *out) { out->Poll = KbdPoll; out->ReadScancode = KbdReadScancode; out->ReadChar = KbdReadChar; + out->GetModifiers = KbdGetModifiers; + out->SetCallback = KbdSetCallback; out->IsKeyDown = KbdIsKeyDown; out->Isr = KbdIsr; + out->Dpc = KbdDpc; out->IrqNumber = 1; } diff --git a/code/hal/hal_main.c b/code/hal/hal_main.c index e1b3cbb..27256e8 100644 --- a/code/hal/hal_main.c +++ b/code/hal/hal_main.c @@ -1,384 +1,381 @@ -// hal_main.c - HAL initialisation and HalGetApi -#include -#include -#include -#include -#include -#include -#include - -// Declared in other hal_*.c files: -void HalScreenInitialize(uintptr_t fb, int w, int h, int stride, HAL_SCREEN* out); -void HalKeyboardInitialize(HAL_KEYBOARD* out); -void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH); -KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out); -void HalPciInitialize(HAL_PCI* out); -void HalDiskInitialize(HAL_DISK* out); -extern void HalSetDebugEnabled(int v); - -// Defined in mm_virt.c - HAL needs it for MMIO window access. -static uint64_t gPhysBase = 0xFFFF880000000000ULL; - -HAL_API gApi; -static int gInitialised = 0; -extern void (*gTimerCallback)(void); - -// --------------------------------------------------------------------------- -// ACPI table walker - minimal, just enough to find APIC and HPET addresses. -// --------------------------------------------------------------------------- -#define ACPI_SIG(a,b,c,d) ((uint32_t)(a)|((uint32_t)(b)<<8)|((uint32_t)(c)<<16)|((uint32_t)(d)<<24)) - -#pragma pack(push, 1) -#pragma warning(disable : 4200) - -// ACPI RSDP (Root System Description Pointer) -typedef struct { - char Signature[8]; - uint8_t Checksum; - char OEMID[6]; - uint8_t Revision; - uint32_t RsdtAddress; - uint32_t Length; - uint64_t XsdtAddress; - uint8_t ExtendedChecksum; - uint8_t Reserved[3]; -} RSDP; - -// ACPI SDT Header -typedef struct { - char Signature[4]; - uint32_t Length; - uint8_t Revision; - uint8_t Checksum; - char OEMID[6]; - char OEMTableID[8]; - uint32_t OEMRevision; - uint32_t CreatorID; - uint32_t CreatorRevision; -} ACPISDTHeader; - -// RSDT (Root System Description Table) -typedef struct { - ACPISDTHeader Header; - uint32_t PointerToOtherSDT[]; -} RSDT; - -// XSDT (Extended System Description Table) -typedef struct { - ACPISDTHeader Header; - uint64_t PointerToOtherSDT[]; -} XSDT; - -// MADT (Multiple APIC Description Table) -typedef struct { - ACPISDTHeader Header; - uint32_t LocalAPICAddress; - uint32_t Flags; - uint8_t Entries[]; -} MADT; - -#pragma pack(pop) - -static uintptr_t gLapicPhys = 0xFEE00000; -static uintptr_t gIoapicPhys = 0xFEC00000; -static uintptr_t gHpetPhys = 0xFED00000; - -// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5) -static uint16_t gPm1aCntBlk = 0; -static uint16_t gPm1bCntBlk = 0; -static uint8_t gSlpTypA = 5; // S5 SLP_TYP for PM1a; 5 is correct for QEMU/most VMs -static uint8_t gSlpTypB = 5; // S5 SLP_TYP for PM1b - -int debugEnabled = 0; -int _fltused; // fuckass -static void hal_putchar(char c) { - gApi.Screen.PrintChar(c, 0x00FF00, 0x000000); -} -void HalDbgPrint(const char* fmt, ...) { - if (!debugEnabled) return; - - va_list args; - va_start(args, fmt); - vprintf(fmt, args); - va_end(args); -} - -static void *HalPhysToVirt(uint64_t phys) { - return (void *)(phys + gPhysBase); -} - -// FADT (Fixed ACPI Description Table) - partial layout up to PM1b_CNT_BLK. -// We only need offsets 40 (DSDT ptr), 64 (PM1a_CNT_BLK), 68 (PM1b_CNT_BLK). -// The full table is ~276 bytes (ACPI 2.0+); we never access beyond offset 68. -#pragma pack(push, 1) -typedef struct { - ACPISDTHeader Header; // 0 (36 bytes) - uint32_t FirmwareCtrl; // 36 - uint32_t Dsdt; // 40 - physical addr of DSDT - uint8_t Reserved1; // 44 - uint8_t PreferredPmProfile; // 45 - uint16_t SciInt; // 46 - uint32_t SmiCmd; // 48 - uint8_t AcpiEnable; // 52 - uint8_t AcpiDisable; // 53 - uint8_t S4BiosReq; // 54 - uint8_t PstateCnt; // 55 - uint32_t Pm1aEvtBlk; // 56 - uint32_t Pm1bEvtBlk; // 60 - uint32_t Pm1aCntBlk; // 64 - I/O port for PM1a control register - uint32_t Pm1bCntBlk; // 68 - I/O port for PM1b control register -} FADT; -#pragma pack(pop) - -// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. -// The encoding looks like: -// 0x08 '_' 'S' '5' '_' ; NameOp + "_S5_" -// 0x12 ; PackageOp + encoded length -// ; usually 0x04 -// 0x0A ; BytePrefix + value (or 0x00 for zero) -// 0x0A -static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) { - for (uint32_t i = 0; i + 9 < len; i++) { - // Match NameOp + "_S5_" + PackageOp - if (aml[i] != 0x08) continue; - if (aml[i+1] != 0x5F || aml[i+2] != 0x53 || - aml[i+3] != 0x35 || aml[i+4] != 0x5F) continue; - if (aml[i+5] != 0x12) continue; - - // PkgLength encoding: low 6 bits = byte count (0 → only this byte); - // bits [7:6] = number of following bytes in the length field. - uint8_t pkgByte = aml[i+6]; - uint32_t pkgExtra = pkgByte >> 6; // 0..3 extra bytes - uint32_t j = i + 7 + pkgExtra; // skip PkgLen field - if (j + 2 >= len) break; - j++; // skip NumElements byte - - // SLP_TYP_A - if (aml[j] == 0x0A && j + 1 < len) { - gSlpTypA = aml[j + 1]; j += 2; - } else if (aml[j] == 0x00) { - gSlpTypA = 0; j++; - } else { - break; // unexpected encoding - keep defaults - } - - // SLP_TYP_B - if (j < len && aml[j] == 0x0A && j + 1 < len) - gSlpTypB = aml[j + 1]; - else if (j < len && aml[j] == 0x00) - gSlpTypB = 0; - - HalDbgPrint("HAL: _S5_ SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n", - gSlpTypA, gSlpTypB); - return; - } - HalDbgPrint("HAL: _S5_ not found in DSDT, using defaults (A=0x%02x B=0x%02x)\n", - gSlpTypA, gSlpTypB); -} - -static int validate_checksum(void* table, uint32_t length) { - uint8_t sum = 0; - uint8_t* bytes = (uint8_t*)table; - for (uint32_t i = 0; i < length; i++) { - sum += bytes[i]; - } - return sum == 0; -} - -static void ParseFadt(const FADT* fadt) { - gPm1aCntBlk = (uint16_t)fadt->Pm1aCntBlk; - gPm1bCntBlk = (uint16_t)fadt->Pm1bCntBlk; - HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%08x\n", - gPm1aCntBlk, gPm1bCntBlk, fadt->Dsdt); - - if (fadt->Dsdt) { - const ACPISDTHeader* dsdt = - (const ACPISDTHeader*)((uintptr_t)fadt->Dsdt + gPhysBase); - if (memcmp(dsdt->Signature, "DSDT", 4) == 0 && - validate_checksum((void*)dsdt, dsdt->Length)) { - const uint8_t* aml = (const uint8_t*)(dsdt + 1); - ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader)); - } - } -} - -static void ParseMadt(MADT* madt) { - gLapicPhys = madt->LocalAPICAddress; - uint8_t* p = (uint8_t *)(madt + 1); - uint8_t* end = (uint8_t *)madt + madt->Header.Length; - while (p < end) { - uint8_t type = p[0], len = p[1]; - if (type == 1) { // I/O APIC entry - gIoapicPhys = *(uint32_t *)(p + 4); - } - p += len ? len : 1; - } -} - -static void WalkAcpi(uintptr_t rsdpPhys) { - if (!rsdpPhys) return; - - // Map RSDP into virtual memory - RSDP* rsdp = (RSDP*)(rsdpPhys + gPhysBase); - - if (memcmp(rsdp->Signature, "RSD PTR ", 8) != 0) { - HalDbgPrint("HAL: Invalid RSDP signature!\n"); - return; - } - - ACPISDTHeader* sdtHeader = NULL; - uint32_t count = 0; - int isXsdt = 0; - - // Safely find and validate the XSDT (ACPI 2.0+) - if (rsdp->Revision >= 2 && rsdp->XsdtAddress != 0) { - XSDT* xsdt = (XSDT*)(rsdp->XsdtAddress + gPhysBase); - - if (validate_checksum(xsdt, xsdt->Header.Length) && - memcmp(xsdt->Header.Signature, "XSDT", 4) == 0) { - - sdtHeader = &xsdt->Header; - count = (xsdt->Header.Length - sizeof(ACPISDTHeader)) / 8; - isXsdt = 1; - } - } - - // Fallback to safely finding and validating the RSDT (ACPI 1.0) - if (sdtHeader == NULL && rsdp->RsdtAddress != 0) { - RSDT* rsdt = (RSDT*)((uintptr_t)rsdp->RsdtAddress + gPhysBase); - - if (validate_checksum(rsdt, rsdt->Header.Length) && - memcmp(rsdt->Header.Signature, "RSDT", 4) == 0) { - - sdtHeader = &rsdt->Header; - count = (rsdt->Header.Length - sizeof(ACPISDTHeader)) / 4; - isXsdt = 0; - } - } - - if (sdtHeader == NULL) { - HalDbgPrint("HAL: No valid XSDT or RSDT found!\n"); - return; - } - - // Walk the tables - for (uint32_t i = 0; i < count; i++) { - uintptr_t tablePhys = 0; - - if (isXsdt) { - tablePhys = ((XSDT*)sdtHeader)->PointerToOtherSDT[i]; - } else { - tablePhys = ((RSDT*)sdtHeader)->PointerToOtherSDT[i]; - } - - // Paranoid alignment check to reject firmware garbage pointers - if (!tablePhys || (tablePhys & 3) != 0) continue; - - ACPISDTHeader* header = (ACPISDTHeader*)(tablePhys + gPhysBase); - - // Ensure the table isn't corrupt before parsing - if (!validate_checksum(header, header->Length)) continue; - - // Parse MADT - if (memcmp(header->Signature, "APIC", 4) == 0) { - ParseMadt((MADT*)header); - } - // Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer - else if (memcmp(header->Signature, "FACP", 4) == 0) { - ParseFadt((const FADT*)header); - } - // Parse HPET - else if (memcmp(header->Signature, "HPET", 4) == 0) { - // HPET base address block is located at offset 44 - // (which is 8 bytes past the 36-byte ACPISDTHeader) - gHpetPhys = *(uint64_t*)((uint8_t*)header + 44) & ~0xFFFULL; - } - } -} - -// --------------------------------------------------------------------------- -// HalShutdown - ACPI S5 soft power-off. Does not return. -// --------------------------------------------------------------------------- -static void HalShutdown(void) { - // Silence! Stop the world! - _disable(); - - // SLP_EN (bit 13) | SLP_TYPx (bits 12:10) - uint16_t valA = (uint16_t)((gSlpTypA << 10) | (1 << 13)); - uint16_t valB = (uint16_t)((gSlpTypB << 10) | (1 << 13)); - - if (gPm1aCntBlk) __outword(gPm1aCntBlk, valA); - if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB); - - // Die! - while (true) __halt(); -} - -// --------------------------------------------------------------------------- -__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) { - if (gInitialised) return STATUS_ALREADY_EXISTS; - gPhysBase = initParams->PhysicalWindowBase; - gApi.PhysToVirt = HalPhysToVirt; - debugEnabled = initParams->DebugEnabled; - - // Screen first so DbgPrint works immediately. - HalScreenInitialize(initParams->FramebufferBase + initParams->PhysicalWindowBase, - (int)initParams->FramebufferWidth, - (int)initParams->FramebufferHeight, - (int)initParams->FramebufferStride, - &gApi.Screen); - - printf_set_putchar(hal_putchar); - - HalDbgPrint("HAL: screen %dx%d\n", - initParams->FramebufferWidth, initParams->FramebufferHeight); - - // Parse ACPI to find LAPIC / IOAPIC / HPET addresses. - if (initParams->RsdpPhysical) WalkAcpi(initParams->RsdpPhysical); - HalDbgPrint("HAL: LAPIC=%016llx IOAPIC=%016llx HPET=%016llx\n", - gLapicPhys, gIoapicPhys, gHpetPhys); - - HalApicInitialize(gLapicPhys, gIoapicPhys); - HalKeyboardInitialize(&gApi.Keyboard); - HalMouseInitialize(&gApi.Mouse, - (int)initParams->FramebufferWidth, - (int)initParams->FramebufferHeight); - - KSTATUS ts = HalTimerInitialize(gLapicPhys, gHpetPhys, &gApi.Timer); - if (!KSUCCESS(ts)) HalDbgPrint("HAL: HPET unavailable, using fallback\n"); - - HalPciInitialize(&gApi.Pci); - HalDiskInitialize(&gApi.Disk); - - gApi.Intc.RouteIrq = HalApicRouteIrq; - gApi.Intc.MaskIrq = HalApicMaskIrq; - gApi.Intc.UnmaskIrq = HalApicUnmaskIrq; - gApi.Intc.SendEoi = HalApicSendEoi; - - gApi.Power.Shutdown = HalShutdown; - - gApi.Version = HAL_API_VERSION; - gInitialised = 1; - - // This shouldn't be here but i have no other way. - // After i send 0xF4 and move the mouse the controller sends an interrupts that doesn't fire - // and because the CPU missed the IRQ 12 edge transition (cli) later when we expect it to fire, it doesn't - // leaving us with no mouse and no keyboard. The dreaded 8042 will hold the CLOCK line low forever. - // To combat this we just go about our day executing the rest of the init and just before the end - // of init we just iterate 256 times to grab whatever the controller left for us ("Drain and Pray"). - for (int i = 0; i < 256; i++) { - if (__inbyte(0x64) & 0x01) { - __inbyte(0x60); // Consume - } - gApi.Timer.Stall(1); // Micro-stalls - } - - HalDbgPrint("HAL: initialisation complete.\n"); - return STATUS_SUCCESS; -} - -__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)) { - gTimerCallback = pKeSchedulerTick; -} - -__declspec(dllexport) const HAL_API* HalGetApi(void) { return &gApi; } +// hal_main.c - HAL initialisation and HalGetApi +#include +#include +#include +#include +#include +#include +#include + +// Declared in other hal_*.c files: +void HalScreenInitialize(uintptr_t fb, int w, int h, int stride, HAL_SCREEN* out); +void HalKeyboardInitialize(HAL_KEYBOARD* out); +void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH); +KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out); +void HalPciInitialize(HAL_PCI* out); +extern void HalSetDebugEnabled(int v); + +// Defined in mm_virt.c - HAL needs it for MMIO window access. +static uint64_t gPhysBase = 0xFFFF880000000000ULL; + +HAL_API gApi; +static int gInitialised = 0; +extern void (*gTimerCallback)(void); + +// --------------------------------------------------------------------------- +// ACPI table walker - minimal, just enough to find APIC and HPET addresses. +// --------------------------------------------------------------------------- +#define ACPI_SIG(a,b,c,d) ((uint32_t)(a)|((uint32_t)(b)<<8)|((uint32_t)(c)<<16)|((uint32_t)(d)<<24)) + +#pragma pack(push, 1) +#pragma warning(disable : 4200) + +// ACPI RSDP (Root System Description Pointer) +typedef struct { + char Signature[8]; + uint8_t Checksum; + char OEMID[6]; + uint8_t Revision; + uint32_t RsdtAddress; + uint32_t Length; + uint64_t XsdtAddress; + uint8_t ExtendedChecksum; + uint8_t Reserved[3]; +} RSDP; + +// ACPI SDT Header +typedef struct { + char Signature[4]; + uint32_t Length; + uint8_t Revision; + uint8_t Checksum; + char OEMID[6]; + char OEMTableID[8]; + uint32_t OEMRevision; + uint32_t CreatorID; + uint32_t CreatorRevision; +} ACPISDTHeader; + +// RSDT (Root System Description Table) +typedef struct { + ACPISDTHeader Header; + uint32_t PointerToOtherSDT[]; +} RSDT; + +// XSDT (Extended System Description Table) +typedef struct { + ACPISDTHeader Header; + uint64_t PointerToOtherSDT[]; +} XSDT; + +// MADT (Multiple APIC Description Table) +typedef struct { + ACPISDTHeader Header; + uint32_t LocalAPICAddress; + uint32_t Flags; + uint8_t Entries[]; +} MADT; + +#pragma pack(pop) + +static uintptr_t gLapicPhys = 0xFEE00000; +static uintptr_t gIoapicPhys = 0xFEC00000; +static uintptr_t gHpetPhys = 0xFED00000; + +// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5) +static uint16_t gPm1aCntBlk = 0; +static uint16_t gPm1bCntBlk = 0; +static uint8_t gSlpTypA = 5; // S5 SLP_TYP for PM1a; 5 is correct for QEMU/most VMs +static uint8_t gSlpTypB = 5; // S5 SLP_TYP for PM1b + +int debugEnabled = 0; +int _fltused; // fuckass +static void hal_putchar(char c) { + gApi.Screen.PrintChar(c, 0x00FF00, 0x000000); +} +void HalDbgPrint(const char* fmt, ...) { + if (!debugEnabled) return; + + va_list args; + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); +} + +static void *HalPhysToVirt(uint64_t phys) { + return (void *)(phys + gPhysBase); +} + +// FADT (Fixed ACPI Description Table) - partial layout up to PM1b_CNT_BLK. +// We only need offsets 40 (DSDT ptr), 64 (PM1a_CNT_BLK), 68 (PM1b_CNT_BLK). +// The full table is ~276 bytes (ACPI 2.0+); we never access beyond offset 68. +#pragma pack(push, 1) +typedef struct { + ACPISDTHeader Header; // 0 (36 bytes) + uint32_t FirmwareCtrl; // 36 + uint32_t Dsdt; // 40 - physical addr of DSDT + uint8_t Reserved1; // 44 + uint8_t PreferredPmProfile; // 45 + uint16_t SciInt; // 46 + uint32_t SmiCmd; // 48 + uint8_t AcpiEnable; // 52 + uint8_t AcpiDisable; // 53 + uint8_t S4BiosReq; // 54 + uint8_t PstateCnt; // 55 + uint32_t Pm1aEvtBlk; // 56 + uint32_t Pm1bEvtBlk; // 60 + uint32_t Pm1aCntBlk; // 64 - I/O port for PM1a control register + uint32_t Pm1bCntBlk; // 68 - I/O port for PM1b control register +} FADT; +#pragma pack(pop) + +// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. +// The encoding looks like: +// 0x08 '_' 'S' '5' '_' ; NameOp + "_S5_" +// 0x12 ; PackageOp + encoded length +// ; usually 0x04 +// 0x0A ; BytePrefix + value (or 0x00 for zero) +// 0x0A +static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) { + for (uint32_t i = 0; i + 9 < len; i++) { + // Match NameOp + "_S5_" + PackageOp + if (aml[i] != 0x08) continue; + if (aml[i+1] != 0x5F || aml[i+2] != 0x53 || + aml[i+3] != 0x35 || aml[i+4] != 0x5F) continue; + if (aml[i+5] != 0x12) continue; + + // PkgLength encoding: low 6 bits = byte count (0 → only this byte); + // bits [7:6] = number of following bytes in the length field. + uint8_t pkgByte = aml[i+6]; + uint32_t pkgExtra = pkgByte >> 6; // 0..3 extra bytes + uint32_t j = i + 7 + pkgExtra; // skip PkgLen field + if (j + 2 >= len) break; + j++; // skip NumElements byte + + // SLP_TYP_A + if (aml[j] == 0x0A && j + 1 < len) { + gSlpTypA = aml[j + 1]; j += 2; + } else if (aml[j] == 0x00) { + gSlpTypA = 0; j++; + } else { + break; // unexpected encoding - keep defaults + } + + // SLP_TYP_B + if (j < len && aml[j] == 0x0A && j + 1 < len) + gSlpTypB = aml[j + 1]; + else if (j < len && aml[j] == 0x00) + gSlpTypB = 0; + + HalDbgPrint("HAL: _S5_ SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n", + gSlpTypA, gSlpTypB); + return; + } + HalDbgPrint("HAL: _S5_ not found in DSDT, using defaults (A=0x%02x B=0x%02x)\n", + gSlpTypA, gSlpTypB); +} + +static int validate_checksum(void* table, uint32_t length) { + uint8_t sum = 0; + uint8_t* bytes = (uint8_t*)table; + for (uint32_t i = 0; i < length; i++) { + sum += bytes[i]; + } + return sum == 0; +} + +static void ParseFadt(const FADT* fadt) { + gPm1aCntBlk = (uint16_t)fadt->Pm1aCntBlk; + gPm1bCntBlk = (uint16_t)fadt->Pm1bCntBlk; + HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%08x\n", + gPm1aCntBlk, gPm1bCntBlk, fadt->Dsdt); + + if (fadt->Dsdt) { + const ACPISDTHeader* dsdt = + (const ACPISDTHeader*)((uintptr_t)fadt->Dsdt + gPhysBase); + if (memcmp(dsdt->Signature, "DSDT", 4) == 0 && + validate_checksum((void*)dsdt, dsdt->Length)) { + const uint8_t* aml = (const uint8_t*)(dsdt + 1); + ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader)); + } + } +} + +static void ParseMadt(MADT* madt) { + gLapicPhys = madt->LocalAPICAddress; + uint8_t* p = (uint8_t *)(madt + 1); + uint8_t* end = (uint8_t *)madt + madt->Header.Length; + while (p < end) { + uint8_t type = p[0], len = p[1]; + if (type == 1) { // I/O APIC entry + gIoapicPhys = *(uint32_t *)(p + 4); + } + p += len ? len : 1; + } +} + +static void WalkAcpi(uintptr_t rsdpPhys) { + if (!rsdpPhys) return; + + // Map RSDP into virtual memory + RSDP* rsdp = (RSDP*)(rsdpPhys + gPhysBase); + + if (memcmp(rsdp->Signature, "RSD PTR ", 8) != 0) { + HalDbgPrint("HAL: Invalid RSDP signature!\n"); + return; + } + + ACPISDTHeader* sdtHeader = NULL; + uint32_t count = 0; + int isXsdt = 0; + + // Safely find and validate the XSDT (ACPI 2.0+) + if (rsdp->Revision >= 2 && rsdp->XsdtAddress != 0) { + XSDT* xsdt = (XSDT*)(rsdp->XsdtAddress + gPhysBase); + + if (validate_checksum(xsdt, xsdt->Header.Length) && + memcmp(xsdt->Header.Signature, "XSDT", 4) == 0) { + + sdtHeader = &xsdt->Header; + count = (xsdt->Header.Length - sizeof(ACPISDTHeader)) / 8; + isXsdt = 1; + } + } + + // Fallback to safely finding and validating the RSDT (ACPI 1.0) + if (sdtHeader == NULL && rsdp->RsdtAddress != 0) { + RSDT* rsdt = (RSDT*)((uintptr_t)rsdp->RsdtAddress + gPhysBase); + + if (validate_checksum(rsdt, rsdt->Header.Length) && + memcmp(rsdt->Header.Signature, "RSDT", 4) == 0) { + + sdtHeader = &rsdt->Header; + count = (rsdt->Header.Length - sizeof(ACPISDTHeader)) / 4; + isXsdt = 0; + } + } + + if (sdtHeader == NULL) { + HalDbgPrint("HAL: No valid XSDT or RSDT found!\n"); + return; + } + + // Walk the tables + for (uint32_t i = 0; i < count; i++) { + uintptr_t tablePhys = 0; + + if (isXsdt) { + tablePhys = ((XSDT*)sdtHeader)->PointerToOtherSDT[i]; + } else { + tablePhys = ((RSDT*)sdtHeader)->PointerToOtherSDT[i]; + } + + // Paranoid alignment check to reject firmware garbage pointers + if (!tablePhys || (tablePhys & 3) != 0) continue; + + ACPISDTHeader* header = (ACPISDTHeader*)(tablePhys + gPhysBase); + + // Ensure the table isn't corrupt before parsing + if (!validate_checksum(header, header->Length)) continue; + + // Parse MADT + if (memcmp(header->Signature, "APIC", 4) == 0) { + ParseMadt((MADT*)header); + } + // Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer + else if (memcmp(header->Signature, "FACP", 4) == 0) { + ParseFadt((const FADT*)header); + } + // Parse HPET + else if (memcmp(header->Signature, "HPET", 4) == 0) { + // HPET base address block is located at offset 44 + // (which is 8 bytes past the 36-byte ACPISDTHeader) + gHpetPhys = *(uint64_t*)((uint8_t*)header + 44) & ~0xFFFULL; + } + } +} + +// --------------------------------------------------------------------------- +// HalShutdown - ACPI S5 soft power-off. Does not return. +// --------------------------------------------------------------------------- +static void HalShutdown(void) { + // Silence! Stop the world! + _disable(); + + // SLP_EN (bit 13) | SLP_TYPx (bits 12:10) + uint16_t valA = (uint16_t)((gSlpTypA << 10) | (1 << 13)); + uint16_t valB = (uint16_t)((gSlpTypB << 10) | (1 << 13)); + + if (gPm1aCntBlk) __outword(gPm1aCntBlk, valA); + if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB); + + // Die! + while (true) __halt(); +} + +// --------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) { + if (gInitialised) return STATUS_ALREADY_EXISTS; + gPhysBase = initParams->PhysicalWindowBase; + gApi.PhysToVirt = HalPhysToVirt; + debugEnabled = initParams->Debug; + + // Screen first so DbgPrint works immediately. + HalScreenInitialize(initParams->FramebufferBase + initParams->PhysicalWindowBase, + (int)initParams->FramebufferWidth, + (int)initParams->FramebufferHeight, + (int)initParams->FramebufferStride, + &gApi.Screen); + + printf_set_putchar(hal_putchar); + + HalDbgPrint("HAL: screen %dx%d\n", + initParams->FramebufferWidth, initParams->FramebufferHeight); + + // Parse ACPI to find LAPIC / IOAPIC / HPET addresses. + if (initParams->RsdpPhysical) WalkAcpi(initParams->RsdpPhysical); + HalDbgPrint("HAL: LAPIC=%016llx IOAPIC=%016llx HPET=%016llx\n", + gLapicPhys, gIoapicPhys, gHpetPhys); + + HalApicInitialize(gLapicPhys, gIoapicPhys); + HalKeyboardInitialize(&gApi.Keyboard); + HalMouseInitialize(&gApi.Mouse, + (int)initParams->FramebufferWidth, + (int)initParams->FramebufferHeight); + + KSTATUS ts = HalTimerInitialize(gLapicPhys, gHpetPhys, &gApi.Timer); + if (!KSUCCESS(ts)) HalDbgPrint("HAL: HPET unavailable, using fallback\n"); + + HalPciInitialize(&gApi.Pci); + gApi.Intc.RouteIrq = HalApicRouteIrq; + gApi.Intc.MaskIrq = HalApicMaskIrq; + gApi.Intc.UnmaskIrq = HalApicUnmaskIrq; + gApi.Intc.SendEoi = HalApicSendEoi; + + gApi.Power.Shutdown = HalShutdown; + + gApi.Version = HAL_API_VERSION; + gInitialised = 1; + + // This shouldn't be here but i have no other way. + // After i send 0xF4 and move the mouse the controller sends an interrupts that doesn't fire + // and because the CPU missed the IRQ 12 edge transition (cli) later when we expect it to fire, it doesn't + // leaving us with no mouse and no keyboard. The dreaded 8042 will hold the CLOCK line low forever. + // To combat this we just go about our day executing the rest of the init and just before the end + // of init we just iterate 256 times to grab whatever the controller left for us ("Drain and Pray"). + for (int i = 0; i < 256; i++) { + if (__inbyte(0x64) & 0x01) { + __inbyte(0x60); // Consume + } + gApi.Timer.Stall(1); // Micro-stalls + } + + HalDbgPrint("HAL: initialisation complete.\n"); + return STATUS_SUCCESS; +} + +__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)) { + gTimerCallback = pKeSchedulerTick; +} + +__declspec(dllexport) const HAL_API* HalGetApi(void) { return &gApi; } diff --git a/code/hal/hal_mouse.c b/code/hal/hal_mouse.c index 727b38c..4f248d8 100644 --- a/code/hal/hal_mouse.c +++ b/code/hal/hal_mouse.c @@ -13,15 +13,15 @@ // Byte 2: delta-Y unsigned magnitude (sign in byte-0 bit 5); PS/2 Y+ = up // // The ISR accumulates the 3-byte packet, converts it to a screen-space delta -// (Y-axis flipped), clamps the absolute position to [0, bounds), enqueues a -// MOUSE_EVENT, and then calls EraseOldCursor / DrawNewCursor via the two -// screen helpers that hal_screen.c exports. +// (Y-axis flipped), and queues the completed packet to the HAL's kernel-queued +// bottom half. Cursor drawing, public event enqueue, and optional callbacks +// run from that bottom half instead of hard IRQ context. // // Threading model // --------------- -// Single-producer (ISR on IRQ 12) / single-consumer ring buffer. -// All ISR-written fields are volatile; on a uniprocessor this is sufficient. -// On SMP, add acquire/release barriers around gHead / gTail. +// Public events are produced by the mouse DPC and consumed by callers. +// Raw completed packets are produced by the ISR and consumed by the same DPC. +// On SMP, add acquire/release barriers around the ring head/tail pairs. #include #include @@ -44,8 +44,13 @@ typedef struct { } MOUSE_EVENT; static MOUSE_EVENT gBuf[MOUSE_BUF_SIZE]; -static volatile int gHead = 0; // consumer read pointer -static volatile int gTail = 0; // producer write pointer (updated by ISR) +static volatile int gHead = 0; // public consumer read pointer +static volatile int gTail = 0; // public producer write pointer (updated by DPC) + +// Completed hardware packets waiting for the DPC bottom half. +static MOUSE_EVENT gDpcBuf[MOUSE_BUF_SIZE]; +static volatile int gDpcHead = 0; +static volatile int gDpcTail = 0; // -- Absolute cursor position & screen bounds ---------------------------------- static volatile int gMouseX = 0; @@ -57,7 +62,7 @@ static volatile int gBoundsH = 1080; static volatile uint8_t gPacket[3]; static volatile int gPacketIdx = 0; // 0, 1, or 2 -// -- Optional callback (called from ISR) --------------------------------------- +// -- Optional callback (called from mouse DPC) --------------------------------- static void (* volatile gCallback)(int dx, int dy, uint8_t buttons) = NULL; // -- Cursor helpers provided by hal_screen.c ----------------------------------- @@ -101,9 +106,45 @@ static uint8_t MouseRecvByte(void) { return __inbyte(PS2_DATA); } +// -- Deferred mouse bottom half ------------------------------------------------- +static void MouseDpc(void) { + for (;;) { + if (gDpcHead == gDpcTail) + break; + + MOUSE_EVENT ev = gDpcBuf[gDpcHead]; + gDpcHead = (gDpcHead + 1) & (int)MOUSE_BUF_MASK; + + // Update absolute position and redraw the cursor outside DIRQL. + HalScreenEraseCursor(); + + int nx = gMouseX + ev.dx; + int ny = gMouseY + ev.dy; + if (nx < 0) nx = 0; + if (nx >= gBoundsW) nx = gBoundsW - 1; + if (ny < 0) ny = 0; + if (ny >= gBoundsH) ny = gBoundsH - 1; + gMouseX = nx; + gMouseY = ny; + + HalScreenDrawCursor(nx, ny); + + // Publish event to the normal consumer ring. + int next = (gTail + 1) & (int)MOUSE_BUF_MASK; + if (next != gHead) { + gBuf[gTail] = ev; + gTail = next; + } + + void (*cb)(int, int, uint8_t) = (void (*)(int, int, uint8_t))gCallback; + if (cb) + cb(ev.dx, ev.dy, ev.buttons); + } +} + // -- Interrupt service routine -------------------------------------------------- // Called with interrupts disabled by KeIrqDispatch (IRQ 12). -// The dispatcher sends the APIC EOI before this function is invoked. +// The dispatcher sends the APIC EOI after this function returns. static void MouseIsr(void) { uint8_t data = __inbyte(PS2_DATA); @@ -130,32 +171,14 @@ static void MouseIsr(void) { int dy = (int)gPacket[2] - ((flags & 0x20u) ? 256 : 0); dy = -dy; // PS/2 Y+ = up; screen Y+ = down, so invert. - // -- Update absolute position (clamped to screen bounds) ------------------ - HalScreenEraseCursor(); - - int nx = gMouseX + dx; - int ny = gMouseY + dy; - if (nx < 0) nx = 0; - if (nx >= gBoundsW) nx = gBoundsW - 1; - if (ny < 0) ny = 0; - if (ny >= gBoundsH) ny = gBoundsH - 1; - gMouseX = nx; - gMouseY = ny; - - HalScreenDrawCursor(nx, ny); - - // -- Enqueue event; silently drop if buffer is full ----------------------- - int next = (gTail + 1) & (int)MOUSE_BUF_MASK; - if (next != gHead) { - gBuf[gTail].dx = dx; - gBuf[gTail].dy = dy; - gBuf[gTail].buttons = buttons; - gTail = next; + // -- Queue completed packet to the DPC bottom half ------------------------- + int next = (gDpcTail + 1) & (int)MOUSE_BUF_MASK; + if (next != gDpcHead) { + gDpcBuf[gDpcTail].dx = dx; + gDpcBuf[gDpcTail].dy = dy; + gDpcBuf[gDpcTail].buttons = buttons; + gDpcTail = next; } - - // -- Optional callback (raw ISR context) ----------------------------------- - void (*cb)(int, int, uint8_t) = (void (*)(int, int, uint8_t))gCallback; - if (cb) cb(dx, dy, buttons); } // -- Consumer API -------------------------------------------------------------- @@ -278,5 +301,6 @@ void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH) { out->SetPosition = MouseSetPosition; out->SetBounds = MouseSetBounds; out->Isr = MouseIsr; + out->Dpc = MouseDpc; out->IrqNumber = 12; } diff --git a/code/hal/io_driver.h b/code/hal/io_driver.h index e86d808..b2289ae 100644 --- a/code/hal/io_driver.h +++ b/code/hal/io_driver.h @@ -1,40 +1,69 @@ -#pragma once -// io_driver.h - driver-side interface for registering disk controllers -// -// DISK_DRIVER.SubmitIrp is the async path. IRP is forward-declared here so -// this header (and hal_disk.c) remain free of kernel sync-object dependencies. -// Drivers that implement SubmitIrp include ke_irp.h themselves. - -#include -#include -#include -#include - -typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API* hal); -typedef void (*DRIVER_UNLOAD_FN)(void); - -// Forward declaration only - avoids dragging ke_event.h into the HAL layer. -struct IRP; - -typedef struct DISK_DRIVER { - void *Ctx; - - // --- Synchronous path (always present) --------------------------------- - KSTATUS (*ReadSectors) (void *ctx, uint64_t lba, uint32_t count, void *buf); - KSTATUS (*WriteSectors)(void *ctx, uint64_t lba, uint32_t count, const void *buf); - - // --- Asynchronous path (optional) -------------------------------------- - // NULL = driver has not implemented async; IoSubmitIrp() falls back to - // the sync path. Returns STATUS_PENDING when the IRP is accepted. - KSTATUS (*SubmitIrp)(void *ctx, struct IRP *irp); - - uint64_t SectorCount; -} DISK_DRIVER; - -__declspec(dllexport) KSTATUS IoRegisterDisk(uint8_t driveIndex, const DISK_DRIVER *driver); - -// Internal dispatch used by ke_irp.c. Calls SubmitIrp if present, otherwise -// runs the sync callbacks inline and writes irp->Status; does NOT touch any -// kernel sync objects so it is safe to call from the HAL layer. -// Returns STATUS_PENDING (async path taken) or STATUS_SUCCESS (ran inline). -__declspec(dllexport) KSTATUS HalDiskDispatch(uint8_t drive, struct IRP *irp); +#pragma once +// io_driver.h - definitive kernel driver/device I/O model +// +// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through +// IoCallDriver()/IoSubmitIrp(); drivers complete accepted IRPs by calling +// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table. + +#include +#include +#include +#include + +typedef struct HAL_API HAL_API; +typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API* hal); +typedef void (*DRIVER_UNLOAD_FN)(void); + +struct IRP; +typedef struct DRIVER_OBJECT DRIVER_OBJECT; +typedef struct DEVICE_OBJECT DEVICE_OBJECT; + +typedef enum IO_MAJOR_FUNCTION { + IRP_MJ_READ = 0, + IRP_MJ_WRITE = 1, + IRP_MJ_DEVICE_CONTROL = 2, + IRP_MJ_MAXIMUM_FUNCTION = 3, +} IO_MAJOR_FUNCTION; + +typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, struct IRP* irp); + +struct DRIVER_OBJECT { + const char* Name; + PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION]; + DRIVER_UNLOAD_FN DriverUnload; +}; + +struct DEVICE_OBJECT { + DRIVER_OBJECT* DriverObject; + void* DeviceExtension; + uint32_t Flags; + uint32_t AlignmentRequirement; + uint64_t SectorCount; + uint8_t DiskIndex; +}; + +#define DEVICE_OBJECT_DISK 0x00000001u +#define DEVICE_OBJECT_REMOVABLE 0x00000002u + +__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver, + size_t extensionSize, + DEVICE_OBJECT** outDevice); +__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device); + +__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, + DEVICE_OBJECT* device); +__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex); +__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex); +__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex); + +__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, + struct IRP* irp); + +__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, + uint64_t lba, + uint32_t count, + void* buffer); +__declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, + uint64_t lba, + uint32_t count, + const void* buffer); diff --git a/code/init/init.c b/code/init/init.c index 519f669..6a1071f 100644 --- a/code/init/init.c +++ b/code/init/init.c @@ -50,7 +50,7 @@ void itoa(int num, char* str) { void _start(void) { - const char* args = SysGetCommandLine(); + const char* args = SysGetPeb()->CommandLine; uint32_t h = 0; uint32_t s = 0; diff --git a/code/kernel/boot_info.h b/code/kernel/boot_info.h index c455ae7..82f789b 100644 --- a/code/kernel/boot_info.h +++ b/code/kernel/boot_info.h @@ -40,5 +40,5 @@ typedef struct BOOT_INFO { uintptr_t PhysicalMapBase; /* From boot.ini */ - uint32_t DebugEnabled; + uint32_t Debug; } BOOT_INFO; diff --git a/code/kernel/io_device.c b/code/kernel/io_device.c new file mode 100644 index 0000000..51e7c89 --- /dev/null +++ b/code/kernel/io_device.c @@ -0,0 +1,114 @@ +// io_device.c - kernel driver/device dispatch and disk registry +#include +#include +#include +#include + +#define MAX_DISK_DEVICES 8 + +static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES]; + +__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver, + size_t extensionSize, + DEVICE_OBJECT** outDevice) { + if (!driver || !outDevice) return STATUS_INVALID_PARAMETER; + + size_t total = sizeof(DEVICE_OBJECT) + extensionSize; + DEVICE_OBJECT* dev = (DEVICE_OBJECT*)MmAllocatePool(total); + if (!dev) return STATUS_NO_MEMORY; + + memset(dev, 0, total); + dev->DriverObject = driver; + if (extensionSize) + dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT)); + dev->AlignmentRequirement = 1; + dev->DiskIndex = 0xFF; + + *outDevice = dev; + return STATUS_SUCCESS; +} + +__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) { + if (!device) return; + MmFreePool(device); +} + +__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, + DEVICE_OBJECT* device) { + if (driveIndex >= MAX_DISK_DEVICES || !device || !device->DriverObject) + return STATUS_INVALID_PARAMETER; + if (gDiskDevices[driveIndex]) + return STATUS_ALREADY_EXISTS; + + device->Flags |= DEVICE_OBJECT_DISK; + device->DiskIndex = driveIndex; + gDiskDevices[driveIndex] = device; + return STATUS_SUCCESS; +} + +__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) { + if (driveIndex >= MAX_DISK_DEVICES) return (DEVICE_OBJECT*)0; + return gDiskDevices[driveIndex]; +} + +__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) { + return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0; +} + +__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) { + DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex); + return dev ? dev->SectorCount : 0; +} + +__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !device->DriverObject || !irp) + return STATUS_INVALID_PARAMETER; + + IO_MAJOR_FUNCTION major = (irp->Flags & IRP_FLAG_WRITE) ? IRP_MJ_WRITE : IRP_MJ_READ; + if (major >= IRP_MJ_MAXIMUM_FUNCTION) + return STATUS_INVALID_PARAMETER; + + PDRIVER_DISPATCH dispatch = device->DriverObject->MajorFunction[major]; + if (!dispatch) + return STATUS_NOT_SUPPORTED; + + return dispatch(device, irp); +} + +__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, + uint64_t lba, + uint32_t count, + void* buffer) { + if (!buffer || count == 0) return STATUS_INVALID_PARAMETER; + + IRP irp; + memset(&irp, 0, sizeof(irp)); + irp.DriveIndex = driveIndex; + irp.Flags = 0; + irp.Lba = lba; + irp.SectorCount = count; + irp.Buffer = buffer; + irp.Status = STATUS_FAILURE; + irp.Information = 0; + + return IoSubmitIrpSync(&irp); +} + +__declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, + uint64_t lba, + uint32_t count, + const void* buffer) { + if (!buffer || count == 0) return STATUS_INVALID_PARAMETER; + + IRP irp; + memset(&irp, 0, sizeof(irp)); + irp.DriveIndex = driveIndex; + irp.Flags = IRP_FLAG_WRITE; + irp.Lba = lba; + irp.SectorCount = count; + irp.Buffer = (void*)buffer; + irp.Status = STATUS_FAILURE; + irp.Information = 0; + + return IoSubmitIrpSync(&irp); +} diff --git a/code/kernel/io_file.c b/code/kernel/io_file.c index 7af6948..175b959 100644 --- a/code/kernel/io_file.c +++ b/code/kernel/io_file.c @@ -6,6 +6,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #define MAX_DRIVES 8 @@ -17,11 +23,389 @@ typedef struct { static VOLUME_ENTRY gVolumes[MAX_DRIVES]; // --------------------------------------------------------------------------- -// Current Working Directory (drive 0 relative, no trailing backslash) -// e.g. "" = root, "apps" = root\apps, "apps\\data" = nested +// Async file I/O worker +// --------------------------------------------------------------------------- +typedef enum IO_ASYNC_FILE_OP { + IoAsyncFileRead = 0, + IoAsyncFileWrite = 1, +} IO_ASYNC_FILE_OP; + +typedef enum IO_ASYNC_FILE_STATE { + IoAsyncStateQueued = 0, + IoAsyncStateRunning = 1, + IoAsyncStateCompleted = 2, + IoAsyncStateCancelled = 3, +} IO_ASYNC_FILE_STATE; + +struct IO_ASYNC_FILE_REQUEST_OBJECT { + OBJECT Header; // ObjectTypeAsyncIo + LIST_ENTRY Link; // protected by gAsyncFileQueueLock + + IO_ASYNC_FILE_OP Op; + IO_ASYNC_FILE_STATE State; + uint8_t CancelRequested; + + FILE_OBJECT* File; // referenced until object destruction + EVENT* CompletionEvent; // optional, referenced until object destruction + KAPC* CompletionApc; // caller-owned APC storage + IO_STATUS_BLOCK* Iosb; // caller/user-owned completion block + + void* Buffer; // caller/user-owned; must remain valid until completion/cancel + size_t Size; + uint64_t Offset; +}; + +static SPINLOCK gAsyncFileInitLock = SPINLOCK_INIT; +static int gAsyncFileInitState; // 0=off, 1=starting, 2=ready +static SPINLOCK gAsyncFileQueueLock = SPINLOCK_INIT; +static LIST_ENTRY gAsyncFileQueue; +static SEMAPHORE* gAsyncFileSemaphore; + +static void KiIoAsyncFileRequestDestroy(OBJECT* obj) { + IO_ASYNC_FILE_REQUEST_OBJECT* req = + CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header); + + if (req->CompletionEvent) { + ObDereferenceObject(&req->CompletionEvent->Header); + req->CompletionEvent = (EVENT*)0; + } + if (req->File) { + ObDereferenceObject(&req->File->Header); + req->File = (FILE_OBJECT*)0; + } +} + +static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, + KSTATUS status, + size_t information) { + if (!req) return; + + uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock); + req->State = (status == STATUS_CANCELLED) ? IoAsyncStateCancelled : IoAsyncStateCompleted; + KeReleaseSpinlock(&gAsyncFileQueueLock, flags); + + if (req->Iosb) { + req->Iosb->Status = status; + req->Iosb->Information = information; + } + + // Match IRP completion ordering: queue APC first, then signal the event. + if (req->CompletionApc) { + if (!KeInsertQueueApc(req->CompletionApc, req->Iosb, (void*)(uintptr_t)status)) { + if (req->CompletionApc->RundownRoutine) + req->CompletionApc->RundownRoutine(req->CompletionApc); + } + } + + if (req->CompletionEvent) + KeSignalEvent(req->CompletionEvent); +} + +static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) { + uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock); + LIST_ENTRY* node = ListPopHead(&gAsyncFileQueue); + if (!node) { + KeReleaseSpinlock(&gAsyncFileQueueLock, flags); + return (IO_ASYNC_FILE_REQUEST_OBJECT*)0; + } + + IO_ASYNC_FILE_REQUEST_OBJECT* req = + CONTAINING_RECORD(node, IO_ASYNC_FILE_REQUEST_OBJECT, Link); + + if (req->CancelRequested) { + req->State = IoAsyncStateCancelled; + } else { + req->State = IoAsyncStateRunning; + } + KeReleaseSpinlock(&gAsyncFileQueueLock, flags); + return req; +} + +static void KiIoAsyncFileWorker(void* arg) { + (void)arg; + + for (;;) { + KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore); + if (!KSUCCESS(s)) { + KeYield(); + continue; + } + + IO_ASYNC_FILE_REQUEST_OBJECT* req = KiIoAsyncFileDequeue(); + if (!req) continue; + + if (req->CancelRequested || req->State == IoAsyncStateCancelled) { + KiIoCompleteAsyncFileRequest(req, STATUS_CANCELLED, 0); + ObDereferenceObject(&req->Header); // queue/worker reference + continue; + } + + size_t done = 0; + if (req->Op == IoAsyncFileRead) { + s = IoReadFileAt(req->File, req->Offset, req->Buffer, + req->Size, &done); + } else { + s = IoWriteFileAt(req->File, req->Offset, req->Buffer, + req->Size, &done); + } + + if (req->CancelRequested) { + // The operation may already have reached the filesystem. The + // completion contract still reports cancellation so waiters/APCs do + // not mistake a cancelled request for useful data. + s = STATUS_CANCELLED; + done = 0; + } + + KiIoCompleteAsyncFileRequest(req, s, done); + ObDereferenceObject(&req->Header); // queue/worker reference + } +} + +static KSTATUS KiIoEnsureAsyncFileIoStarted(void) { + uintptr_t flags = KeAcquireSpinlock(&gAsyncFileInitLock); + if (gAsyncFileInitState == 2) { + KeReleaseSpinlock(&gAsyncFileInitLock, flags); + return STATUS_SUCCESS; + } + if (gAsyncFileInitState == 1) { + KeReleaseSpinlock(&gAsyncFileInitLock, flags); + while (gAsyncFileInitState == 1) + KeYield(); + return (gAsyncFileInitState == 2) ? STATUS_SUCCESS : STATUS_FAILURE; + } + + gAsyncFileInitState = 1; + KeReleaseSpinlock(&gAsyncFileInitLock, flags); + + LIST_INIT(&gAsyncFileQueue); + + KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore); + if (KSUCCESS(s)) { + THREAD* worker = (THREAD*)0; + s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker, + 0, + PriorityNormal, + &worker); + // Keep the creator reference: this worker is intentionally persistent + // for the lifetime of the kernel. + (void)worker; + } + + flags = KeAcquireSpinlock(&gAsyncFileInitLock); + gAsyncFileInitState = KSUCCESS(s) ? 2 : 0; + KeReleaseSpinlock(&gAsyncFileInitLock, flags); + + return s; +} + +static void KiIoNativeFsAsyncComplete(void* completeContext, + KSTATUS status, + size_t information) { + IO_ASYNC_FILE_REQUEST_OBJECT* req = + (IO_ASYNC_FILE_REQUEST_OBJECT*)completeContext; + if (!req) return; + + if (req->CancelRequested) { + status = STATUS_CANCELLED; + information = 0; + } + + KiIoCompleteAsyncFileRequest(req, status, information); + ObDereferenceObject(&req->Header); // native filesystem provider reference +} + +static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op, + FILE_OBJECT* file, + void* buffer, + size_t size, + uint64_t offset, + IO_STATUS_BLOCK* iosb, + EVENT* completionEvent, + KAPC* completionApc, + IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) { + if (!file || !iosb) return STATUS_INVALID_PARAMETER; + if (!buffer && size) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + + iosb->Status = STATUS_PENDING; + iosb->Information = 0; + + IO_ASYNC_FILE_REQUEST_OBJECT* req = + (IO_ASYNC_FILE_REQUEST_OBJECT*)ObAllocateObject(ObjectTypeAsyncIo, + sizeof(IO_ASYNC_FILE_REQUEST_OBJECT)); + if (!req) { + iosb->Status = STATUS_NO_MEMORY; + return STATUS_NO_MEMORY; + } + + req->Header.Destroy = KiIoAsyncFileRequestDestroy; + LIST_INIT(&req->Link); + req->Op = op; + req->State = IoAsyncStateQueued; + req->CancelRequested = 0; + req->File = file; + req->CompletionEvent = completionEvent; + req->CompletionApc = completionApc; + req->Iosb = iosb; + req->Buffer = buffer; + req->Size = size; + req->Offset = offset; + + ObReferenceObject(&file->Header); + if (completionEvent) + ObReferenceObject(&completionEvent->Header); + + IO_FS_ASYNC_COMPLETE complete = KiIoNativeFsAsyncComplete; + KSTATUS ns = STATUS_NOT_SUPPORTED; + bool hasNative = (op == IoAsyncFileRead) ? (file->Fs->ReadAsync != 0) + : (file->Fs->WriteAsync != 0); + + if (hasNative) { + uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock); + req->State = IoAsyncStateRunning; + KeReleaseSpinlock(&gAsyncFileQueueLock, flags); + + ObReferenceObject(&req->Header); // native filesystem provider reference + if (op == IoAsyncFileRead) { + ns = file->Fs->ReadAsync(file->Fs, file->FileCtx, offset, buffer, + size, complete, req); + } else { + ns = file->Fs->WriteAsync(file->Fs, file->FileCtx, offset, buffer, + size, complete, req); + } + if (ns == STATUS_PENDING) { + if (requestOut) { + *requestOut = req; + } else { + ObDereferenceObject(&req->Header); + } + return STATUS_PENDING; + } + + ObDereferenceObject(&req->Header); // provider rejected synchronously + iosb->Status = ns; + ObDereferenceObject(&req->Header); // creator reference + return ns; + } + + KSTATUS s = KiIoEnsureAsyncFileIoStarted(); + if (!KSUCCESS(s)) { + iosb->Status = s; + ObDereferenceObject(&req->Header); + return s; + } + + // Queue ownership keeps the request alive until the worker completes or a + // successful queued cancellation completes it from IoCancelAsyncFileRequest. + ObReferenceObject(&req->Header); + + uintptr_t qflags = KeAcquireSpinlock(&gAsyncFileQueueLock); + ListInsertTail(&gAsyncFileQueue, &req->Link); + KeReleaseSpinlock(&gAsyncFileQueueLock, qflags); + + KeSignalSemaphore(gAsyncFileSemaphore); + + if (requestOut) { + *requestOut = req; // creator reference returned to caller + } else { + ObDereferenceObject(&req->Header); + } + + return STATUS_PENDING; +} + +__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf, + size_t size, uint64_t offset, + IO_STATUS_BLOCK* iosb, + EVENT* completionEvent, + KAPC* completionApc, + IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) { + return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset, + iosb, completionEvent, completionApc, + requestOut); +} + +__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf, + size_t size, uint64_t offset, + IO_STATUS_BLOCK* iosb, + EVENT* completionEvent, + KAPC* completionApc, + IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) { + return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset, + iosb, completionEvent, completionApc, + requestOut); +} + +__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* req) { + return req ? &req->Header : (OBJECT*)0; +} + +__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) { + if (!req) return STATUS_INVALID_PARAMETER; + + IO_ASYNC_FILE_STATE state; + uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock); + state = req->State; + + if (state == IoAsyncStateQueued) { + ListRemove(&req->Link); + req->CancelRequested = 1; + req->State = IoAsyncStateCancelled; + KeReleaseSpinlock(&gAsyncFileQueueLock, flags); + + KiIoCompleteAsyncFileRequest(req, STATUS_CANCELLED, 0); + ObDereferenceObject(&req->Header); // queue reference removed above + return STATUS_SUCCESS; + } + + if (state == IoAsyncStateRunning) { + req->CancelRequested = 1; + KeReleaseSpinlock(&gAsyncFileQueueLock, flags); + return STATUS_PENDING; + } + + KeReleaseSpinlock(&gAsyncFileQueueLock, flags); + return STATUS_OBJECT_DEAD; +} + +__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, + IO_STATUS_BLOCK* iosbOut) { + if (!req || !iosbOut) return STATUS_INVALID_PARAMETER; + if (req->Iosb) { + iosbOut->Status = req->Iosb->Status; + iosbOut->Information = req->Iosb->Information; + } else { + iosbOut->Status = STATUS_FAILURE; + iosbOut->Information = 0; + } + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// Current Working Directory +// Stored as (CurrentDrive, drive-relative path with no trailing backslash). +// Exported as native "N:/path" so personalities can round-trip volumes. // --------------------------------------------------------------------------- #define CWD_MAX 512 +static size_t KiFormatCwd(uint8_t drive, const char* rel, char* buf, size_t size) { + if (!buf || size == 0) return 0; + if (drive > 7) drive = 0; + if (!rel) rel = ""; + + size_t relLen = strlen(rel); + size_t needed = 3 + relLen + 1; + if (needed > size) return needed; + + buf[0] = (char)('0' + drive); + buf[1] = ':'; + buf[2] = '/'; + memcpy(buf + 3, rel, relLen); + buf[3 + relLen] = '\0'; + return needed; +} + __declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) { if (!buf || size == 0) return STATUS_INVALID_PARAMETER; @@ -29,24 +413,26 @@ __declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) { if (!proc) return STATUS_ACCESS_DENIED; uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock); - size_t cwdLen = strlen(proc->CurrentDirectory); - if (cwdLen + 1 > size) { - KeReleaseSpinlock(&proc->CwdLock, flags); - return STATUS_BUFFER_TOO_SMALL; - } - memcpy(buf, proc->CurrentDirectory, cwdLen + 1); + size_t needed = KiFormatCwd(proc->CurrentDrive, proc->CurrentDirectory, buf, size); KeReleaseSpinlock(&proc->CwdLock, flags); - return STATUS_SUCCESS; + return needed && needed <= size ? STATUS_SUCCESS : STATUS_BUFFER_TOO_SMALL; } // Resolve a possibly-relative path component against the CWD. // Writes the fully-resolved, drive-stripped relative path into 'resolved' // (caller supplies buf of at least CWD_MAX bytes). // Returns the drive index. -static uint8_t ResolvePath(const char* path, char* resolved, size_t resolvedSize) { - // Strip explicit drive prefix "N:\" +static uint8_t KiResolvePath(const char* path, char* resolved, size_t resolvedSize) { uint8_t drive = 0; + PROCESS* proc = KeGetCurrentProcess(); + if (proc) { + uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock); + drive = proc->CurrentDrive; + KeReleaseSpinlock(&proc->CwdLock, flags); + } + + // Strip explicit drive prefix "N:\" or "N:/". bool fromDriveRoot = false; if (path[0] >= '0' && path[0] <= '7' && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) { @@ -124,14 +510,15 @@ static uint8_t ResolvePath(const char* path, char* resolved, size_t resolvedSize __declspec(dllexport) KSTATUS IoSetCwd(const char* path) { if (!path) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; PROCESS* proc = KeGetCurrentProcess(); if (!proc) return STATUS_ACCESS_DENIED; // Resolve the new path (this also validates ".." navigation). - // Note: ResolvePath already uses proc->CurrentDirectory internally! + // Note: KiResolvePath already uses proc->CurrentDirectory internally! char resolved[CWD_MAX]; - uint8_t drive = ResolvePath(path, resolved, sizeof(resolved)); + uint8_t drive = KiResolvePath(path, resolved, sizeof(resolved)); // Validate: try to open the directory to confirm it actually exists on disk IO_FILESYSTEM* fs = gVolumes[drive].Fs; @@ -148,16 +535,15 @@ __declspec(dllexport) KSTATUS IoSetCwd(const char* path) { if (len + 1 > sizeof(proc->CurrentDirectory)) return STATUS_BUFFER_TOO_SMALL; uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock); + proc->CurrentDrive = drive; memcpy(proc->CurrentDirectory, resolved, len + 1); // Keep the user-visible PEB mirror in sync. The PEB's CWD buffer is a // fixed SYS_CWD_MAX allocation made at process creation, so the pointer // remains stable for personalities that cache SYS_PEB*. if (proc->UserCwdBase && proc->UserCwdSize) { - size_t copyLen = len; - if (copyLen + 1 > proc->UserCwdSize) copyLen = proc->UserCwdSize - 1; - memcpy((void*)proc->UserCwdBase, resolved, copyLen); - ((char*)proc->UserCwdBase)[copyLen] = '\0'; + size_t formatted = KiFormatCwd(drive, resolved, (char*)proc->UserCwdBase, proc->UserCwdSize); + size_t copyLen = (formatted && formatted <= proc->UserCwdSize) ? formatted - 1 : proc->UserCwdSize - 1; SYS_PEB* peb = (SYS_PEB*)proc->UserPebBase; if (peb && peb->Magic == SYS_PEB_MAGIC) { @@ -189,7 +575,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex, // --------------------------------------------------------------------------- // Destructor // --------------------------------------------------------------------------- -static void FileDestroy(OBJECT* obj) { +static void KiFileDestroy(OBJECT* obj) { FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header); if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx); } @@ -199,9 +585,10 @@ static void FileDestroy(OBJECT* obj) { // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out) { if (!path || !out) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; char relPath[CWD_MAX]; - uint8_t drive = ResolvePath(path, relPath, sizeof(relPath)); + uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath)); IO_FILESYSTEM* fs = gVolumes[drive].Fs; if (!fs) return STATUS_NOT_FOUND; @@ -213,7 +600,7 @@ __declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out) { FILE_OBJECT* f = (FILE_OBJECT *)ObAllocateObject(ObjectTypeFile /* reuse slot */, sizeof(FILE_OBJECT)); if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; } - f->Header.Destroy = FileDestroy; + f->Header.Destroy = KiFileDestroy; f->Fs = fs; f->FileCtx = fileCtx; f->Size = fs->GetSize(fs, fileCtx); @@ -224,20 +611,69 @@ __declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out) { } // --------------------------------------------------------------------------- -// IoReadFile - sequential read at current offset +// Explicit-offset file I/O. Async I/O is built on these routines so it does +// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls. // --------------------------------------------------------------------------- -__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, size_t size, size_t* bytesRead) { +__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, + void* buf, size_t size, + size_t* bytesRead) { if (bytesRead) *bytesRead = 0; if (!file || !buf || !bytesRead) return STATUS_INVALID_PARAMETER; - if (file->Offset >= file->Size || size == 0) return STATUS_SUCCESS; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + if (size == 0) return STATUS_SUCCESS; - // Clamp to remaining bytes. - size_t remaining = (size_t)(file->Size - file->Offset); + uintptr_t flags = KeAcquireSpinlock(&file->Lock); + uint64_t fileSize = file->Size; + KeReleaseSpinlock(&file->Lock, flags); + + if (offset >= fileSize) return STATUS_SUCCESS; + + size_t remaining = (size_t)(fileSize - offset); if (size > remaining) size = remaining; - KSTATUS s = file->Fs->Read(file->Fs, file->FileCtx, - file->Offset, buf, size, bytesRead); - if (KSUCCESS(s)) file->Offset += *bytesRead; + return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead); +} + +__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset, + const void* buf, size_t size, + size_t* bytesWritten) { + if (bytesWritten) *bytesWritten = 0; + if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + if (!file->Fs->Write) return STATUS_NOT_SUPPORTED; + if (size == 0) return STATUS_SUCCESS; + + size_t written = 0; + KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&file->Lock); + uint64_t end = offset + written; + if (end > file->Size) file->Size = end; + KeReleaseSpinlock(&file->Lock, flags); + } + + *bytesWritten = written; + return s; +} + +// --------------------------------------------------------------------------- +// IoReadFile - sequential read at current offset +// --------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, + size_t size, size_t* bytesRead) { + if (bytesRead) *bytesRead = 0; + if (!file || !buf || !bytesRead) return STATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&file->Lock); + uint64_t offset = file->Offset; + KeReleaseSpinlock(&file->Lock, flags); + + KSTATUS s = IoReadFileAt(file, offset, buf, size, bytesRead); + if (KSUCCESS(s)) { + flags = KeAcquireSpinlock(&file->Lock); + file->Offset = offset + *bytesRead; + KeReleaseSpinlock(&file->Lock, flags); + } return s; } @@ -246,19 +682,19 @@ __declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, size_t si // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf, size_t size, size_t* bytesWritten) { + if (bytesWritten) *bytesWritten = 0; if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER; - if (!file->Fs->Write) return STATUS_NOT_SUPPORTED; - size_t written = 0; - KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx, - file->Offset, buf, size, &written); + uintptr_t flags = KeAcquireSpinlock(&file->Lock); + uint64_t offset = file->Offset; + KeReleaseSpinlock(&file->Lock, flags); + + KSTATUS s = IoWriteFileAt(file, offset, buf, size, bytesWritten); if (KSUCCESS(s)) { - file->Offset += written; - // Keep the cached size in sync if the file grew. - if (file->Offset > file->Size) - file->Size = file->Offset; + flags = KeAcquireSpinlock(&file->Lock); + file->Offset = offset + *bytesWritten; + KeReleaseSpinlock(&file->Lock, flags); } - *bytesWritten = written; return s; } @@ -267,9 +703,10 @@ __declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf, // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS IoCreateFile(const char* path, FILE_OBJECT** out) { if (!path || !out) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; char relPath[CWD_MAX]; - uint8_t drive = ResolvePath(path, relPath, sizeof(relPath)); + uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath)); IO_FILESYSTEM* fs = gVolumes[drive].Fs; if (!fs) return STATUS_NOT_FOUND; @@ -282,9 +719,10 @@ __declspec(dllexport) KSTATUS IoCreateFile(const char* path, FILE_OBJECT** out) FILE_OBJECT* f = (FILE_OBJECT*)ObAllocateObject(ObjectTypeFile, sizeof(FILE_OBJECT)); if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; } - f->Header.Destroy = FileDestroy; + f->Header.Destroy = KiFileDestroy; f->Fs = fs; f->FileCtx = fileCtx; + f->Lock = (SPINLOCK)SPINLOCK_INIT; f->Size = 0; f->Offset = 0; @@ -297,9 +735,10 @@ __declspec(dllexport) KSTATUS IoCreateFile(const char* path, FILE_OBJECT** out) // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS IoDeleteFile(const char* path) { if (!path) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; char relPath[CWD_MAX]; - uint8_t drive = ResolvePath(path, relPath, sizeof(relPath)); + uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath)); IO_FILESYSTEM* fs = gVolumes[drive].Fs; if (!fs) return STATUS_NOT_FOUND; @@ -310,11 +749,12 @@ __declspec(dllexport) KSTATUS IoDeleteFile(const char* path) { __declspec(dllexport) KSTATUS IoRenameFile(const char* oldPath, const char* newPath) { if (!oldPath || !newPath) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; char oldRel[CWD_MAX]; char newRel[CWD_MAX]; - uint8_t oldDrive = ResolvePath(oldPath, oldRel, sizeof(oldRel)); - uint8_t newDrive = ResolvePath(newPath, newRel, sizeof(newRel)); + uint8_t oldDrive = KiResolvePath(oldPath, oldRel, sizeof(oldRel)); + uint8_t newDrive = KiResolvePath(newPath, newRel, sizeof(newRel)); if (oldDrive != newDrive) return STATUS_NOT_SUPPORTED; IO_FILESYSTEM* fs = gVolumes[oldDrive].Fs; @@ -326,9 +766,10 @@ __declspec(dllexport) KSTATUS IoRenameFile(const char* oldPath, const char* newP __declspec(dllexport) KSTATUS IoCreateDir(const char* path) { if (!path) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; char relPath[CWD_MAX]; - uint8_t drive = ResolvePath(path, relPath, sizeof(relPath)); + uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath)); IO_FILESYSTEM* fs = gVolumes[drive].Fs; if (!fs) return STATUS_NOT_FOUND; @@ -339,9 +780,10 @@ __declspec(dllexport) KSTATUS IoCreateDir(const char* path) { __declspec(dllexport) KSTATUS IoRemoveDir(const char* path) { if (!path) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; char relPath[CWD_MAX]; - uint8_t drive = ResolvePath(path, relPath, sizeof(relPath)); + uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath)); IO_FILESYSTEM* fs = gVolumes[drive].Fs; if (!fs) return STATUS_NOT_FOUND; @@ -355,6 +797,7 @@ __declspec(dllexport) KSTATUS IoRemoveDir(const char* path) { // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize) { if (!file) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; if (!file->Fs->Truncate) return STATUS_NOT_SUPPORTED; KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize); @@ -370,12 +813,18 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) { if (!file) return STATUS_INVALID_PARAMETER; + uintptr_t flags = KeAcquireSpinlock(&file->Lock); file->Offset = offset; + KeReleaseSpinlock(&file->Lock, flags); return STATUS_SUCCESS; } __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) { - return file ? file->Size : 0; + if (!file) return 0; + uintptr_t flags = KeAcquireSpinlock(&file->Lock); + uint64_t size = file->Size; + KeReleaseSpinlock(&file->Lock, flags); + return size; } __declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) { @@ -386,13 +835,14 @@ __declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) { // Directory enumeration // --------------------------------------------------------------------------- -static void DirDestroy(OBJECT* obj) { +static void KiDirDestroy(OBJECT* obj) { DIR_OBJECT* d = (DIR_OBJECT*)obj; // Header is first member if (d->Fs && d->DirCtx) d->Fs->CloseDir(d->Fs, d->DirCtx); } __declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) { if (!out) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; // NULL or empty path means the CWD itself char relPath[CWD_MAX]; @@ -404,13 +854,10 @@ __declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) { uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock); memcpy(relPath, proc->CurrentDirectory, strlen(proc->CurrentDirectory) + 1); + drive = proc->CurrentDrive; KeReleaseSpinlock(&proc->CwdLock, flags); - - // Note: For now, this assumes drive 0. If you add multi-drive support - // later, you should add a CurrentDrive field to the PROCESS struct. - drive = 0; } else { - drive = ResolvePath(dirPath, relPath, sizeof(relPath)); + drive = KiResolvePath(dirPath, relPath, sizeof(relPath)); } IO_FILESYSTEM* fs = gVolumes[drive].Fs; @@ -424,7 +871,7 @@ __declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) { DIR_OBJECT* d = (DIR_OBJECT*)ObAllocateObject(ObjectTypeDir, sizeof(DIR_OBJECT)); if (!d) { fs->CloseDir(fs, dirCtx); return STATUS_NO_MEMORY; } - d->Header.Destroy = DirDestroy; + d->Header.Destroy = KiDirDestroy; d->Fs = fs; d->DirCtx = dirCtx; @@ -434,6 +881,7 @@ __declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) { __declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) { if (!dir || !entryOut) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; if (!dir->Fs->ReadDir) return STATUS_NOT_SUPPORTED; return dir->Fs->ReadDir(dir->Fs, dir->DirCtx, entryOut); } diff --git a/code/kernel/io_file.h b/code/kernel/io_file.h index 2a8f81c..086246b 100644 --- a/code/kernel/io_file.h +++ b/code/kernel/io_file.h @@ -1,149 +1,199 @@ -#pragma once -// io_file.h - VFS layer -// Filesystem drivers register an IO_FILESYSTEM vtable via IoRegisterFilesystem. -// Kernel and user code open files through IoOpenFile. -#include -#include -#include -#include - -// --------------------------------------------------------------------------- -// KDIR_ENTRY - one directory entry returned by ReadDir / SYS_READ_DIR -// Attribute flags match FAT32/Win32: ATTR_DIRECTORY=0x10, ATTR_ARCHIVE=0x20 -// --------------------------------------------------------------------------- -#define KDIR_NAME_MAX 260 - -typedef struct KDIR_ENTRY { - char Name[KDIR_NAME_MAX]; // null-terminated filename - uint8_t Attr; // FAT attribute byte (0x10=dir, 0x20=file) - uint32_t FileSize; // 0 for directories -} KDIR_ENTRY; - -// --------------------------------------------------------------------------- -// Filesystem driver vtable - filled in by fat32.sys (or any future fs driver) -// --------------------------------------------------------------------------- -typedef struct IO_FILESYSTEM { - // Open a file by absolute path (e.g. "apps\\shell.exe"). - // Returns an opaque per-file driver cookie in* fileCtx. - KSTATUS (*Open)(struct IO_FILESYSTEM* fs, - const char* path, void **fileCtx); - - // Read up to 'size' bytes at 'offset' into 'buf'. - // Sets* bytesRead to actual bytes copied. - KSTATUS (*Read)(struct IO_FILESYSTEM* fs, void* fileCtx, - uint64_t offset, void* buf, - size_t size, size_t* bytesRead); - - KSTATUS (*Write)(struct IO_FILESYSTEM* fs, void* fileCtx, - uint64_t offset, const void* buf, - size_t size, size_t* bytesWritten); - - // Create a new file at 'path'; returns an open fileCtx on success. - KSTATUS (*Create)(struct IO_FILESYSTEM* fs, const char* path, - void** fileCtx); - - // Delete the file at 'path'. - KSTATUS (*Delete)(struct IO_FILESYSTEM* fs, const char* path); - - // Rename or move a file/directory within the same filesystem. - KSTATUS (*Rename)(struct IO_FILESYSTEM* fs, - const char* oldPath, const char* newPath); - - // Create or remove directories. - KSTATUS (*CreateDir)(struct IO_FILESYSTEM* fs, const char* path); - KSTATUS (*RemoveDir)(struct IO_FILESYSTEM* fs, const char* path); - - // Truncate or extend the file to exactly 'newSize' bytes. - KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx, - uint64_t newSize); - - // Return the file size in bytes. - uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx); - - // Release any resources the driver allocated for fileCtx. - void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx); - - // Optional driver-private data. - void* DriverCtx; - - // --------------------------------------------------------------------------- - // Directory enumeration (optional - may be NULL if not supported) - // --------------------------------------------------------------------------- - - // Open a directory iterator for 'dirPath' (relative or drive-prefixed). - // Returns an opaque per-iteration driver cookie in *dirCtx. - KSTATUS (*OpenDir)(struct IO_FILESYSTEM* fs, - const char* dirPath, void** dirCtx); - - // Advance to the next entry. - // Fills 'out' with name, attributes, and size. - // Returns STATUS_NO_MORE_FILES when the directory is exhausted. - KSTATUS (*ReadDir)(struct IO_FILESYSTEM* fs, void* dirCtx, - struct KDIR_ENTRY* out); - - // Release resources allocated by OpenDir. - void (*CloseDir)(struct IO_FILESYSTEM* fs, void* dirCtx); -} IO_FILESYSTEM; - -// --------------------------------------------------------------------------- -// FILE_OBJECT - the kernel object returned to callers -// --------------------------------------------------------------------------- -typedef struct FILE_OBJECT { - OBJECT Header; // must be first - IO_FILESYSTEM* Fs; - void* FileCtx; // driver cookie - uint64_t Size; - uint64_t Offset; // current position for sequential reads -} FILE_OBJECT; - -// --------------------------------------------------------------------------- -// DIR_OBJECT - kernel object for an open directory iterator -// --------------------------------------------------------------------------- -typedef struct DIR_OBJECT { - OBJECT Header; // must be first - IO_FILESYSTEM* Fs; - void* DirCtx; // driver cookie (opaque iteration state) -} DIR_OBJECT; - -typedef struct IO_VOLUME_PARAMS { - uint8_t DiskIndex; // which disk in HAL_DISK table - uint64_t PartitionLba; // starting LBA of this partition - uint64_t PartitionSize; // sectors, for bounds checking -} IO_VOLUME_PARAMS; - -// --------------------------------------------------------------------------- -// Registration - called by fat32.sys from DriverEntry -// --------------------------------------------------------------------------- -// driveIndex matches the index passed to IoRegisterDisk. -__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol); - -// --------------------------------------------------------------------------- -// Kernel-side API -// --------------------------------------------------------------------------- -__declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out); -__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, - size_t size, size_t* bytesRead); -__declspec(dllexport) KSTATUS IoWriteFile (FILE_OBJECT* file, const void* buf, - size_t size, size_t* bytesWritten); -__declspec(dllexport) KSTATUS IoCreateFile (const char* path, FILE_OBJECT** out); -__declspec(dllexport) KSTATUS IoDeleteFile (const char* path); -__declspec(dllexport) KSTATUS IoRenameFile (const char* oldPath, const char* newPath); -__declspec(dllexport) KSTATUS IoCreateDir (const char* path); -__declspec(dllexport) KSTATUS IoRemoveDir (const char* path); -__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize); -__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset); -__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file); -__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file); - -// --------------------------------------------------------------------------- -// Directory enumeration API -// --------------------------------------------------------------------------- -__declspec(dllexport) KSTATUS IoOpenDir (const char* dirPath, DIR_OBJECT** out); -__declspec(dllexport) KSTATUS IoReadDir (DIR_OBJECT* dir, KDIR_ENTRY* entryOut); -__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir); - -// --------------------------------------------------------------------------- -// Current Working Directory -// --------------------------------------------------------------------------- -__declspec(dllexport) KSTATUS IoSetCwd(const char* path); -__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size); +#pragma once +// io_file.h - VFS layer +// Filesystem drivers register an IO_FILESYSTEM vtable via IoRegisterFilesystem. +// Kernel and user code open files through IoOpenFile. +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// KDIR_ENTRY - one directory entry returned by ReadDir / SYS_READ_DIR +// Attribute flags match FAT32/Win32: ATTR_DIRECTORY=0x10, ATTR_ARCHIVE=0x20 +// --------------------------------------------------------------------------- +#define KDIR_NAME_MAX 260 + +typedef struct KDIR_ENTRY { + char Name[KDIR_NAME_MAX]; // null-terminated filename + uint8_t Attr; // FAT attribute byte (0x10=dir, 0x20=file) + uint32_t FileSize; // 0 for directories +} KDIR_ENTRY; + +// Native asynchronous filesystem completion. Filesystems that provide +// ReadAsync/WriteAsync accept the operation, return STATUS_PENDING, and later +// call Complete(CompleteContext, status, information). Filesystems without +// these callbacks fall back to the generic VFS async worker. +typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext, + KSTATUS status, + size_t information); + +// --------------------------------------------------------------------------- +// Filesystem driver vtable - filled in by fat32.sys (or any future fs driver) +// --------------------------------------------------------------------------- +typedef struct IO_FILESYSTEM { + // Open a file by absolute path (e.g. "apps\\shell.exe"). + // Returns an opaque per-file driver cookie in* fileCtx. + KSTATUS (*Open)(struct IO_FILESYSTEM* fs, + const char* path, void **fileCtx); + + // Read up to 'size' bytes at 'offset' into 'buf'. + // Sets* bytesRead to actual bytes copied. + KSTATUS (*Read)(struct IO_FILESYSTEM* fs, void* fileCtx, + uint64_t offset, void* buf, + size_t size, size_t* bytesRead); + + KSTATUS (*Write)(struct IO_FILESYSTEM* fs, void* fileCtx, + uint64_t offset, const void* buf, + size_t size, size_t* bytesWritten); + + KSTATUS (*ReadAsync)(struct IO_FILESYSTEM* fs, void* fileCtx, + uint64_t offset, void* buf, size_t size, + IO_FS_ASYNC_COMPLETE complete, + void* completeContext); + + KSTATUS (*WriteAsync)(struct IO_FILESYSTEM* fs, void* fileCtx, + uint64_t offset, const void* buf, size_t size, + IO_FS_ASYNC_COMPLETE complete, + void* completeContext); + + // Create a new file at 'path'; returns an open fileCtx on success. + KSTATUS (*Create)(struct IO_FILESYSTEM* fs, const char* path, + void** fileCtx); + + // Delete the file at 'path'. + KSTATUS (*Delete)(struct IO_FILESYSTEM* fs, const char* path); + + // Rename or move a file/directory within the same filesystem. + KSTATUS (*Rename)(struct IO_FILESYSTEM* fs, + const char* oldPath, const char* newPath); + + // Create or remove directories. + KSTATUS (*CreateDir)(struct IO_FILESYSTEM* fs, const char* path); + KSTATUS (*RemoveDir)(struct IO_FILESYSTEM* fs, const char* path); + + // Truncate or extend the file to exactly 'newSize' bytes. + KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx, + uint64_t newSize); + + // Return the file size in bytes. + uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx); + + // Release any resources the driver allocated for fileCtx. + void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx); + + // Optional driver-private data. + void* DriverCtx; + + // --------------------------------------------------------------------------- + // Directory enumeration (optional - may be NULL if not supported) + // --------------------------------------------------------------------------- + + // Open a directory iterator for 'dirPath' (relative or drive-prefixed). + // Returns an opaque per-iteration driver cookie in *dirCtx. + KSTATUS (*OpenDir)(struct IO_FILESYSTEM* fs, + const char* dirPath, void** dirCtx); + + // Advance to the next entry. + // Fills 'out' with name, attributes, and size. + // Returns STATUS_NO_MORE_FILES when the directory is exhausted. + KSTATUS (*ReadDir)(struct IO_FILESYSTEM* fs, void* dirCtx, + struct KDIR_ENTRY* out); + + // Release resources allocated by OpenDir. + void (*CloseDir)(struct IO_FILESYSTEM* fs, void* dirCtx); +} IO_FILESYSTEM; + + +// --------------------------------------------------------------------------- +// IO_STATUS_BLOCK - completion status visible to async callers +// --------------------------------------------------------------------------- +typedef struct IO_STATUS_BLOCK { + volatile KSTATUS Status; + volatile size_t Information; +} IO_STATUS_BLOCK; + +struct EVENT; +struct KAPC; +typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT; + +// --------------------------------------------------------------------------- +// FILE_OBJECT - the kernel object returned to callers +// --------------------------------------------------------------------------- +typedef struct FILE_OBJECT { + OBJECT Header; // must be first + IO_FILESYSTEM* Fs; + void* FileCtx; // driver cookie + SPINLOCK Lock; // protects cached Size/Offset updates + uint64_t Size; + uint64_t Offset; // current position for sequential reads +} FILE_OBJECT; + +// --------------------------------------------------------------------------- +// DIR_OBJECT - kernel object for an open directory iterator +// --------------------------------------------------------------------------- +typedef struct DIR_OBJECT { + OBJECT Header; // must be first + IO_FILESYSTEM* Fs; + void* DirCtx; // driver cookie (opaque iteration state) +} DIR_OBJECT; + +typedef struct IO_VOLUME_PARAMS { + uint8_t DiskIndex; // registered disk DEVICE_OBJECT index + uint64_t PartitionLba; // starting LBA of this partition + uint64_t PartitionSize; // sectors, for bounds checking +} IO_VOLUME_PARAMS; + +// --------------------------------------------------------------------------- +// Registration - called by fat32.sys from DriverEntry +// --------------------------------------------------------------------------- +// driveIndex matches the index passed to IoRegisterDiskDevice. +__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol); + +// --------------------------------------------------------------------------- +// Kernel-side API +// --------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out); +__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, + size_t size, size_t* bytesRead); +__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, + void* buf, size_t size, size_t* bytesRead); +__declspec(dllexport) KSTATUS IoWriteFile (FILE_OBJECT* file, const void* buf, + size_t size, size_t* bytesWritten); +__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset, + const void* buf, size_t size, + size_t* bytesWritten); +__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf, + size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb, + struct EVENT* completionEvent, struct KAPC* completionApc, + IO_ASYNC_FILE_REQUEST_OBJECT** requestOut); +__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf, + size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb, + struct EVENT* completionEvent, struct KAPC* completionApc, + IO_ASYNC_FILE_REQUEST_OBJECT** requestOut); +__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* request); +__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request); +__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request, + IO_STATUS_BLOCK* iosbOut); +__declspec(dllexport) KSTATUS IoCreateFile (const char* path, FILE_OBJECT** out); +__declspec(dllexport) KSTATUS IoDeleteFile (const char* path); +__declspec(dllexport) KSTATUS IoRenameFile (const char* oldPath, const char* newPath); +__declspec(dllexport) KSTATUS IoCreateDir (const char* path); +__declspec(dllexport) KSTATUS IoRemoveDir (const char* path); +__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize); +__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset); +__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file); +__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file); + +// --------------------------------------------------------------------------- +// Directory enumeration API +// --------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS IoOpenDir (const char* dirPath, DIR_OBJECT** out); +__declspec(dllexport) KSTATUS IoReadDir (DIR_OBJECT* dir, KDIR_ENTRY* entryOut); +__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir); + +// --------------------------------------------------------------------------- +// Current Working Directory +// --------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS IoSetCwd(const char* path); +__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size); diff --git a/code/kernel/ke_apc.c b/code/kernel/ke_apc.c new file mode 100644 index 0000000..52e438c --- /dev/null +++ b/code/kernel/ke_apc.c @@ -0,0 +1,335 @@ +// ke_apc.c - kernel APC infrastructure +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void KiInitializeApcState(THREAD* thread) { + if (!thread) return; + LIST_INIT(&thread->ApcQueue); + thread->ApcLock = (SPINLOCK)SPINLOCK_INIT; + thread->KernelApcPending = 0; + thread->KernelApcDisable = 0; + thread->ApcDeliveryActive = 0; + + LIST_INIT(&thread->UserApcQueue); + thread->UserApcLock = (SPINLOCK)SPINLOCK_INIT; + thread->UserApcPending = 0; + thread->UserApcDispatcher = 0; + thread->UserApcDeliverable = 0; + thread->UserApcDeliveryActive = 0; +} + +__declspec(dllexport) void KeInitializeApc(KAPC* apc, + THREAD* thread, + PKKERNEL_ROUTINE kernelRoutine, + PKNORMAL_ROUTINE normalRoutine, + void* normalContext) { + if (!apc) return; + + LIST_INIT(&apc->Link); + apc->Thread = thread; + apc->KernelRoutine = kernelRoutine; + apc->NormalRoutine = normalRoutine; + apc->RundownRoutine = (PKRUNDOWN_ROUTINE)0; + apc->NormalContext = normalContext; + apc->SystemArgument1 = (void*)0; + apc->SystemArgument2 = (void*)0; + apc->Inserted = 0; +} + +__declspec(dllexport) void KeSetApcRundownRoutine(KAPC* apc, + PKRUNDOWN_ROUTINE rundownRoutine) { + if (!apc) return; + apc->RundownRoutine = rundownRoutine; +} + +__declspec(dllexport) bool KeInsertQueueApc(KAPC* apc, + void* systemArgument1, + void* systemArgument2) { + if (!apc || !apc->Thread) + return false; + + THREAD* thread = apc->Thread; + if (thread->State == ThreadDead) + return false; + + if (_InterlockedCompareExchange(&apc->Inserted, 1, 0) != 0) + return false; + + apc->SystemArgument1 = systemArgument1; + apc->SystemArgument2 = systemArgument2; + + uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock); + ListInsertTail(&thread->ApcQueue, &apc->Link); + thread->KernelApcPending = 1; + uint8_t wakeAlertable = (thread->State == ThreadWaiting && + thread->AlertableWait && + thread->KernelApcDisable == 0); + KeReleaseSpinlock(&thread->ApcLock, flags); + + // If the target opted into an alertable wait, make the APC itself the + // wait completion. The target thread will deliver the APC before + // KeWaitForEvent(..., true) returns STATUS_KERNEL_APC. + if (wakeAlertable) + KiWakeThreadForKernelApc(thread); + + // If the caller queued an APC to itself at PASSIVE_LEVEL, try to deliver + // immediately. Cross-thread APCs run the next time the target thread + // returns to PASSIVE_LEVEL. + if (thread == KeGetCurrentThread()) + KiDeliverKernelApcs(); + + return true; +} + +__declspec(dllexport) void KeEnterCriticalRegion(void) { + THREAD* thread = KeGetCurrentThread(); + if (!thread) return; + + uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock); + thread->KernelApcDisable++; + KeReleaseSpinlock(&thread->ApcLock, flags); +} + +__declspec(dllexport) void KeLeaveCriticalRegion(void) { + THREAD* thread = KeGetCurrentThread(); + if (!thread) return; + + uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock); + if (thread->KernelApcDisable > 0) + thread->KernelApcDisable--; + int deliver = (thread->KernelApcDisable == 0 && thread->KernelApcPending); + KeReleaseSpinlock(&thread->ApcLock, flags); + + if (deliver) + KiDeliverKernelApcs(); +} + +__declspec(dllexport) bool KeAreKernelApcsDisabled(void) { + THREAD* thread = KeGetCurrentThread(); + if (!thread) return true; + return thread->KernelApcDisable != 0; +} + +static KAPC* KiPopKernelApc(THREAD* thread) { + uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock); + LIST_ENTRY* node = ListPopHead(&thread->ApcQueue); + if (LIST_IS_EMPTY(&thread->ApcQueue)) + thread->KernelApcPending = 0; + KeReleaseSpinlock(&thread->ApcLock, flags); + + if (!node) + return (KAPC*)0; + return CONTAINING_RECORD(node, KAPC, Link); +} + +void KiDeliverKernelApcs(void) { + THREAD* thread = KeGetCurrentThread(); + if (!thread) + return; + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + return; + + if (thread->ApcDeliveryActive || thread->KernelApcDisable != 0 || !thread->KernelApcPending) + return; + + thread->ApcDeliveryActive = 1; + + for (;;) { + if (KeGetCurrentIrql() != PASSIVE_LEVEL || thread->KernelApcDisable != 0) + break; + + KAPC* apc = KiPopKernelApc(thread); + if (!apc) + break; + + apc->Inserted = 0; + + PKNORMAL_ROUTINE normalRoutine = apc->NormalRoutine; + void* normalContext = apc->NormalContext; + void* arg1 = apc->SystemArgument1; + void* arg2 = apc->SystemArgument2; + + if (apc->KernelRoutine) + apc->KernelRoutine(apc, + (void**)&normalRoutine, + &normalContext, + &arg1, + &arg2); + + // After the kernel routine has had a chance to adjust the callout, + // stop touching APC storage before invoking the normal routine. The + // normal routine is allowed to free the APC's containing allocation. + apc->SystemArgument1 = (void*)0; + apc->SystemArgument2 = (void*)0; + + if (normalRoutine && KeGetCurrentIrql() == PASSIVE_LEVEL) + normalRoutine(normalContext, arg1, arg2); + } + + thread->ApcDeliveryActive = 0; +} + + +__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher) { + THREAD* thread = KeGetCurrentThread(); + if (!thread) return STATUS_FAILURE; + if (!dispatcher) return STATUS_INVALID_PARAMETER; + thread->UserApcDispatcher = (uintptr_t)dispatcher; + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread, + void* routine, + void* normalContext, + void* systemArgument1, + void* systemArgument2) { + if (!thread || !routine) + return STATUS_INVALID_PARAMETER; + if (thread->State == ThreadDead) + return STATUS_OBJECT_DEAD; + + KUSER_APC* apc = (KUSER_APC*)MmAllocatePool(sizeof(KUSER_APC)); + if (!apc) + return STATUS_NO_MEMORY; + + memset(apc, 0, sizeof(*apc)); + LIST_INIT(&apc->Link); + apc->Routine = routine; + apc->NormalContext = normalContext; + apc->SystemArgument1 = systemArgument1; + apc->SystemArgument2 = systemArgument2; + + uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock); + ListInsertTail(&thread->UserApcQueue, &apc->Link); + thread->UserApcPending = 1; + uint8_t wakeAlertable = (thread->State == ThreadWaiting && + thread->AlertableWait && + thread->UserApcDispatcher != 0); + KeReleaseSpinlock(&thread->UserApcLock, flags); + + if (wakeAlertable) + KiWakeThreadForUserApc(thread); + + return STATUS_SUCCESS; +} + +static KUSER_APC* KiPopUserApc(THREAD* thread) { + uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock); + LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue); + if (LIST_IS_EMPTY(&thread->UserApcQueue)) + thread->UserApcPending = 0; + KeReleaseSpinlock(&thread->UserApcLock, flags); + + if (!node) + return (KUSER_APC*)0; + return CONTAINING_RECORD(node, KUSER_APC, Link); +} + +void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) { + if (!frame || !KeTrapFrameFromUser(frame)) + return; + + THREAD* thread = KeGetCurrentThread(); + if (!thread) + return; + + if (!thread->UserApcDeliverable || thread->UserApcDeliveryActive || + !thread->UserApcPending || !thread->UserApcDispatcher) + return; + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + return; + + KUSER_APC* apc = KiPopUserApc(thread); + if (!apc) + return; + + uintptr_t oldRsp = (uintptr_t)frame->Rsp; + uintptr_t newRsp = oldRsp - sizeof(SYS_USER_APC_FRAME_K); + newRsp &= ~(uintptr_t)0xFULL; + newRsp -= 8; // make injected dispatcher entry look like a normal call target + + SYS_USER_APC_FRAME_K* userFrame = (SYS_USER_APC_FRAME_K*)newRsp; + userFrame->Magic = SYS_USER_APC_FRAME_MAGIC; + userFrame->OriginalRip = frame->Rip; + userFrame->OriginalRsp = frame->Rsp; + userFrame->OriginalRflags = frame->Rflags; + userFrame->OriginalRax = STATUS_USER_APC; + userFrame->Routine = (uint64_t)(uintptr_t)apc->Routine; + userFrame->NormalContext = (uint64_t)(uintptr_t)apc->NormalContext; + userFrame->SystemArgument1 = (uint64_t)(uintptr_t)apc->SystemArgument1; + userFrame->SystemArgument2 = (uint64_t)(uintptr_t)apc->SystemArgument2; + + frame->Rip = thread->UserApcDispatcher; + frame->Rsp = newRsp; + frame->Rax = STATUS_USER_APC; + + thread->UserApcDeliverable = 0; + thread->UserApcDeliveryActive = 1; + + MmFreePool(apc); +} + +void KiRundownApcState(THREAD* thread) { + if (!thread) return; + + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock); + LIST_ENTRY* node = ListPopHead(&thread->ApcQueue); + if (LIST_IS_EMPTY(&thread->ApcQueue)) + thread->KernelApcPending = 0; + KeReleaseSpinlock(&thread->ApcLock, flags); + + if (!node) break; + KAPC* apc = CONTAINING_RECORD(node, KAPC, Link); + apc->Inserted = 0; + if (apc->RundownRoutine) + apc->RundownRoutine(apc); + } + + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock); + LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue); + if (LIST_IS_EMPTY(&thread->UserApcQueue)) + thread->UserApcPending = 0; + KeReleaseSpinlock(&thread->UserApcLock, flags); + + if (!node) break; + KUSER_APC* apc = CONTAINING_RECORD(node, KUSER_APC, Link); + MmFreePool(apc); + } + + thread->UserApcDispatcher = 0; + thread->UserApcDeliverable = 0; + thread->UserApcDeliveryActive = 0; +} + +__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* userFrame) { + if (!userFrame || userFrame->Magic != SYS_USER_APC_FRAME_MAGIC) + return STATUS_INVALID_PARAMETER; + + THREAD* thread = KeGetCurrentThread(); + if (!thread) + return STATUS_FAILURE; + + KTRAP_FRAME* frame = KeGetCurrentTrapFrame(); + if (!frame || !KeTrapFrameFromUser(frame) || frame->Kind != KTRAP_KIND_SYSCALL) + return STATUS_FAILURE; + + frame->Rip = userFrame->OriginalRip; + frame->Rsp = userFrame->OriginalRsp; + frame->Rflags = userFrame->OriginalRflags; + frame->Rax = userFrame->OriginalRax; + + thread->UserApcDeliveryActive = 0; + thread->UserApcDeliverable = 0; + return userFrame->OriginalRax; +} diff --git a/code/kernel/ke_apc.h b/code/kernel/ke_apc.h new file mode 100644 index 0000000..181ec74 --- /dev/null +++ b/code/kernel/ke_apc.h @@ -0,0 +1,102 @@ +#pragma once +// ke_apc.h - kernel APC infrastructure +// +// Kernel APCs are per-thread callbacks that run in the target thread's +// context when that thread reaches PASSIVE_LEVEL and APC delivery is enabled. +// They are the first APC layer; user APCs will build on this once trap-frame +// return-to-user delivery exists. + +#include +#include +#include +#include + +struct THREAD; +typedef struct THREAD THREAD; +typedef struct KTRAP_FRAME KTRAP_FRAME; + +typedef struct KAPC KAPC; + +typedef void (*PKKERNEL_ROUTINE)(KAPC* apc, + void** normalRoutine, + void** normalContext, + void** systemArgument1, + void** systemArgument2); + +typedef void (*PKNORMAL_ROUTINE)(void* normalContext, + void* systemArgument1, + void* systemArgument2); + +typedef void (*PKRUNDOWN_ROUTINE)(KAPC* apc); + +typedef struct KUSER_APC KUSER_APC; + +typedef struct SYS_USER_APC_FRAME_K { + uint64_t Magic; + uint64_t OriginalRip; + uint64_t OriginalRsp; + uint64_t OriginalRflags; + uint64_t OriginalRax; + uint64_t Routine; + uint64_t NormalContext; + uint64_t SystemArgument1; + uint64_t SystemArgument2; +} SYS_USER_APC_FRAME_K; + +#define SYS_USER_APC_FRAME_MAGIC 0x3143504152555355ULL /* "USURAPC1" little-ish */ + +struct KUSER_APC { + LIST_ENTRY Link; + void* Routine; + void* NormalContext; + void* SystemArgument1; + void* SystemArgument2; +}; + +struct KAPC { + LIST_ENTRY Link; + THREAD* Thread; + PKKERNEL_ROUTINE KernelRoutine; + PKNORMAL_ROUTINE NormalRoutine; + PKRUNDOWN_ROUTINE RundownRoutine; + void* NormalContext; + void* SystemArgument1; + void* SystemArgument2; + volatile long Inserted; +}; + +void KiInitializeApcState(THREAD* thread); + +__declspec(dllexport) void KeInitializeApc(KAPC* apc, + THREAD* thread, + PKKERNEL_ROUTINE kernelRoutine, + PKNORMAL_ROUTINE normalRoutine, + void* normalContext); + +__declspec(dllexport) void KeSetApcRundownRoutine(KAPC* apc, + PKRUNDOWN_ROUTINE rundownRoutine); + +__declspec(dllexport) bool KeInsertQueueApc(KAPC* apc, + void* systemArgument1, + void* systemArgument2); + +__declspec(dllexport) void KeEnterCriticalRegion(void); +__declspec(dllexport) void KeLeaveCriticalRegion(void); +__declspec(dllexport) bool KeAreKernelApcsDisabled(void); + +// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL, +// after context switches, and after waits complete. +void KiDeliverKernelApcs(void); + +__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher); +__declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread, + void* routine, + void* normalContext, + void* systemArgument1, + void* systemArgument2); +__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* frame); + +void KiRundownApcState(THREAD* thread); + +// Called from KiExitTrapFrame before committing the editable user return state. +void KiCheckForUserApcDelivery(KTRAP_FRAME* frame); diff --git a/code/kernel/ke_debug.c b/code/kernel/ke_debug.c index 60550e1..674737b 100644 --- a/code/kernel/ke_debug.c +++ b/code/kernel/ke_debug.c @@ -1,187 +1,187 @@ -// ke_debug.c - debugging utilities -#include -#include -#include -#include -#include -#include -#include -#include -#include -int _fltused; // fuckass - -int gDebugEnabled = 0; -void (*PrintChar)(char, uint32_t, uint32_t); - -void kernel_putchar(char c) { - PrintChar(c, 0x00FF00, 0x000000); -} - -void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) { - gDebugEnabled = v; - PrintChar = PrintPtr; - printf_set_putchar(kernel_putchar); -} - -__declspec(dllexport) void DbgPrint(const char* fmt, ...) { - if (!gDebugEnabled) return; - - static SPINLOCK sl = {0}; - uintptr_t flags = KeAcquireSpinlock(&sl); - va_list va; - va_start(va, fmt); - vprintf(fmt, va); - va_end(va); - KeReleaseSpinlock(&sl, flags); -} - -// KePanic -__declspec(dllexport) _Noreturn void KePanic(const char* fmt, ...) { - // Silence! Stop the world! - _disable(); - - // Print a bright-red banner so the framebuffer is unmistakable. - // We call PrintChar directly so we can set a red foreground. - // (DbgPrint uses green; we want this visually distinct.) - static const char banner[] = "\n*** KERNEL PANIC ***\n"; - for (const char* p = banner; *p; p++) - PrintChar(*p, 0xFF2222, 0x000000); - - // Now print the caller's message through the normal vprintf path. - va_list va; - va_start(va, fmt); - vprintf(fmt, va); - va_end(va); - - // Die! - while (true) __halt(); -} - -// KePrintConsole - write user-visible console text in white-on-black. -// Used by SYS_WRITE_CONSOLE so user programs get a distinct color from -// kernel debug output (green) and panic output (red). -__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) { - if (!buf || !PrintChar) return; - for (size_t i = 0; i < len; i++) - PrintChar(buf[i], 0xFFFFFF, 0x000000); -} - -// --------------------------------------------------------------------------- -// KeBugCheckProcess - user-mode fault handler -// -// Resolves the faulting RIP to a module name, prints a yellow crash banner, -// kills the offending process, then exits the current thread so the scheduler -// can continue running everything else. -// --------------------------------------------------------------------------- - -// Return the short exception name for a vector number. -static const char* FaultVectorName(int vector) { - switch (vector) { - case 0: return "#DE Divide Error"; - case 13: return "#GP General Protection"; - case 14: return "#PF Page Fault"; - default: return "Unknown Exception"; - } -} - -// Walk the process module list to find which image owns rip. -// Returns the module name string, or "unknown" if rip is outside all regions. -static const char* ResolveModule(PROCESS* proc, uintptr_t rip) { - // Check the main executable image first. - if (proc->ImageBase && proc->ImageSize && - rip >= proc->ImageBase && rip < proc->ImageBase + proc->ImageSize) - return proc->Name; - - // Walk the per-process module table (DLLs / shared libraries). - for (uint32_t i = 0; i < proc->ModuleCount; i++) { - uintptr_t base = proc->Modules[i].Base; - size_t size = proc->Modules[i].Size; - if (base && size && rip >= base && rip < base + size) - return proc->Modules[i].Name; - } - - return "unknown"; -} - -static void PrintCrashChar(char c) { - PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic -} - -_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) { - THREAD* thread = KeGetCurrentThread(); - PROCESS* proc = thread ? thread->Owner : (PROCESS*)0; - - // Snapshot everything we need before touching process state. - const char* faultName = FaultVectorName(vector); - const char* procName = proc ? proc->Name : "unknown"; - uint32_t pid = proc ? proc->ProcessId : 0; - const char* modName = proc ? ResolveModule(proc, rip) : "unknown"; - - // Amber crash banner. - static const char banner[] = "\n*** APPLICATION CRASH ***\n"; - for (const char* p = banner; *p; p++) - PrintCrashChar(*p); - - printf_set_putchar(PrintCrashChar); - - if (vector == 14) - printf("Process: %s (PID %u) | Fault: %s\n" - "Module: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n", - procName, pid, faultName, - modName, - (unsigned long long)rip, - (unsigned long long)errCode, - (unsigned long long)cr2); - else - printf("Process: %s (PID %u) | Fault: %s\n" - "Module: %s | RIP: %016llx | Code: %04llx\n", - procName, pid, faultName, - modName, - (unsigned long long)rip, - (unsigned long long)errCode); - - // Restore normal (green) output for subsequent kernel messages. - printf_set_putchar(kernel_putchar); - - // Kill the process and exit this thread. - // KeExitThread sets ThreadDead then calls KeYield; the scheduler will - // never return here, so the process crash cannot affect anything else. - if (proc) KeExitProcess(proc, -1); - if (thread) KeExitThread(thread, -1); - - // Unreachable - KeExitThread never returns for a dead thread. - while (1) __halt(); -} - -// --------------------------------------------------------------------------- -// KeFaultDispatch - single C entry point for all three IDT fault stubs. -// -// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and -// call here. We check the CS RPL to decide: kernel fault → KePanic (halt), -// user fault → KeBugCheckProcess (kill process, resume scheduler). -// --------------------------------------------------------------------------- -__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip, - uintptr_t errCode, uintptr_t cr2, - uintptr_t cs) { - if ((cs & 3) == 0) { - // Kernel-mode fault: unrecoverable. - switch (vector) { - case 0: - KePanic("PANIC: #DE Divide Error | RIP: %016llx\n", - (unsigned long long)rip); - case 13: - KePanic("PANIC: #GP General Protection | RIP: %016llx | Code: %04llx\n", - (unsigned long long)rip, (unsigned long long)errCode); - case 14: - KePanic("PANIC: #PF Page Fault | RIP: %016llx | Code: %04llx | CR2: %016llx\n", - (unsigned long long)rip, (unsigned long long)errCode, - (unsigned long long)cr2); - default: - KePanic("PANIC: Exception #%d | RIP: %016llx\n", - vector, (unsigned long long)rip); - } - } - - // User-mode fault: recoverable - kill the process, resume the scheduler. - KeBugCheckProcess(vector, rip, errCode, cr2); -} +// ke_debug.c - debugging utilities +#include +#include +#include +#include +#include +#include +#include +#include +#include +int _fltused; // fuckass + +int gDebugEnabled = 0; +void (*PrintChar)(char, uint32_t, uint32_t); + +void KiKernelPutChar(char c) { + PrintChar(c, 0x00FF00, 0x000000); +} + +void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) { + gDebugEnabled = v; + PrintChar = PrintPtr; + printf_set_putchar(KiKernelPutChar); +} + +__declspec(dllexport) void DbgPrint(const char* fmt, ...) { + if (!gDebugEnabled) return; + + static SPINLOCK sl = {0}; + uintptr_t flags = KeAcquireSpinlock(&sl); + va_list va; + va_start(va, fmt); + vprintf(fmt, va); + va_end(va); + KeReleaseSpinlock(&sl, flags); +} + +// KePanic +__declspec(dllexport) _Noreturn void KePanic(const char* fmt, ...) { + // Silence! Stop the world! + _disable(); + + // Print a bright-red banner so the framebuffer is unmistakable. + // We call PrintChar directly so we can set a red foreground. + // (DbgPrint uses green; we want this visually distinct.) + static const char banner[] = "\n*** KERNEL PANIC ***\n"; + for (const char* p = banner; *p; p++) + PrintChar(*p, 0xFF2222, 0x000000); + + // Now print the caller's message through the normal vprintf path. + va_list va; + va_start(va, fmt); + vprintf(fmt, va); + va_end(va); + + // Die! + while (true) __halt(); +} + +// KePrintConsole - write user-visible console text in white-on-black. +// Used by SYS_WRITE_CONSOLE so user programs get a distinct color from +// kernel debug output (green) and panic output (red). +__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) { + if (!buf || !PrintChar) return; + for (size_t i = 0; i < len; i++) + PrintChar(buf[i], 0xFFFFFF, 0x000000); +} + +// --------------------------------------------------------------------------- +// KeBugCheckProcess - user-mode fault handler +// +// Resolves the faulting RIP to a module name, prints a yellow crash banner, +// kills the offending process, then exits the current thread so the scheduler +// can continue running everything else. +// --------------------------------------------------------------------------- + +// Return the short exception name for a vector number. +static const char* KiFaultVectorName(int vector) { + switch (vector) { + case 0: return "#DE Divide Error"; + case 13: return "#GP General Protection"; + case 14: return "#PF Page Fault"; + default: return "Unknown Exception"; + } +} + +// Walk the process module list to find which image owns rip. +// Returns the module name string, or "unknown" if rip is outside all regions. +static const char* KiResolveModule(PROCESS* proc, uintptr_t rip) { + // Check the main executable image first. + if (proc->ImageBase && proc->ImageSize && + rip >= proc->ImageBase && rip < proc->ImageBase + proc->ImageSize) + return proc->Name; + + // Walk the per-process module table (DLLs / shared libraries). + for (uint32_t i = 0; i < proc->ModuleCount; i++) { + uintptr_t base = proc->Modules[i].Base; + size_t size = proc->Modules[i].Size; + if (base && size && rip >= base && rip < base + size) + return proc->Modules[i].Name; + } + + return "unknown"; +} + +static void KiPrintCrashChar(char c) { + PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic +} + +_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) { + THREAD* thread = KeGetCurrentThread(); + PROCESS* proc = thread ? thread->Owner : (PROCESS*)0; + + // Snapshot everything we need before touching process state. + const char* faultName = KiFaultVectorName(vector); + const char* procName = proc ? proc->Name : "unknown"; + uint32_t pid = proc ? proc->ProcessId : 0; + const char* modName = proc ? KiResolveModule(proc, rip) : "unknown"; + + // Amber crash banner. + static const char banner[] = "\n*** APPLICATION CRASH ***\n"; + for (const char* p = banner; *p; p++) + KiPrintCrashChar(*p); + + printf_set_putchar(KiPrintCrashChar); + + if (vector == 14) + printf("Process: %s (PID %u) | Fault: %s\n" + "Module: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n", + procName, pid, faultName, + modName, + (unsigned long long)rip, + (unsigned long long)errCode, + (unsigned long long)cr2); + else + printf("Process: %s (PID %u) | Fault: %s\n" + "Module: %s | RIP: %016llx | Code: %04llx\n", + procName, pid, faultName, + modName, + (unsigned long long)rip, + (unsigned long long)errCode); + + // Restore normal (green) output for subsequent kernel messages. + printf_set_putchar(KiKernelPutChar); + + // Kill the process and exit this thread. + // KeExitThread sets ThreadDead then calls KeYield; the scheduler will + // never return here, so the process crash cannot affect anything else. + if (proc) KeExitProcess(proc, -1); + if (thread) KeExitThread(thread, -1); + + // Unreachable - KeExitThread never returns for a dead thread. + while (1) __halt(); +} + +// --------------------------------------------------------------------------- +// KeFaultDispatch - single C entry point for all three IDT fault stubs. +// +// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and +// call here. We check the CS RPL to decide: kernel fault → KePanic (halt), +// user fault → KeBugCheckProcess (kill process, resume scheduler). +// --------------------------------------------------------------------------- +__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip, + uintptr_t errCode, uintptr_t cr2, + uintptr_t cs) { + if ((cs & 3) == 0) { + // Kernel-mode fault: unrecoverable. + switch (vector) { + case 0: + KePanic("PANIC: #DE Divide Error | RIP: %016llx\n", + (unsigned long long)rip); + case 13: + KePanic("PANIC: #GP General Protection | RIP: %016llx | Code: %04llx\n", + (unsigned long long)rip, (unsigned long long)errCode); + case 14: + KePanic("PANIC: #PF Page Fault | RIP: %016llx | Code: %04llx | CR2: %016llx\n", + (unsigned long long)rip, (unsigned long long)errCode, + (unsigned long long)cr2); + default: + KePanic("PANIC: Exception #%d | RIP: %016llx\n", + vector, (unsigned long long)rip); + } + } + + // User-mode fault: recoverable - kill the process, resume the scheduler. + KeBugCheckProcess(vector, rip, errCode, cr2); +} diff --git a/code/kernel/ke_dpc.c b/code/kernel/ke_dpc.c new file mode 100644 index 0000000..e567e17 --- /dev/null +++ b/code/kernel/ke_dpc.c @@ -0,0 +1,104 @@ +// ke_dpc.c - deferred procedure calls +#include +#include +#include +#include + +static LIST_ENTRY gDpcQueue; +static SPINLOCK gDpcLock; +static volatile long gDpcInitialized = 0; +static volatile long gDpcActive = 0; + +void KeInitializeDpcSystem(void) { + LIST_INIT(&gDpcQueue); + gDpcLock = (SPINLOCK)SPINLOCK_INIT; + gDpcActive = 0; + gDpcInitialized = 1; +} + +static void KiEnsureDpcInitialized(void) { + if (gDpcInitialized) + return; + + // Single-CPU bring-up path: this is only a safety net for early callers. + KeInitializeDpcSystem(); +} + +__declspec(dllexport) void KeInitializeDpc(KDPC* dpc, + KDEFERRED_ROUTINE routine, + void* context) { + if (!dpc) + return; + + LIST_INIT(&dpc->Link); + dpc->Routine = routine; + dpc->Context = context; + dpc->SystemArgument1 = (void*)0; + dpc->SystemArgument2 = (void*)0; + dpc->Inserted = 0; +} + +__declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc, + void* systemArgument1, + void* systemArgument2) { + if (!dpc || !dpc->Routine) + return false; + + KiEnsureDpcInitialized(); + + // Coalesce repeated inserts of the same KDPC. The first insert owns the + // argument pair until the DPC is popped by the dispatcher. + if (_InterlockedCompareExchange(&dpc->Inserted, 1, 0) != 0) + return false; + + dpc->SystemArgument1 = systemArgument1; + dpc->SystemArgument2 = systemArgument2; + + uintptr_t flags = KeAcquireSpinlock(&gDpcLock); + ListInsertTail(&gDpcQueue, &dpc->Link); + KeReleaseSpinlock(&gDpcLock, flags); + return true; +} + +void KiDrainDpcQueue(void) { + if (!gDpcInitialized) + return; + + // Never run DPC routines above DISPATCH_LEVEL. If a future nested IRQ path + // reaches here at DIRQL, leave the work pending for the outermost exit. + if (KeGetCurrentIrql() > DISPATCH_LEVEL) + return; + + if (_InterlockedCompareExchange(&gDpcActive, 1, 0) != 0) + return; + + KIRQL oldIrql = KeGetCurrentIrql(); + if (oldIrql < DISPATCH_LEVEL) + KeRaiseIrql(DISPATCH_LEVEL); + + for (;;) { + uintptr_t flags = KeAcquireSpinlock(&gDpcLock); + LIST_ENTRY* node = ListPopHead(&gDpcQueue); + if (!node) { + KeReleaseSpinlock(&gDpcLock, flags); + break; + } + + KDPC* dpc = CONTAINING_RECORD(node, KDPC, Link); + void* arg1 = dpc->SystemArgument1; + void* arg2 = dpc->SystemArgument2; + dpc->SystemArgument1 = (void*)0; + dpc->SystemArgument2 = (void*)0; + dpc->Inserted = 0; + KeReleaseSpinlock(&gDpcLock, flags); + + KeAssertIrql(DISPATCH_LEVEL); + dpc->Routine(dpc, dpc->Context, arg1, arg2); + KeAssertIrql(DISPATCH_LEVEL); + } + + gDpcActive = 0; + + if (oldIrql < DISPATCH_LEVEL) + KeLowerIrql(oldIrql); +} diff --git a/code/kernel/ke_dpc.h b/code/kernel/ke_dpc.h new file mode 100644 index 0000000..f12bd3d --- /dev/null +++ b/code/kernel/ke_dpc.h @@ -0,0 +1,44 @@ +#pragma once +// ke_dpc.h - deferred procedure calls +// +// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC +// after acknowledging/capturing device state; the DPC then runs later at +// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not +// wait, block, or call PASSIVE_LEVEL-only code. + +#include +#include +#include +#include + +typedef struct KDPC KDPC; + +typedef void (*KDEFERRED_ROUTINE)(KDPC* dpc, + void* context, + void* systemArgument1, + void* systemArgument2); + +struct KDPC { + LIST_ENTRY Link; + KDEFERRED_ROUTINE Routine; + void* Context; + void* SystemArgument1; + void* SystemArgument2; + volatile long Inserted; +}; + +// Called during kernel bring-up before interrupts are enabled. +void KeInitializeDpcSystem(void); + +__declspec(dllexport) void KeInitializeDpc(KDPC* dpc, + KDEFERRED_ROUTINE routine, + void* context); + +// Queue a DPC if it is not already queued. Returns false if the DPC was +// already pending or the arguments are invalid. +__declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc, + void* systemArgument1, + void* systemArgument2); + +// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit. +void KiDrainDpcQueue(void); diff --git a/code/kernel/ke_event.c b/code/kernel/ke_event.c index 7cc8b89..2f88e8a 100644 --- a/code/kernel/ke_event.c +++ b/code/kernel/ke_event.c @@ -1,71 +1,81 @@ -// ke_event.c - synchronisation events -#include -#include -#include -#include -#include - -static void EventDestroy(OBJECT* obj) { (void)obj; } - -__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) { - if (!out) return STATUS_INVALID_PARAMETER; - EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT)); - if (!e) return STATUS_NO_MEMORY; - e->Header.Destroy = EventDestroy; - e->Lock = (SPINLOCK)SPINLOCK_INIT; - e->Type = type; - e->Signaled = initialState ? 1 : 0; - LIST_INIT(&e->WaitList); - *out = e; - return STATUS_SUCCESS; -} - -__declspec(dllexport) void KeSignalEvent(EVENT* event) { - if (!event) return; - uintptr_t flags = KeAcquireSpinlock(&event->Lock); - - if (event->Type == EventManualReset) { - event->Signaled = 1; - while (!LIST_IS_EMPTY(&event->WaitList)) { - LIST_ENTRY* node = ListPopHead(&event->WaitList); - THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeReleaseSpinlock(&event->Lock, flags); - KeUnblockThread(t); - flags = KeAcquireSpinlock(&event->Lock); - } - } else { - event->Signaled = 1; - if (!LIST_IS_EMPTY(&event->WaitList)) { - LIST_ENTRY* node = ListPopHead(&event->WaitList); - THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeReleaseSpinlock(&event->Lock, flags); - KeUnblockThread(t); - return; - } - } - KeReleaseSpinlock(&event->Lock, flags); -} - -__declspec(dllexport) void KeClearEvent(EVENT* event) { - if (!event) return; - uintptr_t flags = KeAcquireSpinlock(&event->Lock); - event->Signaled = 0; - KeReleaseSpinlock(&event->Lock, flags); -} - -__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event) { - if (!event) return STATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - - while (true) { - uintptr_t flags = KeAcquireSpinlock(&event->Lock); - if (event->Signaled) { - if (event->Type == EventAutoReset) event->Signaled = 0; - KeReleaseSpinlock(&event->Lock, flags); - return STATUS_SUCCESS; - } - ListInsertTail(&event->WaitList, &self->SchedulerLink); - KeReleaseSpinlock(&event->Lock, flags); - KeBlockThread(self, &event->Header); - } -} +// ke_event.c - synchronisation events +#include +#include +#include +#include +#include +#include +#include + +static void KiEventDestroy(OBJECT* obj) { (void)obj; } + +__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) { + if (!out) return STATUS_INVALID_PARAMETER; + EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT)); + if (!e) return STATUS_NO_MEMORY; + e->Header.Destroy = KiEventDestroy; + e->Lock = (SPINLOCK)SPINLOCK_INIT; + e->Type = type; + e->Signaled = initialState ? 1 : 0; + LIST_INIT(&e->WaitList); + *out = e; + return STATUS_SUCCESS; +} + +__declspec(dllexport) void KeSignalEvent(EVENT* event) { + if (!event) return; + uintptr_t flags = KeAcquireSpinlock(&event->Lock); + + if (event->Type == EventManualReset) { + event->Signaled = 1; + while (!LIST_IS_EMPTY(&event->WaitList)) { + LIST_ENTRY* node = ListPopHead(&event->WaitList); + THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeReleaseSpinlock(&event->Lock, flags); + KeUnblockThreadWithStatus(t, STATUS_SUCCESS); + flags = KeAcquireSpinlock(&event->Lock); + } + } else { + if (!LIST_IS_EMPTY(&event->WaitList)) { + // Auto-reset handoff: transfer the signal directly to exactly one + // waiter. Do not leave Signaled=1, because a later waiter can steal + // the disk/APC completion before the selected thread resumes. + event->Signaled = 0; + LIST_ENTRY* node = ListPopHead(&event->WaitList); + THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeReleaseSpinlock(&event->Lock, flags); + KeUnblockThreadWithStatus(t, STATUS_SUCCESS); + return; + } + event->Signaled = 1; + } + KeReleaseSpinlock(&event->Lock, flags); +} + +__declspec(dllexport) void KeClearEvent(EVENT* event) { + if (!event) return; + uintptr_t flags = KeAcquireSpinlock(&event->Lock); + event->Signaled = 0; + KeReleaseSpinlock(&event->Lock, flags); +} + +__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) { + if (!event) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + if (alertable && KeAreKernelApcsDisabled()) return STATUS_INVALID_PARAMETER; + THREAD* self = KeGetCurrentThread(); + + uintptr_t flags = KeAcquireSpinlock(&event->Lock); + if (event->Signaled) { + if (event->Type == EventAutoReset) event->Signaled = 0; + KeReleaseSpinlock(&event->Lock, flags); + return STATUS_SUCCESS; + } + + KePrepareBlockThread(self, &event->Header, alertable); + ListInsertTail(&event->WaitList, &self->SchedulerLink); + KeSetThreadWaitList(self, &event->WaitList); + KeReleaseSpinlock(&event->Lock, flags); + return KeCommitBlockThread(self, alertable); +} + diff --git a/code/kernel/ke_event.h b/code/kernel/ke_event.h index d492283..681cb25 100644 --- a/code/kernel/ke_event.h +++ b/code/kernel/ke_event.h @@ -1,22 +1,23 @@ -#pragma once -// ke_event.h - auto-reset and manual-reset events -#include -#include -#include -#include -#include - -typedef enum EVENT_TYPE { EventAutoReset, EventManualReset } EVENT_TYPE; - -typedef struct EVENT { - OBJECT Header; - SPINLOCK Lock; - EVENT_TYPE Type; - uint32_t Signaled; - LIST_ENTRY WaitList; -} EVENT; - -__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out); -__declspec(dllexport) void KeSignalEvent(EVENT* event); -__declspec(dllexport) void KeClearEvent(EVENT* event); -__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event); +#pragma once +// ke_event.h - auto-reset and manual-reset events +#include +#include +#include +#include +#include + +typedef enum EVENT_TYPE { EventAutoReset, EventManualReset } EVENT_TYPE; + +typedef struct EVENT { + OBJECT Header; + SPINLOCK Lock; + EVENT_TYPE Type; + uint32_t Signaled; + LIST_ENTRY WaitList; +} EVENT; + +__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out); +__declspec(dllexport) void KeSignalEvent(EVENT* event); +__declspec(dllexport) void KeClearEvent(EVENT* event); +__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable); +__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable); diff --git a/code/kernel/ke_idt.c b/code/kernel/ke_idt.c index da6adb7..9211f07 100644 --- a/code/kernel/ke_idt.c +++ b/code/kernel/ke_idt.c @@ -1,175 +1,175 @@ -// ke_idt.c - GDT, IDT, and TSS initialisation -// -// GDT layout (matches segment selectors in ke_switch.asm / ke_syscall.asm): -// 0x00 null -// 0x08 ring-0 code (64-bit) -// 0x10 ring-0 data -// 0x18 ring-3 code (64-bit, DPL=3) -// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3 -// 0x28 TSS low -// 0x30 TSS high -#include -#include -#include - -// External assembly procedures -extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel); -extern void KeFlushIdt(void* pIdtr); - -extern void KeIrqStub0(void); -extern void KeIrqStub1(void); -extern void KeIrqStub2(void); -extern void KeIrqStub3(void); -extern void KeIrqStub4(void); -extern void KeIrqStub5(void); -extern void KeIrqStub6(void); -extern void KeIrqStub7(void); -extern void KeIrqStub8(void); -extern void KeIrqStub9(void); -extern void KeIrqStub10(void); -extern void KeIrqStub11(void); -extern void KeIrqStub12(void); -extern void KeIrqStub13(void); -extern void KeIrqStub14(void); -extern void KeIrqStub15(void); -extern void KeIrqStub16(void); -extern void KeIrqStub17(void); -extern void KeIrqStub18(void); -extern void KeIrqStub19(void); -extern void KeIrqStub20(void); -extern void KeIrqStub21(void); -extern void KeIrqStub22(void); -extern void KeIrqStub23(void); - -#define NUM_IRQS 24 - -// --------------------------------------------------------------------------- -// GDT -// --------------------------------------------------------------------------- -#define GDT_ENTRIES 8 -#define KERNEL_IST_STACK_SIZE 0x4000 - -static uint64_t gGdt[GDT_ENTRIES]; -static uint8_t gKernelStack[KERNEL_IST_STACK_SIZE]; - -#pragma pack(push, 1) -typedef struct { uint16_t Limit; uint64_t Base; } GDTR; -#pragma pack(pop) - -static void GdtSetEntry(int idx, uint32_t base, uint32_t limit, - uint8_t access, uint8_t flags) { - gGdt[idx] = - ((uint64_t)(limit & 0xFFFF)) | - ((uint64_t)(base & 0xFFFFFF) << 16) | - ((uint64_t)access << 40) | - ((uint64_t)(limit >> 16 & 0xF)<< 48) | - ((uint64_t)(flags & 0xF) << 52) | - ((uint64_t)(base >> 24 & 0xFF)<< 56); -} - -static void GdtSetTss(int idx, uintptr_t base, uint32_t limit) { - // TSS descriptor is 16 bytes (two GDT slots). - gGdt[idx] = 0; - gGdt[idx+1] = 0; - - gGdt[idx] = - ((uint64_t)(limit & 0xFFFF)) | - ((uint64_t)(base & 0xFFFFFF) << 16) | - (0x89ULL << 40) | // P=1,DPL=0,type=TSS64 avail - ((uint64_t)((limit>>16)&0xF) << 48) | - ((uint64_t)(base>>24 & 0xFF) << 56); - - gGdt[idx+1] = (base >> 32); -} - -TSS64 gTssStruct; -TSS64* gTss = &gTssStruct; - -void KeInitializeGdt(void) { - memset(gGdt, 0, sizeof(gGdt)); - memset(&gTssStruct, 0, sizeof(gTssStruct)); - - // 0x00: Null - GdtSetEntry(0, 0, 0, 0, 0); - // 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit) - GdtSetEntry(1, 0, 0xFFFFF, 0x9A, 0xA); - // 0x10: Ring-0 Data - GdtSetEntry(2, 0, 0xFFFFF, 0x92, 0xC); - // 0x18: Ring-3 Code - GdtSetEntry(3, 0, 0xFFFFF, 0xFA, 0xA); - // 0x20: Ring-3 Data - GdtSetEntry(4, 0, 0xFFFFF, 0xF2, 0xC); - - // 0x28: TSS (Occupies two slots: 0x28 and 0x30) - gTssStruct.IoMapBase = sizeof(TSS64); - gTssStruct.RSP0 = (uint64_t)(gKernelStack + KERNEL_IST_STACK_SIZE); // top of stack - - GdtSetTss(5, (uintptr_t)&gTssStruct, sizeof(TSS64) - 1); - - GDTR gdtr = { sizeof(gGdt) - 1, (uint64_t)gGdt }; - - // Call the MASM64 proc to reload everything - // Selectors: CS=0x08, Data=0x10, TSS=0x28 - KeFlushGdt(&gdtr, 0x08, 0x10, 0x28); -} - -// --------------------------------------------------------------------------- -// IDT -// --------------------------------------------------------------------------- -#define IDT_ENTRIES 256 - -#pragma pack(push, 1) -typedef struct { - uint16_t OffsetLow; - uint16_t Selector; - uint8_t Ist; - uint8_t TypeAttr; - uint16_t OffsetMid; - uint32_t OffsetHigh; - uint32_t Reserved; -} IDT_ENTRY; -#pragma pack(pop) - -static IDT_ENTRY gIdt[IDT_ENTRIES]; - -#pragma pack(push, 1) -typedef struct { uint16_t Limit; uint64_t Base; } IDTR; -#pragma pack(pop) - -static void IdtSetGate(int vec, uintptr_t handler, - uint8_t typeAttr, uint8_t ist) { - gIdt[vec].OffsetLow = (uint16_t)(handler & 0xFFFF); - gIdt[vec].Selector = 0x08; // ring-0 code segment - gIdt[vec].Ist = ist; - gIdt[vec].TypeAttr = typeAttr; - gIdt[vec].OffsetMid = (uint16_t)((handler >> 16) & 0xFFFF); - gIdt[vec].OffsetHigh = (uint32_t)((handler >> 32) & 0xFFFFFFFF); - gIdt[vec].Reserved = 0; -} - -void KeInitializeIdt(void) { - memset(gIdt, 0, sizeof(gIdt)); - - IdtSetGate(0, (uintptr_t)KeFaultDE, 0x8E, 0); - IdtSetGate(13, (uintptr_t)KeFaultGP, 0x8E, 0); - IdtSetGate(14, (uintptr_t)KeFaultPF, 0x8E, 0); - IdtSetGate(0x40, (uintptr_t)KeSyscallEntry, 0xEE, 0); - - static void *irqStubs[NUM_IRQS] = { - (void*)KeIrqStub0, (void*)KeIrqStub1, (void*)KeIrqStub2, - (void*)KeIrqStub3, (void*)KeIrqStub4, (void*)KeIrqStub5, - (void*)KeIrqStub6, (void*)KeIrqStub7, (void*)KeIrqStub8, - (void*)KeIrqStub9, (void*)KeIrqStub10, (void*)KeIrqStub11, - (void*)KeIrqStub12, (void*)KeIrqStub13, (void*)KeIrqStub14, - (void*)KeIrqStub15, (void*)KeIrqStub16, (void*)KeIrqStub17, - (void*)KeIrqStub18, (void*)KeIrqStub19, (void*)KeIrqStub20, - (void*)KeIrqStub21, (void*)KeIrqStub22, (void*)KeIrqStub23, - }; - for (int i = 0; i < NUM_IRQS; i++) - IdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], 0x8E, 0); - - IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt }; - - // Call the MASM64 proc to load IDT - KeFlushIdt(&idtr); -} +// ke_idt.c - GDT, IDT, and TSS initialisation +// +// GDT layout (matches segment selectors in ke_switch.asm / ke_syscall.asm): +// 0x00 null +// 0x08 ring-0 code (64-bit) +// 0x10 ring-0 data +// 0x18 ring-3 code (64-bit, DPL=3) +// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3 +// 0x28 TSS low +// 0x30 TSS high +#include +#include +#include + +// External assembly procedures +extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel); +extern void KeFlushIdt(void* pIdtr); + +extern void KeIrqStub0(void); +extern void KeIrqStub1(void); +extern void KeIrqStub2(void); +extern void KeIrqStub3(void); +extern void KeIrqStub4(void); +extern void KeIrqStub5(void); +extern void KeIrqStub6(void); +extern void KeIrqStub7(void); +extern void KeIrqStub8(void); +extern void KeIrqStub9(void); +extern void KeIrqStub10(void); +extern void KeIrqStub11(void); +extern void KeIrqStub12(void); +extern void KeIrqStub13(void); +extern void KeIrqStub14(void); +extern void KeIrqStub15(void); +extern void KeIrqStub16(void); +extern void KeIrqStub17(void); +extern void KeIrqStub18(void); +extern void KeIrqStub19(void); +extern void KeIrqStub20(void); +extern void KeIrqStub21(void); +extern void KeIrqStub22(void); +extern void KeIrqStub23(void); + +#define NUM_IRQS 24 + +// --------------------------------------------------------------------------- +// GDT +// --------------------------------------------------------------------------- +#define GDT_ENTRIES 8 +#define KERNEL_IST_STACK_SIZE 0x4000 + +static uint64_t gGdt[GDT_ENTRIES]; +static uint8_t gKernelStack[KERNEL_IST_STACK_SIZE]; + +#pragma pack(push, 1) +typedef struct { uint16_t Limit; uint64_t Base; } GDTR; +#pragma pack(pop) + +static void KiGdtSetEntry(int idx, uint32_t base, uint32_t limit, + uint8_t access, uint8_t flags) { + gGdt[idx] = + ((uint64_t)(limit & 0xFFFF)) | + ((uint64_t)(base & 0xFFFFFF) << 16) | + ((uint64_t)access << 40) | + ((uint64_t)(limit >> 16 & 0xF)<< 48) | + ((uint64_t)(flags & 0xF) << 52) | + ((uint64_t)(base >> 24 & 0xFF)<< 56); +} + +static void KiGdtSetTss(int idx, uintptr_t base, uint32_t limit) { + // TSS descriptor is 16 bytes (two GDT slots). + gGdt[idx] = 0; + gGdt[idx+1] = 0; + + gGdt[idx] = + ((uint64_t)(limit & 0xFFFF)) | + ((uint64_t)(base & 0xFFFFFF) << 16) | + (0x89ULL << 40) | // P=1,DPL=0,type=TSS64 avail + ((uint64_t)((limit>>16)&0xF) << 48) | + ((uint64_t)(base>>24 & 0xFF) << 56); + + gGdt[idx+1] = (base >> 32); +} + +TSS64 gTssStruct; +TSS64* gTss = &gTssStruct; + +void KeInitializeGdt(void) { + memset(gGdt, 0, sizeof(gGdt)); + memset(&gTssStruct, 0, sizeof(gTssStruct)); + + // 0x00: Null + KiGdtSetEntry(0, 0, 0, 0, 0); + // 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit) + KiGdtSetEntry(1, 0, 0xFFFFF, 0x9A, 0xA); + // 0x10: Ring-0 Data + KiGdtSetEntry(2, 0, 0xFFFFF, 0x92, 0xC); + // 0x18: Ring-3 Code + KiGdtSetEntry(3, 0, 0xFFFFF, 0xFA, 0xA); + // 0x20: Ring-3 Data + KiGdtSetEntry(4, 0, 0xFFFFF, 0xF2, 0xC); + + // 0x28: TSS (Occupies two slots: 0x28 and 0x30) + gTssStruct.IoMapBase = sizeof(TSS64); + gTssStruct.RSP0 = (uint64_t)(gKernelStack + KERNEL_IST_STACK_SIZE); // top of stack + + KiGdtSetTss(5, (uintptr_t)&gTssStruct, sizeof(TSS64) - 1); + + GDTR gdtr = { sizeof(gGdt) - 1, (uint64_t)gGdt }; + + // Call the MASM64 proc to reload everything + // Selectors: CS=0x08, Data=0x10, TSS=0x28 + KeFlushGdt(&gdtr, 0x08, 0x10, 0x28); +} + +// --------------------------------------------------------------------------- +// IDT +// --------------------------------------------------------------------------- +#define IDT_ENTRIES 256 + +#pragma pack(push, 1) +typedef struct { + uint16_t OffsetLow; + uint16_t Selector; + uint8_t Ist; + uint8_t TypeAttr; + uint16_t OffsetMid; + uint32_t OffsetHigh; + uint32_t Reserved; +} IDT_ENTRY; +#pragma pack(pop) + +static IDT_ENTRY gIdt[IDT_ENTRIES]; + +#pragma pack(push, 1) +typedef struct { uint16_t Limit; uint64_t Base; } IDTR; +#pragma pack(pop) + +static void KiIdtSetGate(int vec, uintptr_t handler, + uint8_t typeAttr, uint8_t ist) { + gIdt[vec].OffsetLow = (uint16_t)(handler & 0xFFFF); + gIdt[vec].Selector = 0x08; // ring-0 code segment + gIdt[vec].Ist = ist; + gIdt[vec].TypeAttr = typeAttr; + gIdt[vec].OffsetMid = (uint16_t)((handler >> 16) & 0xFFFF); + gIdt[vec].OffsetHigh = (uint32_t)((handler >> 32) & 0xFFFFFFFF); + gIdt[vec].Reserved = 0; +} + +void KeInitializeIdt(void) { + memset(gIdt, 0, sizeof(gIdt)); + + KiIdtSetGate(0, (uintptr_t)KeFaultDE, 0x8E, 0); + KiIdtSetGate(13, (uintptr_t)KeFaultGP, 0x8E, 0); + KiIdtSetGate(14, (uintptr_t)KeFaultPF, 0x8E, 0); + KiIdtSetGate(0x40, (uintptr_t)KeSyscallEntry, 0xEE, 0); + + static void *irqStubs[NUM_IRQS] = { + (void*)KeIrqStub0, (void*)KeIrqStub1, (void*)KeIrqStub2, + (void*)KeIrqStub3, (void*)KeIrqStub4, (void*)KeIrqStub5, + (void*)KeIrqStub6, (void*)KeIrqStub7, (void*)KeIrqStub8, + (void*)KeIrqStub9, (void*)KeIrqStub10, (void*)KeIrqStub11, + (void*)KeIrqStub12, (void*)KeIrqStub13, (void*)KeIrqStub14, + (void*)KeIrqStub15, (void*)KeIrqStub16, (void*)KeIrqStub17, + (void*)KeIrqStub18, (void*)KeIrqStub19, (void*)KeIrqStub20, + (void*)KeIrqStub21, (void*)KeIrqStub22, (void*)KeIrqStub23, + }; + for (int i = 0; i < NUM_IRQS; i++) + KiIdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], 0x8E, 0); + + IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt }; + + // Call the MASM64 proc to load IDT + KeFlushIdt(&idtr); +} diff --git a/code/kernel/ke_irp.c b/code/kernel/ke_irp.c index c7ab7d8..cded325 100644 --- a/code/kernel/ke_irp.c +++ b/code/kernel/ke_irp.c @@ -10,12 +10,12 @@ // → ke_event.h (KeEventCreate, KeSignalEvent, KeWaitForEvent) // → ke_semaphore.h (KeSignalSemaphore) // → ob_object.h (ObDereferenceObject) -// → io_driver.h (HalDiskDispatch) -// hal_disk.c → io_driver.h only (no ke_event.h, no ke_semaphore.h) +// → io_driver.h (DEVICE_OBJECT / IoCallDriver) #include #include #include +#include #include #include @@ -51,33 +51,53 @@ __declspec(dllexport) IRP *IrpQueueDequeue(IRP_QUEUE *q) { return CONTAINING_RECORD(node, IRP, Link); } +// ------------------------------------------------------------------------- +// IoCompleteRequest +// ------------------------------------------------------------------------- + +static size_t IoIrpTransferBytes(const IRP *irp, KSTATUS status) { + if (!irp || !KSUCCESS(status)) return 0; + return (size_t)irp->SectorCount * 512u; +} + +__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information) { + if (!irp) return; + + irp->Status = status; + irp->Information = information; + + // Queue the APC before waking event waiters so a thread that waits on the + // completion event will see and deliver the APC on its wait-return path. + if (irp->CompletionApc) + KeInsertQueueApc((KAPC *)irp->CompletionApc, irp, (void *)(uintptr_t)status); + if (irp->CompletionEvent) + KeSignalEvent((EVENT *)irp->CompletionEvent); + if (irp->CompletionFn) + irp->CompletionFn(irp, irp->CompletionCtx); +} + // ------------------------------------------------------------------------- // IoSubmitIrp // ------------------------------------------------------------------------- -// Calls HalDiskDispatch() which is HAL-layer safe (no event references). -// If the dispatch ran inline (sync fallback), we fire the completion hooks -// here in the kernel layer where we can legally call KeSignalEvent. +// Dispatches through the registered DEVICE_OBJECT. +// Accepted IRPs are always completed by the target driver via IoCompleteRequest(). __declspec(dllexport) KSTATUS IoSubmitIrp(IRP *irp) { if (!irp) return STATUS_INVALID_PARAMETER; - KSTATUS st = HalDiskDispatch(irp->DriveIndex, irp); + irp->Status = STATUS_FAILURE; + irp->Information = 0; - if (st == STATUS_NOT_FOUND) - return STATUS_NOT_FOUND; + DEVICE_OBJECT *device = IoGetDiskDevice(irp->DriveIndex); + if (!device) return STATUS_NOT_FOUND; - if (st == STATUS_SUCCESS) { - // Sync fallback ran; irp->Status is already written. - // Fire completion hooks now - we are in kernel context so this is safe. - if (irp->CompletionEvent) - KeSignalEvent((EVENT *)irp->CompletionEvent); - if (irp->CompletionFn) - irp->CompletionFn(irp, irp->CompletionCtx); - } + KSTATUS st = IoCallDriver(device, irp); - // Return STATUS_PENDING in both cases so callers have a uniform contract: - // "completion is signalled via the event/callback, not this return value." - return STATUS_PENDING; + // Definitive driver contract: + // STATUS_PENDING means the IRP was accepted and will be completed via + // IoCompleteRequest(), possibly inline before IoCallDriver returns. + // Any other status means the IRP was rejected and no completion occurs. + return st; } // ------------------------------------------------------------------------- @@ -94,19 +114,17 @@ __declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp) { KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev); if (!KSUCCESS(st)) return st; - // Attach our event; preserve any callback the caller may have set. + // Attach our event; preserve any APC/callback the caller may have set. irp->CompletionEvent = ev; st = IoSubmitIrp(irp); - // IoSubmitIrp always returns STATUS_PENDING on the non-error paths. - // The event is signalled either by the driver worker (async) or by - // IoSubmitIrp itself (sync fallback), so KeWaitForEvent will always - // return promptly in the sync-fallback case. - if (st == STATUS_PENDING) - KeWaitForEvent(ev); + if (st == STATUS_PENDING) { + KeWaitForEvent(ev, 0); + st = irp->Status; + } // Release our reference to the temporary event object. ObDereferenceObject(&ev->Header); - return irp->Status; + return st; } diff --git a/code/kernel/ke_irp.h b/code/kernel/ke_irp.h index 411374d..c5f4d99 100644 --- a/code/kernel/ke_irp.h +++ b/code/kernel/ke_irp.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -20,6 +21,7 @@ // We only ever hold pointers in IRP and IRP_QUEUE, so a forward decl suffices. struct EVENT; struct SEMAPHORE; +struct KAPC; // ------------------------------------------------------------------------- // IRP flags @@ -34,7 +36,9 @@ struct SEMAPHORE; typedef struct IRP IRP; -// Completion callback: invoked in driver worker context; must not block. +// Completion callback: invoked by IoCompleteRequest() in the completing +// context. Today that is the driver worker or sync fallback path; future +// DPC/APC work will keep all completion policy centralized here. typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx); // ------------------------------------------------------------------------- @@ -56,10 +60,15 @@ typedef struct IRP { // ---- Completion (caller fills before submit; driver writes Status) ---- volatile KSTATUS Status; + volatile size_t Information; // bytes transferred on success // Forward-declared pointer: callers that create events include ke_event.h. struct EVENT *CompletionEvent; + // Optional kernel APC. If set, IoCompleteRequest queues it to its + // target thread with arg1=irp and arg2=(void*)(uintptr_t)status. + struct KAPC *CompletionApc; + IRP_COMPLETION_FN CompletionFn; void *CompletionCtx; @@ -88,6 +97,10 @@ __declspec(dllexport) IRP *IrpQueueDequeue (IRP_QUEUE *q); // Public dispatch API - implemented in kernel/ke_irp.c // ------------------------------------------------------------------------- +// Complete an IRP exactly once: publish status/information, signal the +// optional completion event, then invoke the optional completion callback. +__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information); + // Submit asynchronously. Returns STATUS_PENDING on success. __declspec(dllexport) KSTATUS IoSubmitIrp (IRP *irp); diff --git a/code/kernel/ke_irq.c b/code/kernel/ke_irq.c index 82c82e6..d7b1d04 100644 --- a/code/kernel/ke_irq.c +++ b/code/kernel/ke_irq.c @@ -1,30 +1,72 @@ -// ke_irq.c - IRQ dispatch, kernel-side -#include -#include - -#define MAX_IRQS 24 - -static void (*gIrqHandlers[MAX_IRQS])(void); -static const HAL_API *gHal; - -void KeIrqInitialize(const HAL_API *hal) { - gHal = hal; -} - -__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) { - if (irq >= MAX_IRQS || !handler) return; - gIrqHandlers[irq] = handler; - gHal->Intc.RouteIrq(irq); -} - -void KeIrqDispatch(uint8_t irq) { - // KeIrqDispatch sends EOI after the handler returns, but the handler switched context before returning. - // The EOI call is still sitting on the idle thread's stack, waiting for that thread to be resumed. - // The LAPIC sees ISR[0x20] still set. Its rule is: don't deliver another interrupt at the same or lower priority until EOI is written. - // So the timer never fires again. The user process runs completely unpreempted until it calls SYS_EXIT_PROCESS, which calls KeYield, which cooperatively switches back - // and only then does the idle thread resume, KeIrqDispatch finally sends its EOI, and the timer starts firing again. - // To fix this we send EOI before calling the handler, not after. - gHal->Intc.SendEoi(); - if (irq < MAX_IRQS && gIrqHandlers[irq]) - gIrqHandlers[irq](); -} +// ke_irq.c - IRQ dispatch, kernel-side +#include +#include +#include +#include +#include + +#define MAX_IRQS 24 + +static void (*gIrqHandlers[MAX_IRQS])(void); +static const HAL_API *gHal; +static volatile uint32_t gInterruptDepth = 0; + +static KIRQL KiIrqToIrql(uint8_t irq) { + KIRQL irql = (KIRQL)(DIRQL_BASE + irq); + if (irql >= HIGH_LEVEL) + irql = (KIRQL)(HIGH_LEVEL - 1); + return irql; +} + +void KeIrqInitialize(const HAL_API *hal) { + gHal = hal; +} + +__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) { + if (irq >= MAX_IRQS || !handler) return; + gIrqHandlers[irq] = handler; + gHal->Intc.RouteIrq(irq); +} + +void KeIrqDispatch(uint8_t irq) { + KIRQL targetIrql = KiIrqToIrql(irq); + KIRQL currentIrql = KeGetCurrentIrql(); + + // Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected + // while IF is clear, but this keeps the software state sane if that changes + // later. + if (targetIrql < currentIrql) + targetIrql = currentIrql; + + KIRQL oldIrql = KeRaiseIrql(targetIrql); + + if (irq < MAX_IRQS && gIrqHandlers[irq]) + gIrqHandlers[irq](); + + // Handlers are no longer allowed to context-switch directly. Timer IRQs + // only request rescheduling; KiExitInterrupt performs the switch after this + // function returns, so the EOI is guaranteed to be sent first. Keep the + // software level at DIRQL until after the controller has been acknowledged. + gHal->Intc.SendEoi(); + + KeLowerIrql(oldIrql); +} + +void KiEnterInterrupt(void) { + gInterruptDepth++; +} + +void KiExitInterrupt(void) { + if (gInterruptDepth) + gInterruptDepth--; + + // A reschedule is only legal after the interrupt unwound back to the + // interrupted thread's original PASSIVE context. If a future path ever + // permits nested interrupting of DISPATCH/DIRQL code, leave the request + // pending instead of switching while a spinlock/high-IRQL region is active. + if (gInterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) { + KiDrainDpcQueue(); + if (KeGetCurrentIrql() == PASSIVE_LEVEL) + KiScheduleFromInterrupt(); + } +} diff --git a/code/kernel/ke_irq.h b/code/kernel/ke_irq.h index a609e51..76aadbd 100644 --- a/code/kernel/ke_irq.h +++ b/code/kernel/ke_irq.h @@ -5,3 +5,10 @@ void KeIrqInitialize(const HAL_API *hal); __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)); void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm + +// Low-level interrupt entry/exit hooks. These are called by the common IRQ +// assembly stub around KeIrqDispatch. The exit hook is where deferred +// interrupt-time scheduling now happens, so IRQ handlers never switch away +// before KeIrqDispatch has returned and sent EOI. +void KiEnterInterrupt(void); +void KiExitInterrupt(void); diff --git a/code/kernel/ke_irql.c b/code/kernel/ke_irql.c new file mode 100644 index 0000000..de50a04 --- /dev/null +++ b/code/kernel/ke_irql.c @@ -0,0 +1,82 @@ +// ke_irql.c - software IRQL skeleton +#include +#include +#include + +// Single-CPU bring-up state. When SMP arrives this becomes per-CPU state. +static volatile KIRQL gCurrentIrql = PASSIVE_LEVEL; + +__declspec(dllexport) const char* KeIrqlName(KIRQL irql) { + switch (irql) { + case PASSIVE_LEVEL: return "PASSIVE_LEVEL"; + case APC_LEVEL: return "APC_LEVEL"; + case DISPATCH_LEVEL: return "DISPATCH_LEVEL"; + case HIGH_LEVEL: return "HIGH_LEVEL"; + default: + if (irql >= DIRQL_BASE && irql < HIGH_LEVEL) + return "DIRQL"; + return "INVALID_IRQL"; + } +} + +__declspec(dllexport) KIRQL KeGetCurrentIrql(void) { + return gCurrentIrql; +} + +__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) { + KIRQL oldIrql = gCurrentIrql; + + if (newIrql > HIGH_LEVEL) { + KePanic("KeRaiseIrql: invalid target IRQL %u\n", (unsigned)newIrql); + } + + if (newIrql < oldIrql) { + KePanic("KeRaiseIrql: attempted raise from %s(%u) down to %s(%u)\n", + KeIrqlName(oldIrql), (unsigned)oldIrql, + KeIrqlName(newIrql), (unsigned)newIrql); + } + + gCurrentIrql = newIrql; + return oldIrql; +} + +__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) { + KIRQL currentIrql = gCurrentIrql; + + if (oldIrql > HIGH_LEVEL) { + KePanic("KeLowerIrql: invalid target IRQL %u\n", (unsigned)oldIrql); + } + + if (oldIrql > currentIrql) { + KePanic("KeLowerIrql: attempted lower from %s(%u) up to %s(%u)\n", + KeIrqlName(currentIrql), (unsigned)currentIrql, + KeIrqlName(oldIrql), (unsigned)oldIrql); + } + + gCurrentIrql = oldIrql; + + if (oldIrql == PASSIVE_LEVEL && currentIrql != PASSIVE_LEVEL) + KiDeliverKernelApcs(); +} + +__declspec(dllexport) void KeAssertIrql(KIRQL expected) { + KIRQL currentIrql = KeGetCurrentIrql(); + if (currentIrql != expected) { + KePanic("IRQL assertion failed: expected %s(%u), current %s(%u)\n", + KeIrqlName(expected), (unsigned)expected, + KeIrqlName(currentIrql), (unsigned)currentIrql); + } +} + +__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) { + KIRQL currentIrql = KeGetCurrentIrql(); + if (currentIrql > maxIrql) { + KePanic("IRQL assertion failed: expected <= %s(%u), current %s(%u)\n", + KeIrqlName(maxIrql), (unsigned)maxIrql, + KeIrqlName(currentIrql), (unsigned)currentIrql); + } +} + +__declspec(dllexport) int KeCanWait(void) { + return KeGetCurrentIrql() == PASSIVE_LEVEL; +} diff --git a/code/kernel/ke_irql.h b/code/kernel/ke_irql.h new file mode 100644 index 0000000..f14b257 --- /dev/null +++ b/code/kernel/ke_irql.h @@ -0,0 +1,34 @@ +#pragma once +// ke_irql.h - software IRQL skeleton +// +// This is intentionally small for the first IRQL/APC bring-up step. It does +// not yet program interrupt hardware or replace the spinlock interrupt-disable +// path. It gives the kernel one authoritative execution-level state plus +// assertions so blocking code can reject illegal callers early. + +#include + +typedef uint8_t KIRQL; + +#define PASSIVE_LEVEL ((KIRQL)0) +#define APC_LEVEL ((KIRQL)1) +#define DISPATCH_LEVEL ((KIRQL)2) +#define DIRQL_BASE ((KIRQL)16) +#define HIGH_LEVEL ((KIRQL)31) + +__declspec(dllexport) KIRQL KeGetCurrentIrql(void); + +// Raise to newIrql and return the previous level. Raising to a lower level is +// a kernel bug and will panic in checked/debug-style builds. +__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql); + +// Restore a previously returned level. Lowering to a higher level is a kernel +// bug and will panic. +__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql); + +__declspec(dllexport) void KeAssertIrql(KIRQL expected); +__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql); +__declspec(dllexport) int KeCanWait(void); + +// Internal/debug helper used by diagnostics and panic messages. +__declspec(dllexport) const char* KeIrqlName(KIRQL irql); diff --git a/code/kernel/ke_mutex.c b/code/kernel/ke_mutex.c index 742ed31..3b66ed8 100644 --- a/code/kernel/ke_mutex.c +++ b/code/kernel/ke_mutex.c @@ -1,66 +1,73 @@ -// ke_mutex.c - binary ownership mutex -#include -#include -#include -#include -#include -#include - -static void MutexDestroy(OBJECT* obj) { (void)obj; } - -KSTATUS MutexCreate(MUTEX **out) { - if (!out) return STATUS_INVALID_PARAMETER; - MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX)); - if (!m) return STATUS_NO_MEMORY; - m->Header.Destroy = MutexDestroy; - m->Lock = (SPINLOCK)SPINLOCK_INIT; - m->Owner = (struct THREAD *)0; - LIST_INIT(&m->WaitList); - *out = m; - return STATUS_SUCCESS; -} - -KSTATUS KeAcquireMutex(MUTEX* mutex) { - if (!mutex) return STATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - - while (true) { - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - if (!mutex->Owner) { - mutex->Owner = self; - KeReleaseSpinlock(&mutex->Lock, flags); - return STATUS_SUCCESS; - } - ListInsertTail(&mutex->WaitList, &self->SchedulerLink); - KeReleaseSpinlock(&mutex->Lock, flags); - KeBlockThread(self, &mutex->Header); - } -} - -KSTATUS KeTryAcquireMutex(MUTEX* mutex) { - if (!mutex) return STATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - if (!mutex->Owner) { - mutex->Owner = self; - KeReleaseSpinlock(&mutex->Lock, flags); - return STATUS_SUCCESS; - } - KeReleaseSpinlock(&mutex->Lock, flags); - return STATUS_FAILURE; -} - -void KeReleaseMutex(MUTEX* mutex) { - if (!mutex) return; - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - mutex->Owner = (struct THREAD *)0; - - if (!LIST_IS_EMPTY(&mutex->WaitList)) { - LIST_ENTRY* node = ListPopHead(&mutex->WaitList); - THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeReleaseSpinlock(&mutex->Lock, flags); - KeUnblockThread(waiter); - return; - } - KeReleaseSpinlock(&mutex->Lock, flags); -} +// ke_mutex.c - binary ownership mutex +#include +#include +#include +#include +#include +#include +#include + +static void KiMutexDestroy(OBJECT* obj) { (void)obj; } + +KSTATUS KeMutexCreate(MUTEX **out) { + if (!out) return STATUS_INVALID_PARAMETER; + MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX)); + if (!m) return STATUS_NO_MEMORY; + m->Header.Destroy = KiMutexDestroy; + m->Lock = (SPINLOCK)SPINLOCK_INIT; + m->Owner = (struct THREAD *)0; + LIST_INIT(&m->WaitList); + *out = m; + return STATUS_SUCCESS; +} + +KSTATUS KeAcquireMutex(MUTEX* mutex) { + if (!mutex) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + THREAD* self = KeGetCurrentThread(); + + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (!mutex->Owner) { + mutex->Owner = self; + KeReleaseSpinlock(&mutex->Lock, flags); + return STATUS_SUCCESS; + } + + KePrepareBlockThread(self, &mutex->Header, 0); + ListInsertTail(&mutex->WaitList, &self->SchedulerLink); + KeSetThreadWaitList(self, &mutex->WaitList); + KeReleaseSpinlock(&mutex->Lock, flags); + return KeCommitBlockThread(self, 0); +} + +KSTATUS KeTryAcquireMutex(MUTEX* mutex) { + if (!mutex) return STATUS_INVALID_PARAMETER; + THREAD* self = KeGetCurrentThread(); + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (!mutex->Owner) { + mutex->Owner = self; + KeReleaseSpinlock(&mutex->Lock, flags); + return STATUS_SUCCESS; + } + KeReleaseSpinlock(&mutex->Lock, flags); + return STATUS_FAILURE; +} + +void KeReleaseMutex(MUTEX* mutex) { + if (!mutex) return; + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + + if (!LIST_IS_EMPTY(&mutex->WaitList)) { + LIST_ENTRY* node = ListPopHead(&mutex->WaitList); + THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + // Direct handoff: the selected waiter owns the mutex when its wait + // returns. Do not drop Owner to NULL, or another thread can steal it. + mutex->Owner = waiter; + KeReleaseSpinlock(&mutex->Lock, flags); + KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS); + return; + } + + mutex->Owner = (struct THREAD *)0; + KeReleaseSpinlock(&mutex->Lock, flags); +} diff --git a/code/kernel/ke_mutex.h b/code/kernel/ke_mutex.h index ad3c466..07ffa24 100644 --- a/code/kernel/ke_mutex.h +++ b/code/kernel/ke_mutex.h @@ -1,18 +1,18 @@ -#pragma once -// ke_mutex.h - binary ownership mutex -#include -#include -#include -#include - -typedef struct MUTEX { - OBJECT Header; - SPINLOCK Lock; - struct THREAD* Owner; /* NULL if unowned */ - LIST_ENTRY WaitList; -} MUTEX; - -KSTATUS MutexCreate(MUTEX **out); -KSTATUS KeAcquireMutex(MUTEX* mutex); -KSTATUS KeTryAcquireMutex(MUTEX* mutex); -void KeReleaseMutex(MUTEX* mutex); +#pragma once +// ke_mutex.h - binary ownership mutex +#include +#include +#include +#include + +typedef struct MUTEX { + OBJECT Header; + SPINLOCK Lock; + struct THREAD* Owner; /* NULL if unowned */ + LIST_ENTRY WaitList; +} MUTEX; + +KSTATUS KeMutexCreate(MUTEX **out); +KSTATUS KeAcquireMutex(MUTEX* mutex); +KSTATUS KeTryAcquireMutex(MUTEX* mutex); +void KeReleaseMutex(MUTEX* mutex); diff --git a/code/kernel/ke_process.c b/code/kernel/ke_process.c index 724aa13..e2418d2 100644 --- a/code/kernel/ke_process.c +++ b/code/kernel/ke_process.c @@ -16,7 +16,7 @@ static LIST_ENTRY gProcessList; static SPINLOCK gProcessListLock; static uint32_t gNextProcessId = 1; -static void ProcessInit(void) { +static void KiProcessInit(void) { static int inited = 0; if (!inited) { LIST_INIT(&gProcessList); @@ -26,7 +26,7 @@ static void ProcessInit(void) { } LIST_ENTRY* KeGetProcessList(void) { - ProcessInit(); + KiProcessInit(); return &gProcessList; } @@ -39,7 +39,7 @@ PROCESS* KeGetCurrentProcess(void) { } size_t KeCopyProcessIds(uint32_t* pidsOut, size_t maxCount) { - ProcessInit(); + KiProcessInit(); uintptr_t flags = KeAcquireSpinlock(&gProcessListLock); size_t count = 0; @@ -56,7 +56,7 @@ size_t KeCopyProcessIds(uint32_t* pidsOut, size_t maxCount) { // --------------------------------------------------------------------------- // Destructor (called by ObDereferenceObject when RefCount hits zero) // --------------------------------------------------------------------------- -static void ProcessDestroy(OBJECT* obj) { +static void KiProcessDestroy(OBJECT* obj) { PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header); // Release all remaining handles. @@ -69,7 +69,7 @@ static void ProcessDestroy(OBJECT* obj) { // Release the main executable image. if (proc->ImageBase && proc->ImageSize) { - MmFreeUserMemory((void *)proc->ImageBase, proc->ImageSize); + MmFreeUserMemory((void *)proc->ImageBase); proc->ImageBase = 0; proc->ImageSize = 0; } @@ -77,7 +77,7 @@ static void ProcessDestroy(OBJECT* obj) { // Release per-process modules. for (uint32_t i = 0; i < proc->ModuleCount; i++) { if (proc->Modules[i].Base && proc->Modules[i].Size) { - MmFreeUserMemory((void *)proc->Modules[i].Base, proc->Modules[i].Size); + MmFreeUserMemory((void *)proc->Modules[i].Base); } proc->Modules[i].Base = 0; proc->Modules[i].Size = 0; @@ -87,22 +87,22 @@ static void ProcessDestroy(OBJECT* obj) { // Release the user-mode process environment. if (proc->UserPebBase && proc->UserPebSize) { - MmFreeUserMemory((void *)proc->UserPebBase, proc->UserPebSize); + MmFreeUserMemory((void *)proc->UserPebBase); proc->UserPebBase = 0; proc->UserPebSize = 0; } if (proc->UserImagePathBase && proc->UserImagePathSize) { - MmFreeUserMemory((void *)proc->UserImagePathBase, proc->UserImagePathSize); + MmFreeUserMemory((void *)proc->UserImagePathBase); proc->UserImagePathBase = 0; proc->UserImagePathSize = 0; } if (proc->UserCmdlineBase && proc->UserCmdlineSize) { - MmFreeUserMemory((void *)proc->UserCmdlineBase, proc->UserCmdlineSize); + MmFreeUserMemory((void *)proc->UserCmdlineBase); proc->UserCmdlineBase = 0; proc->UserCmdlineSize = 0; } if (proc->UserCwdBase && proc->UserCwdSize) { - MmFreeUserMemory((void *)proc->UserCwdBase, proc->UserCwdSize); + MmFreeUserMemory((void *)proc->UserCwdBase); proc->UserCwdBase = 0; proc->UserCwdSize = 0; } @@ -126,14 +126,14 @@ static void ProcessDestroy(OBJECT* obj) { // KeCreateProcess // --------------------------------------------------------------------------- KSTATUS KeCreateProcess(const char* name, PROCESS **out) { - ProcessInit(); + KiProcessInit(); if (!name || !out) return STATUS_INVALID_PARAMETER; PROCESS* proc = (PROCESS *)ObAllocateObject(ObjectTypeProcess, sizeof(PROCESS)); if (!proc) return STATUS_NO_MEMORY; - proc->Header.Destroy = ProcessDestroy; + proc->Header.Destroy = KiProcessDestroy; proc->ProcessId = _InterlockedExchangeAdd((volatile long *)&gNextProcessId, 1); strncpy(proc->Name, name, PROCESS_NAME_MAX - 1); @@ -155,6 +155,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) { proc->UserCwdSize = 0; proc->ModuleCount = 0; proc->CurrentDirectory[0] = '\0'; + proc->CurrentDrive = 0; proc->CwdLock = (SPINLOCK)SPINLOCK_INIT; // Slot 0 is the invalid handle - leave it NULL. @@ -180,7 +181,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) { // --------------------------------------------------------------------------- // KeSetupUserProcessEnvironment // --------------------------------------------------------------------------- -static KSTATUS CopyEnvironmentString(const char* src, uintptr_t* baseOut, size_t* sizeOut) { +static KSTATUS KiCopyEnvironmentString(const char* src, uintptr_t* baseOut, size_t* sizeOut) { if (!baseOut || !sizeOut) return STATUS_INVALID_PARAMETER; if (!src) src = ""; @@ -194,7 +195,7 @@ static KSTATUS CopyEnvironmentString(const char* src, uintptr_t* baseOut, size_t return STATUS_SUCCESS; } -static KSTATUS CopyEnvironmentStringFixed(const char* src, size_t capacity, +static KSTATUS KiCopyEnvironmentStringFixed(const char* src, size_t capacity, uintptr_t* baseOut, size_t* sizeOut) { if (!baseOut || !sizeOut || capacity == 0) return STATUS_INVALID_PARAMETER; if (!src) src = ""; @@ -213,31 +214,48 @@ static KSTATUS CopyEnvironmentStringFixed(const char* src, size_t capacity, return STATUS_SUCCESS; } -static void FreePartialProcessEnvironment(PROCESS* proc) { +static void KiFreePartialProcessEnvironment(PROCESS* proc) { if (!proc) return; if (proc->UserPebBase && proc->UserPebSize) { - MmFreeUserMemory((void*)proc->UserPebBase, proc->UserPebSize); + MmFreeUserMemory((void*)proc->UserPebBase); proc->UserPebBase = 0; proc->UserPebSize = 0; } if (proc->UserImagePathBase && proc->UserImagePathSize) { - MmFreeUserMemory((void*)proc->UserImagePathBase, proc->UserImagePathSize); + MmFreeUserMemory((void*)proc->UserImagePathBase); proc->UserImagePathBase = 0; proc->UserImagePathSize = 0; } if (proc->UserCmdlineBase && proc->UserCmdlineSize) { - MmFreeUserMemory((void*)proc->UserCmdlineBase, proc->UserCmdlineSize); + MmFreeUserMemory((void*)proc->UserCmdlineBase); proc->UserCmdlineBase = 0; proc->UserCmdlineSize = 0; } if (proc->UserCwdBase && proc->UserCwdSize) { - MmFreeUserMemory((void*)proc->UserCwdBase, proc->UserCwdSize); + MmFreeUserMemory((void*)proc->UserCwdBase); proc->UserCwdBase = 0; proc->UserCwdSize = 0; } } +static void KiFormatNativeCwd(const PROCESS* proc, const char* currentDirectory, + char* out, size_t outSize) { + if (!out || outSize == 0) return; + const char* rel = currentDirectory ? currentDirectory : ""; + uint8_t drive = proc ? proc->CurrentDrive : 0; + if (drive > 7) drive = 0; + + if (outSize < 4) { out[0] = '\0'; return; } + out[0] = (char)('0' + drive); + out[1] = ':'; + out[2] = '/'; + size_t len = strlen(rel); + if (len + 4 > outSize) len = outSize - 4; + memcpy(out + 3, rel, len); + out[3 + len] = '\0'; +} + KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc, const char* imagePath, const char* commandLine, @@ -245,17 +263,19 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc, if (!proc || !imagePath) return STATUS_INVALID_PARAMETER; const char* finalCommandLine = (commandLine && commandLine[0]) ? commandLine : imagePath; - const char* finalCwd = currentDirectory ? currentDirectory : proc->CurrentDirectory; + const char* finalCwdRel = currentDirectory ? currentDirectory : proc->CurrentDirectory; + char finalCwd[SYS_CWD_MAX]; + KiFormatNativeCwd(proc, finalCwdRel, finalCwd, sizeof(finalCwd)); - FreePartialProcessEnvironment(proc); + KiFreePartialProcessEnvironment(proc); - KSTATUS s = CopyEnvironmentString(imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize); + KSTATUS s = KiCopyEnvironmentString(imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize); if (!KSUCCESS(s)) goto fail; - s = CopyEnvironmentString(finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize); + s = KiCopyEnvironmentString(finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize); if (!KSUCCESS(s)) goto fail; - s = CopyEnvironmentStringFixed(finalCwd, SYS_CWD_MAX, &proc->UserCwdBase, &proc->UserCwdSize); + s = KiCopyEnvironmentStringFixed(finalCwd, SYS_CWD_MAX, &proc->UserCwdBase, &proc->UserCwdSize); if (!KSUCCESS(s)) goto fail; SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemory(sizeof(SYS_PEB)); @@ -277,7 +297,7 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc, return STATUS_SUCCESS; fail: - FreePartialProcessEnvironment(proc); + KiFreePartialProcessEnvironment(proc); return s; } @@ -290,7 +310,7 @@ void KeExitProcess(PROCESS* process, int exitCode) { // Wake any threads waiting for this process to finish. if (process->ExitEvent) KeSignalEvent(process->ExitEvent); // The last thread exiting via KeExitThread will drop the process refcount. - // This function is a policy hook; actual cleanup is in ProcessDestroy. + // This function is a policy hook; actual cleanup is in KiProcessDestroy. ObDereferenceObject(&process->Header); } @@ -351,7 +371,7 @@ OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) { } PROCESS* KeGetProcessById(uint32_t pid) { - ProcessInit(); + KiProcessInit(); uintptr_t flags = KeAcquireSpinlock(&gProcessListLock); LIST_ENTRY* head = &gProcessList; LIST_ENTRY* node = head->Fwd; diff --git a/code/kernel/ke_process.h b/code/kernel/ke_process.h index ab2d188..e0dab6d 100644 --- a/code/kernel/ke_process.h +++ b/code/kernel/ke_process.h @@ -28,6 +28,7 @@ typedef struct PROCESS { char Name[PROCESS_NAME_MAX]; /* UTF-8 */ char CurrentDirectory[512]; + uint8_t CurrentDrive; SPINLOCK CwdLock; /* Thread list. */ diff --git a/code/kernel/ke_queue.c b/code/kernel/ke_queue.c index c90eaa9..e8d6229 100644 --- a/code/kernel/ke_queue.c +++ b/code/kernel/ke_queue.c @@ -1,85 +1,92 @@ -// ke_queue.c - ring-buffer message queue -#include -#include -#include -#include -#include - -static void QueueDestroy(OBJECT* obj) { (void)obj; } - -KSTATUS QueueCreate(QUEUE **out) { - if (!out) return STATUS_INVALID_PARAMETER; - QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE)); - if (!q) return STATUS_NO_MEMORY; - q->Header.Destroy = QueueDestroy; - q->Lock = (SPINLOCK)SPINLOCK_INIT; - q->Head = q->Tail = q->Count = 0; - LIST_INIT(&q->SendWaiters); - LIST_INIT(&q->RecvWaiters); - *out = q; - return STATUS_SUCCESS; -} - -KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) { - if (!queue || !msg) return STATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - - while (true) { - uintptr_t flags = KeAcquireSpinlock(&queue->Lock); - if (queue->Count < QUEUE_DEPTH) { - queue->Ring[queue->Tail] = *msg; - queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH; - queue->Count++; - - // Wake a receiver if one is waiting. - if (!LIST_IS_EMPTY(&queue->RecvWaiters)) { - LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters); - THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeReleaseSpinlock(&queue->Lock, flags); - KeUnblockThread(t); - } else { - KeReleaseSpinlock(&queue->Lock, flags); - } - return STATUS_SUCCESS; - } - if (!blocking) { - KeReleaseSpinlock(&queue->Lock, flags); - return STATUS_QUEUE_FULL; - } - ListInsertTail(&queue->SendWaiters, &self->SchedulerLink); - KeReleaseSpinlock(&queue->Lock, flags); - KeBlockThread(self, &queue->Header); - } -} - -KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) { - if (!queue || !msg) return STATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - - while (true) { - uintptr_t flags = KeAcquireSpinlock(&queue->Lock); - if (queue->Count > 0) { - *msg = queue->Ring[queue->Head]; - queue->Head = (queue->Head + 1) % QUEUE_DEPTH; - queue->Count--; - - // Wake a sender if one is waiting. - if (!LIST_IS_EMPTY(&queue->SendWaiters)) { - LIST_ENTRY* node = ListPopHead(&queue->SendWaiters); - THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeReleaseSpinlock(&queue->Lock, flags); - KeUnblockThread(t); - } else { - KeReleaseSpinlock(&queue->Lock, flags); - } - return STATUS_SUCCESS; - } - if (!blocking) { - KeReleaseSpinlock(&queue->Lock, flags); - return STATUS_QUEUE_EMPTY; - } - ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink); - KeReleaseSpinlock(&queue->Lock, flags); - KeBlockThread(self, &queue->Header); - } -} +// ke_queue.c - ring-buffer message queue +#include +#include +#include +#include +#include +#include + +static void KiQueueDestroy(OBJECT* obj) { (void)obj; } + +KSTATUS KeQueueCreate(QUEUE **out) { + if (!out) return STATUS_INVALID_PARAMETER; + QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE)); + if (!q) return STATUS_NO_MEMORY; + q->Header.Destroy = KiQueueDestroy; + q->Lock = (SPINLOCK)SPINLOCK_INIT; + q->Head = q->Tail = q->Count = 0; + LIST_INIT(&q->SendWaiters); + LIST_INIT(&q->RecvWaiters); + *out = q; + return STATUS_SUCCESS; +} + +KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) { + if (!queue || !msg) return STATUS_INVALID_PARAMETER; + if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL; + THREAD* self = KeGetCurrentThread(); + + while (true) { + uintptr_t flags = KeAcquireSpinlock(&queue->Lock); + if (queue->Count < QUEUE_DEPTH) { + queue->Ring[queue->Tail] = *msg; + queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH; + queue->Count++; + + // Wake a receiver if one is waiting. + if (!LIST_IS_EMPTY(&queue->RecvWaiters)) { + LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters); + THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeReleaseSpinlock(&queue->Lock, flags); + KeUnblockThread(t); + } else { + KeReleaseSpinlock(&queue->Lock, flags); + } + return STATUS_SUCCESS; + } + if (!blocking) { + KeReleaseSpinlock(&queue->Lock, flags); + return STATUS_QUEUE_FULL; + } + KePrepareBlockThread(self, &queue->Header, 0); + ListInsertTail(&queue->SendWaiters, &self->SchedulerLink); + KeSetThreadWaitList(self, &queue->SendWaiters); + KeReleaseSpinlock(&queue->Lock, flags); + KeCommitBlockThread(self, 0); + } +} + +KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) { + if (!queue || !msg) return STATUS_INVALID_PARAMETER; + if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL; + THREAD* self = KeGetCurrentThread(); + + while (true) { + uintptr_t flags = KeAcquireSpinlock(&queue->Lock); + if (queue->Count > 0) { + *msg = queue->Ring[queue->Head]; + queue->Head = (queue->Head + 1) % QUEUE_DEPTH; + queue->Count--; + + // Wake a sender if one is waiting. + if (!LIST_IS_EMPTY(&queue->SendWaiters)) { + LIST_ENTRY* node = ListPopHead(&queue->SendWaiters); + THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeReleaseSpinlock(&queue->Lock, flags); + KeUnblockThread(t); + } else { + KeReleaseSpinlock(&queue->Lock, flags); + } + return STATUS_SUCCESS; + } + if (!blocking) { + KeReleaseSpinlock(&queue->Lock, flags); + return STATUS_QUEUE_EMPTY; + } + KePrepareBlockThread(self, &queue->Header, 0); + ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink); + KeSetThreadWaitList(self, &queue->RecvWaiters); + KeReleaseSpinlock(&queue->Lock, flags); + KeCommitBlockThread(self, 0); + } +} diff --git a/code/kernel/ke_queue.h b/code/kernel/ke_queue.h index 9a3c48c..f1283c8 100644 --- a/code/kernel/ke_queue.h +++ b/code/kernel/ke_queue.h @@ -1,32 +1,32 @@ -#pragma once -// ke_queue.h - fixed-depth message queue -// Processes share an address space, so a pointer in Param[] is valid -// on both sides. The queue is only a synchronisation and ordering tool. -#include -#include -#include -#include -#include -#include - -#define QUEUE_MESSAGE_SIZE 64 /* bytes, fixed */ -#define QUEUE_DEPTH 256 - -typedef struct QUEUE_MESSAGE { - uint32_t Type; - uint32_t Flags; - uint64_t Param[6]; /* 48 bytes of payload */ -} QUEUE_MESSAGE; /* total: 64 bytes */ - -typedef struct QUEUE { - OBJECT Header; - SPINLOCK Lock; - QUEUE_MESSAGE Ring[QUEUE_DEPTH]; - uint32_t Head, Tail, Count; - LIST_ENTRY SendWaiters; - LIST_ENTRY RecvWaiters; -} QUEUE; - -KSTATUS QueueCreate(QUEUE **out); -KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking); -KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking); +#pragma once +// ke_queue.h - fixed-depth message queue +// Processes share an address space, so a pointer in Param[] is valid +// on both sides. The queue is only a synchronisation and ordering tool. +#include +#include +#include +#include +#include +#include + +#define QUEUE_MESSAGE_SIZE 64 /* bytes, fixed */ +#define QUEUE_DEPTH 256 + +typedef struct QUEUE_MESSAGE { + uint32_t Type; + uint32_t Flags; + uint64_t Param[6]; /* 48 bytes of payload */ +} QUEUE_MESSAGE; /* total: 64 bytes */ + +typedef struct QUEUE { + OBJECT Header; + SPINLOCK Lock; + QUEUE_MESSAGE Ring[QUEUE_DEPTH]; + uint32_t Head, Tail, Count; + LIST_ENTRY SendWaiters; + LIST_ENTRY RecvWaiters; +} QUEUE; + +KSTATUS KeQueueCreate(QUEUE **out); +KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking); +KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking); diff --git a/code/kernel/ke_sched.c b/code/kernel/ke_sched.c index e3452e5..84740c2 100644 --- a/code/kernel/ke_sched.c +++ b/code/kernel/ke_sched.c @@ -1,189 +1,201 @@ -// ke_sched.c - MLFQ scheduler -#include -#include -#include -#include -#include -#include -#include -#include - -#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */ - -static LIST_ENTRY gRunQueue[PriorityCount]; -static SPINLOCK gSchedLock; - -static THREAD* gCurrentThread; -static THREAD* gIdleThread; -static THREAD* gOutgoingThread; -static uint32_t gAgingCounter = 0; - -THREAD* KeGetCurrentThread(void) { return gCurrentThread; } -void KeSetCurrentThread(THREAD* t) { gCurrentThread = t; } - -static void IdleLoop(void) { - while (true) { __halt(); } -} - -void KeInitializeScheduler(void) { - for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]); - gSchedLock = (SPINLOCK)SPINLOCK_INIT; - - PROCESS* idleProc; - KeCreateProcess("idle", &idleProc); - - THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); - idle->Header.Destroy = (void *)0; - idle->ThreadId = 0; - idle->State = ThreadReady; - idle->Priority = PriorityLow; - idle->QuantaConsumed = 0; - idle->Owner = idleProc; - idle->WaitObject = (OBJECT *)0; - idle->SchedLockFlags = 0x202ULL; // IF=1 - - void* kstack = MmAllocatePool(KSTACK_SIZE); - idle->KernelStackBase = kstack; - idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE; - idle->TssRsp0 = idle->KernelStackTop; - - uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop; - ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t); - KCONTEXT* ctx = (KCONTEXT *)ksp; - __stosq((unsigned __int64 *)ctx, 0, sizeof(KCONTEXT) / 8); - ctx->Rip = (uint64_t)IdleLoop; - idle->SavedContext = ctx; - KeInitThreadFpuContext(idle); - - gIdleThread = idle; - gCurrentThread = idle; -} - -// Safely release the lock and clean up any dead threads natively post-switch -void KeSchedulerPostSwitch(void) { - THREAD* cur = KeGetCurrentThread(); - KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags); - - if (gOutgoingThread && gOutgoingThread->State == ThreadDead) { - THREAD* dead = gOutgoingThread; - gOutgoingThread = (THREAD*)0; - ObDereferenceObject(&dead->Header); - } -} - -void KeReadyThread(THREAD* thread) { - if (!thread) return; - thread->State = ThreadReady; - int p = (int)thread->Priority; - if (p < 0) p = 0; - if (p >= PriorityCount) p = PriorityCount - 1; - - uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - ListInsertTail(&gRunQueue[p], &thread->SchedulerLink); - KeReleaseSpinlock(&gSchedLock, flags); -} - -void KeUnreadyThread(THREAD* thread) { - if (!thread) return; - uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - ListRemove(&thread->SchedulerLink); - KeReleaseSpinlock(&gSchedLock, flags); -} - -static THREAD* PickNext(void) { - for (int p = PriorityCount - 1; p >= 0; p--) { - if (!LIST_IS_EMPTY(&gRunQueue[p])) { - LIST_ENTRY* node = gRunQueue[p].Fwd; - ListRemove(node); - return CONTAINING_RECORD(node, THREAD, SchedulerLink); - } - } - return gIdleThread; -} - -void KeSchedulerTick(void) { - THREAD* cur = gCurrentThread; - if (!cur) return; - - uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - cur->SchedLockFlags = flags; // Issue 5 fix - - cur->QuantaConsumed++; - - if (cur != gIdleThread && cur->QuantaConsumed >= DEMOTION_QUANTA) { - cur->QuantaConsumed = 0; - if (cur->Priority > PriorityLow) { - cur->Priority = (THREAD_PRIORITY)(cur->Priority - 1); - } - } - - // Periodic aging pass to prevent indefinite starvation of PriorityLow threads - gAgingCounter++; - if (gAgingCounter >= 100) { - gAgingCounter = 0; - for (int p = 0; p < PriorityHigh; p++) { - while (!LIST_IS_EMPTY(&gRunQueue[p])) { - LIST_ENTRY* node = ListPopHead(&gRunQueue[p]); - THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); - t->Priority = PriorityHigh; - ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink); - } - } - } - - THREAD* next = PickNext(); - if (next == cur) { - if (cur != gIdleThread) { - ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); - } - KeReleaseSpinlock(&gSchedLock, flags); - return; - } - - cur->State = ThreadReady; - next->State = ThreadRunning; - gCurrentThread = next; - - if (cur != gIdleThread && cur->State == ThreadReady) { - ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); - } - - gOutgoingThread = cur; - KeSwitchContext(cur, next); - KeSchedulerPostSwitch(); -} - -__declspec(dllexport) void KeYield(void) { - THREAD* cur = gCurrentThread; - if (!cur) return; - - uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - cur->SchedLockFlags = flags; - - THREAD* next = PickNext(); - if (next == cur) { - // Only re-queue if still running normally; a ThreadWaiting/ThreadDead caller - // has already placed SchedulerLink elsewhere (or is discarding the thread). - if (cur != gIdleThread && cur->State == ThreadRunning) - ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); - KeReleaseSpinlock(&gSchedLock, flags); - return; - } - - next->State = ThreadRunning; - gCurrentThread = next; - - // Only re-queue the outgoing thread if it is still in the running state. - // If KeBlockThread set State = ThreadWaiting, SchedulerLink was already - // inserted into the sync object's wait list - inserting it here too would - // corrupt both lists simultaneously (same intrusive node in two lists). - // If KeExitThread set State = ThreadDead, the thread must not be re-queued. - if (cur != gIdleThread && cur->State == ThreadRunning) { - cur->State = ThreadReady; - ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); - } - - gOutgoingThread = cur; - KeSwitchContext(cur, next); - KeSchedulerPostSwitch(); -} +// ke_sched.c - MLFQ scheduler +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */ + +static LIST_ENTRY gRunQueue[PriorityCount]; +static SPINLOCK gSchedLock; + +static THREAD* gCurrentThread; +static THREAD* gIdleThread; +static THREAD* gOutgoingThread; +static uint32_t gAgingCounter = 0; +static volatile long gReschedulePending = 0; + +THREAD* KeGetCurrentThread(void) { return gCurrentThread; } +void KeSetCurrentThread(THREAD* t) { gCurrentThread = t; } + +static void KiIdleLoop(void) { + while (true) { __halt(); } +} + +void KeInitializeScheduler(void) { + for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]); + gSchedLock = (SPINLOCK)SPINLOCK_INIT; + + PROCESS* idleProc; + KeCreateProcess("idle", &idleProc); + + THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); + idle->Header.Destroy = (void *)0; + idle->ThreadId = 0; + idle->State = ThreadReady; + idle->Priority = PriorityLow; + idle->QuantaConsumed = 0; + idle->Owner = idleProc; + idle->WaitObject = (OBJECT *)0; + idle->WaitStatus = STATUS_SUCCESS; + idle->SchedLockFlags = 0x202ULL; // IF=1 + KiInitializeApcState(idle); + idle->TrapFrame = (KTRAP_FRAME*)0; + + void* kstack = MmAllocatePool(KSTACK_SIZE); + idle->KernelStackBase = kstack; + idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE; + idle->TssRsp0 = idle->KernelStackTop; + + uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop; + ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t); + KCONTEXT* ctx = (KCONTEXT *)ksp; + __stosq((unsigned __int64 *)ctx, 0, sizeof(KCONTEXT) / 8); + ctx->Rip = (uint64_t)KiIdleLoop; + idle->SavedContext = ctx; + KeInitThreadFpuContext(idle); + + gIdleThread = idle; + gCurrentThread = idle; +} + +// Safely release the lock and clean up any dead threads natively post-switch +void KeSchedulerPostSwitch(void) { + THREAD* cur = KeGetCurrentThread(); + KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags); + + KiDeliverKernelApcs(); + + if (gOutgoingThread && gOutgoingThread->State == ThreadDead) { + THREAD* dead = gOutgoingThread; + gOutgoingThread = (THREAD*)0; + ObDereferenceObject(&dead->Header); + } +} + +void KeReadyThread(THREAD* thread) { + if (!thread) return; + thread->State = ThreadReady; + int p = (int)thread->Priority; + if (p < 0) p = 0; + if (p >= PriorityCount) p = PriorityCount - 1; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + ListInsertTail(&gRunQueue[p], &thread->SchedulerLink); + KeReleaseSpinlock(&gSchedLock, flags); +} + +void KeUnreadyThread(THREAD* thread) { + if (!thread) return; + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + ListRemove(&thread->SchedulerLink); + KeReleaseSpinlock(&gSchedLock, flags); +} + +static THREAD* KiPickNext(void) { + for (int p = PriorityCount - 1; p >= 0; p--) { + if (!LIST_IS_EMPTY(&gRunQueue[p])) { + LIST_ENTRY* node = gRunQueue[p].Fwd; + ListRemove(node); + return CONTAINING_RECORD(node, THREAD, SchedulerLink); + } + } + return gIdleThread; +} + +void KiRequestReschedule(void) { + gReschedulePending = 1; +} + +int KiIsReschedulePending(void) { + return gReschedulePending != 0; +} + +static void KiSchedule(void) { + THREAD* cur = gCurrentThread; + if (!cur) return; + + gReschedulePending = 0; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + cur->SchedLockFlags = flags; + + THREAD* next = KiPickNext(); + if (next == cur) { + // Only re-queue if still running normally; a ThreadWaiting/ThreadDead + // caller has already placed SchedulerLink elsewhere (or is discarding + // the thread). + if (cur != gIdleThread && cur->State == ThreadRunning) + ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); + KeReleaseSpinlock(&gSchedLock, flags); + return; + } + + next->State = ThreadRunning; + gCurrentThread = next; + + // Only re-queue the outgoing thread if it is still in the running state. + // If KeBlockThread set State = ThreadWaiting, SchedulerLink was already + // inserted into the sync object's wait list - inserting it here too would + // corrupt both lists simultaneously (same intrusive node in two lists). + // If KeExitThread set State = ThreadDead, the thread must not be re-queued. + if (cur != gIdleThread && cur->State == ThreadRunning) { + cur->State = ThreadReady; + ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); + } + + gOutgoingThread = cur; + KeSwitchContext(cur, next); + KeSchedulerPostSwitch(); +} + +void KiScheduleFromInterrupt(void) { + if (!KiIsReschedulePending()) return; + KiSchedule(); +} + +void KeSchedulerTick(void) { + THREAD* cur = gCurrentThread; + if (!cur) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + cur->SchedLockFlags = flags; // Issue 5 fix + + cur->QuantaConsumed++; + + if (cur != gIdleThread && cur->QuantaConsumed >= DEMOTION_QUANTA) { + cur->QuantaConsumed = 0; + if (cur->Priority > PriorityLow) { + cur->Priority = (THREAD_PRIORITY)(cur->Priority - 1); + } + } + + // Periodic aging pass to prevent indefinite starvation of PriorityLow threads + gAgingCounter++; + if (gAgingCounter >= 100) { + gAgingCounter = 0; + for (int p = 0; p < PriorityHigh; p++) { + while (!LIST_IS_EMPTY(&gRunQueue[p])) { + LIST_ENTRY* node = ListPopHead(&gRunQueue[p]); + THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); + t->Priority = PriorityHigh; + ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink); + } + } + } + + KeReleaseSpinlock(&gSchedLock, flags); + + KiRequestReschedule(); +} + +__declspec(dllexport) void KeYield(void) { + KeAssertIrql(PASSIVE_LEVEL); + KiSchedule(); +} diff --git a/code/kernel/ke_sched.h b/code/kernel/ke_sched.h index c6aeaea..2182951 100644 --- a/code/kernel/ke_sched.h +++ b/code/kernel/ke_sched.h @@ -12,9 +12,18 @@ void KeInitializeScheduler(void); void KeReadyThread(THREAD* thread); void KeUnreadyThread(THREAD* thread); -// Called from the timer ISR every QUANTUM_MS. +// Called from the timer ISR every QUANTUM_MS. This only performs timer +// accounting and requests a reschedule; the actual context switch happens from +// KiExitInterrupt after the IRQ handler has returned and EOI has been sent. void KeSchedulerTick(void); +// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule +// is cheap and can be called from interrupt context. KiScheduleFromInterrupt +// consumes the pending request and performs the context switch, if needed. +void KiRequestReschedule(void); +int KiIsReschedulePending(void); +void KiScheduleFromInterrupt(void); + // Voluntarily yield the CPU. Picks the next runnable thread immediately. __declspec(dllexport) void KeYield(void); diff --git a/code/kernel/ke_selftest.c b/code/kernel/ke_selftest.c new file mode 100644 index 0000000..51508de --- /dev/null +++ b/code/kernel/ke_selftest.c @@ -0,0 +1,702 @@ +// ke_selftest.c - temporary kernel bring-up self-tests for execution/I/O model. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ------------------------------------------------------------------------- +// Kernel APC self-test +// ------------------------------------------------------------------------- + +typedef struct KI_APC_SELFTEST_STATE { + EVENT* ReadyEvent; + EVENT* GoEvent; + EVENT* DoneEvent; + EVENT* FinishedEvent; + + THREAD* WorkerThread; + + volatile int SelfDelivered; + volatile int DeferredDelivered; + volatile int CrossDelivered; +} KI_APC_SELFTEST_STATE; + +static void KiApcSelfTestKernelRoutine(KAPC* apc, + void** normalRoutine, + void** normalContext, + void** systemArgument1, + void** systemArgument2) { + DbgPrint("APC selftest: kernel routine apc=0x%llX normal=0x%llX ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n", + (uint64_t)apc, + (uint64_t)*normalRoutine, + (uint64_t)*normalContext, + (uint64_t)*systemArgument1, + (uint64_t)*systemArgument2, + KeGetCurrentIrql()); +} + +static void KiApcSelfTestNormalRoutine(void* normalContext, + void* systemArgument1, + void* systemArgument2) { + volatile int* counter = (volatile int*)normalContext; + + DbgPrint("APC selftest: normal routine ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n", + (uint64_t)normalContext, + (uint64_t)systemArgument1, + (uint64_t)systemArgument2, + KeGetCurrentIrql()); + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic("APC selftest: normal routine ran above PASSIVE_LEVEL\n"); + + (*counter)++; +} + +static void KiApcSelfTestCrossThreadRoutine(void* normalContext, + void* systemArgument1, + void* systemArgument2) { + KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext; + + DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n", + (uint64_t)systemArgument1, + (uint64_t)systemArgument2, + KeGetCurrentIrql()); + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic("APC selftest: cross-thread APC ran above PASSIVE_LEVEL\n"); + + st->CrossDelivered++; + KeSignalEvent(st->DoneEvent); +} + +static void KiApcSelfTestWorker(void* arg) { + KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)arg; + + DbgPrint("APC selftest: worker started, signaling ready\n"); + KeSignalEvent(st->ReadyEvent); + + DbgPrint("APC selftest: worker waiting for go event\n"); + KSTATUS s = KeWaitForEvent(st->GoEvent, 0); + if (!KSUCCESS(s)) + KePanic("APC selftest: worker wait failed status=%d\n", s); + + // KeCommitBlockThread(..., alertable) should deliver pending kernel APCs before this + // wait returns. If that hook is broken, DoneEvent will never be signaled. + DbgPrint("APC selftest: worker resumed after go event, cross=%d\n", + st->CrossDelivered); + + if (st->CrossDelivered != 1) + KePanic("APC selftest: worker resumed but APC was not delivered\n"); + + KeSignalEvent(st->FinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +typedef struct KI_APC_ALERTABLE_SELFTEST_STATE { + EVENT* ReadyEvent; + EVENT* WaitEvent; + EVENT* FinishedEvent; + THREAD* WorkerThread; + + volatile int ApcDelivered; + volatile KSTATUS WaitStatus; +} KI_APC_ALERTABLE_SELFTEST_STATE; + +static void KiApcAlertableSelfTestNormalRoutine(void* normalContext, + void* systemArgument1, + void* systemArgument2) { + KI_APC_ALERTABLE_SELFTEST_STATE* st = + (KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext; + + DbgPrint("APC selftest: alertable APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n", + (uint64_t)systemArgument1, + (uint64_t)systemArgument2, + KeGetCurrentIrql()); + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic("APC selftest: alertable APC ran above PASSIVE_LEVEL\n"); + + st->ApcDelivered++; +} + +static void KiApcAlertableSelfTestWorker(void* arg) { + KI_APC_ALERTABLE_SELFTEST_STATE* st = + (KI_APC_ALERTABLE_SELFTEST_STATE*)arg; + + DbgPrint("APC selftest: alertable worker ready, entering alertable wait\n"); + KeSignalEvent(st->ReadyEvent); + + KSTATUS s = KeWaitForEvent(st->WaitEvent, 1); + st->WaitStatus = s; + + DbgPrint("APC selftest: alertable wait returned status=%u delivered=%d\n", + s, st->ApcDelivered); + + if (s != STATUS_KERNEL_APC) + KePanic("APC selftest: alertable wait returned wrong status=%u\n", s); + + if (st->ApcDelivered != 1) + KePanic("APC selftest: alertable APC delivery count wrong=%d\n", + st->ApcDelivered); + + KeSignalEvent(st->FinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +static void KiRunKernelApcSelfTest(void) { + DbgPrint("APC selftest: begin\n"); + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic("APC selftest: called above PASSIVE_LEVEL\n"); + + THREAD* self = KeGetCurrentThread(); + if (!self) + KePanic("APC selftest: no current thread\n"); + + // Test 1: queue APC to current thread at PASSIVE_LEVEL. + { + KAPC apc; + volatile int count = 0; + + KeInitializeApc(&apc, + self, + KiApcSelfTestKernelRoutine, + KiApcSelfTestNormalRoutine, + (void*)&count); + + if (!KeInsertQueueApc(&apc, (void*)0x11111111, (void*)0x22222222)) + KePanic("APC selftest: self queue failed\n"); + + if (count != 1) + KePanic("APC selftest: self APC was not delivered immediately, count=%d\n", count); + + DbgPrint("APC selftest: self APC passed\n"); + } + + // Test 2: disable kernel APCs, queue, then re-enable. + { + KAPC apc; + volatile int count = 0; + + KeInitializeApc(&apc, + self, + KiApcSelfTestKernelRoutine, + KiApcSelfTestNormalRoutine, + (void*)&count); + + KeEnterCriticalRegion(); + + if (!KeInsertQueueApc(&apc, (void*)0x33333333, (void*)0x44444444)) + KePanic("APC selftest: deferred queue failed\n"); + + if (count != 0) + KePanic("APC selftest: APC delivered while critical region held, count=%d\n", count); + + KeLeaveCriticalRegion(); + + if (count != 1) + KePanic("APC selftest: deferred APC was not delivered after leave, count=%d\n", count); + + DbgPrint("APC selftest: critical-region deferral passed\n"); + } + + // Test 3: queue APC to another blocked kernel thread, then wake it. + { + KI_APC_SELFTEST_STATE st; + KAPC apc; + memset(&st, 0, sizeof(st)); + + KSTATUS s; + s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent); + if (!KSUCCESS(s)) KePanic("APC selftest: ReadyEvent create failed %d\n", s); + + s = KeEventCreate(EventAutoReset, 0, &st.GoEvent); + if (!KSUCCESS(s)) KePanic("APC selftest: GoEvent create failed %d\n", s); + + s = KeEventCreate(EventAutoReset, 0, &st.DoneEvent); + if (!KSUCCESS(s)) KePanic("APC selftest: DoneEvent create failed %d\n", s); + + s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent); + if (!KSUCCESS(s)) KePanic("APC selftest: FinishedEvent create failed %d\n", s); + + s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker, + (uintptr_t)&st, + PriorityNormal, + &st.WorkerThread); + if (!KSUCCESS(s)) + KePanic("APC selftest: worker create failed %d\n", s); + + s = KeWaitForEvent(st.ReadyEvent, 0); + if (!KSUCCESS(s)) + KePanic("APC selftest: ready wait failed %d\n", s); + + DbgPrint("APC selftest: worker ready, queueing cross-thread APC\n"); + + KeInitializeApc(&apc, + st.WorkerThread, + KiApcSelfTestKernelRoutine, + KiApcSelfTestCrossThreadRoutine, + (void*)&st); + + if (!KeInsertQueueApc(&apc, (void*)0x55555555, (void*)0x66666666)) + KePanic("APC selftest: cross-thread queue failed\n"); + + if (st.CrossDelivered != 0) + KePanic("APC selftest: cross-thread APC delivered before target resumed\n"); + + KeSignalEvent(st.GoEvent); + + s = KeWaitForEvent(st.DoneEvent, 0); + if (!KSUCCESS(s)) + KePanic("APC selftest: done wait failed %d\n", s); + + if (st.CrossDelivered != 1) + KePanic("APC selftest: cross-thread delivery count wrong, count=%d\n", + st.CrossDelivered); + + // The APC signals DoneEvent before the worker returns from the wait. + // Wait for an explicit worker-owned event before dropping stack-backed + // state and event references. THREAD.ExitEvent is not part of this + // kernel-thread contract on all builds. + s = KeWaitForEvent(st.FinishedEvent, 0); + if (!KSUCCESS(s)) + KePanic("APC selftest: worker finished wait failed %d\n", s); + + DbgPrint("APC selftest: cross-thread APC passed\n"); + + ObDereferenceObject(&st.ReadyEvent->Header); + ObDereferenceObject(&st.GoEvent->Header); + ObDereferenceObject(&st.DoneEvent->Header); + ObDereferenceObject(&st.FinishedEvent->Header); + ObDereferenceObject(&st.WorkerThread->Header); + } + + // Test 4: queue APC to a thread blocked in an alertable wait. + // The APC itself must wake the waiter; no event signal is sent. + { + KI_APC_ALERTABLE_SELFTEST_STATE st; + KAPC apc; + memset(&st, 0, sizeof(st)); + st.WaitStatus = STATUS_PENDING; + + KSTATUS s; + s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent); + if (!KSUCCESS(s)) KePanic("APC selftest: alert ReadyEvent create failed %d\n", s); + + s = KeEventCreate(EventAutoReset, 0, &st.WaitEvent); + if (!KSUCCESS(s)) KePanic("APC selftest: alert WaitEvent create failed %d\n", s); + + s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent); + if (!KSUCCESS(s)) KePanic("APC selftest: alert FinishedEvent create failed %d\n", s); + + s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker, + (uintptr_t)&st, + PriorityNormal, + &st.WorkerThread); + if (!KSUCCESS(s)) + KePanic("APC selftest: alertable worker create failed %d\n", s); + + s = KeWaitForEvent(st.ReadyEvent, 0); + if (!KSUCCESS(s)) + KePanic("APC selftest: alert ready wait failed %d\n", s); + + DbgPrint("APC selftest: queueing APC to alertable waiter\n"); + + KeInitializeApc(&apc, + st.WorkerThread, + KiApcSelfTestKernelRoutine, + KiApcAlertableSelfTestNormalRoutine, + (void*)&st); + + if (!KeInsertQueueApc(&apc, (void*)(unsigned long long)0x77777777, (void*)(unsigned long long)0x88888888)) + KePanic("APC selftest: alertable APC queue failed\n"); + + s = KeWaitForEvent(st.FinishedEvent, 0); + if (!KSUCCESS(s)) + KePanic("APC selftest: alert finished wait failed %d\n", s); + + if (st.WaitStatus != STATUS_KERNEL_APC || st.ApcDelivered != 1) + KePanic("APC selftest: alertable result bad status=%u delivered=%d\n", + st.WaitStatus, st.ApcDelivered); + + DbgPrint("APC selftest: alertable wait passed\n"); + + ObDereferenceObject(&st.ReadyEvent->Header); + ObDereferenceObject(&st.WaitEvent->Header); + ObDereferenceObject(&st.FinishedEvent->Header); + ObDereferenceObject(&st.WorkerThread->Header); + } + + DbgPrint("APC selftest: PASS\n"); +} + +// ------------------------------------------------------------------------- +// IRP + kernel APC self-test +// ------------------------------------------------------------------------- + +typedef struct KI_IRP_APC_SELFTEST_STATE { + EVENT* CompletionEvent; + IRP Irp; + KAPC Apc; + uint8_t* Buffer; + + volatile int ApcDelivered; + volatile KSTATUS ApcStatus; + volatile size_t ApcInformation; +} KI_IRP_APC_SELFTEST_STATE; + +static void KiIrpApcSelfTestKernelRoutine(KAPC* apc, + void** normalRoutine, + void** normalContext, + void** systemArgument1, + void** systemArgument2) { + DbgPrint("IRP/APC selftest: kernel routine apc=0x%llX irp=0x%llX status=%u irql=%u\n", + (uint64_t)apc, + (uint64_t)*systemArgument1, + (uint32_t)(uintptr_t)*systemArgument2, + KeGetCurrentIrql()); + + (void)normalRoutine; + (void)normalContext; +} + +static void KiIrpApcSelfTestNormalRoutine(void* normalContext, + void* systemArgument1, + void* systemArgument2) { + KI_IRP_APC_SELFTEST_STATE* st = (KI_IRP_APC_SELFTEST_STATE*)normalContext; + IRP* irp = (IRP*)systemArgument1; + KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2; + + DbgPrint("IRP/APC selftest: APC delivered irp=0x%llX status=%u info=%llu irql=%u\n", + (uint64_t)irp, + status, + (uint64_t)(irp ? irp->Information : 0), + KeGetCurrentIrql()); + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic("IRP/APC selftest: normal routine ran above PASSIVE_LEVEL\n"); + + if (irp != &st->Irp) + KePanic("IRP/APC selftest: APC arg1 was not the submitted IRP\n"); + + st->ApcDelivered++; + st->ApcStatus = status; + st->ApcInformation = irp->Information; +} + +static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) { + DbgPrint("IRP/APC selftest: begin\n"); + + (void)hal; + if (!IoDiskDevicePresent(0)) { + DbgPrint("IRP/APC selftest: SKIP, drive 0 not present\n"); + return; + } + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic("IRP/APC selftest: called above PASSIVE_LEVEL\n"); + + KI_IRP_APC_SELFTEST_STATE st; + memset(&st, 0, sizeof(st)); + + KSTATUS s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent); + if (!KSUCCESS(s)) + KePanic("IRP/APC selftest: completion event create failed %u\n", s); + + st.Buffer = (uint8_t*)MmAllocatePool(512); + if (!st.Buffer) + KePanic("IRP/APC selftest: no buffer\n"); + memset(st.Buffer, 0xCC, 512); + + KeInitializeApc(&st.Apc, + KeGetCurrentThread(), + KiIrpApcSelfTestKernelRoutine, + KiIrpApcSelfTestNormalRoutine, + (void*)&st); + + LIST_INIT(&st.Irp.Link); + st.Irp.DriveIndex = 0; + st.Irp.Flags = 0; + st.Irp.Lba = 0; + st.Irp.SectorCount = 1; + st.Irp.Buffer = st.Buffer; + st.Irp.Status = STATUS_FAILURE; + st.Irp.Information = 0; + st.Irp.CompletionEvent = st.CompletionEvent; + st.Irp.CompletionApc = &st.Apc; + st.Irp.CompletionFn = (IRP_COMPLETION_FN)0; + st.Irp.CompletionCtx = (void*)0; + st.Irp.HwSlot = 0; + st.Irp.DriverCtx = (void*)0; + + DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n"); + s = IoSubmitIrp(&st.Irp); + if (s != STATUS_PENDING) + KePanic("IRP/APC selftest: IoSubmitIrp returned %u\n", s); + + s = KeWaitForEvent(st.CompletionEvent, 0); + if (!KSUCCESS(s)) + KePanic("IRP/APC selftest: completion wait failed %u\n", s); + + // The wait return path should have delivered the pending kernel APC. + if (st.ApcDelivered != 1) + KePanic("IRP/APC selftest: APC was not delivered after I/O wait, count=%d\n", + st.ApcDelivered); + + if (!KSUCCESS(st.Irp.Status) || !KSUCCESS(st.ApcStatus)) + KePanic("IRP/APC selftest: I/O failed irp=%u apc=%u info=%llu\n", + st.Irp.Status, st.ApcStatus, (uint64_t)st.Irp.Information); + + if (st.Irp.Information != 512 || st.ApcInformation != 512) + KePanic("IRP/APC selftest: wrong byte count irp=%llu apc=%llu\n", + (uint64_t)st.Irp.Information, (uint64_t)st.ApcInformation); + + DbgPrint("IRP/APC selftest: first bytes %02X %02X %02X %02X, tail %02X %02X\n", + st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3], + st.Buffer[510], st.Buffer[511]); + + MmFreePool(st.Buffer); + ObDereferenceObject(&st.CompletionEvent->Header); + + DbgPrint("IRP/APC selftest: PASS\n"); +} + + +// ------------------------------------------------------------------------- +// Async file I/O self-test +// ------------------------------------------------------------------------- + +typedef struct KI_FILE_ASYNC_SELFTEST_STATE { + EVENT* CompletionEvent; + FILE_OBJECT* File; + IO_STATUS_BLOCK Iosb; + KAPC Apc; + uint8_t* Buffer; + + volatile int ApcDelivered; + volatile KSTATUS ApcStatus; +} KI_FILE_ASYNC_SELFTEST_STATE; + +static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc, + void** normalRoutine, + void** normalContext, + void** systemArgument1, + void** systemArgument2) { + DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%llX iosb=0x%llX status=%u irql=%u\n", + (uint64_t)apc, + (uint64_t)*systemArgument1, + (uint32_t)(uintptr_t)*systemArgument2, + KeGetCurrentIrql()); + + (void)normalRoutine; + (void)normalContext; +} + +static void KiFileAsyncSelfTestNormalRoutine(void* normalContext, + void* systemArgument1, + void* systemArgument2) { + KI_FILE_ASYNC_SELFTEST_STATE* st = + (KI_FILE_ASYNC_SELFTEST_STATE*)normalContext; + KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2; + + DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%llX status=%u iosb=%u/%llu irql=%u\n", + (uint64_t)systemArgument1, + status, + st->Iosb.Status, + (uint64_t)st->Iosb.Information, + KeGetCurrentIrql()); + + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic("FILE/ASYNC selftest: APC ran above PASSIVE_LEVEL\n"); + + st->ApcDelivered++; + st->ApcStatus = status; +} + +static void KiRunAsyncFileIoSelfTest(void) { + DbgPrint("FILE/ASYNC selftest: begin\n"); + + KI_FILE_ASYNC_SELFTEST_STATE st; + memset(&st, 0, sizeof(st)); + st.Iosb.Status = STATUS_FAILURE; + + KSTATUS s = IoOpenFile("system/init.exe", &st.File); + if (!KSUCCESS(s)) { + DbgPrint("FILE/ASYNC selftest: SKIP, system/init.exe open failed %u\n", s); + return; + } + + s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent); + if (!KSUCCESS(s)) + KePanic("FILE/ASYNC selftest: event create failed %u\n", s); + + st.Buffer = (uint8_t*)MmAllocatePool(512); + if (!st.Buffer) + KePanic("FILE/ASYNC selftest: no buffer\n"); + memset(st.Buffer, 0xCC, 512); + + KeInitializeApc(&st.Apc, + KeGetCurrentThread(), + KiFileAsyncSelfTestKernelRoutine, + KiFileAsyncSelfTestNormalRoutine, + (void*)&st); + + DbgPrint("FILE/ASYNC selftest: submit read system/init.exe offset=0 size=512\n"); + IO_ASYNC_FILE_REQUEST_OBJECT* readReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0; + s = IoReadFileAsyncKernel(st.File, + st.Buffer, + 512, + 0, + &st.Iosb, + st.CompletionEvent, + &st.Apc, + &readReq); + if (s != STATUS_PENDING) + KePanic("FILE/ASYNC selftest: submit returned %u\n", s); + + s = KeWaitForEvent(st.CompletionEvent, 1); + if (s != STATUS_SUCCESS && s != STATUS_KERNEL_APC) + KePanic("FILE/ASYNC selftest: wait returned %u\n", s); + + if (st.Iosb.Status == STATUS_PENDING) { + s = KeWaitForEvent(st.CompletionEvent, 0); + if (!KSUCCESS(s)) + KePanic("FILE/ASYNC selftest: final event wait returned %u\n", s); + } + + if (st.ApcDelivered != 1) + KePanic("FILE/ASYNC selftest: APC not delivered count=%d\n", st.ApcDelivered); + + if (!KSUCCESS(st.Iosb.Status) || !KSUCCESS(st.ApcStatus)) + KePanic("FILE/ASYNC selftest: I/O failed iosb=%u apc=%u info=%llu\n", + st.Iosb.Status, st.ApcStatus, (uint64_t)st.Iosb.Information); + + if (st.Iosb.Information < 2 || st.Buffer[0] != 'M' || st.Buffer[1] != 'Z') + KePanic("FILE/ASYNC selftest: bad file header info=%llu bytes=%02X %02X\n", + (uint64_t)st.Iosb.Information, st.Buffer[0], st.Buffer[1]); + + DbgPrint("FILE/ASYNC selftest: first bytes %02X %02X %02X %02X info=%llu wait=%u\n", + st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3], + (uint64_t)st.Iosb.Information, s); + + if (readReq) + ObDereferenceObject(IoAsyncFileRequestObject(readReq)); + + MmFreePool(st.Buffer); + ObDereferenceObject(&st.CompletionEvent->Header); + IoCloseFile(st.File); + + // Exercise the async write path with an explicit-offset temporary file. + { + static const char payload[] = "async-file-write-selftest\n"; + char verify[sizeof(payload)]; + FILE_OBJECT* wf = (FILE_OBJECT*)0; + FILE_OBJECT* rf = (FILE_OBJECT*)0; + EVENT* wev = (EVENT*)0; + IO_STATUS_BLOCK wiosb; + size_t got = 0; + + memset(verify, 0, sizeof(verify)); + wiosb.Status = STATUS_FAILURE; + wiosb.Information = 0; + + s = IoCreateFile("asyncio.tmp", &wf); + if (!KSUCCESS(s)) + KePanic("FILE/ASYNC selftest: create write temp failed %u\n", s); + + s = KeEventCreate(EventAutoReset, 0, &wev); + if (!KSUCCESS(s)) + KePanic("FILE/ASYNC selftest: write event create failed %u\n", s); + + DbgPrint("FILE/ASYNC selftest: submit write asyncio.tmp size=%llu\n", + (uint64_t)(sizeof(payload) - 1)); + + IO_ASYNC_FILE_REQUEST_OBJECT* writeReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0; + s = IoWriteFileAsyncKernel(wf, + payload, + sizeof(payload) - 1, + 0, + &wiosb, + wev, + (KAPC*)0, + &writeReq); + if (s != STATUS_PENDING) + KePanic("FILE/ASYNC selftest: write submit returned %u\n", s); + + s = KeWaitForEvent(wev, 1); + if (!KSUCCESS(s)) + KePanic("FILE/ASYNC selftest: write wait returned %u\n", s); + + if (!KSUCCESS(wiosb.Status) || wiosb.Information != sizeof(payload) - 1) + KePanic("FILE/ASYNC selftest: write failed status=%u info=%llu\n", + wiosb.Status, (uint64_t)wiosb.Information); + + if (writeReq) + ObDereferenceObject(IoAsyncFileRequestObject(writeReq)); + + IoCloseFile(wf); + wf = (FILE_OBJECT*)0; + + s = IoOpenFile("asyncio.tmp", &rf); + if (!KSUCCESS(s)) + KePanic("FILE/ASYNC selftest: reopen write temp failed %u\n", s); + + s = IoReadFileAt(rf, 0, verify, sizeof(payload) - 1, &got); + if (!KSUCCESS(s) || got != sizeof(payload) - 1 || + memcmp(verify, payload, sizeof(payload) - 1) != 0) { + KePanic("FILE/ASYNC selftest: verify failed status=%u got=%llu\n", + s, (uint64_t)got); + } + + IoCloseFile(rf); + ObDereferenceObject(&wev->Header); + s = IoDeleteFile("asyncio.tmp"); + if (!KSUCCESS(s)) + KePanic("FILE/ASYNC selftest: delete temp failed %u\n", s); + + DbgPrint("FILE/ASYNC selftest: async write verified\n"); + } + + DbgPrint("FILE/ASYNC selftest: PASS\n"); +} + + +static void KiRunTrapFrameSelfTest(void) { + DbgPrint("TRAP selftest: begin\n"); + + if (KeGetCurrentTrapFrame()) + KePanic("TRAP selftest: kernel selftest unexpectedly has active trap frame\n"); + + DbgPrint("TRAP selftest: frame size=%llu query size=%llu\n", + (uint64_t)sizeof(KTRAP_FRAME), + (uint64_t)sizeof(SYS_TRAP_FRAME_INFO_K)); + + if (sizeof(KTRAP_FRAME) != 240) + KePanic("TRAP selftest: KTRAP_FRAME size mismatch %llu\n", + (uint64_t)sizeof(KTRAP_FRAME)); + + DbgPrint("TRAP selftest: PASS\n"); +} + +void KiRunKernelSelfTests(const HAL_API* hal) { + DbgPrint("Kernel selftests: begin\n"); + KiRunKernelApcSelfTest(); + KiRunIrpKernelApcSelfTest(hal); + KiRunAsyncFileIoSelfTest(); + KiRunTrapFrameSelfTest(); + DbgPrint("Kernel selftests: PASS\n"); +} diff --git a/code/kernel/ke_selftest.h b/code/kernel/ke_selftest.h new file mode 100644 index 0000000..1770cc4 --- /dev/null +++ b/code/kernel/ke_selftest.h @@ -0,0 +1,9 @@ +#pragma once +// ke_selftest.h - temporary kernel bring-up self-tests. +// +// These tests intentionally live behind one call site so they can be walled +// off or compiled out later without touching kmain.c again. + +#include + +void KiRunKernelSelfTests(const HAL_API* hal); diff --git a/code/kernel/ke_semaphore.c b/code/kernel/ke_semaphore.c index 8ba70ae..3a464ba 100644 --- a/code/kernel/ke_semaphore.c +++ b/code/kernel/ke_semaphore.c @@ -1,65 +1,71 @@ -// ke_semaphore.c - counting semaphore -#include -#include -#include -#include -#include - -static void SemDestroy(OBJECT* obj) { (void)obj; } - -__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out) { - if (!out || initial < 0 || max < 1 || initial > max) - return STATUS_INVALID_PARAMETER; - SEMAPHORE* s = (SEMAPHORE *)ObAllocateObject(ObjectTypeSemaphore, sizeof(SEMAPHORE)); - if (!s) return STATUS_NO_MEMORY; - s->Header.Destroy = SemDestroy; - s->Lock = (SPINLOCK)SPINLOCK_INIT; - s->Count = initial; - s->MaxCount = max; - LIST_INIT(&s->WaitList); - *out = s; - return STATUS_SUCCESS; -} - -__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) { - if (!sem) return STATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - - while (true) { - uintptr_t flags = KeAcquireSpinlock(&sem->Lock); - if (sem->Count > 0) { - sem->Count--; - KeReleaseSpinlock(&sem->Lock, flags); - return STATUS_SUCCESS; - } - ListInsertTail(&sem->WaitList, &self->SchedulerLink); - KeReleaseSpinlock(&sem->Lock, flags); - KeBlockThread(self, &sem->Header); - } -} - -__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) { - if (!sem) return STATUS_INVALID_PARAMETER; - uintptr_t flags = KeAcquireSpinlock(&sem->Lock); - if (sem->Count > 0) { - sem->Count--; - KeReleaseSpinlock(&sem->Lock, flags); - return STATUS_SUCCESS; - } - KeReleaseSpinlock(&sem->Lock, flags); - return STATUS_FAILURE; -} - -__declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) { - if (!sem) return; - uintptr_t flags = KeAcquireSpinlock(&sem->Lock); - if (!LIST_IS_EMPTY(&sem->WaitList)) { - LIST_ENTRY* node = ListPopHead(&sem->WaitList); - THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeReleaseSpinlock(&sem->Lock, flags); - KeUnblockThread(waiter); - return; - } - if (sem->Count < sem->MaxCount) sem->Count++; - KeReleaseSpinlock(&sem->Lock, flags); -} +// ke_semaphore.c - counting semaphore +#include +#include +#include +#include +#include +#include + +static void KiSemaphoreDestroy(OBJECT* obj) { (void)obj; } + +__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out) { + if (!out || initial < 0 || max < 1 || initial > max) + return STATUS_INVALID_PARAMETER; + SEMAPHORE* s = (SEMAPHORE *)ObAllocateObject(ObjectTypeSemaphore, sizeof(SEMAPHORE)); + if (!s) return STATUS_NO_MEMORY; + s->Header.Destroy = KiSemaphoreDestroy; + s->Lock = (SPINLOCK)SPINLOCK_INIT; + s->Count = initial; + s->MaxCount = max; + LIST_INIT(&s->WaitList); + *out = s; + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) { + if (!sem) return STATUS_INVALID_PARAMETER; + if (!KeCanWait()) return STATUS_INVALID_IRQL; + THREAD* self = KeGetCurrentThread(); + + uintptr_t flags = KeAcquireSpinlock(&sem->Lock); + if (sem->Count > 0) { + sem->Count--; + KeReleaseSpinlock(&sem->Lock, flags); + return STATUS_SUCCESS; + } + + KePrepareBlockThread(self, &sem->Header, 0); + ListInsertTail(&sem->WaitList, &self->SchedulerLink); + KeSetThreadWaitList(self, &sem->WaitList); + KeReleaseSpinlock(&sem->Lock, flags); + return KeCommitBlockThread(self, 0); +} + +__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) { + if (!sem) return STATUS_INVALID_PARAMETER; + uintptr_t flags = KeAcquireSpinlock(&sem->Lock); + if (sem->Count > 0) { + sem->Count--; + KeReleaseSpinlock(&sem->Lock, flags); + return STATUS_SUCCESS; + } + KeReleaseSpinlock(&sem->Lock, flags); + return STATUS_FAILURE; +} + +__declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) { + if (!sem) return; + uintptr_t flags = KeAcquireSpinlock(&sem->Lock); + if (!LIST_IS_EMPTY(&sem->WaitList)) { + // Transfer this semaphore release directly to the selected waiter. The + // count stays unchanged; the waiter returns success without looping and + // consuming a count later. + LIST_ENTRY* node = ListPopHead(&sem->WaitList); + THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + KeReleaseSpinlock(&sem->Lock, flags); + KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS); + return; + } + if (sem->Count < sem->MaxCount) sem->Count++; + KeReleaseSpinlock(&sem->Lock, flags); +} diff --git a/code/kernel/ke_spinlock.h b/code/kernel/ke_spinlock.h index 5d62ad0..5b0ed49 100644 --- a/code/kernel/ke_spinlock.h +++ b/code/kernel/ke_spinlock.h @@ -1,37 +1,89 @@ -#pragma once -// ke_spinlock.h - spinlock for short, non-sleeping critical sections -// Uses MSVC interlocked intrinsics. Never hold across a scheduling boundary. -#include -#include - -typedef struct SPINLOCK { - volatile long Locked; -} SPINLOCK; - -#define SPINLOCK_INIT { 0 } - -// Issue 1 Fix: Save EFLAGS, disable interrupts, and return old flags. -static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) { - uintptr_t flags = __readeflags(); - _disable(); - while (_InterlockedCompareExchange(&sl->Locked, 1, 0) != 0) - _mm_pause(); - return flags; -} - -// Issue 1 Fix: Restore EFLAGS upon release. -static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t flags) { - _InterlockedExchange(&sl->Locked, 0); - __writeeflags(flags); -} - -static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outFlags) { - uintptr_t flags = __readeflags(); - _disable(); - if (_InterlockedCompareExchange(&sl->Locked, 1, 0) == 0) { - *outFlags = flags; - return 1; - } - __writeeflags(flags); - return 0; -} +#pragma once +// ke_spinlock.h - spinlock for short, non-sleeping critical sections +// +// Step 3 IRQL bring-up: +// * Acquiring a spinlock disables maskable interrupts and raises the software +// IRQL to at least DISPATCH_LEVEL. +// * Releasing restores both the previous IRQL and the previous RFLAGS.IF +// state through the returned cookie. +// +// The public type remains uintptr_t so existing callers do not need to change. +// Never hold a spinlock across a wait/blocking operation. +#include +#include +#include + +typedef struct SPINLOCK { + volatile long Locked; +} SPINLOCK; + +#define SPINLOCK_INIT { 0 } + +#define KI_SPINLOCK_IRQL_SHIFT ((unsigned)(sizeof(uintptr_t) * 8u - 8u)) +#define KI_SPINLOCK_FLAGS_MASK (~(((uintptr_t)0xFFu) << KI_SPINLOCK_IRQL_SHIFT)) + +static __forceinline uintptr_t KiPackSpinlockCookie(uintptr_t flags, KIRQL oldIrql) { + return (flags & KI_SPINLOCK_FLAGS_MASK) | + (((uintptr_t)oldIrql) << KI_SPINLOCK_IRQL_SHIFT); +} + +static __forceinline uintptr_t KiSpinlockCookieFlags(uintptr_t cookie) { + return cookie & KI_SPINLOCK_FLAGS_MASK; +} + +static __forceinline KIRQL KiSpinlockCookieIrql(uintptr_t cookie) { + return (KIRQL)(cookie >> KI_SPINLOCK_IRQL_SHIFT); +} + +// Save RFLAGS, mask interrupts, raise to DISPATCH_LEVEL if needed, acquire. +// If already running at DIRQL, the higher IRQL is preserved. +static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) { + uintptr_t flags = __readeflags(); + _disable(); + + KIRQL oldIrql = KeGetCurrentIrql(); + if (oldIrql < DISPATCH_LEVEL) + KeRaiseIrql(DISPATCH_LEVEL); + + while (_InterlockedCompareExchange(&sl->Locked, 1, 0) != 0) + _mm_pause(); + + return KiPackSpinlockCookie(flags, oldIrql); +} + +// Release and restore the caller's previous software IRQL/RFLAGS state. +static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t cookie) { + uintptr_t flags = KiSpinlockCookieFlags(cookie); + KIRQL oldIrql = KiSpinlockCookieIrql(cookie); + + _InterlockedExchange(&sl->Locked, 0); + KeLowerIrql(oldIrql); + __writeeflags(flags); +} + +static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie) { + uintptr_t flags = __readeflags(); + _disable(); + + KIRQL oldIrql = KeGetCurrentIrql(); + if (oldIrql < DISPATCH_LEVEL) + KeRaiseIrql(DISPATCH_LEVEL); + + if (_InterlockedCompareExchange(&sl->Locked, 1, 0) == 0) { + *outCookie = KiPackSpinlockCookie(flags, oldIrql); + return 1; + } + + KeLowerIrql(oldIrql); + __writeeflags(flags); + return 0; +} + +// Transitional names for the IRQL-aware driver model work that follows. +static __forceinline uintptr_t KeAcquireSpinlockRaiseToDpc(SPINLOCK* sl) { + return KeAcquireSpinlock(sl); +} + +static __forceinline void KeReleaseSpinlockFromDpc(SPINLOCK* sl, uintptr_t cookie) { + KeReleaseSpinlock(sl, cookie); +} diff --git a/code/kernel/ke_status.h b/code/kernel/ke_status.h index b7096a2..cc1c6ff 100644 --- a/code/kernel/ke_status.h +++ b/code/kernel/ke_status.h @@ -1,39 +1,46 @@ -#pragma once -// ke_status.h - kernel status codes -// All fallible kernel functions return KSTATUS. -// Success is always zero; failures are positive named constants. -#include - -typedef uint32_t KSTATUS; - -// --- General & Control (0x0000) --- -#define STATUS_SUCCESS ((KSTATUS)0x0000) -#define STATUS_FAILURE ((KSTATUS)0x0001) -#define STATUS_NOT_SUPPORTED ((KSTATUS)0x0002) -#define STATUS_TIMEOUT ((KSTATUS)0x0003) -#define STATUS_PENDING ((KSTATUS)0x0004) - -// --- Validation & Parameters (0x0010) --- -#define STATUS_INVALID_PARAMETER ((KSTATUS)0x0010) -#define STATUS_INVALID_HANDLE ((KSTATUS)0x0011) -#define STATUS_BUFFER_TOO_SMALL ((KSTATUS)0x0012) -#define STATUS_CHECKSUM_MISMATCH ((KSTATUS)0x0013) - -// --- Resource & Access (0x0020) --- -#define STATUS_NO_MEMORY ((KSTATUS)0x0020) -#define STATUS_ACCESS_DENIED ((KSTATUS)0x0021) -#define STATUS_NOT_FOUND ((KSTATUS)0x0022) -#define STATUS_ALREADY_EXISTS ((KSTATUS)0x0023) -#define STATUS_OBJECT_NAME_COLLISION ((KSTATUS)0x0024) - -// --- State & Lifecycle (0x0030) --- -#define STATUS_OBJECT_DEAD ((KSTATUS)0x0030) -#define STATUS_QUEUE_FULL ((KSTATUS)0x0031) -#define STATUS_QUEUE_EMPTY ((KSTATUS)0x0032) - -// --- Hardware & External (0x0040) --- -#define STATUS_DEVICE_ERROR ((KSTATUS)0x0040) - -#define STATUS_NO_MORE_FILES ((KSTATUS)0x0050) - -#define KSUCCESS(s) ((s) == STATUS_SUCCESS) +#pragma once +// ke_status.h - kernel status codes +// All fallible kernel functions return KSTATUS. +// Success is always zero; failures are positive named constants. +#include + +typedef uint32_t KSTATUS; + +// --- General & Control (0x0000) --- +#define STATUS_SUCCESS ((KSTATUS)0x0000) +#define STATUS_FAILURE ((KSTATUS)0x0001) +#define STATUS_NOT_SUPPORTED ((KSTATUS)0x0002) +#define STATUS_TIMEOUT ((KSTATUS)0x0003) +#define STATUS_PENDING ((KSTATUS)0x0004) +#define STATUS_ALERTED ((KSTATUS)0x0005) +#define STATUS_KERNEL_APC ((KSTATUS)0x0006) +#define STATUS_USER_APC ((KSTATUS)0x0007) + +// --- Validation & Parameters (0x0010) --- +#define STATUS_INVALID_PARAMETER ((KSTATUS)0x0010) +#define STATUS_INVALID_HANDLE ((KSTATUS)0x0011) +#define STATUS_BUFFER_TOO_SMALL ((KSTATUS)0x0012) +#define STATUS_CHECKSUM_MISMATCH ((KSTATUS)0x0013) + +// --- Resource & Access (0x0020) --- +#define STATUS_NO_MEMORY ((KSTATUS)0x0020) +#define STATUS_ACCESS_DENIED ((KSTATUS)0x0021) +#define STATUS_NOT_FOUND ((KSTATUS)0x0022) +#define STATUS_ALREADY_EXISTS ((KSTATUS)0x0023) +#define STATUS_OBJECT_NAME_COLLISION ((KSTATUS)0x0024) + +// --- State & Lifecycle (0x0030) --- +#define STATUS_OBJECT_DEAD ((KSTATUS)0x0030) +#define STATUS_QUEUE_FULL ((KSTATUS)0x0031) +#define STATUS_QUEUE_EMPTY ((KSTATUS)0x0032) +#define STATUS_CANCELLED ((KSTATUS)0x0033) + +// --- Hardware & External (0x0040) --- +#define STATUS_DEVICE_ERROR ((KSTATUS)0x0040) + +#define STATUS_NO_MORE_FILES ((KSTATUS)0x0050) + +// --- Execution level (0x0060) --- +#define STATUS_INVALID_IRQL ((KSTATUS)0x0060) + +#define KSUCCESS(s) ((s) == STATUS_SUCCESS) diff --git a/code/kernel/ke_syscall.asm b/code/kernel/ke_syscall.asm index 6934ced..487391c 100644 --- a/code/kernel/ke_syscall.asm +++ b/code/kernel/ke_syscall.asm @@ -1,171 +1,364 @@ -; ke_syscall.asm - INT 0x40 kernel entry (MASM64) -; -; On entry the CPU has pushed (ring-3 -> ring-0 transition): -; SS, RSP, RFLAGS, CS, RIP (on the kernel stack via TSS.RSP0) -; RAX = syscall number -; RCX, RDX, R8, R9 = args 0-3 (MS x64) -; R10, R11 = args 4-5 (our extension) - -EXTERN KeSyscallDispatch:PROC - -.CODE - -KeSyscallEntry PROC - sub rsp, 8 ; 16-byte align the stack frame - - ; We need to allocate 32 bytes for shadow space, plus 24 bytes for - ; the 3 stack arguments (a3, a4, a5), plus 8 bytes of padding to keep - ; the stack 16-byte aligned. Total = 64. - sub rsp, 64 - - ; Place the 5th, 6th, and 7th arguments onto the stack for KeSyscallDispatch - mov [rsp + 32], r9 ; a3 - mov [rsp + 40], r10 ; a4 - mov [rsp + 48], r11 ; a5 - - ; Shift the first 4 registers down one slot - mov r9, r8 ; a2 - mov r8, rdx ; a1 - mov rdx, rcx ; a0 - mov rcx, rax ; nr (syscall number) - - ; Call the C dispatcher - call KeSyscallDispatch - - add rsp, 72 ; Clean up (64 + 8) - iretq -KeSyscallEntry ENDP - -; --------------------------------------------------------------------------- -; Fault stubs - thin wrappers that collect the CPU exception frame and -; forward to KeFaultDispatch (C), which handles both kernel panics and -; user-process bugchecks. -; -; KeFaultDispatch signature (MS x64): -; void KeFaultDispatch(int vector, // rcx -; UINT64 rip, // rdx -; UINT64 errCode, // r8 -; UINT64 cr2, // r9 -; UINT64 cs) // [rsp+20h] (5th arg, on stack) -; --------------------------------------------------------------------------- -EXTERN KeFaultDispatch:PROC - -; Allocation for a 5-argument call (MS x64): -; 32 bytes shadow space (0x00..0x1F) -; 8 bytes for 5th arg (0x20..0x27) -; Total = 0x28 bytes. -; After "and rsp,-16" rsp%16==0; sub 0x28 (40) → rsp%16==8, -; so the subsequent "call" push makes rsp 16-byte aligned at callee entry. ✓ -FAULT_FRAME_SIZE EQU 28h - -; --------------------------------------------------------- -; Divide Error (#0) - NO error code pushed by CPU -; Exception frame at entry: [rsp]=RIP [rsp+8]=CS [rsp+16]=RFLAGS -; --------------------------------------------------------- -KeFaultDE PROC - mov rdx, [rsp] ; Arg2: rip - xor r8, r8 ; Arg3: errCode = 0 (no error code for #DE) - xor r9, r9 ; Arg4: cr2 = 0 (not a page fault) - mov r10, [rsp+8] ; cs (save before we clobber rsp) - mov ecx, 0 ; Arg1: vector = 0 - - and rsp, -16 - sub rsp, FAULT_FRAME_SIZE - mov [rsp+20h], r10 ; Arg5: cs - call KeFaultDispatch ; _Noreturn - never returns -KeFaultDE ENDP - -; --------------------------------------------------------- -; General Protection Fault (#13) - error code pushed by CPU -; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS -; --------------------------------------------------------- -KeFaultGP PROC - mov r8, [rsp] ; Arg3: errCode - mov rdx, [rsp+8] ; Arg2: rip - xor r9, r9 ; Arg4: cr2 = 0 (not a page fault) - mov r10, [rsp+16] ; cs - mov ecx, 13 ; Arg1: vector = 13 - - and rsp, -16 - sub rsp, FAULT_FRAME_SIZE - mov [rsp+20h], r10 ; Arg5: cs - call KeFaultDispatch -KeFaultGP ENDP - -; --------------------------------------------------------- -; Page Fault (#14) - error code pushed by CPU -; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS -; --------------------------------------------------------- -KeFaultPF PROC - mov r8, [rsp] ; Arg3: errCode - mov rdx, [rsp+8] ; Arg2: rip - mov r9, cr2 ; Arg4: cr2 (faulting virtual address) - mov r10, [rsp+16] ; cs - mov ecx, 14 ; Arg1: vector = 14 - - and rsp, -16 - sub rsp, FAULT_FRAME_SIZE - mov [rsp+20h], r10 ; Arg5: cs - call KeFaultDispatch -KeFaultPF ENDP - -EXTERN KeIrqDispatch:PROC - -IrqStub MACRO num -PUBLIC KeIrqStub&num -KeIrqStub&num PROC - push rax - mov al, num - jmp KeIrqCommon -KeIrqStub&num ENDP -ENDM - -IrqStub 0 -IrqStub 1 -IrqStub 2 -IrqStub 3 -IrqStub 4 -IrqStub 5 -IrqStub 6 -IrqStub 7 -IrqStub 8 -IrqStub 9 -IrqStub 10 -IrqStub 11 -IrqStub 12 -IrqStub 13 -IrqStub 14 -IrqStub 15 -IrqStub 16 -IrqStub 17 -IrqStub 18 -IrqStub 19 -IrqStub 20 -IrqStub 21 -IrqStub 22 -IrqStub 23 - -KeIrqCommon PROC - ; al = irq number, rax saved on stack - ; Build a minimal calling-convention-safe frame - push rcx - push rdx - push r8 - push r9 - push r10 - push r11 - movzx ecx, al ; first arg = irq number (rcx, Win64 ABI) - sub rsp, 32 ; shadow space - call KeIrqDispatch - add rsp, 32 - pop r11 - pop r10 - pop r9 - pop r8 - pop rdx - pop rcx - pop rax - iretq -KeIrqCommon ENDP - -END +; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64) +; +; Syscall entry now builds an editable KTRAP_FRAME on the current kernel +; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET +; slots, which is the foundation for future user APC return redirection. + +EXTERN KeSyscallDispatch:PROC +EXTERN KiEnterTrapFrame:PROC +EXTERN KiExitTrapFrame:PROC + +KTRAP_FRAME_SIZE EQU 240 +KTRAP_FRAME_MAGIC EQU 4650545241544B31h +KTRAP_KIND_SYSCALL EQU 1 +KTRAP_KIND_INTERRUPT EQU 2 +KTRAP_MODE_KERNEL EQU 0 +KTRAP_MODE_USER EQU 1 + +TF_Magic EQU 0 +TF_Kind EQU 8 +TF_Vector EQU 12 +TF_PreviousMode EQU 16 +TF_Reserved EQU 20 +TF_Rax EQU 24 +TF_Rcx EQU 32 +TF_Rdx EQU 40 +TF_Rbx EQU 48 +TF_Rbp EQU 56 +TF_Rsi EQU 64 +TF_Rdi EQU 72 +TF_R8 EQU 80 +TF_R9 EQU 88 +TF_R10 EQU 96 +TF_R11 EQU 104 +TF_R12 EQU 112 +TF_R13 EQU 120 +TF_R14 EQU 128 +TF_R15 EQU 136 +TF_Rip EQU 144 +TF_Cs EQU 152 +TF_Rflags EQU 160 +TF_Rsp EQU 168 +TF_Ss EQU 176 +TF_RipSlot EQU 184 +TF_CsSlot EQU 192 +TF_RflagsSlot EQU 200 +TF_RspSlot EQU 208 +TF_SsSlot EQU 216 + +.CODE + +KeSyscallEntry PROC + ; CPU frame at entry, ring 3 -> ring 0: + ; [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS, + ; [rsp+18] user RSP, [rsp+20] SS + ; Save RBX because we use it as the trap-frame pointer and user-mode code + ; sees the syscall stub as a normal function call. + push rbx + sub rsp, KTRAP_FRAME_SIZE + mov rbx, rsp + + mov [rbx + TF_Rax], rax + mov rax, KTRAP_FRAME_MAGIC + mov [rbx + TF_Magic], rax + mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL + mov dword ptr [rbx + TF_Vector], 40h + mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER + mov dword ptr [rbx + TF_Reserved], 0 + + mov [rbx + TF_Rcx], rcx + mov [rbx + TF_Rdx], rdx + mov rax, [rbx + KTRAP_FRAME_SIZE] + mov [rbx + TF_Rbx], rax + mov [rbx + TF_Rbp], rbp + mov [rbx + TF_Rsi], rsi + mov [rbx + TF_Rdi], rdi + mov [rbx + TF_R8], r8 + mov [rbx + TF_R9], r9 + mov [rbx + TF_R10], r10 + mov [rbx + TF_R11], r11 + mov [rbx + TF_R12], r12 + mov [rbx + TF_R13], r13 + mov [rbx + TF_R14], r14 + mov [rbx + TF_R15], r15 + + lea rax, [rbx + KTRAP_FRAME_SIZE + 8] + mov [rbx + TF_RipSlot], rax + mov rdx, [rax] + mov [rbx + TF_Rip], rdx + + lea rax, [rbx + KTRAP_FRAME_SIZE + 16] + mov [rbx + TF_CsSlot], rax + mov rdx, [rax] + mov [rbx + TF_Cs], rdx + + lea rax, [rbx + KTRAP_FRAME_SIZE + 24] + mov [rbx + TF_RflagsSlot], rax + mov rdx, [rax] + mov [rbx + TF_Rflags], rdx + + lea rax, [rbx + KTRAP_FRAME_SIZE + 32] + mov [rbx + TF_RspSlot], rax + mov rdx, [rax] + mov [rbx + TF_Rsp], rdx + + lea rax, [rbx + KTRAP_FRAME_SIZE + 40] + mov [rbx + TF_SsSlot], rax + mov rdx, [rax] + mov [rbx + TF_Ss], rdx + + sub rsp, 32 + mov rcx, rbx + call KiEnterTrapFrame + add rsp, 32 + + ; Re-order original user arguments into KeSyscallDispatch(nr,a0..a5). + ; Original registers were saved in the trap frame before any C calls. + sub rsp, 64 + mov rax, [rbx + TF_R9] + mov [rsp + 32], rax ; a3 + mov rax, [rbx + TF_R10] + mov [rsp + 40], rax ; a4 + mov rax, [rbx + TF_R11] + mov [rsp + 48], rax ; a5 + + mov r9, [rbx + TF_R8] ; a2 + mov r8, [rbx + TF_Rdx] ; a1 + mov rdx, [rbx + TF_Rcx] ; a0 + mov rcx, [rbx + TF_Rax] ; nr + call KeSyscallDispatch + add rsp, 64 + + mov [rbx + TF_Rax], rax + + sub rsp, 32 + mov rcx, rbx + call KiExitTrapFrame + add rsp, 32 + + mov rax, [rbx + TF_Rax] + add rsp, KTRAP_FRAME_SIZE + pop rbx + iretq +KeSyscallEntry ENDP + +; --------------------------------------------------------------------------- +; Fault stubs - thin wrappers that collect the CPU exception frame and +; forward to KeFaultDispatch (C), which handles both kernel panics and +; user-process bugchecks. +; +; KeFaultDispatch signature (MS x64): +; void KeFaultDispatch(int vector, // rcx +; UINT64 rip, // rdx +; UINT64 errCode, // r8 +; UINT64 cr2, // r9 +; UINT64 cs) // [rsp+20h] (5th arg, on stack) +; --------------------------------------------------------------------------- +EXTERN KeFaultDispatch:PROC + +; Allocation for a 5-argument call (MS x64): +; 32 bytes shadow space (0x00..0x1F) +; 8 bytes for 5th arg (0x20..0x27) +; Total = 0x28 bytes. +; After "and rsp,-16" rsp%16==0; sub 0x28 (40) → rsp%16==8, +; so the subsequent "call" push makes rsp 16-byte aligned at callee entry. ✓ +FAULT_FRAME_SIZE EQU 28h + +; --------------------------------------------------------- +; Divide Error (#0) - NO error code pushed by CPU +; Exception frame at entry: [rsp]=RIP [rsp+8]=CS [rsp+16]=RFLAGS +; --------------------------------------------------------- +KeFaultDE PROC + mov rdx, [rsp] ; Arg2: rip + xor r8, r8 ; Arg3: errCode = 0 (no error code for #DE) + xor r9, r9 ; Arg4: cr2 = 0 (not a page fault) + mov r10, [rsp+8] ; cs (save before we clobber rsp) + mov ecx, 0 ; Arg1: vector = 0 + + and rsp, -16 + sub rsp, FAULT_FRAME_SIZE + mov [rsp+20h], r10 ; Arg5: cs + call KeFaultDispatch ; _Noreturn - never returns +KeFaultDE ENDP + +; --------------------------------------------------------- +; General Protection Fault (#13) - error code pushed by CPU +; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS +; --------------------------------------------------------- +KeFaultGP PROC + mov r8, [rsp] ; Arg3: errCode + mov rdx, [rsp+8] ; Arg2: rip + xor r9, r9 ; Arg4: cr2 = 0 (not a page fault) + mov r10, [rsp+16] ; cs + mov ecx, 13 ; Arg1: vector = 13 + + and rsp, -16 + sub rsp, FAULT_FRAME_SIZE + mov [rsp+20h], r10 ; Arg5: cs + call KeFaultDispatch +KeFaultGP ENDP + +; --------------------------------------------------------- +; Page Fault (#14) - error code pushed by CPU +; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS +; --------------------------------------------------------- +KeFaultPF PROC + mov r8, [rsp] ; Arg3: errCode + mov rdx, [rsp+8] ; Arg2: rip + mov r9, cr2 ; Arg4: cr2 (faulting virtual address) + mov r10, [rsp+16] ; cs + mov ecx, 14 ; Arg1: vector = 14 + + and rsp, -16 + sub rsp, FAULT_FRAME_SIZE + mov [rsp+20h], r10 ; Arg5: cs + call KeFaultDispatch +KeFaultPF ENDP + +EXTERN KeIrqDispatch:PROC +EXTERN KiEnterInterrupt:PROC +EXTERN KiExitInterrupt:PROC + +IrqStub MACRO num +PUBLIC KeIrqStub&num +KeIrqStub&num PROC + push rax + mov al, num + jmp KeIrqCommon +KeIrqStub&num ENDP +ENDM + +IrqStub 0 +IrqStub 1 +IrqStub 2 +IrqStub 3 +IrqStub 4 +IrqStub 5 +IrqStub 6 +IrqStub 7 +IrqStub 8 +IrqStub 9 +IrqStub 10 +IrqStub 11 +IrqStub 12 +IrqStub 13 +IrqStub 14 +IrqStub 15 +IrqStub 16 +IrqStub 17 +IrqStub 18 +IrqStub 19 +IrqStub 20 +IrqStub 21 +IrqStub 22 +IrqStub 23 + +KeIrqCommon PROC + ; al = irq number, original rax saved by the per-IRQ stub. + push rcx + push rdx + push r8 + push r9 + push r10 + push r11 + push rbx + + sub rsp, KTRAP_FRAME_SIZE + mov rbx, rsp + + movzx edx, al + mov rax, KTRAP_FRAME_MAGIC + mov [rbx + TF_Magic], rax + mov dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT + mov [rbx + TF_Vector], edx + mov dword ptr [rbx + TF_Reserved], 0 + + ; Saved volatile registers live immediately above the trap frame. + mov rax, [rbx + KTRAP_FRAME_SIZE + 56] + mov [rbx + TF_Rax], rax + mov rax, [rbx + KTRAP_FRAME_SIZE + 48] + mov [rbx + TF_Rcx], rax + mov rax, [rbx + KTRAP_FRAME_SIZE + 40] + mov [rbx + TF_Rdx], rax + mov rax, [rbx + KTRAP_FRAME_SIZE + 0] + mov [rbx + TF_Rbx], rax + mov [rbx + TF_Rbp], rbp + mov [rbx + TF_Rsi], rsi + mov [rbx + TF_Rdi], rdi + mov rax, [rbx + KTRAP_FRAME_SIZE + 32] + mov [rbx + TF_R8], rax + mov rax, [rbx + KTRAP_FRAME_SIZE + 24] + mov [rbx + TF_R9], rax + mov rax, [rbx + KTRAP_FRAME_SIZE + 16] + mov [rbx + TF_R10], rax + mov rax, [rbx + KTRAP_FRAME_SIZE + 8] + mov [rbx + TF_R11], rax + mov [rbx + TF_R12], r12 + mov [rbx + TF_R13], r13 + mov [rbx + TF_R14], r14 + mov [rbx + TF_R15], r15 + + ; CPU interrupt frame starts after the 8 saved registers. + lea rax, [rbx + KTRAP_FRAME_SIZE + 64] + mov [rbx + TF_RipSlot], rax + mov rdx, [rax] + mov [rbx + TF_Rip], rdx + + lea rax, [rbx + KTRAP_FRAME_SIZE + 72] + mov [rbx + TF_CsSlot], rax + mov rdx, [rax] + mov [rbx + TF_Cs], rdx + + lea rax, [rbx + KTRAP_FRAME_SIZE + 80] + mov [rbx + TF_RflagsSlot], rax + mov rdx, [rax] + mov [rbx + TF_Rflags], rdx + + mov rax, [rbx + TF_Cs] + and eax, 3 + cmp eax, 3 + jne short IrqKernelModeFrame + + mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER + lea rax, [rbx + KTRAP_FRAME_SIZE + 88] + mov [rbx + TF_RspSlot], rax + mov rdx, [rax] + mov [rbx + TF_Rsp], rdx + lea rax, [rbx + KTRAP_FRAME_SIZE + 96] + mov [rbx + TF_SsSlot], rax + mov rdx, [rax] + mov [rbx + TF_Ss], rdx + jmp short IrqFrameReady + +IrqKernelModeFrame: + mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_KERNEL + mov qword ptr [rbx + TF_Rsp], 0 + mov qword ptr [rbx + TF_Ss], 0 + mov qword ptr [rbx + TF_RspSlot], 0 + mov qword ptr [rbx + TF_SsSlot], 0 + +IrqFrameReady: + sub rsp, 40 ; 32-byte shadow space + alignment/local slack + mov rcx, rbx + call KiEnterTrapFrame + call KiEnterInterrupt + mov ecx, [rbx + TF_Vector] + call KeIrqDispatch + call KiExitInterrupt + mov rcx, rbx + call KiExitTrapFrame + add rsp, 40 + + add rsp, KTRAP_FRAME_SIZE + pop rbx + pop r11 + pop r10 + pop r9 + pop r8 + pop rdx + pop rcx + pop rax + iretq +KeIrqCommon ENDP + +END diff --git a/code/kernel/ke_syscall.c b/code/kernel/ke_syscall.c index 83d4993..b71321b 100644 --- a/code/kernel/ke_syscall.c +++ b/code/kernel/ke_syscall.c @@ -23,7 +23,7 @@ // // KeCloseHandle(proc, handle) // -> ObDereferenceObject(obj) -// -> obj->Destroy(obj) [type-specific cleanup, e.g. FileDestroy] +// -> obj->Destroy(obj) [type-specific cleanup, e.g. KiFileDestroy] // -> MmFreePool(obj) // // There is no separate SyscallCloseFile. Removing that stub was the right @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -46,12 +47,43 @@ #include #include #include +#include // --------------------------------------------------------------------------- // HAL pointer - set once at boot by KeSyscallSetHal() // --------------------------------------------------------------------------- static const HAL_API* gHal; +typedef struct SYS_IO_STATUS_BLOCK_K { + volatile uint32_t Status; + volatile uint64_t Information; +} SYS_IO_STATUS_BLOCK_K; + +typedef void (*SYS_USER_APC_ROUTINE_K)(void* normalContext, + SYS_IO_STATUS_BLOCK_K* iosb, + uint64_t information); + +typedef struct SYS_ASYNC_FILE_REQUEST_K { + uint32_t FileHandle; + uint32_t EventHandle; + uint32_t Flags; + uint32_t Reserved; + uint64_t Offset; + void* Buffer; + size_t Size; + SYS_IO_STATUS_BLOCK_K* Iosb; + SYS_USER_APC_ROUTINE_K ApcRoutine; + void* ApcContext; + uint32_t* RequestHandleOut; // optional cancellation/query handle output +} SYS_ASYNC_FILE_REQUEST_K; + +typedef struct SYS_USER_ASYNC_APC_CONTEXT { + KAPC KernelApc; + THREAD* TargetThread; + SYS_USER_APC_ROUTINE_K UserRoutine; + void* UserContext; +} SYS_USER_ASYNC_APC_CONTEXT; + void KeSyscallSetHal(const HAL_API* hal) { gHal = hal; } // --------------------------------------------------------------------------- @@ -59,7 +91,7 @@ void KeSyscallSetHal(const HAL_API* hal) { gHal = hal; } // Bumps the refcount on success; caller must ObDereferenceObject when done. // Returns NULL if the handle is invalid or the type does not match. // --------------------------------------------------------------------------- -static OBJECT* HandleToObject(uint32_t handle, OBJECT_TYPE expected) { +static OBJECT* KiHandleToObject(uint32_t handle, OBJECT_TYPE expected) { PROCESS* proc = KeGetCurrentProcess(); if (!proc) return NULL; OBJECT* obj = KeLookupHandle(proc, handle); // bumps refcount @@ -76,7 +108,7 @@ static OBJECT* HandleToObject(uint32_t handle, OBJECT_TYPE expected) { // Takes one reference on behalf of the handle slot. // On failure, drops the reference it would have stored. // --------------------------------------------------------------------------- -static KSTATUS AllocHandleForCurrent(OBJECT* obj, uint32_t* handleOut) { +static KSTATUS KiAllocHandleForCurrent(OBJECT* obj, uint32_t* handleOut) { PROCESS* proc = KeGetCurrentProcess(); if (!proc) return STATUS_FAILURE; KSTATUS s = KeAllocateHandle(proc, obj, handleOut); @@ -173,6 +205,24 @@ static uint64_t SyscallSleep(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_SUCCESS; } +// --------------------------------------------------------------------------- +// SYS_SET_USER_APC_DISPATCHER a0 = dispatcher entry in syscall.dll/user image +// --------------------------------------------------------------------------- +static uint64_t SyscallSetUserApcDispatcher(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KeSetUserApcDispatcher((void*)(uintptr_t)a0); +} + +// --------------------------------------------------------------------------- +// SYS_APC_RETURN a0 = SYS_USER_APC_FRAME* on the user stack +// --------------------------------------------------------------------------- +static uint64_t SyscallApcReturn(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KeUserApcReturn((const SYS_USER_APC_FRAME_K*)(uintptr_t)a0); +} + // --------------------------------------------------------------------------- // SYS_SET_THREAD_PRIORITY (7) a0 = priority // --------------------------------------------------------------------------- @@ -197,12 +247,12 @@ static uint64_t SyscallAlloc(uint64_t a0, uint64_t a1, uint64_t a2, } // --------------------------------------------------------------------------- -// SYS_FREE (9) a0 = ptr, a1 = size +// SYS_FREE a0 = ptr // --------------------------------------------------------------------------- static uint64_t SyscallFree(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a2; (void)a3; (void)a4; (void)a5; - if (a0) MmFreeUserMemory((void*)(uintptr_t)a0, (size_t)a1); + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + if (a0) MmFreeUserMemory((void*)(uintptr_t)a0); return STATUS_SUCCESS; } @@ -224,10 +274,10 @@ static uint64_t SyscallMutexCreate(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; if (!a0) return STATUS_INVALID_PARAMETER; MUTEX* mx = NULL; - KSTATUS s = MutexCreate(&mx); + KSTATUS s = KeMutexCreate(&mx); if (!KSUCCESS(s)) return s; uint32_t h = 0; - s = AllocHandleForCurrent(&mx->Header, &h); // takes ownership of the ref + s = KiAllocHandleForCurrent(&mx->Header, &h); // takes ownership of the ref if (!KSUCCESS(s)) return s; *(uint32_t*)(uintptr_t)a0 = h; return STATUS_SUCCESS; @@ -239,7 +289,7 @@ static uint64_t SyscallMutexCreate(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallMutexAcquire(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - MUTEX* mx = (MUTEX*)HandleToObject((uint32_t)a0, ObjectTypeMutex); + MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex); if (!mx) return STATUS_NOT_FOUND; KSTATUS s = KeAcquireMutex(mx); ObDereferenceObject(&mx->Header); @@ -252,7 +302,7 @@ static uint64_t SyscallMutexAcquire(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - MUTEX* mx = (MUTEX*)HandleToObject((uint32_t)a0, ObjectTypeMutex); + MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex); if (!mx) return STATUS_NOT_FOUND; KSTATUS s = KeTryAcquireMutex(mx); ObDereferenceObject(&mx->Header); @@ -265,7 +315,7 @@ static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallMutexRelease(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - MUTEX* mx = (MUTEX*)HandleToObject((uint32_t)a0, ObjectTypeMutex); + MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex); if (!mx) return STATUS_NOT_FOUND; KeReleaseMutex(mx); ObDereferenceObject(&mx->Header); @@ -283,7 +333,7 @@ static uint64_t SyscallSemCreate(uint64_t a0, uint64_t a1, uint64_t a2, KSTATUS s = KeSemaphoreCreate((int32_t)a0, (int32_t)a1, &sem); if (!KSUCCESS(s)) return s; uint32_t h = 0; - s = AllocHandleForCurrent(&sem->Header, &h); + s = KiAllocHandleForCurrent(&sem->Header, &h); if (!KSUCCESS(s)) return s; *(uint32_t*)(uintptr_t)a2 = h; return STATUS_SUCCESS; @@ -295,7 +345,7 @@ static uint64_t SyscallSemCreate(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallSemWait(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - SEMAPHORE* sem = (SEMAPHORE*)HandleToObject((uint32_t)a0, ObjectTypeSemaphore); + SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore); if (!sem) return STATUS_NOT_FOUND; KSTATUS s = KeWaitSemaphore(sem); ObDereferenceObject(&sem->Header); @@ -308,7 +358,7 @@ static uint64_t SyscallSemWait(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - SEMAPHORE* sem = (SEMAPHORE*)HandleToObject((uint32_t)a0, ObjectTypeSemaphore); + SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore); if (!sem) return STATUS_NOT_FOUND; KSTATUS s = KeTrySemaphore(sem); ObDereferenceObject(&sem->Header); @@ -321,7 +371,7 @@ static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallSemSignal(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - SEMAPHORE* sem = (SEMAPHORE*)HandleToObject((uint32_t)a0, ObjectTypeSemaphore); + SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore); if (!sem) return STATUS_NOT_FOUND; KeSignalSemaphore(sem); ObDereferenceObject(&sem->Header); @@ -339,7 +389,7 @@ static uint64_t SyscallEventCreate(uint64_t a0, uint64_t a1, uint64_t a2, KSTATUS s = KeEventCreate((EVENT_TYPE)a0, (uint32_t)a1, &ev); if (!KSUCCESS(s)) return s; uint32_t h = 0; - s = AllocHandleForCurrent(&ev->Header, &h); + s = KiAllocHandleForCurrent(&ev->Header, &h); if (!KSUCCESS(s)) return s; *(uint32_t*)(uintptr_t)a2 = h; return STATUS_SUCCESS; @@ -351,7 +401,7 @@ static uint64_t SyscallEventCreate(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallEventSignal(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - EVENT* ev = (EVENT*)HandleToObject((uint32_t)a0, ObjectTypeEvent); + EVENT* ev = (EVENT*)KiHandleToObject((uint32_t)a0, ObjectTypeEvent); if (!ev) return STATUS_NOT_FOUND; KeSignalEvent(ev); ObDereferenceObject(&ev->Header); @@ -364,7 +414,7 @@ static uint64_t SyscallEventSignal(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallEventClear(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - EVENT* ev = (EVENT*)HandleToObject((uint32_t)a0, ObjectTypeEvent); + EVENT* ev = (EVENT*)KiHandleToObject((uint32_t)a0, ObjectTypeEvent); if (!ev) return STATUS_NOT_FOUND; KeClearEvent(ev); ObDereferenceObject(&ev->Header); @@ -372,14 +422,20 @@ static uint64_t SyscallEventClear(uint64_t a0, uint64_t a1, uint64_t a2, } // --------------------------------------------------------------------------- -// SYS_EVENT_WAIT (22) a0 = handle +// SYS_EVENT_WAIT a0 = handle, a1 = alertable // --------------------------------------------------------------------------- static uint64_t SyscallEventWait(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - EVENT* ev = (EVENT*)HandleToObject((uint32_t)a0, ObjectTypeEvent); + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a2; (void)a3; (void)a4; (void)a5; + EVENT* ev = (EVENT*)KiHandleToObject((uint32_t)a0, ObjectTypeEvent); if (!ev) return STATUS_NOT_FOUND; - KSTATUS s = KeWaitForEvent(ev); + uint8_t alertable = a1 ? 1 : 0; + KSTATUS s = KeWaitForEvent(ev, alertable); + if (alertable) { + THREAD* thread = KeGetCurrentThread(); + if (thread && thread->UserApcPending) + thread->UserApcDeliverable = 1; + } ObDereferenceObject(&ev->Header); return s; } @@ -392,10 +448,10 @@ static uint64_t SyscallQueueCreate(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; if (!a0) return STATUS_INVALID_PARAMETER; QUEUE* q = NULL; - KSTATUS s = QueueCreate(&q); + KSTATUS s = KeQueueCreate(&q); if (!KSUCCESS(s)) return s; uint32_t h = 0; - s = AllocHandleForCurrent(&q->Header, &h); + s = KiAllocHandleForCurrent(&q->Header, &h); if (!KSUCCESS(s)) return s; *(uint32_t*)(uintptr_t)a0 = h; return STATUS_SUCCESS; @@ -408,7 +464,7 @@ static uint64_t SyscallQueueSend(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a3; (void)a4; (void)a5; if (!a1) return STATUS_INVALID_PARAMETER; - QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a0, ObjectTypeQueue); + QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a0, ObjectTypeQueue); if (!q) return STATUS_NOT_FOUND; KSTATUS s = KeSendMessage(q, (const QUEUE_MESSAGE*)(uintptr_t)a1, (bool)a2); ObDereferenceObject(&q->Header); @@ -422,7 +478,7 @@ static uint64_t SyscallQueueRecv(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a3; (void)a4; (void)a5; if (!a1) return STATUS_INVALID_PARAMETER; - QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a0, ObjectTypeQueue); + QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a0, ObjectTypeQueue); if (!q) return STATUS_NOT_FOUND; KSTATUS s = KeReceiveMessage(q, (QUEUE_MESSAGE*)(uintptr_t)a1, (bool)a2); ObDereferenceObject(&q->Header); @@ -434,7 +490,7 @@ static uint64_t SyscallQueueRecv(uint64_t a0, uint64_t a1, uint64_t a2, // // The one and only close path. KeCloseHandle drops the handle-table reference // on 'obj'. If that was the last reference, ObDereferenceObject runs the -// type-specific Destroy callback (FileDestroy, etc.) and frees the pool memory. +// type-specific Destroy callback (KiFileDestroy, etc.) and frees the pool memory. // No per-type close syscall needed or wanted. // --------------------------------------------------------------------------- static uint64_t SyscallCloseHandle(uint64_t a0, uint64_t a1, uint64_t a2, @@ -457,10 +513,10 @@ static uint64_t SyscallOpenFile(uint64_t a0, uint64_t a1, uint64_t a2, KSTATUS s = IoOpenFile((const char*)(uintptr_t)a0, &file); if (!KSUCCESS(s)) return s; uint32_t h = 0; - s = AllocHandleForCurrent(&file->Header, &h); + s = KiAllocHandleForCurrent(&file->Header, &h); if (!KSUCCESS(s)) { - // AllocHandleForCurrent already dropped the ref on failure; - // the refcount is now 0, so ObDereferenceObject will run FileDestroy. + // KiAllocHandleForCurrent already dropped the ref on failure; + // the refcount is now 0, so ObDereferenceObject will run KiFileDestroy. ObDereferenceObject(&file->Header); return s; } @@ -475,7 +531,7 @@ static uint64_t SyscallReadFile(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a4; (void)a5; if (!a1) return STATUS_INVALID_PARAMETER; - FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile); + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile); if (!file) return STATUS_NOT_FOUND; size_t got = 0; KSTATUS s = IoReadFile(file, (void*)(uintptr_t)a1, (size_t)a2, &got); @@ -491,7 +547,7 @@ static uint64_t SyscallWriteFile(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a4; (void)a5; if (!a1) return STATUS_INVALID_PARAMETER; - FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile); + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile); if (!file) return STATUS_NOT_FOUND; size_t written = 0; KSTATUS s = IoWriteFile(file, (const void*)(uintptr_t)a1, (size_t)a2, &written); @@ -500,6 +556,189 @@ static uint64_t SyscallWriteFile(uint64_t a0, uint64_t a1, uint64_t a2, return s; } +static void SyscallUserAsyncKernelRoutine(KAPC* apc, + void** normalRoutine, + void** normalContext, + void** systemArgument1, + void** systemArgument2) { + (void)normalRoutine; + (void)normalContext; + (void)systemArgument1; + (void)systemArgument2; + + SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc); + DbgPrint("USER/APC async: kernel routine ctx=0x%llX iosb=0x%llX status=%u info=%llu irql=%u\n", + (uint64_t)ctx, + (uint64_t)(uintptr_t)apc->SystemArgument1, + (uint32_t)(uintptr_t)apc->SystemArgument2, + apc->SystemArgument1 ? ((SYS_IO_STATUS_BLOCK_K*)apc->SystemArgument1)->Information : 0, + KeGetCurrentIrql()); +} + +static void SyscallUserAsyncApcContextFree(SYS_USER_ASYNC_APC_CONTEXT* ctx) { + if (!ctx) return; + if (ctx->TargetThread) { + ObDereferenceObject(&ctx->TargetThread->Header); + ctx->TargetThread = (THREAD*)0; + } + MmFreePool(ctx); +} + +static void SyscallUserAsyncApcRundown(KAPC* apc) { + if (!apc) return; + SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc); + SyscallUserAsyncApcContextFree(ctx); +} + +static void SyscallUserAsyncNormalRoutine(void* normalContext, + void* systemArgument1, + void* systemArgument2) { + SYS_USER_ASYNC_APC_CONTEXT* ctx = (SYS_USER_ASYNC_APC_CONTEXT*)normalContext; + SYS_IO_STATUS_BLOCK_K* iosb = (SYS_IO_STATUS_BLOCK_K*)systemArgument1; + uint64_t information = iosb ? iosb->Information : 0; + (void)systemArgument2; + + if (ctx && ctx->UserRoutine) { + KSTATUS s = KeQueueUserApc(ctx->TargetThread, + (void*)ctx->UserRoutine, + ctx->UserContext, + iosb, + (void*)(uintptr_t)information); + if (!KSUCCESS(s)) { + DbgPrint("USER/APC async: KeQueueUserApc failed status=%u\n", s); + } + } + + SyscallUserAsyncApcContextFree(ctx); +} + +static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) { + if (!a0) return STATUS_INVALID_PARAMETER; + + const SYS_ASYNC_FILE_REQUEST_K* req = + (const SYS_ASYNC_FILE_REQUEST_K*)(uintptr_t)a0; + + if (req->Flags || req->Reserved) return STATUS_INVALID_PARAMETER; + if (!req->Iosb) return STATUS_INVALID_PARAMETER; + if (!req->Buffer && req->Size) return STATUS_INVALID_PARAMETER; + + FILE_OBJECT* file = + (FILE_OBJECT*)KiHandleToObject(req->FileHandle, ObjectTypeFile); + if (!file) return STATUS_NOT_FOUND; + + EVENT* event = (EVENT*)0; + if (req->EventHandle) { + event = (EVENT*)KiHandleToObject(req->EventHandle, ObjectTypeEvent); + if (!event) { + ObDereferenceObject(&file->Header); + return STATUS_NOT_FOUND; + } + } + + KAPC* completionApc = (KAPC*)0; + SYS_USER_ASYNC_APC_CONTEXT* apcCtx = (SYS_USER_ASYNC_APC_CONTEXT*)0; + if (req->ApcRoutine) { + THREAD* thread = KeGetCurrentThread(); + if (!thread || !thread->UserApcDispatcher) { + if (event) ObDereferenceObject(&event->Header); + ObDereferenceObject(&file->Header); + return STATUS_INVALID_PARAMETER; + } + + apcCtx = (SYS_USER_ASYNC_APC_CONTEXT*)MmAllocatePool(sizeof(SYS_USER_ASYNC_APC_CONTEXT)); + if (!apcCtx) { + if (event) ObDereferenceObject(&event->Header); + ObDereferenceObject(&file->Header); + return STATUS_NO_MEMORY; + } + + memset(apcCtx, 0, sizeof(*apcCtx)); + ObReferenceObject(&thread->Header); + apcCtx->TargetThread = thread; + apcCtx->UserRoutine = req->ApcRoutine; + apcCtx->UserContext = req->ApcContext; + KeInitializeApc(&apcCtx->KernelApc, + thread, + SyscallUserAsyncKernelRoutine, + SyscallUserAsyncNormalRoutine, + apcCtx); + KeSetApcRundownRoutine(&apcCtx->KernelApc, SyscallUserAsyncApcRundown); + completionApc = &apcCtx->KernelApc; + } + + IO_ASYNC_FILE_REQUEST_OBJECT* asyncReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0; + KSTATUS s; + if (write) { + s = IoWriteFileAsyncKernel(file, req->Buffer, req->Size, req->Offset, + (IO_STATUS_BLOCK*)req->Iosb, event, + completionApc, + &asyncReq); + } else { + s = IoReadFileAsyncKernel(file, req->Buffer, req->Size, req->Offset, + (IO_STATUS_BLOCK*)req->Iosb, event, + completionApc, + &asyncReq); + } + + if (s == STATUS_PENDING && req->RequestHandleOut && asyncReq) { + uint32_t requestHandle = 0; + KSTATUS hs = KiAllocHandleForCurrent(IoAsyncFileRequestObject(asyncReq), &requestHandle); + if (!KSUCCESS(hs)) { + (void)IoCancelAsyncFileRequest(asyncReq); + ObDereferenceObject(IoAsyncFileRequestObject(asyncReq)); + if (event) ObDereferenceObject(&event->Header); + ObDereferenceObject(&file->Header); + return hs; + } + *req->RequestHandleOut = requestHandle; + } else if (req->RequestHandleOut) { + *req->RequestHandleOut = 0; + } + + if (asyncReq) + ObDereferenceObject(IoAsyncFileRequestObject(asyncReq)); // drop creator reference + + if (!KSUCCESS(s) && s != STATUS_PENDING && apcCtx) + SyscallUserAsyncApcContextFree(apcCtx); + + if (event) ObDereferenceObject(&event->Header); + ObDereferenceObject(&file->Header); + return s; +} + +// --------------------------------------------------------------------------- +// SYS_READ_FILE_ASYNC (43) a0=*SYS_ASYNC_FILE_REQUEST +// --------------------------------------------------------------------------- +static uint64_t SyscallReadFileAsync(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return SyscallAsyncFileCommon(a0, false); +} + +// --------------------------------------------------------------------------- +// SYS_WRITE_FILE_ASYNC (44) a0=*SYS_ASYNC_FILE_REQUEST +// --------------------------------------------------------------------------- +static uint64_t SyscallWriteFileAsync(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return SyscallAsyncFileCommon(a0, true); +} + +// --------------------------------------------------------------------------- +// SYS_CANCEL_IO a0 = async request handle +// --------------------------------------------------------------------------- +static uint64_t SyscallCancelIo(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + IO_ASYNC_FILE_REQUEST_OBJECT* req = + (IO_ASYNC_FILE_REQUEST_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeAsyncIo); + if (!req) return STATUS_INVALID_HANDLE; + + KSTATUS s = IoCancelAsyncFileRequest(req); + ObDereferenceObject(IoAsyncFileRequestObject(req)); + return s; +} + // --------------------------------------------------------------------------- // SYS_CREATE_FILE (30) a0=*path a1=*handleOut // --------------------------------------------------------------------------- @@ -511,7 +750,7 @@ static uint64_t SyscallCreateFile(uint64_t a0, uint64_t a1, uint64_t a2, KSTATUS s = IoCreateFile((const char*)(uintptr_t)a0, &file); if (!KSUCCESS(s)) return s; uint32_t h = 0; - s = AllocHandleForCurrent(&file->Header, &h); + s = KiAllocHandleForCurrent(&file->Header, &h); if (!KSUCCESS(s)) { ObDereferenceObject(&file->Header); return s; @@ -536,7 +775,7 @@ static uint64_t SyscallDeleteFile(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallTruncateFile(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a2; (void)a3; (void)a4; (void)a5; - FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile); + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile); if (!file) return STATUS_NOT_FOUND; KSTATUS s = IoTruncateFile(file, (uint64_t)a1); ObDereferenceObject(&file->Header); @@ -549,9 +788,11 @@ static uint64_t SyscallTruncateFile(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallGetFilePos(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile); + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile); if (!file) return (uint64_t)-1; + uintptr_t flags = KeAcquireSpinlock(&file->Lock); uint64_t pos = file->Offset; + KeReleaseSpinlock(&file->Lock, flags); ObDereferenceObject(&file->Header); return pos; } @@ -562,7 +803,7 @@ static uint64_t SyscallGetFilePos(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallSeekFile(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a2; (void)a3; (void)a4; (void)a5; - FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile); + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile); if (!file) return STATUS_NOT_FOUND; KSTATUS s = IoSeekFile(file, (uint64_t)a1); ObDereferenceObject(&file->Header); @@ -575,7 +816,7 @@ static uint64_t SyscallSeekFile(uint64_t a0, uint64_t a1, uint64_t a2, static uint64_t SyscallGetFileSize(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile); + FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile); if (!file) return 0; uint64_t sz = IoGetFileSize(file); ObDereferenceObject(&file->Header); @@ -738,6 +979,15 @@ static uint64_t SyscallDebugPrint(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_SUCCESS; } +// --------------------------------------------------------------------------- +// SYS_QUERY_TRAP_FRAME a0=*SYS_TRAP_FRAME_INFO +// --------------------------------------------------------------------------- +static uint64_t SyscallQueryTrapFrame(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KeQueryCurrentTrapFrame((SYS_TRAP_FRAME_INFO_K*)(uintptr_t)a0); +} + // --------------------------------------------------------------------------- // SYS_CREATE_PROCESS (50) a0=*path a1=*handleOut a2=*cmdline (optional) // --------------------------------------------------------------------------- @@ -775,6 +1025,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, uintptr_t flags = KeAcquireSpinlock(&parent->CwdLock); strncpy(proc->CurrentDirectory, parent->CurrentDirectory, sizeof(proc->CurrentDirectory) - 1); proc->CurrentDirectory[sizeof(proc->CurrentDirectory) - 1] = '\0'; + proc->CurrentDrive = parent->CurrentDrive; KeReleaseSpinlock(&parent->CwdLock, flags); } @@ -789,7 +1040,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, s = LdrResolveImportsForProcess(proc, imageBase); if (!KSUCCESS(s)) { - MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase)); + MmFreeUserMemory((void*)imageBase); ObDereferenceObject(&proc->Header); return s; } @@ -802,7 +1053,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, // to finish wiring it up. PROCESS* caller = KeGetCurrentProcess(); if (!caller) { - MmFreeUserMemory((void*)imageBase, proc->ImageSize); + MmFreeUserMemory((void*)imageBase); ObDereferenceObject(&proc->Header); return STATUS_FAILURE; } @@ -813,7 +1064,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, KeCloseHandle(caller, handle); proc->ImageBase = 0; proc->ImageSize = 0; - MmFreeUserMemory((void*)imageBase, proc->ImageSize); + MmFreeUserMemory((void*)imageBase); ObDereferenceObject(&proc->Header); return s; } @@ -825,7 +1076,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, uintptr_t entryPoint = LdrGetEntryPoint(imageBase); s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread); if (!KSUCCESS(s)) { - MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase)); + MmFreeUserMemory((void*)imageBase); ObDereferenceObject(&proc->Header); return s; } @@ -841,10 +1092,10 @@ static uint64_t SyscallWaitForProcess(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - PROCESS* proc = (PROCESS*)HandleToObject((uint32_t)a0, ObjectTypeProcess); + PROCESS* proc = (PROCESS*)KiHandleToObject((uint32_t)a0, ObjectTypeProcess); if (!proc) return STATUS_INVALID_HANDLE; - KSTATUS s = KeWaitForEvent(proc->ExitEvent); + KSTATUS s = KeWaitForEvent(proc->ExitEvent, 0); ObDereferenceObject(&proc->Header); return s; } @@ -856,7 +1107,7 @@ static uint64_t SyscallGetProcessExitCode(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a2; (void)a3; (void)a4; (void)a5; - PROCESS* proc = (PROCESS*)HandleToObject((uint32_t)a0, ObjectTypeProcess); + PROCESS* proc = (PROCESS*)KiHandleToObject((uint32_t)a0, ObjectTypeProcess); if (!proc) return STATUS_INVALID_HANDLE; int* exitCodeOut = (int*)(uintptr_t)a1; @@ -915,12 +1166,12 @@ static uint64_t SyscallReadDir(uint64_t a0, uint64_t a1, uint64_t a2, // a1 = KDIR_ENTRY* buffer // Look up a DIR_OBJECT using ObjectTypeDir (not ObjectTypeFile) - DIR_OBJECT* dir = (DIR_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeDir); + DIR_OBJECT* dir = (DIR_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeDir); if (!dir) return STATUS_INVALID_PARAMETER; KSTATUS s = IoReadDir(dir, (void*)(uintptr_t)a1); - ObDereferenceObject(&dir->Header); // Drop the ref taken by HandleToObject + ObDereferenceObject(&dir->Header); // Drop the ref taken by KiHandleToObject return s; } @@ -986,7 +1237,7 @@ static uint64_t SyscallWaitThread(uint64_t a0, uint64_t a1, uint64_t a2, return STATUS_NOT_FOUND; } - KSTATUS s = KeWaitForEvent(exitEvent); + KSTATUS s = KeWaitForEvent(exitEvent, 0); ObDereferenceObject(&exitEvent->Header); return s; } @@ -1000,7 +1251,7 @@ static uint64_t SyscallGetProcessPid(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a2; (void)a3; (void)a4; (void)a5; - PROCESS* proc = (PROCESS*)HandleToObject((uint32_t)a0, ObjectTypeProcess); + PROCESS* proc = (PROCESS*)KiHandleToObject((uint32_t)a0, ObjectTypeProcess); uint32_t* pidOut = (uint32_t*)(uintptr_t)a1; if (!proc) return STATUS_INVALID_HANDLE; if (!pidOut) { ObDereferenceObject(&proc->Header); return STATUS_INVALID_PARAMETER; } @@ -1048,9 +1299,9 @@ static uint64_t SyscallPublishQueue(uint64_t a0, uint64_t a1, uint64_t a2, const char* name = (const char*)(uintptr_t)a0; if (!name || !name[0]) return STATUS_INVALID_PARAMETER; - // Resolve the handle to a QUEUE object. HandleToObject bumps the + // Resolve the handle to a QUEUE object. KiHandleToObject bumps the // refcount by one; we keep that ref in the registry slot. - QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a1, ObjectTypeQueue); + QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a1, ObjectTypeQueue); if (!q) return STATUS_INVALID_HANDLE; uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock); @@ -1113,9 +1364,9 @@ static uint64_t SyscallOpenNamedQueue(uint64_t a0, uint64_t a1, uint64_t a2, if (!q) return STATUS_NOT_FOUND; - // AllocHandleForCurrent consumes the ref we just bumped. + // KiAllocHandleForCurrent consumes the ref we just bumped. // On failure it calls ObDereferenceObject internally. - return AllocHandleForCurrent(&q->Header, handleOut); + return KiAllocHandleForCurrent(&q->Header, handleOut); } // --------------------------------------------------------------------------- @@ -1131,7 +1382,7 @@ static uint64_t SyscallQueueDuplicate(uint64_t a0, uint64_t a1, uint64_t a2, (void)a3; (void)a4; (void)a5; if (!a2) return STATUS_INVALID_PARAMETER; - QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a0, ObjectTypeQueue); + QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a0, ObjectTypeQueue); if (!q) return STATUS_NOT_FOUND; PROCESS* target = KeGetProcessById((uint32_t)a1); @@ -1250,82 +1501,89 @@ typedef uint64_t (*SYSCALL_HANDLER)(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { - /* 00 SYS_EXIT_PROCESS */ SyscallExitProcess, - /* 01 SYS_EXIT_THREAD */ SyscallExitThread, - /* 02 SYS_CREATE_THREAD */ SyscallCreateThread, - /* 03 SYS_GET_CURRENT_TID */ SyscallGetCurrentTid, - /* 04 SYS_GET_CURRENT_PID */ SyscallGetCurrentPid, - /* 05 SYS_YIELD */ SyscallYield, - /* 06 SYS_SLEEP */ SyscallSleep, - /* 07 SYS_SET_THREAD_PRIORITY */ SyscallSetThreadPriority, - /* 08 SYS_ALLOC */ SyscallAlloc, - /* 09 SYS_FREE */ SyscallFree, - /* 10 SYS_REALLOC */ SyscallRealloc, - /* 11 SYS_MUTEX_CREATE */ SyscallMutexCreate, - /* 12 SYS_MUTEX_ACQUIRE */ SyscallMutexAcquire, - /* 13 SYS_MUTEX_TRY_ACQUIRE */ SyscallMutexTryAcquire, - /* 14 SYS_MUTEX_RELEASE */ SyscallMutexRelease, - /* 15 SYS_SEM_CREATE */ SyscallSemCreate, - /* 16 SYS_SEM_WAIT */ SyscallSemWait, - /* 17 SYS_SEM_TRY_WAIT */ SyscallSemTryWait, - /* 18 SYS_SEM_SIGNAL */ SyscallSemSignal, - /* 19 SYS_EVENT_CREATE */ SyscallEventCreate, - /* 20 SYS_EVENT_SIGNAL */ SyscallEventSignal, - /* 21 SYS_EVENT_CLEAR */ SyscallEventClear, - /* 22 SYS_EVENT_WAIT */ SyscallEventWait, - /* 23 SYS_QUEUE_CREATE */ SyscallQueueCreate, - /* 24 SYS_QUEUE_SEND */ SyscallQueueSend, - /* 25 SYS_QUEUE_RECV */ SyscallQueueRecv, - /* 26 SYS_CLOSE_HANDLE */ SyscallCloseHandle, - /* 27 SYS_OPEN_FILE */ SyscallOpenFile, - /* 28 SYS_READ_FILE */ SyscallReadFile, - /* 29 SYS_WRITE_FILE */ SyscallWriteFile, - /* 30 SYS_CREATE_FILE */ SyscallCreateFile, - /* 31 SYS_DELETE_FILE */ SyscallDeleteFile, - /* 32 SYS_TRUNCATE_FILE */ SyscallTruncateFile, - /* 33 SYS_GET_FILE_POS */ SyscallGetFilePos, - /* 34 SYS_SEEK_FILE */ SyscallSeekFile, - /* 35 SYS_GET_FILE_SIZE */ SyscallGetFileSize, - /* 36 SYS_SCREEN_BLIT */ SyscallScreenBlit, - /* 37 SYS_SCREEN_CLEAR */ SyscallScreenClear, - /* 38 SYS_SCREEN_GET_SIZE */ SyscallScreenGetSize, - /* 39 SYS_SCREEN_PUT_CHAR */ SyscallScreenPutChar, - /* 40 SYS_KB_POLL */ SyscallKbPoll, - /* 41 SYS_KB_IS_KEY_DOWN */ SyscallKbIsKeyDown, - /* 42 SYS_KB_READ_CHAR */ SyscallKbReadChar, - /* 43 SYS_KB_READ_SCANCODE */ SyscallKbReadScancode, - /* 44 SYS_KB_GET_MODIFIERS */ SyscallKbGetModifiers, - /* 45 SYS_MOUSE_POLL */ SyscallMousePoll, - /* 46 SYS_MOUSE_READ */ SyscallMouseRead, - /* 47 SYS_MOUSE_GET_POS */ SyscallMouseGetPos, - /* 48 SYS_GET_TICK */ SyscallGetTick, - /* 49 SYS_DEBUG_PRINT */ SyscallDebugPrint, - /* 50 SYS_CREATE_PROCESS */ SyscallCreateProcess, - /* 51 SYS_WAIT_FOR_PROCESS */ SyscallWaitForProcess, - /* 52 SYS_GET_PROCESS_EXIT_CODE */ SyscallGetProcessExitCode, - /* 53 SYS_WRITE_CONSOLE */ SyscallWriteConsole, - /* 54 SYS_OPEN_DIR */ SyscallOpenDir, - /* 55 SYS_READ_DIR */ SyscallReadDir, - /* 56 SYS_SET_CWD */ SyscallSetCwd, - /* 57 SYS_GET_CWD */ SyscallGetCwd, - /* 58 SYS_WAIT_THREAD */ SyscallWaitThread, - /* 59 SYS_GET_PROCESS_PID */ SyscallGetProcessPid, - /* 60 SYS_PUBLISH_QUEUE */ SyscallPublishQueue, - /* 61 SYS_OPEN_NAMED_QUEUE */ SyscallOpenNamedQueue, - /* 62 SYS_QUEUE_DUPLICATE */ SyscallQueueDuplicate, - /* 63 SYS_NAMED_QUEUE_OWNER_PID */ SyscallNamedQueueOwnerPid, - /* 64 SYS_KILL_PROCESS */ SyscallKillProcess, - /* 65 SYS_GET_PROCESS_LIST */ SyscallGetProcessList, - /* 66 SYS_RENAME_FILE */ SyscallRenameFile, - /* 67 SYS_CREATE_DIR */ SyscallCreateDir, - /* 68 SYS_REMOVE_DIR */ SyscallRemoveDir, + /* 00 SYS_EXIT_PROCESS */ SyscallExitProcess, + /* 01 SYS_EXIT_THREAD */ SyscallExitThread, + /* 02 SYS_CREATE_PROCESS */ SyscallCreateProcess, + /* 03 SYS_WAIT_FOR_PROCESS */ SyscallWaitForProcess, + /* 04 SYS_GET_PROCESS_EXIT_CODE */ SyscallGetProcessExitCode, + /* 05 SYS_KILL_PROCESS */ SyscallKillProcess, + /* 06 SYS_GET_PROCESS_LIST */ SyscallGetProcessList, + /* 07 SYS_GET_PROCESS_PID */ SyscallGetProcessPid, + /* 08 SYS_CREATE_THREAD */ SyscallCreateThread, + /* 09 SYS_WAIT_THREAD */ SyscallWaitThread, + /* 10 SYS_GET_CURRENT_TID */ SyscallGetCurrentTid, + /* 11 SYS_GET_CURRENT_PID */ SyscallGetCurrentPid, + /* 12 SYS_SET_THREAD_PRIORITY */ SyscallSetThreadPriority, + /* 13 SYS_YIELD */ SyscallYield, + /* 14 SYS_SLEEP */ SyscallSleep, + /* 15 SYS_SET_USER_APC_DISPATCHER */ SyscallSetUserApcDispatcher, + /* 16 SYS_APC_RETURN */ SyscallApcReturn, + /* 17 SYS_ALLOC */ SyscallAlloc, + /* 18 SYS_FREE */ SyscallFree, + /* 19 SYS_REALLOC */ SyscallRealloc, + /* 20 SYS_CLOSE_HANDLE */ SyscallCloseHandle, + /* 21 SYS_MUTEX_CREATE */ SyscallMutexCreate, + /* 22 SYS_MUTEX_ACQUIRE */ SyscallMutexAcquire, + /* 23 SYS_MUTEX_TRY_ACQUIRE */ SyscallMutexTryAcquire, + /* 24 SYS_MUTEX_RELEASE */ SyscallMutexRelease, + /* 25 SYS_SEM_CREATE */ SyscallSemCreate, + /* 26 SYS_SEM_WAIT */ SyscallSemWait, + /* 27 SYS_SEM_TRY_WAIT */ SyscallSemTryWait, + /* 28 SYS_SEM_SIGNAL */ SyscallSemSignal, + /* 29 SYS_EVENT_CREATE */ SyscallEventCreate, + /* 30 SYS_EVENT_SIGNAL */ SyscallEventSignal, + /* 31 SYS_EVENT_CLEAR */ SyscallEventClear, + /* 32 SYS_EVENT_WAIT */ SyscallEventWait, + /* 34 SYS_QUEUE_CREATE */ SyscallQueueCreate, + /* 35 SYS_QUEUE_SEND */ SyscallQueueSend, + /* 36 SYS_QUEUE_RECV */ SyscallQueueRecv, + /* 37 SYS_PUBLISH_QUEUE */ SyscallPublishQueue, + /* 38 SYS_OPEN_NAMED_QUEUE */ SyscallOpenNamedQueue, + /* 39 SYS_QUEUE_DUPLICATE */ SyscallQueueDuplicate, + /* 40 SYS_NAMED_QUEUE_OWNER_PID */ SyscallNamedQueueOwnerPid, + /* 41 SYS_OPEN_FILE */ SyscallOpenFile, + /* 42 SYS_CREATE_FILE */ SyscallCreateFile, + /* 43 SYS_READ_FILE */ SyscallReadFile, + /* 44 SYS_WRITE_FILE */ SyscallWriteFile, + /* 45 SYS_READ_FILE_ASYNC */ SyscallReadFileAsync, + /* 46 SYS_WRITE_FILE_ASYNC */ SyscallWriteFileAsync, + /* 47 SYS_CANCEL_IO */ SyscallCancelIo, + /* 48 SYS_SEEK_FILE */ SyscallSeekFile, + /* 49 SYS_GET_FILE_POS */ SyscallGetFilePos, + /* 50 SYS_GET_FILE_SIZE */ SyscallGetFileSize, + /* 51 SYS_TRUNCATE_FILE */ SyscallTruncateFile, + /* 52 SYS_DELETE_FILE */ SyscallDeleteFile, + /* 53 SYS_RENAME_FILE */ SyscallRenameFile, + /* 54 SYS_OPEN_DIR */ SyscallOpenDir, + /* 55 SYS_READ_DIR */ SyscallReadDir, + /* 56 SYS_CREATE_DIR */ SyscallCreateDir, + /* 57 SYS_REMOVE_DIR */ SyscallRemoveDir, + /* 58 SYS_SET_CWD */ SyscallSetCwd, + /* 59 SYS_GET_CWD */ SyscallGetCwd, + /* 60 SYS_WRITE_CONSOLE */ SyscallWriteConsole, + /* 61 SYS_SCREEN_BLIT */ SyscallScreenBlit, + /* 62 SYS_SCREEN_CLEAR */ SyscallScreenClear, + /* 63 SYS_SCREEN_GET_SIZE */ SyscallScreenGetSize, + /* 64 SYS_SCREEN_PUT_CHAR */ SyscallScreenPutChar, + /* 65 SYS_KB_POLL */ SyscallKbPoll, + /* 66 SYS_KB_IS_KEY_DOWN */ SyscallKbIsKeyDown, + /* 67 SYS_KB_READ_CHAR */ SyscallKbReadChar, + /* 68 SYS_KB_READ_SCANCODE */ SyscallKbReadScancode, + /* 69 SYS_KB_GET_MODIFIERS */ SyscallKbGetModifiers, + /* 70 SYS_MOUSE_POLL */ SyscallMousePoll, + /* 71 SYS_MOUSE_READ */ SyscallMouseRead, + /* 72 SYS_MOUSE_GET_POS */ SyscallMouseGetPos, + /* 73 SYS_GET_TICK */ SyscallGetTick, + /* 74 SYS_DEBUG_PRINT */ SyscallDebugPrint, + /* 75 SYS_QUERY_TRAP_FRAME */ SyscallQueryTrapFrame, }; // --------------------------------------------------------------------------- // Entry point - called from ke_syscall.asm after argument re-ordering // --------------------------------------------------------------------------- -KSTATUS KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1, +uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { + KiValidateSyscallTrapFrame(nr, a0, a1, a2, a3, a4, a5); if (nr >= SYS_MAX || !gSyscallTable[nr]) return STATUS_NOT_SUPPORTED; - return (KSTATUS)gSyscallTable[nr](a0, a1, a2, a3, a4, a5); + return gSyscallTable[nr](a0, a1, a2, a3, a4, a5); } diff --git a/code/kernel/ke_thread.c b/code/kernel/ke_thread.c index 1fe2feb..9340018 100644 --- a/code/kernel/ke_thread.c +++ b/code/kernel/ke_thread.c @@ -1,9 +1,12 @@ // ke_thread.c - thread model #include +#include #include #include #include +#include #include +#include #include #include #include @@ -11,12 +14,26 @@ #define FPU_FCW_DEFAULT 0x037Fu #define FPU_MXCSR_DEFAULT 0x1F80u +// These offsets are consumed directly by ke_switch.asm. Keep the asserts +// here so future THREAD layout edits fail at build time instead of corrupting +// context switches at runtime. +#define STATIC_ASSERT_OFFSET(type, field, expected) \ + _Static_assert(offsetof(type, field) == (expected), \ + #type "." #field " offset must match assembly") + +STATIC_ASSERT_OFFSET(THREAD, SavedContext, 88); +STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 104); +STATIC_ASSERT_OFFSET(THREAD, Teb, 112); +STATIC_ASSERT_OFFSET(THREAD, FpuContext, 168); + static uint32_t gNextThreadId = 1; extern void KeThreadEntry(void); -static void ThreadDestroy(OBJECT* obj) { +static void KiThreadDestroy(OBJECT* obj) { THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header); + KiRundownApcState(thread); + if (thread->ExitEvent) { ObDereferenceObject(&thread->ExitEvent->Header); thread->ExitEvent = (EVENT*)0; @@ -37,13 +54,13 @@ static void ThreadDestroy(OBJECT* obj) { } if (thread->Teb && thread->TebSize) { - MmFreeUserMemory(thread->Teb, thread->TebSize); + MmFreeUserMemory(thread->Teb); thread->Teb = (void*)0; thread->TebSize = 0; } if (thread->UserStackBase) { - MmFreeUserMemory(thread->UserStackBase, USTACK_SIZE); + MmFreeUserMemory(thread->UserStackBase); thread->UserStackBase = (void *)0; } } @@ -56,13 +73,18 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); if (!thread) return STATUS_NO_MEMORY; - thread->Header.Destroy = ThreadDestroy; + thread->Header.Destroy = KiThreadDestroy; thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1); thread->State = ThreadReady; thread->Priority = priority; thread->QuantaConsumed = 0; thread->WaitObject = (OBJECT *)0; + thread->WaitStatus = STATUS_SUCCESS; + thread->WaitListHead = (LIST_ENTRY*)0; + thread->AlertableWait = 0; thread->SchedLockFlags = 0x202ULL; + KiInitializeApcState(thread); + thread->TrapFrame = (KTRAP_FRAME*)0; void* kstack = MmAllocatePool(KSTACK_SIZE); if (!kstack) { @@ -88,7 +110,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemory(sizeof(SYS_TEB)); if (!teb) { - MmFreeUserMemory(ustack, USTACK_SIZE); + MmFreeUserMemory(ustack); thread->UserStackBase = (void*)0; MmFreePool(kstack); thread->KernelStackBase = (void*)0; @@ -132,10 +154,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui EVENT* exitEvent; KSTATUS s = KeEventCreate(EventManualReset, 0, &exitEvent); if (!KSUCCESS(s)) { - if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb, thread->TebSize); + if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb); thread->Teb = (void*)0; thread->TebSize = 0; - MmFreeUserMemory(ustack, USTACK_SIZE); + MmFreeUserMemory(ustack); thread->UserStackBase = (void*)0; MmFreePool(kstack); thread->KernelStackBase = (void*)0; @@ -161,6 +183,8 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) { (void)code; if (!thread) return; + KiRundownApcState(thread); + // Wake any pthread_join waiters before the thread is marked dead and // potentially freed. Manual-reset means all concurrent waiters unblock. if (thread->ExitEvent) @@ -197,13 +221,18 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); if (!thread) return STATUS_NO_MEMORY; - thread->Header.Destroy = ThreadDestroy; + thread->Header.Destroy = KiThreadDestroy; thread->ThreadId = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1); thread->State = ThreadReady; thread->Priority = priority; thread->QuantaConsumed = 0; thread->WaitObject = (OBJECT*)0; + thread->WaitStatus = STATUS_SUCCESS; + thread->WaitListHead = (LIST_ENTRY*)0; + thread->AlertableWait = 0; thread->SchedLockFlags = 0x202ULL; + KiInitializeApcState(thread); + thread->TrapFrame = (KTRAP_FRAME*)0; thread->UserStackBase = (void*)0; // no user stack thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx thread->TebSize = 0; @@ -245,16 +274,101 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, return STATUS_SUCCESS; } -void KeBlockThread(THREAD* thread, OBJECT* waitObject) { - thread->State = ThreadWaiting; - thread->WaitObject = waitObject; +void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) { + // Called with the synchronization object's spinlock held, so the caller is + // normally at DISPATCH_LEVEL here. This function only marks intent to + // block; the actual scheduling point is KeCommitBlockThread at PASSIVE. + KeAssertIrqlAtOrBelow(DISPATCH_LEVEL); + if (!thread) return; - KeYield(); + thread->State = ThreadWaiting; + thread->WaitObject = waitObject; + thread->WaitStatus = STATUS_PENDING; + thread->WaitListHead = (LIST_ENTRY*)0; + thread->AlertableWait = alertable ? 1 : 0; } -void KeUnblockThread(THREAD* thread) { - thread->WaitObject = (OBJECT *)0; - thread->State = ThreadReady; + +void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) { + KeAssertIrqlAtOrBelow(DISPATCH_LEVEL); + if (!thread) return; + thread->WaitListHead = waitListHead; +} + +static void KiFinishWait(THREAD* thread, KSTATUS status) { + thread->WaitObject = (OBJECT*)0; + thread->WaitStatus = status; + thread->WaitListHead = (LIST_ENTRY*)0; + thread->AlertableWait = 0; +} + +KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { + KeAssertIrql(PASSIVE_LEVEL); + if (!thread) return STATUS_INVALID_PARAMETER; + + if (alertable) + thread->AlertableWait = 1; + + // A signal/APC may have arrived after the waiter was placed on the + // object's wait list but before this thread reached the scheduling point. + // In that case KeUnblockThreadWithStatus completed the wait in-place and + // there is nothing left to block on. + if (thread->State != ThreadWaiting) { + KSTATUS status = thread->WaitStatus; + thread->AlertableWait = 0; + return status; + } + + // If a user APC was already pending before the thread fully blocked, + // consume the alertable wait immediately instead of going to sleep. + if (alertable && thread->UserApcPending && thread->UserApcDispatcher) { + if (thread->WaitListHead) + ListRemove(&thread->SchedulerLink); + KiFinishWait(thread, STATUS_USER_APC); + thread->State = ThreadRunning; + return STATUS_USER_APC; + } + + KeYield(); + + // Kernel APCs are delivered before an alertable wait returns + // STATUS_KERNEL_APC to its caller. + KiDeliverKernelApcs(); + + thread->AlertableWait = 0; + return thread->WaitStatus; +} + + +KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) { + KePrepareBlockThread(thread, waitObject, alertable); + return KeCommitBlockThread(thread, alertable); +} + + +void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) { + if (!thread) return; + + // If the selected waiter is still the currently executing thread, the + // signal landed in the prepare->commit window. Complete the wait in-place + // instead of inserting the running thread into the ready queue. + if (thread == KeGetCurrentThread()) { + KiFinishWait(thread, status); + if (thread->State == ThreadWaiting) + thread->State = ThreadRunning; + return; + } + + // The same object can be signaled after a timeout/cancel path exists, or a + // buggy caller can try to wake a non-waiting thread. Do not enqueue such a + // thread twice. + if (thread->State != ThreadWaiting) { + thread->WaitStatus = status; + return; + } + + KiFinishWait(thread, status); + thread->State = ThreadReady; if (thread->Priority < PriorityHigh) { thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1); @@ -264,6 +378,35 @@ void KeUnblockThread(THREAD* thread) { KeReadyThread(thread); } +void KeUnblockThread(THREAD* thread) { + KeUnblockThreadWithStatus(thread, STATUS_SUCCESS); +} + +static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) { + if (!thread) return STATUS_INVALID_PARAMETER; + + if (thread->State != ThreadWaiting || !thread->AlertableWait) + return STATUS_PENDING; + + if (thread->WaitListHead) + ListRemove(&thread->SchedulerLink); + + KeUnblockThreadWithStatus(thread, status); + return STATUS_SUCCESS; +} + +KSTATUS KeAlertThread(THREAD* thread) { + return KiAlertThreadWithStatus(thread, STATUS_ALERTED); +} + +void KiWakeThreadForKernelApc(THREAD* thread) { + (void)KiAlertThreadWithStatus(thread, STATUS_KERNEL_APC); +} + +void KiWakeThreadForUserApc(THREAD* thread) { + (void)KiAlertThreadWithStatus(thread, STATUS_USER_APC); +} + void KeInitThreadFpuContext(THREAD* thread) { uintptr_t raw = diff --git a/code/kernel/ke_thread.h b/code/kernel/ke_thread.h index 73463cd..eea084b 100644 --- a/code/kernel/ke_thread.h +++ b/code/kernel/ke_thread.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #define KSTACK_SIZE (64 * 1024) /* 64 KB kernel stack */ #define USTACK_SIZE (512 * 1024) /* 512 KB user stack */ @@ -77,6 +79,36 @@ typedef struct THREAD { void* FpuContext; uint8_t FpuStorage[528]; + + // Set by wait primitives before blocking and completed by the object + // that wakes the thread. Keep this field after the assembly-visible + // context-switch fields: ke_switch.asm hard-codes offsets through + // FpuContext. + KSTATUS WaitStatus; + LIST_ENTRY* WaitListHead; + uint8_t AlertableWait; + + // Kernel APC state. APCs are queued to a target thread and delivered + // when that thread reaches PASSIVE_LEVEL with APC delivery enabled. + LIST_ENTRY ApcQueue; + SPINLOCK ApcLock; + volatile long KernelApcPending; + int32_t KernelApcDisable; + uint8_t ApcDeliveryActive; + + // Non-NULL while executing on a syscall/interrupt/fault entry frame. + // The pointed-to frame lives on this thread's kernel stack. + KTRAP_FRAME* TrapFrame; + + // User APC state. User APCs are queued by kernel completions, but are + // delivered only on an alertable return to user mode through the registered + // user-mode dispatcher stub. + LIST_ENTRY UserApcQueue; + SPINLOCK UserApcLock; + volatile long UserApcPending; + uintptr_t UserApcDispatcher; + uint8_t UserApcDeliverable; + uint8_t UserApcDeliveryActive; } THREAD; __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg, @@ -93,8 +125,26 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, __declspec(dllexport) void KeExitThread(THREAD* thread, int code); // Block / unblock primitives used by sync objects. -void KeBlockThread(THREAD* thread, OBJECT* waitObject); +// +// Correct wait pattern: +// object lock held +// KePrepareBlockThread(self, object, alertable); +// insert self into object's wait list +// release object lock +// KeCommitBlockThread(self, alertable); +// +// The split form prevents the classic lost wakeup where an ISR/DPC signals +// the object after the waiter is queued but before the waiter marks itself +// waiting. Alertable waits are explicit at every call site. +void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable); +void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead); +KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable); +KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable); +void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status); void KeUnblockThread(THREAD* thread); +KSTATUS KeAlertThread(THREAD* thread); +void KiWakeThreadForKernelApc(THREAD* thread); +void KiWakeThreadForUserApc(THREAD* thread); // Assembly stub - implemented in ke_switch.asm. void KeSwitchContext(THREAD* from, THREAD* to); diff --git a/code/kernel/ke_trap.c b/code/kernel/ke_trap.c new file mode 100644 index 0000000..2b300f1 --- /dev/null +++ b/code/kernel/ke_trap.c @@ -0,0 +1,148 @@ +// ke_trap.c - current-thread trap-frame tracking and validation +#include +#include +#include +#include +#include +#include +#include + +#define STATIC_ASSERT_OFFSET(type, field, expected) \ + _Static_assert(offsetof(type, field) == (expected), \ + #type "." #field " offset must match assembly") + +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Magic, 0); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Kind, 8); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Vector, 12); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousMode, 16); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rax, 24); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rcx, 32); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rdx, 40); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rbx, 48); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rbp, 56); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rsi, 64); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rdi, 72); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R8, 80); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R9, 88); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R10, 96); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R11, 104); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R12, 112); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R13, 120); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R14, 128); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, R15, 136); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rip, 144); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Cs, 152); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rflags, 160); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rsp, 168); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Ss, 176); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, RipSlot, 184); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, CsSlot, 192); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, RflagsSlot, 200); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, RspSlot, 208); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, SsSlot, 216); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousFrame, 224); +STATIC_ASSERT_OFFSET(KTRAP_FRAME, Reserved2, 232); +_Static_assert(sizeof(KTRAP_FRAME) == 240, "KTRAP_FRAME size must match ke_syscall.asm"); + +int KeTrapFrameFromUser(const KTRAP_FRAME* frame) { + if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) return 0; + return (frame->Cs & 3u) == 3u; +} + +KTRAP_FRAME* KeGetCurrentTrapFrame(void) { + THREAD* t = KeGetCurrentThread(); + if (!t) return (KTRAP_FRAME*)0; + return t->TrapFrame; +} + +void KiEnterTrapFrame(KTRAP_FRAME* frame) { + if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) + KePanic("KiEnterTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame); + + THREAD* t = KeGetCurrentThread(); + if (!t) return; + + frame->PreviousMode = KeTrapFrameFromUser(frame) ? KTRAP_MODE_USER : KTRAP_MODE_KERNEL; + frame->PreviousFrame = t->TrapFrame; + t->TrapFrame = frame; +} + +void KiExitTrapFrame(KTRAP_FRAME* frame) { + if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) + KePanic("KiExitTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame); + + // Give alertable user-mode returns a chance to redirect through the + // registered user APC dispatcher before committing IRET slots. + KiCheckForUserApcDelivery(frame); + + // Commit editable return state. Kernel-mode interrupts have no RSP/SS + // hardware slots, so those pointers are NULL in that case. + if (frame->RipSlot) *frame->RipSlot = frame->Rip; + if (frame->CsSlot) *frame->CsSlot = frame->Cs; + if (frame->RflagsSlot) *frame->RflagsSlot = frame->Rflags; + if (frame->RspSlot) *frame->RspSlot = frame->Rsp; + if (frame->SsSlot) *frame->SsSlot = frame->Ss; + + THREAD* t = KeGetCurrentThread(); + if (t) { + if (t->TrapFrame != frame) + KePanic("KiExitTrapFrame: trap-frame stack mismatch top=0x%llX frame=0x%llX\n", + (uint64_t)t->TrapFrame, (uint64_t)frame); + t->TrapFrame = frame->PreviousFrame; + } +} + +void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1, + uint64_t a2, uint64_t a3, uint64_t a4, + uint64_t a5) { + KTRAP_FRAME* f = KeGetCurrentTrapFrame(); + if (!f) + KePanic("syscall trap validation: no current trap frame for nr=%llu\n", nr); + if (f->Magic != KTRAP_FRAME_MAGIC) + KePanic("syscall trap validation: bad magic 0x%llX\n", f->Magic); + if (f->Kind != KTRAP_KIND_SYSCALL) + KePanic("syscall trap validation: kind=%u, expected syscall\n", f->Kind); + if (!KeTrapFrameFromUser(f) || f->PreviousMode != KTRAP_MODE_USER) + KePanic("syscall trap validation: syscall did not come from user cs=0x%llX mode=%u\n", + f->Cs, f->PreviousMode); + if (f->Rax != nr || f->Rcx != a0 || f->Rdx != a1 || f->R8 != a2 || + f->R9 != a3 || f->R10 != a4 || f->R11 != a5) { + KePanic("syscall trap validation: arg mismatch nr=%llu tf=%llu/%llX/%llX/%llX/%llX/%llX/%llX call=%llu/%llX/%llX/%llX/%llX/%llX/%llX\n", + nr, + f->Rax, f->Rcx, f->Rdx, f->R8, f->R9, f->R10, f->R11, + nr, a0, a1, a2, a3, a4, a5); + } +} + +KSTATUS KeQueryCurrentTrapFrame(SYS_TRAP_FRAME_INFO_K* out) { + if (!out) return STATUS_INVALID_PARAMETER; + + KTRAP_FRAME* f = KeGetCurrentTrapFrame(); + if (!f || f->Magic != KTRAP_FRAME_MAGIC) + return STATUS_FAILURE; + + uint32_t size = out->Size; + if (size < sizeof(SYS_TRAP_FRAME_INFO_K)) { + out->Size = sizeof(SYS_TRAP_FRAME_INFO_K); + return STATUS_BUFFER_TOO_SMALL; + } + + memset(out, 0, sizeof(*out)); + out->Size = sizeof(*out); + out->Kind = f->Kind; + out->Vector = f->Vector; + out->PreviousMode = f->PreviousMode; + out->Rip = f->Rip; + out->Cs = f->Cs; + out->Rflags = f->Rflags; + out->Rsp = f->Rsp; + out->Ss = f->Ss; + out->Rax = f->Rax; + out->Rcx = f->Rcx; + out->Rdx = f->Rdx; + out->R8 = f->R8; + out->R9 = f->R9; + out->R10 = f->R10; + out->R11 = f->R11; + return STATUS_SUCCESS; +} diff --git a/code/kernel/ke_trap.h b/code/kernel/ke_trap.h new file mode 100644 index 0000000..2c8b17e --- /dev/null +++ b/code/kernel/ke_trap.h @@ -0,0 +1,88 @@ +#pragma once +// ke_trap.h - trap-frame metadata for syscall/interrupt return paths +// +// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally +// an editable return frame: future user APC delivery can rewrite Rip/Rsp in the +// frame, and KiExitTrapFrame will commit those values back to the hardware IRET +// slots before returning to user mode. + +#include +#include + +#define KTRAP_FRAME_MAGIC 0x4650545241544B31ULL + +#define KTRAP_KIND_NONE 0u +#define KTRAP_KIND_SYSCALL 1u +#define KTRAP_KIND_INTERRUPT 2u + +#define KTRAP_MODE_KERNEL 0u +#define KTRAP_MODE_USER 1u + +typedef struct KTRAP_FRAME { + uint64_t Magic; + uint32_t Kind; + uint32_t Vector; + uint32_t PreviousMode; + uint32_t Reserved; + + uint64_t Rax; + uint64_t Rcx; + uint64_t Rdx; + uint64_t Rbx; + uint64_t Rbp; + uint64_t Rsi; + uint64_t Rdi; + uint64_t R8; + uint64_t R9; + uint64_t R10; + uint64_t R11; + uint64_t R12; + uint64_t R13; + uint64_t R14; + uint64_t R15; + + uint64_t Rip; + uint64_t Cs; + uint64_t Rflags; + uint64_t Rsp; + uint64_t Ss; + + uint64_t* RipSlot; + uint64_t* CsSlot; + uint64_t* RflagsSlot; + uint64_t* RspSlot; + uint64_t* SsSlot; + + struct KTRAP_FRAME* PreviousFrame; + uint64_t Reserved2; +} KTRAP_FRAME; + +typedef struct SYS_TRAP_FRAME_INFO_K { + uint32_t Size; + uint32_t Kind; + uint32_t Vector; + uint32_t PreviousMode; + + uint64_t Rip; + uint64_t Cs; + uint64_t Rflags; + uint64_t Rsp; + uint64_t Ss; + + uint64_t Rax; + uint64_t Rcx; + uint64_t Rdx; + uint64_t R8; + uint64_t R9; + uint64_t R10; + uint64_t R11; +} SYS_TRAP_FRAME_INFO_K; + +void KiEnterTrapFrame(KTRAP_FRAME* frame); +void KiExitTrapFrame(KTRAP_FRAME* frame); +KTRAP_FRAME* KeGetCurrentTrapFrame(void); +int KeTrapFrameFromUser(const KTRAP_FRAME* frame); +void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1, + uint64_t a2, uint64_t a3, uint64_t a4, + uint64_t a5); +KSTATUS KeQueryCurrentTrapFrame(SYS_TRAP_FRAME_INFO_K* out); diff --git a/code/kernel/ke_userenv.h b/code/kernel/ke_userenv.h index e6d71bd..ebf8bce 100644 --- a/code/kernel/ke_userenv.h +++ b/code/kernel/ke_userenv.h @@ -8,7 +8,7 @@ #define SYS_PEB_MAGIC 0x30424550u /* "PEB0" */ #define SYS_TEB_MAGIC 0x30424554u /* "TEB0" */ #define SYS_ENV_VERSION 1u -#define SYS_CWD_MAX 512u +#define SYS_CWD_MAX 520u typedef struct SYS_PEB { uint32_t Magic; diff --git a/code/kernel/kmain.c b/code/kernel/kmain.c index 5c9defd..eb95b75 100644 --- a/code/kernel/kmain.c +++ b/code/kernel/kmain.c @@ -11,12 +11,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include typedef struct { @@ -30,7 +32,49 @@ typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *); // threading a parameter through every call chain. static const HAL_API* gKmainHal; -static int PageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) { +// Kernel-owned DPC wrappers for HAL input devices. The HAL keeps the tiny +// hardware-specific ISR fast and exposes a bottom-half entry point; the kernel +// owns the actual DPC objects and scheduling policy. +static KDPC gKeyboardDpc; +static KDPC gMouseDpc; + +static void KeyboardDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)context; + (void)systemArgument1; + (void)systemArgument2; + + if (gKmainHal && gKmainHal->Keyboard.Dpc) + gKmainHal->Keyboard.Dpc(); +} + +static void KiMouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)context; + (void)systemArgument1; + (void)systemArgument2; + + if (gKmainHal && gKmainHal->Mouse.Dpc) + gKmainHal->Mouse.Dpc(); +} + +static void KeyboardIrqWrapper(void) { + if (gKmainHal && gKmainHal->Keyboard.Isr) + gKmainHal->Keyboard.Isr(); + + if (gKmainHal && gKmainHal->Keyboard.Dpc) + KeInsertQueueDpc(&gKeyboardDpc, (void*)0, (void*)0); +} + +static void KiMouseIrqWrapper(void) { + if (gKmainHal && gKmainHal->Mouse.Isr) + gKmainHal->Mouse.Isr(); + + if (gKmainHal && gKmainHal->Mouse.Dpc) + KeInsertQueueDpc(&gMouseDpc, (void*)0, (void*)0); +} + +static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) { for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) { const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i]; if (r->Type != MemoryBootloader) continue; @@ -44,7 +88,7 @@ static int PageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS pa return 0; } -static void ReclaimBootloaderMemory(const BOOT_INFO* bootInfo) { +static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) { // Reclaim only the raw buffers backing boot-loaded .sys modules. // The kernel/HAL images, page tables, and bootloader code remain live. for (uint32_t i = 2; i < bootInfo->ModuleCount; i++) { @@ -55,7 +99,7 @@ static void ReclaimBootloaderMemory(const BOOT_INFO* bootInfo) { uintptr_t end = (uintptr_t)((mod->PhysBase + mod->ImageSize + PAGE_SIZE - 1) & PAGE_MASK); for (uintptr_t page = base; page < end; page += PAGE_SIZE) { - if (!PageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue; + if (!KiPageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue; MmFreeFrame((PHYSICAL_ADDRESS)page); } } @@ -64,7 +108,7 @@ static void ReclaimBootloaderMemory(const BOOT_INFO* bootInfo) { // --------------------------------------------------------------------------- // Phase Helper: Resolve Imports for core modules // --------------------------------------------------------------------------- -static void ResolveSystemImports(BOOT_INFO* bootInfo) { +static void KiResolveSystemImports(BOOT_INFO* bootInfo) { // Only resolve imports for kernel.exe and hal.dll (typically indices 0 and 1). // The bootloader maps these, but drivers are now passed as raw physical files. uint32_t coreModuleCount = (bootInfo->ModuleCount > 2) ? 2 : bootInfo->ModuleCount; @@ -79,7 +123,7 @@ static void ResolveSystemImports(BOOT_INFO* bootInfo) { // --------------------------------------------------------------------------- // Phase Helper: Initialize dynamically loaded .sys drivers // --------------------------------------------------------------------------- -static void DriversIniCb(const char* section, const char* key, +static void KiDriversIniCb(const char* section, const char* key, const char* val, void* ud) { (void)val; if (!strcmp(section, "runtime")) { @@ -93,8 +137,8 @@ static void DriversIniCb(const char* section, const char* key, } } -static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { - DbgPrint("InitializeDrivers: Initializing drivers...\n"); +static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { + DbgPrint("KiInitializeDrivers: Initializing drivers...\n"); // boot drivers (provided by the bootloader) for (uint32_t i = 0; i < bootInfo->ModuleCount; i++) { @@ -104,7 +148,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { // or have no physical data (invalid) if (mod->VirtBase != 0 || mod->PhysBase == 0) continue; - DbgPrint("InitializeDrivers: Loading boot driver %s\n", mod->Name); + DbgPrint("KiInitializeDrivers: Loading boot driver %s\n", mod->Name); // Calculate virtual address of the raw file in the physical window void* rawPeBuffer = (void*)(bootInfo->PhysicalMapBase + mod->PhysBase); @@ -145,7 +189,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { s = IoOpenFile("drivers\\drivers.ini", &f); if (s != STATUS_SUCCESS) { - KePanic("InitializeDrivers: Failed to open drivers.ini (status %d)\n", s); + KePanic("KiInitializeDrivers: Failed to open drivers.ini (status %d)\n", s); } uint64_t size = IoGetFileSize(f) + 1; @@ -153,14 +197,14 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { size_t got; if (!KSUCCESS(IoReadFile(f, mem, size, &got))) { IoCloseFile(f); - KePanic("InitializeDrivers: Failed to read drivers.ini\n"); + KePanic("KiInitializeDrivers: Failed to read drivers.ini\n"); } mem[got] = '\0'; IoCloseFile(f); char** runtimeKeys = NULL; - IniParse(mem, DriversIniCb, &runtimeKeys); + IniParse(mem, KiDriversIniCb, &runtimeKeys); for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) { DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]); @@ -179,7 +223,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { s = IoOpenFile(path, &f); if (s != STATUS_SUCCESS) { - DbgPrint("InitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path); + DbgPrint("KiInitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path); continue; // Skip this driver but keep going with others } @@ -188,7 +232,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { uint64_t drvSize = IoGetFileSize(f); void* drvMem = MmAllocatePool(drvSize); if (!KSUCCESS(IoReadFile(f, drvMem, drvSize, &got))) { - DbgPrint("InitializeDrivers: Failed to read runtime driver: %s\n", runtimeKeys[i]); + DbgPrint("KiInitializeDrivers: Failed to read runtime driver: %s\n", runtimeKeys[i]); IoCloseFile(f); continue; } @@ -196,14 +240,14 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { uintptr_t mappedBase = 0; if (!KSUCCESS(LdrLoadImage(drvMem, drvSize, &mappedBase, false))) { // Load into kernel space - DbgPrint("InitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]); + DbgPrint("KiInitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]); MmFreePool(drvMem); continue; } MmFreePool(drvMem); if (!KSUCCESS(LdrResolveImports(mappedBase))) { - DbgPrint("InitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]); + DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]); continue; } @@ -211,7 +255,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { KSTATUS ds = entry(hal); if (!KSUCCESS(ds)) { - DbgPrint("InitializeDrivers: Driver %s failed to init!\n", runtimeKeys[i]); + DbgPrint("KiInitializeDrivers: Driver %s failed to init!\n", runtimeKeys[i]); } } @@ -235,23 +279,23 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) { // Returns the new PROCESS* with one reference owned by the caller. // Returns NULL on failure. // --------------------------------------------------------------------------- -static PROCESS* LaunchUserProcess(const char* filePath, const char* args) { +static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { KSTATUS s; FILE_OBJECT* f; // Read the executable file into a raw buffer. s = IoOpenFile(filePath, &f); - DbgPrint("LaunchUserProcess: IoOpenFile: %d\n", s); + DbgPrint("KiLaunchUserProcess: IoOpenFile: %d\n", s); if (s != STATUS_SUCCESS) { DbgPrint("Failed to load file: \"%s\"!\n", filePath); return 0; } uint64_t size = IoGetFileSize(f); - DbgPrint("LaunchUserProcess: IoGetFileSize: %llu\n", size); + DbgPrint("KiLaunchUserProcess: IoGetFileSize: %llu\n", size); void* mem = MmAllocatePool(size); - DbgPrint("LaunchUserProcess: MmAllocatePool: 0x%llX\n", mem); + DbgPrint("KiLaunchUserProcess: MmAllocatePool: 0x%llX\n", mem); if (!mem) { IoCloseFile(f); return 0; @@ -259,7 +303,7 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) { size_t got = 0; s = IoReadFile(f, mem, size, &got); - DbgPrint("LaunchUserProcess: IoReadFile: %d\n", s); + DbgPrint("KiLaunchUserProcess: IoReadFile: %d\n", s); IoCloseFile(f); if (!KSUCCESS(s)) { MmFreePool(mem); @@ -280,7 +324,7 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) { s = KeSetupUserProcessEnvironment(proc, filePath, args, proc->CurrentDirectory); if (!KSUCCESS(s)) { - DbgPrint("LaunchUserProcess: Failed to build user process environment. Status: %d\n", s); + DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s); MmFreePool(mem); ObDereferenceObject(&proc->Header); return 0; @@ -291,15 +335,15 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) { s = LdrLoadImage(mem, size, &imageBase, true); MmFreePool(mem); if (!KSUCCESS(s)) { - DbgPrint("LaunchUserProcess: LdrLoadImage failed with status: %d\n", s); + DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s); ObDereferenceObject(&proc->Header); return 0; } s = LdrResolveImportsForProcess(proc, imageBase); if (!KSUCCESS(s)) { - DbgPrint("LaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s); - MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase)); + DbgPrint("KiLaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s); + MmFreeUserMemory((void*)imageBase); ObDereferenceObject(&proc->Header); return 0; } @@ -308,24 +352,24 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) { proc->ImageSize = LdrGetImageSize(imageBase); uintptr_t entryPoint = LdrGetEntryPoint(imageBase); - DbgPrint("LaunchUserProcess: PE Image mapped! Base 0x%llx, Entry 0x%llX\n", imageBase, entryPoint); + DbgPrint("KiLaunchUserProcess: PE Image mapped! Base 0x%llx, Entry 0x%llX\n", imageBase, entryPoint); // Create the main thread last. KeCreateThread queues it before returning. s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread); if (!KSUCCESS(s)) { - DbgPrint("LaunchUserProcess: Failed to create thread. Status: %d\n", s); + DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s); proc->ImageBase = 0; proc->ImageSize = 0; - MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase)); + MmFreeUserMemory((void*)imageBase); ObDereferenceObject(&proc->Header); return 0; } - DbgPrint("LaunchUserProcess: Thread created and queued!\n"); + DbgPrint("KiLaunchUserProcess: Thread created and queued!\n"); // Caller owns the reference returned by KeCreateProcess. return proc; } -static void PrintProcessList() { +static void KiPrintProcessList() { LIST_ENTRY* listHead = KeGetProcessList(); LIST_ENTRY* current; @@ -349,10 +393,10 @@ void SystemInitThread(void* arg) { DbgPrint("SystemInitThread: Thread started. Igniting the system...\n"); - InitializeDrivers(args->bootInfo, args->hal); + KiInitializeDrivers(args->bootInfo, args->hal); - ResolveSystemImports(args->bootInfo); - ReclaimBootloaderMemory(args->bootInfo); + KiResolveSystemImports(args->bootInfo); + KiReclaimBootloaderMemory(args->bootInfo); args->hal->Screen.Clear(0x000000); @@ -362,16 +406,19 @@ void SystemInitThread(void* arg) { DbgPrint(" -> Kernel Pool: %.2f MB\n", (float)MmGetKernelPoolUsage() / (1024.0f * 1024.0f)); DbgPrint(" -> User Heap: %.2f MB\n", (float)MmGetUserHeapUsage() / (1024.0f * 1024.0f)); + if (args->bootInfo->Debug & 2) + KiRunKernelSelfTests(args->hal); + // Launch User-Mode init.exe and restart it until it exits cleanly. while (true) { - PROCESS* shellProc = LaunchUserProcess("system/init.exe", args->bootInfo->CommandLine); + PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine); if (!shellProc) { DbgPrint("SystemInitThread: Failed to start system/init.exe; retrying.\n"); KeYield(); continue; } - KeWaitForEvent(shellProc->ExitEvent); + KeWaitForEvent(shellProc->ExitEvent, 0); int exitCode = shellProc->ExitCode; DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode); ObDereferenceObject(&shellProc->Header); @@ -421,7 +468,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) { .FramebufferHeight = bootInfo->FramebufferHeight, .FramebufferStride = bootInfo->FramebufferStride, .RsdpPhysical = bootInfo->RsdpPhysical, - .DebugEnabled = bootInfo->DebugEnabled + .Debug = bootInfo->Debug }; KSTATUS s = HalInitialize(&halParams); @@ -432,13 +479,19 @@ extern void KernelMain(BOOT_INFO* bootInfo) { const HAL_API* hal = HalGetApi(); gKmainHal = hal; - KeInitDebug(hal->Screen.PrintChar, bootInfo->DebugEnabled); // Enable debug + KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1); // Enable debug KeIrqInitialize(hal); + KeInitializeDpcSystem(); - KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr); - KeIrqRegister(hal->Keyboard.IrqNumber, hal->Keyboard.Isr); - KeIrqRegister(hal->Mouse.IrqNumber, hal->Mouse.Isr); + KeInitializeDpc(&gKeyboardDpc, KeyboardDpcRoutine, (void*)0); + KeInitializeDpc(&gMouseDpc, KiMouseDpcRoutine, (void*)0); + + KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr); + KeIrqRegister(hal->Keyboard.IrqNumber, + hal->Keyboard.Dpc ? KeyboardIrqWrapper : hal->Keyboard.Isr); + KeIrqRegister(hal->Mouse.IrqNumber, + hal->Mouse.Dpc ? KiMouseIrqWrapper : hal->Mouse.Isr); HalRegisterScheduler(KeSchedulerTick); @@ -447,8 +500,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) { // // Dynamic Linking & Driver Execution // - ResolveSystemImports(bootInfo); - //InitializeDrivers(bootInfo, hal); + KiResolveSystemImports(bootInfo); + //KiInitializeDrivers(bootInfo, hal); // // Scheduler Prep & Status Update diff --git a/code/kernel/ldr_image.c b/code/kernel/ldr_image.c index 3807481..006c265 100644 --- a/code/kernel/ldr_image.c +++ b/code/kernel/ldr_image.c @@ -1,516 +1,516 @@ -// ldr_image.c - PE32+ loader and export resolver -#include -#include -#include -#include -#include -#include -#include -#include - -// --------------------------------------------------------------------------- -// PE32+ structures (just what we need) -// --------------------------------------------------------------------------- -#define PE_SIGNATURE 0x00004550UL // 'PE\0\0' -#define IMAGE_FILE_MACHINE_AMD64 0x8664 - -#pragma pack(push, 1) -typedef struct { - uint16_t Magic; - uint8_t MajorLinkerVersion; - uint8_t MinorLinkerVersion; - uint32_t SizeOfCode; - uint32_t SizeOfInitializedData; - uint32_t SizeOfUninitializedData; - uint32_t AddressOfEntryPoint; - uint32_t BaseOfCode; - uint64_t ImageBase; - uint32_t SectionAlignment; - uint32_t FileAlignment; - uint16_t MajorOSVersion; - uint16_t MinorOSVersion; - uint16_t MajorImageVersion; - uint16_t MinorImageVersion; - uint16_t MajorSubsystemVersion; - uint16_t MinorSubsystemVersion; - uint32_t Win32VersionValue; - uint32_t SizeOfImage; - uint32_t SizeOfHeaders; - uint32_t CheckSum; - uint16_t Subsystem; - uint16_t DllCharacteristics; - uint64_t SizeOfStackReserve; - uint64_t SizeOfStackCommit; - uint64_t SizeOfHeapReserve; - uint64_t SizeOfHeapCommit; - uint32_t LoaderFlags; - uint32_t NumberOfRvaAndSizes; - struct { uint32_t VirtualAddress; uint32_t Size; } DataDirectory[16]; -} OPT_HEADER64; - -typedef struct { - uint32_t Signature; - uint16_t Machine; - uint16_t NumberOfSections; - uint32_t TimeDateStamp; - uint32_t PointerToSymbolTable; - uint32_t NumberOfSymbols; - uint16_t SizeOfOptionalHeader; - uint16_t Characteristics; - OPT_HEADER64 OptionalHeader; -} PE_HEADER; - -typedef struct { - char Name[8]; - uint32_t VirtualSize; - uint32_t VirtualAddress; - uint32_t SizeOfRawData; - uint32_t PointerToRawData; - uint32_t PointerToRelocations; - uint32_t PointerToLinenumbers; - uint16_t NumberOfRelocations; - uint16_t NumberOfLinenumbers; - uint32_t Characteristics; -} SECTION_HEADER; - -typedef struct { - uint32_t Characteristics; - uint32_t TimeDateStamp; - uint16_t MajorVersion; - uint16_t MinorVersion; - uint32_t Name; - uint32_t Base; - uint32_t NumberOfFunctions; - uint32_t NumberOfNames; - uint32_t AddressOfFunctions; - uint32_t AddressOfNames; - uint32_t AddressOfNameOrdinals; -} EXPORT_DIR; - -typedef struct { - uint32_t OriginalFirstThunk; - uint32_t TimeDateStamp; - uint32_t ForwarderChain; - uint32_t Name; - uint32_t FirstThunk; -} IMPORT_DESC; - -typedef struct { - uint32_t VirtualAddress; - uint32_t SizeOfBlock; -} BASE_RELOC; -#pragma pack(pop) - -#define IMAGE_REL_BASED_DIR64 10 - -// --------------------------------------------------------------------------- -// Module registry -// --------------------------------------------------------------------------- -#define MAX_MODULES 32 - -typedef struct { - char Name[32]; - uintptr_t Base; - size_t Size; -} MODULE_ENTRY; - -static MODULE_ENTRY gModules[MAX_MODULES]; -static uint32_t gModuleCount = 0; - -static uintptr_t FindModuleBaseInTable(const MODULE_ENTRY* modules, uint32_t count, const char* name) { - for (uint32_t i = 0; i < count; i++) { - if (_stricmp(modules[i].Name, name) == 0) return modules[i].Base; - } - return 0; -} - -static void RegisterModuleEntry(MODULE_ENTRY* entry, const char* name, uintptr_t base, size_t size) { - strncpy(entry->Name, name, 31); - entry->Name[31] = '\0'; - entry->Base = base; - entry->Size = size; -} - -void LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) { - for (uint32_t i = 0; i < bootInfo->ModuleCount && gModuleCount < MAX_MODULES; i++) { - // Only register modules that have a virtual base (Kernel, HAL) - if (bootInfo->Modules[i].VirtBase != 0) { - RegisterModuleEntry(&gModules[gModuleCount], - bootInfo->Modules[i].Name, - bootInfo->Modules[i].VirtBase, - bootInfo->Modules[i].ImageSize); - gModuleCount++; - } - } -} - -void LdrRegisterModule(const char* name, uintptr_t base) { - if (gModuleCount < MAX_MODULES) { - RegisterModuleEntry(&gModules[gModuleCount], name, base, 0); - gModuleCount++; - } -} - -static uintptr_t FindGlobalModuleBase(const char* name) { - return FindModuleBaseInTable(gModules, gModuleCount, name); -} - -static uintptr_t FindProcessModuleBaseInTable( - const PROCESS_MODULE* modules, - uint32_t count, - const char* name -) { - for (uint32_t i = 0; i < count; i++) { - if (_stricmp(modules[i].Name, name) == 0) { - return modules[i].Base; - } - } - return 0; -} - -static uintptr_t FindProcessOrGlobalModuleBase(PROCESS* process, const char* name) { - if (process) { - uintptr_t base = FindProcessModuleBaseInTable(process->Modules, process->ModuleCount, name); - if (base) return base; - } - return FindGlobalModuleBase(name); -} - -static void* GetExportAtBase(uintptr_t base, const char* functionName) { - if (!base) return (void *)0; - - uint32_t peOffset = *(uint32_t *)(base + 0x3C); - PE_HEADER* pe = (PE_HEADER *)(base + peOffset); - if (pe->Signature != PE_SIGNATURE) return (void *)0; - - OPT_HEADER64* opt = &pe->OptionalHeader; - if (opt->NumberOfRvaAndSizes < 1) return (void *)0; - - uint32_t expRva = opt->DataDirectory[0].VirtualAddress; - if (!expRva) return (void *)0; - - EXPORT_DIR* exp = (EXPORT_DIR *)(base + expRva); - uint32_t* names = (uint32_t *)(base + exp->AddressOfNames); - uint16_t* ordinals = (uint16_t *)(base + exp->AddressOfNameOrdinals); - uint32_t* functions = (uint32_t *)(base + exp->AddressOfFunctions); - - for (uint32_t i = 0; i < exp->NumberOfNames; i++) { - const char* name = (const char *)(base + names[i]); - if (strcmp(name, functionName) == 0) { - uint16_t ord = ordinals[i]; - return (void *)(base + functions[ord]); - } - } - return (void *)0; -} - -// --------------------------------------------------------------------------- -// Export lookup -// --------------------------------------------------------------------------- -void* LdrGetExport(const char* moduleName, const char* functionName) { - uintptr_t base = FindGlobalModuleBase(moduleName); - return GetExportAtBase(base, functionName); -} - -void* LdrGetExportForProcess(PROCESS* process, const char* moduleName, const char* functionName) { - uintptr_t base = FindProcessOrGlobalModuleBase(process, moduleName); - return GetExportAtBase(base, functionName); -} - -size_t LdrGetImageSize(uintptr_t base) { - uint32_t peOffset = *(uint32_t *)(base + 0x3C); - PE_HEADER* pe = (PE_HEADER *)(base + peOffset); - if (pe->Signature != PE_SIGNATURE) return 0; - return (size_t)pe->OptionalHeader.SizeOfImage; -} - -// --------------------------------------------------------------------------- -// Entry point lookup -// --------------------------------------------------------------------------- -uintptr_t LdrGetEntryPoint(uintptr_t base) { - uint32_t pe_offset = *(uint32_t*)(base + 0x3C); - uint32_t entry_rva = *(uint32_t*)(base + pe_offset + 0x28); - return base + entry_rva; -} - -// --------------------------------------------------------------------------- -// Base Relocation -// --------------------------------------------------------------------------- -static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) { - uint32_t peOffset = *(uint32_t *)(base + 0x3C); - PE_HEADER* pe = (PE_HEADER *)(base + peOffset); - OPT_HEADER64* opt = &pe->OptionalHeader; - - if (opt->NumberOfRvaAndSizes < 6) return STATUS_SUCCESS; - uint32_t relocRva = opt->DataDirectory[5].VirtualAddress; - uint32_t relocSize = opt->DataDirectory[5].Size; - if (!relocRva || !relocSize) return STATUS_SUCCESS; - - BASE_RELOC* block = (BASE_RELOC *)(base + relocRva); - uintptr_t end = (uintptr_t)block + relocSize; - - while ((uintptr_t)block < end && block->SizeOfBlock > 0) { - uint32_t count = (block->SizeOfBlock - sizeof(BASE_RELOC)) / sizeof(uint16_t); - uint16_t* entries = (uint16_t *)(block + 1); - - for (uint32_t i = 0; i < count; i++) { - int type = entries[i] >> 12; - int offset = entries[i] & 0xFFF; - - // For 64-bit PEs, we mostly care about DIR64 (Type 10) - if (type == IMAGE_REL_BASED_DIR64) { - uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset); - *ptr += delta; - } - } - block = (BASE_RELOC *)((uint8_t *)block + block->SizeOfBlock); - } - return STATUS_SUCCESS; -} - -// --------------------------------------------------------------------------- -// Import resolution -// --------------------------------------------------------------------------- -typedef void* (*EXPORT_LOOKUP_FN)(void* ctx, const char* moduleName, const char* functionName); - -static void* GlobalExportLookup(void* ctx, const char* moduleName, const char* functionName) { - (void)ctx; - return LdrGetExport(moduleName, functionName); -} - -static void* ProcessExportLookup(void* ctx, const char* moduleName, const char* functionName) { - return LdrGetExportForProcess((PROCESS*)ctx, moduleName, functionName); -} - -static KSTATUS LdrResolveImportsCommon( - PROCESS* process, - uintptr_t imageBase, - EXPORT_LOOKUP_FN lookup, - void* ctx -) { - uint32_t peOffset = *(uint32_t *)(imageBase + 0x3C); - PE_HEADER* pe = (PE_HEADER *)(imageBase + peOffset); - OPT_HEADER64* opt = &pe->OptionalHeader; - - if (opt->NumberOfRvaAndSizes < 2) - return STATUS_SUCCESS; - - uint32_t impRva = opt->DataDirectory[1].VirtualAddress; - if (!impRva) - return STATUS_SUCCESS; - - IMPORT_DESC* imp = (IMPORT_DESC *)(imageBase + impRva); - - while (imp->Name) { - const char* dllName = (const char *)(imageBase + imp->Name); - - // Win32 compatibility remap - if (!_stricmp(dllName, "kernel32.dll") || - !_stricmp(dllName, "user32.dll") || - !_stricmp(dllName, "ole32.dll") || - !_stricmp(dllName, "winmm.dll")) { - dllName = "win32.dll"; - } - - // - // Ensure dependency is loaded into the process - // - if (process) { - uintptr_t modBase = - FindProcessOrGlobalModuleBase(process, dllName); - - if (!modBase) { - KSTATUS s = - LdrLoadModuleForProcess(process, dllName, &modBase); - - if (!KSUCCESS(s)) { - DbgPrint( - "LdrResolveImports: Failed to load dependency %s (%d)\n", - dllName, - s - ); - return s; - } - } - } - - uint64_t* thunk = - (uint64_t *)(imageBase + imp->FirstThunk); - - uint64_t* origThunk = - (uint64_t *)(imageBase + - (imp->OriginalFirstThunk - ? imp->OriginalFirstThunk - : imp->FirstThunk)); - - while (*origThunk) { - if (*origThunk & (1ULL << 63)) { - DbgPrint( - "LdrResolveImports: Ordinal imports not supported! (%s)\n", - dllName - ); - return STATUS_NOT_SUPPORTED; - } - - const char* funcName = - (const char *)(imageBase + - (*origThunk & 0xFFFFFFFF) + 2); - - void* addr = lookup(ctx, dllName, funcName); - - if (!addr) { - DbgPrint( - "LdrResolveImports: Function not found! (%s!%s)\n", - dllName, - funcName - ); - return STATUS_NOT_FOUND; - } - - *thunk = (uint64_t)addr; - - thunk++; - origThunk++; - } - - imp++; - } - - return STATUS_SUCCESS; -} - -KSTATUS LdrResolveImports(uintptr_t imageBase) { - return LdrResolveImportsCommon( - (PROCESS*)0, - imageBase, - GlobalExportLookup, - (void*)0 - ); -} - -KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) { - return LdrResolveImportsCommon( - process, - imageBase, - ProcessExportLookup, - process - ); -} - -static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) { - if (!process || !name || !base) return STATUS_INVALID_PARAMETER; - - for (uint32_t i = 0; i < process->ModuleCount; i++) { - if (_stricmp(process->Modules[i].Name, name) == 0) { - return STATUS_SUCCESS; - } - } - - if (process->ModuleCount >= PROCESS_MODULE_MAX) return STATUS_NO_MEMORY; - - PROCESS_MODULE* mod = &process->Modules[process->ModuleCount++]; - strncpy(mod->Name, name, 31); - mod->Name[31] = '\0'; - mod->Base = base; - mod->Size = LdrGetImageSize(base); - return STATUS_SUCCESS; -} - -KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_t* baseOut) { - if (!process || !filename) return STATUS_INVALID_PARAMETER; - - for (uint32_t i = 0; i < process->ModuleCount; i++) { - if (_stricmp(process->Modules[i].Name, filename) == 0) { - if (baseOut) *baseOut = process->Modules[i].Base; - return STATUS_SUCCESS; - } - } - - FILE_OBJECT* f; - KSTATUS s = IoOpenFile(filename, &f); - if (!KSUCCESS(s)) { - char path[260]; - memcpy(path, "0:/system/", 10); - memcpy(path + 10, filename, strnlen(filename, 248) + 1); - path[259] = '\0'; - s = IoOpenFile(path, &f); - if (!KSUCCESS(s)) { - DbgPrint("LdrLoadModuleForProcess: Failed to open %s (%d)\n", filename, s); - return s; - } - } - - uint64_t fileSize = IoGetFileSize(f); - void* mem = MmAllocatePool((size_t)fileSize); - if (!mem) { - IoCloseFile(f); - return STATUS_NO_MEMORY; - } - - size_t got = 0; - s = IoReadFile(f, mem, (size_t)fileSize, &got); - IoCloseFile(f); - if (!KSUCCESS(s)) { - MmFreePool(mem); - return s; - } - - uintptr_t imageBase = 0; - s = LdrLoadImage(mem, (size_t)fileSize, &imageBase, true); - MmFreePool(mem); - if (!KSUCCESS(s)) { - return s; - } - - s = LdrResolveImportsForProcess(process, imageBase); - if (!KSUCCESS(s)) { - MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase)); - return s; - } - - s = LdrRegisterProcessModule(process, filename, imageBase); - if (!KSUCCESS(s)) { - MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase)); - return s; - } - - if (baseOut) *baseOut = imageBase; - return STATUS_SUCCESS; -} - -// LdrLoadImage - load a PE from a raw buffer into virtual space. -KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) { - if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER; - - uint32_t peOffset = *(uint32_t *)((uint8_t *)buffer + 0x3C); - PE_HEADER* pe = (PE_HEADER *)((uint8_t *)buffer + peOffset); - if (pe->Signature != PE_SIGNATURE) return STATUS_FAILURE; - - OPT_HEADER64* opt = &pe->OptionalHeader; - size_t imageSize = opt->SizeOfImage; - - void* mem = user ? MmAllocateUserMemory(imageSize) : MmAllocatePool(imageSize); - if (!mem) return STATUS_NO_MEMORY; - memset(mem, 0, imageSize); - - memcpy(mem, buffer, opt->SizeOfHeaders); - SECTION_HEADER* sec = (SECTION_HEADER *)((uint8_t *)&pe->OptionalHeader + pe->SizeOfOptionalHeader); - for (int i = 0; i < pe->NumberOfSections; i++) { - void* dst = (uint8_t *)mem + sec[i].VirtualAddress; - const void* src = (const uint8_t *)buffer + sec[i].PointerToRawData; - if (sec[i].SizeOfRawData) { - memcpy(dst, src, sec[i].SizeOfRawData < sec[i].VirtualSize ? sec[i].SizeOfRawData : sec[i].VirtualSize); - } - - uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0); - MmSetPageProtect((uintptr_t)mem + sec[i].VirtualAddress, sec[i].VirtualSize, prot); - } - - uintptr_t delta = (uintptr_t)mem - opt->ImageBase; - if (delta != 0) { - LdrApplyRelocations((uintptr_t)mem, delta); - } - - *baseOut = (uintptr_t)mem; - return STATUS_SUCCESS; -} +// ldr_image.c - PE32+ loader and export resolver +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// PE32+ structures (just what we need) +// --------------------------------------------------------------------------- +#define PE_SIGNATURE 0x00004550UL // 'PE\0\0' +#define IMAGE_FILE_MACHINE_AMD64 0x8664 + +#pragma pack(push, 1) +typedef struct { + uint16_t Magic; + uint8_t MajorLinkerVersion; + uint8_t MinorLinkerVersion; + uint32_t SizeOfCode; + uint32_t SizeOfInitializedData; + uint32_t SizeOfUninitializedData; + uint32_t AddressOfEntryPoint; + uint32_t BaseOfCode; + uint64_t ImageBase; + uint32_t SectionAlignment; + uint32_t FileAlignment; + uint16_t MajorOSVersion; + uint16_t MinorOSVersion; + uint16_t MajorImageVersion; + uint16_t MinorImageVersion; + uint16_t MajorSubsystemVersion; + uint16_t MinorSubsystemVersion; + uint32_t Win32VersionValue; + uint32_t SizeOfImage; + uint32_t SizeOfHeaders; + uint32_t CheckSum; + uint16_t Subsystem; + uint16_t DllCharacteristics; + uint64_t SizeOfStackReserve; + uint64_t SizeOfStackCommit; + uint64_t SizeOfHeapReserve; + uint64_t SizeOfHeapCommit; + uint32_t LoaderFlags; + uint32_t NumberOfRvaAndSizes; + struct { uint32_t VirtualAddress; uint32_t Size; } DataDirectory[16]; +} OPT_HEADER64; + +typedef struct { + uint32_t Signature; + uint16_t Machine; + uint16_t NumberOfSections; + uint32_t TimeDateStamp; + uint32_t PointerToSymbolTable; + uint32_t NumberOfSymbols; + uint16_t SizeOfOptionalHeader; + uint16_t Characteristics; + OPT_HEADER64 OptionalHeader; +} PE_HEADER; + +typedef struct { + char Name[8]; + uint32_t VirtualSize; + uint32_t VirtualAddress; + uint32_t SizeOfRawData; + uint32_t PointerToRawData; + uint32_t PointerToRelocations; + uint32_t PointerToLinenumbers; + uint16_t NumberOfRelocations; + uint16_t NumberOfLinenumbers; + uint32_t Characteristics; +} SECTION_HEADER; + +typedef struct { + uint32_t Characteristics; + uint32_t TimeDateStamp; + uint16_t MajorVersion; + uint16_t MinorVersion; + uint32_t Name; + uint32_t Base; + uint32_t NumberOfFunctions; + uint32_t NumberOfNames; + uint32_t AddressOfFunctions; + uint32_t AddressOfNames; + uint32_t AddressOfNameOrdinals; +} EXPORT_DIR; + +typedef struct { + uint32_t OriginalFirstThunk; + uint32_t TimeDateStamp; + uint32_t ForwarderChain; + uint32_t Name; + uint32_t FirstThunk; +} IMPORT_DESC; + +typedef struct { + uint32_t VirtualAddress; + uint32_t SizeOfBlock; +} BASE_RELOC; +#pragma pack(pop) + +#define IMAGE_REL_BASED_DIR64 10 + +// --------------------------------------------------------------------------- +// Module registry +// --------------------------------------------------------------------------- +#define MAX_MODULES 32 + +typedef struct { + char Name[32]; + uintptr_t Base; + size_t Size; +} MODULE_ENTRY; + +static MODULE_ENTRY gModules[MAX_MODULES]; +static uint32_t gModuleCount = 0; + +static uintptr_t KiFindModuleBaseInTable(const MODULE_ENTRY* modules, uint32_t count, const char* name) { + for (uint32_t i = 0; i < count; i++) { + if (_stricmp(modules[i].Name, name) == 0) return modules[i].Base; + } + return 0; +} + +static void KiRegisterModuleEntry(MODULE_ENTRY* entry, const char* name, uintptr_t base, size_t size) { + strncpy(entry->Name, name, 31); + entry->Name[31] = '\0'; + entry->Base = base; + entry->Size = size; +} + +void LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) { + for (uint32_t i = 0; i < bootInfo->ModuleCount && gModuleCount < MAX_MODULES; i++) { + // Only register modules that have a virtual base (Kernel, HAL) + if (bootInfo->Modules[i].VirtBase != 0) { + KiRegisterModuleEntry(&gModules[gModuleCount], + bootInfo->Modules[i].Name, + bootInfo->Modules[i].VirtBase, + bootInfo->Modules[i].ImageSize); + gModuleCount++; + } + } +} + +void LdrRegisterModule(const char* name, uintptr_t base) { + if (gModuleCount < MAX_MODULES) { + KiRegisterModuleEntry(&gModules[gModuleCount], name, base, 0); + gModuleCount++; + } +} + +static uintptr_t KiFindGlobalModuleBase(const char* name) { + return KiFindModuleBaseInTable(gModules, gModuleCount, name); +} + +static uintptr_t KiFindProcessModuleBaseInTable( + const PROCESS_MODULE* modules, + uint32_t count, + const char* name +) { + for (uint32_t i = 0; i < count; i++) { + if (_stricmp(modules[i].Name, name) == 0) { + return modules[i].Base; + } + } + return 0; +} + +static uintptr_t KiFindProcessOrGlobalModuleBase(PROCESS* process, const char* name) { + if (process) { + uintptr_t base = KiFindProcessModuleBaseInTable(process->Modules, process->ModuleCount, name); + if (base) return base; + } + return KiFindGlobalModuleBase(name); +} + +static void* KiGetExportAtBase(uintptr_t base, const char* functionName) { + if (!base) return (void *)0; + + uint32_t peOffset = *(uint32_t *)(base + 0x3C); + PE_HEADER* pe = (PE_HEADER *)(base + peOffset); + if (pe->Signature != PE_SIGNATURE) return (void *)0; + + OPT_HEADER64* opt = &pe->OptionalHeader; + if (opt->NumberOfRvaAndSizes < 1) return (void *)0; + + uint32_t expRva = opt->DataDirectory[0].VirtualAddress; + if (!expRva) return (void *)0; + + EXPORT_DIR* exp = (EXPORT_DIR *)(base + expRva); + uint32_t* names = (uint32_t *)(base + exp->AddressOfNames); + uint16_t* ordinals = (uint16_t *)(base + exp->AddressOfNameOrdinals); + uint32_t* functions = (uint32_t *)(base + exp->AddressOfFunctions); + + for (uint32_t i = 0; i < exp->NumberOfNames; i++) { + const char* name = (const char *)(base + names[i]); + if (strcmp(name, functionName) == 0) { + uint16_t ord = ordinals[i]; + return (void *)(base + functions[ord]); + } + } + return (void *)0; +} + +// --------------------------------------------------------------------------- +// Export lookup +// --------------------------------------------------------------------------- +void* LdrGetExport(const char* moduleName, const char* functionName) { + uintptr_t base = KiFindGlobalModuleBase(moduleName); + return KiGetExportAtBase(base, functionName); +} + +void* LdrGetExportForProcess(PROCESS* process, const char* moduleName, const char* functionName) { + uintptr_t base = KiFindProcessOrGlobalModuleBase(process, moduleName); + return KiGetExportAtBase(base, functionName); +} + +size_t LdrGetImageSize(uintptr_t base) { + uint32_t peOffset = *(uint32_t *)(base + 0x3C); + PE_HEADER* pe = (PE_HEADER *)(base + peOffset); + if (pe->Signature != PE_SIGNATURE) return 0; + return (size_t)pe->OptionalHeader.SizeOfImage; +} + +// --------------------------------------------------------------------------- +// Entry point lookup +// --------------------------------------------------------------------------- +uintptr_t LdrGetEntryPoint(uintptr_t base) { + uint32_t pe_offset = *(uint32_t*)(base + 0x3C); + uint32_t entry_rva = *(uint32_t*)(base + pe_offset + 0x28); + return base + entry_rva; +} + +// --------------------------------------------------------------------------- +// Base Relocation +// --------------------------------------------------------------------------- +static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) { + uint32_t peOffset = *(uint32_t *)(base + 0x3C); + PE_HEADER* pe = (PE_HEADER *)(base + peOffset); + OPT_HEADER64* opt = &pe->OptionalHeader; + + if (opt->NumberOfRvaAndSizes < 6) return STATUS_SUCCESS; + uint32_t relocRva = opt->DataDirectory[5].VirtualAddress; + uint32_t relocSize = opt->DataDirectory[5].Size; + if (!relocRva || !relocSize) return STATUS_SUCCESS; + + BASE_RELOC* block = (BASE_RELOC *)(base + relocRva); + uintptr_t end = (uintptr_t)block + relocSize; + + while ((uintptr_t)block < end && block->SizeOfBlock > 0) { + uint32_t count = (block->SizeOfBlock - sizeof(BASE_RELOC)) / sizeof(uint16_t); + uint16_t* entries = (uint16_t *)(block + 1); + + for (uint32_t i = 0; i < count; i++) { + int type = entries[i] >> 12; + int offset = entries[i] & 0xFFF; + + // For 64-bit PEs, we mostly care about DIR64 (Type 10) + if (type == IMAGE_REL_BASED_DIR64) { + uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset); + *ptr += delta; + } + } + block = (BASE_RELOC *)((uint8_t *)block + block->SizeOfBlock); + } + return STATUS_SUCCESS; +} + +// --------------------------------------------------------------------------- +// Import resolution +// --------------------------------------------------------------------------- +typedef void* (*EXPORT_LOOKUP_FN)(void* ctx, const char* moduleName, const char* functionName); + +static void* KiGlobalExportLookup(void* ctx, const char* moduleName, const char* functionName) { + (void)ctx; + return LdrGetExport(moduleName, functionName); +} + +static void* KiProcessExportLookup(void* ctx, const char* moduleName, const char* functionName) { + return LdrGetExportForProcess((PROCESS*)ctx, moduleName, functionName); +} + +static KSTATUS LdrResolveImportsCommon( + PROCESS* process, + uintptr_t imageBase, + EXPORT_LOOKUP_FN lookup, + void* ctx +) { + uint32_t peOffset = *(uint32_t *)(imageBase + 0x3C); + PE_HEADER* pe = (PE_HEADER *)(imageBase + peOffset); + OPT_HEADER64* opt = &pe->OptionalHeader; + + if (opt->NumberOfRvaAndSizes < 2) + return STATUS_SUCCESS; + + uint32_t impRva = opt->DataDirectory[1].VirtualAddress; + if (!impRva) + return STATUS_SUCCESS; + + IMPORT_DESC* imp = (IMPORT_DESC *)(imageBase + impRva); + + while (imp->Name) { + const char* dllName = (const char *)(imageBase + imp->Name); + + // Win32 compatibility remap + if (!_stricmp(dllName, "kernel32.dll") || + !_stricmp(dllName, "user32.dll") || + !_stricmp(dllName, "ole32.dll") || + !_stricmp(dllName, "winmm.dll")) { + dllName = "win32.dll"; + } + + // + // Ensure dependency is loaded into the process + // + if (process) { + uintptr_t modBase = + KiFindProcessOrGlobalModuleBase(process, dllName); + + if (!modBase) { + KSTATUS s = + LdrLoadModuleForProcess(process, dllName, &modBase); + + if (!KSUCCESS(s)) { + DbgPrint( + "LdrResolveImports: Failed to load dependency %s (%d)\n", + dllName, + s + ); + return s; + } + } + } + + uint64_t* thunk = + (uint64_t *)(imageBase + imp->FirstThunk); + + uint64_t* origThunk = + (uint64_t *)(imageBase + + (imp->OriginalFirstThunk + ? imp->OriginalFirstThunk + : imp->FirstThunk)); + + while (*origThunk) { + if (*origThunk & (1ULL << 63)) { + DbgPrint( + "LdrResolveImports: Ordinal imports not supported! (%s)\n", + dllName + ); + return STATUS_NOT_SUPPORTED; + } + + const char* funcName = + (const char *)(imageBase + + (*origThunk & 0xFFFFFFFF) + 2); + + void* addr = lookup(ctx, dllName, funcName); + + if (!addr) { + DbgPrint( + "LdrResolveImports: Function not found! (%s!%s)\n", + dllName, + funcName + ); + return STATUS_NOT_FOUND; + } + + *thunk = (uint64_t)addr; + + thunk++; + origThunk++; + } + + imp++; + } + + return STATUS_SUCCESS; +} + +KSTATUS LdrResolveImports(uintptr_t imageBase) { + return LdrResolveImportsCommon( + (PROCESS*)0, + imageBase, + KiGlobalExportLookup, + (void*)0 + ); +} + +KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) { + return LdrResolveImportsCommon( + process, + imageBase, + KiProcessExportLookup, + process + ); +} + +static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) { + if (!process || !name || !base) return STATUS_INVALID_PARAMETER; + + for (uint32_t i = 0; i < process->ModuleCount; i++) { + if (_stricmp(process->Modules[i].Name, name) == 0) { + return STATUS_SUCCESS; + } + } + + if (process->ModuleCount >= PROCESS_MODULE_MAX) return STATUS_NO_MEMORY; + + PROCESS_MODULE* mod = &process->Modules[process->ModuleCount++]; + strncpy(mod->Name, name, 31); + mod->Name[31] = '\0'; + mod->Base = base; + mod->Size = LdrGetImageSize(base); + return STATUS_SUCCESS; +} + +KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_t* baseOut) { + if (!process || !filename) return STATUS_INVALID_PARAMETER; + + for (uint32_t i = 0; i < process->ModuleCount; i++) { + if (_stricmp(process->Modules[i].Name, filename) == 0) { + if (baseOut) *baseOut = process->Modules[i].Base; + return STATUS_SUCCESS; + } + } + + FILE_OBJECT* f; + KSTATUS s = IoOpenFile(filename, &f); + if (!KSUCCESS(s)) { + char path[260]; + memcpy(path, "0:/system/", 10); + memcpy(path + 10, filename, strnlen(filename, 248) + 1); + path[259] = '\0'; + s = IoOpenFile(path, &f); + if (!KSUCCESS(s)) { + DbgPrint("LdrLoadModuleForProcess: Failed to open %s (%d)\n", filename, s); + return s; + } + } + + uint64_t fileSize = IoGetFileSize(f); + void* mem = MmAllocatePool((size_t)fileSize); + if (!mem) { + IoCloseFile(f); + return STATUS_NO_MEMORY; + } + + size_t got = 0; + s = IoReadFile(f, mem, (size_t)fileSize, &got); + IoCloseFile(f); + if (!KSUCCESS(s)) { + MmFreePool(mem); + return s; + } + + uintptr_t imageBase = 0; + s = LdrLoadImage(mem, (size_t)fileSize, &imageBase, true); + MmFreePool(mem); + if (!KSUCCESS(s)) { + return s; + } + + s = LdrResolveImportsForProcess(process, imageBase); + if (!KSUCCESS(s)) { + MmFreeUserMemory((void*)imageBase); + return s; + } + + s = LdrRegisterProcessModule(process, filename, imageBase); + if (!KSUCCESS(s)) { + MmFreeUserMemory((void*)imageBase); + return s; + } + + if (baseOut) *baseOut = imageBase; + return STATUS_SUCCESS; +} + +// LdrLoadImage - load a PE from a raw buffer into virtual space. +KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) { + if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER; + + uint32_t peOffset = *(uint32_t *)((uint8_t *)buffer + 0x3C); + PE_HEADER* pe = (PE_HEADER *)((uint8_t *)buffer + peOffset); + if (pe->Signature != PE_SIGNATURE) return STATUS_FAILURE; + + OPT_HEADER64* opt = &pe->OptionalHeader; + size_t imageSize = opt->SizeOfImage; + + void* mem = user ? MmAllocateUserMemory(imageSize) : MmAllocatePool(imageSize); + if (!mem) return STATUS_NO_MEMORY; + memset(mem, 0, imageSize); + + memcpy(mem, buffer, opt->SizeOfHeaders); + SECTION_HEADER* sec = (SECTION_HEADER *)((uint8_t *)&pe->OptionalHeader + pe->SizeOfOptionalHeader); + for (int i = 0; i < pe->NumberOfSections; i++) { + void* dst = (uint8_t *)mem + sec[i].VirtualAddress; + const void* src = (const uint8_t *)buffer + sec[i].PointerToRawData; + if (sec[i].SizeOfRawData) { + memcpy(dst, src, sec[i].SizeOfRawData < sec[i].VirtualSize ? sec[i].SizeOfRawData : sec[i].VirtualSize); + } + + uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0); + MmSetPageProtect((uintptr_t)mem + sec[i].VirtualAddress, sec[i].VirtualSize, prot); + } + + uintptr_t delta = (uintptr_t)mem - opt->ImageBase; + if (delta != 0) { + LdrApplyRelocations((uintptr_t)mem, delta); + } + + *baseOut = (uintptr_t)mem; + return STATUS_SUCCESS; +} diff --git a/code/kernel/mm_phys.c b/code/kernel/mm_phys.c index a46d4b1..e7133d1 100644 --- a/code/kernel/mm_phys.c +++ b/code/kernel/mm_phys.c @@ -14,7 +14,7 @@ // --------------------------------------------------------------------------- static uint64_t gPhysBase = 0; // set by MmInitializePhysical -static inline void* PhysToVirt(PHYSICAL_ADDRESS phys) { +static inline void* KiPhysToVirt(PHYSICAL_ADDRESS phys) { return (void *)(phys + gPhysBase); } @@ -33,7 +33,7 @@ typedef struct { static FRAME_STACK gFrameStack; // Push a physical frame address onto the stack. -static void PushFrame(PHYSICAL_ADDRESS frame) { +static void KiPushFrame(PHYSICAL_ADDRESS frame) { uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock); if (gFrameStack.Top < gFrameStack.Capacity) { gFrameStack.Stack[gFrameStack.Top++] = frame; @@ -72,7 +72,7 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) { return false; } - gFrameStack.Stack = (PHYSICAL_ADDRESS *)PhysToVirt(stackPhys); + gFrameStack.Stack = (PHYSICAL_ADDRESS *)KiPhysToVirt(stackPhys); gFrameStack.Top = 0; gFrameStack.Capacity = maxFrames; gFrameStack.Total = 0; @@ -91,7 +91,7 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) { for (PHYSICAL_ADDRESS frame = base; frame < end; frame += PAGE_SIZE) { if (frame >= stackPhys && frame < stackEnd) continue; - PushFrame(frame); + KiPushFrame(frame); } } diff --git a/code/kernel/mm_virt.c b/code/kernel/mm_virt.c index b17d3bf..ffd6668 100644 --- a/code/kernel/mm_virt.c +++ b/code/kernel/mm_virt.c @@ -16,17 +16,18 @@ #include #include #include +#include // --------------------------------------------------------------------------- // Helpers - physical window access // --------------------------------------------------------------------------- static uint64_t gPhysBase = PHYS_MAP_BASE; -static inline void* PhysToVirt(PHYSICAL_ADDRESS p) { +static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) { return (void *)(p + gPhysBase); } -static inline uint64_t* PtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) { - return (uint64_t *)PhysToVirt(tablePhys) + idx; +static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) { + return (uint64_t *)KiPhysToVirt(tablePhys) + idx; } // --------------------------------------------------------------------------- @@ -43,16 +44,16 @@ static PHYSICAL_ADDRESS gPml4Phys = 0; // Ensure a page-table entry at 'parentPhys[index]' points to a child table. // Allocates and zeros a new frame if the entry is not present. // Returns the physical address of the child table, or 0 on OOM. -static PHYSICAL_ADDRESS EnsureTable(PHYSICAL_ADDRESS parentPhys, +static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys, unsigned index, uint64_t extraFlags) { - uint64_t* entry = PtePtr(parentPhys, index); + uint64_t* entry = KiPtePtr(parentPhys, index); if (*entry & PAGE_PRESENT) { *entry |= extraFlags; return* entry & ~0xFFFULL; } PHYSICAL_ADDRESS childPhys = MmAllocateFrame(); if (!childPhys) return 0; - memset(PhysToVirt(childPhys), 0, PAGE_SIZE); + memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE); *entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags; return childPhys; } @@ -63,27 +64,27 @@ static PHYSICAL_ADDRESS EnsureTable(PHYSICAL_ADDRESS parentPhys, __declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags) { unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0; - PHYSICAL_ADDRESS pdpt = EnsureTable(gPml4Phys, PML4_IDX(virt), userFlag); + PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag); if (!pdpt) return STATUS_NO_MEMORY; - PHYSICAL_ADDRESS pdt = EnsureTable(pdpt, PDPT_IDX(virt), userFlag); + PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag); if (!pdt) return STATUS_NO_MEMORY; - PHYSICAL_ADDRESS pt = EnsureTable(pdt, PDT_IDX(virt), userFlag); + PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag); if (!pt) return STATUS_NO_MEMORY; - uint64_t* pte = PtePtr(pt, PT_IDX(virt)); + uint64_t* pte = KiPtePtr(pt, PT_IDX(virt)); *pte = phys | flags; __invlpg((void *)virt); return STATUS_SUCCESS; } __declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) { - uint64_t* e4 = PtePtr(gPml4Phys, PML4_IDX(virt)); + uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt)); if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND; - uint64_t* e3 = PtePtr(*e4 & ~0xFFFULL, PDPT_IDX(virt)); + uint64_t* e3 = KiPtePtr(*e4 & ~0xFFFULL, PDPT_IDX(virt)); if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND; - uint64_t* e2 = PtePtr(*e3 & ~0xFFFULL, PDT_IDX(virt)); + uint64_t* e2 = KiPtePtr(*e3 & ~0xFFFULL, PDT_IDX(virt)); if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND; - uint64_t* pte = PtePtr(*e2 & ~0xFFFULL, PT_IDX(virt)); + uint64_t* pte = KiPtePtr(*e2 & ~0xFFFULL, PT_IDX(virt)); *pte = 0; __invlpg((void *)virt); return STATUS_SUCCESS; @@ -97,13 +98,13 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK; for (uintptr_t va = base; va < end; va += PAGE_SIZE) { - uint64_t* e4 = PtePtr(gPml4Phys, PML4_IDX(va)); + uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va)); if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND; - uint64_t* e3 = PtePtr(*e4 & ~0xFFFULL, PDPT_IDX(va)); + uint64_t* e3 = KiPtePtr(*e4 & ~0xFFFULL, PDPT_IDX(va)); if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND; - uint64_t* e2 = PtePtr(*e3 & ~0xFFFULL, PDT_IDX(va)); + uint64_t* e2 = KiPtePtr(*e3 & ~0xFFFULL, PDT_IDX(va)); if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND; - uint64_t* pte = PtePtr(*e2 & ~0xFFFULL, PT_IDX(va)); + uint64_t* pte = KiPtePtr(*e2 & ~0xFFFULL, PT_IDX(va)); if (!(*pte & PAGE_PRESENT)) return STATUS_NOT_FOUND; // Preserve the physical address, replace everything else. @@ -123,10 +124,10 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) { uint64_t pd_idx = (virt >> 21) & 0x1FF; uint64_t pt_idx = (virt >> 12) & 0x1FF; - uint64_t* pml4 = PhysToVirt(gPml4Phys); + uint64_t* pml4 = KiPhysToVirt(gPml4Phys); if (!(pml4[pml4_idx] & PAGE_PRESENT)) return 0; - uint64_t* pdpt = PhysToVirt(pml4[pml4_idx] & ~0xFFFULL); + uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & ~0xFFFULL); if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) return 0; // Check for 1GB huge pages (bit 7 - PS) @@ -134,7 +135,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) { return (pdpt[pdpt_idx] & 0x000FFFFFC0000000ULL) + (virt & 0x3FFFFFFF); } - uint64_t* pd = PhysToVirt(pdpt[pdpt_idx] & ~0xFFFULL); + uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & ~0xFFFULL); if (!(pd[pd_idx] & PAGE_PRESENT)) return 0; // Check for 2MB huge pages (bit 7 - PS) @@ -142,7 +143,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) { return (pd[pd_idx] & 0x000FFFFFFFE00000ULL) + (virt & 0x1FFFFF); } - uint64_t* pt = PhysToVirt(pd[pd_idx] & ~0xFFFULL); + uint64_t* pt = KiPhysToVirt(pd[pd_idx] & ~0xFFFULL); if (!(pt[pt_idx] & PAGE_PRESENT)) return 0; // Extract the physical frame and add the offset within the page @@ -177,16 +178,16 @@ typedef struct BLOCK_FTR { #define OVERHEAD (HDR_SIZE + FTR_SIZE) #define MIN_BLOCK (OVERHEAD + sizeof(void*)) -static inline BLOCK_FTR* HdrToFtr(BLOCK_HDR* h) { +static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) { return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size); } -static inline BLOCK_HDR* FtrToHdr(BLOCK_FTR* f) { +static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) { return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size); } -static inline BLOCK_HDR* NextBlock(BLOCK_HDR* h) { +static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) { return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE); } -static inline BLOCK_FTR* PrevFtr(BLOCK_HDR* h) { +static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) { return (BLOCK_FTR *)((char *)h - FTR_SIZE); } @@ -200,7 +201,7 @@ typedef struct HEAP { } HEAP; // Extend heap by at least 'need' bytes (rounded up to page boundary). -static int HeapGrow(HEAP* h, size_t need) { +static int KiHeapGrow(HEAP* h, size_t need) { size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); if (h->Brk + grow > h->Limit) return 0; @@ -223,23 +224,23 @@ static int HeapGrow(HEAP* h, size_t need) { blk->NextFree = h->FreeList; if (h->FreeList) h->FreeList->PrevFree = blk; h->FreeList = blk; - HdrToFtr(blk)->Magic = BLOCK_MAGIC; - HdrToFtr(blk)->Free = 1; - HdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = 1; + KiHdrToFtr(blk)->Size = blk->Size; h->Brk += grow; return 1; } // Unlink a free block from the free list. -static void FreeListRemove(HEAP* h, BLOCK_HDR* blk) { +static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) { if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree; else h->FreeList = blk->NextFree; if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree; blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0; } -static void HeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) { +static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) { if (!blk || blk->Free == 0) return; while (h->Brk > h->Start) { @@ -264,13 +265,13 @@ static void HeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) { } if (blk->Free) { - HdrToFtr(blk)->Size = blk->Size; - HdrToFtr(blk)->Magic = BLOCK_MAGIC; - HdrToFtr(blk)->Free = 1; + KiHdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = 1; } } -static void* HeapAlloc(HEAP* h, size_t size) { +static void* KiHeapAlloc(HEAP* h, size_t size) { if (!size) return (void *)0; size = (size + 7) & ~7ULL; // 8-byte align @@ -286,11 +287,11 @@ static void* HeapAlloc(HEAP* h, size_t size) { } if (!best) { - if (!HeapGrow(h, size + OVERHEAD)) { + if (!KiHeapGrow(h, size + OVERHEAD)) { KeReleaseSpinlock(&h->Lock, flags); return (void *)0; } - // Re-search; HeapGrow inserted at the front. + // Re-search; KiHeapGrow inserted at the front. best = h->FreeList; while (best && best->Size < size) best = best->NextFree; if (!best) { @@ -299,7 +300,7 @@ static void* HeapAlloc(HEAP* h, size_t size) { } } - FreeListRemove(h, best); + KiFreeListRemove(h, best); // Split the block if the remainder is big enough. if (best->Size >= size + OVERHEAD + sizeof(void*)) { @@ -311,24 +312,24 @@ static void* HeapAlloc(HEAP* h, size_t size) { rem->NextFree = h->FreeList; if (h->FreeList) h->FreeList->PrevFree = rem; h->FreeList = rem; - HdrToFtr(rem)->Magic = BLOCK_MAGIC; - HdrToFtr(rem)->Free = 1; - HdrToFtr(rem)->Size = rem->Size; + KiHdrToFtr(rem)->Magic = BLOCK_MAGIC; + KiHdrToFtr(rem)->Free = 1; + KiHdrToFtr(rem)->Size = rem->Size; best->Size = size; } best->Free = 0; - HdrToFtr(best)->Free = 0; - HdrToFtr(best)->Magic = BLOCK_MAGIC; - HdrToFtr(best)->Size = best->Size; + KiHdrToFtr(best)->Free = 0; + KiHdrToFtr(best)->Magic = BLOCK_MAGIC; + KiHdrToFtr(best)->Size = best->Size; memset((char *)best + HDR_SIZE, 0, size); KeReleaseSpinlock(&h->Lock, flags); return (char *)best + HDR_SIZE; } -static void HeapFree(HEAP* h, void* ptr) { +static void KiHeapFree(HEAP* h, void* ptr) { if (!ptr) return; BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE); if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt @@ -338,25 +339,25 @@ static void HeapFree(HEAP* h, void* ptr) { blk->Free = 1; // Coalesce with next block if it is free. - BLOCK_HDR* next = NextBlock(blk); + BLOCK_HDR* next = KiNextBlock(blk); if ((uintptr_t)next < h->Brk && next->Magic == BLOCK_MAGIC && next->Free) { - FreeListRemove(h, next); + KiFreeListRemove(h, next); blk->Size += OVERHEAD + next->Size; - HdrToFtr(blk)->Size = blk->Size; - HdrToFtr(blk)->Magic = BLOCK_MAGIC; - HdrToFtr(blk)->Free = 1; + KiHdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = 1; } // Coalesce with previous block if it is free. if ((uintptr_t)blk > h->Start) { - BLOCK_FTR* prevFtr = PrevFtr(blk); + BLOCK_FTR* prevFtr = KiPrevFtr(blk); if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free) { - BLOCK_HDR* prev = FtrToHdr(prevFtr); - FreeListRemove(h, prev); + BLOCK_HDR* prev = KiFtrToHdr(prevFtr); + KiFreeListRemove(h, prev); prev->Size += OVERHEAD + blk->Size; - HdrToFtr(prev)->Size = prev->Size; - HdrToFtr(prev)->Magic = BLOCK_MAGIC; - HdrToFtr(prev)->Free = 1; + KiHdrToFtr(prev)->Size = prev->Size; + KiHdrToFtr(prev)->Magic = BLOCK_MAGIC; + KiHdrToFtr(prev)->Free = 1; blk = prev; } } @@ -367,23 +368,23 @@ static void HeapFree(HEAP* h, void* ptr) { if (h->FreeList) h->FreeList->PrevFree = blk; h->FreeList = blk; - HdrToFtr(blk)->Free = 1; - HdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = 1; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; - HeapTryDecommitTail(h, blk); + KiHeapTryDecommitTail(h, blk); KeReleaseSpinlock(&h->Lock, flags); } -static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) { +static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) { // If ptr is NULL, realloc acts like malloc if (!ptr) { - return HeapAlloc(h, new_size); + return KiHeapAlloc(h, new_size); } // If size is 0, realloc acts like free if (new_size == 0) { - HeapFree(h, ptr); + KiHeapFree(h, ptr); return (void*)0; } @@ -395,18 +396,18 @@ static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) { return (void*)0; } - // Align the requested size (matches HeapAlloc 8-byte alignment) + // Align the requested size (matches KiHeapAlloc 8-byte alignment) size_t aligned_size = (new_size + 7) & ~7ULL; // If the current block is already big enough, we can reuse it in-place. // (Note: To save space, you could optionally split the block here if - // the new size is significantly smaller, similar to HeapAlloc's logic). + // the new size is significantly smaller, similar to KiHeapAlloc's logic). if (blk->Size >= aligned_size) { return ptr; } // The block is too small. Allocate a new, larger block. - void* new_ptr = HeapAlloc(h, aligned_size); + void* new_ptr = KiHeapAlloc(h, aligned_size); if (!new_ptr) { return (void*)0; // Out of memory, original block is left untouched } @@ -417,7 +418,7 @@ static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) { memcpy(new_ptr, ptr, blk->Size); // Free the old block - HeapFree(h, ptr); + KiHeapFree(h, ptr); return new_ptr; } @@ -449,7 +450,7 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) { // Pre-commit the requested pool size. if (poolSize) { poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - HeapGrow(&gPoolHeap, poolSize); + KiHeapGrow(&gPoolHeap, poolSize); } // User heap. @@ -468,16 +469,27 @@ void MmActivate(void) { // --------------------------------------------------------------------------- // Public pool API // --------------------------------------------------------------------------- -__declspec(dllexport) void* MmAllocatePool(size_t size) { return HeapAlloc(&gPoolHeap, size); } -__declspec(dllexport) void MmFreePool(void* ptr) { HeapFree(&gPoolHeap, ptr); } -__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return HeapRealloc(&gPoolHeap, ptr, newSize); } +__declspec(dllexport) void* MmAllocatePool(size_t size) { return KiHeapAlloc(&gPoolHeap, size); } +__declspec(dllexport) void MmFreePool(void* ptr) { KiHeapFree(&gPoolHeap, ptr); } +__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); } // --------------------------------------------------------------------------- // Public user heap API // --------------------------------------------------------------------------- -__declspec(dllexport) void* MmAllocateUserMemory(size_t size) { return HeapAlloc(&gUserHeap, size); } -__declspec(dllexport) void MmFreeUserMemory(void* ptr, size_t size) { (void)size; HeapFree(&gUserHeap, ptr); } -__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { return HeapRealloc(&gUserHeap, ptr, newSize); } +__declspec(dllexport) void* MmAllocateUserMemory(size_t size) { + KeAssertIrql(PASSIVE_LEVEL); + return KiHeapAlloc(&gUserHeap, size); +} + +__declspec(dllexport) void MmFreeUserMemory(void* ptr) { + KeAssertIrql(PASSIVE_LEVEL); + KiHeapFree(&gUserHeap, ptr); +} + +__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { + KeAssertIrql(PASSIVE_LEVEL); + return KiHeapRealloc(&gUserHeap, ptr, newSize); +} size_t MmGetKernelPoolUsage(void) { // Returns bytes committed to the kernel pool diff --git a/code/kernel/mm_virt.h b/code/kernel/mm_virt.h index b335942..98fc0b9 100644 --- a/code/kernel/mm_virt.h +++ b/code/kernel/mm_virt.h @@ -1,73 +1,73 @@ -#pragma once -// mm_virt.h - virtual memory manager (VMM) -// Manages the single shared PML4, kernel pool, and user heap. -#include -#include -#include -#include - -// --------------------------------------------------------------------------- -// Page flags (x86-64 PTE bits) -// --------------------------------------------------------------------------- -#define PAGE_PRESENT (1ULL << 0) -#define PAGE_WRITE (1ULL << 1) -#define PAGE_USER (1ULL << 2) -#define PAGE_NOCACHE (1ULL << 4) -#define PAGE_NX (1ULL << 63) - -// --------------------------------------------------------------------------- -// Virtual address space constants -// --------------------------------------------------------------------------- -#define USER_BASE 0x0000000000001000ULL -#define USER_LIMIT 0x00007FFFFFFFF000ULL - -#define KERNEL_BASE 0xFFFF800000000000ULL -#define PHYS_MAP_BASE 0xFFFF880000000000ULL -#define POOL_BASE 0xFFFFC00000000000ULL -#define POOL_LIMIT 0xFFFFE00000000000ULL - -// --------------------------------------------------------------------------- -// Initialisation -// --------------------------------------------------------------------------- -// Called by KernelMain after MmInitializePhysical. -// bootPml4Phys: the PML4 set up by the bootloader. -// poolSize: initial reservation for kernel pool (bytes, page-aligned). -void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize); - -// Install the shared PML4 into CR3. Called once at the end of KernelMain init. -void MmActivate(void); - -// --------------------------------------------------------------------------- -// Page mapping -// --------------------------------------------------------------------------- -__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags); -__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt); - -// Modify protection flags on an already-mapped virtual range. -// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags -// entirely (except the physical address, which is preserved). -// Returns STATUS_NOT_FOUND if any page in the range is not mapped. -__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags); - -// Walk the page tables to translate a virtual address to physical. -// Returns 0 if the address is not mapped. -__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt); - -// --------------------------------------------------------------------------- -// Kernel pool - ring 0 only, never PAGE_USER. -// Best-fit free-list allocator, thread-safe via spinlock. -// --------------------------------------------------------------------------- -__declspec(dllexport) void* MmAllocatePool(size_t size); -__declspec(dllexport) void MmFreePool(void* ptr); -__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize); - -// --------------------------------------------------------------------------- -// User heap - shared flat space, ring 3 accessible. -// All allocations are PAGE_PRESENT | PAGE_WRITE | PAGE_USER. -// --------------------------------------------------------------------------- -__declspec(dllexport) void* MmAllocateUserMemory(size_t size); -__declspec(dllexport) void MmFreeUserMemory(void* ptr, size_t size); -__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize); - -size_t MmGetKernelPoolUsage(void); -size_t MmGetUserHeapUsage(void); +#pragma once +// mm_virt.h - virtual memory manager (VMM) +// Manages the single shared PML4, kernel pool, and user heap. +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Page flags (x86-64 PTE bits) +// --------------------------------------------------------------------------- +#define PAGE_PRESENT (1ULL << 0) +#define PAGE_WRITE (1ULL << 1) +#define PAGE_USER (1ULL << 2) +#define PAGE_NOCACHE (1ULL << 4) +#define PAGE_NX (1ULL << 63) + +// --------------------------------------------------------------------------- +// Virtual address space constants +// --------------------------------------------------------------------------- +#define USER_BASE 0x0000000000001000ULL +#define USER_LIMIT 0x00007FFFFFFFF000ULL + +#define KERNEL_BASE 0xFFFF800000000000ULL +#define PHYS_MAP_BASE 0xFFFF880000000000ULL +#define POOL_BASE 0xFFFFC00000000000ULL +#define POOL_LIMIT 0xFFFFE00000000000ULL + +// --------------------------------------------------------------------------- +// Initialisation +// --------------------------------------------------------------------------- +// Called by KernelMain after MmInitializePhysical. +// bootPml4Phys: the PML4 set up by the bootloader. +// poolSize: initial reservation for kernel pool (bytes, page-aligned). +void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize); + +// Install the shared PML4 into CR3. Called once at the end of KernelMain init. +void MmActivate(void); + +// --------------------------------------------------------------------------- +// Page mapping +// --------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags); +__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt); + +// Modify protection flags on an already-mapped virtual range. +// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags +// entirely (except the physical address, which is preserved). +// Returns STATUS_NOT_FOUND if any page in the range is not mapped. +__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags); + +// Walk the page tables to translate a virtual address to physical. +// Returns 0 if the address is not mapped. +__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt); + +// --------------------------------------------------------------------------- +// Kernel pool - ring 0 only, never PAGE_USER. +// Best-fit free-list allocator, thread-safe via spinlock. +// --------------------------------------------------------------------------- +__declspec(dllexport) void* MmAllocatePool(size_t size); +__declspec(dllexport) void MmFreePool(void* ptr); +__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize); + +// --------------------------------------------------------------------------- +// User heap - shared flat space, ring 3 accessible. +// All allocations are PAGE_PRESENT | PAGE_WRITE | PAGE_USER. +// --------------------------------------------------------------------------- +__declspec(dllexport) void* MmAllocateUserMemory(size_t size); +__declspec(dllexport) void MmFreeUserMemory(void* ptr); +__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize); + +size_t MmGetKernelPoolUsage(void); +size_t MmGetUserHeapUsage(void); diff --git a/code/kernel/ob_object.h b/code/kernel/ob_object.h index 5a79a4c..4ae0e6a 100644 --- a/code/kernel/ob_object.h +++ b/code/kernel/ob_object.h @@ -1,34 +1,35 @@ -#pragma once -// ob_object.h - executive object model -// Every resource the kernel tracks is a typed, reference-counted OBJECT. -// The OBJECT header must be the first member of any derived struct. -#include -#include -#include - -typedef enum OBJECT_TYPE { - ObjectTypeProcess = 1, - ObjectTypeThread = 2, - ObjectTypeMutex = 3, - ObjectTypeSemaphore = 4, - ObjectTypeEvent = 5, - ObjectTypeQueue = 6, - ObjectTypeFile = 7, - ObjectTypeDir = 8, -} OBJECT_TYPE; - -typedef struct OBJECT { - OBJECT_TYPE Type; - volatile long RefCount; // interlocked - void (*Destroy)(struct OBJECT* self); -} OBJECT; - -// Increment the reference count. Call whenever ownership is shared. -void ObReferenceObject(OBJECT* object); - -// Decrement. Calls Destroy() and frees the object when count reaches zero. -void ObDereferenceObject(OBJECT* object); - -// Allocate a zeroed object of the given size from kernel pool. -// Caller must fill Type and Destroy before publishing the pointer. -OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size); +#pragma once +// ob_object.h - executive object model +// Every resource the kernel tracks is a typed, reference-counted OBJECT. +// The OBJECT header must be the first member of any derived struct. +#include +#include +#include + +typedef enum OBJECT_TYPE { + ObjectTypeProcess = 1, + ObjectTypeThread = 2, + ObjectTypeMutex = 3, + ObjectTypeSemaphore = 4, + ObjectTypeEvent = 5, + ObjectTypeQueue = 6, + ObjectTypeFile = 7, + ObjectTypeDir = 8, + ObjectTypeAsyncIo = 9, +} OBJECT_TYPE; + +typedef struct OBJECT { + OBJECT_TYPE Type; + volatile long RefCount; // interlocked + void (*Destroy)(struct OBJECT* self); +} OBJECT; + +// Increment the reference count. Call whenever ownership is shared. +void ObReferenceObject(OBJECT* object); + +// Decrement. Calls Destroy() and frees the object when count reaches zero. +void ObDereferenceObject(OBJECT* object); + +// Allocate a zeroed object of the given size from kernel pool. +// Caller must fill Type and Destroy before publishing the pointer. +OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size); diff --git a/code/kernel/syscall_nr.h b/code/kernel/syscall_nr.h index a8b678c..8b1c0de 100644 --- a/code/kernel/syscall_nr.h +++ b/code/kernel/syscall_nr.h @@ -1,178 +1,85 @@ #pragma once -// syscall_nr.h - system call numbers -// Shared between kernel (ke_syscall.c) and user mode (syscall.dll). -// Keep in sync with gSyscallTable in ke_syscall.c and SYS_MAX below. -// -// Calling convention (INT 0x40, MS x64): +// syscall_nr.h +// INT 0x40 calling convention, MS x64: // RAX = syscall number -// RCX, RDX, R8, R9 = args 0-3 -// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6 in syscall_stubs.asm) -// Return value in RAX (KSTATUS or data). -// -// Handle semantics: -// Every kernel object (file, mutex, semaphore, event, queue) is reference- -// counted. SYS_CLOSE_HANDLE is the ONE way to close any handle regardless -// of the object type behind it. KeCloseHandle calls ObDereferenceObject, -// which fires the type-specific Destroy callback (e.g. FileDestroy) -// automatically when the refcount reaches zero. There is no separate -// SYS_CLOSE_FILE - that was redundant. +// RCX, RDX, R8, R9 = args 0-3 +// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6) +// RAX = return value -// --------------------------------------------------------------------------- -// Process & Thread -// --------------------------------------------------------------------------- -#define SYS_EXIT_PROCESS 0 // (int exitCode) -> void -#define SYS_EXIT_THREAD 1 // (int exitCode) -> void -#define SYS_CREATE_THREAD 2 // (uint64_t entry, uint64_t arg, uint32_t priority, uint32_t* tidOut) -> KSTATUS -#define SYS_GET_CURRENT_TID 3 // () -> uint32_t -#define SYS_GET_CURRENT_PID 4 // () -> uint32_t -#define SYS_YIELD 5 // () -> void -#define SYS_SLEEP 6 // (uint32_t ms) -> void -#define SYS_SET_THREAD_PRIORITY 7 // (uint32_t priority) -> KSTATUS +#define SYS_EXIT_PROCESS 0 +#define SYS_EXIT_THREAD 1 +#define SYS_CREATE_PROCESS 2 +#define SYS_WAIT_FOR_PROCESS 3 +#define SYS_GET_PROCESS_EXIT_CODE 4 +#define SYS_KILL_PROCESS 5 +#define SYS_GET_PROCESS_LIST 6 +#define SYS_GET_PROCESS_PID 7 +#define SYS_CREATE_THREAD 8 +#define SYS_WAIT_THREAD 9 +#define SYS_GET_CURRENT_TID 10 +#define SYS_GET_CURRENT_PID 11 +#define SYS_SET_THREAD_PRIORITY 12 +#define SYS_YIELD 13 +#define SYS_SLEEP 14 +#define SYS_SET_USER_APC_DISPATCHER 15 +#define SYS_APC_RETURN 16 +#define SYS_ALLOC 17 +#define SYS_FREE 18 +#define SYS_REALLOC 19 +#define SYS_CLOSE_HANDLE 20 +#define SYS_MUTEX_CREATE 21 +#define SYS_MUTEX_ACQUIRE 22 +#define SYS_MUTEX_TRY_ACQUIRE 23 +#define SYS_MUTEX_RELEASE 24 +#define SYS_SEM_CREATE 25 +#define SYS_SEM_WAIT 26 +#define SYS_SEM_TRY_WAIT 27 +#define SYS_SEM_SIGNAL 28 +#define SYS_EVENT_CREATE 29 +#define SYS_EVENT_SIGNAL 30 +#define SYS_EVENT_CLEAR 31 +#define SYS_EVENT_WAIT 32 +#define SYS_QUEUE_CREATE 33 +#define SYS_QUEUE_SEND 34 +#define SYS_QUEUE_RECV 35 +#define SYS_PUBLISH_QUEUE 36 +#define SYS_OPEN_NAMED_QUEUE 37 +#define SYS_QUEUE_DUPLICATE 38 +#define SYS_NAMED_QUEUE_OWNER_PID 39 +#define SYS_OPEN_FILE 40 +#define SYS_CREATE_FILE 41 +#define SYS_READ_FILE 42 +#define SYS_WRITE_FILE 43 +#define SYS_READ_FILE_ASYNC 44 +#define SYS_WRITE_FILE_ASYNC 45 +#define SYS_CANCEL_IO 46 +#define SYS_SEEK_FILE 47 +#define SYS_GET_FILE_POS 48 +#define SYS_GET_FILE_SIZE 49 +#define SYS_TRUNCATE_FILE 50 +#define SYS_DELETE_FILE 51 +#define SYS_RENAME_FILE 52 +#define SYS_OPEN_DIR 53 +#define SYS_READ_DIR 54 +#define SYS_CREATE_DIR 55 +#define SYS_REMOVE_DIR 56 +#define SYS_SET_CWD 57 +#define SYS_GET_CWD 58 +#define SYS_WRITE_CONSOLE 59 +#define SYS_SCREEN_BLIT 60 +#define SYS_SCREEN_CLEAR 61 +#define SYS_SCREEN_GET_SIZE 62 +#define SYS_SCREEN_PUT_CHAR 63 +#define SYS_KB_POLL 64 +#define SYS_KB_IS_KEY_DOWN 65 +#define SYS_KB_READ_CHAR 66 +#define SYS_KB_READ_SCANCODE 67 +#define SYS_KB_GET_MODIFIERS 68 +#define SYS_MOUSE_POLL 69 +#define SYS_MOUSE_READ 70 +#define SYS_MOUSE_GET_POS 71 +#define SYS_GET_TICK 72 +#define SYS_DEBUG_PRINT 73 +#define SYS_QUERY_TRAP_FRAME 74 -// --------------------------------------------------------------------------- -// Memory -// --------------------------------------------------------------------------- -#define SYS_ALLOC 8 // (size_t size) -> void* (NULL on failure) -#define SYS_FREE 9 // (void* ptr, size_t size) -> void -#define SYS_REALLOC 10 // (void* ptr, size_t newSize) -> void* - -// --------------------------------------------------------------------------- -// Synchronisation Objects -// --------------------------------------------------------------------------- -#define SYS_MUTEX_CREATE 11 // (uint32_t* handleOut) -> KSTATUS -#define SYS_MUTEX_ACQUIRE 12 // (uint32_t handle) -> KSTATUS -#define SYS_MUTEX_TRY_ACQUIRE 13 // (uint32_t handle) -> KSTATUS -#define SYS_MUTEX_RELEASE 14 // (uint32_t handle) -> KSTATUS - -#define SYS_SEM_CREATE 15 // (int32_t initial, int32_t max, uint32_t* handleOut) -> KSTATUS -#define SYS_SEM_WAIT 16 // (uint32_t handle) -> KSTATUS -#define SYS_SEM_TRY_WAIT 17 // (uint32_t handle) -> KSTATUS -#define SYS_SEM_SIGNAL 18 // (uint32_t handle) -> KSTATUS - -#define SYS_EVENT_CREATE 19 // (uint32_t* handleOut) -> KSTATUS -#define SYS_EVENT_SIGNAL 20 // (uint32_t handle) -> KSTATUS -#define SYS_EVENT_CLEAR 21 // (uint32_t handle) -> KSTATUS -#define SYS_EVENT_WAIT 22 // (uint32_t handle) -> KSTATUS - -// --------------------------------------------------------------------------- -// Queue -// --------------------------------------------------------------------------- -#define SYS_QUEUE_CREATE 23 // (uint32_t* handleOut) -> KSTATUS -#define SYS_QUEUE_SEND 24 // (uint32_t handle, void* msg, int blocking) -> KSTATUS -#define SYS_QUEUE_RECV 25 // (uint32_t handle, void* msg, int blocking) -> KSTATUS - -// --------------------------------------------------------------------------- -// Handle (unified close for all object types) -// --------------------------------------------------------------------------- -#define SYS_CLOSE_HANDLE 26 // (uint32_t handle) -> KSTATUS - -// --------------------------------------------------------------------------- -// File I/O -// (Close a file handle via SYS_CLOSE_HANDLE, same as any other object.) -// --------------------------------------------------------------------------- -#define SYS_OPEN_FILE 27 // (const char* path, uint32_t* handleOut) -> KSTATUS -#define SYS_READ_FILE 28 // (uint32_t handle, void* buf, size_t size, size_t* bytesReadOut) -> KSTATUS -#define SYS_WRITE_FILE 29 // (uint32_t handle, const void* buf, size_t size, size_t* bytesWrittenOut) -> KSTATUS -#define SYS_CREATE_FILE 30 // (const char* path, uint32_t* handleOut) -> KSTATUS -#define SYS_DELETE_FILE 31 // (const char* path) -> KSTATUS -#define SYS_TRUNCATE_FILE 32 // (uint32_t handle, uint64_t newSize) -> KSTATUS -#define SYS_GET_FILE_POS 33 // (uint32_t handle) -> uint64_t -#define SYS_SEEK_FILE 34 // (uint32_t handle, uint64_t offset) -> KSTATUS -#define SYS_GET_FILE_SIZE 35 // (uint32_t handle) -> uint64_t - -// --------------------------------------------------------------------------- -// Screen / Graphics -// --------------------------------------------------------------------------- -#define SYS_SCREEN_BLIT 36 // (int x, int y, int w, int h, const uint32_t* pixels) -> void -#define SYS_SCREEN_CLEAR 37 // (uint32_t color) -> void -#define SYS_SCREEN_GET_SIZE 38 // (int* widthOut, int* heightOut) -> void -#define SYS_SCREEN_PUT_CHAR 39 // (char c, int x, int y, uint32_t fg, uint32_t bg) -> void - -// --------------------------------------------------------------------------- -// Keyboard -// --------------------------------------------------------------------------- -#define SYS_KB_POLL 40 // () -> uint32_t (1 if key available, 0 otherwise) -#define SYS_KB_IS_KEY_DOWN 41 // (int scancode) -> uint32_t (1 if down, 0 otherwise) -#define SYS_KB_READ_CHAR 42 // () -> char (blocks, yields while waiting) -#define SYS_KB_READ_SCANCODE 43 // () -> uint8_t (blocks, yields while waiting) -#define SYS_KB_GET_MODIFIERS 44 // () -> uint32_t (current modifier bitmask) - -// --------------------------------------------------------------------------- -// Mouse -// --------------------------------------------------------------------------- -#define SYS_MOUSE_POLL 45 // () -> uint32_t (1 if packet waiting, 0 otherwise) -#define SYS_MOUSE_READ 46 // (int* dx, int* dy, uint8_t* buttons) -> uint32_t (1=ok, 0=empty) -#define SYS_MOUSE_GET_POS 47 // (int* x, int* y) -> void - -// --------------------------------------------------------------------------- -// Timer -// --------------------------------------------------------------------------- -#define SYS_GET_TICK 48 // () -> uint64_t (milliseconds since boot) - -// --------------------------------------------------------------------------- -// Debug -// --------------------------------------------------------------------------- -#define SYS_DEBUG_PRINT 49 // (const char* str) -> void - -// --------------------------------------------------------------------------- -// Process launch & lifecycle -// --------------------------------------------------------------------------- -#define SYS_CREATE_PROCESS 50 // (const char* path, const char* args, uint32_t* pidOut) -> KSTATUS -#define SYS_WAIT_FOR_PROCESS 51 // (uint32_t pid) -> KSTATUS -#define SYS_GET_PROCESS_EXIT_CODE 52 // (uint32_t pid, int* exitCodeOut) -> KSTATUS - -// --------------------------------------------------------------------------- -// Console output (white text, for user programs) -// --------------------------------------------------------------------------- -#define SYS_WRITE_CONSOLE 53 // (const char* str, size_t len) -> void - -// --------------------------------------------------------------------------- -// Directory enumeration -// --------------------------------------------------------------------------- -#define SYS_OPEN_DIR 54 // (const char* path, uint32_t* handleOut) -> KSTATUS -#define SYS_READ_DIR 55 // (uint32_t handle, DirEntry* entryOut) -> KSTATUS - -// --------------------------------------------------------------------------- -// Current Working Directory -// --------------------------------------------------------------------------- -#define SYS_SET_CWD 56 // (const char* path) -> KSTATUS -#define SYS_GET_CWD 57 // (char* buf, size_t buflen) -> KSTATUS - -// --------------------------------------------------------------------------- -// Thread join (used by pthread_join in posix.dll) -// --------------------------------------------------------------------------- -#define SYS_WAIT_THREAD 58 // (uint32_t tid) -> KSTATUS - -// --------------------------------------------------------------------------- -// Process PID query (used by posix_spawn -> waitpid in posix.dll) -// --------------------------------------------------------------------------- -#define SYS_GET_PROCESS_PID 59 // (uint32_t pid) -> KSTATUS - -// --------------------------------------------------------------------------- -// Named Queue Registry -// SysPublishQueue - register a queue under a name (call once at startup) -// SysOpenNamedQueue - resolve a name to a handle (retry until SUCCESS) -// --------------------------------------------------------------------------- -#define SYS_PUBLISH_QUEUE 60 // (const char* name, uint32_t queueHandle) -> KSTATUS -#define SYS_OPEN_NAMED_QUEUE 61 // (const char* name, uint32_t* handleOut) -> KSTATUS - -// --------------------------------------------------------------------------- -// Handle duplication -// --------------------------------------------------------------------------- -#define SYS_QUEUE_DUPLICATE 62 // (uint32_t handle, uint32_t targetPid, uint32_t* handleOut) -> KSTATUS -#define SYS_NAMED_QUEUE_OWNER_PID 63 // (const char* name, uint32_t* pidOut) -> KSTATUS - -// --------------------------------------------------------------------------- -// Process utilities -// --------------------------------------------------------------------------- -#define SYS_KILL_PROCESS 64 // (uint32_t pid, int exitCode) -> KSTATUS -#define SYS_GET_PROCESS_LIST 65 // (uint32_t* pidsOut, size_t maxCount, size_t* countOut) -> KSTATUS - -// --------------------------------------------------------------------------- -// Filesystem mutations -// --------------------------------------------------------------------------- -#define SYS_RENAME_FILE 66 // (const char* oldPath, const char* newPath) -> KSTATUS -#define SYS_CREATE_DIR 67 // (const char* path) -> KSTATUS -#define SYS_REMOVE_DIR 68 // (const char* path) -> KSTATUS - -#define SYS_MAX 69 +#define SYS_MAX 75 diff --git a/code/posix/posix_dir.c b/code/posix/posix_dir.c index 40b64b4..543202d 100644 --- a/code/posix/posix_dir.c +++ b/code/posix/posix_dir.c @@ -65,6 +65,6 @@ __declspec(dllexport) struct dirent* readdir(DIR* d) { __declspec(dllexport) int closedir(DIR* d) { if (!d) { errno = EBADF; return -1; } SysCloseHandle(d->KHandle); - SysFree(d, sizeof(DIR)); + SysFree(d); return 0; } diff --git a/code/posix/posix_path.c b/code/posix/posix_path.c index 974deb1..db2b795 100644 --- a/code/posix/posix_path.c +++ b/code/posix/posix_path.c @@ -3,29 +3,36 @@ #include #include "posix_path.h" -// The drive letter used as the POSIX filesystem root. -// A future SYS_GET_BOOT_DRIVE could populate this; hardcoded for v1. -static const char gRootDrive = '0'; - static void FlipSlashes(char* s) { for (; *s; s++) if (*s == '/') *s = '\\'; } +static int IsDriveMount(const char* p) { + return p && p[0] == '/' && p[1] >= '0' && p[1] <= '7' && + (p[2] == '\0' || p[2] == '/'); +} + const char* PosixTranslatePath(const char* in, char* out, size_t outSize) { if (!in || !out || outSize < 4) return out; - // Synthetic devices if (strcmp(in, "/dev/null") == 0) return SYNTH_DEVNULL; if (strcmp(in, "/dev/zero") == 0) return SYNTH_DEVZERO; - // /proc/self/* - read-only synthesized files; flag with a sentinel so - // posix_io.c can intercept them without opening a real file. - // (In v1 only /proc/self/pid is handled; others fall through to ENOENT.) + if (IsDriveMount(in)) { + out[0] = in[1]; + out[1] = ':'; + out[2] = '/'; + const char* rest = (in[2] == '/') ? in + 3 : ""; + size_t restLen = strlen(rest); + if (restLen + 4 > outSize) restLen = outSize - 4; + memcpy(out + 3, rest, restLen); + out[3 + restLen] = '\0'; + FlipSlashes(out); + return out; + } if (in[0] == '/') { - // Absolute POSIX path -> C:\... - // outSize must fit "C:\" + rest - out[0] = gRootDrive; + out[0] = '0'; out[1] = ':'; out[2] = '/'; size_t rest = strlen(in + 1); @@ -33,7 +40,6 @@ const char* PosixTranslatePath(const char* in, char* out, size_t outSize) { memcpy(out + 3, in + 1, rest); out[3 + rest] = '\0'; } else { - // Relative - pass through with slash flip only size_t len = strlen(in); if (len + 1 > outSize) len = outSize - 1; memcpy(out, in, len); @@ -47,16 +53,27 @@ const char* PosixTranslatePath(const char* in, char* out, size_t outSize) { void NativeToPosix(const char* in, char* out, size_t outSize) { if (!in || !out || outSize == 0) return; - // Strip "X:\" prefix if present const char* src = in; - if (src[0] != '\0' && src[1] == ':' && (src[2] == '\\' || src[2] == '/')) - src += 2; // skip "C:", leave the leading backslash + char drive = 0; + if (src[0] >= '0' && src[0] <= '7' && src[1] == ':' && + (src[2] == '\\' || src[2] == '/')) { + drive = src[0]; + src += 3; + } - size_t len = strlen(src); - if (len + 1 > outSize) len = outSize - 1; - memcpy(out, src, len); - out[len] = '\0'; + size_t pos = 0; + if (drive && drive != '0') { + if (outSize < 4) { out[0] = '\0'; return; } + out[pos++] = '/'; + out[pos++] = drive; + if (*src) out[pos++] = '/'; + } else { + out[pos++] = '/'; + } - // Backslash -> forward slash - for (char* p = out; *p; p++) if (*p == '\\') *p = '/'; + while (*src && pos + 1 < outSize) { + out[pos++] = (*src == '\\') ? '/' : *src; + src++; + } + out[pos] = '\0'; } diff --git a/code/posix/posix_thread.c b/code/posix/posix_thread.c index c2b8444..47e2296 100644 --- a/code/posix/posix_thread.c +++ b/code/posix/posix_thread.c @@ -56,7 +56,7 @@ static void PthreadTrampoline(uintptr_t raw) { PTHREAD_SLOT* slot = PthreadSlotFind(pid, tid); if (!slot) slot = PthreadSlotAlloc(pid, tid); - SysFree(tr, sizeof(PTHREAD_TRAMP)); + SysFree(tr); void* retval = fn(arg); @@ -86,7 +86,7 @@ __declspec(dllexport) int pthread_create(pthread_t* t, const pthread_attr_t* att uint32_t s = SysCreateThread((uint64_t)PthreadTrampoline, (uint64_t)tr, 1 /*PriorityNormal*/, &tid); if (s != 0) { - SysFree(tr, sizeof(PTHREAD_TRAMP)); + SysFree(tr); return EAGAIN; } // The trampoline allocates the retval slot itself after it knows its real diff --git a/code/shell_native/shell_native.c b/code/shell_native/shell_native.c index 1154d5c..77db322 100644 --- a/code/shell_native/shell_native.c +++ b/code/shell_native/shell_native.c @@ -89,9 +89,6 @@ static void cmd_dir(const char* path) { KDIR_ENTRY ke; while (SysReadDir(hDir, &ke) == 0) { - // Skip hidden/system markers if desired (e.g., those starting with .) - if (ke.Name[0] == '.') continue; - // Corrected Attribute Check: 0x10 is the Directory bit in FAT if (ke.Attr & 0x10) { print(" "); @@ -173,7 +170,7 @@ static int runCommand(const char* cmd) { } int main(void) { - const char* cmdline = SysGetCommandLine(); + const char* cmdline = SysGetPeb()->CommandLine; char lineBuf[MAX_CMD]; print("shell: command line: "); diff --git a/code/shell_posix/shell_posix.c b/code/shell_posix/shell_posix.c index b8be491..576e807 100644 --- a/code/shell_posix/shell_posix.c +++ b/code/shell_posix/shell_posix.c @@ -119,7 +119,6 @@ static void cmd_dir(const char* path) { if (!d) { print("dir: cannot access directory\n"); return; } dirent* entry; while ((entry = readdir(d)) != NULL) { - if (entry->d_name[0] == '.') continue; print(entry->d_type == 1 ? " " : " "); print(entry->d_name); print("\n"); } @@ -166,7 +165,7 @@ static int runCommand(const char* cmd) { int main(void) { PosixFdInit(); - const char* cmdline = SysGetCommandLine(); + const char* cmdline = SysGetPeb()->CommandLine; char lineBuf[MAX_CMD]; print("shell: command line: "); diff --git a/code/shell_win32/shell_win32.c b/code/shell_win32/shell_win32.c index c930b9a..c395bb7 100644 --- a/code/shell_win32/shell_win32.c +++ b/code/shell_win32/shell_win32.c @@ -102,7 +102,6 @@ static void cmd_dir(const char* path) { HANDLE hFind = FindFirstFileA(search_path, &fd); if (hFind == INVALID_HANDLE_VALUE) { print("dir: failed to read directory\n"); return; } do { - if (fd.cFileName[0] == '.') continue; // Skip . and .. print(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ? " " : " "); print(fd.cFileName); print("\n"); } while (FindNextFileA(hFind, &fd)); @@ -151,7 +150,7 @@ int main(void) { hIn = GetStdHandle(STD_INPUT_HANDLE); hOut = GetStdHandle(STD_OUTPUT_HANDLE); - const char* cmdline = SysGetCommandLine(); + const char* cmdline = GetCommandLineA(); char lineBuf[MAX_CMD]; print("shell: command line: "); diff --git a/code/syscall/syscall.h b/code/syscall/syscall.h index 29e0353..843fd88 100644 --- a/code/syscall/syscall.h +++ b/code/syscall/syscall.h @@ -6,6 +6,12 @@ #include #include +#define SYS_STATUS_SUCCESS 0u +#define SYS_STATUS_PENDING 4u +#define SYS_STATUS_ALERTED 5u +#define SYS_STATUS_KERNEL_APC 6u +#define SYS_STATUS_USER_APC 7u +#define SYS_STATUS_CANCELLED 0x33u // --------------------------------------------------------------------------- // User process/thread environment @@ -15,7 +21,7 @@ #define SYS_PEB_MAGIC 0x30424550u /* "PEB0" */ #define SYS_TEB_MAGIC 0x30424554u /* "TEB0" */ #define SYS_ENV_VERSION 1u -#define SYS_CWD_MAX 512u +#define SYS_CWD_MAX 520u typedef struct SYS_PEB { uint32_t Magic; @@ -49,121 +55,181 @@ static inline SYS_TEB* SysGetTeb(void) { return (SYS_TEB*)(uintptr_t)__readgsqword(offsetof(SYS_TEB, Self)); } -static inline const char* SysGetCommandLine(void) { - SYS_PEB* peb = SysGetPeb(); - if (!peb || peb->Magic != SYS_PEB_MAGIC) return ""; - return peb->CommandLine ? peb->CommandLine : ""; -} +// --------------------------------------------------------------------------- +// APC and async I/O ABI +// --------------------------------------------------------------------------- +typedef struct SYS_IO_STATUS_BLOCK { + volatile uint32_t Status; + volatile uint64_t Information; +} SYS_IO_STATUS_BLOCK; + +typedef void (*SYS_USER_APC_ROUTINE)(void* normalContext, + SYS_IO_STATUS_BLOCK* iosb, + uint64_t information); + +typedef struct SYS_ASYNC_FILE_REQUEST { + uint32_t FileHandle; + uint32_t EventHandle; // optional; 0 means no event signal + uint32_t Flags; // reserved, must be 0 for now + uint32_t Reserved; + uint64_t Offset; // explicit byte offset; does not touch file pointer + void* Buffer; + size_t Size; + SYS_IO_STATUS_BLOCK* Iosb; // required; initialized to SYS_STATUS_PENDING + SYS_USER_APC_ROUTINE ApcRoutine; // optional; requires SysSetUserApcDispatcher + void* ApcContext; + uint32_t* RequestHandleOut; // optional; receives cancel/query handle +} SYS_ASYNC_FILE_REQUEST; + +typedef struct SYS_USER_APC_FRAME { + uint64_t Magic; + uint64_t OriginalRip; + uint64_t OriginalRsp; + uint64_t OriginalRflags; + uint64_t OriginalRax; + uint64_t Routine; + uint64_t NormalContext; + uint64_t SystemArgument1; + uint64_t SystemArgument2; +} SYS_USER_APC_FRAME; + +#define SYS_USER_APC_FRAME_MAGIC 0x3143504152555355ULL + +void SysUserApcDispatcher(void); + // --------------------------------------------------------------------------- -// Thread & Process Management (0-7) +// Trap-frame query ABI // --------------------------------------------------------------------------- -void SysExitProcess(int exitCode); // 00 -void SysExitThread(int exitCode); // 01 -uint32_t SysCreateThread(uintptr_t entry, uintptr_t arg, uint32_t priority, uint32_t* tidOut); // 02 -uint32_t SysGetCurrentTid(void); // 03 -uint32_t SysGetCurrentPid(void); // 04 -uint32_t SysYield(void); // 05 -uint32_t SysSleep(uint64_t ms); // 06 -uint32_t SysSetThreadPriority(uint32_t priority); // 07 +typedef struct SYS_TRAP_FRAME_INFO { + uint32_t Size; + uint32_t Kind; + uint32_t Vector; + uint32_t PreviousMode; + + uint64_t Rip; + uint64_t Cs; + uint64_t Rflags; + uint64_t Rsp; + uint64_t Ss; + + uint64_t Rax; + uint64_t Rcx; + uint64_t Rdx; + uint64_t R8; + uint64_t R9; + uint64_t R10; + uint64_t R11; +} SYS_TRAP_FRAME_INFO; // --------------------------------------------------------------------------- -// Memory Management (8-10) +// Process & thread lifecycle (0-14) // --------------------------------------------------------------------------- -void* SysAlloc(size_t size); // 08 -uint32_t SysFree(void* ptr, size_t size); // 09 -void* SysRealloc(void* ptr, size_t newSize); // 10 +void SysExitProcess(int exitCode); +void SysExitThread(int exitCode); +uint32_t SysCreateProcess(const char* path, uint32_t* handleOut, const char* cmdline); +uint32_t SysWaitForProcess(uint32_t handle); +uint32_t SysGetProcessExitCode(uint32_t handle, int* exitCodeOut); +uint32_t SysKillProcess(uint32_t pid, int exitCode); +uint32_t SysGetProcessList(uint32_t* pidsOut, size_t maxCount, size_t* countOut); +uint32_t SysGetProcessPid(uint32_t handle, uint32_t* pidOut); + +uint32_t SysCreateThread(uintptr_t entry, uintptr_t arg, uint32_t priority, uint32_t* tidOut); +uint32_t SysWaitThread(uint32_t tid); +uint32_t SysGetCurrentTid(void); +uint32_t SysGetCurrentPid(void); +uint32_t SysSetThreadPriority(uint32_t priority); +uint32_t SysYield(void); +uint32_t SysSleep(uint64_t ms); +uint32_t SysSetUserApcDispatcher(void* dispatcher); +uint32_t SysApcReturn(const SYS_USER_APC_FRAME* frame); // --------------------------------------------------------------------------- -// Synchronization Primitives (11-22) +// Memory and handles // --------------------------------------------------------------------------- -uint32_t SysMutexCreate(uint32_t* handleOut); // 11 -uint32_t SysMutexAcquire(uint32_t handle); // 12 -uint32_t SysMutexTryAcquire(uint32_t handle); // 13 -uint32_t SysMutexRelease(uint32_t handle); // 14 - -uint32_t SysSemCreate(int32_t initial, int32_t max, uint32_t* handleOut); // 15 -uint32_t SysSemWait(uint32_t handle); // 16 -uint32_t SysSemTryWait(uint32_t handle); // 17 -uint32_t SysSemSignal(uint32_t handle); // 18 - -uint32_t SysEventCreate(uint32_t type, uint32_t initialState, uint32_t* handleOut); // 19 -uint32_t SysEventSignal(uint32_t handle); // 20 -uint32_t SysEventClear(uint32_t handle); // 21 -uint32_t SysEventWait(uint32_t handle); // 22 +void* SysAlloc(size_t size); +uint32_t SysFree(void* ptr); +void* SysRealloc(void* ptr, size_t newSize); +uint32_t SysCloseHandle(uint32_t handle); // --------------------------------------------------------------------------- -// Anonymous Queues & Unified Handle Path (23-26) +// Synchronization primitives (19-31) // --------------------------------------------------------------------------- -uint32_t SysQueueCreate(uint32_t* handleOut); // 23 -uint32_t SysQueueSend(uint32_t handle, const void* msg, bool blocking); // 24 -uint32_t SysQueueRecv(uint32_t handle, void* msg, bool blocking); // 25 -uint32_t SysCloseHandle(uint32_t handle); // 26 +uint32_t SysMutexCreate(uint32_t* handleOut); +uint32_t SysMutexAcquire(uint32_t handle); +uint32_t SysMutexTryAcquire(uint32_t handle); +uint32_t SysMutexRelease(uint32_t handle); + +uint32_t SysSemCreate(int32_t initial, int32_t max, uint32_t* handleOut); +uint32_t SysSemWait(uint32_t handle); +uint32_t SysSemTryWait(uint32_t handle); +uint32_t SysSemSignal(uint32_t handle); + +uint32_t SysEventCreate(uint32_t type, uint32_t initialState, uint32_t* handleOut); +uint32_t SysEventSignal(uint32_t handle); +uint32_t SysEventClear(uint32_t handle); +uint32_t SysEventWait(uint32_t handle, uint32_t alertable); // --------------------------------------------------------------------------- -// File System (27-35) +// Queues and named queues (32-38) // --------------------------------------------------------------------------- -uint32_t SysOpenFile(const char* path, uint32_t* handleOut); // 27 -uint32_t SysReadFile(uint32_t handle, void* buf, size_t size, size_t* bytesRead); // 28 -uint32_t SysWriteFile(uint32_t handle, const void* buf, size_t size, size_t* written); // 29 -uint32_t SysCreateFile(const char* path, uint32_t* handleOut); // 30 -uint32_t SysDeleteFile(const char* path); // 31 -uint32_t SysTruncateFile(uint32_t handle, uint64_t newSize); // 32 -uint64_t SysGetFilePos(uint32_t handle); // 33 -uint32_t SysSeekFile(uint32_t handle, uint64_t offset); // 34 -uint64_t SysGetFileSize(uint32_t handle); // 35 +uint32_t SysQueueCreate(uint32_t* handleOut); +uint32_t SysQueueSend(uint32_t handle, const void* msg, bool blocking); +uint32_t SysQueueRecv(uint32_t handle, void* msg, bool blocking); +uint32_t SysPublishQueue(const char* name, uint32_t handle); +uint32_t SysOpenNamedQueue(const char* name, uint32_t* handleOut); +uint32_t SysQueueDuplicate(uint32_t handle, uint32_t targetPid, uint32_t* handleOut); +uint32_t SysNamedQueueOwnerPid(const char* name, uint32_t* pidOut); // --------------------------------------------------------------------------- -// Graphics & Input (36-47) +// Files, directories, and current directory (39-56) // --------------------------------------------------------------------------- -uint32_t SysScreenBlit(int x, int y, int w, int h, const uint32_t* pixels); // 36 -uint32_t SysScreenClear(uint32_t color); // 37 -uint32_t SysScreenGetSize(int* widthOut, int* heightOut); // 38 -uint32_t SysScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg); // 39 +uint32_t SysOpenFile(const char* path, uint32_t* handleOut); +uint32_t SysCreateFile(const char* path, uint32_t* handleOut); +uint32_t SysReadFile(uint32_t handle, void* buf, size_t size, size_t* bytesRead); +uint32_t SysWriteFile(uint32_t handle, const void* buf, size_t size, size_t* written); +uint32_t SysReadFileAsync(const SYS_ASYNC_FILE_REQUEST* request); +uint32_t SysWriteFileAsync(const SYS_ASYNC_FILE_REQUEST* request); +uint32_t SysCancelIo(uint32_t asyncRequestHandle); +uint32_t SysSeekFile(uint32_t handle, uint64_t offset); +uint64_t SysGetFilePos(uint32_t handle); +uint64_t SysGetFileSize(uint32_t handle); +uint32_t SysTruncateFile(uint32_t handle, uint64_t newSize); +uint32_t SysDeleteFile(const char* path); +uint32_t SysRenameFile(const char* oldPath, const char* newPath); -uint64_t SysKbPoll(void); // 40 -uint64_t SysKbIsKeyDown(uint8_t scancode, bool extended); // 41 -char SysKbReadChar(void); // 42 -uint8_t SysKbReadScancode(void); // 43 -uint32_t SysKbGetModifiers(void); // 44 - -uint64_t SysMousePoll(void); // 45 -uint64_t SysMouseRead(int* dx, int* dy, uint8_t* buttons); // 46 -void SysMouseGetPos(int* x, int* y); // 47 +uint32_t SysOpenDir(const char* path, uint32_t* handleOut); +uint32_t SysReadDir(uint32_t handle, void* dirEntryBuffer); +uint32_t SysCreateDir(const char* path); +uint32_t SysRemoveDir(const char* path); +uint32_t SysSetCwd(const char* path); +uint32_t SysGetCwd(char* buf, size_t size); // --------------------------------------------------------------------------- -// System Info & Process Management (48-65) +// Console and graphics (57-61) // --------------------------------------------------------------------------- -uint64_t SysGetTick(void); // 48 -uint32_t SysDebugPrint(const char* str); // 49 -uint32_t SysCreateProcess(const char* path, uint32_t* handleOut, const char* cmdline); // 50 -uint32_t SysWaitForProcess(uint32_t handle); // 51 -uint32_t SysGetProcessExitCode(uint32_t handle, int* exitCodeOut); // 52 -uint32_t SysWriteConsole(const char* buf, size_t len); // 53 +uint32_t SysWriteConsole(const char* buf, size_t len); +uint32_t SysScreenBlit(int x, int y, int w, int h, const uint32_t* pixels); +uint32_t SysScreenClear(uint32_t color); +uint32_t SysScreenGetSize(int* widthOut, int* heightOut); +uint32_t SysScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg); // --------------------------------------------------------------------------- -// Directories & Environment (54-57) +// Input (62-69) // --------------------------------------------------------------------------- -uint32_t SysOpenDir(const char* path, uint32_t* handleOut); // 54 -uint32_t SysReadDir(uint32_t handle, void* dirEntryBuffer); // 55 -uint32_t SysSetCwd(const char* path); // 56 -uint32_t SysGetCwd(char* buf, size_t size); // 57 +uint64_t SysKbPoll(void); +uint64_t SysKbIsKeyDown(uint8_t scancode, bool extended); +char SysKbReadChar(void); +uint8_t SysKbReadScancode(void); +uint32_t SysKbGetModifiers(void); + +uint64_t SysMousePoll(void); +uint64_t SysMouseRead(int* dx, int* dy, uint8_t* buttons); +void SysMouseGetPos(int* x, int* y); // --------------------------------------------------------------------------- -// Process/Thread Helpers & Named Queues (58-65) +// Time and debug (70-72) // --------------------------------------------------------------------------- -uint32_t SysWaitThread(uint32_t tid); // 58 -uint32_t SysGetProcessPid(uint32_t handle, uint32_t* pidOut); // 59 -uint32_t SysPublishQueue(const char* name, uint32_t handle); // 60 -uint32_t SysOpenNamedQueue(const char* name, uint32_t* handleOut); // 61 -uint32_t SysQueueDuplicate(uint32_t handle, uint32_t targetPid, uint32_t* handleOut); // 62 -uint32_t SysNamedQueueOwnerPid(const char* name, uint32_t* pidOut); // 63 -uint32_t SysKillProcess(uint32_t pid, int exitCode); // 64 -uint32_t SysGetProcessList(uint32_t* pidsOut, size_t maxCount, size_t* countOut); // 65 - -// --------------------------------------------------------------------------- -// Filesystem mutations (66-68) -// --------------------------------------------------------------------------- -uint32_t SysRenameFile(const char* oldPath, const char* newPath); // 66 -uint32_t SysCreateDir(const char* path); // 67 -uint32_t SysRemoveDir(const char* path); // 68 +uint64_t SysGetTick(void); +uint32_t SysDebugPrint(const char* str); +uint32_t SysQueryTrapFrame(SYS_TRAP_FRAME_INFO* info); diff --git a/code/syscall/syscall_nr.inc b/code/syscall/syscall_nr.inc index 8c46a93..2110169 100644 --- a/code/syscall/syscall_nr.inc +++ b/code/syscall/syscall_nr.inc @@ -1,73 +1,79 @@ -; syscall_nr.inc - auto-synced with syscall_nr.h and ke_syscall.c +; syscall_nr.inc -SYS_EXIT_PROCESS EQU 0 -SYS_EXIT_THREAD EQU 1 -SYS_CREATE_THREAD EQU 2 -SYS_GET_CURRENT_TID EQU 3 -SYS_GET_CURRENT_PID EQU 4 -SYS_YIELD EQU 5 -SYS_SLEEP EQU 6 -SYS_SET_THREAD_PRIORITY EQU 7 -SYS_ALLOC EQU 8 -SYS_FREE EQU 9 -SYS_REALLOC EQU 10 -SYS_MUTEX_CREATE EQU 11 -SYS_MUTEX_ACQUIRE EQU 12 -SYS_MUTEX_TRY_ACQUIRE EQU 13 -SYS_MUTEX_RELEASE EQU 14 -SYS_SEM_CREATE EQU 15 -SYS_SEM_WAIT EQU 16 -SYS_SEM_TRY_WAIT EQU 17 -SYS_SEM_SIGNAL EQU 18 -SYS_EVENT_CREATE EQU 19 -SYS_EVENT_SIGNAL EQU 20 -SYS_EVENT_CLEAR EQU 21 -SYS_EVENT_WAIT EQU 22 -SYS_QUEUE_CREATE EQU 23 -SYS_QUEUE_SEND EQU 24 -SYS_QUEUE_RECV EQU 25 -SYS_CLOSE_HANDLE EQU 26 -SYS_OPEN_FILE EQU 27 -SYS_READ_FILE EQU 28 -SYS_WRITE_FILE EQU 29 -SYS_CREATE_FILE EQU 30 -SYS_DELETE_FILE EQU 31 -SYS_TRUNCATE_FILE EQU 32 -SYS_GET_FILE_POS EQU 33 -SYS_SEEK_FILE EQU 34 -SYS_GET_FILE_SIZE EQU 35 -SYS_SCREEN_BLIT EQU 36 -SYS_SCREEN_CLEAR EQU 37 -SYS_SCREEN_GET_SIZE EQU 38 -SYS_SCREEN_PUT_CHAR EQU 39 -SYS_KB_POLL EQU 40 -SYS_KB_IS_KEY_DOWN EQU 41 -SYS_KB_READ_CHAR EQU 42 -SYS_KB_READ_SCANCODE EQU 43 -SYS_KB_GET_MODIFIERS EQU 44 -SYS_MOUSE_POLL EQU 45 -SYS_MOUSE_READ EQU 46 -SYS_MOUSE_GET_POS EQU 47 -SYS_GET_TICK EQU 48 -SYS_DEBUG_PRINT EQU 49 -SYS_CREATE_PROCESS EQU 50 -SYS_WAIT_FOR_PROCESS EQU 51 -SYS_GET_PROCESS_EXIT_CODE EQU 52 -SYS_WRITE_CONSOLE EQU 53 -SYS_OPEN_DIR EQU 54 -SYS_READ_DIR EQU 55 -SYS_SET_CWD EQU 56 -SYS_GET_CWD EQU 57 -SYS_WAIT_THREAD EQU 58 -SYS_GET_PROCESS_PID EQU 59 -SYS_PUBLISH_QUEUE EQU 60 -SYS_OPEN_NAMED_QUEUE EQU 61 -SYS_QUEUE_DUPLICATE EQU 62 -SYS_NAMED_QUEUE_OWNER_PID EQU 63 -SYS_KILL_PROCESS EQU 64 -SYS_GET_PROCESS_LIST EQU 65 -SYS_RENAME_FILE EQU 66 -SYS_CREATE_DIR EQU 67 -SYS_REMOVE_DIR EQU 68 +SYS_EXIT_PROCESS EQU 0 +SYS_EXIT_THREAD EQU 1 +SYS_CREATE_PROCESS EQU 2 +SYS_WAIT_FOR_PROCESS EQU 3 +SYS_GET_PROCESS_EXIT_CODE EQU 4 +SYS_KILL_PROCESS EQU 5 +SYS_GET_PROCESS_LIST EQU 6 +SYS_GET_PROCESS_PID EQU 7 +SYS_CREATE_THREAD EQU 8 +SYS_WAIT_THREAD EQU 9 +SYS_GET_CURRENT_TID EQU 10 +SYS_GET_CURRENT_PID EQU 11 +SYS_SET_THREAD_PRIORITY EQU 12 +SYS_YIELD EQU 13 +SYS_SLEEP EQU 14 +SYS_SET_USER_APC_DISPATCHER EQU 15 +SYS_APC_RETURN EQU 16 +SYS_ALLOC EQU 17 +SYS_FREE EQU 18 +SYS_REALLOC EQU 19 +SYS_CLOSE_HANDLE EQU 20 +SYS_MUTEX_CREATE EQU 21 +SYS_MUTEX_ACQUIRE EQU 22 +SYS_MUTEX_TRY_ACQUIRE EQU 23 +SYS_MUTEX_RELEASE EQU 24 +SYS_SEM_CREATE EQU 25 +SYS_SEM_WAIT EQU 26 +SYS_SEM_TRY_WAIT EQU 27 +SYS_SEM_SIGNAL EQU 28 +SYS_EVENT_CREATE EQU 29 +SYS_EVENT_SIGNAL EQU 30 +SYS_EVENT_CLEAR EQU 31 +SYS_EVENT_WAIT EQU 32 +SYS_QUEUE_CREATE EQU 33 +SYS_QUEUE_SEND EQU 34 +SYS_QUEUE_RECV EQU 35 +SYS_PUBLISH_QUEUE EQU 36 +SYS_OPEN_NAMED_QUEUE EQU 37 +SYS_QUEUE_DUPLICATE EQU 38 +SYS_NAMED_QUEUE_OWNER_PID EQU 39 +SYS_OPEN_FILE EQU 40 +SYS_CREATE_FILE EQU 41 +SYS_READ_FILE EQU 42 +SYS_WRITE_FILE EQU 43 +SYS_READ_FILE_ASYNC EQU 44 +SYS_WRITE_FILE_ASYNC EQU 45 +SYS_CANCEL_IO EQU 46 +SYS_SEEK_FILE EQU 47 +SYS_GET_FILE_POS EQU 48 +SYS_GET_FILE_SIZE EQU 49 +SYS_TRUNCATE_FILE EQU 50 +SYS_DELETE_FILE EQU 51 +SYS_RENAME_FILE EQU 52 +SYS_OPEN_DIR EQU 53 +SYS_READ_DIR EQU 54 +SYS_CREATE_DIR EQU 55 +SYS_REMOVE_DIR EQU 56 +SYS_SET_CWD EQU 57 +SYS_GET_CWD EQU 58 +SYS_WRITE_CONSOLE EQU 59 +SYS_SCREEN_BLIT EQU 60 +SYS_SCREEN_CLEAR EQU 61 +SYS_SCREEN_GET_SIZE EQU 62 +SYS_SCREEN_PUT_CHAR EQU 63 +SYS_KB_POLL EQU 64 +SYS_KB_IS_KEY_DOWN EQU 65 +SYS_KB_READ_CHAR EQU 66 +SYS_KB_READ_SCANCODE EQU 67 +SYS_KB_GET_MODIFIERS EQU 68 +SYS_MOUSE_POLL EQU 69 +SYS_MOUSE_READ EQU 70 +SYS_MOUSE_GET_POS EQU 71 +SYS_GET_TICK EQU 72 +SYS_DEBUG_PRINT EQU 73 +SYS_QUERY_TRAP_FRAME EQU 74 -SYS_MAX EQU 69 +SYS_MAX EQU 75 diff --git a/code/syscall/syscall_stubs.asm b/code/syscall/syscall_stubs.asm index 906c10e..dd2a9bf 100644 --- a/code/syscall/syscall_stubs.asm +++ b/code/syscall/syscall_stubs.asm @@ -22,6 +22,16 @@ INCLUDE syscall_nr.inc +SYS_USER_APC_FRAME_Magic EQU 0 +SYS_USER_APC_FRAME_OriginalRip EQU 8 +SYS_USER_APC_FRAME_OriginalRsp EQU 16 +SYS_USER_APC_FRAME_OriginalRflags EQU 24 +SYS_USER_APC_FRAME_OriginalRax EQU 32 +SYS_USER_APC_FRAME_Routine EQU 40 +SYS_USER_APC_FRAME_NormalContext EQU 48 +SYS_USER_APC_FRAME_SystemArgument1 EQU 56 +SYS_USER_APC_FRAME_SystemArgument2 EQU 64 + .CODE ; --------------------------------------------------------------------------- @@ -47,169 +57,144 @@ Sys&name ENDP ENDM ; --------------------------------------------------------------------------- -; Process & Thread +; Process & thread lifecycle ; --------------------------------------------------------------------------- -SYSCALL_STUB2 ExitProcess, SYS_EXIT_PROCESS ; (int exitCode) -> void -SYSCALL_STUB2 ExitThread, SYS_EXIT_THREAD ; (int exitCode) -> void -SYSCALL_STUB2 CreateThread, SYS_CREATE_THREAD ; (uint64_t entry, uint64_t arg, uint32_t priority, uint32_t* tidOut) -> KSTATUS -SYSCALL_STUB2 GetCurrentTid, SYS_GET_CURRENT_TID ; () -> uint32_t -SYSCALL_STUB2 GetCurrentPid, SYS_GET_CURRENT_PID ; () -> uint32_t -SYSCALL_STUB2 Yield, SYS_YIELD ; () -> void -SYSCALL_STUB2 Sleep, SYS_SLEEP ; (uint32_t ms) -> void -SYSCALL_STUB2 SetThreadPriority, SYS_SET_THREAD_PRIORITY ; (uint32_t priority) -> KSTATUS +SYSCALL_STUB2 ExitProcess, SYS_EXIT_PROCESS +SYSCALL_STUB2 ExitThread, SYS_EXIT_THREAD +SYSCALL_STUB2 CreateProcess, SYS_CREATE_PROCESS +SYSCALL_STUB2 WaitForProcess, SYS_WAIT_FOR_PROCESS +SYSCALL_STUB2 GetProcessExitCode, SYS_GET_PROCESS_EXIT_CODE +SYSCALL_STUB2 KillProcess, SYS_KILL_PROCESS +SYSCALL_STUB2 GetProcessList, SYS_GET_PROCESS_LIST +SYSCALL_STUB2 GetProcessPid, SYS_GET_PROCESS_PID + +SYSCALL_STUB2 CreateThread, SYS_CREATE_THREAD +SYSCALL_STUB2 WaitThread, SYS_WAIT_THREAD +SYSCALL_STUB2 GetCurrentTid, SYS_GET_CURRENT_TID +SYSCALL_STUB2 GetCurrentPid, SYS_GET_CURRENT_PID +SYSCALL_STUB2 SetThreadPriority, SYS_SET_THREAD_PRIORITY +SYSCALL_STUB2 Yield, SYS_YIELD +SYSCALL_STUB2 Sleep, SYS_SLEEP +SYSCALL_STUB2 SetUserApcDispatcher, SYS_SET_USER_APC_DISPATCHER +SYSCALL_STUB2 ApcReturn, SYS_APC_RETURN ; --------------------------------------------------------------------------- -; Memory +; User APC dispatcher ; --------------------------------------------------------------------------- -SYSCALL_STUB2 Alloc, SYS_ALLOC ; (size_t size) -> void* (NULL on failure) -SYSCALL_STUB2 Free, SYS_FREE ; (void* ptr, size_t size) -> void -SYSCALL_STUB2 Realloc, SYS_REALLOC ; (void* ptr, size_t newSize) -> void* +PUBLIC SysUserApcDispatcher +SysUserApcDispatcher PROC EXPORT + ; Kernel enters here with RSP pointing at SYS_USER_APC_FRAME. There is no + ; return address because this is an IRET redirection. Preserve RBX because + ; the original user context resumes after SysApcReturn. + push rbx + lea rbx, [rsp + 8] ; original injected RSP = frame pointer + + sub rsp, 32 ; shadow space; RSP is already call-aligned + mov rcx, [rbx + SYS_USER_APC_FRAME_NormalContext] + mov rdx, [rbx + SYS_USER_APC_FRAME_SystemArgument1] + mov r8, [rbx + SYS_USER_APC_FRAME_SystemArgument2] + mov rax, [rbx + SYS_USER_APC_FRAME_Routine] + call rax + add rsp, 32 + + mov rcx, rbx ; SysApcReturn(frame) + mov rbx, [rsp] ; restore original user RBX + add rsp, 8 + mov rax, SYS_APC_RETURN + int 40h + +SysUserApcDispatcher_Failed: + jmp SysUserApcDispatcher_Failed +SysUserApcDispatcher ENDP ; --------------------------------------------------------------------------- -; Mutex +; Memory and handles ; --------------------------------------------------------------------------- -SYSCALL_STUB2 MutexCreate, SYS_MUTEX_CREATE ; (uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 MutexAcquire, SYS_MUTEX_ACQUIRE ; (uint32_t handle) -> KSTATUS -SYSCALL_STUB2 MutexTryAcquire, SYS_MUTEX_TRY_ACQUIRE ; (uint32_t handle) -> KSTATUS -SYSCALL_STUB2 MutexRelease, SYS_MUTEX_RELEASE ; (uint32_t handle) -> KSTATUS +SYSCALL_STUB2 Alloc, SYS_ALLOC +SYSCALL_STUB2 Free, SYS_FREE +SYSCALL_STUB2 Realloc, SYS_REALLOC +SYSCALL_STUB2 CloseHandle, SYS_CLOSE_HANDLE ; --------------------------------------------------------------------------- -; Semaphore +; Synchronization objects ; --------------------------------------------------------------------------- -SYSCALL_STUB2 SemCreate, SYS_SEM_CREATE ; (int32_t initial, int32_t max, uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 SemWait, SYS_SEM_WAIT ; (uint32_t handle) -> KSTATUS -SYSCALL_STUB2 SemTryWait, SYS_SEM_TRY_WAIT ; (uint32_t handle) -> KSTATUS -SYSCALL_STUB2 SemSignal, SYS_SEM_SIGNAL ; (uint32_t handle) -> KSTATUS +SYSCALL_STUB2 MutexCreate, SYS_MUTEX_CREATE +SYSCALL_STUB2 MutexAcquire, SYS_MUTEX_ACQUIRE +SYSCALL_STUB2 MutexTryAcquire, SYS_MUTEX_TRY_ACQUIRE +SYSCALL_STUB2 MutexRelease, SYS_MUTEX_RELEASE + +SYSCALL_STUB2 SemCreate, SYS_SEM_CREATE +SYSCALL_STUB2 SemWait, SYS_SEM_WAIT +SYSCALL_STUB2 SemTryWait, SYS_SEM_TRY_WAIT +SYSCALL_STUB2 SemSignal, SYS_SEM_SIGNAL + +SYSCALL_STUB2 EventCreate, SYS_EVENT_CREATE +SYSCALL_STUB2 EventSignal, SYS_EVENT_SIGNAL +SYSCALL_STUB2 EventClear, SYS_EVENT_CLEAR +SYSCALL_STUB2 EventWait, SYS_EVENT_WAIT ; --------------------------------------------------------------------------- -; Event +; Queues and named queues ; --------------------------------------------------------------------------- -SYSCALL_STUB2 EventCreate, SYS_EVENT_CREATE ; (uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 EventSignal, SYS_EVENT_SIGNAL ; (uint32_t handle) -> KSTATUS -SYSCALL_STUB2 EventClear, SYS_EVENT_CLEAR ; (uint32_t handle) -> KSTATUS -SYSCALL_STUB2 EventWait, SYS_EVENT_WAIT ; (uint32_t handle) -> KSTATUS +SYSCALL_STUB2 QueueCreate, SYS_QUEUE_CREATE +SYSCALL_STUB2 QueueSend, SYS_QUEUE_SEND +SYSCALL_STUB2 QueueRecv, SYS_QUEUE_RECV +SYSCALL_STUB2 PublishQueue, SYS_PUBLISH_QUEUE +SYSCALL_STUB2 OpenNamedQueue, SYS_OPEN_NAMED_QUEUE +SYSCALL_STUB2 QueueDuplicate, SYS_QUEUE_DUPLICATE +SYSCALL_STUB2 NamedQueueOwnerPid, SYS_NAMED_QUEUE_OWNER_PID ; --------------------------------------------------------------------------- -; Queue +; Files, directories, and current directory ; --------------------------------------------------------------------------- -SYSCALL_STUB2 QueueCreate, SYS_QUEUE_CREATE ; (uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 QueueSend, SYS_QUEUE_SEND ; (uint32_t handle, void* msg, int blocking) -> KSTATUS -SYSCALL_STUB2 QueueRecv, SYS_QUEUE_RECV ; (uint32_t handle, void* msg, int blocking) -> KSTATUS +SYSCALL_STUB2 OpenFile, SYS_OPEN_FILE +SYSCALL_STUB2 CreateFile, SYS_CREATE_FILE +SYSCALL_STUB2 ReadFile, SYS_READ_FILE +SYSCALL_STUB2 WriteFile, SYS_WRITE_FILE +SYSCALL_STUB2 ReadFileAsync, SYS_READ_FILE_ASYNC +SYSCALL_STUB2 WriteFileAsync, SYS_WRITE_FILE_ASYNC +SYSCALL_STUB2 CancelIo, SYS_CANCEL_IO +SYSCALL_STUB2 SeekFile, SYS_SEEK_FILE +SYSCALL_STUB2 GetFilePos, SYS_GET_FILE_POS +SYSCALL_STUB2 GetFileSize, SYS_GET_FILE_SIZE +SYSCALL_STUB2 TruncateFile, SYS_TRUNCATE_FILE +SYSCALL_STUB2 DeleteFile, SYS_DELETE_FILE +SYSCALL_STUB2 RenameFile, SYS_RENAME_FILE + +SYSCALL_STUB2 OpenDir, SYS_OPEN_DIR +SYSCALL_STUB2 ReadDir, SYS_READ_DIR +SYSCALL_STUB2 CreateDir, SYS_CREATE_DIR +SYSCALL_STUB2 RemoveDir, SYS_REMOVE_DIR +SYSCALL_STUB2 SetCwd, SYS_SET_CWD +SYSCALL_STUB2 GetCwd, SYS_GET_CWD ; --------------------------------------------------------------------------- -; Handle (unified - use for files, mutexes, semaphores, events, queues) +; Console and graphics ; --------------------------------------------------------------------------- -SYSCALL_STUB2 CloseHandle, SYS_CLOSE_HANDLE ; (uint32_t handle) -> KSTATUS +SYSCALL_STUB2 WriteConsole, SYS_WRITE_CONSOLE +SYSCALL_STUB6 ScreenBlit, SYS_SCREEN_BLIT +SYSCALL_STUB2 ScreenClear, SYS_SCREEN_CLEAR +SYSCALL_STUB2 ScreenGetSize, SYS_SCREEN_GET_SIZE +SYSCALL_STUB6 ScreenPutChar, SYS_SCREEN_PUT_CHAR ; --------------------------------------------------------------------------- -; File I/O -; (To close: SysCloseHandle(handle) - no dedicated SysCloseFile) +; Input ; --------------------------------------------------------------------------- -SYSCALL_STUB2 OpenFile, SYS_OPEN_FILE ; (const char* path, uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 ReadFile, SYS_READ_FILE ; (uint32_t handle, void* buf, size_t size, size_t* bytesReadOut) -> KSTATUS -SYSCALL_STUB2 WriteFile, SYS_WRITE_FILE ; (uint32_t handle, const void* buf, size_t size, size_t* bytesWrittenOut) -> KSTATUS -SYSCALL_STUB2 CreateFile, SYS_CREATE_FILE ; (const char* path, uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 DeleteFile, SYS_DELETE_FILE ; (const char* path) -> KSTATUS -SYSCALL_STUB2 TruncateFile, SYS_TRUNCATE_FILE ; (uint32_t handle, uint64_t newSize) -> KSTATUS -SYSCALL_STUB2 GetFilePos, SYS_GET_FILE_POS ; (uint32_t handle) -> uint64_t -SYSCALL_STUB2 SeekFile, SYS_SEEK_FILE ; (uint32_t handle, uint64_t offset) -> KSTATUS -SYSCALL_STUB2 GetFileSize, SYS_GET_FILE_SIZE ; (uint32_t handle) -> uint64_t +SYSCALL_STUB2 KbPoll, SYS_KB_POLL +SYSCALL_STUB2 KbIsKeyDown, SYS_KB_IS_KEY_DOWN +SYSCALL_STUB2 KbReadChar, SYS_KB_READ_CHAR +SYSCALL_STUB2 KbReadScancode, SYS_KB_READ_SCANCODE +SYSCALL_STUB2 KbGetModifiers, SYS_KB_GET_MODIFIERS + +SYSCALL_STUB2 MousePoll, SYS_MOUSE_POLL +SYSCALL_STUB2 MouseRead, SYS_MOUSE_READ +SYSCALL_STUB2 MouseGetPos, SYS_MOUSE_GET_POS ; --------------------------------------------------------------------------- -; Screen -; SysScreenBlit has 5 args (x, y, w, h, pixels) - needs SYSCALL_STUB6. -; SysScreenPutChar has 5 args (c, x, y, fg, bg) - needs SYSCALL_STUB6. +; Time and debug ; --------------------------------------------------------------------------- -SYSCALL_STUB6 ScreenBlit, SYS_SCREEN_BLIT ; (int x, int y, int w, int h, const uint32_t* pixels) -> void -SYSCALL_STUB2 ScreenClear, SYS_SCREEN_CLEAR ; (uint32_t color) -> void -SYSCALL_STUB2 ScreenGetSize, SYS_SCREEN_GET_SIZE ; (int* widthOut, int* heightOut) -> void -SYSCALL_STUB6 ScreenPutChar, SYS_SCREEN_PUT_CHAR ; (char c, int x, int y, uint32_t fg, uint32_t bg) -> void - -; --------------------------------------------------------------------------- -; Keyboard -; --------------------------------------------------------------------------- -SYSCALL_STUB2 KbPoll, SYS_KB_POLL ; () -> uint32_t (1 if key available, 0 otherwise) -SYSCALL_STUB2 KbIsKeyDown, SYS_KB_IS_KEY_DOWN ; (int scancode) -> uint32_t (1 if down, 0 otherwise) -SYSCALL_STUB2 KbReadChar, SYS_KB_READ_CHAR ; () -> char (blocks, yields while waiting) -SYSCALL_STUB2 KbReadScancode, SYS_KB_READ_SCANCODE ; () -> uint8_t (blocks, yields while waiting) -SYSCALL_STUB2 KbGetModifiers, SYS_KB_GET_MODIFIERS ; () -> uint32_t (current modifier bitmask) - -; --------------------------------------------------------------------------- -; Mouse -; --------------------------------------------------------------------------- -SYSCALL_STUB2 MousePoll, SYS_MOUSE_POLL ; () -> uint32_t (1 if packet waiting, 0 otherwise) -SYSCALL_STUB2 MouseRead, SYS_MOUSE_READ ; (int* dx, int* dy, uint8_t* buttons) -> uint32_t (1=ok, 0=empty) -SYSCALL_STUB2 MouseGetPos, SYS_MOUSE_GET_POS ; (int* x, int* y) -> void - -; --------------------------------------------------------------------------- -; Timer -; --------------------------------------------------------------------------- -SYSCALL_STUB2 GetTick, SYS_GET_TICK ; () -> uint64_t (milliseconds since boot) - -; --------------------------------------------------------------------------- -; Debug -; --------------------------------------------------------------------------- -SYSCALL_STUB2 DebugPrint, SYS_DEBUG_PRINT ; (const char* str) -> void - -; --------------------------------------------------------------------------- -; Process launch & lifecycle -; --------------------------------------------------------------------------- -SYSCALL_STUB2 CreateProcess, SYS_CREATE_PROCESS ; (const char* path, const char* args, uint32_t* pidOut) -> KSTATUS -SYSCALL_STUB2 WaitForProcess, SYS_WAIT_FOR_PROCESS ; (uint32_t pid) -> KSTATUS -SYSCALL_STUB2 GetProcessExitCode, SYS_GET_PROCESS_EXIT_CODE ; (uint32_t pid, int* exitCodeOut) -> KSTATUS - -; --------------------------------------------------------------------------- -; Console output (white text, for user programs) -; --------------------------------------------------------------------------- -SYSCALL_STUB2 WriteConsole, SYS_WRITE_CONSOLE ; (const char* str, size_t len) -> void - -; --------------------------------------------------------------------------- -; Directory enumeration -; --------------------------------------------------------------------------- -SYSCALL_STUB2 OpenDir, SYS_OPEN_DIR ; (const char* path, uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 ReadDir, SYS_READ_DIR ; (uint32_t handle, DirEntry* entryOut) -> KSTATUS - -; --------------------------------------------------------------------------- -; Current Working Directory -; --------------------------------------------------------------------------- -SYSCALL_STUB2 SetCwd, SYS_SET_CWD ; (const char* path) -> KSTATUS -SYSCALL_STUB2 GetCwd, SYS_GET_CWD ; (char* buf, size_t buflen) -> KSTATUS - -; --------------------------------------------------------------------------- -; Thread join (used by pthread_join in posix.dll) -; --------------------------------------------------------------------------- -SYSCALL_STUB2 WaitThread, SYS_WAIT_THREAD ; (uint32_t tid) -> KSTATUS - -; --------------------------------------------------------------------------- -; Process PID query (used by posix_spawn -> waitpid in posix.dll) -; --------------------------------------------------------------------------- -SYSCALL_STUB2 GetProcessPid, SYS_GET_PROCESS_PID ; (uint32_t pid) -> KSTATUS - -; --------------------------------------------------------------------------- -; Named Queue Registry -; SysPublishQueue - register a queue under a name (call once at startup) -; SysOpenNamedQueue - resolve a name to a handle (retry until SUCCESS) -; --------------------------------------------------------------------------- -SYSCALL_STUB2 PublishQueue, SYS_PUBLISH_QUEUE ; (const char* name, uint32_t queueHandle) -> KSTATUS -SYSCALL_STUB2 OpenNamedQueue, SYS_OPEN_NAMED_QUEUE ; (const char* name, uint32_t* handleOut) -> KSTATUS - -; --------------------------------------------------------------------------- -; Handle duplication -; --------------------------------------------------------------------------- -SYSCALL_STUB2 QueueDuplicate, SYS_QUEUE_DUPLICATE ; (uint32_t handle, uint32_t targetPid, uint32_t* handleOut) -> KSTATUS -SYSCALL_STUB2 NamedQueueOwnerPid, SYS_NAMED_QUEUE_OWNER_PID ; (const char* name, uint32_t* pidOut) -> KSTATUS - -; --------------------------------------------------------------------------- -; Process utilities -; --------------------------------------------------------------------------- -SYSCALL_STUB2 KillProcess, SYS_KILL_PROCESS ; (uint32_t pid, int exitCode) -> KSTATUS -SYSCALL_STUB2 GetProcessList, SYS_GET_PROCESS_LIST ; (uint32_t* pidsOut, size_t maxCount, size_t* countOut) -> KSTATUS - -; --------------------------------------------------------------------------- -; Filesystem mutations -; --------------------------------------------------------------------------- -SYSCALL_STUB2 RenameFile, SYS_RENAME_FILE ; (const char* oldPath, const char* newPath) -> KSTATUS -SYSCALL_STUB2 CreateDir, SYS_CREATE_DIR ; (const char* path) -> KSTATUS -SYSCALL_STUB2 RemoveDir, SYS_REMOVE_DIR ; (const char* path) -> KSTATUS +SYSCALL_STUB2 GetTick, SYS_GET_TICK +SYSCALL_STUB2 DebugPrint, SYS_DEBUG_PRINT +SYSCALL_STUB2 QueryTrapFrame, SYS_QUERY_TRAP_FRAME END diff --git a/code/win32/gdi32.c b/code/win32/gdi32.c index 2e0d17c..5dd847e 100644 --- a/code/win32/gdi32.c +++ b/code/win32/gdi32.c @@ -92,13 +92,13 @@ __declspec(dllexport) int StretchDIBits( // Only reallocate if the new size is LARGER than what we already have int requiredPixels = DestWidth * DestHeight; if (requiredPixels > currentDestCapacity) { - SysFree(destBuffer, 0); + SysFree(destBuffer); destBuffer = (uint32_t*)SysAlloc(requiredPixels * sizeof(uint32_t)); currentDestCapacity = requiredPixels; } if (DestWidth > currentXMapCapacity) { - SysFree(srcX_map, 0); + SysFree(srcX_map); srcX_map = (int*)SysAlloc(DestWidth * sizeof(int)); currentXMapCapacity = DestWidth; } diff --git a/code/win32/kernel32.c b/code/win32/kernel32.c index 2c6fa29..09ea325 100644 --- a/code/win32/kernel32.c +++ b/code/win32/kernel32.c @@ -197,7 +197,7 @@ __declspec(dllexport) BOOL HeapFree(void* heap, DWORD flags, void* ptr) { (void)heap; (void)flags; // Win32's HeapFree doesn't need the size; we pass 0 and the kernel's // allocator handles it (the block header carries the size). - if (ptr) SysFree(ptr, 0); + if (ptr) SysFree(ptr); return TRUE; } @@ -1163,30 +1163,33 @@ __declspec(dllexport) BOOL SetCurrentDirectoryA(const char* lpPathName) { __declspec(dllexport) DWORD GetCurrentDirectoryA(DWORD nBufferLength, char* lpBuffer) { if (!lpBuffer || nBufferLength == 0) { - // Return required size (including NUL). Max CWD is 512 bytes + "C:\". + // Return required size (including NUL). Max CWD is 512 bytes + "C:\\". return 515; } - // Retrieve the raw relative CWD from the kernel (e.g. "" or "apps\\data") + // Kernel returns native "N:/path". Present it as a Win32 drive letter. char raw[512]; uint32_t s = SysGetCwd(raw, sizeof(raw)); if (s != 0) { gLastError = 1; return 0; } - // Prepend the Windows-style drive letter so callers see "C:\apps\data" - // rather than just "apps\data". char full[515]; - if (raw[0] == '\0') { - // Root - full[0] = 'C'; full[1] = ':'; full[2] = '\\'; full[3] = '\0'; - } else { - // "C:\" - full[0] = 'C'; full[1] = ':'; full[2] = '\\'; - size_t rawLen = strlen(raw); - if (rawLen + 4 > sizeof(full)) rawLen = sizeof(full) - 4; - memcpy(full + 3, raw, rawLen); - full[3 + rawLen] = '\0'; + char drive = 'C'; + const char* rel = raw; + if (raw[0] >= '0' && raw[0] <= '7' && raw[1] == ':' && + (raw[2] == '\\' || raw[2] == '/')) { + drive = (char)('C' + (raw[0] - '0')); + rel = raw + 3; } + full[0] = drive; + full[1] = ':'; + full[2] = '\\'; + size_t rawLen = strlen(rel); + if (rawLen + 4 > sizeof(full)) rawLen = sizeof(full) - 4; + memcpy(full + 3, rel, rawLen); + full[3 + rawLen] = '\0'; + for (char* p = full + 3; *p; p++) if (*p == '/') *p = '\\'; + size_t needed = strlen(full) + 1; if ((DWORD)needed > nBufferLength) { gLastError = 122; // ERROR_INSUFFICIENT_BUFFER diff --git a/code/win32/ole32.c b/code/win32/ole32.c index c1be4ef..4046d84 100644 --- a/code/win32/ole32.c +++ b/code/win32/ole32.c @@ -65,7 +65,7 @@ __declspec(dllexport) HRESULT CoCreateInstance(const GUID* clsid, IUnknown* oute // Memory // --------------------------------------------------------------------------- __declspec(dllexport) void CoTaskMemFree(void* pv) { - if (pv) SysFree(pv, 0); + if (pv) SysFree(pv); } // --------------------------------------------------------------------------- diff --git a/readme.md b/readme.md index 6c13c94..be067e4 100644 --- a/readme.md +++ b/readme.md @@ -1,87 +1,197 @@ # NT-Lite -- an x86-64 kernel -## The Monster From ~~Redmond~~ Kraków +## The Monster From ~~Redmond~~ Cracow -Boots. Runs processes. Mimics Windows to some extent. +Boots. Runs processes. Loads drivers. Plays video. Runs Doom. Has opinions. ## what is it? -NT-Lite is a 64-bit operating system kernel written from scratch. It includes a UEFI bootloader, physical and virtual memory managers, a preemptive scheduler, a HAL, a VFS, a PE loader, and partial Win32 and POSIX layers. It boots into a shell. +NT-Lite is a 64-bit operating system kernel written from scratch. It has a UEFI bootloader, physical and virtual memory managers, a preemptive scheduler, a HAL, a VFS, a PE loader, dynamic drivers, partial Win32 and POSIX personalities, and enough asynchronous I/O machinery to make the kernel feel a little less like a toy and a little more like a tiny monster. -The codebase is roughly 20k lines of C and assembly. +It boots into a shell, launches user processes, survives application crashes, and can run things it probably should not be able to run yet. + +The codebase is still small enough to understand, but large enough to hurt your feelings. ## features - UEFI bootloader +- higher-half x86-64 kernel - physical memory manager -- virtual memory manager with kernel pool -- HAL for screen, keyboard, mouse, timer, disk, PCI, and APIC -- preemptive scheduler with priority levels +- virtual memory manager with kernel pool and user heap +- single address-space design with user/supervisor page permissions +- HAL for screen, keyboard, mouse, timer, PCI, APIC, HPET, AHCI, and IDE +- preemptive priority scheduler - processes and threads +- object handles +- events, semaphores, mutexes, queues, and alertable waits +- IRQL model: `PASSIVE_LEVEL`, `APC_LEVEL`, `DISPATCH_LEVEL`, and device IRQLs +- interrupt-exit scheduling instead of context-switching inside hard IRQ handlers +- DPC queue for interrupt bottom halves +- kernel APCs +- user APCs through a real trap-frame return dispatcher +- async file I/O with completion events, APC callbacks, IOSBs, and cancellation - VFS with IRPs -- drive-letter paths such as `0:/apps/shell.exe` +- centralized `IoCompleteRequest()` +- `DRIVER_OBJECT` / `DEVICE_OBJECT` driver model +- `IoCallDriver()` dispatch path +- AHCI driver, interrupt-driven +- IDE driver, PIO/DMA +- FAT32 filesystem driver +- VFAT long filename support for real dotfiles like `.doomrc` +- drive-number paths such as `0:/apps/shell.exe` - INI-based boot and driver configuration -- PE import resolution and dynamic driver loading +- PE import resolution and dynamic `.sys` driver loading - partial Win32 personality -- partial POSIX personality +- partial (stub/microsoft-level) POSIX personality +- self-tests for APCs, async disk I/O, async file I/O, trap frames, and cancellation +- Doom, because priorities matter ## architecture notes ### memory layout -NT-Lite uses a single virtual address space per booted system rather than a fully separate kernel address space and user address space. The distinction between kernel and user mode comes from page permissions and CPU privilege checks, not from having a different page table for the kernel. +Single address space for the whole system. No separate kernel/user page tables. Isolation comes from compilers, page permissions and privilege levels, not from switching CR3. -The kernel is mapped into the higher half of virtual memory and user processes only receive the regions they are explicitly granted. Kernel pages are marked supervisor-only, so user-mode code cannot read or write them directly. User pages are mapped with user permissions and are isolated from one another by the kernel’s allocator, VMM, and process creation path. +Kernel lives in the higher half. User processes only see what they are explicitly given. Kernel pages are supervisor-only so user-mode code cannot touch them. User pages are isolated from each other through the allocator and process creation path, not by hardware address-space separation. One rogue pointer can't crash the kernel but it might corrupt other processes. + +Is this how a serious modern OS should isolate processes? No. + +Is it very convenient while building? Absolutely. ### boot chain -The bootloader reads configuration, loads the core images, maps the higher-half layout, and hands control to `KernelMain`. After that, the kernel takes over interrupt setup, paging, scheduler bring-up, runtime driver loading, and user process launch. +Bootloader loads config, pulls in the core images, sets up the higher-half mapping, then jumps to `KernelMain`. -### drivers +From there: -Drivers are PE32+ `.sys` files with a conventional `DriverEntry` export that accepts a pointer to `HAL_API`. +```text +HAL init +interrupts +paging +scheduler +boot drivers +filesystems +runtime drivers +self-tests +init.exe +shell +``` -Drivers are split into two groups: +### execution model -- essential boot drivers, which are loaded before the kernel starts doing real work -- runtime drivers, which are loaded later from disk once storage and the VFS are available +NT-Lite has a Windows-flavored execution model, but a simplified one. -The driver model is intentionally small. Disk drivers expose a `DISK_DRIVER` vtable and register with `IoRegisterDisk(index, &driver)`. Filesystem drivers expose an `IO_FILESYSTEM` vtable and register with `IoRegisterFilesystem(index, &fs, &vol)`. +```text +PASSIVE_LEVEL normal thread code; may wait, allocate, touch filesystems +APC_LEVEL APC delivery / guarded regions +DISPATCH_LEVEL scheduler, DPCs, spinlocks; must not block +DIRQL hardware IRQ handlers; acknowledge device, queue bottom half, leave +``` -Currently shipping: AHCI (interrupts), IDE (PIO/DMA), and FAT32. +The timer interrupt don't context-switch directly inside the IRQ handler. Interrupts unwind through a common exit path, which can drain DPCs, deliver APCs when legal, and perform deferred scheduling. + +### DPCs and APCs + +Hard IRQ handlers stay small. They acknowledge hardware, capture status, and queue DPCs. + +DPCs run at `DISPATCH_LEVEL` and do the bottom-half work: input callbacks, disk completion wakeups, and other things that should not happen at device IRQL. + +Kernel APCs run later at `PASSIVE_LEVEL` in the target thread. User APCs are delivered by editing the saved user trap frame and returning through a registered user-mode dispatcher stub in `syscall.dll`. + +The important bit: + +```text +interrupt -> DPC -> IoCompleteRequest -> APC -> alertable wait/user callback +``` + +That is a real path, not a TODO scribbled in the margin. ### I/O and filesystems -All file access flows through the VFS layer. That keeps disk access, filesystem parsing, and path translation separate instead of spreading storage logic across the rest of the kernel. +Everything goes through the VFS. Disk access, filesystem parsing, path resolution, and process personalities are separate layers instead of one giant bowl of sadness. -Paths are not interpreted as Windows paths by default. A drive such as `0:/` is a mounted namespace, while the Win32 and POSIX personalities translate their own path styles into the same underlying VFS. +Disk drivers expose devices. The I/O manager calls drivers through `IoCallDriver()`. Completion goes through `IoCompleteRequest()`. + +Async file I/O exists. It is currently worker-backed at the filesystem layer, which means the user-facing async contract is real. We are somewhere between right and dangerous. + +Async requests can complete by: + +- signaling an event +- writing an `IO_STATUS_BLOCK` +- queuing a kernel APC +- queuing a user APC +- being cancelled + +### drivers + +PE32+ `.sys` files. The entry point, preferably named: + +```c +DriverEntry(HAL_API* hal) +``` + +Two loading phases: + +```text +boot drivers loaded before the kernel does any real work +runtime drivers loaded later, once storage and VFS are up +``` + +Disk drivers register real device objects, and I/O goes through the driver object/device object dispatch path. + +Currently shipping: + +```text +AHCI interrupt-driven SATA +IDE PIO/DMA +FAT32 filesystem driver with VFAT long filename support +``` ### processes, threads, and syscalls -User processes are launched from PE images, imports are resolved before the first thread is queued, and the syscall layer provides the boundary between user code and kernel services. `syscall.dll` is the shared entry point for the supported personalities. +PE images get loaded, imports resolved, then the first thread is queued. `syscall.dll` is the shared syscall entry point, and both personalities go through it. -The scheduler is preemptive and priority-based, so threads do not depend on cooperative yielding to keep the system responsive. Process state is tracked separately from thread state, which allows a process to exit, be reaped, or be restarted by `init` without bringing down the rest of the system. +Scheduler is preemptive and priority-based. Process state and thread state are tracked separately, so a process can die or get reaped without taking the whole system down. `init` can restart things if needed. + +Syscalls build real trap frames. That matters because user APC delivery needs to safely redirect the return path back into user mode and then restore the interrupted context with `SysApcReturn`. ### personalities -The Win32 personality supports programs that expect drive letters, familiar process APIs, and a Windows-style runtime. The POSIX personality maps the same kernel primitives into a tree-shaped namespace and a different userspace contract. +The Win32 personality supports programs that expect drive letters, familiar process APIs, and a Windows-style runtime. -## how it boots +The POSIX personality maps the same kernel primitives into a tree-shaped namespace and a different userspace contract. -1. UEFI loads `boot.ini` and reads `drivers.ini` -2. the bootloader maps the kernel, HAL, and boot drivers into the upper half -3. a direct physical window is mapped at `0xFFFF880000000000` -4. control passes to `KernelMain` -5. PMM, VMM, GDT, IDT, HAL, IRQs, and scheduler initialization run -6. runtime drivers are loaded from disk -7. `shell.exe` is launched +Binary compatibility is limited and a non-goal. -## how to build +## async I/O example -- open the VS x64 build tools as administrator -- change into this directory -- run `build.bat` -- use one of `run.bat` (IDE), `rund.bat` (IDE, debug), `runa.bat` (AHCI), or `runad.bat` (AHCI, debug) +User-mode async I/O looks roughly like this: + +```c +SYS_IO_STATUS_BLOCK iosb; +uint32_t done; +SysEventCreate(0, 0, &done); + +SYS_ASYNC_FILE_REQUEST req = { + .FileHandle = file, + .EventHandle = done, + .Offset = 0, + .Buffer = buffer, + .Size = 512, + .Iosb = &iosb, + .RequestHandleOut = 0, + .ApcRoutine = MyCompletionRoutine, + .ApcContext = &something, +}; + +uint32_t s = SysReadFileAsync(&req); +if (s == SYS_STATUS_PENDING) + s = SysEventWait(done, 1); // alertable +``` + +If the request completes by user APC, the alertable wait returns `SYS_STATUS_USER_APC`, the callback runs in user mode, and the original execution continues afterward. + +That sentence used to be science fiction. Now it is boring. Excellent. ## paths @@ -89,211 +199,81 @@ The Win32 personality supports programs that expect drive letters, familiar proc "apps/shell.exe" -> current working directory "0:/apps/shell.exe" -> drive 0 "2:/data/config.ini" -> drive 2 +"C:\apps\doom.exe" -> drive 0, mapping is simple c->0, d->1, etc. +"/apps/video.exe" -> drive 0 +"/1/apps/video.exe" -> drive 1 ``` -Drives 0-7 are available. A drive only works if a filesystem is registered on it. The Win32 personality translates drive letters to drive numbers, while the POSIX personality maps drives onto a tree. +Drives `0`-`7` are available. A drive only works if a filesystem is registered on it. The Win32 personality translates drive letters to drive numbers, while the POSIX personality maps drives onto a tree. + +`0:/` is a mounted namespace, not a Windows path. It merely enjoys cosplay. + +## how to build + +- open the VS x64 build tools as administrator +- change into this directory +- run `build.bat` +- use one of: + - `run.bat` — IDE + - `rund.bat` — IDE, debug + - `runa.bat` — AHCI + - `runad.bat` — AHCI, debug + +## how it looks when built + +```text +efi/boot/ + bootx64.efi +boot.ini +drivers/ + drivers.ini + ahci.sys + fat32.sys + ide.sys +system/ + colonel.exe + hal.dll + init.exe + syscall.dll + win32.dll + posix.dll +apps/ + shell_native.exe + shell_win32.exe + shell_posix.exe + async_apc.exe + async_cancel.exe + doom.exe + video.exe +``` + +## known truths + +- FAT32 works well enough to save `.doomrc` properly. +- Doom runs. +- Video playback works. +- Async file I/O exists. +- Process isolation is not strong. +- This is not production software. +- This is absolutely the wrong way to spend a weekend. ## screenshots ![video.exe running bink video 2](image.png) -
Boot logs - -```text -BdsDxe: loading Boot0001 "UEFI QEMU HARDDISK QM00005 " from PciRoot(0x0)/Pci(0x4,0x0)/Sata(0x0,0xFFFF,0x0) -BdsDxe: starting Boot0001 "UEFI QEMU HARDDISK QM00005 " from PciRoot(0x0)/Pci(0x4,0x0)/Sata(0x0,0xFFFF,0x0) -1 -2 -3 -4 -Loading colonel.exe... -Loading hal.dll... -5 -6 -7 -8 -Found RSDP at: 0xF77E014 -9 -HAL: screen 1280x800 -HAL: FADT PM1a_CNT=0xb004 PM1b_CNT=0x0000 DSDT=0x0f77a000 -HAL: _S5_ SLP_TYP_A=0x00 SLP_TYP_B=0x00 -HAL: LAPIC=00000000fee00000 IOAPIC=00000000fec00000 HPET=00000000fed00000 -HAL: lapicPhys=0xFEE00000, lapicVirt=0xFFFF8800FEE00000, hpetPhys=0xFED00000 -HAL: Attempting to calibrate APIC timer via HPET... -HAL: APIC timer: 62981 ticks/ms -HAL: initialisation complete. -KernelMain: Handing over to scheduler... -SystemInitThread: Thread started. Igniting the system... -InitializeDrivers: Initializing drivers... -InitializeDrivers: Executing DriverEntry for ide.sys -IDE: Init -IDE: No drive on primary channel -InitializeDrivers: Driver ide.sys failed to init! -InitializeDrivers: Executing DriverEntry for ahci.sys -AHCI: driver loading... -AHCI: controller at PCI 00:04.0 -AHCI: PCI IRQ line = 11 -AHCI: ABAR phys=0x81060000 version=0x00010000 -AHCI: cap=0xc0141f05 ports=6 slots=32 64bit=1 -AHCI: port 0: link up, initializing... -AHCI: port COMRESET done: sig=0x00000101 ssts=0x00000113 -AHCI: drive 0: 524288 sectors (256 MB) -AHCI: drive 0 registered -AHCI: port 1: no link (ssts=0x00000000) -AHCI: port 2: no link (ssts=0x00000000) -AHCI: port 3: no link (ssts=0x00000000) -AHCI: port 4: no link (ssts=0x00000000) -AHCI: port 5: no link (ssts=0x00000000) -AHCI: ISR registered on IRQ 11; initialised 1 drive(s) -InitializeDrivers: Executing DriverEntry for fat32.sys -FAT32: Init -FAT32: Init complete (status=0, clusters=128480). -Found runtime driver: usb.sys -Test runtime driver -SystemInitThread: Loading system DLLs... -Memory Status: - -> Free (PMM): 151.71 MB - -> Kernel Pool: 4.00 MB - -> User Heap: 0.55 MB -LaunchUserProcess: IoOpenFile: 0 -LaunchUserProcess: IoGetFileSize: 5120 -LaunchUserProcess: MmAllocateUserMemory: 0x8E020 -LaunchUserProcess: IoReadFile: 0 -LaunchUserProcess: IoCloseFile -Created process (PID 2) with command line: init.exe -LaunchUserProcess: PE Image mapped! Base 0x90020, Entry 0x91600 -LaunchUserProcess: Thread created and queued! -[init] starting -[init] waiting -shell: ready -> help -Available commands: - help - Show this list - dir - List files - cd - Change directory - type - Display file contents - del - Delete a file - write - Create a file with text - exit - Close the shell -> dir -System Volume Information -drivers -efi -colonel.exe -hal.dll -syscall.dll -win32.dll -posix.dll -init.exe -shell_win32.exe -shell_posix.exe -shell_native.exe -doom.exe -video.exe -boot.ini -bink2w64.dll -video.bk2 -doom1.wad -> shell_win32.exe -shell: ready -shell_win32.exe -> doom.exe -doom: starting -getenv: Unknown variable DOOMWADDIR - DOOM Shareware Startup v1.10 -V_Init: allocate screens. -M_LoadDefaults: Load system defaults. -Z_Init: Init zone memory allocation daemon. -W_Init: Init WADfiles. - adding ./doom1.wad -=========================================================================== - Shareware! -=========================================================================== -M_Init: Init miscellaneous info. -R_Init: Init Doom refresh daemon - [.. ] -InitTextures -InitFlats........ -InitSprites -InitColormaps -R_InitData -R_InitPointToAngle -R_InitTables -R_InitPlanes -R_InitLightTables -R_InitSkyMap -R_InitTranslationsTables -P_Init: Init Playloop state. -I_Init: Setting up machine state. -I_InitSound: I_InitSound: pre-cached all sound data -I_InitSound: sound module ready -D_CheckNetGame: Checking network game status. -startskill 2 deathmatch: 0 startmap: 1 startepisode: 1 -player 1 of 1 (1 nodes) -S_Init: Setting up sound. -S_Init: default sfx volume 8 -HU_Init: Setting up heads up display. -ST_Init: Init status bar. -doom: entering game loop -I_ShutdownSound: NOT finishing pending sounds -> shell_posix.exe -shell: ready -> help -Available commands: - help - Show this list - dir [path] - List directory contents - cd - Change directory - type - Display file contents - del - Delete a file - write - Create a file with text - exit - Close the shell -> dir - System Volume Information - drivers - efi - colonel.exe - hal.dll - syscall.dll - win32.dll - posix.dll - init.exe - shell_win32.exe - shell_posix.exe - shell_native.exe - doom.exe - video.exe - boot.ini - bink2w64.dll - video.bk2 - doom1.wad - .doo -> video.exe -video: starting -video.exe -video: video.bk2 loaded -shell: goodbye -> exit -shell: goodbye -> exit -shell: goodbye -> exit -shell: goodbye -[init] done -ProcessWatcherThread: Process "init.exe" (PID 2) exited with code 0. -ProcessWatcherThread: Shutting down. -``` - -
- ## faq Q: Why? -A: Learning. Fun. +A: Learning. Fun. Spite. Q: Does it run Doom? A: `doom.exe` -Q: How often does it crash? -A: Yes. +Q: Does Doom save its config correctly? +A: Yes. `.doomrc`, not cursed `.DOO` nonsense. VFAT forever! + +Q: Does it have async I/O? +A: Yes. Events, IOSBs, user APC callbacks, cancellation. Very suspiciously serious. Q: Is the Win32 layer real? A: Real enough to load binaries that expect it. Not real enough to run Notepad. @@ -301,11 +281,17 @@ A: Real enough to load binaries that expect it. Not real enough to run Notepad. Q: Are processes isolated? A: No. But IPC is free. -Q: Is this finished? -A: No. +Q: How often does it crash? +A: Less often than before, which is how civilizations begin. -Q: Will it corrupt my data? -A: It shouldn't. +Q: Can it recover from a crash? +A: Usually. Unless the kernel shat itself. + +Q: Is this finished? +A: No. And it never will be. + +Q: Will it corrupt my data? +A: It should not. Q: Why is the path syntax so weird? A: It makes sense to me. @@ -314,9 +300,9 @@ Q: Why are the logs so verbose? A: How else do I debug it? Q: What should I read first? -A: This document. +A: This document. Then the scheduler. Then whatever hurt you last. Q: Can I use this in production? A: Don't. -#### Projekt dedykowany dla szanownego Pana profesora Leszka Wójcika przez którego znienawidziłem Unixa. +#### Project dedicated to the honorable, esteemed Professor Leszek Wójcik through whom I came to hate Unix.