diff --git a/.gitignore b/.gitignore index 359d1b1..0075d3e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,6 @@ build.obj bin/ obj/ qemu.log -disk.vhd +*.vhd code/build/build.exe code/build/build.exe.old \ No newline at end of file diff --git a/OVMF/OVMF_VARS_4M_d.fd b/OVMF/OVMF_VARS_4M_d.fd index 5ff0a80..57dd62c 100644 Binary files a/OVMF/OVMF_VARS_4M_d.fd and b/OVMF/OVMF_VARS_4M_d.fd differ diff --git a/code/boot.ini b/code/boot.ini index d7cc45b..8c0cd3c 100644 --- a/code/boot.ini +++ b/code/boot.ini @@ -1,5 +1,5 @@ [kernel] ; bitmask: 1 = debug prints, 2 = self check -debug = 0 +debug = 1 [init] commandline = apps/shell_native.exe diff --git a/code/boot/uefi/main.c b/code/boot/uefi/main.c index 74eae96..5ed1eb6 100644 --- a/code/boot/uefi/main.c +++ b/code/boot/uefi/main.c @@ -20,6 +20,7 @@ #include #include #include +#pragma intrinsic(_disable) int _fltused = 0; @@ -287,6 +288,21 @@ static void DriversIniCb(const char* section, const char* key, // --------------------------------------------------------------------------- // EFI Memory Map -> MEMORY_MAP conversion // --------------------------------------------------------------------------- + +static EFI_PHYSICAL_ADDRESS AllocateApTrampolinePageBelow1M(void) { + // AP startup via SIPI requires the first instruction page to live below + // 1 MiB. Some real UEFI firmware exposes no ConventionalMemory below + // 640 KiB after ExitBootServices, so reserve a loader-data page here while + // boot services can still satisfy AllocateMaxAddress. + EFI_PHYSICAL_ADDRESS phys = 0x0009F000ULL; + EFI_STATUS st = gBS->AllocatePages(AllocateMaxAddress, EfiLoaderData, 1, &phys); + if (EFI_ERROR(st) || phys == 0 || phys >= 0x000A0000ULL || (phys & 0xFFFULL)) { + return 0; + } + gBS->SetMem((VOID*)phys, PAGE_SIZE, 0); + return phys; +} + static void ConvertMemoryMap(MEMORY_MAP* out) { UINTN mapSize = 0, descSz = 0, mapKey; UINT32 descVer; gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer); @@ -738,6 +754,8 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable //Print(L"7\n"); + bi->ApTrampolinePhys = (uintptr_t)AllocateApTrampolinePageBelow1M(); + // --- Map physical RAM as direct window AND identity map --- UINTN mapSize = 0, descSz = 0, mapKey; UINT32 descVer; gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer); @@ -880,7 +898,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable } // Print it out to verify the bootloader actually found it before launching the kernel - //Print(L"Found RSDP at: 0x%llx\n", bi->RsdpPhysical); + //Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical); bi->PhysicalMapBase = PHYS_MAP_BASE; bi->Debug = (UINT32)ps.debug; diff --git a/code/build/build.c b/code/build/build.c index b7a5cad..edfc4fe 100644 --- a/code/build/build.c +++ b/code/build/build.c @@ -1,20 +1,48 @@ #define NOB_IMPLEMENTATION #include "nob.h" -// Replicate your compiler and linker flags -const char *CFLAGS = "/nologo /W3 /WX /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include /Icode\\kernel\\include"; -const char *LWIP_CFLAGS = "/nologo /W3 /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include /Icode\\kernel\\include /Icode\\drivers\\net\\lwip\\include /Icode\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706"; -const char *LFLAGS = "/nologo /NODEFAULTLIB /DEBUG"; +// Replicate your compiler and linker flags. Keep the literals in macros so +// they can be used in file-scope aggregate initializers. MSVC C rejects +// initializing a static object with the value of another object, even if that +// object is a const char*. +#define CFLAGS_LITERAL "/nologo /W3 /WX /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi- /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include /Icode\\kernel\\include" +#define LWIP_CFLAGS_LITERAL CFLAGS_LITERAL " /Icode\\drivers\\net\\lwip\\include /Icode\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706" +#define LFLAGS_LITERAL "/nologo /NODEFAULTLIB /DEBUG" + +const char *CFLAGS = CFLAGS_LITERAL; +const char *LWIP_CFLAGS = LWIP_CFLAGS_LITERAL; +const char *LFLAGS = LFLAGS_LITERAL; + +typedef struct Driver_Target { + const char *name; + const char *obj_dir; + const char *sources; + const char *cflags; +} Driver_Target; + +#define DRIVER_TARGET(name_, obj_dir_, sources_) { (name_), (obj_dir_), (sources_), CFLAGS_LITERAL } +#define DRIVER_TARGET_FLAGS(name_, obj_dir_, sources_, cflags_) { (name_), (obj_dir_), (sources_), (cflags_) } + +static const Driver_Target STANDALONE_DRIVERS[] = { + DRIVER_TARGET("ps2", "obj\\drivers\\ps2", "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"), + DRIVER_TARGET("ide", "obj\\drivers\\ide", "code\\drivers\\storage\\ide\\ide.c code\\drivers\\storage\\ide\\ide_bus.c"), + DRIVER_TARGET("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c"), + DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"), + DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c"), + DRIVER_TARGET("rtl816x", "obj\\drivers\\rtl816x", "code\\drivers\\net\\rtl816x\\rtl816x.c code\\drivers\\net\\rtl816x\\rtl816x_pci.c"), + DRIVER_TARGET("rtl8139", "obj\\drivers\\rtl8139", "code\\drivers\\net\\rtl8139\\rtl8139.c code\\drivers\\net\\rtl8139\\rtl8139_pci.c"), +}; void create_layout() { const char *dirs[] = { - "obj", "bin", "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal", - "obj\\syscall", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init", - "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", - "obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget", + "obj", "bin", "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal", + "obj\\syscall", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init", + "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", + "obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget", "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video" }; for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) nob_mkdir_if_not_exists(dirs[i]); + for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) nob_mkdir_if_not_exists(STANDALONE_DRIVERS[i].obj_dir); } // Helper to run a shell command string and wait for it @@ -60,7 +88,7 @@ int main(int argc, char **argv) { "cl /nologo /c /Foobj\\libc.obj code\\common\\libc.c", // Bootloader - nob_temp_sprintf("cl %s /MP /c /Icode\\boot\\uefi\\include /Foobj\\boot\\ code\\boot\\uefi\\*.c && link %s /SUBSYSTEM:EFI_APPLICATION /ENTRY:UefiMain /OUT:bin\\bootx64.efi code\\boot\\uefi\\efilib.lib obj\\ini_parser.obj obj\\libc.obj obj\\boot\\*.obj", CFLAGS, LFLAGS), + nob_temp_sprintf("cl %s /MP /c /Icode\\boot\\uefi\\include /Foobj\\boot\\ code\\boot\\uefi\\*.c && link %s /FORCE:MULTIPLE /SUBSYSTEM:EFI_APPLICATION /ENTRY:UefiMain /OUT:bin\\bootx64.efi obj\\libc.obj obj\\ini_parser.obj code\\boot\\uefi\\efilib.lib obj\\boot\\*.obj", CFLAGS, LFLAGS), // HAL nob_temp_sprintf("cl %s /MP /c /Foobj\\hal\\ code\\kernel\\hal\\*.c && link %s /DLL /OUT:bin\\hal.dll /ENTRY:HalInitialize /IGNORE:4216 obj\\printf.obj obj\\libc.obj obj\\hal\\*.obj", CFLAGS, LFLAGS), @@ -82,23 +110,13 @@ int main(int argc, char **argv) { nob_log(NOB_INFO, "--- Compiling Drivers and Apps in Parallel ---"); Nob_Procs procs = {0}; - // Queue standalone Drivers - const char *drivers[][2] = { - {"ide", "storage\\ide"}, - {"ahci", "storage\\ahci"}, - {"fat32", "fs\\fat32"}, - {"usb", "bus\\usb"}, - {"rtl816x", "net\\rtl816x"}, - {"rtl8139", "net\\rtl8139"} - }; - - for (size_t i = 0; i < NOB_ARRAY_LEN(drivers); i++) { - const char *name = drivers[i][0]; - const char *path = drivers[i][1]; - + // Queue standalone drivers. Each driver is now a target with one or more + // translation units, but still links to one .sys image with DriverEntry. + for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) { + const Driver_Target *driver = &STANDALONE_DRIVERS[i]; const char *cmd = nob_temp_sprintf( - "cl %s /c /Foobj\\drivers\\%s.obj code\\drivers\\%s\\%s.c && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj obj\\drivers\\%s.obj", - CFLAGS, name, path, name, name, name + "cl %s /MP /c /Fo%s\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj %s\\*.obj", + driver->cflags, driver->obj_dir, driver->sources, driver->name, driver->obj_dir ); nob_da_append(&procs, run_async(cmd)); } @@ -179,7 +197,8 @@ int main(int argc, char **argv) { {"bin\\win32.dll", "Z:\\system\\win32.dll"}, {"bin\\posix.dll", "Z:\\system\\posix.dll"}, {"code\\settings.ini", "Z:\\system\\settings.ini"}, {"bin\\init.exe", "Z:\\system\\init.exe"}, - {"bin\\ide.sys", "Z:\\drivers\\ide.sys"}, {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"}, + {"bin\\ps2.sys", "Z:\\drivers\\ps2.sys"}, {"bin\\ide.sys", "Z:\\drivers\\ide.sys"}, + {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"}, {"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"}, {"bin\\lwip.sys", "Z:\\drivers\\lwip.sys"}, {"bin\\usb.sys", "Z:\\drivers\\usb.sys"}, {"bin\\rtl816x.sys", "Z:\\drivers\\rtl816x.sys"}, {"bin\\rtl8139.sys", "Z:\\drivers\\rtl8139.sys"}, {"code\\drivers\\drivers.ini", "Z:\\drivers\\drivers.ini"}, diff --git a/code/common/libc.c b/code/common/libc.c index 3a39c2e..fc76cc9 100644 --- a/code/common/libc.c +++ b/code/common/libc.c @@ -1,6 +1,7 @@ // everyone needs a little libc #include #include +#include char* strcpy(char* dest, const char* src) { char* d = dest; @@ -100,22 +101,110 @@ char* strrchr(const char* s, int c) { return (char*)last; } +void* memset(void* dest, int c, size_t n) { + unsigned char* d = (unsigned char*)dest; + size_t qwords = n / 8; + size_t tail = n % 8; + + if (qwords > 0) { + uint64_t value = (unsigned char)c; + value |= value << 8; + value |= value << 16; + value |= value << 32; + __stosq((unsigned __int64*)d, value, qwords); + d += qwords * 8; + } + + if (tail > 0) { + __stosb(d, (unsigned char)c, tail); + } + + return dest; +} + +void* memcpy(void* dest, const void* src, size_t n) { + unsigned char* d = (unsigned char*)dest; + const unsigned char* s = (const unsigned char*)src; + size_t qwords = n / 8; + size_t tail = n % 8; + + if (qwords > 0) { + __movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords); + d += qwords * 8; + s += qwords * 8; + } + + if (tail > 0) { + __movsb(d, s, tail); + } + + return dest; +} + void* memmove(void* dest, const void* src, size_t n) { - uint8_t* d = (uint8_t *)dest; - const uint8_t* s = (const uint8_t *)src; + unsigned char* d = (unsigned char*)dest; + const unsigned char* s = (const unsigned char*)src; + if (d < s) { - for (size_t i = 0; i < n; i++) d[i] = s[i]; + // Forward copy: safe to use chunked __movsq + tail + size_t qwords = n / 8; + size_t tail = n % 8; + if (qwords > 0) { + __movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords); + d += qwords * 8; + s += qwords * 8; + } + if (tail > 0) { + __movsb(d, s, tail); + } } else if (d > s) { - for (size_t i = n; i > 0; i--) d[i - 1] = s[i - 1]; + // Backward copy: must process from end to avoid corruption + size_t qwords = n / 8; + size_t tail = n % 8; + + // Copy tail bytes first from the very end + if (tail > 0) { + for (size_t i = tail; i > 0; i--) { + d[qwords * 8 + i - 1] = s[qwords * 8 + i - 1]; + } + } + + // Copy 64-bit chunks backward + if (qwords > 0) { + unsigned __int64* dq = (unsigned __int64*)d; + const unsigned __int64* sq = (const unsigned __int64*)s; + for (size_t i = qwords; i > 0; i--) { + dq[i - 1] = sq[i - 1]; + } + } } return dest; } int memcmp(const void* s1, const void* s2, size_t n) { - const uint8_t* a = (const uint8_t *)s1; - const uint8_t* b = (const uint8_t *)s2; - for (size_t i = 0; i < n; i++) { - if (a[i] != b[i]) return a[i] - b[i]; + const unsigned __int64* a64 = (const unsigned __int64*)s1; + const unsigned __int64* b64 = (const unsigned __int64*)s2; + size_t qwords = n / 8; + size_t tail = n % 8; + + // Compare 8-byte blocks + for (size_t i = 0; i < qwords; i++) { + if (a64[i] != b64[i]) { + // Locate the exact mismatching byte within the quadword + const unsigned char* a8 = (const unsigned char*)&a64[i]; + const unsigned char* b8 = (const unsigned char*)&b64[i]; + for (size_t j = 0; j < 8; j++) { + if (a8[j] != b8[j]) return (int)a8[j] - (int)b8[j]; + } + } } + + // Compare remaining tail bytes + const unsigned char* a8 = (const unsigned char*)(a64 + qwords); + const unsigned char* b8 = (const unsigned char*)(b64 + qwords); + for (size_t i = 0; i < tail; i++) { + if (a8[i] != b8[i]) return (int)a8[i] - (int)b8[i]; + } + return 0; } diff --git a/code/drivers/bus/usb/usb.c b/code/drivers/bus/usb/usb.c index 9a6feb6..a84e49d 100644 --- a/code/drivers/bus/usb/usb.c +++ b/code/drivers/bus/usb/usb.c @@ -1,9 +1,86 @@ -#include -#include -#include +// usb.c - USB bus driver entry point and PCI controller discovery. +#include "usb.h" + +XHCI_CONTROLLER gXhci; +bool gHaveXhci = false; + +const char* UsbControllerName(uint8_t progIf) { + switch (progIf) { + case 0x00: return "UHCI"; + case 0x10: return "OHCI"; + case 0x20: return "EHCI"; + case 0x30: return "xHCI"; + default: return "USB"; + } +} + + +void UsbDiscoverControllers(const HAL_API* hal) { + uint32_t found = 0; + uint32_t xhciOk = 0; + + for (uint16_t bus = 0; bus < 256; bus++) { + for (uint8_t dev = 0; dev < 32; dev++) { + for (uint8_t fn = 0; fn < 8; fn++) { + uint32_t vd = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_VENDOR_DEVICE); + if ((vd & 0xFFFFu) == PCI_VENDOR_INVALID) + continue; + + uint32_t classReg = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_CLASS); + uint8_t classCode = (uint8_t)(classReg >> 24); + uint8_t subclass = (uint8_t)(classReg >> 16); + uint8_t progIf = (uint8_t)(classReg >> 8); + if (classCode != PCI_CLASS_SERIAL_BUS || subclass != PCI_SUBCLASS_USB) + continue; + + USB_BAR bar0; + bool hasBar0 = UsbReadBar(hal, (uint8_t)bus, dev, fn, 0, &bar0); + uint32_t rawBar0 = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_BAR0); + uint32_t rawBar1 = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_BAR0 + 4u); + uint8_t irq = hal->Pci.GetIrqLine((uint8_t)bus, dev, fn); + + DbgPrint("usb: found %s controller at %u:%u.%u vendor=%04x device=%04x irq=%u bar0raw=%08x bar1raw=%08x", + UsbControllerName(progIf), + bus, dev, fn, + vd & 0xFFFFu, vd >> 16, + irq, rawBar0, rawBar1); + if (hasBar0 && bar0.Kind == UsbBarMmio) { + DbgPrint(" mmio=0x%016llX\n", (unsigned long long)bar0.Base); + } else if (hasBar0 && bar0.Kind == UsbBarIo) { + DbgPrint(" io=0x%016llX\n", (unsigned long long)bar0.Base); + } else { + DbgPrint(" no-bar0\n"); + } + + if (progIf == PCI_PROGIF_XHCI) { + KSTATUS st = XhciInitialise(hal, (uint8_t)bus, dev, fn, + (uint16_t)(vd & 0xFFFFu), + (uint16_t)(vd >> 16)); + if (KSUCCESS(st) || st == STATUS_ALREADY_EXISTS) xhciOk++; + else DbgPrint("usb: xHCI init failed st=0x%x\n", st); + } + found++; + } + } + } + + if (!found) + DbgPrint("usb: no PCI USB controllers discovered\n"); + else + DbgPrint("usb: discovery complete; controllers=%u xhci_initialised=%u\n", found, xhciOk); +} + KSTATUS DriverEntry(const HAL_API* hal) { - (void)hal; - DbgPrint("Test runtime driver\n"); + if (!hal) return STATUS_INVALID_PARAMETER; + DbgPrint("usb: discovery starting\n"); + + KSTATUS st = UsbRegisterClassMatchers(); + if (!KSUCCESS(st)) { + DbgPrint("usb: class matcher registration failed st=0x%x\n", st); + return st; + } + + UsbDiscoverControllers(hal); return STATUS_SUCCESS; } diff --git a/code/drivers/bus/usb/usb.h b/code/drivers/bus/usb/usb.h new file mode 100644 index 0000000..d0bba62 --- /dev/null +++ b/code/drivers/bus/usb/usb.h @@ -0,0 +1,593 @@ +// usb.h - internal USB/xHCI driver interfaces shared across usb.sys translation units. +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef USB_DEBUG_VERBOSE +#define USB_DEBUG_VERBOSE 0 +#endif + +#if USB_DEBUG_VERBOSE +#define UsbTrace DbgPrint +#else +static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } +#endif + + +#define PCI_VENDOR_INVALID 0xFFFFu +#define PCI_CLASS_SERIAL_BUS 0x0Cu +#define PCI_SUBCLASS_USB 0x03u +#define PCI_PROGIF_XHCI 0x30u +#define PCI_REG_VENDOR_DEVICE 0x00u +#define PCI_REG_COMMAND 0x04u +#define PCI_REG_CLASS 0x08u +#define PCI_REG_BAR0 0x10u +#define PCI_COMMAND_MEMORY (1u << 1) +#define PCI_COMMAND_MASTER (1u << 2) +#define PCI_COMMAND_INTX_DISABLE (1u << 10) + +#define XHCI_CAP_CAPLENGTH 0x00u +#define XHCI_CAP_HCSPARAMS1 0x04u +#define XHCI_CAP_HCSPARAMS2 0x08u +#define XHCI_CAP_HCCPARAMS1 0x10u +#define XHCI_CAP_DBOFF 0x14u +#define XHCI_CAP_RTSOFF 0x18u + +#define XHCI_OP_USBCMD 0x00u +#define XHCI_OP_USBSTS 0x04u +#define XHCI_OP_PAGESIZE 0x08u +#define XHCI_OP_DNCTRL 0x14u +#define XHCI_OP_CRCR 0x18u +#define XHCI_OP_DCBAAP 0x30u +#define XHCI_OP_CONFIG 0x38u +#define XHCI_OP_PORT_BASE 0x400u +#define XHCI_OP_PORT_STRIDE 0x10u + +#define XHCI_USBCMD_RS (1u << 0) +#define XHCI_USBCMD_HCRST (1u << 1) +#define XHCI_USBCMD_INTE (1u << 2) +#define XHCI_USBCMD_HSEE (1u << 3) + +#define XHCI_USBSTS_HCH (1u << 0) +#define XHCI_USBSTS_HSE (1u << 2) +#define XHCI_USBSTS_EINT (1u << 3) +#define XHCI_USBSTS_PCD (1u << 4) +#define XHCI_USBSTS_CNR (1u << 11) +#define XHCI_USBSTS_HCE (1u << 12) +#define XHCI_USBSTS_CLEAR_MASK (XHCI_USBSTS_HSE | XHCI_USBSTS_EINT | \ + XHCI_USBSTS_PCD | XHCI_USBSTS_HCE) + +#define XHCI_IMAN_IP (1u << 0) +#define XHCI_IMAN_IE (1u << 1) + +#define XHCI_PORTSC_CCS (1u << 0) +#define XHCI_PORTSC_PED (1u << 1) +#define XHCI_PORTSC_OCA (1u << 3) +#define XHCI_PORTSC_PR (1u << 4) +#define XHCI_PORTSC_PP (1u << 9) +#define XHCI_PORTSC_SPEED_SHIFT 10u +#define XHCI_PORTSC_SPEED_MASK (0xFu << XHCI_PORTSC_SPEED_SHIFT) +#define XHCI_PORTSC_CSC (1u << 17) +#define XHCI_PORTSC_PEC (1u << 18) +#define XHCI_PORTSC_WRC (1u << 19) +#define XHCI_PORTSC_OCC (1u << 20) +#define XHCI_PORTSC_PRC (1u << 21) +#define XHCI_PORTSC_PLC (1u << 22) +#define XHCI_PORTSC_CEC (1u << 23) +#define XHCI_PORTSC_CHANGE_MASK (XHCI_PORTSC_CSC | XHCI_PORTSC_PEC | \ + XHCI_PORTSC_WRC | XHCI_PORTSC_OCC | \ + XHCI_PORTSC_PRC | XHCI_PORTSC_PLC | \ + XHCI_PORTSC_CEC) + +#define XHCI_EXT_CAP_LEGACY 0x01u +#define XHCI_EXT_CAP_PROTOCOL 0x02u +#define XHCI_LEGSUP_BIOS_OWNED (1u << 16) +#define XHCI_LEGSUP_OS_OWNED (1u << 24) + +#define XHCI_TRB_TYPE_SHIFT 10u +#define XHCI_TRB_TYPE_MASK 0x3Fu +#define XHCI_TRB_CYCLE (1u << 0) +#define XHCI_TRB_LINK_TC (1u << 1) +#define XHCI_TRB_TYPE_LINK 6u +#define XHCI_TRB_TYPE_NOOP_CMD 23u +#define XHCI_TRB_TYPE_CMD_DONE 33u +#define XHCI_TRB_TYPE_PORT_SC 34u +#define XHCI_TRB_TYPE_TRANSFER 32u +#define XHCI_TRB_TYPE_NORMAL 1u +#define XHCI_TRB_TYPE_SETUP 2u +#define XHCI_TRB_TYPE_DATA 3u +#define XHCI_TRB_TYPE_STATUS 4u +#define XHCI_TRB_TYPE_ENABLE_SLOT 9u +#define XHCI_TRB_TYPE_ADDRESS_DEVICE 11u +#define XHCI_TRB_TYPE_CONFIGURE_ENDPOINT 12u + + +#define XHCI_CC_SUCCESS 1u + +#define XHCI_TRB_ISP (1u << 2) +#define XHCI_TRB_IOC (1u << 5) +#define XHCI_TRB_IDT (1u << 6) +#define XHCI_TRB_DATA_DIR_IN (1u << 16) +#define XHCI_TRB_SETUP_TRT_NONE (0u << 16) +#define XHCI_TRB_SETUP_TRT_IN (3u << 16) +#define XHCI_TRB_STATUS_DIR_IN (1u << 16) + +#define XHCI_HCC1_CSZ (1u << 2) + +#define XHCI_CONTEXT_ENTRIES(n) ((uint32_t)(n) << 27) +#define XHCI_SLOT_SPEED(s) ((uint32_t)(s) << 20) +#define XHCI_SLOT_ROOT_PORT(p) ((uint32_t)(p) << 16) +#define XHCI_EP_TYPE_CONTROL (4u << 3) +#define XHCI_EP_TYPE_BULK_OUT (2u << 3) +#define XHCI_EP_TYPE_BULK_IN (6u << 3) +#define XHCI_EP_TYPE_INT_IN (7u << 3) +#define XHCI_EP_CERR_3 (3u << 1) +#define XHCI_EP_DCS 1ull + +#define USB_REQ_GET_DESCRIPTOR 0x06u +#define USB_REQ_SET_CONFIGURATION 0x09u +#define USB_REQ_SET_IDLE 0x0Au +#define USB_REQ_SET_PROTOCOL 0x0Bu +#define USB_DESC_DEVICE 0x01u +#define USB_DESC_CONFIGURATION 0x02u +#define USB_DESC_INTERFACE 0x04u +#define USB_DESC_ENDPOINT 0x05u +#define USB_DESC_HID 0x21u +#define USB_DESC_CS_INTERFACE 0x24u +#define USB_DIR_OUT 0x00u +#define USB_DIR_IN 0x80u +#define USB_TYPE_STANDARD 0x00u +#define USB_TYPE_CLASS 0x20u +#define USB_RECIP_DEVICE 0x00u +#define USB_RECIP_INTERFACE 0x01u +#define USB_ENDPOINT_DIR_IN 0x80u +#define USB_ENDPOINT_NUMBER_MASK 0x0Fu +#define USB_ENDPOINT_ATTR_TYPE_MASK 0x03u +#define USB_ENDPOINT_TYPE_CONTROL 0x00u +#define USB_ENDPOINT_TYPE_ISOCH 0x01u +#define USB_ENDPOINT_TYPE_BULK 0x02u +#define USB_ENDPOINT_TYPE_INTERRUPT 0x03u +#define USB_CLASS_HID 0x03u +#define USB_CLASS_MASS_STORAGE 0x08u +#define USB_MASS_SUBCLASS_SCSI 0x06u +#define USB_MASS_PROTOCOL_BOT 0x50u +#define USB_HID_SUBCLASS_BOOT 0x01u +#define USB_HID_PROTOCOL_KEYBOARD 0x01u +#define USB_HID_PROTOCOL_MOUSE 0x02u +#define USB_MASS_BOT_CBW_SIGNATURE 0x43425355u +#define USB_MASS_BOT_CSW_SIGNATURE 0x53425355u +#define USB_MASS_BOT_CSW_PASSED 0u +#define USB_MASS_BOT_CSW_FAILED 1u + +#define SCSI_TEST_UNIT_READY 0x00u +#define SCSI_REQUEST_SENSE 0x03u +#define SCSI_INQUIRY 0x12u +#define SCSI_READ_10 0x28u +#define SCSI_WRITE_10 0x2Au +#define SCSI_READ_CAPACITY_10 0x25u + + +#define XHCI_RING_TRBS 256u +#define XHCI_EVENT_TRBS 256u +#define XHCI_DCBAA_ENTRIES 256u +#define XHCI_MMIO_FLOOR_BYTES 0x10000u +#define XHCI_MAX_SCRATCHPADS 1024u +#define XHCI_MAX_TRACKED_SLOTS 64u +#define XHCI_MAX_ROOT_PORTS 255u +#define XHCI_EP0_RING_TRBS 256u +#define XHCI_HID_RING_TRBS 256u +#define XHCI_BULK_RING_TRBS 256u +#define XHCI_MAX_HID_REPORT 16u +#define USB_STORAGE_DMA_PAGES 1u +#define USB_STORAGE_MAX_BYTES (USB_STORAGE_DMA_PAGES * PAGE_SIZE) +#define USB_STORAGE_SECTOR_SIZE 512u +#define USB_MAX_TRACKED_INTERFACES 8u +#define USB_MAX_TRACKED_ENDPOINTS 8u + +#pragma pack(push, 1) +typedef struct XHCI_TRB { + uint32_t ParameterLo; + uint32_t ParameterHi; + uint32_t Status; + uint32_t Control; +} XHCI_TRB; + +typedef struct XHCI_ERST_ENTRY { + uint64_t RingSegmentBase; + uint32_t RingSegmentSize; + uint32_t Reserved; +} XHCI_ERST_ENTRY; + +typedef struct USB_SETUP_PACKET { + uint8_t bmRequestType; + uint8_t bRequest; + uint16_t wValue; + uint16_t wIndex; + uint16_t wLength; +} USB_SETUP_PACKET; + +typedef struct USB_DEVICE_DESCRIPTOR { + uint8_t bLength; + uint8_t bDescriptorType; + uint16_t bcdUSB; + uint8_t bDeviceClass; + uint8_t bDeviceSubClass; + uint8_t bDeviceProtocol; + uint8_t bMaxPacketSize0; + uint16_t idVendor; + uint16_t idProduct; + uint16_t bcdDevice; + uint8_t iManufacturer; + uint8_t iProduct; + uint8_t iSerialNumber; + uint8_t bNumConfigurations; +} USB_DEVICE_DESCRIPTOR; + +typedef struct USB_CONFIGURATION_DESCRIPTOR { + uint8_t bLength; + uint8_t bDescriptorType; + uint16_t wTotalLength; + uint8_t bNumInterfaces; + uint8_t bConfigurationValue; + uint8_t iConfiguration; + uint8_t bmAttributes; + uint8_t bMaxPower; +} USB_CONFIGURATION_DESCRIPTOR; + +typedef struct USB_INTERFACE_DESCRIPTOR { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bInterfaceNumber; + uint8_t bAlternateSetting; + uint8_t bNumEndpoints; + uint8_t bInterfaceClass; + uint8_t bInterfaceSubClass; + uint8_t bInterfaceProtocol; + uint8_t iInterface; +} USB_INTERFACE_DESCRIPTOR; + +typedef struct USB_ENDPOINT_DESCRIPTOR { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bEndpointAddress; + uint8_t bmAttributes; + uint16_t wMaxPacketSize; + uint8_t bInterval; +} USB_ENDPOINT_DESCRIPTOR; + +typedef struct USB_MASS_CBW { + uint32_t Signature; + uint32_t Tag; + uint32_t DataTransferLength; + uint8_t Flags; + uint8_t Lun; + uint8_t CommandBlockLength; + uint8_t CommandBlock[16]; +} USB_MASS_CBW; + +typedef struct USB_MASS_CSW { + uint32_t Signature; + uint32_t Tag; + uint32_t DataResidue; + uint8_t Status; +} USB_MASS_CSW; +#pragma pack(pop) + +typedef struct USB_TRACKED_INTERFACE { + bool Present; + USB_INTERFACE_DESCRIPTOR Descriptor; + uint8_t EndpointCount; + USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS]; +} USB_TRACKED_INTERFACE; + +typedef enum USB_BAR_KIND { + UsbBarNone, + UsbBarIo, + UsbBarMmio +} USB_BAR_KIND; + +typedef struct USB_BAR { + USB_BAR_KIND Kind; + uint64_t Base; + uint8_t Index; +} USB_BAR; + +typedef struct XHCI_DEVICE { + bool Allocated; + bool Addressed; + uint8_t SlotId; + uint8_t Port; + uint8_t Speed; + uint16_t MaxPacketSize0; + uint16_t VendorId; + uint16_t ProductId; + uint16_t BcdUsb; + uint16_t BcdDevice; + uint8_t DeviceClass; + uint8_t DeviceSubclass; + uint8_t DeviceProtocol; + uint8_t ConfigurationValue; + bool Configured; + + uint8_t InterfaceCount; + USB_TRACKED_INTERFACE Interfaces[USB_MAX_TRACKED_INTERFACES]; + + bool HidCandidate; + bool HidActive; + bool HidKeyboard; + bool HidMouse; + bool HidArmed; + uint8_t HidInterface; + uint8_t HidProtocol; + uint8_t HidEndpointAddress; + uint8_t HidEndpointId; + uint8_t HidInterval; + uint16_t HidMaxPacket; + uint16_t HidReportLength; + uint32_t HidEnqueue; + uint8_t HidCycle; + uint8_t HidLastModifiers; + uint8_t HidLastKeys[6]; + uint8_t HidLastButtons; + uint64_t HidLastTrbPhys; + + bool MassCandidate; + bool MassActive; + uint8_t MassInterface; + uint8_t MassBulkInAddress; + uint8_t MassBulkOutAddress; + uint8_t MassBulkInEndpointId; + uint8_t MassBulkOutEndpointId; + uint16_t MassBulkInMaxPacket; + uint16_t MassBulkOutMaxPacket; + uint32_t MassInEnqueue; + uint8_t MassInCycle; + uint32_t MassOutEnqueue; + uint8_t MassOutCycle; + uint32_t MassTag; + uint32_t MassBlockSize; + uint64_t MassBlockCount; + DEVICE_OBJECT* MassDeviceObject; + volatile long MassIoBusy; + + PHYSICAL_ADDRESS InputContextPhys; + PHYSICAL_ADDRESS DeviceContextPhys; + PHYSICAL_ADDRESS Ep0RingPhys; + PHYSICAL_ADDRESS HidRingPhys; + PHYSICAL_ADDRESS MassInRingPhys; + PHYSICAL_ADDRESS MassOutRingPhys; + PHYSICAL_ADDRESS DescriptorPhys; + PHYSICAL_ADDRESS HidBufferPhys; + PHYSICAL_ADDRESS MassBufferPhys; + + uint8_t* InputContext; + uint8_t* DeviceContext; + XHCI_TRB* Ep0Ring; + XHCI_TRB* HidRing; + XHCI_TRB* MassInRing; + XHCI_TRB* MassOutRing; + uint8_t* DescriptorBuffer; + uint8_t* HidBuffer; + uint8_t* MassBuffer; + + uint32_t Ep0Enqueue; + uint8_t Ep0Cycle; +} XHCI_DEVICE; + +typedef struct XHCI_COMMAND_RESULT { + uint32_t CompletionCode; + uint8_t SlotId; +} XHCI_COMMAND_RESULT; + +typedef struct XHCI_TRANSFER_RESULT { + uint32_t CompletionCode; + uint8_t SlotId; + uint8_t EndpointId; + uint32_t Residue; +} XHCI_TRANSFER_RESULT; + +typedef struct XHCI_WAIT_TARGET { + uint64_t CommandPhys; + uint64_t TransferPhys; + uint8_t TransferSlotId; + uint8_t TransferEndpointId; + bool CommandMatched; + bool TransferMatched; + XHCI_COMMAND_RESULT Command; + XHCI_TRANSFER_RESULT Transfer; +} XHCI_WAIT_TARGET; + +typedef struct XHCI_CONTROLLER { + const HAL_API* Hal; + uint8_t Bus; + uint8_t Dev; + uint8_t Fn; + uint8_t Irq; + bool UsingMsi; + + uint64_t MmioPhys; + uint32_t MmioBytes; + volatile uint8_t* Mmio; + volatile uint8_t* Cap; + volatile uint8_t* Op; + volatile uint8_t* Runtime; + volatile uint8_t* Doorbells; + + uint8_t CapLength; + uint16_t Version; + uint32_t Hcs1; + uint32_t Hcs2; + uint32_t Hcc1; + uint32_t DbOff; + uint32_t RtsOff; + uint8_t MaxSlots; + uint16_t MaxIntrs; + uint8_t MaxPorts; + uint16_t ScratchpadCount; + uint8_t PortProtocolMajor[XHCI_MAX_ROOT_PORTS + 1u]; + uint8_t PortProtocolMinor[XHCI_MAX_ROOT_PORTS + 1u]; + + PHYSICAL_ADDRESS DcbaaPhys; + PHYSICAL_ADDRESS ScratchpadArrayPhys; + PHYSICAL_ADDRESS ScratchpadPhys[XHCI_MAX_SCRATCHPADS]; + PHYSICAL_ADDRESS CommandRingPhys; + PHYSICAL_ADDRESS EventRingPhys; + PHYSICAL_ADDRESS ErstPhys; + + uint64_t* Dcbaa; + uint64_t* ScratchpadArray; + XHCI_TRB* CommandRing; + XHCI_TRB* EventRing; + XHCI_ERST_ENTRY* Erst; + + uint32_t CommandEnqueue; + uint8_t CommandCycle; + uint32_t EventDequeue; + uint8_t EventCycle; + + uint8_t ContextBytes; + XHCI_DEVICE Devices[XHCI_MAX_TRACKED_SLOTS + 1u]; + uint32_t HidDeviceCount; + + KDPC EventDpc; + volatile bool InterruptsEnabled; + volatile uint32_t InterruptCount; + volatile uint32_t EventDpcCount; + SPINLOCK EventLock; + + volatile XHCI_WAIT_TARGET* ActiveWait; + volatile bool Running; +} XHCI_CONTROLLER; + + +extern XHCI_CONTROLLER gXhci; +extern bool gHaveXhci; +extern DRIVER_OBJECT gUsbStorageDriver; + +uint32_t UsbAlignUp32(uint32_t value, uint32_t align); +void XhciBarrier(void); +uint8_t XhciRead8(volatile uint8_t* base, uint32_t reg); +uint16_t XhciRead16(volatile uint8_t* base, uint32_t reg); +uint32_t XhciRead32(volatile uint8_t* base, uint32_t reg); +void XhciWrite32(volatile uint8_t* base, uint32_t reg, uint32_t value); +uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg); +void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value); +void UsbDelayUs(const HAL_API* hal, uint32_t us); +bool UsbWaitSet32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs); +bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs); +bool UsbReadBar(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, USB_BAR* out); +KSTATUS UsbMapMmio(const HAL_API* hal, uint64_t physBase, uint32_t bytes, volatile uint8_t** outMmio); +void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys); +uint16_t XhciScratchpadCount(uint32_t hcs2); +uint32_t XhciTrbType(const XHCI_TRB* trb); +uint32_t XhciCompletionCode(const XHCI_TRB* trb); +uint8_t XhciEventSlotId(const XHCI_TRB* trb); +uint8_t XhciEventEndpointId(const XHCI_TRB* trb); +uint32_t XhciTransferResidue(const XHCI_TRB* trb); +uint32_t* XhciInputControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +uint32_t* XhciInputSlotContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +uint32_t* XhciInputEndpointContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId); +uint32_t XhciEpMaxPacket(uint8_t speed); +uint64_t UsbSetupPacketToU64(const USB_SETUP_PACKET* setup); +const char* XhciPortSpeedName(uint32_t speed); +void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci); +uint8_t XhciPortProtocolMajor(XHCI_CONTROLLER* xhci, uint8_t port, uint32_t portsc); +const char* UsbEndpointTypeName(uint8_t type); +const char* UsbDirectionName(uint8_t endpointAddress); +const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto); +void XhciLogPortStatus(XHCI_CONTROLLER* xhci); +void XhciAckInterrupter0(XHCI_CONTROLLER* xhci); +void XhciUpdateErdp(XHCI_CONTROLLER* xhci); +bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target); +bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target); +bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target); +KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs); +void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2); +void XhciIsr(void); +KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci); +KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci); +KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci); +KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci); +KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci); +uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index); +KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, XHCI_COMMAND_RESULT* outResult); +KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci); +KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId); +KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS XhciStart(XHCI_CONTROLLER* xhci); +void XhciEnableInterrupts(XHCI_CONTROLLER* xhci); +uint32_t XhciPortReg(uint8_t port); +void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port); +KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed); +KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index); +KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys); +KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length); +KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup); +KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc); +KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength); +uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress); +uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval); +KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys); +bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key); +INPUT_KEY XhciHidModifierKey(uint8_t bit); +void XhciSubmitHidKey(INPUT_KEY key, bool pressed); +void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len); +void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len); +KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +void XhciArmAllHid(XHCI_CONTROLLER* xhci); +void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue); +KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci); +KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub, uint8_t endpointType, bool wantIn); +bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context); +KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context); +void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle); +KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn); +uint32_t UsbBe32(const uint8_t* p); +void UsbStoreBe32(uint8_t* p, uint32_t v); +void UsbStoreBe16(uint8_t* p, uint16_t v); +KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, bool write, uint64_t lba, uint32_t sectors, void* buffer); +void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +void UsbStorageReleaseIo(XHCI_DEVICE* dev); +KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp); +KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +uint8_t UsbFindFreeDiskIndex(void); +KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev); +bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context); +KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context); +KSTATUS UsbRegisterClassMatchers(void); +KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationValue); +void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_CONFIGURATION_DESCRIPTOR* config, uint16_t totalLength); +void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); +KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_DEVICE_DESCRIPTOR* devDesc); +KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port); +void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci); +KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device); diff --git a/code/drivers/bus/usb/usb_core.c b/code/drivers/bus/usb/usb_core.c new file mode 100644 index 0000000..52a25b3 --- /dev/null +++ b/code/drivers/bus/usb/usb_core.c @@ -0,0 +1,361 @@ +// usb_core.c - USB descriptor/configuration parsing, interface publishing, and root-port enumeration. +#include "usb.h" + +KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + uint8_t configurationValue) { + USB_SETUP_PACKET setup; + setup.bmRequestType = USB_DIR_OUT; + setup.bRequest = USB_REQ_SET_CONFIGURATION; + setup.wValue = configurationValue; + setup.wIndex = 0; + setup.wLength = 0; + + KSTATUS st = XhciControlNoData(xhci, dev, &setup); + if (!KSUCCESS(st)) { + DbgPrint("usb: SET_CONFIGURATION slot=%u value=%u failed st=0x%x\n", + dev->SlotId, configurationValue, st); + return st; + } + + dev->ConfigurationValue = configurationValue; + dev->Configured = true; + UsbTrace("usb: SET_CONFIGURATION slot=%u value=%u complete\n", + dev->SlotId, configurationValue); + return STATUS_SUCCESS; +} + + +void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + const USB_CONFIGURATION_DESCRIPTOR* config, + uint16_t totalLength) { + (void)xhci; + + memset(dev->Interfaces, 0, sizeof(dev->Interfaces)); + dev->InterfaceCount = 0; + + UsbTrace("usb: config slot=%u value=%u total=%u interfaces=%u attrs=%02x maxpower=%umA\n", + dev->SlotId, + config->bConfigurationValue, + totalLength, + config->bNumInterfaces, + config->bmAttributes, + (uint32_t)config->bMaxPower * 2u); + + uint16_t off = config->bLength; + uint8_t currentInterface = 0xFFu; + uint8_t currentClass = 0; + uint8_t currentSubClass = 0; + uint8_t currentProtocol = 0; + bool hidCandidate = false; + bool massCandidate = false; + uint8_t hidInterruptIn = 0; + uint8_t massBulkIn = 0; + uint8_t massBulkOut = 0; + USB_TRACKED_INTERFACE* tracked = (USB_TRACKED_INTERFACE*)0; + + while (off + 2u <= totalLength) { + const uint8_t* desc = dev->DescriptorBuffer + off; + uint8_t len = desc[0]; + uint8_t type = desc[1]; + + if (len < 2 || off + len > totalLength) { + DbgPrint("usb: descriptor parse stopped slot=%u off=%u len=%u type=%u total=%u\n", + dev->SlotId, off, len, type, totalLength); + break; + } + + if (type == USB_DESC_INTERFACE && len >= sizeof(USB_INTERFACE_DESCRIPTOR)) { + const USB_INTERFACE_DESCRIPTOR* iface = (const USB_INTERFACE_DESCRIPTOR*)desc; + currentInterface = iface->bInterfaceNumber; + currentClass = iface->bInterfaceClass; + currentSubClass = iface->bInterfaceSubClass; + currentProtocol = iface->bInterfaceProtocol; + hidCandidate = (currentClass == USB_CLASS_HID); + massCandidate = (currentClass == USB_CLASS_MASS_STORAGE && + currentSubClass == USB_MASS_SUBCLASS_SCSI && + currentProtocol == USB_MASS_PROTOCOL_BOT); + hidInterruptIn = 0; + massBulkIn = 0; + massBulkOut = 0; + tracked = (USB_TRACKED_INTERFACE*)0; + + if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) { + tracked = &dev->Interfaces[dev->InterfaceCount++]; + memset(tracked, 0, sizeof(*tracked)); + tracked->Present = true; + memcpy(&tracked->Descriptor, iface, sizeof(*iface)); + } else { + DbgPrint("usb: slot=%u too many interfaces, interface %u not published\n", + dev->SlotId, iface->bInterfaceNumber); + } + + UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)\n", + dev->SlotId, + iface->bInterfaceNumber, + iface->bAlternateSetting, + iface->bNumEndpoints, + currentClass, + currentSubClass, + currentProtocol, + UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol)); + } else if (type == USB_DESC_ENDPOINT && len >= sizeof(USB_ENDPOINT_DESCRIPTOR)) { + const USB_ENDPOINT_DESCRIPTOR* ep = (const USB_ENDPOINT_DESCRIPTOR*)desc; + uint8_t epType = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK; + uint8_t epNumber = ep->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK; + bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0; + + if (tracked && tracked->EndpointCount < USB_MAX_TRACKED_ENDPOINTS) { + memcpy(&tracked->Endpoints[tracked->EndpointCount++], ep, sizeof(*ep)); + } else if (tracked) { + DbgPrint("usb: slot=%u if=%u too many endpoints, endpoint %02x not published\n", + dev->SlotId, currentInterface, ep->bEndpointAddress); + } + + UsbTrace("usb: endpoint slot=%u if=%u addr=%02x ep=%u dir=%s type=%s maxpkt=%u interval=%u\n", + dev->SlotId, + currentInterface, + ep->bEndpointAddress, + epNumber, + UsbDirectionName(ep->bEndpointAddress), + UsbEndpointTypeName(epType), + ep->wMaxPacketSize, + ep->bInterval); + + if (hidCandidate && in && epType == USB_ENDPOINT_TYPE_INTERRUPT && hidInterruptIn == 0) { + hidInterruptIn = ep->bEndpointAddress; + } + if (massCandidate && epType == USB_ENDPOINT_TYPE_BULK) { + if (in && massBulkIn == 0) massBulkIn = ep->bEndpointAddress; + if (!in && massBulkOut == 0) massBulkOut = ep->bEndpointAddress; + } + } else if (type == USB_DESC_HID) { + UsbTrace("usb: hid descriptor slot=%u if=%u len=%u\n", + dev->SlotId, currentInterface, len); + } else if (type == USB_DESC_CS_INTERFACE) { + UsbTrace("usb: class-specific interface descriptor slot=%u if=%u len=%u subtype=%u\n", + dev->SlotId, currentInterface, len, len >= 3 ? desc[2] : 0); + } + + uint16_t next = (uint16_t)(off + len); + if (hidCandidate && hidInterruptIn && + (next >= totalLength || (next + 1u < totalLength && dev->DescriptorBuffer[next + 1u] == USB_DESC_INTERFACE))) { + UsbTrace("usb: class candidate slot=%u if=%u hid interrupt-in=%02x subclass=%02x proto=%02x\n", + dev->SlotId, + currentInterface, + hidInterruptIn, + currentSubClass, + currentProtocol); + hidCandidate = false; + } + if (massCandidate && massBulkIn && massBulkOut && + (next >= totalLength || (next + 1u < totalLength && dev->DescriptorBuffer[next + 1u] == USB_DESC_INTERFACE))) { + UsbTrace("usb: class candidate slot=%u if=%u mass-storage bulk-in=%02x bulk-out=%02x\n", + dev->SlotId, + currentInterface, + massBulkIn, + massBulkOut); + massCandidate = false; + } + + off = next; + } + + if (hidCandidate && hidInterruptIn) { + UsbTrace("usb: class candidate slot=%u if=%u hid interrupt-in=%02x subclass=%02x proto=%02x\n", + dev->SlotId, + currentInterface, + hidInterruptIn, + currentSubClass, + currentProtocol); + } + if (massCandidate && massBulkIn && massBulkOut) { + UsbTrace("usb: class candidate slot=%u if=%u mass-storage bulk-in=%02x bulk-out=%02x\n", + dev->SlotId, + currentInterface, + massBulkIn, + massBulkOut); + } +} + + +void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + for (uint8_t i = 0; i < dev->InterfaceCount; i++) { + USB_TRACKED_INTERFACE* iface = &dev->Interfaces[i]; + if (!iface->Present) + continue; + + IO_PUBLISHED_DEVICE pub; + memset(&pub, 0, sizeof(pub)); + pub.BusType = IoBusUsb; + pub.Name = "usb-interface"; + pub.BusContext = xhci; + pub.DeviceContext = dev; + + pub.u.Usb.VendorId = dev->VendorId; + pub.u.Usb.ProductId = dev->ProductId; + pub.u.Usb.BcdUsb = dev->BcdUsb; + pub.u.Usb.BcdDevice = dev->BcdDevice; + pub.u.Usb.DeviceClass = dev->DeviceClass; + pub.u.Usb.DeviceSubclass = dev->DeviceSubclass; + pub.u.Usb.DeviceProtocol = dev->DeviceProtocol; + pub.u.Usb.SlotId = dev->SlotId; + pub.u.Usb.Port = dev->Port; + pub.u.Usb.Speed = dev->Speed; + pub.u.Usb.ConfigurationValue = dev->ConfigurationValue; + pub.u.Usb.InterfaceNumber = iface->Descriptor.bInterfaceNumber; + pub.u.Usb.AlternateSetting = iface->Descriptor.bAlternateSetting; + pub.u.Usb.InterfaceClass = iface->Descriptor.bInterfaceClass; + pub.u.Usb.InterfaceSubclass = iface->Descriptor.bInterfaceSubClass; + pub.u.Usb.InterfaceProtocol = iface->Descriptor.bInterfaceProtocol; + pub.u.Usb.EndpointCount = iface->EndpointCount; + if (pub.u.Usb.EndpointCount > IO_PUBLISHED_USB_MAX_ENDPOINTS) + pub.u.Usb.EndpointCount = IO_PUBLISHED_USB_MAX_ENDPOINTS; + + for (uint8_t ep = 0; ep < pub.u.Usb.EndpointCount; ep++) { + pub.u.Usb.Endpoints[ep].Address = iface->Endpoints[ep].bEndpointAddress; + pub.u.Usb.Endpoints[ep].Attributes = iface->Endpoints[ep].bmAttributes; + pub.u.Usb.Endpoints[ep].MaxPacketSize = iface->Endpoints[ep].wMaxPacketSize; + pub.u.Usb.Endpoints[ep].Interval = iface->Endpoints[ep].bInterval; + } + + KSTATUS st = IoPublishDevice(&pub); + if (!KSUCCESS(st)) { + DbgPrint("usb: publish interface slot=%u if=%u failed st=0x%x\n", + dev->SlotId, iface->Descriptor.bInterfaceNumber, st); + } + } +} + + +KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + const USB_DEVICE_DESCRIPTOR* devDesc) { + if (devDesc->bNumConfigurations == 0) { + DbgPrint("usb: slot=%u has no configurations\n", dev->SlotId); + return STATUS_NOT_FOUND; + } + + USB_CONFIGURATION_DESCRIPTOR config; + memset(&config, 0, sizeof(config)); + uint16_t totalLength = 0; + KSTATUS st = XhciGetConfigurationDescriptor(xhci, dev, 0, &config, &totalLength); + if (!KSUCCESS(st)) { + DbgPrint("usb: slot=%u configuration descriptor read failed st=0x%x\n", + dev->SlotId, st); + return st; + } + + XhciLogConfigurationTree(xhci, dev, &config, totalLength); + + if (config.bConfigurationValue == 0) { + DbgPrint("usb: slot=%u configuration value 0 is invalid for SET_CONFIGURATION\n", + dev->SlotId); + return STATUS_DEVICE_ERROR; + } + + st = XhciSetConfiguration(xhci, dev, config.bConfigurationValue); + if (!KSUCCESS(st)) return st; + + XhciPublishUsbInterfaces(xhci, dev); + + return STATUS_SUCCESS; +} + + +KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) { + uint8_t speed = 0; + KSTATUS st = XhciResetPort(xhci, port, &speed); + if (!KSUCCESS(st)) return st; + + uint8_t slotId = 0; + st = XhciEnableSlot(xhci, &slotId); + if (!KSUCCESS(st)) return st; + + XHCI_DEVICE* dev = &xhci->Devices[slotId]; + memset(dev, 0, sizeof(*dev)); + dev->Allocated = true; + dev->SlotId = slotId; + dev->Port = port; + dev->Speed = speed; + + st = XhciAllocateDeviceMemory(xhci, dev); + if (!KSUCCESS(st)) return st; + + XhciBuildDefaultControlContext(xhci, dev); + + st = XhciAddressDevice(xhci, dev); + if (!KSUCCESS(st)) return st; + dev->Addressed = true; + + USB_DEVICE_DESCRIPTOR desc; + memset(&desc, 0, sizeof(desc)); + st = XhciGetDeviceDescriptor(xhci, dev, &desc); + if (!KSUCCESS(st)) { + DbgPrint("usb: port %u slot %u descriptor read failed st=0x%x\n", + port, slotId, st); + return st; + } + + dev->MaxPacketSize0 = desc.bMaxPacketSize0; + dev->VendorId = desc.idVendor; + dev->ProductId = desc.idProduct; + dev->BcdUsb = desc.bcdUSB; + dev->BcdDevice = desc.bcdDevice; + dev->DeviceClass = desc.bDeviceClass; + dev->DeviceSubclass = desc.bDeviceSubClass; + dev->DeviceProtocol = desc.bDeviceProtocol; + + DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n", + port, + slotId, + XhciPortSpeedName(speed), + speed, + desc.idVendor, + desc.idProduct, + desc.bDeviceClass, + desc.bDeviceSubClass, + desc.bDeviceProtocol, + desc.bMaxPacketSize0, + desc.bNumConfigurations, + desc.bcdUSB); + + st = XhciDiscoverAndSetConfiguration(xhci, dev, &desc); + if (!KSUCCESS(st)) return st; + + return STATUS_SUCCESS; +} + + +void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) { + uint32_t attempted = 0; + uint32_t configured = 0; + + for (uint8_t port = 1; port <= xhci->MaxPorts; port++) { + uint32_t portsc = XhciRead32(xhci->Op, XhciPortReg(port)); + if (!(portsc & XHCI_PORTSC_CCS)) + continue; + + attempted++; + KSTATUS st = XhciEnumeratePort(xhci, port); + if (KSUCCESS(st)) configured++; + else DbgPrint("usb: root-port %u enumeration failed st=0x%x\n", port, st); + } + + DbgPrint("usb: xHCI root-port enumeration complete; attempted=%u configured=%u\n", + attempted, configured); +} + + +KSTATUS UsbRegisterClassMatchers(void) { + memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver)); + gUsbStorageDriver.Name = "usbstor"; + gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite; + gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite; + + KSTATUS st = IoRegisterDriverMatcher("usbhid", IoBusUsb, UsbHidMatch, UsbHidBind, NULL); + if (!KSUCCESS(st)) return st; + + st = IoRegisterDriverMatcher("usbstor", IoBusUsb, UsbStorageMatch, UsbStorageBind, NULL); + if (!KSUCCESS(st)) return st; + + return STATUS_SUCCESS; +} diff --git a/code/drivers/bus/usb/usb_hid.c b/code/drivers/bus/usb/usb_hid.c new file mode 100644 index 0000000..0cf00f9 --- /dev/null +++ b/code/drivers/bus/usb/usb_hid.c @@ -0,0 +1,367 @@ +// usb_hid.c - USB HID boot keyboard/mouse class binding and report handling. +#include "usb.h" + +bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) { + if (key == 0 || key == 1) return false; + for (uint8_t i = 0; i < 6; i++) { + if (keys[i] == key) return true; + } + return false; +} + + +INPUT_KEY XhciHidModifierKey(uint8_t bit) { + switch (bit) { + case 0: return INPUT_KEY_LEFT_CTRL; + case 1: return INPUT_KEY_LEFT_SHIFT; + case 2: return INPUT_KEY_LEFT_ALT; + case 3: return INPUT_KEY_LEFT_GUI; + case 4: return INPUT_KEY_RIGHT_CTRL; + case 5: return INPUT_KEY_RIGHT_SHIFT; + case 6: return INPUT_KEY_RIGHT_ALT; + case 7: return INPUT_KEY_RIGHT_GUI; + default: return INPUT_KEY_NONE; + } +} + + +void XhciSubmitHidKey(INPUT_KEY key, bool pressed) { + if (key == INPUT_KEY_NONE) return; + INPUT_KEY_EVENT ev; + ev.Key = key; + ev.Pressed = pressed; + KeInputSubmitKey(&ev); +} + + +void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) { + if (len < 8) return; + + uint8_t modifiers = report[0]; + for (uint8_t bit = 0; bit < 8; bit++) { + uint8_t mask = (uint8_t)(1u << bit); + bool wasDown = (dev->HidLastModifiers & mask) != 0; + bool isDown = (modifiers & mask) != 0; + if (wasDown != isDown) + XhciSubmitHidKey(XhciHidModifierKey(bit), isDown); + } + + const uint8_t* keys = &report[2]; + for (uint8_t i = 0; i < 6; i++) { + uint8_t oldKey = dev->HidLastKeys[i]; + if (oldKey > 1 && !XhciHidKeyInReport(keys, oldKey)) + XhciSubmitHidKey((INPUT_KEY)oldKey, false); + } + for (uint8_t i = 0; i < 6; i++) { + uint8_t newKey = keys[i]; + if (newKey > 1 && !XhciHidKeyInReport(dev->HidLastKeys, newKey)) + XhciSubmitHidKey((INPUT_KEY)newKey, true); + } + + dev->HidLastModifiers = modifiers; + memcpy(dev->HidLastKeys, keys, sizeof(dev->HidLastKeys)); +} + + +void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) { + if (len < 3) return; + + int dx = (int)(int8_t)report[1]; + int dy = (int)(int8_t)report[2]; + int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0; + uint8_t buttons = report[0] & 0x07u; + + if (dx || dy || wheel || buttons != dev->HidLastButtons) { + INPUT_MOUSE_EVENT ev; + ev.dx = dx; + ev.dy = dy; + ev.wheel = wheel; + ev.buttons = buttons; + KeInputSubmitMouse(&ev); + } + + dev->HidLastButtons = buttons; +} + + +KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + if (!dev->HidActive || dev->HidArmed || !dev->HidRing || !dev->HidBuffer) + return STATUS_SUCCESS; + + memset(dev->HidBuffer, 0, dev->HidReportLength); + + uint64_t trbPhys = 0; + KSTATUS st = XhciRingAppend(dev->HidRing, + dev->HidRingPhys, + XHCI_HID_RING_TRBS, + &dev->HidEnqueue, + &dev->HidCycle, + dev->HidBufferPhys, + dev->HidReportLength, + (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_ISP | + XHCI_TRB_IOC, + &trbPhys); + if (!KSUCCESS(st)) return st; + + dev->HidLastTrbPhys = trbPhys; + dev->HidArmed = true; + XhciBarrier(); + XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, dev->HidEndpointId); + return STATUS_SUCCESS; +} + + +void XhciArmAllHid(XHCI_CONTROLLER* xhci) { + for (uint8_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++) { + XHCI_DEVICE* dev = &xhci->Devices[slot]; + if (!dev->HidActive) continue; + KSTATUS st = XhciArmHidInterruptIn(xhci, dev); + if (!KSUCCESS(st)) { + DbgPrint("usb: HID arm failed slot=%u ep=%u st=0x%x\n", + dev->SlotId, dev->HidEndpointId, st); + } + } +} + + +void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, + uint32_t completionCode, uint8_t slotId, + uint8_t endpointId, uint32_t residue) { + if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return; + + XHCI_DEVICE* dev = &xhci->Devices[slotId]; + if (!dev->HidActive || endpointId != dev->HidEndpointId || trbPhys != dev->HidLastTrbPhys) + return; + + dev->HidArmed = false; + if (completionCode != XHCI_CC_SUCCESS) { + DbgPrint("usb: HID transfer failed slot=%u ep=%u cc=%u residue=%u\n", + slotId, endpointId, completionCode, residue); + return; + } + + uint16_t length = dev->HidReportLength; + if (residue < length) length = (uint16_t)(length - residue); + + if (dev->HidKeyboard) + XhciProcessBootKeyboardReport(dev, dev->HidBuffer, length); + else if (dev->HidMouse) + XhciProcessBootMouseReport(dev, dev->HidBuffer, length); + + KSTATUS st = XhciArmHidInterruptIn(xhci, dev); + if (!KSUCCESS(st)) { + DbgPrint("usb: HID re-arm failed slot=%u ep=%u st=0x%x\n", + slotId, endpointId, st); + } +} + + +KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) { + if (xhci->HidDeviceCount == 0) + return STATUS_SUCCESS; + + XhciArmAllHid(xhci); + DbgPrint("usb: HID interrupt input armed devices=%u\n", xhci->HidDeviceCount); + return STATUS_SUCCESS; +} + + +KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + USB_SETUP_PACKET setup; + setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE; + setup.bRequest = USB_REQ_SET_PROTOCOL; + setup.wValue = 0; // boot protocol + setup.wIndex = dev->HidInterface; + setup.wLength = 0; + + KSTATUS st = XhciControlNoData(xhci, dev, &setup); + if (!KSUCCESS(st)) { + DbgPrint("usb: HID SET_PROTOCOL boot failed slot=%u if=%u st=0x%x\n", + dev->SlotId, dev->HidInterface, st); + return st; + } + + setup.bRequest = USB_REQ_SET_IDLE; + setup.wValue = 0; + st = XhciControlNoData(xhci, dev, &setup); + if (!KSUCCESS(st)) { + // SET_IDLE is useful for keyboards but not worth rejecting an otherwise + // usable boot device in the early stack. + DbgPrint("usb: HID SET_IDLE failed slot=%u if=%u st=0x%x; continuing\n", + dev->SlotId, dev->HidInterface, st); + } + + DbgPrint("usb: HID boot protocol selected slot=%u if=%u proto=%u\n", + dev->SlotId, dev->HidInterface, dev->HidProtocol); + return STATUS_SUCCESS; +} + + +KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + if (dev->HidRing && dev->HidBuffer) return STATUS_SUCCESS; + + dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingPhys); + dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferPhys); + if (!dev->HidRing || !dev->HidBuffer) + return STATUS_NO_MEMORY; + + dev->HidEnqueue = 0; + dev->HidCycle = 1; + + XHCI_TRB* link = &dev->HidRing[XHCI_HID_RING_TRBS - 1u]; + link->ParameterLo = (uint32_t)dev->HidRingPhys; + link->ParameterHi = (uint32_t)(dev->HidRingPhys >> 32); + link->Status = 0; + link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_LINK_TC | + dev->HidCycle; + + memset(dev->HidLastKeys, 0, sizeof(dev->HidLastKeys)); + dev->HidLastModifiers = 0; + dev->HidLastButtons = 0; + return STATUS_SUCCESS; +} + + +KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + if (!dev->HidCandidate) + return STATUS_SUCCESS; + + KSTATUS st = XhciAllocateHidMemory(xhci, dev); + if (!KSUCCESS(st)) return st; + + uint8_t dci = dev->HidEndpointId; + if (dci < 2 || dci > 31) + return STATUS_INVALID_PARAMETER; + + memset(dev->InputContext, 0, PAGE_SIZE); + uint32_t* icc = XhciInputControlContext(xhci, dev); + uint32_t* slot = XhciInputSlotContext(xhci, dev); + uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci); + + icc[0] = 0; + icc[1] = (1u << 0) | (1u << dci); + + slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed); + slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port); + slot[2] = 0; + slot[3] = 0; + + uint16_t maxPacket = dev->HidMaxPacket & 0x07FFu; + if (maxPacket == 0) maxPacket = dev->HidKeyboard ? 8u : 4u; + uint8_t interval = XhciInterruptInterval(dev->Speed, dev->HidInterval); + + ep[0] = (uint32_t)interval << 16; + ep[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_INT_IN | ((uint32_t)maxPacket << 16); + ep[2] = (uint32_t)(dev->HidRingPhys | XHCI_EP_DCS); + ep[3] = (uint32_t)((dev->HidRingPhys | XHCI_EP_DCS) >> 32); + ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16); + + XHCI_COMMAND_RESULT result; + memset(&result, 0, sizeof(result)); + st = XhciSubmitCommand( + xhci, + dev->InputContextPhys, + 0, + (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | + ((uint32_t)dev->SlotId << 24), + &result); + if (!KSUCCESS(st)) { + DbgPrint("usb: Configure Endpoint failed slot=%u ep=%u st=0x%x cc=%u\n", + dev->SlotId, dci, st, result.CompletionCode); + return st; + } + + dev->HidReportLength = maxPacket; + if (dev->HidKeyboard && dev->HidReportLength < 8) dev->HidReportLength = 8; + if (dev->HidMouse && dev->HidReportLength < 3) dev->HidReportLength = 3; + if (dev->HidReportLength > XHCI_MAX_HID_REPORT) dev->HidReportLength = XHCI_MAX_HID_REPORT; + + st = XhciSetHidBootProtocol(xhci, dev); + if (!KSUCCESS(st)) return st; + + dev->HidActive = true; + xhci->HidDeviceCount++; + uint32_t inputId = 0; + KeInputRegisterDevice(dev->HidKeyboard ? "usb-hid-keyboard" : "usb-hid-mouse", 0, &inputId); + + DbgPrint("usb: HID boot %s bound slot=%u if=%u ep=%02x dci=%u maxpkt=%u interval=%u input=%u\n", + dev->HidKeyboard ? "keyboard" : "mouse", + dev->SlotId, + dev->HidInterface, + dev->HidEndpointAddress, + dci, + maxPacket, + dev->HidInterval, + inputId); + return STATUS_SUCCESS; +} + + +const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub, + uint8_t endpointType, + bool wantIn) { + if (!pub || pub->BusType != IoBusUsb) return (const IO_PUBLISHED_USB_ENDPOINT*)0; + + for (uint8_t i = 0; i < pub->u.Usb.EndpointCount; i++) { + const IO_PUBLISHED_USB_ENDPOINT* ep = &pub->u.Usb.Endpoints[i]; + uint8_t type = ep->Attributes & USB_ENDPOINT_ATTR_TYPE_MASK; + bool in = (ep->Address & USB_ENDPOINT_DIR_IN) != 0; + if (type == endpointType && in == wantIn) + return ep; + } + + return (const IO_PUBLISHED_USB_ENDPOINT*)0; +} + + +bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context) { + (void)context; + if (!pub || pub->BusType != IoBusUsb) + return false; + if (pub->u.Usb.InterfaceClass != USB_CLASS_HID) + return false; + if (pub->u.Usb.InterfaceSubclass != USB_HID_SUBCLASS_BOOT) + return false; + if (pub->u.Usb.InterfaceProtocol != USB_HID_PROTOCOL_KEYBOARD && + pub->u.Usb.InterfaceProtocol != USB_HID_PROTOCOL_MOUSE) + return false; + + return UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_INTERRUPT, true) != (const IO_PUBLISHED_USB_ENDPOINT*)0; +} + + +KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context) { + (void)context; + if (!pub || !pub->BusContext || !pub->DeviceContext) + return STATUS_INVALID_PARAMETER; + + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext; + XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext; + + if (dev->HidActive || dev->HidCandidate) + return STATUS_ALREADY_EXISTS; + + const IO_PUBLISHED_USB_ENDPOINT* ep = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_INTERRUPT, true); + if (!ep) + return STATUS_NOT_FOUND; + + dev->HidCandidate = true; + dev->HidKeyboard = (pub->u.Usb.InterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD); + dev->HidMouse = (pub->u.Usb.InterfaceProtocol == USB_HID_PROTOCOL_MOUSE); + dev->HidInterface = pub->u.Usb.InterfaceNumber; + dev->HidProtocol = pub->u.Usb.InterfaceProtocol; + dev->HidEndpointAddress = ep->Address; + dev->HidEndpointId = XhciEndpointIdFromAddress(ep->Address); + dev->HidInterval = ep->Interval; + dev->HidMaxPacket = ep->MaxPacketSize; + + KSTATUS st = XhciConfigureHidEndpoint(xhci, dev); + if (!KSUCCESS(st)) { + dev->HidCandidate = false; + dev->HidKeyboard = false; + dev->HidMouse = false; + } + return st; +} diff --git a/code/drivers/bus/usb/usb_storage.c b/code/drivers/bus/usb/usb_storage.c new file mode 100644 index 0000000..be13c16 --- /dev/null +++ b/code/drivers/bus/usb/usb_storage.c @@ -0,0 +1,472 @@ +// usb_storage.c - USB mass-storage BOT/SCSI class binding and disk bridge. +#include "usb.h" + +DRIVER_OBJECT gUsbStorageDriver; + +KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + PHYSICAL_ADDRESS phys, uint32_t length, + uint32_t* outActual) { + return XhciBulkTransfer(xhci, + dev, + dev->MassBulkInEndpointId, + dev->MassInRing, + dev->MassInRingPhys, + &dev->MassInEnqueue, + &dev->MassInCycle, + phys, + length, + outActual); +} + + +KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + PHYSICAL_ADDRESS phys, uint32_t length, + uint32_t* outActual) { + return XhciBulkTransfer(xhci, + dev, + dev->MassBulkOutEndpointId, + dev->MassOutRing, + dev->MassOutRingPhys, + &dev->MassOutEnqueue, + &dev->MassOutCycle, + phys, + length, + outActual); +} + + +KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + const uint8_t* cdb, uint8_t cdbLen, + void* data, PHYSICAL_ADDRESS dataPhys, + uint32_t dataLength, bool dataIn) { + if (!xhci || !dev || !cdb || cdbLen == 0 || cdbLen > 16) + return STATUS_INVALID_PARAMETER; + if (dataLength && (!data || !dataPhys)) + return STATUS_INVALID_PARAMETER; + + USB_MASS_CBW* cbw = (USB_MASS_CBW*)dev->DescriptorBuffer; + memset(cbw, 0, sizeof(*cbw)); + cbw->Signature = USB_MASS_BOT_CBW_SIGNATURE; + cbw->Tag = ++dev->MassTag; + cbw->DataTransferLength = dataLength; + cbw->Flags = dataIn ? USB_DIR_IN : USB_DIR_OUT; + cbw->Lun = 0; + cbw->CommandBlockLength = cdbLen; + memcpy(cbw->CommandBlock, cdb, cdbLen); + + uint32_t actual = 0; + KSTATUS st = UsbStorageBulkOut(xhci, dev, dev->DescriptorPhys, sizeof(*cbw), &actual); + if (!KSUCCESS(st) || actual != sizeof(*cbw)) + return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st; + + if (dataLength) { + actual = 0; + if (dataIn) + st = UsbStorageBulkIn(xhci, dev, dataPhys, dataLength, &actual); + else + st = UsbStorageBulkOut(xhci, dev, dataPhys, dataLength, &actual); + if (!KSUCCESS(st)) + return st; + } + + USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer; + memset(csw, 0, sizeof(*csw)); + actual = 0; + st = UsbStorageBulkIn(xhci, dev, dev->DescriptorPhys, sizeof(*csw), &actual); + if (!KSUCCESS(st) || actual != sizeof(*csw)) + return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st; + + if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != dev->MassTag) { + DbgPrint("usb-storage: invalid CSW slot=%u sig=%08x tag=%08x expected=%08x\n", + dev->SlotId, csw->Signature, csw->Tag, dev->MassTag); + return STATUS_DEVICE_ERROR; + } + if (csw->Status != USB_MASS_BOT_CSW_PASSED) { + DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u\n", + cdb[0], dev->SlotId, csw->Status, csw->DataResidue); + return STATUS_DEVICE_ERROR; + } + + return STATUS_SUCCESS; +} + + +uint32_t UsbBe32(const uint8_t* p) { + return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | + ((uint32_t)p[2] << 8) | (uint32_t)p[3]; +} + + +void UsbStoreBe32(uint8_t* p, uint32_t v) { + p[0] = (uint8_t)(v >> 24); + p[1] = (uint8_t)(v >> 16); + p[2] = (uint8_t)(v >> 8); + p[3] = (uint8_t)v; +} + + +void UsbStoreBe16(uint8_t* p, uint16_t v) { + p[0] = (uint8_t)(v >> 8); + p[1] = (uint8_t)v; +} + + +KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 }; + memset(dev->MassBuffer, 0, 36); + KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + dev->MassBuffer, dev->MassBufferPhys, + 36, true); + if (!KSUCCESS(st)) return st; + + char vendor[9]; + char product[17]; + memcpy(vendor, dev->MassBuffer + 8, 8); vendor[8] = 0; + memcpy(product, dev->MassBuffer + 16, 16); product[16] = 0; + DbgPrint("usb-storage: INQUIRY slot=%u vendor='%s' product='%s'\n", + dev->SlotId, vendor, product); + return STATUS_SUCCESS; +} + + +KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 }; + memset(dev->MassBuffer, 0, 18); + return UsbStorageBotCommand(xhci, dev, rs, sizeof(rs), + dev->MassBuffer, dev->MassBufferPhys, 18, true); +} + + +KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + uint8_t cdb[6] = { SCSI_TEST_UNIT_READY, 0, 0, 0, 0, 0 }; + + KSTATUS last = STATUS_DEVICE_ERROR; + for (uint32_t attempt = 0; attempt < 10; attempt++) { + KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + (void*)0, 0, 0, false); + if (KSUCCESS(st)) return STATUS_SUCCESS; + + last = st; + (void)UsbStorageRequestSense(xhci, dev); + UsbDelayUs(xhci->Hal, 10000u); + } + + return last; +} + + +KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + memset(dev->MassBuffer, 0, 8); + KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + dev->MassBuffer, dev->MassBufferPhys, + 8, true); + if (!KSUCCESS(st)) return st; + + uint32_t lastLba = UsbBe32(dev->MassBuffer + 0); + uint32_t blockSize = UsbBe32(dev->MassBuffer + 4); + if (blockSize == 0) + return STATUS_DEVICE_ERROR; + + dev->MassBlockSize = blockSize; + dev->MassBlockCount = (uint64_t)lastLba + 1ull; + DbgPrint("usb-storage: capacity slot=%u blocks=%llu block_size=%u (%llu MB)\n", + dev->SlotId, + (unsigned long long)dev->MassBlockCount, + dev->MassBlockSize, + (unsigned long long)((dev->MassBlockCount * (uint64_t)dev->MassBlockSize) / (1024ull * 1024ull))); + return STATUS_SUCCESS; +} + + +KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + bool write, uint64_t lba, + uint32_t sectors, void* buffer) { + if (!xhci || !dev || !buffer || sectors == 0) + return STATUS_INVALID_PARAMETER; + if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) + return STATUS_NOT_SUPPORTED; + if (lba > 0xFFFFFFFFull || sectors > 0xFFFFu) + return STATUS_INVALID_PARAMETER; + + uint32_t bytes = sectors * USB_STORAGE_SECTOR_SIZE; + if (bytes > USB_STORAGE_MAX_BYTES) + return STATUS_INVALID_PARAMETER; + + uint8_t cdb[10]; + memset(cdb, 0, sizeof(cdb)); + cdb[0] = write ? SCSI_WRITE_10 : SCSI_READ_10; + UsbStoreBe32(&cdb[2], (uint32_t)lba); + UsbStoreBe16(&cdb[7], (uint16_t)sectors); + + if (write) + memcpy(dev->MassBuffer, buffer, bytes); + + KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb), + dev->MassBuffer, dev->MassBufferPhys, + bytes, !write); + if (!KSUCCESS(st)) return st; + + if (!write) + memcpy(buffer, dev->MassBuffer, bytes); + return STATUS_SUCCESS; +} + + +void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + while (_InterlockedCompareExchange(&dev->MassIoBusy, 1, 0) != 0) { + UsbDelayUs(xhci->Hal, 10); + } +} + + +void UsbStorageReleaseIo(XHCI_DEVICE* dev) { + _InterlockedExchange(&dev->MassIoBusy, 0); +} + + +KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp || !device->DeviceExtension) + return STATUS_INVALID_PARAMETER; + + XHCI_DEVICE* dev = (XHCI_DEVICE*)device->DeviceExtension; + if (!dev->MassActive || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) { + IoCompleteRequest(irp, STATUS_DEVICE_ERROR, 0); + return STATUS_PENDING; + } + + uint8_t* buffer = (uint8_t*)irp->Buffer; + uint64_t lba = irp->Lba; + uint32_t remaining = irp->SectorCount; + uint32_t done = 0; + bool write = (irp->Flags & IRP_FLAG_WRITE) != 0; + uint32_t maxSectors = USB_STORAGE_MAX_BYTES / USB_STORAGE_SECTOR_SIZE; + + KSTATUS st = STATUS_SUCCESS; + UsbStorageAcquireIo(&gXhci, dev); + while (remaining) { + uint32_t chunk = remaining; + if (chunk > maxSectors) chunk = maxSectors; + st = UsbStorageReadWriteSectors(&gXhci, dev, write, lba, chunk, + buffer + (done * USB_STORAGE_SECTOR_SIZE)); + if (!KSUCCESS(st)) + break; + lba += chunk; + done += chunk; + remaining -= chunk; + } + UsbStorageReleaseIo(dev); + + IoCompleteRequest(irp, st, KSUCCESS(st) ? (size_t)done * USB_STORAGE_SECTOR_SIZE : 0); + return STATUS_PENDING; +} + + +KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer) + return STATUS_SUCCESS; + + dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingPhys); + dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingPhys); + dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferPhys); + if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer) + return STATUS_NO_MEMORY; + + dev->MassInEnqueue = 0; + dev->MassInCycle = 1; + dev->MassOutEnqueue = 0; + dev->MassOutCycle = 1; + dev->MassTag = 0x1000u + dev->SlotId; + XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingPhys, XHCI_BULK_RING_TRBS, dev->MassInCycle); + XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingPhys, XHCI_BULK_RING_TRBS, dev->MassOutCycle); + return STATUS_SUCCESS; +} + + +KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + if (!dev->MassCandidate) + return STATUS_SUCCESS; + + KSTATUS st = XhciAllocateMassStorageMemory(xhci, dev); + if (!KSUCCESS(st)) return st; + + uint8_t inDci = dev->MassBulkInEndpointId; + uint8_t outDci = dev->MassBulkOutEndpointId; + if (inDci < 2 || inDci > 31 || outDci < 2 || outDci > 31) + return STATUS_INVALID_PARAMETER; + + memset(dev->InputContext, 0, PAGE_SIZE); + uint32_t* icc = XhciInputControlContext(xhci, dev); + uint32_t* slot = XhciInputSlotContext(xhci, dev); + uint32_t* epIn = XhciInputEndpointContext(xhci, dev, inDci); + uint32_t* epOut = XhciInputEndpointContext(xhci, dev, outDci); + + icc[0] = 0; + icc[1] = (1u << 0) | (1u << inDci) | (1u << outDci); + + uint8_t maxDci = inDci > outDci ? inDci : outDci; + slot[0] = XHCI_CONTEXT_ENTRIES(maxDci) | XHCI_SLOT_SPEED(dev->Speed); + slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port); + slot[2] = 0; + slot[3] = 0; + + uint16_t inMaxPacket = dev->MassBulkInMaxPacket & 0x07FFu; + uint16_t outMaxPacket = dev->MassBulkOutMaxPacket & 0x07FFu; + if (inMaxPacket == 0) inMaxPacket = 512; + if (outMaxPacket == 0) outMaxPacket = 512; + + epIn[0] = 0; + epIn[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_IN | ((uint32_t)inMaxPacket << 16); + epIn[2] = (uint32_t)(dev->MassInRingPhys | XHCI_EP_DCS); + epIn[3] = (uint32_t)((dev->MassInRingPhys | XHCI_EP_DCS) >> 32); + epIn[4] = inMaxPacket; + + epOut[0] = 0; + epOut[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_OUT | ((uint32_t)outMaxPacket << 16); + epOut[2] = (uint32_t)(dev->MassOutRingPhys | XHCI_EP_DCS); + epOut[3] = (uint32_t)((dev->MassOutRingPhys | XHCI_EP_DCS) >> 32); + epOut[4] = outMaxPacket; + + XHCI_COMMAND_RESULT result; + memset(&result, 0, sizeof(result)); + st = XhciSubmitCommand( + xhci, + dev->InputContextPhys, + 0, + (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | + ((uint32_t)dev->SlotId << 24), + &result); + if (!KSUCCESS(st)) { + DbgPrint("usb-storage: Configure Endpoint failed slot=%u in=%u out=%u st=0x%x cc=%u\n", + dev->SlotId, inDci, outDci, st, result.CompletionCode); + return st; + } + + UsbTrace("usb-storage: bulk endpoints configured slot=%u if=%u in=%02x/dci=%u out=%02x/dci=%u\n", + dev->SlotId, + dev->MassInterface, + dev->MassBulkInAddress, + inDci, + dev->MassBulkOutAddress, + outDci); + return STATUS_SUCCESS; +} + + +uint8_t UsbFindFreeDiskIndex(void) { + for (uint8_t i = 0; i < 8; i++) { + if (!IoDiskDevicePresent(i)) + return i; + } + return 0xFFu; +} + + +KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) { + if (!dev || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE || dev->MassBlockCount == 0) + return STATUS_NOT_SUPPORTED; + + DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0; + KSTATUS st = IoCreateDevice(&gUsbStorageDriver, 0, &disk); + if (!KSUCCESS(st)) return st; + + disk->DeviceExtension = dev; + disk->SectorCount = dev->MassBlockCount; + disk->AlignmentRequirement = USB_STORAGE_SECTOR_SIZE; + disk->Flags |= DEVICE_OBJECT_REMOVABLE; + + uint8_t index = UsbFindFreeDiskIndex(); + if (index == 0xFFu) { + IoDeleteDevice(disk); + return STATUS_NO_MEMORY; + } + + st = IoRegisterDiskDevice(index, disk); + if (!KSUCCESS(st)) { + IoDeleteDevice(disk); + return st; + } + + dev->MassDeviceObject = disk; + DbgPrint("usb-storage: registered disk %u slot=%u sectors=%llu\n", + index, dev->SlotId, (unsigned long long)dev->MassBlockCount); + return STATUS_SUCCESS; +} + + +bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) { + (void)context; + if (!pub || pub->BusType != IoBusUsb) + return false; + if (pub->u.Usb.InterfaceClass != USB_CLASS_MASS_STORAGE) + return false; + if (pub->u.Usb.InterfaceSubclass != USB_MASS_SUBCLASS_SCSI) + return false; + if (pub->u.Usb.InterfaceProtocol != USB_MASS_PROTOCOL_BOT) + return false; + + return UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true) && + UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false); +} + + +KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context) { + (void)context; + if (!pub || !pub->BusContext || !pub->DeviceContext) + return STATUS_INVALID_PARAMETER; + + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext; + XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext; + if (dev->MassActive || dev->MassCandidate) + return STATUS_ALREADY_EXISTS; + + const IO_PUBLISHED_USB_ENDPOINT* bulkIn = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true); + const IO_PUBLISHED_USB_ENDPOINT* bulkOut = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false); + if (!bulkIn || !bulkOut) + return STATUS_NOT_FOUND; + + dev->MassCandidate = true; + dev->MassInterface = pub->u.Usb.InterfaceNumber; + dev->MassBulkInAddress = bulkIn->Address; + dev->MassBulkOutAddress = bulkOut->Address; + dev->MassBulkInEndpointId = XhciEndpointIdFromAddress(bulkIn->Address); + dev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(bulkOut->Address); + dev->MassBulkInMaxPacket = bulkIn->MaxPacketSize; + dev->MassBulkOutMaxPacket = bulkOut->MaxPacketSize; + + KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, dev); + if (!KSUCCESS(st)) goto fail; + + st = UsbStorageInquiry(xhci, dev); + if (!KSUCCESS(st)) goto fail; + + st = UsbStorageTestUnitReady(xhci, dev); + if (!KSUCCESS(st)) { + DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", dev->SlotId, st); + goto fail; + } + + st = UsbStorageReadCapacity(xhci, dev); + if (!KSUCCESS(st)) goto fail; + if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) { + DbgPrint("usb-storage: unsupported block size slot=%u block_size=%u\n", + dev->SlotId, dev->MassBlockSize); + st = STATUS_NOT_SUPPORTED; + goto fail; + } + + dev->MassActive = true; + st = UsbStorageRegisterDisk(dev); + if (!KSUCCESS(st)) goto fail; + + DbgPrint("usb-storage: bound slot=%u if=%u\n", dev->SlotId, dev->MassInterface); + return STATUS_SUCCESS; + +fail: + dev->MassCandidate = false; + dev->MassActive = false; + DbgPrint("usb-storage: bind failed slot=%u if=%u st=0x%x\n", + dev->SlotId, pub->u.Usb.InterfaceNumber, st); + return st; +} + diff --git a/code/drivers/bus/usb/usb_xhci.c b/code/drivers/bus/usb/usb_xhci.c new file mode 100644 index 0000000..dc66833 --- /dev/null +++ b/code/drivers/bus/usb/usb_xhci.c @@ -0,0 +1,1511 @@ +// usb_xhci.c - xHCI host-controller, MMIO, rings, commands, event dispatch, and generic transfers. +#include "usb.h" + +uint32_t UsbAlignUp32(uint32_t value, uint32_t align) { + return (value + align - 1u) & ~(align - 1u); +} + + +void XhciBarrier(void) { + _ReadWriteBarrier(); + _mm_mfence(); + _ReadWriteBarrier(); +} + + +uint8_t XhciRead8(volatile uint8_t* base, uint32_t reg) { + return *(volatile uint8_t*)(base + reg); +} + + +uint16_t XhciRead16(volatile uint8_t* base, uint32_t reg) { + return *(volatile uint16_t*)(base + reg); +} + + +uint32_t XhciRead32(volatile uint8_t* base, uint32_t reg) { + return *(volatile uint32_t*)(base + reg); +} + + +void XhciWrite32(volatile uint8_t* base, uint32_t reg, uint32_t value) { + *(volatile uint32_t*)(base + reg) = value; +} + + +uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg) { + uint32_t lo = XhciRead32(base, reg); + uint32_t hi = XhciRead32(base, reg + 4u); + return ((uint64_t)hi << 32) | lo; +} + + +void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value) { + XhciWrite32(base, reg, (uint32_t)value); + XhciWrite32(base, reg + 4u, (uint32_t)(value >> 32)); +} + + +void UsbDelayUs(const HAL_API* hal, uint32_t us) { + if (hal && hal->Timer.Stall) hal->Timer.Stall(us); + else { + for (uint32_t i = 0; i < us * 32u; i++) _mm_pause(); + } +} + + +bool UsbWaitSet32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, + uint32_t mask, uint32_t timeoutUs) { + for (uint32_t i = 0; i < timeoutUs; i++) { + if ((XhciRead32(base, reg) & mask) == mask) return true; + UsbDelayUs(hal, 1); + } + return false; +} + + +bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, + uint32_t mask, uint32_t timeoutUs) { + for (uint32_t i = 0; i < timeoutUs; i++) { + if ((XhciRead32(base, reg) & mask) == 0) return true; + UsbDelayUs(hal, 1); + } + return false; +} + + +bool UsbReadBar(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, + uint8_t index, USB_BAR* out) { + uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u); + uint32_t lo = hal->Pci.Read32(bus, dev, fn, reg); + + out->Kind = UsbBarNone; + out->Base = 0; + out->Index = index; + + if (lo == 0 || lo == 0xFFFFFFFFu) + return false; + + if (lo & 1u) { + out->Kind = UsbBarIo; + out->Base = (uint64_t)(lo & ~0x3u); + return out->Base != 0; + } + + uint64_t base = (uint64_t)(lo & ~0xFu); + uint32_t type = (lo >> 1) & 0x3u; + if (type == 0x2u && index < 5) { + uint32_t hi = hal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u)); + base |= ((uint64_t)hi << 32); + } + + out->Kind = UsbBarMmio; + out->Base = base; + return base != 0; +} + + +KSTATUS UsbMapMmio(const HAL_API* hal, uint64_t physBase, + uint32_t bytes, volatile uint8_t** outMmio) { + uint64_t pageBase = physBase & PAGE_MASK; + uint64_t pageOff = physBase - pageBase; + uint32_t total = UsbAlignUp32((uint32_t)pageOff + bytes, (uint32_t)PAGE_SIZE); + uint32_t pages = total / (uint32_t)PAGE_SIZE; + + for (uint32_t i = 0; i < pages; i++) { + PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE; + uintptr_t virt = (uintptr_t)hal->PhysToVirt(phys); + KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); + if (st == STATUS_ALREADY_EXISTS) + st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); + if (!KSUCCESS(st)) { + DbgPrint("usb: xHCI MmMapPage failed phys=0x%016llX st=0x%x\n", + (unsigned long long)phys, st); + return st; + } + } + + *outMmio = (volatile uint8_t*)((uintptr_t)hal->PhysToVirt(pageBase) + pageOff); + return STATUS_SUCCESS; +} + + +void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, + PHYSICAL_ADDRESS* outPhys) { + if (!outPhys || pages == 0) + return NULL; + + // Most xHCI runtime objects are one page. Do not route those through the + // physically-contiguous allocator: on real machines that allocator is a + // scarce DMA primitive, while a single frame is already contiguous. QEMU + // hid this because it had simple low memory; laptop firmware does not. + PHYSICAL_ADDRESS phys = 0; + if (pages == 1) { + phys = MmAllocateFrame(); + } else { + phys = MmAllocateContiguous(pages); + } + + if (!phys) { + DbgPrint("usb: DMA allocation failed pages=%llu\n", + (unsigned long long)pages); + return NULL; + } + + void* virt = xhci->Hal->PhysToVirt(phys); + memset(virt, 0, pages * (size_t)PAGE_SIZE); + *outPhys = phys; + return virt; +} + + +uint16_t XhciScratchpadCount(uint32_t hcs2) { + return (uint16_t)((((hcs2 >> 16) & 0x3E0u) | ((hcs2 >> 27) & 0x1Fu)) & 0x3FFu); +} + + +uint32_t XhciTrbType(const XHCI_TRB* trb) { + return (trb->Control >> XHCI_TRB_TYPE_SHIFT) & XHCI_TRB_TYPE_MASK; +} + + +uint32_t XhciCompletionCode(const XHCI_TRB* trb) { + return (trb->Status >> 24) & 0xFFu; +} + + +uint8_t XhciEventSlotId(const XHCI_TRB* trb) { + return (uint8_t)(trb->Control >> 24); +} + + +uint8_t XhciEventEndpointId(const XHCI_TRB* trb) { + return (uint8_t)((trb->Control >> 16) & 0x1Fu); +} + + +uint32_t XhciTransferResidue(const XHCI_TRB* trb) { + return trb->Status & 0xFFFFFFu; +} + + +uint32_t* XhciInputControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + (void)xhci; + return (uint32_t*)dev->InputContext; +} + + +uint32_t* XhciInputSlotContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + return (uint32_t*)(dev->InputContext + xhci->ContextBytes); +} + + +uint32_t* XhciInputEndpointContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + uint8_t endpointId) { + return (uint32_t*)(dev->InputContext + (uint32_t)(endpointId + 1u) * xhci->ContextBytes); +} + + +uint32_t XhciEpMaxPacket(uint8_t speed) { + switch (speed) { + case 2: return 8; // low-speed + case 3: return 64; // high-speed + case 4: return 512; // SuperSpeed + case 5: return 512; // SuperSpeed+ + case 1: + default: return 8; // full-speed before descriptor refinement + } +} + + +uint64_t UsbSetupPacketToU64(const USB_SETUP_PACKET* setup) { + uint64_t v = 0; + v |= (uint64_t)setup->bmRequestType; + v |= (uint64_t)setup->bRequest << 8; + v |= (uint64_t)setup->wValue << 16; + v |= (uint64_t)setup->wIndex << 32; + v |= (uint64_t)setup->wLength << 48; + return v; +} + + +const char* XhciPortSpeedName(uint32_t speed) { + switch (speed) { + case 1: return "full"; + case 2: return "low"; + case 3: return "high"; + case 4: return "super"; + case 5: return "super+"; + default: return "unknown"; + } +} + + + +void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) { + memset(xhci->PortProtocolMajor, 0, sizeof(xhci->PortProtocolMajor)); + memset(xhci->PortProtocolMinor, 0, sizeof(xhci->PortProtocolMinor)); + + uint32_t xecp = (xhci->Hcc1 >> 16) & 0xFFFFu; + for (uint32_t guard = 0; xecp && guard < 64; guard++) { + uint32_t off = xecp << 2; + if (off + 12u > xhci->MmioBytes) + break; + + uint32_t cap = XhciRead32(xhci->Mmio, off); + uint8_t id = (uint8_t)(cap & 0xFFu); + uint8_t next = (uint8_t)((cap >> 8) & 0xFFu); + + if (id == XHCI_EXT_CAP_PROTOCOL) { + uint8_t minor = (uint8_t)((cap >> 16) & 0xFFu); + uint8_t major = (uint8_t)((cap >> 24) & 0xFFu); + uint32_t name = XhciRead32(xhci->Mmio, off + 4u); + uint32_t ports = XhciRead32(xhci->Mmio, off + 8u); + uint8_t portOffset = (uint8_t)(ports & 0xFFu); + uint8_t portCount = (uint8_t)((ports >> 8) & 0xFFu); + + UsbTrace("usb: xHCI protocol cap +0x%x name='%c%c%c%c' usb=%u.%u ports=%u..%u\n", + off, + (char)(name & 0xFFu), + (char)((name >> 8) & 0xFFu), + (char)((name >> 16) & 0xFFu), + (char)((name >> 24) & 0xFFu), + major, minor, portOffset, + portCount ? (uint8_t)(portOffset + portCount - 1u) : portOffset); + + if (portOffset != 0 && portCount != 0) { + for (uint32_t i = 0; i < portCount; i++) { + uint32_t port = (uint32_t)portOffset + i; + if (port <= XHCI_MAX_ROOT_PORTS) { + xhci->PortProtocolMajor[port] = major; + xhci->PortProtocolMinor[port] = minor; + } + } + } + } + + xecp = next; + } +} + + +uint8_t XhciPortProtocolMajor(XHCI_CONTROLLER* xhci, uint8_t port, uint32_t portsc) { + if (port <= XHCI_MAX_ROOT_PORTS && xhci->PortProtocolMajor[port]) + return xhci->PortProtocolMajor[port]; + + uint32_t speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; + if (speed >= 4u) + return 3u; + if (speed >= 1u && speed <= 3u) + return 2u; + return 0u; +} + +const char* UsbEndpointTypeName(uint8_t type) { + switch (type & USB_ENDPOINT_ATTR_TYPE_MASK) { + case USB_ENDPOINT_TYPE_CONTROL: return "control"; + case USB_ENDPOINT_TYPE_ISOCH: return "isochronous"; + case USB_ENDPOINT_TYPE_BULK: return "bulk"; + case USB_ENDPOINT_TYPE_INTERRUPT: return "interrupt"; + default: return "unknown"; + } +} + + +const char* UsbDirectionName(uint8_t endpointAddress) { + return (endpointAddress & USB_ENDPOINT_DIR_IN) ? "in" : "out"; +} + + +const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto) { + if (klass == USB_CLASS_HID) { + if (subclass == 1 && proto == 1) return "hid-boot-keyboard"; + if (subclass == 1 && proto == 2) return "hid-boot-mouse"; + return "hid"; + } + if (klass == USB_CLASS_MASS_STORAGE) { + if (subclass == USB_MASS_SUBCLASS_SCSI && proto == USB_MASS_PROTOCOL_BOT) + return "mass-storage-bot"; + return "mass-storage"; + } + return "class"; +} + + +void XhciLogPortStatus(XHCI_CONTROLLER* xhci) { + for (uint8_t port = 1; port <= xhci->MaxPorts; port++) { + uint32_t off = XHCI_OP_PORT_BASE + ((uint32_t)port - 1u) * XHCI_OP_PORT_STRIDE; + uint32_t portsc = XhciRead32(xhci->Op, off); + uint32_t speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; + + if ((portsc & (XHCI_PORTSC_CCS | XHCI_PORTSC_CHANGE_MASK)) || port <= 8u) { + uint8_t proto = XhciPortProtocolMajor(xhci, port, portsc); + UsbTrace("usb: xHCI port %u: portsc=%08x ccs=%u ped=%u pp=%u proto=usb%u speed=%s(%u)%s%s\n", + port, portsc, + (portsc & XHCI_PORTSC_CCS) ? 1u : 0u, + (portsc & XHCI_PORTSC_PED) ? 1u : 0u, + (portsc & XHCI_PORTSC_PP) ? 1u : 0u, + proto, + XhciPortSpeedName(speed), speed, + (portsc & XHCI_PORTSC_OCA) ? " over-current" : "", + (portsc & XHCI_PORTSC_PR) ? " resetting" : ""); + } + } +} + + +void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) { + if (!xhci->Runtime) return; + volatile uint8_t* ir0 = xhci->Runtime + 0x20u; + uint32_t iman = XhciRead32(ir0, 0x00u); + XhciWrite32(ir0, 0x00u, (iman & XHCI_IMAN_IE) | XHCI_IMAN_IP); +} + + +void XhciUpdateErdp(XHCI_CONTROLLER* xhci) { + volatile uint8_t* ir0 = xhci->Runtime + 0x20u; + uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB); + XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear +} + + +bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { + bool processed = false; + + for (;;) { + XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue]; + uint32_t control = ev->Control; + uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE); + if (cycle != xhci->EventCycle) + break; + + processed = true; + uint32_t type = XhciTrbType(ev); + uint32_t cc = XhciCompletionCode(ev); + uint64_t param = ((uint64_t)ev->ParameterHi << 32) | ev->ParameterLo; + + XHCI_WAIT_TARGET* waiter = target ? target : (XHCI_WAIT_TARGET*)xhci->ActiveWait; + + if (type == XHCI_TRB_TYPE_CMD_DONE) { + uint8_t slotId = XhciEventSlotId(ev); + UsbTrace("usb: xHCI command completion ptr=0x%016llX cc=%u slot=%u\n", + (unsigned long long)param, cc, slotId); + if (waiter && waiter->CommandPhys && param == waiter->CommandPhys) { + waiter->CommandMatched = true; + waiter->Command.CompletionCode = cc; + waiter->Command.SlotId = slotId; + } + } else if (type == XHCI_TRB_TYPE_TRANSFER) { + uint8_t slotId = XhciEventSlotId(ev); + uint8_t epId = XhciEventEndpointId(ev); + uint32_t residue = XhciTransferResidue(ev); + bool claimedByWaiter = false; + bool targetMatches = waiter && waiter->TransferPhys && param == waiter->TransferPhys && + slotId == waiter->TransferSlotId && epId == waiter->TransferEndpointId; + if (cc != XHCI_CC_SUCCESS) { + UsbTrace("usb: xHCI transfer event trb=0x%016llX cc=%u slot=%u ep=%u residue=%u\n", + (unsigned long long)param, cc, slotId, epId, residue); + } + if (targetMatches) { + waiter->TransferMatched = true; + waiter->Transfer.CompletionCode = cc; + waiter->Transfer.SlotId = slotId; + waiter->Transfer.EndpointId = epId; + waiter->Transfer.Residue = residue; + claimedByWaiter = true; + } + if (!claimedByWaiter) + XhciHandleAsyncTransferEvent(xhci, param, cc, slotId, epId, residue); + } else if (type == XHCI_TRB_TYPE_PORT_SC) { + uint32_t portId = ev->ParameterLo >> 24; + UsbTrace("usb: xHCI port-status-change event port=%u cc=%u\n", portId, cc); + } else { + DbgPrint("usb: xHCI event type=%u cc=%u p=0x%016llX status=%08x ctrl=%08x\n", + type, cc, (unsigned long long)param, ev->Status, ev->Control); + } + + xhci->EventDequeue++; + if (xhci->EventDequeue == XHCI_EVENT_TRBS) { + xhci->EventDequeue = 0; + xhci->EventCycle ^= 1u; + } + } + + if (processed) + XhciUpdateErdp(xhci); + return processed; +} + + +bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { + if (!xhci) return false; + uintptr_t flags = KeAcquireSpinlock(&xhci->EventLock); + bool processed = XhciDrainEventRing(xhci, target); + KeReleaseSpinlock(&xhci->EventLock, flags); + return processed; +} + + +bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) { + if (!target) return false; + if (target->CommandPhys && !target->CommandMatched) return false; + if (target->TransferPhys && !target->TransferMatched) return false; + return true; +} + + +KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, + uint32_t timeoutUs) { + if (!xhci || !target) + return STATUS_INVALID_PARAMETER; + + xhci->ActiveWait = target; + for (uint32_t i = 0; i < timeoutUs; i++) { + if (XhciWaitTargetComplete(target)) { + xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0; + return STATUS_SUCCESS; + } + + // The IRQ/DPC path normally owns the event ring after enumeration, but + // synchronous storage/control waits must not depend exclusively on an + // interrupt being delivered. Opportunistically drain the ring from the + // waiter too; EventLock serializes this with the DPC. + if (!xhci->InterruptsEnabled || (i & 0x7u) == 0) { + XhciDrainEventRingLocked(xhci, target); + } else if ((XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_EINT) && (i & 0x3Fu) == 0) { + KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); + } + + UsbDelayUs(xhci->Hal, 1); + } + + xhci->ActiveWait = (volatile XHCI_WAIT_TARGET*)0; + return STATUS_TIMEOUT; +} + + +void XhciEventDpcRoutine(KDPC* dpc, void* context, + void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)systemArgument1; + (void)systemArgument2; + + XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)context; + if (!xhci || !xhci->Running || !xhci->Runtime) + return; + + xhci->EventDpcCount++; + while (XhciDrainEventRingLocked(xhci, NULL)) { + // Keep draining until the software dequeue catches up to the producer. + } +} + + +void XhciIsr(void) { + XHCI_CONTROLLER* xhci = &gXhci; + if (!xhci->Running || !xhci->Op || !xhci->Runtime) + return; + + uint32_t status = XhciRead32(xhci->Op, XHCI_OP_USBSTS); + if (!(status & (XHCI_USBSTS_EINT | XHCI_USBSTS_PCD | XHCI_USBSTS_HSE | XHCI_USBSTS_HCE))) + return; + + xhci->InterruptCount++; + XhciWrite32(xhci->Op, XHCI_OP_USBSTS, status & XHCI_USBSTS_CLEAR_MASK); + XhciAckInterrupter0(xhci); + KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); +} + + +KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) { + uint32_t xecp = (xhci->Hcc1 >> 16) & 0xFFFFu; + if (!xecp) return STATUS_SUCCESS; + + for (uint32_t guard = 0; xecp && guard < 64; guard++) { + uint32_t off = xecp << 2; + if (off + 8u > xhci->MmioBytes) + break; + + uint32_t cap = XhciRead32(xhci->Mmio, off); + uint8_t id = (uint8_t)(cap & 0xFFu); + uint8_t next = (uint8_t)((cap >> 8) & 0xFFu); + + if (id == XHCI_EXT_CAP_LEGACY) { + UsbTrace("usb: xHCI legacy-support cap at +0x%x legsup=%08x\n", off, cap); + if (cap & XHCI_LEGSUP_BIOS_OWNED) { + XhciWrite32(xhci->Mmio, off, cap | XHCI_LEGSUP_OS_OWNED); + bool ok = false; + for (uint32_t i = 0; i < 1000000u; i++) { + uint32_t cur = XhciRead32(xhci->Mmio, off); + if ((cur & XHCI_LEGSUP_BIOS_OWNED) == 0 && + (cur & XHCI_LEGSUP_OS_OWNED)) { + ok = true; + break; + } + UsbDelayUs(xhci->Hal, 1); + } + if (!ok) { + DbgPrint("usb: xHCI BIOS handoff timed out; continuing anyway\n"); + } + } + + // Clear legacy SMI enables/status. RW1C status bits tolerate this. + XhciWrite32(xhci->Mmio, off + 4u, 0xFFFFFFFFu); + return STATUS_SUCCESS; + } + + xecp = next; + } + + return STATUS_SUCCESS; +} + + +KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) { + uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); + if (cmd & XHCI_USBCMD_RS) { + XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS); + if (!UsbWaitSet32(xhci->Hal, xhci->Op, XHCI_OP_USBSTS, + XHCI_USBSTS_HCH, 1000000u)) { + DbgPrint("usb: xHCI stop timeout usbsts=%08x\n", + XhciRead32(xhci->Op, XHCI_OP_USBSTS)); + return STATUS_TIMEOUT; + } + } + + XhciWrite32(xhci->Op, XHCI_OP_USBCMD, XHCI_USBCMD_HCRST); + if (!UsbWaitClear32(xhci->Hal, xhci->Op, XHCI_OP_USBCMD, + XHCI_USBCMD_HCRST, 1000000u)) { + DbgPrint("usb: xHCI reset timeout usbcmd=%08x\n", + XhciRead32(xhci->Op, XHCI_OP_USBCMD)); + return STATUS_TIMEOUT; + } + + if (!UsbWaitClear32(xhci->Hal, xhci->Op, XHCI_OP_USBSTS, + XHCI_USBSTS_CNR, 1000000u)) { + DbgPrint("usb: xHCI controller-not-ready timeout usbsts=%08x\n", + XhciRead32(xhci->Op, XHCI_OP_USBSTS)); + return STATUS_TIMEOUT; + } + + XhciWrite32(xhci->Op, XHCI_OP_USBSTS, + XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_CLEAR_MASK); + return STATUS_SUCCESS; +} + + +KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { + if (xhci->ScratchpadCount == 0) return STATUS_SUCCESS; + if (xhci->ScratchpadCount > XHCI_MAX_SCRATCHPADS) { + DbgPrint("usb: xHCI wants %u scratchpads, capped driver supports %u\n", + xhci->ScratchpadCount, XHCI_MAX_SCRATCHPADS); + return STATUS_NOT_SUPPORTED; + } + + xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayPhys); + if (!xhci->ScratchpadArray) { + DbgPrint("usb: xHCI scratchpad pointer-array allocation failed count=%u\n", + xhci->ScratchpadCount); + return STATUS_NO_MEMORY; + } + + for (uint16_t i = 0; i < xhci->ScratchpadCount; i++) { + void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadPhys[i]); + if (!page) { + DbgPrint("usb: xHCI scratchpad buffer allocation failed index=%u/%u\n", + i, xhci->ScratchpadCount); + return STATUS_NO_MEMORY; + } + (void)page; + xhci->ScratchpadArray[i] = xhci->ScratchpadPhys[i]; + } + + xhci->Dcbaa[0] = xhci->ScratchpadArrayPhys; + UsbTrace("usb: xHCI allocated %u scratchpad buffer(s) array=0x%016llX\n", + xhci->ScratchpadCount, + (unsigned long long)xhci->ScratchpadArrayPhys); + return STATUS_SUCCESS; +} + + +KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { + xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys); + xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys); + xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingPhys); + xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstPhys); + + if (!xhci->Dcbaa || !xhci->CommandRing || !xhci->EventRing || !xhci->Erst) { + DbgPrint("usb: xHCI runtime DMA allocation failed dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", + xhci->Dcbaa, xhci->CommandRing, xhci->EventRing, xhci->Erst); + return STATUS_NO_MEMORY; + } + + xhci->CommandEnqueue = 0; + xhci->CommandCycle = 1; + xhci->EventDequeue = 0; + xhci->EventCycle = 1; + + XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; + link->ParameterLo = (uint32_t)xhci->CommandRingPhys; + link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); + link->Status = 0; + link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_LINK_TC | + xhci->CommandCycle; + + xhci->Erst[0].RingSegmentBase = xhci->EventRingPhys; + xhci->Erst[0].RingSegmentSize = XHCI_EVENT_TRBS; + xhci->Erst[0].Reserved = 0; + + KSTATUS st = XhciAllocateScratchpads(xhci); + if (!KSUCCESS(st)) return st; + + UsbTrace("usb: xHCI dma dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", + (unsigned long long)xhci->DcbaaPhys, + (unsigned long long)xhci->CommandRingPhys, + (unsigned long long)xhci->EventRingPhys, + (unsigned long long)xhci->ErstPhys); + return STATUS_SUCCESS; +} + + +KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) { + uint8_t slots = xhci->MaxSlots; + if (slots == 0) return STATUS_DEVICE_ERROR; + + XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaPhys); + XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingPhys | xhci->CommandCycle); + XhciWrite32(xhci->Op, XHCI_OP_DNCTRL, 0); + XhciWrite32(xhci->Op, XHCI_OP_CONFIG, slots); + + volatile uint8_t* ir0 = xhci->Runtime + 0x20u; + XhciWrite32(ir0, 0x00u, XHCI_IMAN_IP); // clear pending, leave disabled until smoke test passes + XhciWrite32(ir0, 0x04u, 0); // no interrupt moderation yet + XhciWrite32(ir0, 0x08u, 1); // one ERST entry + XhciWrite64(ir0, 0x10u, xhci->ErstPhys); + XhciWrite64(ir0, 0x18u, xhci->EventRingPhys); + + XhciBarrier(); + return STATUS_SUCCESS; +} + + +uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) { + return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB); +} + + +KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, + uint64_t parameter, + uint32_t status, + uint32_t controlWithoutCycle, + XHCI_COMMAND_RESULT* outResult) { + if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) + return STATUS_QUEUE_FULL; + + uint32_t index = xhci->CommandEnqueue++; + XHCI_TRB* trb = &xhci->CommandRing[index]; + uint64_t trbPhys = XhciCommandTrbPhys(xhci, index); + + trb->ParameterLo = (uint32_t)parameter; + trb->ParameterHi = (uint32_t)(parameter >> 32); + trb->Status = status; + XhciBarrier(); + trb->Control = controlWithoutCycle | xhci->CommandCycle; + + XHCI_WAIT_TARGET target; + memset(&target, 0, sizeof(target)); + target.CommandPhys = trbPhys; + xhci->ActiveWait = ⌖ + + XhciBarrier(); + XhciWrite32(xhci->Doorbells, 0, 0); // host-controller doorbell, command ring + + KSTATUS waitStatus = XhciWaitForTarget(xhci, &target, 500000u); + if (KSUCCESS(waitStatus) && target.CommandMatched) { + if (outResult) *outResult = target.Command; + return (target.Command.CompletionCode == XHCI_CC_SUCCESS) + ? STATUS_SUCCESS + : STATUS_DEVICE_ERROR; + } + + DbgPrint("usb: xHCI command timeout type=%u usbsts=%08x crcr=0x%016llX\n", + (controlWithoutCycle >> XHCI_TRB_TYPE_SHIFT) & XHCI_TRB_TYPE_MASK, + XhciRead32(xhci->Op, XHCI_OP_USBSTS), + (unsigned long long)XhciRead64(xhci->Op, XHCI_OP_CRCR)); + return waitStatus; +} + + +KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci) { + XHCI_COMMAND_RESULT result; + KSTATUS st = XhciSubmitCommand( + xhci, + 0, + 0, + (XHCI_TRB_TYPE_NOOP_CMD << XHCI_TRB_TYPE_SHIFT), + &result); + + if (KSUCCESS(st)) { + UsbTrace("usb: xHCI No-Op command completed successfully\n"); + return STATUS_SUCCESS; + } + + DbgPrint("usb: xHCI No-Op command failed st=0x%x cc=%u\n", + st, result.CompletionCode); + return st; +} + + +KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) { + XHCI_COMMAND_RESULT result; + memset(&result, 0, sizeof(result)); + KSTATUS st = XhciSubmitCommand( + xhci, + 0, + 0, + (XHCI_TRB_TYPE_ENABLE_SLOT << XHCI_TRB_TYPE_SHIFT), + &result); + if (!KSUCCESS(st)) { + DbgPrint("usb: xHCI Enable Slot failed st=0x%x cc=%u\n", + st, result.CompletionCode); + return st; + } + if (result.SlotId == 0 || result.SlotId > XHCI_MAX_TRACKED_SLOTS) { + DbgPrint("usb: xHCI Enable Slot returned unsupported slot=%u\n", result.SlotId); + return STATUS_DEVICE_ERROR; + } + *outSlotId = result.SlotId; + UsbTrace("usb: xHCI Enable Slot -> slot %u\n", result.SlotId); + return STATUS_SUCCESS; +} + + +KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + XHCI_COMMAND_RESULT result; + memset(&result, 0, sizeof(result)); + KSTATUS st = XhciSubmitCommand( + xhci, + dev->InputContextPhys, + 0, + (XHCI_TRB_TYPE_ADDRESS_DEVICE << XHCI_TRB_TYPE_SHIFT) | + ((uint32_t)dev->SlotId << 24), + &result); + if (!KSUCCESS(st)) { + DbgPrint("usb: xHCI Address Device slot=%u failed st=0x%x cc=%u\n", + dev->SlotId, st, result.CompletionCode); + return st; + } + UsbTrace("usb: xHCI Address Device slot=%u complete\n", dev->SlotId); + return STATUS_SUCCESS; +} + + +KSTATUS XhciStart(XHCI_CONTROLLER* xhci) { + uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); + cmd |= XHCI_USBCMD_RS; + XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd); + + if (!UsbWaitClear32(xhci->Hal, xhci->Op, XHCI_OP_USBSTS, + XHCI_USBSTS_HCH, 1000000u)) { + DbgPrint("usb: xHCI start timeout usbsts=%08x\n", + XhciRead32(xhci->Op, XHCI_OP_USBSTS)); + return STATUS_TIMEOUT; + } + + xhci->Running = true; + return STATUS_SUCCESS; +} + + +void XhciEnableInterrupts(XHCI_CONTROLLER* xhci) { + volatile uint8_t* ir0 = xhci->Runtime + 0x20u; + XhciWrite32(xhci->Op, XHCI_OP_USBSTS, + XhciRead32(xhci->Op, XHCI_OP_USBSTS) & XHCI_USBSTS_CLEAR_MASK); + XhciWrite32(ir0, 0x00u, XHCI_IMAN_IE | XHCI_IMAN_IP); + uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); + XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd | XHCI_USBCMD_INTE | XHCI_USBCMD_HSEE); +} + + +uint32_t XhciPortReg(uint8_t port) { + return XHCI_OP_PORT_BASE + ((uint32_t)port - 1u) * XHCI_OP_PORT_STRIDE; +} + + +void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port) { + uint32_t off = XhciPortReg(port); + uint32_t portsc = XhciRead32(xhci->Op, off); + uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK; + if (changes) { + /* + * PORTSC is a mixed register. The change bits are RW1C, but many + * other bits are either RO or have side effects when written as one + * (PED disables the port; PR starts a reset). Preserve only Port + * Power and write one to the RW1C change bits. + */ + XhciWrite32(xhci->Op, off, (portsc & XHCI_PORTSC_PP) | changes); + } +} + + +KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) { + uint32_t off = XhciPortReg(port); + uint32_t portsc = XhciRead32(xhci->Op, off); + if (!(portsc & XHCI_PORTSC_CCS)) + return STATUS_NOT_FOUND; + + uint32_t speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; + uint8_t protocol = XhciPortProtocolMajor(xhci, port, portsc); + + XhciClearPortChanges(xhci, port); + portsc = XhciRead32(xhci->Op, off); + + if (protocol >= 3u || speed >= 4u) { + /* + * SuperSpeed ports are enabled by link training. Do not issue a + * USB2-style PORTSC.PR reset; on real Intel controllers that can make + * the link drop during early enumeration. If firmware/controller + * already has PR set, just wait for it to settle. + */ + bool enabled = false; + for (uint32_t i = 0; i < 1000000u; i++) { + portsc = XhciRead32(xhci->Op, off); + if (!(portsc & XHCI_PORTSC_CCS)) + break; + if (!(portsc & XHCI_PORTSC_PR) && (portsc & XHCI_PORTSC_PED)) { + enabled = true; + break; + } + XhciDrainEventRingLocked(xhci, NULL); + UsbDelayUs(xhci->Hal, 1); + } + + XhciClearPortChanges(xhci, port); + portsc = XhciRead32(xhci->Op, off); + speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; + if (outSpeed) *outSpeed = (uint8_t)speed; + + UsbTrace("usb: xHCI port %u link ready portsc=%08x ped=%u proto=usb%u speed=%s(%u)\n", + port, portsc, + (portsc & XHCI_PORTSC_PED) ? 1u : 0u, + protocol, XhciPortSpeedName(speed), speed); + + if (!enabled || !(portsc & XHCI_PORTSC_CCS) || !(portsc & XHCI_PORTSC_PED)) + return STATUS_DEVICE_ERROR; + return STATUS_SUCCESS; + } + + uint32_t reset = (portsc & XHCI_PORTSC_PP) | XHCI_PORTSC_PR; + XhciWrite32(xhci->Op, off, reset); + + bool resetDone = false; + for (uint32_t i = 0; i < 500000u; i++) { + portsc = XhciRead32(xhci->Op, off); + if (!(portsc & XHCI_PORTSC_PR) && (portsc & XHCI_PORTSC_PRC)) { + resetDone = true; + break; + } + XhciDrainEventRingLocked(xhci, NULL); + UsbDelayUs(xhci->Hal, 1); + } + + if (!resetDone) { + DbgPrint("usb: xHCI port %u reset timeout portsc=%08x\n", port, portsc); + return STATUS_TIMEOUT; + } + + XhciClearPortChanges(xhci, port); + portsc = XhciRead32(xhci->Op, off); + speed = (portsc & XHCI_PORTSC_SPEED_MASK) >> XHCI_PORTSC_SPEED_SHIFT; + if (outSpeed) *outSpeed = (uint8_t)speed; + + UsbTrace("usb: xHCI port %u reset complete portsc=%08x ped=%u proto=usb%u speed=%s(%u)\n", + port, portsc, + (portsc & XHCI_PORTSC_PED) ? 1u : 0u, + protocol, XhciPortSpeedName(speed), speed); + + if (!(portsc & XHCI_PORTSC_CCS) || !(portsc & XHCI_PORTSC_PED)) + return STATUS_DEVICE_ERROR; + + return STATUS_SUCCESS; +} + + +KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys); + dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys); + dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingPhys); + dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorPhys); + + if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) + return STATUS_NO_MEMORY; + + dev->Ep0Enqueue = 0; + dev->Ep0Cycle = 1; + + XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; + link->ParameterLo = (uint32_t)dev->Ep0RingPhys; + link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); + link->Status = 0; + link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_LINK_TC | + dev->Ep0Cycle; + + return STATUS_SUCCESS; +} + + +void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { + memset(dev->InputContext, 0, PAGE_SIZE); + memset(dev->DeviceContext, 0, PAGE_SIZE); + + uint32_t* icc = XhciInputControlContext(xhci, dev); + uint32_t* slot = XhciInputSlotContext(xhci, dev); + uint32_t* ep0 = XhciInputEndpointContext(xhci, dev, 1); + + icc[0] = 0; // drop flags + icc[1] = 0x3u; // add slot context and endpoint 0 context + + uint32_t maxPacket = XhciEpMaxPacket(dev->Speed); + dev->MaxPacketSize0 = (uint16_t)maxPacket; + + slot[0] = XHCI_CONTEXT_ENTRIES(1) | XHCI_SLOT_SPEED(dev->Speed); + // Slot context dword 1 owns Root Hub Port Number bits 23:16. + // Putting the port number in dword 2 makes Address Device fail with a + // context/TRB error on QEMU xHCI. + slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port); + slot[2] = 0; + slot[3] = 0; + + ep0[0] = 0; + ep0[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_CONTROL | (maxPacket << 16); + ep0[2] = (uint32_t)(dev->Ep0RingPhys | XHCI_EP_DCS); + ep0[3] = (uint32_t)((dev->Ep0RingPhys | XHCI_EP_DCS) >> 32); + ep0[4] = 8; + + xhci->Dcbaa[dev->SlotId] = dev->DeviceContextPhys; + XhciBarrier(); +} + + +uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) { + return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB); +} + + +KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, + uint64_t parameter, + uint32_t status, + uint32_t controlWithoutCycle, + uint64_t* outPhys) { + if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) + return STATUS_QUEUE_FULL; + + uint32_t index = dev->Ep0Enqueue++; + XHCI_TRB* trb = &dev->Ep0Ring[index]; + uint64_t phys = XhciEp0TrbPhys(dev, index); + + trb->ParameterLo = (uint32_t)parameter; + trb->ParameterHi = (uint32_t)(parameter >> 32); + trb->Status = status; + XhciBarrier(); + trb->Control = controlWithoutCycle | dev->Ep0Cycle; + + if (outPhys) *outPhys = phys; + return STATUS_SUCCESS; +} + + +KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + const USB_SETUP_PACKET* setup, + void* buffer, + PHYSICAL_ADDRESS bufferPhys, + uint16_t length) { + if (!buffer || length == 0 || length > PAGE_SIZE) + return STATUS_INVALID_PARAMETER; + + memset(buffer, 0, length); + + uint64_t setupValue = UsbSetupPacketToU64(setup); + KSTATUS st = XhciEp0AppendTrb( + dev, + setupValue, + 8, + (XHCI_TRB_TYPE_SETUP << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_IDT | + XHCI_TRB_SETUP_TRT_IN, + NULL); + if (!KSUCCESS(st)) return st; + + st = XhciEp0AppendTrb( + dev, + bufferPhys, + length, + (XHCI_TRB_TYPE_DATA << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_DATA_DIR_IN, + NULL); + if (!KSUCCESS(st)) return st; + + uint64_t statusTrbPhys = 0; + st = XhciEp0AppendTrb( + dev, + 0, + 0, + (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_IOC, + &statusTrbPhys); + if (!KSUCCESS(st)) return st; + + XhciBarrier(); + XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); // endpoint ID 1 = EP0 + + XHCI_WAIT_TARGET target; + memset(&target, 0, sizeof(target)); + target.TransferPhys = statusTrbPhys; + target.TransferSlotId = dev->SlotId; + target.TransferEndpointId = 1; + + st = XhciWaitForTarget(xhci, &target, 1000000u); + if (!KSUCCESS(st)) { + DbgPrint("usb: xHCI control IN timeout slot=%u len=%u\n", dev->SlotId, length); + return st; + } + if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS) + return STATUS_SUCCESS; + + DbgPrint("usb: xHCI control IN failed slot=%u cc=%u residue=%u\n", + dev->SlotId, + target.Transfer.CompletionCode, + target.Transfer.Residue); + return STATUS_DEVICE_ERROR; +} + + +KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + const USB_SETUP_PACKET* setup) { + uint64_t setupValue = UsbSetupPacketToU64(setup); + KSTATUS st = XhciEp0AppendTrb( + dev, + setupValue, + 8, + (XHCI_TRB_TYPE_SETUP << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_IDT | + XHCI_TRB_SETUP_TRT_NONE, + NULL); + if (!KSUCCESS(st)) return st; + + uint64_t statusTrbPhys = 0; + st = XhciEp0AppendTrb( + dev, + 0, + 0, + (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_STATUS_DIR_IN | + XHCI_TRB_IOC, + &statusTrbPhys); + if (!KSUCCESS(st)) return st; + + XhciBarrier(); + XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, 1u); + + XHCI_WAIT_TARGET target; + memset(&target, 0, sizeof(target)); + target.TransferPhys = statusTrbPhys; + target.TransferSlotId = dev->SlotId; + target.TransferEndpointId = 1; + + st = XhciWaitForTarget(xhci, &target, 1000000u); + if (!KSUCCESS(st)) { + DbgPrint("usb: xHCI control no-data timeout slot=%u req=%u value=%04x\n", + dev->SlotId, setup->bRequest, setup->wValue); + return st; + } + if (target.Transfer.CompletionCode == XHCI_CC_SUCCESS) + return STATUS_SUCCESS; + + DbgPrint("usb: xHCI control no-data failed slot=%u cc=%u residue=%u\n", + dev->SlotId, + target.Transfer.CompletionCode, + target.Transfer.Residue); + return STATUS_DEVICE_ERROR; +} + + +KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + USB_DEVICE_DESCRIPTOR* outDesc) { + USB_SETUP_PACKET setup; + setup.bmRequestType = USB_DIR_IN; + setup.bRequest = USB_REQ_GET_DESCRIPTOR; + setup.wValue = (uint16_t)(USB_DESC_DEVICE << 8); + setup.wIndex = 0; + setup.wLength = sizeof(USB_DEVICE_DESCRIPTOR); + + KSTATUS st = XhciControlIn(xhci, dev, &setup, + dev->DescriptorBuffer, + dev->DescriptorPhys, + sizeof(USB_DEVICE_DESCRIPTOR)); + if (!KSUCCESS(st)) return st; + + memcpy(outDesc, dev->DescriptorBuffer, sizeof(*outDesc)); + if (outDesc->bLength < sizeof(USB_DEVICE_DESCRIPTOR) || + outDesc->bDescriptorType != USB_DESC_DEVICE) { + DbgPrint("usb: invalid device descriptor slot=%u len=%u type=%u\n", + dev->SlotId, outDesc->bLength, outDesc->bDescriptorType); + return STATUS_DEVICE_ERROR; + } + + return STATUS_SUCCESS; +} + + +KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + uint8_t configurationIndex, + USB_CONFIGURATION_DESCRIPTOR* outConfig, + uint16_t* outTotalLength) { + USB_SETUP_PACKET setup; + setup.bmRequestType = USB_DIR_IN; + setup.bRequest = USB_REQ_GET_DESCRIPTOR; + setup.wValue = (uint16_t)((USB_DESC_CONFIGURATION << 8) | configurationIndex); + setup.wIndex = 0; + setup.wLength = sizeof(USB_CONFIGURATION_DESCRIPTOR); + + KSTATUS st = XhciControlIn(xhci, dev, &setup, + dev->DescriptorBuffer, + dev->DescriptorPhys, + sizeof(USB_CONFIGURATION_DESCRIPTOR)); + if (!KSUCCESS(st)) return st; + + memcpy(outConfig, dev->DescriptorBuffer, sizeof(*outConfig)); + if (outConfig->bLength < sizeof(USB_CONFIGURATION_DESCRIPTOR) || + outConfig->bDescriptorType != USB_DESC_CONFIGURATION || + outConfig->wTotalLength < sizeof(USB_CONFIGURATION_DESCRIPTOR)) { + DbgPrint("usb: invalid config descriptor slot=%u index=%u len=%u type=%u total=%u\n", + dev->SlotId, + configurationIndex, + outConfig->bLength, + outConfig->bDescriptorType, + outConfig->wTotalLength); + return STATUS_DEVICE_ERROR; + } + + uint16_t total = outConfig->wTotalLength; + if (total > PAGE_SIZE) { + DbgPrint("usb: config descriptor slot=%u index=%u total=%u too large, capped at %u\n", + dev->SlotId, configurationIndex, total, (uint32_t)PAGE_SIZE); + total = (uint16_t)PAGE_SIZE; + } + + setup.wLength = total; + st = XhciControlIn(xhci, dev, &setup, + dev->DescriptorBuffer, + dev->DescriptorPhys, + total); + if (!KSUCCESS(st)) return st; + + memcpy(outConfig, dev->DescriptorBuffer, sizeof(*outConfig)); + if (outConfig->bLength < sizeof(USB_CONFIGURATION_DESCRIPTOR) || + outConfig->bDescriptorType != USB_DESC_CONFIGURATION) { + DbgPrint("usb: invalid full config descriptor slot=%u index=%u len=%u type=%u\n", + dev->SlotId, + configurationIndex, + outConfig->bLength, + outConfig->bDescriptorType); + return STATUS_DEVICE_ERROR; + } + + if (outTotalLength) *outTotalLength = total; + return STATUS_SUCCESS; +} + + +uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress) { + uint8_t number = endpointAddress & USB_ENDPOINT_NUMBER_MASK; + if (number == 0) return 1; + return (uint8_t)(number * 2u + ((endpointAddress & USB_ENDPOINT_DIR_IN) ? 1u : 0u)); +} + + +uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) { + if (usbInterval == 0) usbInterval = 1; + + if (speed == 3 || speed == 4 || speed == 5) { + // High-speed and newer interrupt endpoints already encode bInterval as + // an exponent in USB descriptors. xHCI stores the zero-based exponent. + if (usbInterval > 16) usbInterval = 16; + return (uint8_t)(usbInterval - 1u); + } + + // Full/low-speed interrupt endpoints express bInterval in milliseconds. + // xHCI wants the nearest exponent in 125us units. + uint32_t units125us = (uint32_t)usbInterval * 8u; + uint8_t interval = 0; + uint32_t v = 1; + while (v < units125us && interval < 15u) { + v <<= 1; + interval++; + } + return interval; +} + + +KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, + uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, + uint64_t parameter, uint32_t status, + uint32_t controlWithoutCycle, uint64_t* outPhys) { + if (!ring || !enqueue || !cycle || trbCount < 2) + return STATUS_INVALID_PARAMETER; + + if (*enqueue >= trbCount - 1u) { + XHCI_TRB* link = &ring[trbCount - 1u]; + link->ParameterLo = (uint32_t)ringPhys; + link->ParameterHi = (uint32_t)(ringPhys >> 32); + link->Status = 0; + XhciBarrier(); + link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_LINK_TC | + *cycle; + *enqueue = 0; + *cycle ^= 1u; + } + + uint32_t index = (*enqueue)++; + XHCI_TRB* trb = &ring[index]; + uint64_t phys = ringPhys + (uint64_t)index * sizeof(XHCI_TRB); + + trb->ParameterLo = (uint32_t)parameter; + trb->ParameterHi = (uint32_t)(parameter >> 32); + trb->Status = status; + XhciBarrier(); + trb->Control = controlWithoutCycle | *cycle; + + if (outPhys) *outPhys = phys; + return STATUS_SUCCESS; +} + + +void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, + uint32_t trbCount, uint8_t cycle) { + memset(ring, 0, trbCount * sizeof(XHCI_TRB)); + XHCI_TRB* link = &ring[trbCount - 1u]; + link->ParameterLo = (uint32_t)ringPhys; + link->ParameterHi = (uint32_t)(ringPhys >> 32); + link->Status = 0; + link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_LINK_TC | + cycle; +} + + +KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, + uint8_t endpointId, + XHCI_TRB* ring, + PHYSICAL_ADDRESS ringPhys, + uint32_t* enqueue, + uint8_t* cycle, + PHYSICAL_ADDRESS bufferPhys, + uint32_t length, + uint32_t* outActual) { + if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferPhys || length == 0) + return STATUS_INVALID_PARAMETER; + if (length > USB_STORAGE_MAX_BYTES) + return STATUS_INVALID_PARAMETER; + + uint64_t trbPhys = 0; + KSTATUS st = XhciRingAppend(ring, + ringPhys, + XHCI_BULK_RING_TRBS, + enqueue, + cycle, + bufferPhys, + length, + (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | + XHCI_TRB_ISP | + XHCI_TRB_IOC, + &trbPhys); + if (!KSUCCESS(st)) return st; + + XHCI_WAIT_TARGET target; + memset(&target, 0, sizeof(target)); + target.TransferPhys = trbPhys; + target.TransferSlotId = dev->SlotId; + target.TransferEndpointId = endpointId; + + xhci->ActiveWait = ⌖ + XhciBarrier(); + XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, endpointId); + + st = XhciWaitForTarget(xhci, &target, 1000000u); + if (!KSUCCESS(st)) { + DbgPrint("usb-storage: bulk transfer timeout slot=%u ep=%u len=%u\n", + dev->SlotId, endpointId, length); + return st; + } + if (target.Transfer.CompletionCode != XHCI_CC_SUCCESS) { + DbgPrint("usb-storage: bulk transfer failed slot=%u ep=%u cc=%u residue=%u\n", + dev->SlotId, + endpointId, + target.Transfer.CompletionCode, + target.Transfer.Residue); + return STATUS_DEVICE_ERROR; + } + + if (outActual) + *outActual = length - target.Transfer.Residue; + return STATUS_SUCCESS; +} + + +KSTATUS XhciInitialise(const HAL_API* hal, + uint8_t bus, uint8_t dev, uint8_t fn, + uint16_t vendor, uint16_t device) { + if (gHaveXhci) { + DbgPrint("usb: additional xHCI controller at %u:%u.%u ignored for now\n", + bus, dev, fn); + return STATUS_ALREADY_EXISTS; + } + + USB_BAR bar; + if (!UsbReadBar(hal, bus, dev, fn, 0, &bar) || bar.Kind != UsbBarMmio) { + DbgPrint("usb: xHCI controller has no usable MMIO BAR0\n"); + return STATUS_NOT_SUPPORTED; + } + + memset(&gXhci, 0, sizeof(gXhci)); + XHCI_CONTROLLER* xhci = &gXhci; + KeInitializeDpc(&xhci->EventDpc, XhciEventDpcRoutine, xhci); + xhci->Hal = hal; + xhci->Bus = bus; + xhci->Dev = dev; + xhci->Fn = fn; + xhci->Irq = hal->Pci.GetIrqLine(bus, dev, fn); + xhci->MmioPhys = bar.Base; + + uint32_t pciCmd = hal->Pci.Read32(bus, dev, fn, PCI_REG_COMMAND); + pciCmd |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; + hal->Pci.Write32(bus, dev, fn, PCI_REG_COMMAND, pciCmd); + + if (xhci->Irq == 0xFFu) { + uint8_t msiIrq = 0xFFu; + KSTATUS irqSt = KeIrqAllocateMsiVector(&msiIrq); + if (!KSUCCESS(irqSt)) { + DbgPrint("usb: xHCI has no PCI INTx IRQ line and MSI vector allocation failed st=0x%x\n", + irqSt); + return irqSt; + } + if (!hal->Pci.EnableMsi) { + DbgPrint("usb: xHCI has no PCI INTx IRQ line and HAL MSI support is unavailable\n"); + return STATUS_NOT_SUPPORTED; + } + irqSt = hal->Pci.EnableMsi(bus, dev, fn, msiIrq); + if (!KSUCCESS(irqSt)) { + DbgPrint("usb: xHCI has no PCI INTx IRQ line and MSI enable failed st=0x%x\n", + irqSt); + return irqSt; + } + xhci->Irq = msiIrq; + xhci->UsingMsi = true; + DbgPrint("usb: xHCI using MSI irqslot=%u vector=0x%02x\n", + xhci->Irq, 0x20u + xhci->Irq); + } else { + pciCmd = hal->Pci.Read32(bus, dev, fn, PCI_REG_COMMAND); + pciCmd &= ~PCI_COMMAND_INTX_DISABLE; + hal->Pci.Write32(bus, dev, fn, PCI_REG_COMMAND, pciCmd); + } + + KSTATUS st = UsbMapMmio(hal, xhci->MmioPhys, PAGE_SIZE, &xhci->Mmio); + if (!KSUCCESS(st)) return st; + + xhci->Cap = xhci->Mmio; + uint32_t cap0 = XhciRead32(xhci->Cap, XHCI_CAP_CAPLENGTH); + xhci->CapLength = (uint8_t)(cap0 & 0xFFu); + xhci->Version = (uint16_t)(cap0 >> 16); + xhci->Hcs1 = XhciRead32(xhci->Cap, XHCI_CAP_HCSPARAMS1); + xhci->Hcs2 = XhciRead32(xhci->Cap, XHCI_CAP_HCSPARAMS2); + xhci->Hcc1 = XhciRead32(xhci->Cap, XHCI_CAP_HCCPARAMS1); + xhci->DbOff = XhciRead32(xhci->Cap, XHCI_CAP_DBOFF) & ~0x3u; + xhci->RtsOff = XhciRead32(xhci->Cap, XHCI_CAP_RTSOFF) & ~0x1Fu; + xhci->MaxSlots = (uint8_t)(xhci->Hcs1 & 0xFFu); + xhci->MaxIntrs = (uint16_t)((xhci->Hcs1 >> 8) & 0x7FFu); + xhci->MaxPorts = (uint8_t)((xhci->Hcs1 >> 24) & 0xFFu); + xhci->ScratchpadCount = XhciScratchpadCount(xhci->Hcs2); + xhci->ContextBytes = (xhci->Hcc1 & XHCI_HCC1_CSZ) ? 64u : 32u; + + if (xhci->CapLength < 0x20 || xhci->DbOff == 0 || xhci->RtsOff == 0 || + xhci->MaxSlots == 0 || xhci->MaxPorts == 0) { + DbgPrint("usb: xHCI invalid caps caplen=%u dboff=0x%x rtsoff=0x%x hcs1=%08x\n", + xhci->CapLength, xhci->DbOff, xhci->RtsOff, xhci->Hcs1); + return STATUS_DEVICE_ERROR; + } + + uint32_t portEnd = xhci->CapLength + XHCI_OP_PORT_BASE + + (uint32_t)xhci->MaxPorts * XHCI_OP_PORT_STRIDE; + uint32_t needBytes = portEnd; + if (xhci->DbOff + 0x100u > needBytes) needBytes = xhci->DbOff + 0x100u; + if (xhci->RtsOff + 0x1000u > needBytes) needBytes = xhci->RtsOff + 0x1000u; + if (needBytes < XHCI_MMIO_FLOOR_BYTES) needBytes = XHCI_MMIO_FLOOR_BYTES; + needBytes = UsbAlignUp32(needBytes, (uint32_t)PAGE_SIZE); + + st = UsbMapMmio(hal, xhci->MmioPhys, needBytes, &xhci->Mmio); + if (!KSUCCESS(st)) return st; + xhci->MmioBytes = needBytes; + xhci->Cap = xhci->Mmio; + xhci->Op = xhci->Mmio + xhci->CapLength; + xhci->Runtime = xhci->Mmio + xhci->RtsOff; + xhci->Doorbells = xhci->Mmio + xhci->DbOff; + + DbgPrint("usb: xHCI init %u:%u.%u vendor=%04x device=%04x irq=%u%s mmio=0x%016llX caplen=%u version=%04x\n", + bus, dev, fn, vendor, device, xhci->Irq, + xhci->UsingMsi ? "(msi)" : "", + (unsigned long long)xhci->MmioPhys, xhci->CapLength, xhci->Version); + DbgPrint("usb: xHCI caps slots=%u intrs=%u ports=%u ctx=%u scratchpads=%u hcs2=%08x hcc1=%08x dboff=0x%x rtsoff=0x%x pagesize=%08x\n", + xhci->MaxSlots, xhci->MaxIntrs, xhci->MaxPorts, xhci->ContextBytes, + xhci->ScratchpadCount, xhci->Hcs2, xhci->Hcc1, xhci->DbOff, + xhci->RtsOff, XhciRead32(xhci->Op, XHCI_OP_PAGESIZE)); + + XhciParseSupportedProtocols(xhci); + + st = XhciBiosHandoff(xhci); + if (!KSUCCESS(st)) return st; + + st = XhciStopAndReset(xhci); + if (!KSUCCESS(st)) return st; + + uint32_t pageSize = XhciRead32(xhci->Op, XHCI_OP_PAGESIZE); + if ((pageSize & 1u) == 0) { + DbgPrint("usb: xHCI does not advertise 4K pages pagesize=%08x\n", pageSize); + return STATUS_NOT_SUPPORTED; + } + + st = XhciAllocateRuntimeMemory(xhci); + if (!KSUCCESS(st)) return st; + + st = XhciProgramRuntime(xhci); + if (!KSUCCESS(st)) return st; + + st = XhciStart(xhci); + if (!KSUCCESS(st)) return st; + + st = XhciSubmitNoopCommand(xhci); + if (!KSUCCESS(st)) return st; + + XhciLogPortStatus(xhci); + XhciEnumerateRootPorts(xhci); + + if (xhci->HidDeviceCount > 0) { + st = XhciStartHidInterrupts(xhci); + if (!KSUCCESS(st)) return st; + } + + KeIrqRegister(xhci->Irq, XhciIsr); + XhciEnableInterrupts(xhci); + xhci->InterruptsEnabled = true; + KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0); + DbgPrint("usb: xHCI ISR registered on IRQ %u%s; event ring handled by DPC\n", + xhci->Irq, xhci->UsingMsi ? "(msi)" : ""); + + gHaveXhci = true; + DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage is next\n", + xhci->HidDeviceCount); + return STATUS_SUCCESS; +} diff --git a/code/drivers/drivers.ini b/code/drivers/drivers.ini index f76c9f0..d1b3560 100644 --- a/code/drivers/drivers.ini +++ b/code/drivers/drivers.ini @@ -1,9 +1,10 @@ [boot] +ps2.sys ide.sys ahci.sys +usb.sys fat32.sys [runtime] lwip.sys -usb.sys rtl8139.sys diff --git a/code/drivers/fs/fat32/fat32.c b/code/drivers/fs/fat32/fat32.c index 332199e..87cda9e 100644 --- a/code/drivers/fs/fat32/fat32.c +++ b/code/drivers/fs/fat32/fat32.c @@ -1,54 +1,5 @@ -// fat32.c - FAT32 filesystem driver -// Read + write support: file data, cluster allocation, directory entries. -// -// Performance improvements over the original: -// -// 1. FAT sector cache (FAT_CACHE_SLOTS write-back entries). -// FatEntry() and WriteFatEntry() go through the cache; the hot FAT -// region stays in RAM instead of being read from disk on every chain -// traversal step. A 32-slot cache covers 32 x (512/4) = 4096 -// consecutive cluster entries — enough to keep the entire FAT of a -// small partition in memory, and to avoid repeated reads even on large -// ones. -// -// 2. Fat32ReadDir() cluster buffer cached in FAT32_DIR_CTX. -// The original re-read the same cluster from disk on every call -// (allocate -> read -> return one entry -> free). Now the buffer is -// allocated once in Fat32OpenDir, freed in Fat32CloseDir, and only -// re-read when the iteration moves to a new cluster. -// -// 3. AllocCluster() next-free hint. -// The search now starts from the cluster after the last allocation -// (volume NextFreeHint) and wraps around once, so repeated allocations -// on a fragmented volume don't rescan from sector 0 every time. -// -// 4. Contiguous-cluster run coalescing in Fat32Read() and Fat32Write(). -// After fetching a cluster's FAT entry the code peeks ahead: if the -// next cluster is physically adjacent (cluster+1) it extends the run -// rather than issuing a separate ReadSectors/WriteSectors call. A -// freshly-written unfragmented file now transfers in one or a handful -// of large DMA bursts instead of SectorsPerCluster-sized slices. -// -// 5. FAT write batching via a dirty-sector cache. -// WriteFatEntry() marks modified FAT cache slots dirty instead of -// flushing immediately. FlushFatCache() (called from Fat32Close, -// Fat32Truncate, and Fat32Delete) writes each dirty slot once for -// every FAT copy, collapsing the 2x(read+write) per-entry cost of the -// original into a single write per dirty sector per FAT copy. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +// fat32.c - FAT32 filesystem core, directory, cache, and I/O implementation. +#include "fat32.h" // --------------------------------------------------------------------------- // On-disk structures @@ -1042,7 +993,7 @@ static int FindFreeDirentRun(FAT32_VOL *v, uint32_t dirCluster, uint32_t needed, // --------------------------------------------------------------------------- // Open / Close // --------------------------------------------------------------------------- -static KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { +KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0) return STATUS_INVALID_PARAMETER; @@ -1059,7 +1010,7 @@ static KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx return STATUS_SUCCESS; } -static void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) { +void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) { (void)fs; FAT32_FILE *f = (FAT32_FILE *)fileCtx; FlushFatCache(f->Vol); @@ -1079,7 +1030,7 @@ static void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) { #define MAX_RUN_SECTORS 128u -static KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, +KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead) { (void)fs; @@ -1174,7 +1125,7 @@ static KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, // Same strategy as read. Whole-cluster-aligned runs skip the read-modify- // write; partial head/tail clusters still require it. -static KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, +KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten) { (void)fs; @@ -1279,9 +1230,9 @@ static KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, // --------------------------------------------------------------------------- // Create // --------------------------------------------------------------------------- -static KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize); +KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize); -static KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { +KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0) return STATUS_INVALID_PARAMETER; @@ -1385,7 +1336,7 @@ static KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileC // --------------------------------------------------------------------------- // Truncate // --------------------------------------------------------------------------- -static KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) { +KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) { (void)fs; FAT32_FILE *f = (FAT32_FILE *)fileCtx; FAT32_VOL *v = f->Vol; @@ -1445,7 +1396,7 @@ update_size: // --------------------------------------------------------------------------- // Delete // --------------------------------------------------------------------------- -static KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) { +KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) { if (!fs || !SysSvIsValid(path) || path.Length == 0) return STATUS_INVALID_PARAMETER; @@ -1467,7 +1418,7 @@ static KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) { return STATUS_SUCCESS; } -static uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) { +uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) { (void)fs; return ((FAT32_FILE *)fileCtx)->FileSize; } @@ -1489,7 +1440,7 @@ typedef struct { uint8_t *ClusterBuf; // allocated once in OpenDir, freed in CloseDir } FAT32_DIR_CTX; -static KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx) { +KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx) { if (!fs || !dirCtx || !SysSvIsValid(dirPath)) return STATUS_INVALID_PARAMETER; @@ -1520,7 +1471,7 @@ static KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **d return STATUS_SUCCESS; } -static KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) { +KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) { (void)fs; FAT32_DIR_CTX *ctx = (FAT32_DIR_CTX *)dirCtx; FAT32_VOL *v = ctx->Vol; @@ -1575,7 +1526,7 @@ static KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) { return STATUS_NO_MORE_FILES; } -static void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx) { +void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx) { (void)fs; FAT32_DIR_CTX *ctx = (FAT32_DIR_CTX *)dirCtx; MmFreePool(ctx->ClusterBuf); @@ -1934,7 +1885,7 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) { Fat32AsyncFinish(ctx, s, ctx->Done); } -static KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, +KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext) { @@ -2087,7 +2038,7 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) { Fat32AsyncFinish(ctx, s, ctx->Done); } -static KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, +KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext) { @@ -2174,22 +2125,101 @@ static KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, static FAT32_VOL gVols[FAT32_MAX_VOLUMES]; static IO_FILESYSTEM gFs[FAT32_MAX_VOLUMES]; -static void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) { - memset(fs, 0, sizeof(*fs)); - fs->Open = Fat32Open; - fs->Read = Fat32Read; - fs->Write = Fat32Write; - fs->ReadAsync = Fat32ReadAsync; - fs->WriteAsync = Fat32WriteAsync; - fs->Create = Fat32Create; - fs->Delete = Fat32Delete; - fs->Truncate = Fat32Truncate; - fs->GetSize = Fat32GetSize; - fs->Close = Fat32Close; - fs->DriverCtx = vol; - fs->OpenDir = Fat32OpenDir; - fs->ReadDir = Fat32ReadDir; - fs->CloseDir = Fat32CloseDir; + +static uint32_t gMountedCount; +static bool gDiskScanned[FAT32_MAX_VOLUMES]; + +static uint32_t Fat32TryMountDisk(uint8_t disk) { + if (disk >= FAT32_MAX_VOLUMES) + return 0; + if (gDiskScanned[disk]) + return 0; + gDiskScanned[disk] = true; + + if (!IoDiskDevicePresent(disk)) + return 0; + + uint32_t mountedHere = 0; + FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK]; + uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK); + if (!partCount) { + DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk); + return 0; + } + + for (uint32_t part = 0; part < partCount && gMountedCount < FAT32_MAX_VOLUMES; part++) { + uint8_t volumeIndex = (uint8_t)gMountedCount; + FAT32_VOL* v = &gVols[volumeIndex]; + memset(v, 0, sizeof(*v)); + v->DriveIndex = disk; + v->PartitionLba = parts[part].StartLba; + v->PartitionSize = parts[part].SectorCount; + + uint8_t sector[512]; + if (!KSUCCESS(ReadSectors(v, v->PartitionLba, 1, sector))) { + DbgPrint("FAT32: disk %u part %u: failed to read VBR\n", disk, part); + continue; + } + + BPB *bpb = (BPB *)sector; + if (!IsFat32Vbr(v, v->PartitionLba)) { + DbgPrint("FAT32: disk %u part %u: unsupported FAT variant\n", disk, part); + continue; + } + + v->BytesPerSector = bpb->BytesPerSector; + v->SectorsPerCluster = bpb->SectorsPerCluster; + v->BytesPerCluster = bpb->BytesPerSector * bpb->SectorsPerCluster; + v->ReservedSectors = bpb->ReservedSectors; + v->NumFats = bpb->NumFats; + v->FatSize = bpb->FatSize32; + v->RootCluster = bpb->RootCluster; + v->DataStart = v->PartitionLba + bpb->ReservedSectors + (uint64_t)bpb->NumFats * bpb->FatSize32; + v->NextFreeHint = 2; + + uint32_t totalSectors = bpb->TotalSectors16 ? bpb->TotalSectors16 : bpb->TotalSectors32; + uint32_t dataSectors = totalSectors - bpb->ReservedSectors - bpb->NumFats * bpb->FatSize32; + v->TotalClusters = dataSectors / bpb->SectorsPerCluster; + if (v->PartitionSize == 0 || v->PartitionSize > totalSectors) + v->PartitionSize = totalSectors; + + Fat32InitVtable(&gFs[volumeIndex], v); + + IO_VOLUME_PARAMS volParams; + volParams.DiskIndex = v->DriveIndex; + volParams.PartitionLba = v->PartitionLba; + volParams.PartitionSize = v->PartitionSize; + + KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams); + if (KSUCCESS(status)) { + gMountedCount++; + mountedHere++; + DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n", + disk, part, volumeIndex, (unsigned long long)v->PartitionLba, v->TotalClusters); + } else { + DbgPrint("FAT32: disk %u part %u mount as drive %u failed status=%d.\n", + disk, part, volumeIndex, status); + } + } + + if (partCount && !mountedHere && gMountedCount >= FAT32_MAX_VOLUMES) { + DbgPrint("FAT32: disk %u: volume table full\n", disk); + } + + return mountedHere; +} + +static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* context) { + (void)device; + (void)context; + uint32_t before = gMountedCount; + uint32_t mounted = Fat32TryMountDisk(driveIndex); + if (mounted) { + DbgPrint("FAT32: hotplug scan mounted %u new volume(s); total=%u\n", + mounted, gMountedCount); + } else if (gMountedCount != before) { + DbgPrint("FAT32: hotplug scan total=%u\n", gMountedCount); + } } KSTATUS DriverEntry(const HAL_API *hal) { @@ -2198,84 +2228,19 @@ KSTATUS DriverEntry(const HAL_API *hal) { FatCacheInvalidate(); (void)hal; + gMountedCount = 0; + memset(gDiskScanned, 0, sizeof(gDiskScanned)); - uint32_t mounted = 0; - KSTATUS lastStatus = STATUS_NOT_SUPPORTED; - - for (uint8_t disk = 0; disk < FAT32_MAX_VOLUMES && mounted < FAT32_MAX_VOLUMES; disk++) { - if (!IoDiskDevicePresent(disk)) - continue; - - FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK]; - uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK); - if (!partCount) { - DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk); - continue; - } - - for (uint32_t part = 0; part < partCount && mounted < FAT32_MAX_VOLUMES; part++) { - uint8_t volumeIndex = (uint8_t)mounted; - FAT32_VOL* v = &gVols[volumeIndex]; - memset(v, 0, sizeof(*v)); - v->DriveIndex = disk; - v->PartitionLba = parts[part].StartLba; - v->PartitionSize = parts[part].SectorCount; - - uint8_t sector[512]; - if (!KSUCCESS(ReadSectors(v, v->PartitionLba, 1, sector))) { - lastStatus = STATUS_DEVICE_ERROR; - DbgPrint("FAT32: disk %u part %u: failed to read VBR\n", disk, part); - continue; - } - - BPB *bpb = (BPB *)sector; - if (!IsFat32Vbr(v, v->PartitionLba)) { - lastStatus = STATUS_NOT_SUPPORTED; - DbgPrint("FAT32: disk %u part %u: unsupported FAT variant\n", disk, part); - continue; - } - - v->BytesPerSector = bpb->BytesPerSector; - v->SectorsPerCluster = bpb->SectorsPerCluster; - v->BytesPerCluster = bpb->BytesPerSector * bpb->SectorsPerCluster; - v->ReservedSectors = bpb->ReservedSectors; - v->NumFats = bpb->NumFats; - v->FatSize = bpb->FatSize32; - v->RootCluster = bpb->RootCluster; - v->DataStart = v->PartitionLba + bpb->ReservedSectors + (uint64_t)bpb->NumFats * bpb->FatSize32; - v->NextFreeHint = 2; - - uint32_t totalSectors = bpb->TotalSectors16 ? bpb->TotalSectors16 : bpb->TotalSectors32; - uint32_t dataSectors = totalSectors - bpb->ReservedSectors - bpb->NumFats * bpb->FatSize32; - v->TotalClusters = dataSectors / bpb->SectorsPerCluster; - if (v->PartitionSize == 0 || v->PartitionSize > totalSectors) - v->PartitionSize = totalSectors; - - Fat32InitVtable(&gFs[volumeIndex], v); - - IO_VOLUME_PARAMS volParams; - volParams.DiskIndex = v->DriveIndex; - volParams.PartitionLba = v->PartitionLba; - volParams.PartitionSize = v->PartitionSize; - - KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams); - if (KSUCCESS(status)) { - mounted++; - DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n", - disk, part, volumeIndex, (unsigned long long)v->PartitionLba, v->TotalClusters); - } else { - lastStatus = status; - DbgPrint("FAT32: disk %u part %u mount as drive %u failed status=%d.\n", - disk, part, volumeIndex, status); - } - } + KSTATUS cbStatus = IoRegisterDiskArrivalCallback(Fat32DiskArrival, (void*)0); + if (!KSUCCESS(cbStatus)) { + DbgPrint("FAT32: disk-arrival callback registration failed st=0x%x\n", cbStatus); } - if (!mounted) { + if (!gMountedCount) { DbgPrint("FAT32: no mountable FAT32 volumes found.\n"); - return lastStatus; + return STATUS_NOT_SUPPORTED; } - DbgPrint("FAT32: Init complete; mounted %u volume(s).\n", mounted); + DbgPrint("FAT32: Init complete; mounted %u volume(s).\n", gMountedCount); return STATUS_SUCCESS; } diff --git a/code/drivers/fs/fat32/fat32.h b/code/drivers/fs/fat32/fat32.h new file mode 100644 index 0000000..be93f55 --- /dev/null +++ b/code/drivers/fs/fat32/fat32.h @@ -0,0 +1,34 @@ +// fat32.h - shared FAT32 driver internals. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct FAT32_VOL_S FAT32_VOL; + +KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx); +void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx); +KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead); +KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten); +KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext); +KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext); +KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx); +KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path); +KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize); +uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx); +KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx); +KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out); +void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx); + +void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol); diff --git a/code/drivers/fs/fat32/fat32_mount.c b/code/drivers/fs/fat32/fat32_mount.c new file mode 100644 index 0000000..bffc562 --- /dev/null +++ b/code/drivers/fs/fat32/fat32_mount.c @@ -0,0 +1,20 @@ +// fat32_mount.c - filesystem vtable/mount wiring. +#include "fat32.h" + +void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) { + memset(fs, 0, sizeof(*fs)); + fs->Open = Fat32Open; + fs->Read = Fat32Read; + fs->Write = Fat32Write; + fs->ReadAsync = Fat32ReadAsync; + fs->WriteAsync = Fat32WriteAsync; + fs->Create = Fat32Create; + fs->Delete = Fat32Delete; + fs->Truncate = Fat32Truncate; + fs->GetSize = Fat32GetSize; + fs->Close = Fat32Close; + fs->DriverCtx = vol; + fs->OpenDir = Fat32OpenDir; + fs->ReadDir = Fat32ReadDir; + fs->CloseDir = Fat32CloseDir; +} diff --git a/code/drivers/input/ps2/ps2.c b/code/drivers/input/ps2/ps2.c new file mode 100644 index 0000000..24f532f --- /dev/null +++ b/code/drivers/input/ps2/ps2.c @@ -0,0 +1,263 @@ +// ps2.c - PS/2 keyboard/mouse event decoding and kernel input binding. +#include "ps2.h" + +static KDPC gMouseDpc; +static volatile bool gKeyboardNextExtended; +static volatile uint8_t gMousePacket[3]; +static volatile int gMousePacketIndex; +static PS2_MOUSE_PACKET gMouseDpcQueue[MOUSE_DPC_QUEUE_SIZE]; +static volatile int gMouseDpcHead; +static volatile int gMouseDpcTail; + +static INPUT_KEY Ps2Set1ToKey(uint8_t make, bool extended) { + if (extended) { + switch (make) { + case 0x1C: return INPUT_KEY_KP_ENTER; + case 0x1D: return INPUT_KEY_RIGHT_CTRL; + case 0x35: return INPUT_KEY_KP_DIVIDE; + case 0x38: return INPUT_KEY_RIGHT_ALT; + case 0x47: return INPUT_KEY_HOME; + case 0x48: return INPUT_KEY_UP; + case 0x49: return INPUT_KEY_PAGE_UP; + case 0x4B: return INPUT_KEY_LEFT; + case 0x4D: return INPUT_KEY_RIGHT; + case 0x4F: return INPUT_KEY_END; + case 0x50: return INPUT_KEY_DOWN; + case 0x51: return INPUT_KEY_PAGE_DOWN; + case 0x52: return INPUT_KEY_INSERT; + case 0x53: return INPUT_KEY_DELETE; + default: return INPUT_KEY_NONE; + } + } + + switch (make) { + case 0x01: return INPUT_KEY_ESCAPE; + case 0x02: return INPUT_KEY_1; + case 0x03: return INPUT_KEY_2; + case 0x04: return INPUT_KEY_3; + case 0x05: return INPUT_KEY_4; + case 0x06: return INPUT_KEY_5; + case 0x07: return INPUT_KEY_6; + case 0x08: return INPUT_KEY_7; + case 0x09: return INPUT_KEY_8; + case 0x0A: return INPUT_KEY_9; + case 0x0B: return INPUT_KEY_0; + case 0x0C: return INPUT_KEY_MINUS; + case 0x0D: return INPUT_KEY_EQUAL; + case 0x0E: return INPUT_KEY_BACKSPACE; + case 0x0F: return INPUT_KEY_TAB; + case 0x10: return INPUT_KEY_Q; + case 0x11: return INPUT_KEY_W; + case 0x12: return INPUT_KEY_E; + case 0x13: return INPUT_KEY_R; + case 0x14: return INPUT_KEY_T; + case 0x15: return INPUT_KEY_Y; + case 0x16: return INPUT_KEY_U; + case 0x17: return INPUT_KEY_I; + case 0x18: return INPUT_KEY_O; + case 0x19: return INPUT_KEY_P; + case 0x1A: return INPUT_KEY_LEFT_BRACKET; + case 0x1B: return INPUT_KEY_RIGHT_BRACKET; + case 0x1C: return INPUT_KEY_ENTER; + case 0x1D: return INPUT_KEY_LEFT_CTRL; + case 0x1E: return INPUT_KEY_A; + case 0x1F: return INPUT_KEY_S; + case 0x20: return INPUT_KEY_D; + case 0x21: return INPUT_KEY_F; + case 0x22: return INPUT_KEY_G; + case 0x23: return INPUT_KEY_H; + case 0x24: return INPUT_KEY_J; + case 0x25: return INPUT_KEY_K; + case 0x26: return INPUT_KEY_L; + case 0x27: return INPUT_KEY_SEMICOLON; + case 0x28: return INPUT_KEY_APOSTROPHE; + case 0x29: return INPUT_KEY_GRAVE; + case 0x2A: return INPUT_KEY_LEFT_SHIFT; + case 0x2B: return INPUT_KEY_BACKSLASH; + case 0x2C: return INPUT_KEY_Z; + case 0x2D: return INPUT_KEY_X; + case 0x2E: return INPUT_KEY_C; + case 0x2F: return INPUT_KEY_V; + case 0x30: return INPUT_KEY_B; + case 0x31: return INPUT_KEY_N; + case 0x32: return INPUT_KEY_M; + case 0x33: return INPUT_KEY_COMMA; + case 0x34: return INPUT_KEY_PERIOD; + case 0x35: return INPUT_KEY_SLASH; + case 0x36: return INPUT_KEY_RIGHT_SHIFT; + case 0x37: return INPUT_KEY_KP_MULTIPLY; + case 0x38: return INPUT_KEY_LEFT_ALT; + case 0x39: return INPUT_KEY_SPACE; + case 0x3A: return INPUT_KEY_CAPS_LOCK; + case 0x3B: return INPUT_KEY_F1; + case 0x3C: return INPUT_KEY_F2; + case 0x3D: return INPUT_KEY_F3; + case 0x3E: return INPUT_KEY_F4; + case 0x3F: return INPUT_KEY_F5; + case 0x40: return INPUT_KEY_F6; + case 0x41: return INPUT_KEY_F7; + case 0x42: return INPUT_KEY_F8; + case 0x43: return INPUT_KEY_F9; + case 0x44: return INPUT_KEY_F10; + case 0x45: return INPUT_KEY_NUM_LOCK; + case 0x47: return INPUT_KEY_KP_7; + case 0x48: return INPUT_KEY_KP_8; + case 0x49: return INPUT_KEY_KP_9; + case 0x4A: return INPUT_KEY_KP_MINUS; + case 0x4B: return INPUT_KEY_KP_4; + case 0x4C: return INPUT_KEY_KP_5; + case 0x4D: return INPUT_KEY_KP_6; + case 0x4E: return INPUT_KEY_KP_PLUS; + case 0x4F: return INPUT_KEY_KP_1; + case 0x50: return INPUT_KEY_KP_2; + case 0x51: return INPUT_KEY_KP_3; + case 0x52: return INPUT_KEY_KP_0; + case 0x53: return INPUT_KEY_KP_PERIOD; + case 0x57: return INPUT_KEY_F11; + case 0x58: return INPUT_KEY_F12; + default: return INPUT_KEY_NONE; + } +} + +static void Ps2KeyboardIsr(void) { + uint8_t sc = __inbyte(PS2_DATA); + if (sc == SC1_EXT_PREFIX) { + gKeyboardNextExtended = true; + return; + } + + bool extended = gKeyboardNextExtended; + bool released = (sc & SC1_BREAK_BIT) != 0; + uint8_t make = sc & SC1_MAKE_MASK; + gKeyboardNextExtended = false; + + INPUT_KEY key = Ps2Set1ToKey(make, extended); + if (key == INPUT_KEY_NONE) return; + + INPUT_KEY_EVENT event = { + .Key = key, + .Pressed = !released, + }; + KeInputSubmitKey(&event); +} + +static void Ps2MouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)context; + (void)systemArgument1; + (void)systemArgument2; + + while (gMouseDpcHead != gMouseDpcTail) { + PS2_MOUSE_PACKET packet = gMouseDpcQueue[gMouseDpcHead]; + gMouseDpcHead = (gMouseDpcHead + 1) & (int)MOUSE_DPC_QUEUE_MASK; + + INPUT_MOUSE_EVENT event = { + .dx = packet.dx, + .dy = packet.dy, + .wheel = 0, + .buttons = packet.buttons, + }; + KeInputSubmitMouse(&event); + } +} + +static void Ps2MouseIsr(void) { + uint8_t data = __inbyte(PS2_DATA); + + if (gMousePacketIndex == 0 && !(data & 0x08u)) + return; + + gMousePacket[gMousePacketIndex++] = data; + if (gMousePacketIndex < 3) + return; + gMousePacketIndex = 0; + + uint8_t flags = gMousePacket[0]; + if (flags & 0xC0u) + return; + + int dx = (int)gMousePacket[1] - ((flags & 0x10u) ? 256 : 0); + int dy = (int)gMousePacket[2] - ((flags & 0x20u) ? 256 : 0); + dy = -dy; + + int next = (gMouseDpcTail + 1) & (int)MOUSE_DPC_QUEUE_MASK; + if (next != gMouseDpcHead) { + gMouseDpcQueue[gMouseDpcTail].dx = dx; + gMouseDpcQueue[gMouseDpcTail].dy = dy; + gMouseDpcQueue[gMouseDpcTail].buttons = flags & 0x07u; + gMouseDpcTail = next; + KeInsertQueueDpc(&gMouseDpc, 0, 0); + } +} + +static void Ps2ControllerPrepare(void) { + Ps2WriteCommand(0xADu); // disable keyboard port + Ps2WriteCommand(0xA7u); // disable aux port + Ps2FlushOutput(); + + uint8_t commandByte = Ps2ReadCommandByte(); + commandByte &= (uint8_t)~0x01u; // keep keyboard IRQ disabled during polled init + commandByte &= (uint8_t)~0x02u; // keep mouse IRQ disabled during polled init + commandByte |= 0x40u; // translate keyboard stream to set 1 + commandByte &= (uint8_t)~0x10u; // keyboard clock enabled + commandByte &= (uint8_t)~0x20u; // mouse clock enabled + Ps2WriteCommandByte(commandByte); + + Ps2WriteCommand(0xAEu); // enable keyboard port + Ps2WriteCommand(0xA8u); // enable aux port + Ps2FlushOutput(); +} + +static void Ps2ControllerEnableInterrupts(void) { + uint8_t commandByte = Ps2ReadCommandByte(); + commandByte |= 0x01u; + commandByte |= 0x02u; + Ps2WriteCommandByte(commandByte); +} + +static void Ps2KeyboardInitialize(void) { + // Enable scanning. If the keyboard is absent, the timeout/ACK failure is + // non-fatal; the IRQ path simply never produces events. + if (!Ps2SendKeyboard(0xF4u)) + DbgPrint("ps2: keyboard enable did not ACK\n"); +} + +static void Ps2MouseInitialize(void) { + if (!Ps2SendMouse(0xFFu)) { + DbgPrint("ps2: mouse reset did not ACK\n"); + return; + } + + (void)Ps2ReadData(); // BAT completion, usually 0xAA + (void)Ps2ReadData(); // device id, usually 0x00 + + if (!Ps2SendMouse(0xF6u)) + DbgPrint("ps2: mouse set-defaults did not ACK\n"); + if (!Ps2SendMouse(0xF4u)) + DbgPrint("ps2: mouse enable did not ACK\n"); +} + +KSTATUS DriverEntry(const HAL_API* hal) { + (void)hal; + + uint32_t keyboardId = 0, mouseId = 0; + KeInputRegisterDevice("ps2-keyboard", 0, &keyboardId); + KeInputRegisterDevice("ps2-mouse", 0, &mouseId); + + KeInitializeDpc(&gMouseDpc, Ps2MouseDpcRoutine, 0); + + // Keep 8042 IRQ generation disabled while doing polled controller/device + // setup. The old ordering routed IRQ1/IRQ12 first, which let ACK/BAT bytes + // race the ISR and get consumed as fake key/mouse input before the polled + // init path could read them. + Ps2ControllerPrepare(); + Ps2KeyboardInitialize(); + Ps2MouseInitialize(); + + KeIrqRegister(PS2_IRQ_KEYBOARD, Ps2KeyboardIsr); + KeIrqRegister(PS2_IRQ_MOUSE, Ps2MouseIsr); + Ps2ControllerEnableInterrupts(); + + DbgPrint("ps2: initialized keyboard=%u mouse=%u\n", keyboardId, mouseId); + return STATUS_SUCCESS; +} diff --git a/code/drivers/input/ps2/ps2.h b/code/drivers/input/ps2/ps2.h new file mode 100644 index 0000000..ec52d41 --- /dev/null +++ b/code/drivers/input/ps2/ps2.h @@ -0,0 +1,48 @@ +// ps2.h - shared i8042/PS/2 driver internals. +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define PS2_DATA 0x60u +#define PS2_STAT 0x64u +#define PS2_CMD 0x64u + +#define PS2_STAT_OUT_FULL 0x01u +#define PS2_STAT_IN_FULL 0x02u + +#define PS2_IRQ_KEYBOARD 1u +#define PS2_IRQ_MOUSE 12u + +#define SC1_BREAK_BIT 0x80u +#define SC1_MAKE_MASK 0x7Fu +#define SC1_EXT_PREFIX 0xE0u + +#define MOUSE_DPC_QUEUE_SIZE 64u +#define MOUSE_DPC_QUEUE_MASK (MOUSE_DPC_QUEUE_SIZE - 1u) + +typedef struct PS2_MOUSE_PACKET { + int dx; + int dy; + uint8_t buttons; +} PS2_MOUSE_PACKET; + +void Ps2Pause(void); +void Ps2WaitInput(void); +bool Ps2WaitOutput(void); +void Ps2FlushOutput(void); +uint8_t Ps2ReadData(void); +void Ps2WriteCommand(uint8_t command); +void Ps2WriteData(uint8_t data); +uint8_t Ps2ReadCommandByte(void); +void Ps2WriteCommandByte(uint8_t commandByte); +bool Ps2SendKeyboard(uint8_t byte); +bool Ps2SendMouse(uint8_t byte); diff --git a/code/drivers/input/ps2/ps2_i8042.c b/code/drivers/input/ps2/ps2_i8042.c new file mode 100644 index 0000000..fb0ff0c --- /dev/null +++ b/code/drivers/input/ps2/ps2_i8042.c @@ -0,0 +1,64 @@ +// ps2_i8042.c - 8042 controller and device command helpers. +#include "ps2.h" + +void Ps2Pause(void) { + _mm_pause(); +} + +void Ps2WaitInput(void) { + int timeout = 100000; + while ((__inbyte(PS2_STAT) & PS2_STAT_IN_FULL) && --timeout) { + Ps2Pause(); + } +} + +bool Ps2WaitOutput(void) { + int timeout = 5000000; + while (!(__inbyte(PS2_STAT) & PS2_STAT_OUT_FULL) && --timeout) { + Ps2Pause(); + } + return timeout != 0; +} + +void Ps2FlushOutput(void) { + int timeout = 100000; + while ((__inbyte(PS2_STAT) & PS2_STAT_OUT_FULL) && --timeout) { + (void)__inbyte(PS2_DATA); + } +} + +uint8_t Ps2ReadData(void) { + if (!Ps2WaitOutput()) return 0xFFu; + return __inbyte(PS2_DATA); +} + +void Ps2WriteCommand(uint8_t command) { + Ps2WaitInput(); + __outbyte(PS2_CMD, command); +} + +void Ps2WriteData(uint8_t data) { + Ps2WaitInput(); + __outbyte(PS2_DATA, data); +} + +uint8_t Ps2ReadCommandByte(void) { + Ps2WriteCommand(0x20u); + return Ps2ReadData(); +} + +void Ps2WriteCommandByte(uint8_t commandByte) { + Ps2WriteCommand(0x60u); + Ps2WriteData(commandByte); +} + +bool Ps2SendKeyboard(uint8_t byte) { + Ps2WriteData(byte); + return Ps2ReadData() == 0xFAu; +} + +bool Ps2SendMouse(uint8_t byte) { + Ps2WriteCommand(0xD4u); + Ps2WriteData(byte); + return Ps2ReadData() == 0xFAu; +} diff --git a/code/drivers/net/lwip/lwip_sys_arch.c b/code/drivers/net/lwip/lwip_sys_arch.c index a77d8cf..7c6aa8e 100644 --- a/code/drivers/net/lwip/lwip_sys_arch.c +++ b/code/drivers/net/lwip/lwip_sys_arch.c @@ -4,7 +4,8 @@ #include #include #include - +#pragma intrinsic(_disable) +#pragma intrinsic(_enable) #include #include #include diff --git a/code/drivers/net/rtl8139/rtl8139.c b/code/drivers/net/rtl8139/rtl8139.c index 5ee7562..47da4e2 100644 --- a/code/drivers/net/rtl8139/rtl8139.c +++ b/code/drivers/net/rtl8139/rtl8139.c @@ -1,150 +1,9 @@ -// rtl8139.c - Realtek RTL8139 / QEMU rtl8139 PCI fast Ethernet driver. -// -// Scope for this first version: -// * PCI ID 10ec:8139, legacy INTx interrupts. -// * MMIO preferred, I/O BAR fallback. -// * Classic RTL8139 register interface: one RX ring buffer and four TX slots. -// * No checksum offload, C+ mode, VLAN, multicast hash programming, or power mgmt. -// -// This is intentionally a separate mini-driver from rtl816x.sys. RTL8139 and -// RTL8111/8168 share a vendor but not a useful packet-DMA programming model. +// rtl8139.c - packet I/O, interrupt, and DriverEntry path. +#include "rtl8139.h" -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define RTL8139_VENDOR_REALTEK 0x10ECu -#define RTL8139_DEVICE_ID 0x8139u - -#define PCI_REG_COMMAND 0x04u -#define PCI_REG_BAR0 0x10u -#define PCI_COMMAND_IO (1u << 0) -#define PCI_COMMAND_MEMORY (1u << 1) -#define PCI_COMMAND_MASTER (1u << 2) -#define PCI_COMMAND_INTX_DISABLE (1u << 10) - -#define RTL8139_REG_IDR0 0x00u -#define RTL8139_REG_MAR0 0x08u -#define RTL8139_REG_TSD0 0x10u -#define RTL8139_REG_TSAD0 0x20u -#define RTL8139_REG_RBSTART 0x30u -#define RTL8139_REG_CMD 0x37u -#define RTL8139_REG_CAPR 0x38u -#define RTL8139_REG_CBR 0x3Au -#define RTL8139_REG_IMR 0x3Cu -#define RTL8139_REG_ISR 0x3Eu -#define RTL8139_REG_TCR 0x40u -#define RTL8139_REG_RCR 0x44u -#define RTL8139_REG_CONFIG1 0x52u - -#define RTL8139_CMD_BUFE 0x01u -#define RTL8139_CMD_TE 0x04u -#define RTL8139_CMD_RE 0x08u -#define RTL8139_CMD_RST 0x10u - -#define RTL8139_ISR_ROK 0x0001u -#define RTL8139_ISR_RER 0x0002u -#define RTL8139_ISR_TOK 0x0004u -#define RTL8139_ISR_TER 0x0008u -#define RTL8139_ISR_RXOVW 0x0010u -#define RTL8139_ISR_PUN 0x0020u -#define RTL8139_ISR_FOVW 0x0040u -#define RTL8139_ISR_LENCHG 0x2000u -#define RTL8139_ISR_TIMEOUT 0x4000u -#define RTL8139_ISR_SYSERR 0x8000u - -#define RTL8139_INTR_ENABLES (RTL8139_ISR_ROK | RTL8139_ISR_RER | RTL8139_ISR_TOK | \ - RTL8139_ISR_TER | RTL8139_ISR_RXOVW | RTL8139_ISR_PUN | \ - RTL8139_ISR_FOVW | RTL8139_ISR_LENCHG | RTL8139_ISR_SYSERR) - -#define RTL8139_RCR_AAP (1u << 0) -#define RTL8139_RCR_APM (1u << 1) -#define RTL8139_RCR_AM (1u << 2) -#define RTL8139_RCR_AB (1u << 3) -#define RTL8139_RCR_AR (1u << 4) -#define RTL8139_RCR_AER (1u << 5) -#define RTL8139_RCR_WRAP (1u << 7) -#define RTL8139_RCR_MXDMA_UNLIMITED (7u << 8) -#define RTL8139_RCR_RBLEN_8K (0u << 11) - -#define RTL8139_TCR_MXDMA_UNLIMITED (7u << 8) -#define RTL8139_TCR_IFG_NORMAL (3u << 24) - -#define RTL8139_TSD_SIZE_MASK 0x00001FFFu -#define RTL8139_TSD_OWN (1u << 13) -#define RTL8139_TSD_TUN (1u << 14) -#define RTL8139_TSD_TOK (1u << 15) -#define RTL8139_TSD_TABT (1u << 30) - -#define RTL8139_RX_OK 0x0001u -#define RTL8139_RX_BAD_BITS 0x003Eu // FAE | CRC | LONG | RUNT | ISE; MAR/PAM/BAR are valid match flags. - -#define RTL8139_TX_SLOTS 4u -#define RTL8139_TX_BUF_SIZE 2048u -#define RTL8139_RX_RING_SIZE 8192u -#define RTL8139_RX_PAD 16u -#define RTL8139_RX_GUARD 1500u -#define RTL8139_RX_ALLOC_SIZE (RTL8139_RX_RING_SIZE + RTL8139_RX_PAD + RTL8139_RX_GUARD) -#define RTL8139_RX_PAGES ((RTL8139_RX_ALLOC_SIZE + 4095u) / 4096u) -#define RTL8139_TX_PAGES (((RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE) + 4095u) / 4096u) -#define RTL8139_MMIO_PAGES 1u -#define RTL8139_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u) - -#define RTL8139_REG_TSD(slot) (RTL8139_REG_TSD0 + ((slot) * 4u)) -#define RTL8139_REG_TSAD(slot) (RTL8139_REG_TSAD0 + ((slot) * 4u)) - -typedef enum RTL8139_BAR_KIND { - Rtl8139BarNone = 0, - Rtl8139BarIo, - Rtl8139BarMmio, -} RTL8139_BAR_KIND; - -typedef struct RTL8139_BAR { - RTL8139_BAR_KIND Kind; - uint64_t Base; - uint8_t Index; -} RTL8139_BAR; - -typedef struct RTL8139_DEVICE { - uint8_t Bus; - uint8_t Dev; - uint8_t Fn; - uint8_t Irq; - - bool UseMmio; - volatile uint8_t* Mmio; - uint16_t IoBase; - - PHYSICAL_ADDRESS RxPhys; - PHYSICAL_ADDRESS TxPhys; - uint8_t* RxBuffer; - uint8_t* TxBuffers; - uint8_t RxScratch[RTL8139_TX_BUF_SIZE]; - - uint32_t RxOffset; - uint32_t TxSlot; - SPINLOCK TxLock; - KDPC Dpc; - volatile uint16_t PendingIsr; - - NET_DEVICE NetDevice; -} RTL8139_DEVICE; - -static const HAL_API* gHal; -static RTL8139_DEVICE gDevice; -static bool gDevicePresent = false; +const HAL_API* gHal; +RTL8139_DEVICE gDevice; +bool gDevicePresent = false; static __forceinline void Rtl8139MemoryBarrier(void) { _ReadWriteBarrier(); @@ -185,112 +44,9 @@ static void Rtl8139Write32(RTL8139_DEVICE* r, uint32_t reg, uint32_t value) { else __outdword((unsigned short)(r->IoBase + reg), value); } -static bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out) { - uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u); - uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg); - out->Kind = Rtl8139BarNone; - out->Base = 0; - out->Index = index; - if (lo == 0 || lo == 0xFFFFFFFFu) - return false; - if (lo & 1u) { - uint32_t base = lo & ~0x3u; - if (!base) return false; - out->Kind = Rtl8139BarIo; - out->Base = base; - return true; - } - - uint64_t base = (uint64_t)(lo & ~0xFu); - uint32_t type = (lo >> 1) & 0x3u; - if (type == 0x2u && index < 5) { - uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u)); - base |= ((uint64_t)hi << 32); - } - - if (!base) return false; - out->Kind = Rtl8139BarMmio; - out->Base = base; - return true; -} - -static KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio) { - uint64_t pageBase = physBase & PAGE_MASK; - uint64_t pageOff = physBase - pageBase; - - for (uint32_t i = 0; i < RTL8139_MMIO_PAGES; i++) { - PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE; - uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys); - KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); - if (st == STATUS_ALREADY_EXISTS) - st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); - if (!KSUCCESS(st)) - return st; - } - - *outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff); - return STATUS_SUCCESS; -} - -static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) { - RTL8139_BAR best = { Rtl8139BarNone, 0, 0 }; - RTL8139_BAR fallback = { Rtl8139BarNone, 0, 0 }; - - for (uint8_t i = 0; i < 6; i++) { - RTL8139_BAR bar; - if (!Rtl8139ReadBar(r->Bus, r->Dev, r->Fn, i, &bar)) - continue; - - if (bar.Kind == Rtl8139BarMmio) { - best = bar; - break; - } - if (bar.Kind == Rtl8139BarIo && fallback.Kind == Rtl8139BarNone) - fallback = bar; - - if (bar.Kind == Rtl8139BarMmio) { - uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u)); - if (((lo >> 1) & 0x3u) == 0x2u) - i++; - } - } - - if (best.Kind == Rtl8139BarNone) - best = fallback; - - if (best.Kind == Rtl8139BarMmio) { - KSTATUS st = Rtl8139MapMmio(best.Base, &r->Mmio); - if (!KSUCCESS(st)) - return st; - r->UseMmio = true; - DbgPrint("RTL8139: BAR%u MMIO base=0x%llx virt=%p\n", - best.Index, (unsigned long long)best.Base, r->Mmio); - return STATUS_SUCCESS; - } - - if (best.Kind == Rtl8139BarIo) { - if (best.Base > 0xFFFFu) - return STATUS_NOT_SUPPORTED; - r->IoBase = (uint16_t)best.Base; - r->UseMmio = false; - DbgPrint("RTL8139: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase); - return STATUS_SUCCESS; - } - - return STATUS_NOT_FOUND; -} - -static void Rtl8139EnablePci(RTL8139_DEVICE* r) { - uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND); - cmd |= PCI_COMMAND_MASTER; - if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY; - else cmd |= PCI_COMMAND_IO; - cmd &= ~PCI_COMMAND_INTX_DISABLE; - gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd); -} static KSTATUS Rtl8139Reset(RTL8139_DEVICE* r) { Rtl8139Write8(r, RTL8139_REG_CMD, RTL8139_CMD_RST); @@ -328,7 +84,7 @@ static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) { memset(r->RxBuffer, 0, PAGE_SIZE * RTL8139_RX_PAGES); memset(r->TxBuffers, 0, PAGE_SIZE * RTL8139_TX_PAGES); - DbgPrint("RTL8139: rx=0x%llx tx=0x%llx\n", + DbgPrint("RTL8139: rx=0x%016llX tx=0x%016llX\n", (unsigned long long)r->RxPhys, (unsigned long long)r->TxPhys); return STATUS_SUCCESS; diff --git a/code/drivers/net/rtl8139/rtl8139.h b/code/drivers/net/rtl8139/rtl8139.h new file mode 100644 index 0000000..6192405 --- /dev/null +++ b/code/drivers/net/rtl8139/rtl8139.h @@ -0,0 +1,145 @@ +// rtl8139.h - shared rtl8139 driver internals. +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTL8139_VENDOR_REALTEK 0x10ECu +#define RTL8139_DEVICE_ID 0x8139u + +#define PCI_REG_COMMAND 0x04u +#define PCI_REG_BAR0 0x10u +#define PCI_COMMAND_IO (1u << 0) +#define PCI_COMMAND_MEMORY (1u << 1) +#define PCI_COMMAND_MASTER (1u << 2) +#define PCI_COMMAND_INTX_DISABLE (1u << 10) + +#define RTL8139_REG_IDR0 0x00u +#define RTL8139_REG_MAR0 0x08u +#define RTL8139_REG_TSD0 0x10u +#define RTL8139_REG_TSAD0 0x20u +#define RTL8139_REG_RBSTART 0x30u +#define RTL8139_REG_CMD 0x37u +#define RTL8139_REG_CAPR 0x38u +#define RTL8139_REG_CBR 0x3Au +#define RTL8139_REG_IMR 0x3Cu +#define RTL8139_REG_ISR 0x3Eu +#define RTL8139_REG_TCR 0x40u +#define RTL8139_REG_RCR 0x44u +#define RTL8139_REG_CONFIG1 0x52u + +#define RTL8139_CMD_BUFE 0x01u +#define RTL8139_CMD_TE 0x04u +#define RTL8139_CMD_RE 0x08u +#define RTL8139_CMD_RST 0x10u + +#define RTL8139_ISR_ROK 0x0001u +#define RTL8139_ISR_RER 0x0002u +#define RTL8139_ISR_TOK 0x0004u +#define RTL8139_ISR_TER 0x0008u +#define RTL8139_ISR_RXOVW 0x0010u +#define RTL8139_ISR_PUN 0x0020u +#define RTL8139_ISR_FOVW 0x0040u +#define RTL8139_ISR_LENCHG 0x2000u +#define RTL8139_ISR_TIMEOUT 0x4000u +#define RTL8139_ISR_SYSERR 0x8000u + +#define RTL8139_INTR_ENABLES (RTL8139_ISR_ROK | RTL8139_ISR_RER | RTL8139_ISR_TOK | \ + RTL8139_ISR_TER | RTL8139_ISR_RXOVW | RTL8139_ISR_PUN | \ + RTL8139_ISR_FOVW | RTL8139_ISR_LENCHG | RTL8139_ISR_SYSERR) + +#define RTL8139_RCR_AAP (1u << 0) +#define RTL8139_RCR_APM (1u << 1) +#define RTL8139_RCR_AM (1u << 2) +#define RTL8139_RCR_AB (1u << 3) +#define RTL8139_RCR_AR (1u << 4) +#define RTL8139_RCR_AER (1u << 5) +#define RTL8139_RCR_WRAP (1u << 7) +#define RTL8139_RCR_MXDMA_UNLIMITED (7u << 8) +#define RTL8139_RCR_RBLEN_8K (0u << 11) + +#define RTL8139_TCR_MXDMA_UNLIMITED (7u << 8) +#define RTL8139_TCR_IFG_NORMAL (3u << 24) + +#define RTL8139_TSD_SIZE_MASK 0x00001FFFu +#define RTL8139_TSD_OWN (1u << 13) +#define RTL8139_TSD_TUN (1u << 14) +#define RTL8139_TSD_TOK (1u << 15) +#define RTL8139_TSD_TABT (1u << 30) + +#define RTL8139_RX_OK 0x0001u +#define RTL8139_RX_BAD_BITS 0x003Eu // FAE | CRC | LONG | RUNT | ISE; MAR/PAM/BAR are valid match flags. + +#define RTL8139_TX_SLOTS 4u +#define RTL8139_TX_BUF_SIZE 2048u +#define RTL8139_RX_RING_SIZE 8192u +#define RTL8139_RX_PAD 16u +#define RTL8139_RX_GUARD 1500u +#define RTL8139_RX_ALLOC_SIZE (RTL8139_RX_RING_SIZE + RTL8139_RX_PAD + RTL8139_RX_GUARD) +#define RTL8139_RX_PAGES ((RTL8139_RX_ALLOC_SIZE + 4095u) / 4096u) +#define RTL8139_TX_PAGES (((RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE) + 4095u) / 4096u) +#define RTL8139_MMIO_PAGES 1u +#define RTL8139_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u) + +#define RTL8139_REG_TSD(slot) (RTL8139_REG_TSD0 + ((slot) * 4u)) +#define RTL8139_REG_TSAD(slot) (RTL8139_REG_TSAD0 + ((slot) * 4u)) + +typedef enum RTL8139_BAR_KIND { + Rtl8139BarNone = 0, + Rtl8139BarIo, + Rtl8139BarMmio, +} RTL8139_BAR_KIND; + +typedef struct RTL8139_BAR { + RTL8139_BAR_KIND Kind; + uint64_t Base; + uint8_t Index; +} RTL8139_BAR; + +typedef struct RTL8139_DEVICE { + uint8_t Bus; + uint8_t Dev; + uint8_t Fn; + uint8_t Irq; + + bool UseMmio; + volatile uint8_t* Mmio; + uint16_t IoBase; + + PHYSICAL_ADDRESS RxPhys; + PHYSICAL_ADDRESS TxPhys; + uint8_t* RxBuffer; + uint8_t* TxBuffers; + uint8_t RxScratch[RTL8139_TX_BUF_SIZE]; + + uint32_t RxOffset; + uint32_t TxSlot; + SPINLOCK TxLock; + KDPC Dpc; + volatile uint16_t PendingIsr; + + NET_DEVICE NetDevice; +} RTL8139_DEVICE; + + +extern const HAL_API* gHal; +extern RTL8139_DEVICE gDevice; +extern bool gDevicePresent; + +bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out); +KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio); +KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r); +void Rtl8139EnablePci(RTL8139_DEVICE* r); diff --git a/code/drivers/net/rtl8139/rtl8139_pci.c b/code/drivers/net/rtl8139/rtl8139_pci.c new file mode 100644 index 0000000..7179817 --- /dev/null +++ b/code/drivers/net/rtl8139/rtl8139_pci.c @@ -0,0 +1,112 @@ +// rtl8139_pci.c - PCI BAR/MMIO front-end for rtl8139. +#include "rtl8139.h" + +bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out) { + uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u); + uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg); + + out->Kind = Rtl8139BarNone; + out->Base = 0; + out->Index = index; + + if (lo == 0 || lo == 0xFFFFFFFFu) + return false; + + if (lo & 1u) { + uint32_t base = lo & ~0x3u; + if (!base) return false; + out->Kind = Rtl8139BarIo; + out->Base = base; + return true; + } + + uint64_t base = (uint64_t)(lo & ~0xFu); + uint32_t type = (lo >> 1) & 0x3u; + if (type == 0x2u && index < 5) { + uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u)); + base |= ((uint64_t)hi << 32); + } + + if (!base) return false; + out->Kind = Rtl8139BarMmio; + out->Base = base; + return true; +} + + +KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio) { + uint64_t pageBase = physBase & PAGE_MASK; + uint64_t pageOff = physBase - pageBase; + + for (uint32_t i = 0; i < RTL8139_MMIO_PAGES; i++) { + PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE; + uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys); + KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); + if (st == STATUS_ALREADY_EXISTS) + st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); + if (!KSUCCESS(st)) + return st; + } + + *outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff); + return STATUS_SUCCESS; +} + + +KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) { + RTL8139_BAR best = { Rtl8139BarNone, 0, 0 }; + RTL8139_BAR fallback = { Rtl8139BarNone, 0, 0 }; + + for (uint8_t i = 0; i < 6; i++) { + RTL8139_BAR bar; + if (!Rtl8139ReadBar(r->Bus, r->Dev, r->Fn, i, &bar)) + continue; + + if (bar.Kind == Rtl8139BarMmio) { + best = bar; + break; + } + if (bar.Kind == Rtl8139BarIo && fallback.Kind == Rtl8139BarNone) + fallback = bar; + + if (bar.Kind == Rtl8139BarMmio) { + uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u)); + if (((lo >> 1) & 0x3u) == 0x2u) + i++; + } + } + + if (best.Kind == Rtl8139BarNone) + best = fallback; + + if (best.Kind == Rtl8139BarMmio) { + KSTATUS st = Rtl8139MapMmio(best.Base, &r->Mmio); + if (!KSUCCESS(st)) + return st; + r->UseMmio = true; + DbgPrint("RTL8139: BAR%u MMIO base=0x%016llX virt=0x%016llX\n", + best.Index, (unsigned long long)best.Base, r->Mmio); + return STATUS_SUCCESS; + } + + if (best.Kind == Rtl8139BarIo) { + if (best.Base > 0xFFFFu) + return STATUS_NOT_SUPPORTED; + r->IoBase = (uint16_t)best.Base; + r->UseMmio = false; + DbgPrint("RTL8139: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase); + return STATUS_SUCCESS; + } + + return STATUS_NOT_FOUND; +} + + +void Rtl8139EnablePci(RTL8139_DEVICE* r) { + uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND); + cmd |= PCI_COMMAND_MASTER; + if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY; + else cmd |= PCI_COMMAND_IO; + cmd &= ~PCI_COMMAND_INTX_DISABLE; + gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd); +} diff --git a/code/drivers/net/rtl816x/rtl816x.c b/code/drivers/net/rtl816x/rtl816x.c index 02fb231..98eb07f 100644 --- a/code/drivers/net/rtl816x/rtl816x.c +++ b/code/drivers/net/rtl816x/rtl816x.c @@ -1,168 +1,9 @@ -// rtl816x.c - first-pass Realtek RTL8111/8168/8411 PCIe Gigabit NIC driver. -// -// Scope for this version: -// * RTL8168/RTL8169-compatible PCI IDs, legacy INTx interrupt delivery. -// * MMIO preferred, I/O BAR fallback. -// * Descriptor-mode RX/TX with plain Ethernet frames. -// * No checksum offload, TSO, VLAN, jumbo frames, MSI/MSI-X, or EEE. -// -// The driver registers a NET_DEVICE so a later ARP/IPv4/ICMP stack can attach -// without knowing Realtek register details. +// rtl816x.c - packet I/O, interrupt, and DriverEntry path. +#include "rtl816x.h" -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define RTL_VENDOR_REALTEK 0x10ECu -#define RTL_DEVICE_8168 0x8168u -#define RTL_DEVICE_8169 0x8169u - -#define PCI_REG_COMMAND 0x04u -#define PCI_REG_BAR0 0x10u -#define PCI_REG_IRQ 0x3Cu -#define PCI_COMMAND_IO (1u << 0) -#define PCI_COMMAND_MEMORY (1u << 1) -#define PCI_COMMAND_MASTER (1u << 2) -#define PCI_COMMAND_INTX_DISABLE (1u << 10) - -#define RTL_REG_MAC0 0x00u -#define RTL_REG_MAR0 0x08u -#define RTL_REG_TNPDS_LO 0x20u -#define RTL_REG_TNPDS_HI 0x24u -#define RTL_REG_THPDS_LO 0x28u -#define RTL_REG_THPDS_HI 0x2Cu -#define RTL_REG_CHIP_CMD 0x37u -#define RTL_REG_TX_POLL 0x38u -#define RTL_REG_INTR_MASK 0x3Cu -#define RTL_REG_INTR_STAT 0x3Eu -#define RTL_REG_TX_CONFIG 0x40u -#define RTL_REG_RX_CONFIG 0x44u -#define RTL_REG_CFG9346 0x50u -#define RTL_REG_CONFIG1 0x52u -#define RTL_REG_PHY_STATUS 0x6Cu -#define RTL_REG_RX_MAXSIZE 0xDAu -#define RTL_REG_CPLUS_CMD 0xE0u -#define RTL_REG_RDSAR_LO 0xE4u -#define RTL_REG_RDSAR_HI 0xE8u - -#define RTL_CMD_RESET 0x10u -#define RTL_CMD_RX_ENABLE 0x08u -#define RTL_CMD_TX_ENABLE 0x04u - -#define RTL_TXPOLL_NPQ 0x40u - -#define RTL_ISR_ROK 0x0001u -#define RTL_ISR_RER 0x0002u -#define RTL_ISR_TOK 0x0004u -#define RTL_ISR_TER 0x0008u -#define RTL_ISR_RXOVW 0x0010u -#define RTL_ISR_LINKCHG 0x0020u -#define RTL_ISR_FOVW 0x0040u -#define RTL_ISR_TDU 0x0080u -#define RTL_ISR_SWINT 0x0100u -#define RTL_ISR_TIMEOUT 0x4000u -#define RTL_ISR_SYSERR 0x8000u - -#define RTL_INTR_ENABLES (RTL_ISR_ROK | RTL_ISR_RER | RTL_ISR_TOK | RTL_ISR_TER | \ - RTL_ISR_RXOVW | RTL_ISR_LINKCHG | RTL_ISR_FOVW | \ - RTL_ISR_TDU | RTL_ISR_SYSERR) - -#define RTL_RXCFG_AAP (1u << 0) // accept all physical packets/promiscuous -#define RTL_RXCFG_APM (1u << 1) // accept physical match -#define RTL_RXCFG_AM (1u << 2) // accept multicast -#define RTL_RXCFG_AB (1u << 3) // accept broadcast -#define RTL_RXCFG_AR (1u << 4) // accept runt -#define RTL_RXCFG_AER (1u << 5) // accept errored -#define RTL_RXCFG_DMA_UNLIMITED (7u << 8) - -#define RTL_TXCFG_DMA_UNLIMITED (7u << 8) -#define RTL_TXCFG_IFG (3u << 24) - -#define RTL_DESC_OWN (1u << 31) -#define RTL_DESC_EOR (1u << 30) -#define RTL_DESC_FS (1u << 29) -#define RTL_DESC_LS (1u << 28) -#define RTL_RXDESC_RES (1u << 21) -#define RTL_RXDESC_RWT (1u << 22) -#define RTL_RXDESC_CRC (1u << 19) -#define RTL_RXDESC_RUNT (1u << 20) -#define RTL_DESC_LENGTH_MASK 0x00003FFFu - -#define RTL_RX_COUNT 64u -#define RTL_TX_COUNT 64u -#define RTL_BUF_SIZE 2048u -#define RTL_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u) -#define RTL_RING_PAGES 1u -#define RTL_RXBUF_PAGES ((RTL_RX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u) -#define RTL_TXBUF_PAGES ((RTL_TX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u) -#define RTL_MMIO_PAGES 1u - -#pragma pack(push, 1) -typedef struct RTL_DESC { - volatile uint32_t opts1; - volatile uint32_t opts2; - volatile uint32_t addrLo; - volatile uint32_t addrHi; -} RTL_DESC; -#pragma pack(pop) - -typedef enum RTL_BAR_KIND { - RtlBarNone = 0, - RtlBarIo, - RtlBarMmio, -} RTL_BAR_KIND; - -typedef struct RTL_BAR { - RTL_BAR_KIND Kind; - uint64_t Base; - uint8_t Index; -} RTL_BAR; - -typedef struct RTL816X_DEVICE { - uint8_t Bus; - uint8_t Dev; - uint8_t Fn; - uint8_t Irq; - uint16_t DeviceId; - - bool UseMmio; - volatile uint8_t* Mmio; - uint16_t IoBase; - - RTL_DESC* RxRing; - RTL_DESC* TxRing; - PHYSICAL_ADDRESS RxRingPhys; - PHYSICAL_ADDRESS TxRingPhys; - - uint8_t* RxBuffers; - uint8_t* TxBuffers; - PHYSICAL_ADDRESS RxBuffersPhys; - PHYSICAL_ADDRESS TxBuffersPhys; - - uint32_t RxIndex; - uint32_t TxIndex; - SPINLOCK TxLock; - KDPC Dpc; - volatile uint16_t PendingIsr; - - NET_DEVICE NetDevice; -} RTL816X_DEVICE; - -static const HAL_API* gHal; -static RTL816X_DEVICE gDevice; -static bool gDevicePresent = false; +const HAL_API* gHal; +RTL816X_DEVICE gDevice; +bool gDevicePresent = false; static __forceinline void RtlMemoryBarrier(void) { _ReadWriteBarrier(); @@ -203,125 +44,10 @@ static void RtlWrite32(RTL816X_DEVICE* r, uint32_t reg, uint32_t value) { else __outdword((unsigned short)(r->IoBase + reg), value); } -static bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId) { - if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8168, outBus, outDev, outFn)) { - *outDeviceId = RTL_DEVICE_8168; - return true; - } - if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8169, outBus, outDev, outFn)) { - *outDeviceId = RTL_DEVICE_8169; - return true; - } - return false; -} -static bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out) { - uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u); - uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg); - out->Kind = RtlBarNone; - out->Base = 0; - out->Index = index; - if (lo == 0 || lo == 0xFFFFFFFFu) - return false; - if (lo & 1u) { - uint32_t base = lo & ~0x3u; - if (!base) return false; - out->Kind = RtlBarIo; - out->Base = base; - return true; - } - - uint64_t base = (uint64_t)(lo & ~0xFu); - uint32_t type = (lo >> 1) & 0x3u; - if (type == 0x2u && index < 5) { - uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u)); - base |= ((uint64_t)hi << 32); - } - - if (!base) return false; - out->Kind = RtlBarMmio; - out->Base = base; - return true; -} - -static KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio) { - uint64_t pageBase = physBase & PAGE_MASK; - uint64_t pageOff = physBase - pageBase; - - for (uint32_t i = 0; i < RTL_MMIO_PAGES; i++) { - PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE; - uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys); - KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); - if (st == STATUS_ALREADY_EXISTS) - st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); - if (!KSUCCESS(st)) - return st; - } - - *outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff); - return STATUS_SUCCESS; -} - -static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) { - RTL_BAR best = { RtlBarNone, 0, 0 }; - RTL_BAR fallback = { RtlBarNone, 0, 0 }; - - for (uint8_t i = 0; i < 6; i++) { - RTL_BAR bar; - if (!RtlReadBar(r->Bus, r->Dev, r->Fn, i, &bar)) - continue; - - if (bar.Kind == RtlBarMmio) { - best = bar; - break; - } - if (bar.Kind == RtlBarIo && fallback.Kind == RtlBarNone) - fallback = bar; - - // Skip the high dword of 64-bit memory BARs. - if (bar.Kind == RtlBarMmio) { - uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u)); - if (((lo >> 1) & 0x3u) == 0x2u) - i++; - } - } - - if (best.Kind == RtlBarNone) - best = fallback; - - if (best.Kind == RtlBarMmio) { - KSTATUS st = RtlMapMmio(best.Base, &r->Mmio); - if (!KSUCCESS(st)) - return st; - r->UseMmio = true; - DbgPrint("RTL816X: BAR%u MMIO base=0x%llx virt=%p\n", - best.Index, (unsigned long long)best.Base, r->Mmio); - return STATUS_SUCCESS; - } - - if (best.Kind == RtlBarIo) { - if (best.Base > 0xFFFFu) - return STATUS_NOT_SUPPORTED; - r->IoBase = (uint16_t)best.Base; - r->UseMmio = false; - DbgPrint("RTL816X: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase); - return STATUS_SUCCESS; - } - - return STATUS_NOT_FOUND; -} - -static void RtlEnablePci(RTL816X_DEVICE* r) { - uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND); - cmd |= PCI_COMMAND_MASTER; - if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY; - else cmd |= PCI_COMMAND_IO; - cmd &= ~PCI_COMMAND_INTX_DISABLE; - gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd); -} static KSTATUS RtlReset(RTL816X_DEVICE* r) { RtlWrite8(r, RTL_REG_CHIP_CMD, RTL_CMD_RESET); @@ -359,7 +85,7 @@ static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) { memset(r->RxBuffers, 0, PAGE_SIZE * RTL_RXBUF_PAGES); memset(r->TxBuffers, 0, PAGE_SIZE * RTL_TXBUF_PAGES); - DbgPrint("RTL816X: rxRing=0x%llx txRing=0x%llx rxBuf=0x%llx txBuf=0x%llx\n", + DbgPrint("RTL816X: rxRing=0x%016llX txRing=0x%016llX rxBuf=0x%016llX txBuf=0x%016llX\n", (unsigned long long)r->RxRingPhys, (unsigned long long)r->TxRingPhys, (unsigned long long)r->RxBuffersPhys, diff --git a/code/drivers/net/rtl816x/rtl816x.h b/code/drivers/net/rtl816x/rtl816x.h new file mode 100644 index 0000000..780ff1c --- /dev/null +++ b/code/drivers/net/rtl816x/rtl816x.h @@ -0,0 +1,164 @@ +// rtl816x.h - shared rtl816x driver internals. +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTL_VENDOR_REALTEK 0x10ECu +#define RTL_DEVICE_8168 0x8168u +#define RTL_DEVICE_8169 0x8169u + +#define PCI_REG_COMMAND 0x04u +#define PCI_REG_BAR0 0x10u +#define PCI_REG_IRQ 0x3Cu +#define PCI_COMMAND_IO (1u << 0) +#define PCI_COMMAND_MEMORY (1u << 1) +#define PCI_COMMAND_MASTER (1u << 2) +#define PCI_COMMAND_INTX_DISABLE (1u << 10) + +#define RTL_REG_MAC0 0x00u +#define RTL_REG_MAR0 0x08u +#define RTL_REG_TNPDS_LO 0x20u +#define RTL_REG_TNPDS_HI 0x24u +#define RTL_REG_THPDS_LO 0x28u +#define RTL_REG_THPDS_HI 0x2Cu +#define RTL_REG_CHIP_CMD 0x37u +#define RTL_REG_TX_POLL 0x38u +#define RTL_REG_INTR_MASK 0x3Cu +#define RTL_REG_INTR_STAT 0x3Eu +#define RTL_REG_TX_CONFIG 0x40u +#define RTL_REG_RX_CONFIG 0x44u +#define RTL_REG_CFG9346 0x50u +#define RTL_REG_CONFIG1 0x52u +#define RTL_REG_PHY_STATUS 0x6Cu +#define RTL_REG_RX_MAXSIZE 0xDAu +#define RTL_REG_CPLUS_CMD 0xE0u +#define RTL_REG_RDSAR_LO 0xE4u +#define RTL_REG_RDSAR_HI 0xE8u + +#define RTL_CMD_RESET 0x10u +#define RTL_CMD_RX_ENABLE 0x08u +#define RTL_CMD_TX_ENABLE 0x04u + +#define RTL_TXPOLL_NPQ 0x40u + +#define RTL_ISR_ROK 0x0001u +#define RTL_ISR_RER 0x0002u +#define RTL_ISR_TOK 0x0004u +#define RTL_ISR_TER 0x0008u +#define RTL_ISR_RXOVW 0x0010u +#define RTL_ISR_LINKCHG 0x0020u +#define RTL_ISR_FOVW 0x0040u +#define RTL_ISR_TDU 0x0080u +#define RTL_ISR_SWINT 0x0100u +#define RTL_ISR_TIMEOUT 0x4000u +#define RTL_ISR_SYSERR 0x8000u + +#define RTL_INTR_ENABLES (RTL_ISR_ROK | RTL_ISR_RER | RTL_ISR_TOK | RTL_ISR_TER | \ + RTL_ISR_RXOVW | RTL_ISR_LINKCHG | RTL_ISR_FOVW | \ + RTL_ISR_TDU | RTL_ISR_SYSERR) + +#define RTL_RXCFG_AAP (1u << 0) // accept all physical packets/promiscuous +#define RTL_RXCFG_APM (1u << 1) // accept physical match +#define RTL_RXCFG_AM (1u << 2) // accept multicast +#define RTL_RXCFG_AB (1u << 3) // accept broadcast +#define RTL_RXCFG_AR (1u << 4) // accept runt +#define RTL_RXCFG_AER (1u << 5) // accept errored +#define RTL_RXCFG_DMA_UNLIMITED (7u << 8) + +#define RTL_TXCFG_DMA_UNLIMITED (7u << 8) +#define RTL_TXCFG_IFG (3u << 24) + +#define RTL_DESC_OWN (1u << 31) +#define RTL_DESC_EOR (1u << 30) +#define RTL_DESC_FS (1u << 29) +#define RTL_DESC_LS (1u << 28) +#define RTL_RXDESC_RES (1u << 21) +#define RTL_RXDESC_RWT (1u << 22) +#define RTL_RXDESC_CRC (1u << 19) +#define RTL_RXDESC_RUNT (1u << 20) +#define RTL_DESC_LENGTH_MASK 0x00003FFFu + +#define RTL_RX_COUNT 64u +#define RTL_TX_COUNT 64u +#define RTL_BUF_SIZE 2048u +#define RTL_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u) +#define RTL_RING_PAGES 1u +#define RTL_RXBUF_PAGES ((RTL_RX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u) +#define RTL_TXBUF_PAGES ((RTL_TX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u) +#define RTL_MMIO_PAGES 1u + +#pragma pack(push, 1) +typedef struct RTL_DESC { + volatile uint32_t opts1; + volatile uint32_t opts2; + volatile uint32_t addrLo; + volatile uint32_t addrHi; +} RTL_DESC; +#pragma pack(pop) + +typedef enum RTL_BAR_KIND { + RtlBarNone = 0, + RtlBarIo, + RtlBarMmio, +} RTL_BAR_KIND; + +typedef struct RTL_BAR { + RTL_BAR_KIND Kind; + uint64_t Base; + uint8_t Index; +} RTL_BAR; + +typedef struct RTL816X_DEVICE { + uint8_t Bus; + uint8_t Dev; + uint8_t Fn; + uint8_t Irq; + uint16_t DeviceId; + + bool UseMmio; + volatile uint8_t* Mmio; + uint16_t IoBase; + + RTL_DESC* RxRing; + RTL_DESC* TxRing; + PHYSICAL_ADDRESS RxRingPhys; + PHYSICAL_ADDRESS TxRingPhys; + + uint8_t* RxBuffers; + uint8_t* TxBuffers; + PHYSICAL_ADDRESS RxBuffersPhys; + PHYSICAL_ADDRESS TxBuffersPhys; + + uint32_t RxIndex; + uint32_t TxIndex; + SPINLOCK TxLock; + KDPC Dpc; + volatile uint16_t PendingIsr; + + NET_DEVICE NetDevice; +} RTL816X_DEVICE; + + +extern const HAL_API* gHal; +extern RTL816X_DEVICE gDevice; +extern bool gDevicePresent; + +bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId); +bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out); +KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio); +KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r); +void RtlEnablePci(RTL816X_DEVICE* r); diff --git a/code/drivers/net/rtl816x/rtl816x_pci.c b/code/drivers/net/rtl816x/rtl816x_pci.c new file mode 100644 index 0000000..58f89a2 --- /dev/null +++ b/code/drivers/net/rtl816x/rtl816x_pci.c @@ -0,0 +1,126 @@ +// rtl816x_pci.c - PCI BAR/MMIO front-end for rtl816x. +#include "rtl816x.h" + +bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId) { + if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8168, outBus, outDev, outFn)) { + *outDeviceId = RTL_DEVICE_8168; + return true; + } + if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8169, outBus, outDev, outFn)) { + *outDeviceId = RTL_DEVICE_8169; + return true; + } + return false; +} + + +bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out) { + uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u); + uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg); + + out->Kind = RtlBarNone; + out->Base = 0; + out->Index = index; + + if (lo == 0 || lo == 0xFFFFFFFFu) + return false; + + if (lo & 1u) { + uint32_t base = lo & ~0x3u; + if (!base) return false; + out->Kind = RtlBarIo; + out->Base = base; + return true; + } + + uint64_t base = (uint64_t)(lo & ~0xFu); + uint32_t type = (lo >> 1) & 0x3u; + if (type == 0x2u && index < 5) { + uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u)); + base |= ((uint64_t)hi << 32); + } + + if (!base) return false; + out->Kind = RtlBarMmio; + out->Base = base; + return true; +} + + +KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio) { + uint64_t pageBase = physBase & PAGE_MASK; + uint64_t pageOff = physBase - pageBase; + + for (uint32_t i = 0; i < RTL_MMIO_PAGES; i++) { + PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE; + uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys); + KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); + if (st == STATUS_ALREADY_EXISTS) + st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE); + if (!KSUCCESS(st)) + return st; + } + + *outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff); + return STATUS_SUCCESS; +} + + +KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) { + RTL_BAR best = { RtlBarNone, 0, 0 }; + RTL_BAR fallback = { RtlBarNone, 0, 0 }; + + for (uint8_t i = 0; i < 6; i++) { + RTL_BAR bar; + if (!RtlReadBar(r->Bus, r->Dev, r->Fn, i, &bar)) + continue; + + if (bar.Kind == RtlBarMmio) { + best = bar; + break; + } + if (bar.Kind == RtlBarIo && fallback.Kind == RtlBarNone) + fallback = bar; + + // Skip the high dword of 64-bit memory BARs. + if (bar.Kind == RtlBarMmio) { + uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u)); + if (((lo >> 1) & 0x3u) == 0x2u) + i++; + } + } + + if (best.Kind == RtlBarNone) + best = fallback; + + if (best.Kind == RtlBarMmio) { + KSTATUS st = RtlMapMmio(best.Base, &r->Mmio); + if (!KSUCCESS(st)) + return st; + r->UseMmio = true; + DbgPrint("RTL816X: BAR%u MMIO base=0x%016llX virt=0x%016llX\n", + best.Index, (unsigned long long)best.Base, r->Mmio); + return STATUS_SUCCESS; + } + + if (best.Kind == RtlBarIo) { + if (best.Base > 0xFFFFu) + return STATUS_NOT_SUPPORTED; + r->IoBase = (uint16_t)best.Base; + r->UseMmio = false; + DbgPrint("RTL816X: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase); + return STATUS_SUCCESS; + } + + return STATUS_NOT_FOUND; +} + + +void RtlEnablePci(RTL816X_DEVICE* r) { + uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND); + cmd |= PCI_COMMAND_MASTER; + if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY; + else cmd |= PCI_COMMAND_IO; + cmd &= ~PCI_COMMAND_INTX_DISABLE; + gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd); +} diff --git a/code/drivers/storage/ahci/ahci.c b/code/drivers/storage/ahci/ahci.c index 98d28ad..940509a 100644 --- a/code/drivers/storage/ahci/ahci.c +++ b/code/drivers/storage/ahci/ahci.c @@ -1,436 +1,18 @@ -// ahci.c - AHCI driver, interrupt-driven with per-drive async IRP queue. -// Supports SATA drives only (no ATAPI). Registers each found drive with the -// kernel disk device registry via IoRegisterDiskDevice(). -// -// Memory model: -// MmAllocateContiguous(n) returns a physical address for n*4K pages. -// hal->PhysToVirt(phys) converts to kernel virtual address. -// DMA buffers are bounce-buffered (allocated via MmAllocateContiguous) to -// avoid needing a VirtToPhys conversion for arbitrary kernel pointers. -// -// Async I/O model (IRP path): -// Each drive has a dedicated kernel worker thread (AhciWorker) that blocks -// on a per-drive counting semaphore. Callers submit IRPs through IoCallDriver, -// which dispatches to AHCI and enqueues the IRP. The worker dequeues -// one IRP at a time, runs the synchronous hardware transfer (blocking the -// worker, NOT the submitting thread), then writes irp->Status and fires -// irp->CompletionEvent / irp->CompletionFn. -// -// This gives true async I/O from the caller's perspective: IoSubmitIrp() -// returns STATUS_PENDING immediately. Multiple callers can have outstanding -// requests against different drives concurrently without stalling each other. -// -// NCQ extension (future work - see "NCQ TODO" comments): -// To saturate modern SSDs, replace the serialised single-slot model with -// Native Command Queuing: -// • Allocate per-slot bounce buffers (NUM_NCQ_SLOTS × SLOT_BOUNCE_BYTES). -// • Use ATA_CMD_READ_FPDMA_QUEUED / WRITE_FPDMA_QUEUED. -// • Encode the slot tag in FIS.countl bits 7:3. -// • Issue via port->sact (NCQ active) rather than port->ci. -// • In AhciIsr: completed slots = prev_sact ^ port->sact; look up -// slot_irp[slot] and fire each completion from within the ISR. -// • The worker becomes a pure dispatcher: grab a free slot, build the -// command, issue it, return immediately. No per-slot KeWaitForEvent. -// -// Interrupt model (unchanged): -// GHC.IE and per-port PORT_IE are enabled after each port is initialised. -// AhciIsr fires on the PCI IRQ line, captures/clears port status, and queues -// a per-drive DPC. The DPC signals completionEvent outside hard IRQ context. -// port_issue_cmd blocks the worker thread on KeWaitForEvent instead of -// spinning on CI. port_issue_cmd_poll is kept for early init (IDENTIFY) -// before the scheduler is live. +// ahci.c - AHCI disk command path and DriverEntry. +#include "ahci.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// --------------------------------------------------------------------------- -typedef enum { - FIS_TYPE_REG_H2D = 0x27, - FIS_TYPE_REG_D2H = 0x34, - FIS_TYPE_DMA_ACT = 0x39, - FIS_TYPE_DMA_SETUP = 0x41, - FIS_TYPE_DATA = 0x46, - FIS_TYPE_BIST = 0x58, - FIS_TYPE_PIO_SETUP = 0x5F, - FIS_TYPE_DEV_BITS = 0xA1, -} FIS_TYPE; - -#pragma pack(push, 1) -// Register FIS - Host to Device (H2D) -typedef struct { - uint8_t fis_type; // FIS_TYPE_REG_H2D - uint8_t pmport : 4; - uint8_t rsv0 : 3; - uint8_t c : 1; // 1 = Command, 0 = Control - uint8_t command; - uint8_t featurel; - // DW1 - uint8_t lba0, lba1, lba2; - uint8_t device; - // DW2 - uint8_t lba3, lba4, lba5; - uint8_t featureh; - // DW3 - uint8_t countl, counth; - uint8_t icc; - uint8_t control; - // DW4 - uint8_t rsv1[4]; -} FIS_REG_H2D; - -// Register FIS - Device to Host (D2H) -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 2; - uint8_t i : 1; - uint8_t rsv1 : 1; - uint8_t status; - uint8_t error; - uint8_t lba0, lba1, lba2; - uint8_t device; - uint8_t lba3, lba4, lba5; - uint8_t rsv2; - uint8_t countl, counth; - uint8_t rsv3[2]; - uint8_t rsv4[4]; -} FIS_REG_D2H; - -// DMA Setup FIS -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 1; - uint8_t d : 1; - uint8_t i : 1; - uint8_t a : 1; - uint8_t rsved[2]; - uint64_t DMAbufferID; - uint32_t rsvd; - uint32_t DMAbufOffset; - uint32_t TransferCount; - uint32_t resvd; -} FIS_DMA_SETUP; - -// PIO Setup FIS -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 1; - uint8_t d : 1; - uint8_t i : 1; - uint8_t rsv1 : 1; - uint8_t status; - uint8_t error; - uint8_t lba0, lba1, lba2; - uint8_t device; - uint8_t lba3, lba4, lba5; - uint8_t rsv2; - uint8_t countl, counth; - uint8_t rsv3; - uint8_t e_status; - uint16_t tc; - uint8_t rsv4[2]; -} FIS_PIO_SETUP; - -// Set Device Bits FIS -typedef struct { - uint8_t fis_type; - uint8_t pmport : 4; - uint8_t rsv0 : 2; - uint8_t i : 1; - uint8_t n : 1; - uint8_t statusl : 4; - uint8_t rsv1 : 4; - uint8_t statush : 4; - uint8_t rsv2 : 4; - uint8_t error; - uint32_t protocol; -} FIS_DEV_BITS; - -// --------------------------------------------------------------------------- -// HBA structures (volatile - MMIO) -// --------------------------------------------------------------------------- - -// Received FIS structure in host memory -typedef volatile struct { - FIS_DMA_SETUP dsfis; // 0x00 DMA Setup FIS - uint8_t pad0[4]; - FIS_PIO_SETUP psfis; // 0x20 PIO Setup FIS - uint8_t pad1[12]; - FIS_REG_D2H rfis; // 0x40 Register D2H FIS - uint8_t pad2[4]; - FIS_DEV_BITS sdbfis; // 0x58 Set Device Bits FIS - uint8_t ufis[64]; // 0x60 Unknown FIS - uint8_t rsv[0x100-0xA0]; -} HBA_FIS; - -// Command List header (one entry = one slot) -typedef struct { - // DW0 - uint8_t cfl : 5; // Command FIS length in DWORDs (2..16) - uint8_t a : 1; // ATAPI - uint8_t w : 1; // Write (H2D=1, D2H=0) - uint8_t p : 1; // Prefetchable - uint8_t r : 1; // Reset - uint8_t b : 1; // BIST - uint8_t c : 1; // Clear busy on R_OK - uint8_t rsv0 : 1; - uint8_t pmp : 4; // Port multiplier port - uint16_t prdtl; // PRDT length in entries - // DW1 - volatile uint32_t prdbc; // PRDT byte count transferred - // DW2,3 - uint32_t ctba; // Command table base address (128-byte aligned) - uint32_t ctbau; // Upper 32 bits - // DW4-7 - uint32_t rsv1[4]; -} HBA_CMD_HEADER; - -// Physical Region Descriptor Table entry -typedef struct { - uint32_t dba; // Data base address - uint32_t dbau; // Upper 32 bits - uint32_t rsv0; - uint32_t dbc : 22; // Byte count (0-based, so actual = dbc+1) - uint32_t rsv1 : 9; - uint32_t i : 1; // Interrupt on completion -} HBA_PRDT_ENTRY; - -// Command Table -typedef struct { - uint8_t cfis[64]; // Command FIS (up to 64 bytes) - uint8_t acmd[16]; // ATAPI command (12 or 16 bytes) - uint8_t rsv[48]; // Reserved - HBA_PRDT_ENTRY prdt_entry[1]; // PRDT entries (variable length) -} HBA_CMD_TBL; -#pragma pack(pop) - -// Port registers -typedef volatile struct { - uint32_t clb; // 0x00 Command list base (1K-aligned) - uint32_t clbu; // 0x04 Command list base upper - uint32_t fb; // 0x08 FIS base (256-byte aligned) - uint32_t fbu; // 0x0C FIS base upper - uint32_t is; // 0x10 Interrupt status - uint32_t ie; // 0x14 Interrupt enable - uint32_t cmd; // 0x18 Command and status - uint32_t rsv0; // 0x1C Reserved - uint32_t tfd; // 0x20 Task file data - uint32_t sig; // 0x24 Signature - uint32_t ssts; // 0x28 SATA status - uint32_t sctl; // 0x2C SATA control - uint32_t serr; // 0x30 SATA error - uint32_t sact; // 0x34 SATA active - uint32_t ci; // 0x38 Command issue - uint32_t sntf; // 0x3C SATA notification - uint32_t fbs; // 0x40 FIS-based switch control - uint32_t rsv1[11]; - uint32_t vendor[4]; -} HBA_PORT; - -// Generic Host Control registers (starts at ABAR) -typedef volatile struct { - uint32_t cap; // 0x00 Host capability - uint32_t ghc; // 0x04 Global host control - uint32_t is; // 0x08 Interrupt status - uint32_t pi; // 0x0C Ports implemented - uint32_t vs; // 0x10 Version - uint32_t ccc_ctl; // 0x14 CCC control - uint32_t ccc_pts; // 0x18 CCC ports - uint32_t em_loc; // 0x1C Enclosure mgmt location - uint32_t em_ctl; // 0x20 Enclosure mgmt control - uint32_t cap2; // 0x24 Extended capabilities - uint32_t bohc; // 0x28 BIOS/OS handoff control - uint8_t rsv[0xA0-0x2C]; - uint8_t vendor[0x100-0xA0]; - HBA_PORT ports[32]; // 0x100 Port control registers -} HBA_MEM; - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -// PCI -#define PCI_AHCI_CLASS 0x01 -#define PCI_AHCI_SUBCLASS 0x06 -#define PCI_CMD_REG 0x04 -#define PCI_BAR5_REG 0x24 -#define PCI_CMD_BUSMASTER (1 << 2) -#define PCI_CMD_MEMSPACE (1 << 1) - -// GHC register bits -#define GHC_AE (1u << 31) // AHCI Enable -#define GHC_IE (1u << 1) // Interrupt Enable -#define GHC_HR (1u << 0) // HBA Reset - -// BOHC register bits -#define BOHC_BOS (1u << 0) // BIOS-owned semaphore -#define BOHC_OOS (1u << 1) // OS-owned semaphore -#define BOHC_BB (1u << 4) // BIOS Busy - -// CAP2 bits -#define CAP2_BOH (1u << 0) // BIOS/OS handoff capable - -// Port CMD register bits -#define PORT_CMD_ST (1u << 0) // Start -#define PORT_CMD_FRE (1u << 4) // FIS Receive Enable -#define PORT_CMD_FR (1u << 14) // FIS Receive Running -#define PORT_CMD_CR (1u << 15) // Command List Running -#define PORT_CMD_POD (1u << 2) // Power On Device -#define PORT_CMD_SUD (1u << 1) // Spin-Up Device -#define PORT_CMD_ICC_ACTIVE (0x1u << 28) - -// Port IS register bits (interrupt status) -#define PORT_IS_TFES (1u << 30) // Task File Error Status -#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error -#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error -#define PORT_IS_IFS (1u << 27) // Interface Fatal Error - -// Port SSTS register -#define PORT_DET_PRESENT 0x3 -#define PORT_IPM_ACTIVE 0x1 - -// Port signatures -#define SATA_SIG_ATA 0x00000101 -#define SATA_SIG_ATAPI 0xEB140101 -#define SATA_SIG_SEMB 0xC33C0101 -#define SATA_SIG_PM 0x96690101 - -// ATA commands -#define ATA_CMD_READ_DMA_EX 0x25 -#define ATA_CMD_WRITE_DMA_EX 0x35 -#define ATA_CMD_IDENTIFY 0xEC -#define ATA_CMD_FLUSH_EXT 0xEA - -// ATA status bits -#define ATA_SR_BUSY 0x80 -#define ATA_SR_DRQ 0x08 -#define ATA_SR_ERR 0x01 - -// IDENTIFY word offsets -#define ATA_IDENT_W83 83 // bit 10 = 48-bit LBA supported -#define ATA_IDENT_W100 100 // 48-bit LBA sector count (4 words) -#define ATA_IDENT_W60 60 // 28-bit LBA sector count (2 words) - -// Layout constants (per port, single contiguous physical allocation) -// Offset 0: Command List 1024 bytes (32 × 32-byte headers) -// Offset 1024: Received FIS 256 bytes -// Offset 2048: Command Tables 32 × 256 bytes = 8192 bytes -// Total: 10240 bytes → 3 pages (12288 bytes) -#define PORT_PAGES_STRUCT 3 -#define PORT_OFF_CMDLIST 0 -#define PORT_OFF_FIS 1024 -#define PORT_OFF_CMDTBLS 2048 -#define PORT_CMDTBL_SIZE 256 // 64 CFIS + 16 ACMD + 48 rsv + 8×16 PRDT -#define PORT_PRDT_COUNT 8 // PRDT entries per command table - -// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages -#define BOUNCE_SECTORS 128 -#define BOUNCE_BYTES (BOUNCE_SECTORS * 512) -#define BOUNCE_PAGES (BOUNCE_BYTES / 4096) - -// Polling timeout (spin iterations) -#define POLL_TIMEOUT 1000000 - -#define MAX_AHCI_DRIVES 8 -#define NUM_CMD_SLOTS 32 - -// Map MMIO physical range into the physical window as uncacheable. -// Must be called before any pointer dereference via PhysToVirt(). -// AHCI MMIO layout: 0x100 GHC + 32*0x80 ports = 0x1100 bytes → 2 pages minimum. -// We map 8 pages (32KB) without probing the BAR size. -#define AHCI_MMIO_PAGES 8 - -// --------------------------------------------------------------------------- -// Driver-global state -// --------------------------------------------------------------------------- - -static const HAL_API *g_hal; -static HBA_MEM *g_hba; // HBA MMIO base (set during init) -static uint8_t g_irq; // PCI IRQ line for this HBA -static DRIVER_OBJECT g_driver; // AHCI disk driver object - -// Per-drive context -typedef struct { - HBA_PORT *port; // MMIO port registers (virtual) - uint64_t sectorCount; // Total LBA sectors on device - uint8_t portNo; // Physical port index (for ISR dispatch) - - // Structures block (command list + FIS + cmd tables) - uint64_t structPhys; // Physical base - void *structVirt; // Virtual base - - // Bounce buffer for DMA data - uint64_t bouncePhys; - void *bounceVirt; - - // Signaled by the completion DPC when the port raises an interrupt. - // Auto-reset: one Wait consumes one completion. - EVENT *completionEvent; - - // Last IS snapshot captured by the ISR (for error checking post-wait). - volatile uint32_t lastIs; - - // Bottom-half used to wake the worker outside hard IRQ/DIRQL context. - KDPC completionDpc; - - // ----------------------------------------------------------------------- - // IRP queue - async dispatch - // ----------------------------------------------------------------------- - // AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP - // and signals the semaphore to wake the worker. - IRP_QUEUE irpQueue; - THREAD *workerThread; - DEVICE_OBJECT *deviceObject; - - // NCQ TODO: for multi-slot NCQ, add: - // uint32_t slotMask; // bitmask of in-flight slots - // IRP *slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP - // uint64_t slotBouncePhys[NUM_CMD_SLOTS]; - // void *slotBounceVirt[NUM_CMD_SLOTS]; - // volatile uint32_t prevSact; // previous port->sact snapshot -} AHCI_DRIVE; - -static AHCI_DRIVE g_drives[MAX_AHCI_DRIVES]; -static int g_driveCount; +const HAL_API *g_hal; +HBA_MEM *g_hba; +uint8_t g_irq; +DRIVER_OBJECT g_driver; +AHCI_DRIVE g_drives[MAX_AHCI_DRIVES]; +int g_driveCount; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- -static KSTATUS ahci_map_mmio(uint64_t physBase) { - for (int i = 0; i < AHCI_MMIO_PAGES; i++) { - uint64_t phys = physBase + (uint64_t)i * 4096; - uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys); - uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; - KSTATUS st = MmMapPage(virt, phys, flags); - // Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags - // is possible for the first page if the bootloader was overly generous. - // Use MmSetPageProtect to force NOCACHE in that case. - if (st == STATUS_ALREADY_EXISTS) { - st = MmSetPageProtect(virt, 4096, flags); - } - if (!KSUCCESS(st)) { - DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%llx, st=0x%x)\n", - i, (unsigned long long)phys, st); - return st; - } - } - return STATUS_SUCCESS; -} // Accessors for port structure memory at known offsets from structVirt/Phys static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) { @@ -460,29 +42,7 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { // Port engine start / stop (polling safe) // --------------------------------------------------------------------------- -static void port_stop(HBA_PORT *port) { - // Clear ST (bit 0) - port->cmd &= ~PORT_CMD_ST; - // Wait for CR (bit 15) to clear (command list DMA stops) - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(port->cmd & PORT_CMD_CR)) break; - } - // Clear FRE (bit 4) - port->cmd &= ~PORT_CMD_FRE; - // Wait for FR (bit 14) to clear - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(port->cmd & PORT_CMD_FR)) break; - } -} -static void port_start(HBA_PORT *port) { - // Wait until CR is clear before setting ST - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(port->cmd & PORT_CMD_CR)) break; - } - port->cmd |= PORT_CMD_FRE; - port->cmd |= PORT_CMD_ST; -} // --------------------------------------------------------------------------- // Port memory initialisation @@ -878,100 +438,16 @@ static void AhciWorker(uintptr_t arg) { // AHCI controller initialisation // --------------------------------------------------------------------------- -static KSTATUS hba_reset(HBA_MEM *hba) { - // BIOS/OS handoff (spec §10.6.3) - if (hba->cap2 & CAP2_BOH) { - hba->bohc |= BOHC_OOS; - // Wait for BIOS to clear BOS - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(hba->bohc & BOHC_BOS)) break; - } - // Give the BIOS another moment to clear Busy - for (int i = 0; i < 800000; i++) { - if (!(hba->bohc & BOHC_BB)) break; - } - } - - // Enable AHCI mode before reset so the reset actually takes effect - hba->ghc |= GHC_AE; - - // HBA reset - hba->ghc |= GHC_HR; - for (int i = 0; i < POLL_TIMEOUT; i++) { - if (!(hba->ghc & GHC_HR)) break; - } - if (hba->ghc & GHC_HR) { - DbgPrint("AHCI: HBA reset timed out\n"); - return STATUS_TIMEOUT; - } - - // Re-enable AHCI mode (reset cleared it), then enable global interrupts. - // GHC.IE lets the HBA assert its PCI interrupt line when a port fires. - hba->ghc |= GHC_AE | GHC_IE; - return STATUS_SUCCESS; -} // Before memory init: only check physical link. Signature is not yet valid. -static bool port_link_up(HBA_PORT *port) { - uint32_t det = port->ssts & 0x0F; - uint32_t ipm = (port->ssts >> 8) & 0x0F; - return (det == PORT_DET_PRESENT && ipm == PORT_IPM_ACTIVE); -} // After COMRESET + FIS receive: check signature for SATA disk. -static bool port_is_sata_disk(HBA_PORT *port) { - return port->sig == SATA_SIG_ATA; -} -static void ahci_delay_ms(uint32_t ms) { - for (uint32_t i = 0; i < ms * 1000; i++) - __outbyte(0x80, 0); -} // Issue a COMRESET on the port, wait for link re-establishment and D2H FIS. // FRE must already be set (port_init_memory does this) so the HBA can // capture the signature FIS the device sends after the reset. -static KSTATUS port_comreset(AHCI_DRIVE *drv) { - HBA_PORT *port = drv->port; - - port->cmd &= ~PORT_CMD_ST; - for (int i = 0; i < POLL_TIMEOUT; i++) - if (!(port->cmd & PORT_CMD_CR)) break; - - // Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms - port->sctl = (port->sctl & ~0x0Fu) | 0x1u; - ahci_delay_ms(1); - - // Deassert: DET = 0 - port->sctl = port->sctl & ~0x0Fu; - - // Poll for DET = 3, up to 1 second - bool link_ok = false; - for (int i = 0; i < 1000; i++) { - if ((port->ssts & 0x0Fu) == PORT_DET_PRESENT) { link_ok = true; break; } - ahci_delay_ms(1); - } - if (!link_ok) { - DbgPrint("AHCI: COMRESET: link did not come back (ssts=0x%x)\n", port->ssts); - return STATUS_TIMEOUT; - } - - port->serr = (uint32_t)-1; - port->is = (uint32_t)-1; - port->cmd |= PORT_CMD_ST; - - // Wait up to 500ms for a valid signature (D2H FIS from device) - for (int i = 0; i < 500; i++) { - uint32_t sig = port->sig; - if (sig != 0xFFFFFFFFu && sig != 0x00000000u) break; - ahci_delay_ms(1); - } - - DbgPrint("AHCI: port COMRESET done: sig=0x%08x ssts=0x%08x\n", - port->sig, port->ssts); - return STATUS_SUCCESS; -} // PORT_IE bits we care about: // DHRS (bit 0) - D2H Register FIS received - normal completion @@ -1032,7 +508,7 @@ KSTATUS AhciInitialize(const HAL_API *hal) { if (!KSUCCESS(st)) return st; g_hba = (HBA_MEM *)hal->PhysToVirt(abarPhys); - DbgPrint("AHCI: ABAR phys=0x%llx version=0x%08x\n", + DbgPrint("AHCI: ABAR phys=0x%016llX version=0x%08x\n", (unsigned long long)abarPhys, g_hba->vs); // ----------------------------------------------------------------------- diff --git a/code/drivers/storage/ahci/ahci.h b/code/drivers/storage/ahci/ahci.h new file mode 100644 index 0000000..abedc39 --- /dev/null +++ b/code/drivers/storage/ahci/ahci.h @@ -0,0 +1,373 @@ +// ahci.h - shared AHCI driver internals. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +typedef enum { + FIS_TYPE_REG_H2D = 0x27, + FIS_TYPE_REG_D2H = 0x34, + FIS_TYPE_DMA_ACT = 0x39, + FIS_TYPE_DMA_SETUP = 0x41, + FIS_TYPE_DATA = 0x46, + FIS_TYPE_BIST = 0x58, + FIS_TYPE_PIO_SETUP = 0x5F, + FIS_TYPE_DEV_BITS = 0xA1, +} FIS_TYPE; + +#pragma pack(push, 1) +// Register FIS - Host to Device (H2D) +typedef struct { + uint8_t fis_type; // FIS_TYPE_REG_H2D + uint8_t pmport : 4; + uint8_t rsv0 : 3; + uint8_t c : 1; // 1 = Command, 0 = Control + uint8_t command; + uint8_t featurel; + // DW1 + uint8_t lba0, lba1, lba2; + uint8_t device; + // DW2 + uint8_t lba3, lba4, lba5; + uint8_t featureh; + // DW3 + uint8_t countl, counth; + uint8_t icc; + uint8_t control; + // DW4 + uint8_t rsv1[4]; +} FIS_REG_H2D; + +// Register FIS - Device to Host (D2H) +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 2; + uint8_t i : 1; + uint8_t rsv1 : 1; + uint8_t status; + uint8_t error; + uint8_t lba0, lba1, lba2; + uint8_t device; + uint8_t lba3, lba4, lba5; + uint8_t rsv2; + uint8_t countl, counth; + uint8_t rsv3[2]; + uint8_t rsv4[4]; +} FIS_REG_D2H; + +// DMA Setup FIS +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 1; + uint8_t d : 1; + uint8_t i : 1; + uint8_t a : 1; + uint8_t rsved[2]; + uint64_t DMAbufferID; + uint32_t rsvd; + uint32_t DMAbufOffset; + uint32_t TransferCount; + uint32_t resvd; +} FIS_DMA_SETUP; + +// PIO Setup FIS +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 1; + uint8_t d : 1; + uint8_t i : 1; + uint8_t rsv1 : 1; + uint8_t status; + uint8_t error; + uint8_t lba0, lba1, lba2; + uint8_t device; + uint8_t lba3, lba4, lba5; + uint8_t rsv2; + uint8_t countl, counth; + uint8_t rsv3; + uint8_t e_status; + uint16_t tc; + uint8_t rsv4[2]; +} FIS_PIO_SETUP; + +// Set Device Bits FIS +typedef struct { + uint8_t fis_type; + uint8_t pmport : 4; + uint8_t rsv0 : 2; + uint8_t i : 1; + uint8_t n : 1; + uint8_t statusl : 4; + uint8_t rsv1 : 4; + uint8_t statush : 4; + uint8_t rsv2 : 4; + uint8_t error; + uint32_t protocol; +} FIS_DEV_BITS; + +// --------------------------------------------------------------------------- +// HBA structures (volatile - MMIO) +// --------------------------------------------------------------------------- + +// Received FIS structure in host memory +typedef volatile struct { + FIS_DMA_SETUP dsfis; // 0x00 DMA Setup FIS + uint8_t pad0[4]; + FIS_PIO_SETUP psfis; // 0x20 PIO Setup FIS + uint8_t pad1[12]; + FIS_REG_D2H rfis; // 0x40 Register D2H FIS + uint8_t pad2[4]; + FIS_DEV_BITS sdbfis; // 0x58 Set Device Bits FIS + uint8_t ufis[64]; // 0x60 Unknown FIS + uint8_t rsv[0x100-0xA0]; +} HBA_FIS; + +// Command List header (one entry = one slot) +typedef struct { + // DW0 + uint8_t cfl : 5; // Command FIS length in DWORDs (2..16) + uint8_t a : 1; // ATAPI + uint8_t w : 1; // Write (H2D=1, D2H=0) + uint8_t p : 1; // Prefetchable + uint8_t r : 1; // Reset + uint8_t b : 1; // BIST + uint8_t c : 1; // Clear busy on R_OK + uint8_t rsv0 : 1; + uint8_t pmp : 4; // Port multiplier port + uint16_t prdtl; // PRDT length in entries + // DW1 + volatile uint32_t prdbc; // PRDT byte count transferred + // DW2,3 + uint32_t ctba; // Command table base address (128-byte aligned) + uint32_t ctbau; // Upper 32 bits + // DW4-7 + uint32_t rsv1[4]; +} HBA_CMD_HEADER; + +// Physical Region Descriptor Table entry +typedef struct { + uint32_t dba; // Data base address + uint32_t dbau; // Upper 32 bits + uint32_t rsv0; + uint32_t dbc : 22; // Byte count (0-based, so actual = dbc+1) + uint32_t rsv1 : 9; + uint32_t i : 1; // Interrupt on completion +} HBA_PRDT_ENTRY; + +// Command Table +typedef struct { + uint8_t cfis[64]; // Command FIS (up to 64 bytes) + uint8_t acmd[16]; // ATAPI command (12 or 16 bytes) + uint8_t rsv[48]; // Reserved + HBA_PRDT_ENTRY prdt_entry[1]; // PRDT entries (variable length) +} HBA_CMD_TBL; +#pragma pack(pop) + +// Port registers +typedef volatile struct { + uint32_t clb; // 0x00 Command list base (1K-aligned) + uint32_t clbu; // 0x04 Command list base upper + uint32_t fb; // 0x08 FIS base (256-byte aligned) + uint32_t fbu; // 0x0C FIS base upper + uint32_t is; // 0x10 Interrupt status + uint32_t ie; // 0x14 Interrupt enable + uint32_t cmd; // 0x18 Command and status + uint32_t rsv0; // 0x1C Reserved + uint32_t tfd; // 0x20 Task file data + uint32_t sig; // 0x24 Signature + uint32_t ssts; // 0x28 SATA status + uint32_t sctl; // 0x2C SATA control + uint32_t serr; // 0x30 SATA error + uint32_t sact; // 0x34 SATA active + uint32_t ci; // 0x38 Command issue + uint32_t sntf; // 0x3C SATA notification + uint32_t fbs; // 0x40 FIS-based switch control + uint32_t rsv1[11]; + uint32_t vendor[4]; +} HBA_PORT; + +// Generic Host Control registers (starts at ABAR) +typedef volatile struct { + uint32_t cap; // 0x00 Host capability + uint32_t ghc; // 0x04 Global host control + uint32_t is; // 0x08 Interrupt status + uint32_t pi; // 0x0C Ports implemented + uint32_t vs; // 0x10 Version + uint32_t ccc_ctl; // 0x14 CCC control + uint32_t ccc_pts; // 0x18 CCC ports + uint32_t em_loc; // 0x1C Enclosure mgmt location + uint32_t em_ctl; // 0x20 Enclosure mgmt control + uint32_t cap2; // 0x24 Extended capabilities + uint32_t bohc; // 0x28 BIOS/OS handoff control + uint8_t rsv[0xA0-0x2C]; + uint8_t vendor[0x100-0xA0]; + HBA_PORT ports[32]; // 0x100 Port control registers +} HBA_MEM; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +// PCI +#define PCI_AHCI_CLASS 0x01 +#define PCI_AHCI_SUBCLASS 0x06 +#define PCI_CMD_REG 0x04 +#define PCI_BAR5_REG 0x24 +#define PCI_CMD_BUSMASTER (1 << 2) +#define PCI_CMD_MEMSPACE (1 << 1) + +// GHC register bits +#define GHC_AE (1u << 31) // AHCI Enable +#define GHC_IE (1u << 1) // Interrupt Enable +#define GHC_HR (1u << 0) // HBA Reset + +// BOHC register bits +#define BOHC_BOS (1u << 0) // BIOS-owned semaphore +#define BOHC_OOS (1u << 1) // OS-owned semaphore +#define BOHC_BB (1u << 4) // BIOS Busy + +// CAP2 bits +#define CAP2_BOH (1u << 0) // BIOS/OS handoff capable + +// Port CMD register bits +#define PORT_CMD_ST (1u << 0) // Start +#define PORT_CMD_FRE (1u << 4) // FIS Receive Enable +#define PORT_CMD_FR (1u << 14) // FIS Receive Running +#define PORT_CMD_CR (1u << 15) // Command List Running +#define PORT_CMD_POD (1u << 2) // Power On Device +#define PORT_CMD_SUD (1u << 1) // Spin-Up Device +#define PORT_CMD_ICC_ACTIVE (0x1u << 28) + +// Port IS register bits (interrupt status) +#define PORT_IS_TFES (1u << 30) // Task File Error Status +#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error +#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error +#define PORT_IS_IFS (1u << 27) // Interface Fatal Error + +// Port SSTS register +#define PORT_DET_PRESENT 0x3 +#define PORT_IPM_ACTIVE 0x1 + +// Port signatures +#define SATA_SIG_ATA 0x00000101 +#define SATA_SIG_ATAPI 0xEB140101 +#define SATA_SIG_SEMB 0xC33C0101 +#define SATA_SIG_PM 0x96690101 + +// ATA commands +#define ATA_CMD_READ_DMA_EX 0x25 +#define ATA_CMD_WRITE_DMA_EX 0x35 +#define ATA_CMD_IDENTIFY 0xEC +#define ATA_CMD_FLUSH_EXT 0xEA + +// ATA status bits +#define ATA_SR_BUSY 0x80 +#define ATA_SR_DRQ 0x08 +#define ATA_SR_ERR 0x01 + +// IDENTIFY word offsets +#define ATA_IDENT_W83 83 // bit 10 = 48-bit LBA supported +#define ATA_IDENT_W100 100 // 48-bit LBA sector count (4 words) +#define ATA_IDENT_W60 60 // 28-bit LBA sector count (2 words) + +// Layout constants (per port, single contiguous physical allocation) +// Offset 0: Command List 1024 bytes (32 × 32-byte headers) +// Offset 1024: Received FIS 256 bytes +// Offset 2048: Command Tables 32 × 256 bytes = 8192 bytes +// Total: 10240 bytes → 3 pages (12288 bytes) +#define PORT_PAGES_STRUCT 3 +#define PORT_OFF_CMDLIST 0 +#define PORT_OFF_FIS 1024 +#define PORT_OFF_CMDTBLS 2048 +#define PORT_CMDTBL_SIZE 256 // 64 CFIS + 16 ACMD + 48 rsv + 8×16 PRDT +#define PORT_PRDT_COUNT 8 // PRDT entries per command table + +// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages +#define BOUNCE_SECTORS 128 +#define BOUNCE_BYTES (BOUNCE_SECTORS * 512) +#define BOUNCE_PAGES (BOUNCE_BYTES / 4096) + +// Polling timeout (spin iterations) +#define POLL_TIMEOUT 1000000 + +#define MAX_AHCI_DRIVES 8 +#define NUM_CMD_SLOTS 32 + +// Map MMIO physical range into the physical window as uncacheable. +// Must be called before any pointer dereference via PhysToVirt(). +// AHCI MMIO layout: 0x100 GHC + 32*0x80 ports = 0x1100 bytes → 2 pages minimum. +// We map 8 pages (32KB) without probing the BAR size. +#define AHCI_MMIO_PAGES 8 + +// Per-drive context +typedef struct { + HBA_PORT *port; // MMIO port registers (virtual) + uint64_t sectorCount; // Total LBA sectors on device + uint8_t portNo; // Physical port index (for ISR dispatch) + + // Structures block (command list + FIS + cmd tables) + uint64_t structPhys; // Physical base + void* structVirt; // Virtual base + + // Bounce buffer for DMA data + uint64_t bouncePhys; + void* bounceVirt; + + // Signaled by the completion DPC when the port raises an interrupt. + // Auto-reset: one Wait consumes one completion. + EVENT *completionEvent; + + // Last IS snapshot captured by the ISR (for error checking post-wait). + volatile uint32_t lastIs; + + // Bottom-half used to wake the worker outside hard IRQ/DIRQL context. + KDPC completionDpc; + + // ----------------------------------------------------------------------- + // IRP queue - async dispatch + // ----------------------------------------------------------------------- + // AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP + // and signals the semaphore to wake the worker. + IRP_QUEUE irpQueue; + THREAD* workerThread; + DEVICE_OBJECT* deviceObject; + + // NCQ TODO: for multi-slot NCQ, add: + // uint32_t slotMask; // bitmask of in-flight slots + // IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP + // uint64_t slotBouncePhys[NUM_CMD_SLOTS]; + // void* slotBounceVirt[NUM_CMD_SLOTS]; + // volatile uint32_t prevSact; // previous port->sact snapshot +} AHCI_DRIVE; + + +extern const HAL_API *g_hal; +extern HBA_MEM *g_hba; +extern uint8_t g_irq; +extern DRIVER_OBJECT g_driver; +extern AHCI_DRIVE g_drives[MAX_AHCI_DRIVES]; +extern int g_driveCount; + +KSTATUS ahci_map_mmio(uint64_t physBase); +void port_stop(HBA_PORT *port); +void port_start(HBA_PORT *port); +KSTATUS hba_reset(HBA_MEM *hba); +bool port_link_up(HBA_PORT *port); +bool port_is_sata_disk(HBA_PORT *port); +void ahci_delay_ms(uint32_t ms); +KSTATUS port_comreset(AHCI_DRIVE *drv); diff --git a/code/drivers/storage/ahci/ahci_port.c b/code/drivers/storage/ahci/ahci_port.c new file mode 100644 index 0000000..6936bb3 --- /dev/null +++ b/code/drivers/storage/ahci/ahci_port.c @@ -0,0 +1,143 @@ +// ahci_port.c - AHCI MMIO mapping, HBA reset, and port link bring-up helpers. +#include "ahci.h" + +KSTATUS ahci_map_mmio(uint64_t physBase) { + for (int i = 0; i < AHCI_MMIO_PAGES; i++) { + uint64_t phys = physBase + (uint64_t)i * 4096; + uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys); + uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; + KSTATUS st = MmMapPage(virt, phys, flags); + // Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags + // is possible for the first page if the bootloader was overly generous. + // Use MmSetPageProtect to force NOCACHE in that case. + if (st == STATUS_ALREADY_EXISTS) { + st = MmSetPageProtect(virt, 4096, flags); + } + if (!KSUCCESS(st)) { + DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n", + i, (unsigned long long)phys, st); + return st; + } + } + return STATUS_SUCCESS; +} + + +void port_stop(HBA_PORT *port) { + // Clear ST (bit 0) + port->cmd &= ~PORT_CMD_ST; + // Wait for CR (bit 15) to clear (command list DMA stops) + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(port->cmd & PORT_CMD_CR)) break; + } + // Clear FRE (bit 4) + port->cmd &= ~PORT_CMD_FRE; + // Wait for FR (bit 14) to clear + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(port->cmd & PORT_CMD_FR)) break; + } +} + + +void port_start(HBA_PORT *port) { + // Wait until CR is clear before setting ST + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(port->cmd & PORT_CMD_CR)) break; + } + port->cmd |= PORT_CMD_FRE; + port->cmd |= PORT_CMD_ST; +} + + +KSTATUS hba_reset(HBA_MEM *hba) { + // BIOS/OS handoff (spec §10.6.3) + if (hba->cap2 & CAP2_BOH) { + hba->bohc |= BOHC_OOS; + // Wait for BIOS to clear BOS + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(hba->bohc & BOHC_BOS)) break; + } + // Give the BIOS another moment to clear Busy + for (int i = 0; i < 800000; i++) { + if (!(hba->bohc & BOHC_BB)) break; + } + } + + // Enable AHCI mode before reset so the reset actually takes effect + hba->ghc |= GHC_AE; + + // HBA reset + hba->ghc |= GHC_HR; + for (int i = 0; i < POLL_TIMEOUT; i++) { + if (!(hba->ghc & GHC_HR)) break; + } + if (hba->ghc & GHC_HR) { + DbgPrint("AHCI: HBA reset timed out\n"); + return STATUS_TIMEOUT; + } + + // Re-enable AHCI mode (reset cleared it), then enable global interrupts. + // GHC.IE lets the HBA assert its PCI interrupt line when a port fires. + hba->ghc |= GHC_AE | GHC_IE; + return STATUS_SUCCESS; +} + + +bool port_link_up(HBA_PORT *port) { + uint32_t det = port->ssts & 0x0F; + uint32_t ipm = (port->ssts >> 8) & 0x0F; + return (det == PORT_DET_PRESENT && ipm == PORT_IPM_ACTIVE); +} + + +bool port_is_sata_disk(HBA_PORT *port) { + return port->sig == SATA_SIG_ATA; +} + + +void ahci_delay_ms(uint32_t ms) { + for (uint32_t i = 0; i < ms * 1000; i++) + __outbyte(0x80, 0); +} + + +KSTATUS port_comreset(AHCI_DRIVE *drv) { + HBA_PORT *port = drv->port; + + port->cmd &= ~PORT_CMD_ST; + for (int i = 0; i < POLL_TIMEOUT; i++) + if (!(port->cmd & PORT_CMD_CR)) break; + + // Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms + port->sctl = (port->sctl & ~0x0Fu) | 0x1u; + ahci_delay_ms(1); + + // Deassert: DET = 0 + port->sctl = port->sctl & ~0x0Fu; + + // Poll for DET = 3, up to 1 second + bool link_ok = false; + for (int i = 0; i < 1000; i++) { + if ((port->ssts & 0x0Fu) == PORT_DET_PRESENT) { link_ok = true; break; } + ahci_delay_ms(1); + } + if (!link_ok) { + DbgPrint("AHCI: COMRESET: link did not come back (ssts=0x%x)\n", port->ssts); + return STATUS_TIMEOUT; + } + + port->serr = (uint32_t)-1; + port->is = (uint32_t)-1; + port->cmd |= PORT_CMD_ST; + + // Wait up to 500ms for a valid signature (D2H FIS from device) + for (int i = 0; i < 500; i++) { + uint32_t sig = port->sig; + if (sig != 0xFFFFFFFFu && sig != 0x00000000u) break; + ahci_delay_ms(1); + } + + DbgPrint("AHCI: port COMRESET done: sig=0x%08x ssts=0x%08x\n", + port->sig, port->ssts); + return STATUS_SUCCESS; +} diff --git a/code/drivers/storage/ide/ide.c b/code/drivers/storage/ide/ide.c index ffdd6e7..142d52c 100644 --- a/code/drivers/storage/ide/ide.c +++ b/code/drivers/storage/ide/ide.c @@ -1,130 +1,26 @@ -// ide.c — IDE/ATA driver: PIO (sync fallback) + Bus Master DMA (async) -// -// Architecture -// ------------ -// The dispatch path mirrors AHCI: -// IoCallDriver → IdeDispatchReadWrite → IRP queue or inline PIO -// IdeWorker → dequeues IRP, calls IdeDmaRw(), completes the IRP -// IdeIsr → clears BM status, queues a DPC -// IdeDpc → signals gDmaEvent to unblock the worker -// -// Graceful degradation: if Bus Master init fails (no PCI IDE controller, -// BAR4 not assigned, PRDT allocation above 4 GB, etc.) the same device -// object remains registered and dispatch completes IRPs inline through PIO. -// -// PRDT layout -// ----------- -// One page allocated via MmAllocateContiguous. We build up to PRDT_MAX_ENTRIES -// entries by walking the transfer buffer in page-sized chunks and looking up -// each page's physical address via MmGetPhysicalAddress. Adjacent physical -// pages are coalesced into a single PRD entry where possible. -// -// LBA48 vs LBA28 -// -------------- -// DMA commands exist in both modes: CMD_READ/WRITE_DMA_EXT (0x25/0x35) for -// LBA48 and CMD_READ/WRITE_DMA (0xC8/0xCA) for LBA28. The existing -// gSupportsLba48 flag picks the right variant. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// --------------------------------------------------------------------------- -// IDE register map (primary channel, base 0x1F0) -// --------------------------------------------------------------------------- -#define IDE_DATA 0x1F0 -#define IDE_ERR 0x1F1 -#define IDE_SECCNT 0x1F2 -#define IDE_LBA0 0x1F3 -#define IDE_LBA1 0x1F4 -#define IDE_LBA2 0x1F5 -#define IDE_DRIVE 0x1F6 -#define IDE_STATUS 0x1F7 -#define IDE_CMD 0x1F7 -#define IDE_ALT 0x3F6 - -#define IDE_BUSY 0x80 -#define IDE_READY 0x40 -#define IDE_ERR_F 0x01 -#define IDE_DRQ 0x08 - -// ATA commands -#define CMD_READ_EXT 0x24 // PIO 48-bit read -#define CMD_WRITE_EXT 0x34 // PIO 48-bit write -#define CMD_READ_DMA_EXT 0x25 // DMA 48-bit read -#define CMD_WRITE_DMA_EXT 0x35 // DMA 48-bit write -#define CMD_READ_DMA 0xC8 // DMA 28-bit read -#define CMD_WRITE_DMA 0xCA // DMA 28-bit write -#define CMD_IDENT 0xEC - -// IDE primary IRQ -#define IDE_IRQ_PRIMARY 14 - -// --------------------------------------------------------------------------- -// Bus Master IDE register offsets (relative to gBmBase, primary channel) -// --------------------------------------------------------------------------- -#define BM_CMD_REG 0x00 // command -#define BM_STATUS_REG 0x02 // status -#define BM_PRDT_REG 0x04 // PRDT physical address (32-bit) - -// BM_CMD_REG bits -#define BM_CMD_START 0x01 // start DMA transfer -#define BM_CMD_READ 0x08 // 1 = disk→memory (read), 0 = memory→disk (write) - -// BM_STATUS_REG bits -#define BM_STATUS_ACTIVE 0x01 // DMA active -#define BM_STATUS_ERROR 0x02 // DMA error (write 1 to clear) -#define BM_STATUS_IRQ 0x04 // interrupt pending (write 1 to clear) - -// --------------------------------------------------------------------------- -// Physical Region Descriptor (PRDT entry, 8 bytes) -// The PRDT itself must not cross a 64 KB boundary and must be 4-byte aligned. -// We allocate a full page so both constraints are trivially satisfied. -// --------------------------------------------------------------------------- -#pragma pack(push, 1) -typedef struct { - uint32_t physAddr; // physical base of this memory region - uint16_t byteCount; // byte count; 0 means 64 KB - uint16_t flags; // bit 15: End Of Table -} PRD; -#pragma pack(pop) - -#define PRD_EOT 0x8000u -// 512 entries × 8 bytes = 4096 bytes = exactly one page; supports 512×64 KB = 32 MB -#define PRDT_MAX_ENTRIES 512 +// ide.c - IDE/ATA disk I/O path and DriverEntry. +#include "ide.h" // --------------------------------------------------------------------------- // Module state // --------------------------------------------------------------------------- -static const HAL_API *gHal; -static DRIVER_OBJECT gDriver; -static DEVICE_OBJECT *gDevice = NULL; -static bool gDmaEnabled = false; -static uint64_t gSectorCount = 0; -static bool gSupportsLba48 = false; +const HAL_API *gHal; +DRIVER_OBJECT gDriver; +DEVICE_OBJECT *gDevice = NULL; +bool gDmaEnabled = false; +uint64_t gSectorCount = 0; +bool gSupportsLba48 = false; // DMA hardware state -static uint16_t gBmBase = 0; // Bus Master I/O base port (BAR4 & ~3) -static PRD *gPrdtVirt = NULL; // kernel VA of PRDT page -static uint32_t gPrdtPhys = 0; // 32-bit physical address of PRDT +uint16_t gBmBase = 0; // Bus Master I/O base port (BAR4 & ~3) +PRD *gPrdtVirt = NULL; // kernel VA of PRDT page +uint32_t gPrdtPhys = 0; // 32-bit physical address of PRDT // Async worker synchronisation -static IRP_QUEUE gIrpQueue; -static SEMAPHORE *gIrpSemaphore = NULL; -static EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer -static KDPC gDmaDpc; // bottom half for DMA completion +IRP_QUEUE gIrpQueue; +SEMAPHORE *gIrpSemaphore = NULL; +EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer +KDPC gDmaDpc; // bottom half for DMA completion // --------------------------------------------------------------------------- // Bus Master I/O helpers @@ -255,7 +151,7 @@ static uint32_t BuildPrdt(void *virtBuf, uint32_t byteLen) { PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); if (phys >> 32) { // Physical address above 4 GB — BM IDE's 32-bit PRDT can't reach it. - DbgPrint("IDE DMA: buffer page at VA 0x%llx has phys 0x%llx > 4 GB\n", + DbgPrint("IDE DMA: buffer page at VA 0x%016llX has phys 0x%016llX > 4 GB\n", (uint64_t)va, (uint64_t)phys); return 0; } @@ -294,7 +190,7 @@ static uint32_t BuildPrdt(void *virtBuf, uint32_t byteLen) { // IDE DMA completion DPC. The ISR owns hardware acknowledgement; this bottom // half wakes the worker outside hard IRQ/DIRQL context. // --------------------------------------------------------------------------- -static void IdeDmaCompletionDpc(KDPC *dpc, void *context, +void IdeDmaCompletionDpc(KDPC *dpc, void *context, void *systemArgument1, void *systemArgument2) { (void)dpc; (void)context; @@ -312,7 +208,7 @@ static void IdeDmaCompletionDpc(KDPC *dpc, void *context, // interrupt bits here and queue a DPC to wake the worker thread. // The worker reads the final status and fires IRP completion hooks. // --------------------------------------------------------------------------- -static void IdeIsr(void) { +void IdeIsr(void) { uint8_t bmSt = BmRead8(BM_STATUS_REG); if (!(bmSt & BM_STATUS_IRQ)) return; // not our interrupt (spurious or secondary channel) @@ -417,7 +313,7 @@ static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) { // Blocks on gIrpSemaphore. Each IrpQueueEnqueue increments it by one, so // the worker wakes once per submitted IRP. Mirrors AhciWorker exactly. // --------------------------------------------------------------------------- -static void IdeWorker(uintptr_t arg) { +void IdeWorker(uintptr_t arg) { (void)arg; while (true) { KeWaitSemaphore(gIrpSemaphore); @@ -473,90 +369,6 @@ static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) { // Returns true on success. On any failure, logs a message and returns false; // dispatch will complete IRPs inline through PIO. // --------------------------------------------------------------------------- -static bool IdeDmaInit(void) { - // ---- Find PCI IDE controller (class 0x01, subclass 0x01) -------------- - uint8_t bus, dev, fn; - if (!gHal->Pci.FindClass(0x01, 0x01, &bus, &dev, &fn)) { - DbgPrint("IDE DMA: No PCI IDE controller found — using PIO\n"); - return false; - } - - // ---- Read BAR4 (Bus Master base, I/O space) --------------------------- - // PCI config offset 0x20. Bit 0 must be 1 for I/O BAR. - uint32_t bar4 = gHal->Pci.Read32(bus, dev, fn, 0x20); - if (!(bar4 & 0x1)) { - DbgPrint("IDE DMA: BAR4 is a memory BAR (0x%08x) — unexpected, using PIO\n", bar4); - return false; - } - uint16_t bmBase = (uint16_t)(bar4 & 0xFFFC); - if (bmBase == 0) { - DbgPrint("IDE DMA: BAR4 is zero (firmware didn't assign it) — using PIO\n"); - return false; - } - DbgPrint("IDE DMA: Bus Master base = 0x%04X (bus %u, dev %u, fn %u)\n", - bmBase, bus, dev, fn); - - // ---- Enable Bus Mastering in PCI Command register (bit 2) ------------ - uint32_t pciCmd = gHal->Pci.Read32(bus, dev, fn, 0x04); - gHal->Pci.Write32(bus, dev, fn, 0x04, pciCmd | (1u << 2)); - - // ---- Allocate PRDT page (must be 32-bit physical, 4-byte aligned) ---- - PHYSICAL_ADDRESS prdtPhys = MmAllocateContiguous(1); - if (!prdtPhys) { - DbgPrint("IDE DMA: Failed to allocate PRDT page\n"); - return false; - } - if (prdtPhys >> 32) { - // MmAllocateContiguous returned memory above 4 GB — BM IDE can't - // address it. This shouldn't happen on normal x86 systems with early - // boot allocations but we check defensively. - DbgPrint("IDE DMA: PRDT page is above 4 GB (0x%llx) — using PIO\n", - (uint64_t)prdtPhys); - return false; - } - void *prdtVirt = gHal->PhysToVirt(prdtPhys); - DbgPrint("IDE DMA: PRDT at phys 0x%08x, virt %p\n", (uint32_t)prdtPhys, prdtVirt); - - // ---- Create synchronisation objects ------------------------------------ - SEMAPHORE *sem = NULL; - if (!KSUCCESS(KeSemaphoreCreate(0, 0x7FFFFFFF, &sem))) { - DbgPrint("IDE DMA: Failed to create IRP semaphore\n"); - return false; - } - - EVENT *dmaEv = NULL; - if (!KSUCCESS(KeEventCreate(EventAutoReset, 0, &dmaEv))) { - DbgPrint("IDE DMA: Failed to create DMA completion event\n"); - return false; - } - - // ---- Commit module state ----------------------------------------------- - gBmBase = bmBase; - gPrdtPhys = (uint32_t)prdtPhys; - gPrdtVirt = (PRD *)prdtVirt; - gIrpSemaphore = sem; - gDmaEvent = dmaEv; - KeInitializeDpc(&gDmaDpc, IdeDmaCompletionDpc, (void *)0); - - IrpQueueInit(&gIrpQueue, sem); - - // ---- Register IRQ 14 -------------------------------------------------- - KeIrqRegister(IDE_IRQ_PRIMARY, IdeIsr); - DbgPrint("IDE DMA: IRQ %u registered\n", IDE_IRQ_PRIMARY); - - // ---- Spawn worker thread ----------------------------------------------- - KSTATUS st = KeCreateKernelThread((uintptr_t)IdeWorker, 0, - PriorityHigh, (THREAD **)NULL); - if (!KSUCCESS(st)) { - DbgPrint("IDE DMA: Worker thread creation failed (0x%x)\n", st); - // Not fatal for init — the PRDT/IRQ are live but async callers will - // never get completions. Better to surface the failure clearly. - return false; - } - - DbgPrint("IDE DMA: Init complete\n"); - return true; -} // --------------------------------------------------------------------------- // DriverEntry diff --git a/code/drivers/storage/ide/ide.h b/code/drivers/storage/ide/ide.h new file mode 100644 index 0000000..dd46269 --- /dev/null +++ b/code/drivers/storage/ide/ide.h @@ -0,0 +1,102 @@ +// ide.h - shared IDE/ATA driver internals. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// IDE register map (primary channel, base 0x1F0) +// --------------------------------------------------------------------------- +#define IDE_DATA 0x1F0 +#define IDE_ERR 0x1F1 +#define IDE_SECCNT 0x1F2 +#define IDE_LBA0 0x1F3 +#define IDE_LBA1 0x1F4 +#define IDE_LBA2 0x1F5 +#define IDE_DRIVE 0x1F6 +#define IDE_STATUS 0x1F7 +#define IDE_CMD 0x1F7 +#define IDE_ALT 0x3F6 + +#define IDE_BUSY 0x80 +#define IDE_READY 0x40 +#define IDE_ERR_F 0x01 +#define IDE_DRQ 0x08 + +// ATA commands +#define CMD_READ_EXT 0x24 // PIO 48-bit read +#define CMD_WRITE_EXT 0x34 // PIO 48-bit write +#define CMD_READ_DMA_EXT 0x25 // DMA 48-bit read +#define CMD_WRITE_DMA_EXT 0x35 // DMA 48-bit write +#define CMD_READ_DMA 0xC8 // DMA 28-bit read +#define CMD_WRITE_DMA 0xCA // DMA 28-bit write +#define CMD_IDENT 0xEC + +// IDE primary IRQ +#define IDE_IRQ_PRIMARY 14 + +// --------------------------------------------------------------------------- +// Bus Master IDE register offsets (relative to gBmBase, primary channel) +// --------------------------------------------------------------------------- +#define BM_CMD_REG 0x00 // command +#define BM_STATUS_REG 0x02 // status +#define BM_PRDT_REG 0x04 // PRDT physical address (32-bit) + +// BM_CMD_REG bits +#define BM_CMD_START 0x01 // start DMA transfer +#define BM_CMD_READ 0x08 // 1 = disk→memory (read), 0 = memory→disk (write) + +// BM_STATUS_REG bits +#define BM_STATUS_ACTIVE 0x01 // DMA active +#define BM_STATUS_ERROR 0x02 // DMA error (write 1 to clear) +#define BM_STATUS_IRQ 0x04 // interrupt pending (write 1 to clear) + +// --------------------------------------------------------------------------- +// Physical Region Descriptor (PRDT entry, 8 bytes) +// The PRDT itself must not cross a 64 KB boundary and must be 4-byte aligned. +// We allocate a full page so both constraints are trivially satisfied. +// --------------------------------------------------------------------------- +#pragma pack(push, 1) +typedef struct { + uint32_t physAddr; // physical base of this memory region + uint16_t byteCount; // byte count; 0 means 64 KB + uint16_t flags; // bit 15: End Of Table +} PRD; +#pragma pack(pop) + +#define PRD_EOT 0x8000u +// 512 entries × 8 bytes = 4096 bytes = exactly one page; supports 512×64 KB = 32 MB +#define PRDT_MAX_ENTRIES 512 + + +extern const HAL_API *gHal; +extern DRIVER_OBJECT gDriver; +extern DEVICE_OBJECT *gDevice; +extern bool gDmaEnabled; +extern uint64_t gSectorCount; +extern bool gSupportsLba48; +extern uint16_t gBmBase; +extern PRD *gPrdtVirt; +extern uint32_t gPrdtPhys; +extern IRP_QUEUE gIrpQueue; +extern SEMAPHORE *gIrpSemaphore; +extern EVENT *gDmaEvent; +extern KDPC gDmaDpc; + +void IdeDmaCompletionDpc(KDPC *dpc, void *context, void *systemArgument1, void *systemArgument2); +void IdeIsr(void); +void IdeWorker(uintptr_t arg); +bool IdeDmaInit(void); diff --git a/code/drivers/storage/ide/ide_bus.c b/code/drivers/storage/ide/ide_bus.c new file mode 100644 index 0000000..add0410 --- /dev/null +++ b/code/drivers/storage/ide/ide_bus.c @@ -0,0 +1,87 @@ +// ide_bus.c - Bus-master IDE PCI/DMA initialisation. +#include "ide.h" + +bool IdeDmaInit(void) { + // ---- Find PCI IDE controller (class 0x01, subclass 0x01) -------------- + uint8_t bus, dev, fn; + if (!gHal->Pci.FindClass(0x01, 0x01, &bus, &dev, &fn)) { + DbgPrint("IDE DMA: No PCI IDE controller found — using PIO\n"); + return false; + } + + // ---- Read BAR4 (Bus Master base, I/O space) --------------------------- + // PCI config offset 0x20. Bit 0 must be 1 for I/O BAR. + uint32_t bar4 = gHal->Pci.Read32(bus, dev, fn, 0x20); + if (!(bar4 & 0x1)) { + DbgPrint("IDE DMA: BAR4 is a memory BAR (0x%08x) — unexpected, using PIO\n", bar4); + return false; + } + uint16_t bmBase = (uint16_t)(bar4 & 0xFFFC); + if (bmBase == 0) { + DbgPrint("IDE DMA: BAR4 is zero (firmware didn't assign it) — using PIO\n"); + return false; + } + DbgPrint("IDE DMA: Bus Master base = 0x%04X (bus %u, dev %u, fn %u)\n", + bmBase, bus, dev, fn); + + // ---- Enable Bus Mastering in PCI Command register (bit 2) ------------ + uint32_t pciCmd = gHal->Pci.Read32(bus, dev, fn, 0x04); + gHal->Pci.Write32(bus, dev, fn, 0x04, pciCmd | (1u << 2)); + + // ---- Allocate PRDT page (must be 32-bit physical, 4-byte aligned) ---- + PHYSICAL_ADDRESS prdtPhys = MmAllocateContiguous(1); + if (!prdtPhys) { + DbgPrint("IDE DMA: Failed to allocate PRDT page\n"); + return false; + } + if (prdtPhys >> 32) { + // MmAllocateContiguous returned memory above 4 GB — BM IDE can't + // address it. This shouldn't happen on normal x86 systems with early + // boot allocations but we check defensively. + DbgPrint("IDE DMA: PRDT page is above 4 GB (0x%016llX) — using PIO\n", + (uint64_t)prdtPhys); + return false; + } + void *prdtVirt = gHal->PhysToVirt(prdtPhys); + DbgPrint("IDE DMA: PRDT at phys 0x%08x, virt 0x%016llX\n", (uint32_t)prdtPhys, prdtVirt); + + // ---- Create synchronisation objects ------------------------------------ + SEMAPHORE *sem = NULL; + if (!KSUCCESS(KeSemaphoreCreate(0, 0x7FFFFFFF, &sem))) { + DbgPrint("IDE DMA: Failed to create IRP semaphore\n"); + return false; + } + + EVENT *dmaEv = NULL; + if (!KSUCCESS(KeEventCreate(EventAutoReset, 0, &dmaEv))) { + DbgPrint("IDE DMA: Failed to create DMA completion event\n"); + return false; + } + + // ---- Commit module state ----------------------------------------------- + gBmBase = bmBase; + gPrdtPhys = (uint32_t)prdtPhys; + gPrdtVirt = (PRD *)prdtVirt; + gIrpSemaphore = sem; + gDmaEvent = dmaEv; + KeInitializeDpc(&gDmaDpc, IdeDmaCompletionDpc, (void *)0); + + IrpQueueInit(&gIrpQueue, sem); + + // ---- Register IRQ 14 -------------------------------------------------- + KeIrqRegister(IDE_IRQ_PRIMARY, IdeIsr); + DbgPrint("IDE DMA: IRQ %u registered\n", IDE_IRQ_PRIMARY); + + // ---- Spawn worker thread ----------------------------------------------- + KSTATUS st = KeCreateKernelThread((uintptr_t)IdeWorker, 0, + PriorityHigh, (THREAD **)NULL); + if (!KSUCCESS(st)) { + DbgPrint("IDE DMA: Worker thread creation failed (0x%x)\n", st); + // Not fatal for init — the PRDT/IRQ are live but async callers will + // never get completions. Better to surface the failure clearly. + return false; + } + + DbgPrint("IDE DMA: Init complete\n"); + return true; +} diff --git a/code/kernel/hal/hal_apic.c b/code/kernel/hal/hal_apic.c index 9e57b4c..1806388 100644 --- a/code/kernel/hal/hal_apic.c +++ b/code/kernel/hal/hal_apic.c @@ -16,11 +16,23 @@ static volatile uint32_t* gLapic = (volatile uint32_t *)0; #define LAPIC_ICR_HI 0x310 #define LAPIC_ENABLE 0x100 // bit in spurious vector register +#define IA32_APIC_BASE 0x0000001Bu +#define IA32_APIC_BASE_ENABLE (1ULL << 11) +#define IA32_APIC_BASE_X2APIC (1ULL << 10) +#define IA32_X2APIC_ID 0x00000802u +#define IA32_X2APIC_EOI 0x0000080Bu +#define IA32_X2APIC_SVR 0x0000080Fu +#define IA32_X2APIC_ICR 0x00000830u + +#define CPUID_FEATURE_ECX_X2APIC (1u << 21) + #define APIC_VECTOR_RESCHEDULE 0xF0u // ---- I/O APIC ---- static volatile uint32_t* gIoapic = (volatile uint32_t *)0; static uint32_t gBspLapicId = 0; +static uint8_t gX2ApicSupported = 0; +static uint8_t gX2ApicEnabled = 0; #define IOAPIC_IDX 0x00 #define IOAPIC_DATA 0x10 @@ -33,6 +45,14 @@ static void IoapicWrite(uint8_t reg, uint32_t val) { *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val; } +static void HalApicDetectMode(void) { + int regs[4] = {0, 0, 0, 0}; + __cpuid(regs, 1); + gX2ApicSupported = (regs[2] & CPUID_FEATURE_ECX_X2APIC) ? 1u : 0u; + + uint64_t apicBase = __readmsr(IA32_APIC_BASE); + gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u; +} static void HalApicStallFallback(uint32_t loops) { for (volatile uint32_t i = 0; i < loops; i++) { @@ -46,6 +66,9 @@ static void HalApicStallUs(uint32_t us) { } static void HalApicWaitIcrIdle(void) { + if (gX2ApicEnabled) + return; + for (uint32_t i = 0; i < 1000000u; i++) { if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12))) return; @@ -54,6 +77,11 @@ static void HalApicWaitIcrIdle(void) { } static void HalApicSendIpi(uint32_t lapicId, uint32_t low) { + if (gX2ApicEnabled) { + __writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low); + return; + } + HalApicWaitIcrIdle(); LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24; LAPIC_REG(LAPIC_ICR_LO) = low; @@ -63,52 +91,70 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) { // IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high). #define IOREDTBL_LO(n) (0x10 + 2*(n)) #define IOREDTBL_HI(n) (0x11 + 2*(n)) -// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> 0x21, ... +// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ... #define IRQ_VECTOR_BASE 0x20 __declspec(dllexport) uint32_t HalGetBspLapicId(void) { return gBspLapicId; } -__declspec(dllexport) void HalApicEnableCurrentCpu(void) { - if (!gLapic) return; - LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF; +__declspec(dllexport) uint8_t HalApicUsingX2Apic(void) { + return gX2ApicEnabled; } +__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId) { + if (gX2ApicEnabled) + return 1u; + return (lapicId < 256u) ? 1u : 0u; +} + +__declspec(dllexport) void HalApicEnableCurrentCpu(void) { + uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE; + + if (gX2ApicEnabled) { + apicBase |= IA32_APIC_BASE_X2APIC; + __writemsr(IA32_APIC_BASE, apicBase); + __writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu); + return; + } + + __writemsr(IA32_APIC_BASE, apicBase); + if (gLapic) + LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF; +} __declspec(dllexport) void HalApicSendPanicIpi(void) { - if (!gLapic) return; + if (!gLapic && !gX2ApicEnabled) return; - // Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed + // Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed // vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still - // enter the panic path. The NMI vector is already installed in the IDT on + // enter the panic path. The NMI vector is already installed in the IDT on // every CPU that has reached kernel-owned state. - HalApicWaitIcrIdle(); - LAPIC_REG(LAPIC_ICR_HI) = 0; - LAPIC_REG(LAPIC_ICR_LO) = (3u << 18) | (4u << 8); // all-excluding-self, NMI - HalApicWaitIcrIdle(); + HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI } __declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) { - if (!gLapic) return; + if (!gLapic && !gX2ApicEnabled) return; HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination } __declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) { - if (!gLapic) return; + if (!gLapic && !gX2ApicEnabled) return; // TLB shootdown must reach CPUs that may be spinning with IF=0 inside // high-IRQL/spinlock regions, so use NMI delivery rather than a fixed - // vector. The NMI stub handles active shootdowns and returns; unexpected + // vector. The NMI stub handles active shootdowns and returns; unexpected // NMIs still go through the normal fault/panic path. HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination } __declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) { - if (!gLapic || !startupVector || lapicId == gBspLapicId) + if ((!gLapic && !gX2ApicEnabled) || !startupVector || lapicId == gBspLapicId) return STATUS_INVALID_PARAMETER; + if (!HalApicCanAddressId(lapicId)) + return STATUS_NOT_SUPPORTED; - // INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12. + // INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12. HalApicSendIpi(lapicId, 0x0000C500u); // INIT, level assert, level-triggered HalApicStallUs(10000); HalApicSendIpi(lapicId, 0x00008500u); // INIT, level deassert @@ -124,7 +170,7 @@ void HalApicRouteIrq(uint8_t irq) { // Full initial setup of the redirection entry. // Sets the vector and clears the mask bit in one write. uint32_t vector = IRQ_VECTOR_BASE + irq; // e.g. IRQ 1 -> vector 0x21 - IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24); // external IRQs target the BSP + IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24); // external IRQs target the BSP IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed } @@ -138,29 +184,37 @@ void HalApicUnmaskIrq(uint8_t irq) { uint32_t lo = IoapicRead(IOREDTBL_LO(irq)); lo &= ~(1 << 16); // clear mask bit lo = (lo & ~0xFF) | (IRQ_VECTOR_BASE + irq); - IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24); + IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24); IoapicWrite(IOREDTBL_LO(irq), lo); } void HalApicSendEoi(void) { + if (gX2ApicEnabled) { + __writemsr(IA32_X2APIC_EOI, 0); + return; + } LAPIC_REG(LAPIC_EOI) = 0; } void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) { + HalApicDetectMode(); + gLapic = (volatile uint32_t *)(gApi.PhysToVirt(lapicPhys)); gIoapic = (volatile uint32_t *)(gApi.PhysToVirt(ioapicPhys)); // The BSP is the CPU executing HAL init. Keep all external/device IRQs // routed to this LAPIC; CPU-local IPIs use the local APIC directly. - gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF; + if (gX2ApicEnabled) + gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID); + else + gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF; - // Enable Local APIC, set spurious vector to 0xFF. - LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF; + HalApicEnableCurrentCpu(); // Mask all IOAPIC lines initially. uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF; for (uint8_t i = 0; i <= maxRedir; i++) { IoapicWrite(IOREDTBL_LO(i), (1 << 16) | (IRQ_VECTOR_BASE + i)); - IoapicWrite(IOREDTBL_HI(i), gBspLapicId << 24); + IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24); } } diff --git a/code/kernel/hal/hal_keyboard.c b/code/kernel/hal/hal_keyboard.c index 2ed4581..9d6890b 100644 --- a/code/kernel/hal/hal_keyboard.c +++ b/code/kernel/hal/hal_keyboard.c @@ -1,204 +1 @@ -// hal_keyboard.c - PS/2 keyboard, interrupt-driven, with live state tracking -#include -#include -#include -#include - -// -- Port addresses ------------------------------------------------------------ -#define PS2_DATA 0x60u -#define PS2_STAT 0x64u -#define PS2_CMD 0x64u - -// -- Ring buffer (for ASCII/typing) ------------------------------------------- -#define KBD_BUF_SIZE 256u -#define KBD_BUF_MASK (KBD_BUF_SIZE - 1u) - -static uint8_t gBuf[KBD_BUF_SIZE]; -static volatile int gHead = 0; -static volatile int gTail = 0; - -// -- Deferred callback queue --------------------------------------------------- -// The hard IRQ path updates the key state and stores completed make codes. -// Optional client callbacks are delivered later from the kernel-queued DPC, -// never directly from DIRQL/ISR context. -#define KBD_DPC_BUF_SIZE 128u -#define KBD_DPC_BUF_MASK (KBD_DPC_BUF_SIZE - 1u) - -static uint8_t gDpcBuf[KBD_DPC_BUF_SIZE]; -static volatile int gDpcHead = 0; -static volatile int gDpcTail = 0; - -// -- Live Key State (for GetAsyncKeyState) ------------------------------------ -// These arrays track whether a key is currently held down. -static volatile uint8_t gKeyState[128]; // Standard scan codes -static volatile uint8_t gKeyStateExt[128]; // Extended (E0) scan codes - -// -- Scan-code set 1 constants ------------------------------------------------- -#define SC1_BREAK_BIT 0x80u -#define SC1_MAKE_MASK 0x7Fu -#define SC1_EXT_PREFIX 0xE0u - -// -- Modifier flags ------------------------------------------------------------ -#define MOD_LSHIFT (1u << 0) -#define MOD_RSHIFT (1u << 1) -#define MOD_LCTRL (1u << 2) -#define MOD_RCTRL (1u << 3) -#define MOD_LALT (1u << 4) -#define MOD_RALT (1u << 5) -#define MOD_CAPS (1u << 6) -#define MOD_NUMLOCK (1u << 7) - -#define MOD_SHIFT (MOD_LSHIFT | MOD_RSHIFT) -#define MOD_CTRL (MOD_LCTRL | MOD_RCTRL) -#define MOD_ALT (MOD_LALT | MOD_RALT) - -static volatile uint32_t gModifiers = 0; -static volatile bool gNextExtended = false; -static void (* volatile gCallback)(uint8_t) = NULL; - -// -- ASCII Translation Tables -------------------------------------------------- -static const char gUnshifted[128] = { - 0, 0, '1','2','3','4','5','6','7','8','9','0','-','=','\b','\t', - 'q','w','e','r','t','y','u','i','o','p','[',']','\n', 0, 'a','s', - 'd','f','g','h','j','k','l',';','\'','`', 0, '\\','z','x','c','v', - 'b','n','m',',','.','/', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.' -}; - -static const char gShifted[128] = { - 0, 0, '!','@','#','$','%','^','&','*','(',')','_','+','\b','\t', - 'Q','W','E','R','T','Y','U','I','O','P','{','}','\n', 0, 'A','S', - 'D','F','G','H','J','K','L',':','"','~', 0, '|','Z','X','C','V', - 'B','N','M','<','>','?', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.' -}; - -// -- Deferred callback delivery ------------------------------------------------ -static void KbdDpc(void) { - for (;;) { - if (gDpcHead == gDpcTail) - break; - - uint8_t make = gDpcBuf[gDpcHead]; - gDpcHead = (gDpcHead + 1) & (int)KBD_DPC_BUF_MASK; - - void (*cb)(uint8_t) = (void (*)(uint8_t))gCallback; - if (cb) - cb(make); - } -} - -// -- ISR: The Engine ----------------------------------------------------------- -void KbdIsr(void) { - uint8_t sc = __inbyte(PS2_DATA); - - if (sc == SC1_EXT_PREFIX) { - gNextExtended = true; - return; - } - - bool extended = gNextExtended; - bool is_break = (sc & SC1_BREAK_BIT) != 0; - uint8_t make = sc & SC1_MAKE_MASK; - gNextExtended = false; - - // Update the live state arrays - if (extended) { - gKeyStateExt[make] = is_break ? 0 : 1; - } else { - gKeyState[make] = is_break ? 0 : 1; - } - - // Update modifiers - if (!extended) { - switch (make) { - case 0x2A: gModifiers = is_break ? (gModifiers & ~MOD_LSHIFT) : (gModifiers | MOD_LSHIFT); break; - case 0x36: gModifiers = is_break ? (gModifiers & ~MOD_RSHIFT) : (gModifiers | MOD_RSHIFT); break; - case 0x1D: gModifiers = is_break ? (gModifiers & ~MOD_LCTRL) : (gModifiers | MOD_LCTRL); break; - case 0x38: gModifiers = is_break ? (gModifiers & ~MOD_LALT) : (gModifiers | MOD_LALT); break; - case 0x3A: if (!is_break) gModifiers ^= MOD_CAPS; break; - case 0x45: if (!is_break) gModifiers ^= MOD_NUMLOCK; break; - } - } else { - switch (make) { - case 0x1D: gModifiers = is_break ? (gModifiers & ~MOD_RCTRL) : (gModifiers | MOD_RCTRL); break; - case 0x38: gModifiers = is_break ? (gModifiers & ~MOD_RALT) : (gModifiers | MOD_RALT); break; - } - } - - // Buffer management (only for 'make' codes) - if (!is_break) { - int next = (gTail + 1) & (int)KBD_BUF_MASK; - if (next != gHead) { - gBuf[gTail] = make; - gTail = next; - } - - // Queue optional callback delivery to the DPC path. Keep a separate - // callback queue so the public ReadScancode ring remains independent. - if (gCallback) { - int dpcNext = (gDpcTail + 1) & (int)KBD_DPC_BUF_MASK; - if (dpcNext != gDpcHead) { - gDpcBuf[gDpcTail] = make; - gDpcTail = dpcNext; - } - } - } -} - -// -- Public API ---------------------------------------------------------------- - -// Use this inside your GetAsyncKeyState implementation! -bool KbdIsKeyDown(uint8_t scancode, bool extended) { - if (extended) return gKeyStateExt[scancode & 0x7F] != 0; - return gKeyState[scancode & 0x7F] != 0; -} - -static uint8_t KbdPoll(void) { - return (gHead == gTail) ? 0 : gBuf[gHead]; -} - -static uint8_t KbdReadScancode(void) { - while (gHead == gTail) { _mm_pause(); } - uint8_t sc = gBuf[gHead]; - gHead = (gHead + 1) & (int)KBD_BUF_MASK; - return sc; -} - -static char KbdReadChar(void) { - uint8_t make = KbdReadScancode(); - if (make >= 128) return 0; - - uint32_t mods = gModifiers; - bool shifted = (mods & MOD_SHIFT) != 0; - - if (mods & MOD_CAPS) { - char base = gUnshifted[make]; - if (base >= 'a' && base <= 'z') shifted = !shifted; - } - return shifted ? gShifted[make] : gUnshifted[make]; -} - -static uint32_t KbdGetModifiers(void) { - return gModifiers; -} - -static void KbdSetCallback(void (*cb)(uint8_t scancode)) { - gCallback = cb; -} - -void HalKeyboardInitialize(HAL_KEYBOARD *out) { - while (__inbyte(PS2_STAT) & 2); - __outbyte(PS2_CMD, 0xA8); - while (__inbyte(PS2_STAT) & 1) { __inbyte(PS2_DATA); } - - out->Poll = KbdPoll; - out->ReadScancode = KbdReadScancode; - out->ReadChar = KbdReadChar; - out->GetModifiers = KbdGetModifiers; - out->SetCallback = KbdSetCallback; - out->IsKeyDown = KbdIsKeyDown; - out->Isr = KbdIsr; - out->Dpc = KbdDpc; - out->IrqNumber = 1; -} +// hal_keyboard.c - input moved out of HAL into kernel input core and ps2.sys. diff --git a/code/kernel/hal/hal_main.c b/code/kernel/hal/hal_main.c index 5d2dc8c..ee89a26 100644 --- a/code/kernel/hal/hal_main.c +++ b/code/kernel/hal/hal_main.c @@ -5,12 +5,11 @@ #include #include #include +#pragma intrinsic(_disable) #include -// Declared in other hal_*.c files: -void HalScreenInitialize(uintptr_t fb, int w, int h, int stride, HAL_SCREEN* out); -void HalKeyboardInitialize(HAL_KEYBOARD* out); -void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH); +void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride, + HAL_SCREEN* out); KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out); void HalPciInitialize(HAL_PCI* out); extern void HalSetDebugEnabled(int v); @@ -242,7 +241,7 @@ static void ParseMadt(MADT* madt) { uint32_t lapicId = *(uint32_t *)(p + 4); uint32_t flags = *(uint32_t *)(p + 8); uint32_t uid = *(uint32_t *)(p + 12); - if ((flags & 1u) && lapicId < 256u) + if (flags & 1u) HalAddCpuRecord(uid, lapicId, flags); } @@ -395,7 +394,7 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) { debugEnabled = initParams->Debug; // Screen first so DbgPrint works immediately. - HalScreenInitialize(initParams->FramebufferBase + initParams->PhysicalWindowBase, + HalScreenInitialize(initParams->AllocatePool, initParams->FramebufferBase + initParams->PhysicalWindowBase, (int)initParams->FramebufferWidth, (int)initParams->FramebufferHeight, (int)initParams->FramebufferStride, @@ -412,10 +411,9 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) { gLapicPhys, gIoapicPhys, gHpetPhys); HalApicInitialize(gLapicPhys, gIoapicPhys); - HalKeyboardInitialize(&gApi.Keyboard); - HalMouseInitialize(&gApi.Mouse, - (int)initParams->FramebufferWidth, - (int)initParams->FramebufferHeight); + HalDbgPrint("HAL: local APIC mode=%s BSP_ID=%u\n", + HalApicUsingX2Apic() ? "x2APIC" : "xAPIC", + HalGetBspLapicId()); KSTATUS ts = HalTimerInitialize(gLapicPhys, gHpetPhys, &gApi.Timer); if (!KSUCCESS(ts)) HalDbgPrint("HAL: HPET unavailable, using fallback\n"); diff --git a/code/kernel/hal/hal_mouse.c b/code/kernel/hal/hal_mouse.c index 71f2a35..6f60f47 100644 --- a/code/kernel/hal/hal_mouse.c +++ b/code/kernel/hal/hal_mouse.c @@ -1,305 +1 @@ -// hal_mouse.c - PS/2 auxiliary mouse driver -// -// Sits on IRQ 12, sharing the 8042 controller with the keyboard (IRQ 1). -// hal_keyboard.c has already issued command 0xA8 (enable aux port) by the -// time HalMouseInitialize is called, so we only need to: -// 1. Enable aux interrupts in the 8042 command byte (bit 1, clears bit 5). -// 2. Send the standard mouse init sequence (reset → defaults → enable). -// 3. Wire up the ISR and expose the HAL_MOUSE vtable. -// -// Packet format - standard 3-byte PS/2 mouse (no scroll wheel): -// Byte 0: [YOV][XOV][YS][XS][1][M][R][L] -// Byte 1: delta-X unsigned magnitude (sign in byte-0 bit 4) -// Byte 2: delta-Y unsigned magnitude (sign in byte-0 bit 5); PS/2 Y+ = up -// -// The ISR accumulates the 3-byte packet, converts it to a screen-space delta -// (Y-axis flipped), and queues the completed packet to the HAL's kernel-queued -// bottom half. Cursor drawing, public event enqueue, and optional callbacks -// run from that bottom half instead of hard IRQ context. -// -// Threading model -// --------------- -// Public events are produced by the mouse DPC and consumed by callers. -// Raw completed packets are produced by the ISR and consumed by the same DPC. -// On SMP, add acquire/release barriers around the ring head/tail pairs. - -#include -#include -#include -#include -#include - -// -- PS/2 port addresses ------------------------------------------------------- -#define PS2_DATA 0x60u -#define PS2_STAT 0x64u -#define PS2_CMD 0x64u - -// -- Ring buffer --------------------------------------------------------------- -#define MOUSE_BUF_SIZE 64u -#define MOUSE_BUF_MASK (MOUSE_BUF_SIZE - 1u) - -typedef struct { - int dx, dy; - uint8_t buttons; // bit0=left, bit1=right, bit2=middle -} MOUSE_EVENT; - -static MOUSE_EVENT gBuf[MOUSE_BUF_SIZE]; -static volatile int gHead = 0; // public consumer read pointer -static volatile int gTail = 0; // public producer write pointer (updated by DPC) - -// Completed hardware packets waiting for the DPC bottom half. -static MOUSE_EVENT gDpcBuf[MOUSE_BUF_SIZE]; -static volatile int gDpcHead = 0; -static volatile int gDpcTail = 0; - -// -- Absolute cursor position & screen bounds ---------------------------------- -static volatile int gMouseX = 0; -static volatile int gMouseY = 0; -static volatile int gBoundsW = 1920; -static volatile int gBoundsH = 1080; - -// -- 3-byte packet assembly ---------------------------------------------------- -static volatile uint8_t gPacket[3]; -static volatile int gPacketIdx = 0; // 0, 1, or 2 - -// -- Optional callback (called from mouse DPC) --------------------------------- -static void (* volatile gCallback)(int dx, int dy, uint8_t buttons) = NULL; - -// -- Cursor helpers provided by hal_screen.c ----------------------------------- -// Declared here; defined in hal_screen.c. -extern void HalScreenDrawCursor(int x, int y); -extern void HalScreenEraseCursor(void); - -// -- 8042 low-level helpers ---------------------------------------------------- -// Wait until the 8042's input buffer is empty (bit 1 of status = 0) so we -// can safely write to PS2_DATA or PS2_CMD. -static void Ps2WaitInput(void) { - int timeout = 100000; - while ((__inbyte(PS2_STAT) & 0x02u) && --timeout) { /* spin */ } -} - -// Wait until the 8042's output buffer has data (bit 0 of status = 1). -static void Ps2WaitOutput(void) { - int timeout = 5000000; - while (!(__inbyte(PS2_STAT) & 0x01u) && --timeout) { /* spin */ } -} - -// Reads from PS2_DATA until the output buffer is empty -static void Ps2FlushOutput(void) { - int timeout = 100000; - while ((__inbyte(PS2_STAT) & 0x01u) && --timeout) { - __inbyte(PS2_DATA); - } -} - -// Route the next write to the auxiliary (mouse) device rather than the keyboard. -static void MouseSendByte(uint8_t byte) { - Ps2WaitInput(); - __outbyte(PS2_CMD, 0xD4u); // "write to mouse" prefix - Ps2WaitInput(); - __outbyte(PS2_DATA, byte); -} - -// Read one byte from the PS/2 output buffer (shared by keyboard and mouse). -static uint8_t MouseRecvByte(void) { - Ps2WaitOutput(); - return __inbyte(PS2_DATA); -} - -// -- Deferred mouse bottom half ------------------------------------------------- -static void MouseDpc(void) { - for (;;) { - if (gDpcHead == gDpcTail) - break; - - MOUSE_EVENT ev = gDpcBuf[gDpcHead]; - gDpcHead = (gDpcHead + 1) & (int)MOUSE_BUF_MASK; - - // Update absolute position and redraw the cursor outside DIRQL. - HalScreenEraseCursor(); - - int nx = gMouseX + ev.dx; - int ny = gMouseY + ev.dy; - if (nx < 0) nx = 0; - if (nx >= gBoundsW) nx = gBoundsW - 1; - if (ny < 0) ny = 0; - if (ny >= gBoundsH) ny = gBoundsH - 1; - gMouseX = nx; - gMouseY = ny; - - HalScreenDrawCursor(nx, ny); - - // Publish event to the normal consumer ring. - int next = (gTail + 1) & (int)MOUSE_BUF_MASK; - if (next != gHead) { - gBuf[gTail] = ev; - gTail = next; - } - - void (*cb)(int, int, uint8_t) = (void (*)(int, int, uint8_t))gCallback; - if (cb) - cb(ev.dx, ev.dy, ev.buttons); - } -} - -// -- Interrupt service routine -------------------------------------------------- -// Called with interrupts disabled by KeIrqDispatch (IRQ 12). -// The dispatcher sends the APIC EOI after this function returns. -static void MouseIsr(void) { - uint8_t data = __inbyte(PS2_DATA); - - // Synchronisation guard: the first byte of every packet has bit 3 set. - // If we lose sync, wait for the next valid first byte before assembling. - if (gPacketIdx == 0) { - if (!(data & 0x08u)) return; // not a valid first byte - discard - } - - gPacket[gPacketIdx++] = data; - if (gPacketIdx < 3) return; // packet not yet complete - gPacketIdx = 0; - - // -- Decode flags ---------------------------------------------------------- - uint8_t flags = gPacket[0]; - - // Overflow bits: if either is set the movement data is meaningless. - if (flags & 0xC0u) return; - - uint8_t buttons = flags & 0x07u; - - // Convert unsigned magnitudes + sign bits to signed 9-bit values. - int dx = (int)gPacket[1] - ((flags & 0x10u) ? 256 : 0); - int dy = (int)gPacket[2] - ((flags & 0x20u) ? 256 : 0); - dy = -dy; // PS/2 Y+ = up; screen Y+ = down, so invert. - - // -- Queue completed packet to the DPC bottom half ------------------------- - int next = (gDpcTail + 1) & (int)MOUSE_BUF_MASK; - if (next != gDpcHead) { - gDpcBuf[gDpcTail].dx = dx; - gDpcBuf[gDpcTail].dy = dy; - gDpcBuf[gDpcTail].buttons = buttons; - gDpcTail = next; - } -} - -// -- Consumer API -------------------------------------------------------------- -static bool MousePoll(void) { - return gHead != gTail; -} - -static bool MouseRead(int* dx, int* dy, uint8_t* buttons) { - if (gHead == gTail) return false; - *dx = gBuf[gHead].dx; - *dy = gBuf[gHead].dy; - *buttons = gBuf[gHead].buttons; - gHead = (gHead + 1) & (int)MOUSE_BUF_MASK; - return true; -} - -static void MouseGetPosition(int* x, int* y) { - *x = gMouseX; - *y = gMouseY; -} - -static void MouseSetPosition(int x, int y) { - gMouseX = x; - gMouseY = y; -} - -static void MouseSetBounds(int w, int h) { - gBoundsW = w; - gBoundsH = h; -} - -static void MouseSetCallback(void (*cb)(int dx, int dy, uint8_t buttons)) { - gCallback = cb; -} - -// -- Initializer --------------------------------------------------------------- -// Called from HalInitialize in hal_main.c after the screen is ready. -// hal_keyboard.c has already sent 0xA8 (enable aux port) to port 0x64. -void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH) { - gBoundsW = screenW; - gBoundsH = screenH; - gMouseX = screenW / 2; - gMouseY = screenH / 2; - - // ----------------------------------------------------------------- - // Step 0: Deafen the controller and flush - // ----------------------------------------------------------------- - Ps2WaitInput(); - __outbyte(PS2_CMD, 0xA7u); // Disable Aux port (ignores mouse) - Ps2WaitInput(); - __outbyte(PS2_CMD, 0xADu); // Disable Main port (ignores keyboard) - - // Now flush the buffer. Because the ports are disabled, we guarantee - // no new data can sneak in behind our backs. - Ps2FlushOutput(); - - // ----------------------------------------------------------------- - // Step 1: Safely configure the 8042 Command Byte - // ----------------------------------------------------------------- - Ps2WaitInput(); - __outbyte(PS2_CMD, 0x20u); // read current command byte - Ps2WaitOutput(); - uint8_t cmd = __inbyte(PS2_DATA); - - cmd |= 0x02u; // enable aux interrupt - cmd &= ~0x20u; // clear "disable aux clock" - - Ps2WaitInput(); - __outbyte(PS2_CMD, 0x60u); // write command byte - Ps2WaitInput(); - __outbyte(PS2_DATA, cmd); - - // Re-enable both ports now that config is safely written - Ps2WaitInput(); - __outbyte(PS2_CMD, 0xA8u); // Enable Aux port - Ps2WaitInput(); - __outbyte(PS2_CMD, 0xAEu); // Enable Main port - - // ----------------------------------------------------------------- - // Step 2: Reset mouse (0xFF). - // ----------------------------------------------------------------- - MouseSendByte(0xFFu); // reset - - // The BIOS or an earlier phase might have left a movement packet - // partially in flight right as we re-enabled the port. - // We loop and discard anything that isn't the 0xFA ACK. - uint8_t ack; - int retries = 10; - do { - ack = MouseRecvByte(); - } while (ack != 0xFAu && --retries > 0); - - // The output timeout covers the physical hardware self-test latency. - MouseRecvByte(); // 0xAA - BAT passed - MouseRecvByte(); // 0x00 - device ID - - // ----------------------------------------------------------------- - // Step 3: Set default parameters (sampling rate 100, 4 counts/mm). - // Mouse replies: ACK. - // ----------------------------------------------------------------- - MouseSendByte(0xF6u); // set defaults - MouseRecvByte(); // ACK - - // ----------------------------------------------------------------- - // Step 4: Enable data reporting. - // Mouse will now send 3-byte packets on every movement/button event. - // ----------------------------------------------------------------- - MouseSendByte(0xF4u); // enable data reporting - MouseRecvByte(); // ACK - - // Draw the initial cursor at the centre of the screen. - HalScreenDrawCursor(gMouseX, gMouseY); - - // Populate vtable. - out->Poll = MousePoll; - out->Read = MouseRead; - out->SetCallback = MouseSetCallback; - out->GetPosition = MouseGetPosition; - out->SetPosition = MouseSetPosition; - out->SetBounds = MouseSetBounds; - out->Isr = MouseIsr; - out->Dpc = MouseDpc; - out->IrqNumber = 12; -} +// hal_mouse.c - input moved out of HAL into kernel input core and ps2.sys. diff --git a/code/kernel/hal/hal_pci.c b/code/kernel/hal/hal_pci.c index 7b55ede..c6d3198 100644 --- a/code/kernel/hal/hal_pci.c +++ b/code/kernel/hal/hal_pci.c @@ -1,10 +1,27 @@ // hal_pci.c - PCI configuration space via Mechanism 1 port I/O #include #include +#include +#include #define PCI_CONFIG_ADDR 0xCF8 #define PCI_CONFIG_DATA 0xCFC +#define PCI_REG_COMMAND_STATUS 0x04u +#define PCI_REG_CAP_PTR 0x34u +#define PCI_STATUS_CAP_LIST (1u << 20) +#define PCI_COMMAND_INTX_DISABLE (1u << 10) + +#define PCI_CAP_ID_MSI 0x05u +#define PCI_CAP_ID_MSIX 0x11u +#define PCI_MSI_CTRL_ENABLE (1u << 0) +#define PCI_MSI_CTRL_MME_MASK (7u << 4) +#define PCI_MSI_CTRL_64BIT (1u << 7) +#define PCI_MSI_CTRL_PVM (1u << 8) + +#define PCI_MSI_ADDRESS_BASE 0xFEE00000u +#define PCI_IRQ_VECTOR_BASE 0x20u + static uint32_t MakeAddr(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) { return 0x80000000UL | ((uint32_t)bus << 16) @@ -24,6 +41,25 @@ static void PciWrite32(uint8_t bus, uint8_t dev, uint8_t fn, __outdword(PCI_CONFIG_DATA, val); } +static uint8_t PciRead8(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) { + uint32_t v = PciRead32(bus, dev, fn, reg); + return (uint8_t)(v >> ((reg & 3u) * 8u)); +} + +static uint16_t PciRead16(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) { + uint32_t v = PciRead32(bus, dev, fn, reg); + return (uint16_t)(v >> ((reg & 2u) * 8u)); +} + +static void PciWrite16(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg, uint16_t value) { + uint8_t aligned = reg & ~3u; + uint32_t shift = (reg & 2u) * 8u; + uint32_t old = PciRead32(bus, dev, fn, aligned); + old &= ~(0xFFFFu << shift); + old |= ((uint32_t)value << shift); + PciWrite32(bus, dev, fn, aligned, old); +} + static bool PciFindDevice(uint16_t vendor, uint16_t device, uint8_t* outBus, uint8_t* outDev, uint8_t* outFn) { for (int bus = 0; bus < 256; bus++) @@ -61,10 +97,82 @@ static uint8_t PciGetIrqLine(uint8_t bus, uint8_t dev, uint8_t fn) { return (uint8_t)(val & 0xFF); // 0xFF = not connected } +static uint8_t PciFindCapability(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t capId) { + uint32_t status = PciRead32(bus, dev, fn, PCI_REG_COMMAND_STATUS); + if ((status & PCI_STATUS_CAP_LIST) == 0) + return 0; + + uint8_t cap = PciRead8(bus, dev, fn, PCI_REG_CAP_PTR) & 0xFCu; + for (uint32_t guard = 0; cap && guard < 48u; guard++) { + uint32_t hdr = PciRead32(bus, dev, fn, cap); + uint8_t id = (uint8_t)(hdr & 0xFFu); + uint8_t next = (uint8_t)((hdr >> 8) & 0xFCu); + if (id == capId) + return cap; + cap = next; + } + + return 0; +} + +static KSTATUS PciEnableMsi(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t irqSlot) { + if (irqSlot >= 24u) + return STATUS_INVALID_PARAMETER; + + uint8_t cap = PciFindCapability(bus, dev, fn, PCI_CAP_ID_MSI); + if (!cap) { + HalDbgPrint("PCI %u:%u.%u: MSI capability not present\n", bus, dev, fn); + return STATUS_NOT_SUPPORTED; + } + + uint32_t bspLapic = HalGetBspLapicId(); + if (!HalApicUsingX2Apic() && bspLapic >= 256u) + return STATUS_NOT_SUPPORTED; + if (HalApicUsingX2Apic() && bspLapic >= 256u) { + // Extended x2APIC MSI destinations need interrupt-remapping awareness. + // Keep the first MSI implementation xAPIC-compatible and explicit. + HalDbgPrint("PCI %u:%u.%u: MSI x2APIC destination %u not supported yet\n", + bus, dev, fn, bspLapic); + return STATUS_NOT_SUPPORTED; + } + + uint16_t control = PciRead16(bus, dev, fn, (uint8_t)(cap + 2u)); + bool is64 = (control & PCI_MSI_CTRL_64BIT) != 0; + bool pvm = (control & PCI_MSI_CTRL_PVM) != 0; + uint8_t vector = (uint8_t)(PCI_IRQ_VECTOR_BASE + irqSlot); + uint32_t address = PCI_MSI_ADDRESS_BASE | ((bspLapic & 0xFFu) << 12); + uint16_t data = vector; + + PciWrite32(bus, dev, fn, (uint8_t)(cap + 4u), address); + if (is64) { + PciWrite32(bus, dev, fn, (uint8_t)(cap + 8u), 0); + PciWrite16(bus, dev, fn, (uint8_t)(cap + 12u), data); + if (pvm) + PciWrite32(bus, dev, fn, (uint8_t)(cap + 16u), 0); + } else { + PciWrite16(bus, dev, fn, (uint8_t)(cap + 8u), data); + if (pvm) + PciWrite32(bus, dev, fn, (uint8_t)(cap + 12u), 0); + } + + control &= (uint16_t)~PCI_MSI_CTRL_MME_MASK; // enable exactly one message + control |= PCI_MSI_CTRL_ENABLE; + PciWrite16(bus, dev, fn, (uint8_t)(cap + 2u), control); + + uint32_t cmd = PciRead32(bus, dev, fn, PCI_REG_COMMAND_STATUS); + cmd |= PCI_COMMAND_INTX_DISABLE; + PciWrite32(bus, dev, fn, PCI_REG_COMMAND_STATUS, cmd); + + HalDbgPrint("PCI %u:%u.%u: MSI enabled cap=0x%02x vector=0x%02x irqslot=%u dest=%u %s\n", + bus, dev, fn, cap, vector, irqSlot, bspLapic, is64 ? "64-bit" : "32-bit"); + return STATUS_SUCCESS; +} + void HalPciInitialize(HAL_PCI* out) { out->Read32 = PciRead32; out->Write32 = PciWrite32; out->FindDevice = PciFindDevice; out->FindClass = PciFindClass; out->GetIrqLine = PciGetIrqLine; + out->EnableMsi = PciEnableMsi; } diff --git a/code/kernel/hal/hal_screen.c b/code/kernel/hal/hal_screen.c index b2c79ce..1a83193 100644 --- a/code/kernel/hal/hal_screen.c +++ b/code/kernel/hal/hal_screen.c @@ -7,8 +7,10 @@ #include #include #include +#include static uint32_t* gFb = (uint32_t *)0; +static uint32_t* gBackbuffer = (uint32_t *)0; static int gWidth = 0; static int gHeight = 0; static int gStride = 0; // pixels per row @@ -20,14 +22,39 @@ static int gCursorX = 0, gCursorY = 0; // --------------------------------------------------------------------------- // 8x16 VGA font - printable ASCII only (glyph 0 = space = 0x20) // --------------------------------------------------------------------------- -#include // see font8x16.h below +#include // see font8x16.h // --------------------------------------------------------------------------- // Internals // --------------------------------------------------------------------------- +static inline uint32_t* ScreenBackbufferPixel(int x, int y) { + return gBackbuffer ? (gBackbuffer + y * gStride + x) : (gFb + y * gStride + x); +} + static inline void PutPixelRaw(int x, int y, uint32_t color) { - if ((unsigned)x < (unsigned)gWidth && (unsigned)y < (unsigned)gHeight) - gFb[y * gStride + x] = color; + if ((unsigned)x >= (unsigned)gWidth || (unsigned)y >= (unsigned)gHeight) + return; + + ScreenBackbufferPixel(x, y)[0] = color; + gFb[y * gStride + x] = color; +} + +static void ScreenFlushRows(int startY, int rows) { + if (!gBackbuffer || rows <= 0) + return; + + if (startY < 0) { + rows += startY; + startY = 0; + } + if (startY >= gHeight || rows <= 0) + return; + if (startY + rows > gHeight) + rows = gHeight - startY; + + size_t rowBytes = (size_t)gStride * sizeof(uint32_t); + memcpy(gFb + startY * gStride, gBackbuffer + startY * gStride, + (size_t)rows * rowBytes); } // --------------------------------------------------------------------------- @@ -36,7 +63,14 @@ static inline void PutPixelRaw(int x, int y, uint32_t color) { static void ScreenClear(uint32_t color) { for (int y = 0; y < gHeight; y++) for (int x = 0; x < gWidth; x++) - gFb[y * gStride + x] = color; + ScreenBackbufferPixel(x, y)[0] = color; + ScreenFlushRows(0, gHeight); + if (!gBackbuffer) { + // ScreenFlushRows is a no-op without a backbuffer. + for (int y = 0; y < gHeight; y++) + for (int x = 0; x < gWidth; x++) + gFb[y * gStride + x] = color; + } gCursorX = 0; gCursorY = 0; } @@ -66,25 +100,13 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) { // Calculate bytes to copy per row size_t bytesPerRow = (size_t)drawW * sizeof(uint32_t); - // If the width we are drawing exactly matches the physical screen stride, - // AND it matches the original source image width, memory is perfectly contiguous. - if (drawW == gStride && drawW == w) { - uint32_t* dst = gFb + (startY * gStride) + startX; - const uint32_t* src = pixels + (srcOffsetY * w) + srcOffsetX; - - memcpy(dst, src, bytesPerRow * drawH); - return; - } - - // Copy row-by-row for (int row = 0; row < drawH; row++) { - // Destination in the actual framebuffer (accounting for stride) - uint32_t* dst = gFb + ((startY + row) * gStride) + startX; - - // Source from the provided pixel buffer + uint32_t* shadow = ScreenBackbufferPixel(startX, startY + row); + uint32_t* fb = gFb + ((startY + row) * gStride) + startX; const uint32_t* src = pixels + ((srcOffsetY + row) * w) + srcOffsetX; - - memcpy(dst, src, bytesPerRow); + + memcpy(shadow, src, bytesPerRow); + memcpy(fb, src, bytesPerRow); } } @@ -92,12 +114,17 @@ static void ScreenGetSize(int* w, int* h) { *w = gWidth; *h = gHeight; } static void ScreenScroll(int lines) { if (lines <= 0 || lines >= gHeight) { ScreenClear(0); return; } - int rowBytes = gStride * sizeof(uint32_t); - memmove(gFb, gFb + lines * gStride, + + size_t rowBytes = (size_t)gStride * sizeof(uint32_t); + uint32_t* shadow = gBackbuffer ? gBackbuffer : gFb; + + memmove(shadow, shadow + lines * gStride, (size_t)(gHeight - lines) * rowBytes); // Clear the revealed rows at the bottom. - memset(gFb + (gHeight - lines) * gStride, 0, + memset(shadow + (gHeight - lines) * gStride, 0, (size_t)lines * rowBytes); + + ScreenFlushRows(0, gHeight); } static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) { @@ -195,7 +222,7 @@ static void CursorSaveBg(int x, int y) { int fx = x + col; uint32_t px = 0; if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) - px = gFb[fy * gStride + fx]; + px = ScreenBackbufferPixel(fx, fy)[0]; gCursorSavedBg[row * CURSOR_W + col] = px; } } @@ -207,8 +234,12 @@ static void CursorRestoreBg(int x, int y) { int fy = y + row; for (int col = 0; col < CURSOR_W; col++) { int fx = x + col; - if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) - gFb[fy * gStride + fx] = gCursorSavedBg[row * CURSOR_W + col]; + if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) { + uint32_t px = gBackbuffer + ? ScreenBackbufferPixel(fx, fy)[0] + : gCursorSavedBg[row * CURSOR_W + col]; + gFb[fy * gStride + fx] = px; + } } } } @@ -248,8 +279,8 @@ void HalScreenDrawCursor(int x, int y) { if ((src >> 24) == 0) continue; // fully transparent, skip int fx = x + col; if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) { - uint32_t* dst = &gFb[fy * gStride + fx]; - *dst = BlendPixel(*dst, src); + uint32_t dst = ScreenBackbufferPixel(fx, fy)[0]; + gFb[fy * gStride + fx] = BlendPixel(dst, src); } } } @@ -266,12 +297,21 @@ void HalScreenEraseCursor(void) { // --------------------------------------------------------------------------- // Populate the HAL_SCREEN vtable // --------------------------------------------------------------------------- -void HalScreenInitialize(uintptr_t fb, int w, int h, int stride, - HAL_SCREEN* out) { +void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride, + HAL_SCREEN* out) { gFb = (uint32_t *)fb; gWidth = w; gHeight = h; gStride = stride; + + size_t backbufferPixels = (size_t)gStride * (size_t)gHeight; + size_t backbufferBytes = backbufferPixels * sizeof(uint32_t); + if (gStride > 0 && gHeight > 0 && + backbufferPixels / (size_t)gStride == (size_t)gHeight && + backbufferBytes / sizeof(uint32_t) == backbufferPixels) { + gBackbuffer = (uint32_t*)AllocatePool(backbufferBytes); + } + out->Clear = ScreenClear; out->PutPixel = ScreenPutPixel; out->Blit = ScreenBlit; diff --git a/code/kernel/hal/hal_timer.c b/code/kernel/hal/hal_timer.c index d10f7bb..5cb7d8e 100644 --- a/code/kernel/hal/hal_timer.c +++ b/code/kernel/hal/hal_timer.c @@ -51,7 +51,7 @@ static uint64_t QueryTicks(void) { } // Fallback: combine the coarse periodic tick with the current LAPIC - // countdown. Read the coarse tick twice so an interrupt between gTicks and + // countdown. Read the coarse tick twice so an interrupt between gTicks and // LAPIC_TIMER_CUR cannot make time jump backwards by almost one quantum. uint64_t baseTicks1; uint64_t baseTicks2; @@ -177,7 +177,7 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) { KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out) { gLapic = (volatile uint32_t *)gApi.PhysToVirt(lapicPhys); - HalDbgPrint("HAL: lapicPhys=0x%llX, lapicVirt=0x%llX, hpetPhys=0x%llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys); + HalDbgPrint("HAL: lapicPhys=0x%016llX, lapicVirt=0x%016llX, hpetPhys=0x%016llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys); uint32_t ticksPerMs = 0; if (hpetPhys) { diff --git a/code/kernel/include/kernel/hal/hal_apic.h b/code/kernel/include/kernel/hal/hal_apic.h index 81d9d13..e68c7c5 100644 --- a/code/kernel/include/kernel/hal/hal_apic.h +++ b/code/kernel/include/kernel/hal/hal_apic.h @@ -28,6 +28,8 @@ typedef struct HAL_CPU_TOPOLOGY { __declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out); __declspec(dllexport) uint32_t HalGetBspLapicId(void); +__declspec(dllexport) uint8_t HalApicUsingX2Apic(void); +__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId); __declspec(dllexport) void HalApicEnableCurrentCpu(void); __declspec(dllexport) void HalApicSendPanicIpi(void); __declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId); diff --git a/code/kernel/include/kernel/io_file.h b/code/kernel/include/kernel/io_file.h index cb180d7..b6ced40 100644 --- a/code/kernel/include/kernel/io_file.h +++ b/code/kernel/include/kernel/io_file.h @@ -14,9 +14,9 @@ typedef SYS_DIR_ENTRY KDIR_ENTRY; #define KDIR_NAME_MAX SYS_DIR_NAME_MAX -// Native asynchronous filesystem completion. Filesystems that provide +// Native asynchronous filesystem completion. Filesystems that provide // ReadAsync/WriteAsync accept the operation, return STATUS_PENDING, and later -// call Complete(CompleteContext, status, information). Filesystems without +// call Complete(CompleteContext, status, information). Filesystems without // these callbacks fall back to the generic VFS async worker. typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext, KSTATUS status, diff --git a/code/kernel/include/kernel/ke_apc.h b/code/kernel/include/kernel/ke_apc.h index d6d9d91..5c7c626 100644 --- a/code/kernel/include/kernel/ke_apc.h +++ b/code/kernel/include/kernel/ke_apc.h @@ -84,7 +84,7 @@ __declspec(dllexport) void KeEnterCriticalRegion(void); __declspec(dllexport) void KeLeaveCriticalRegion(void); __declspec(dllexport) bool KeAreKernelApcsDisabled(void); -// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL, +// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL, // after context switches, and after waits complete. void KiDeliverKernelApcs(void); diff --git a/code/kernel/include/kernel/ke_debug.h b/code/kernel/include/kernel/ke_debug.h index 5d87d80..d83b0ff 100644 --- a/code/kernel/include/kernel/ke_debug.h +++ b/code/kernel/include/kernel/ke_debug.h @@ -26,8 +26,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void); // Fault dispatch - called from the IDT fault stubs (ke_syscall.asm). // // The old scalar entry only reported RIP/error/CR2, which is not enough for -// debugging null indirect calls. The frame entry snapshots user GPRs plus the -// CPU exception frame, including user RSP. For a CALL through a null function +// debugging null indirect calls. The frame entry snapshots user GPRs plus the +// CPU exception frame, including user RSP. For a CALL through a null function // pointer, RIP is 0 and [RSP] is usually the return address after the bad call; // that is the useful caller site. // --------------------------------------------------------------------------- diff --git a/code/kernel/include/kernel/ke_dpc.h b/code/kernel/include/kernel/ke_dpc.h index ee4b75e..a1530eb 100644 --- a/code/kernel/include/kernel/ke_dpc.h +++ b/code/kernel/include/kernel/ke_dpc.h @@ -1,9 +1,9 @@ #pragma once // ke_dpc.h - deferred procedure calls // -// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC +// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC // after acknowledging/capturing device state; the DPC then runs later at -// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not +// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not // wait, block, or call PASSIVE_LEVEL-only code. #include @@ -34,11 +34,11 @@ __declspec(dllexport) void KeInitializeDpc(KDPC* dpc, KDEFERRED_ROUTINE routine, void* context); -// Queue a DPC if it is not already queued. Returns false if the DPC was +// Queue a DPC if it is not already queued. Returns false if the DPC was // already pending or the arguments are invalid. __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc, void* systemArgument1, void* systemArgument2); -// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit. +// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit. void KiDrainDpcQueue(void); diff --git a/code/kernel/include/kernel/ke_input.h b/code/kernel/include/kernel/ke_input.h new file mode 100644 index 0000000..c2b318b --- /dev/null +++ b/code/kernel/include/kernel/ke_input.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include +#include +#include +#include + +void KeInputInitialize(const HAL_API* hal); + +__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId); +__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event); +__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event); + +INPUT_KEY KeInputKbPoll(void); +bool KeInputIsKeyDown(INPUT_KEY key); +INPUT_KEY KeInputReadKey(void); +char KeInputReadChar(void); +uint32_t KeInputGetModifiers(void); + +bool KeInputMousePoll(void); +bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons); +void KeInputMouseGetPosition(int* x, int* y); +void KeInputMouseSetPosition(int x, int y); +void KeInputMouseSetBounds(int w, int h); diff --git a/code/kernel/include/kernel/ke_irp.h b/code/kernel/include/kernel/ke_irp.h index bd6c993..a9288b6 100644 --- a/code/kernel/include/kernel/ke_irp.h +++ b/code/kernel/include/kernel/ke_irp.h @@ -55,7 +55,7 @@ typedef struct IRP { uint32_t Flags; // IRP_FLAG_* uint64_t Lba; uint32_t SectorCount; - void *Buffer; // kernel VA; valid until completion + void* Buffer; // kernel VA; valid until completion // ---- Completion (caller fills before submit; driver writes Status) ---- @@ -65,17 +65,17 @@ typedef struct IRP { // Forward-declared pointer: callers that create events include ke_event.h. struct EVENT *CompletionEvent; - // Optional kernel APC. If set, IoCompleteRequest queues it to its + // Optional kernel APC. If set, IoCompleteRequest queues it to its // target thread with arg1=irp and arg2=(void*)(uintptr_t)status. struct KAPC *CompletionApc; IRP_COMPLETION_FN CompletionFn; - void *CompletionCtx; + void* CompletionCtx; // ---- Driver-internal -------------------------------------------------- uint8_t HwSlot; - void *DriverCtx; + void* DriverCtx; } IRP; // ------------------------------------------------------------------------- diff --git a/code/kernel/include/kernel/ke_irq.h b/code/kernel/include/kernel/ke_irq.h index 98e3c4d..77709e2 100644 --- a/code/kernel/include/kernel/ke_irq.h +++ b/code/kernel/include/kernel/ke_irq.h @@ -1,13 +1,15 @@ #pragma once #include #include +#include void KeIrqInitialize(const HAL_API *hal); +__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq); __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)); void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm -// Low-level interrupt entry/exit hooks. These are called by the common IRQ -// assembly stub around KeIrqDispatch. The exit hook is where deferred +// Low-level interrupt entry/exit hooks. These are called by the common IRQ +// assembly stub around KeIrqDispatch. The exit hook is where deferred // interrupt-time scheduling now happens, so IRQ handlers never switch away // before KeIrqDispatch has returned and sent EOI. void KiEnterInterrupt(void); diff --git a/code/kernel/include/kernel/ke_irql.h b/code/kernel/include/kernel/ke_irql.h index f14b257..fcd4ec9 100644 --- a/code/kernel/include/kernel/ke_irql.h +++ b/code/kernel/include/kernel/ke_irql.h @@ -1,9 +1,9 @@ #pragma once // ke_irql.h - software IRQL skeleton // -// This is intentionally small for the first IRQL/APC bring-up step. It does +// This is intentionally small for the first IRQL/APC bring-up step. It does // not yet program interrupt hardware or replace the spinlock interrupt-disable -// path. It gives the kernel one authoritative execution-level state plus +// path. It gives the kernel one authoritative execution-level state plus // assertions so blocking code can reject illegal callers early. #include @@ -18,11 +18,11 @@ typedef uint8_t KIRQL; __declspec(dllexport) KIRQL KeGetCurrentIrql(void); -// Raise to newIrql and return the previous level. Raising to a lower level is +// Raise to newIrql and return the previous level. Raising to a lower level is // a kernel bug and will panic in checked/debug-style builds. __declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql); -// Restore a previously returned level. Lowering to a higher level is a kernel +// Restore a previously returned level. Lowering to a higher level is a kernel // bug and will panic. __declspec(dllexport) void KeLowerIrql(KIRQL oldIrql); diff --git a/code/kernel/include/kernel/ke_process.h b/code/kernel/include/kernel/ke_process.h index 879b2fa..5b492c6 100644 --- a/code/kernel/include/kernel/ke_process.h +++ b/code/kernel/include/kernel/ke_process.h @@ -75,6 +75,12 @@ typedef struct PROCESS { uintptr_t UserCwdBase; size_t UserCwdSize; + /* Per-process user virtual regions. The actual VA arena is global because + NT-Lite currently uses one shared address space; this list tracks + ownership so leaked user mappings can be reclaimed on process exit. */ + LIST_ENTRY UserRegions; + SPINLOCK UserRegionLock; + /* Per-process loaded modules (user processes only). */ PROCESS_MODULE Modules[PROCESS_MODULE_MAX]; uint32_t ModuleCount; diff --git a/code/kernel/include/kernel/ke_sched.h b/code/kernel/include/kernel/ke_sched.h index 056b591..62e3a15 100644 --- a/code/kernel/include/kernel/ke_sched.h +++ b/code/kernel/include/kernel/ke_sched.h @@ -37,15 +37,15 @@ KSTATUS KiCreateIdleThreadForCpu(struct KCPU* cpu); void KeReadyThread(THREAD* thread); void KeUnreadyThread(THREAD* thread); -// Called from the timer ISR every SCHED_TICK_MS. This performs timer +// Called from the timer ISR every SCHED_TICK_MS. This performs timer // accounting and requests a reschedule only when the current thread has -// exhausted its scheduler budget, or when idle work becomes available. The +// exhausted its scheduler budget, or when idle work becomes available. The // actual context switch happens from // KiExitInterrupt after the IRQ handler has returned and EOI has been sent. void KeSchedulerTick(void); -// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule -// is cheap and can be called from interrupt context. KiScheduleFromInterrupt +// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule +// is cheap and can be called from interrupt context. KiScheduleFromInterrupt // consumes the pending request and performs the context switch, if needed. void KiRequestReschedule(void); int KiIsReschedulePending(void); diff --git a/code/kernel/include/kernel/ke_smp.h b/code/kernel/include/kernel/ke_smp.h index 69c5310..f1e0162 100644 --- a/code/kernel/include/kernel/ke_smp.h +++ b/code/kernel/include/kernel/ke_smp.h @@ -2,6 +2,9 @@ #include #include +// Provide an optional bootloader-reserved low-memory trampoline page. +void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys); + // Start application processors with hardware INIT/SIPI and park them in // kernel-owned state. APs are not runnable until scheduler admission succeeds. KSTATUS KeStartApplicationProcessors(void); diff --git a/code/kernel/include/kernel/ke_spinlock.h b/code/kernel/include/kernel/ke_spinlock.h index 5a7e3e9..653ce21 100644 --- a/code/kernel/include/kernel/ke_spinlock.h +++ b/code/kernel/include/kernel/ke_spinlock.h @@ -11,6 +11,7 @@ // Never hold a spinlock across a wait/blocking operation. #include #include +#pragma intrinsic(_disable) #include typedef struct SPINLOCK { diff --git a/code/kernel/include/kernel/ke_thread.h b/code/kernel/include/kernel/ke_thread.h index 4f226b5..2a14837 100644 --- a/code/kernel/include/kernel/ke_thread.h +++ b/code/kernel/include/kernel/ke_thread.h @@ -71,7 +71,7 @@ typedef struct THREAD { uintptr_t UserArg; // Saved scheduler-lock cookie for this thread's suspended switch - // continuation. This is deliberately thread-owned, not CPU-owned: + // continuation. This is deliberately thread-owned, not CPU-owned: // after KeSwitchContext() resumes a saved KiSchedule() continuation, the // resumed thread must restore the IF/IRQL state captured when that same // thread switched out. @@ -86,7 +86,7 @@ typedef struct THREAD { void* FpuContext; uint8_t FpuStorage[528]; - // Scheduler accounting. QuantaConsumed is kept in the assembly-stable + // Scheduler accounting. QuantaConsumed is kept in the assembly-stable // portion of THREAD for layout compatibility; the dynamic scheduler uses // these fields for per-dispatch budget and CPU-burst tracking. uint32_t QuantumTicks; @@ -105,14 +105,14 @@ typedef struct THREAD { uint64_t AffinityMask; // Set by wait primitives before blocking and completed by the object - // that wakes the thread. Keep this field after the assembly-visible + // that wakes the thread. Keep this field after the assembly-visible // context-switch fields: ke_switch.asm hard-codes offsets through // FpuContext. KSTATUS WaitStatus; LIST_ENTRY* WaitListHead; uint8_t AlertableWait; - // Kernel APC state. APCs are queued to a target thread and delivered + // Kernel APC state. APCs are queued to a target thread and delivered // when that thread reaches PASSIVE_LEVEL with APC delivery enabled. LIST_ENTRY ApcQueue; SPINLOCK ApcLock; @@ -124,7 +124,7 @@ typedef struct THREAD { // The pointed-to frame lives on this thread's kernel stack. KTRAP_FRAME* TrapFrame; - // User APC state. User APCs are queued by kernel completions, but are + // User APC state. User APCs are queued by kernel completions, but are // delivered only on an alertable return to user mode through the registered // user-mode dispatcher stub. LIST_ENTRY UserApcQueue; @@ -162,7 +162,7 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code); // // The split form prevents the classic lost wakeup where an ISR/DPC signals // the object after the waiter is queued but before the waiter marks itself -// waiting. Alertable waits are explicit at every call site. +// waiting. Alertable waits are explicit at every call site. void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable); void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead); KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable); diff --git a/code/kernel/include/kernel/ke_userenv.h b/code/kernel/include/kernel/ke_userenv.h index ea49a2d..a5cd02e 100644 --- a/code/kernel/include/kernel/ke_userenv.h +++ b/code/kernel/include/kernel/ke_userenv.h @@ -32,7 +32,7 @@ typedef struct SYS_TEB { uint32_t ThreadId; uint32_t Magic; - // Native per-thread extension pointer. The kernel leaves this NULL; + // Native per-thread extension pointer. The kernel leaves this NULL; // user-mode personalities/libraries may hang their own thread-local // state here without baking subsystem semantics into SYS_TEB. void* SubsystemData; diff --git a/code/kernel/include/kernel/ldr_image.h b/code/kernel/include/kernel/ldr_image.h index cffafa5..c40eb87 100644 --- a/code/kernel/include/kernel/ldr_image.h +++ b/code/kernel/include/kernel/ldr_image.h @@ -6,12 +6,16 @@ #include #include +struct PROCESS; + // Map a PE32+ image from 'buffer' into the kernel virtual address space. // On success, *baseOut receives the virtual base address. KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user); +KSTATUS LdrLoadImageForProcess(struct PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut); // Return the SizeOfImage recorded in the PE header for an already-mapped image. size_t LdrGetImageSize(uintptr_t base); +size_t LdrGetMappedImageSize(struct PROCESS* process, uintptr_t base); // Look up a named export in a loaded module. // 'moduleName' must match a name in BOOT_INFO.Modules (e.g., "hal.dll"). @@ -24,11 +28,11 @@ uintptr_t LdrGetEntryPoint(uintptr_t base); KSTATUS LdrResolveImports(uintptr_t imageBase); // Resolve imports using a process-local module registry. -struct PROCESS; KSTATUS LdrResolveImportsForProcess(struct PROCESS* process, uintptr_t imageBase); // Apply final page permissions for a loaded image after relocations/imports. KSTATUS LdrProtectImage(uintptr_t imageBase, bool user); +KSTATUS LdrProtectImageForProcess(struct PROCESS* process, uintptr_t imageBase, bool user); // Load a DLL for a specific process, resolve it against that process's // loaded modules, and register it in the process-local registry. diff --git a/code/kernel/include/kernel/mm_phys.h b/code/kernel/include/kernel/mm_phys.h index b8009d8..2f95c81 100644 --- a/code/kernel/include/kernel/mm_phys.h +++ b/code/kernel/include/kernel/mm_phys.h @@ -17,8 +17,10 @@ bool MmInitializePhysical(const MEMORY_MAP* map, // Allocate / free a single 4 KB frame. // MmAllocateFrame returns 0 on failure. -PHYSICAL_ADDRESS MmAllocateFrame(void); +__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void); PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive); +PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive); void MmFreeFrame(PHYSICAL_ADDRESS frame); // Allocate 'count' physically contiguous 4 KB frames. diff --git a/code/kernel/include/kernel/mm_virt.h b/code/kernel/include/kernel/mm_virt.h index 513c3c4..13dafaf 100644 --- a/code/kernel/include/kernel/mm_virt.h +++ b/code/kernel/include/kernel/mm_virt.h @@ -6,6 +6,25 @@ #include #include +struct PROCESS; +typedef struct PROCESS PROCESS; + +typedef enum USER_REGION_TYPE { + UserRegionGeneric = 0, + UserRegionHeap, + UserRegionImage, + UserRegionStack, + UserRegionTeb, + UserRegionPeb, + UserRegionEnvironment, + UserRegionMappedFile, + UserRegionShared, +} USER_REGION_TYPE; + +#define USER_REGION_GUARD_LOW 0x00000001u +#define USER_REGION_GUARD_HIGH 0x00000002u +#define USER_REGION_ZERO 0x00000004u + // --------------------------------------------------------------------------- // Page flags (x86-64 PTE bits) // --------------------------------------------------------------------------- @@ -32,10 +51,10 @@ // Called by KernelMain after MmInitializePhysical. // bootPml4Phys: the PML4 set up by the bootloader. // poolSize: initial reservation for kernel pool (bytes, page-aligned). -void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize); +void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize); // Install the shared PML4 into CR3. Called once at the end of KernelMain init. -void MmActivate(void); +void MmActivate(void); // --------------------------------------------------------------------------- // Page mapping @@ -89,16 +108,37 @@ __declspec(dllexport) void MmFreePool(void* ptr); __declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize); // --------------------------------------------------------------------------- -// User heap - shared flat space, ring 3 accessible. -// Allocations are writable user pages with NX set by default; PE image -// sections opt back into execution through the loader. +// User virtual regions. +// The kernel has one shared user VA arena, but every user mapping is owned by +// a PROCESS so process teardown can reclaim leaked memory. Guard pages are +// reserved VA with no present PTE. // --------------------------------------------------------------------------- +KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment, + uint64_t pageFlags, uint32_t regionFlags, + USER_REGION_TYPE type, void** outBase); +KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags); +KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags); +KSTATUS MmDestroyUserRegion(PROCESS* process, void* base); +void MmDestroyAllUserRegions(PROCESS* process); +size_t MmGetUserRegionSize(PROCESS* process, const void* base); +size_t MmGetUserRegionSizeByBase(const void* base); + +// Debug-only ownership/address diagnostics used by fault and launch paths. +void MmDebugDescribeAddress(uintptr_t addr); +void MmDebugDumpUserRegions(PROCESS* process); + +__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size); +__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr); +__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize); + +// Convenience wrappers for syscalls/current-process allocations. __declspec(dllexport) void* MmAllocateUserMemory(size_t size); __declspec(dllexport) void MmFreeUserMemory(void* ptr); __declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize); -// Page-aligned user image mapping. PE section protection is page based, so -// executable images must not be allocated from the normal sub-page heap. +// Page-aligned user image mapping. PE section protection is page based. +__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size); +__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size); __declspec(dllexport) void* MmAllocateUserImage(size_t size); __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size); diff --git a/code/kernel/include/kernel/syscall_nr.h b/code/kernel/include/kernel/syscall_nr.h index 959c97d..cf83049 100644 --- a/code/kernel/include/kernel/syscall_nr.h +++ b/code/kernel/include/kernel/syscall_nr.h @@ -73,7 +73,7 @@ #define SYS_KB_POLL 64 #define SYS_KB_IS_KEY_DOWN 65 #define SYS_KB_READ_CHAR 66 -#define SYS_KB_READ_SCANCODE 67 +#define SYS_KB_READ_KEY 67 #define SYS_KB_GET_MODIFIERS 68 #define SYS_MOUSE_POLL 69 #define SYS_MOUSE_READ 70 diff --git a/code/kernel/io_device.c b/code/kernel/io_device.c index 02b3227..bcd08f1 100644 --- a/code/kernel/io_device.c +++ b/code/kernel/io_device.c @@ -3,10 +3,137 @@ #include #include #include +#include + + +#define MAX_PUBLISHED_DEVICES 64 +#define MAX_DRIVER_MATCHERS 32 + +typedef struct IO_DRIVER_MATCHER_ENTRY { + bool Used; + const char* DriverName; + IO_BUS_TYPE BusType; + IO_DRIVER_MATCH_CALLBACK Match; + IO_DRIVER_BIND_CALLBACK Bind; + void* Context; +} IO_DRIVER_MATCHER_ENTRY; + +static IO_PUBLISHED_DEVICE gPublishedDevices[MAX_PUBLISHED_DEVICES]; +static uint32_t gPublishedDeviceCount; +static IO_DRIVER_MATCHER_ENTRY gDriverMatchers[MAX_DRIVER_MATCHERS]; +static uint32_t gDriverMatcherCount; + +static const char* IoBusTypeName(IO_BUS_TYPE busType) { + switch (busType) { + case IoBusPci: return "pci"; + case IoBusUsb: return "usb"; + default: return "unknown"; + } +} + +static bool IoTryBindDevice(IO_PUBLISHED_DEVICE* device, + IO_DRIVER_MATCHER_ENTRY* matcher) { + if (!device || !matcher || !matcher->Used || device->Bound) + return false; + if (matcher->BusType != IoBusUnknown && matcher->BusType != device->BusType) + return false; + if (!matcher->Match || !matcher->Bind) + return false; + if (!matcher->Match(device, matcher->Context)) + return false; + + KSTATUS st = matcher->Bind(device, matcher->Context); + if (KSUCCESS(st)) { + device->Bound = true; + device->BoundDriverName = matcher->DriverName; + DbgPrint("io: %s device %s bound by %s\n", + IoBusTypeName(device->BusType), + device->Name ? device->Name : "", + matcher->DriverName ? matcher->DriverName : ""); + return true; + } + + DbgPrint("io: %s matcher %s accepted %s but bind failed st=0x%x\n", + IoBusTypeName(device->BusType), + matcher->DriverName ? matcher->DriverName : "", + device->Name ? device->Name : "", + st); + return false; +} + +__declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName, + IO_BUS_TYPE busType, + IO_DRIVER_MATCH_CALLBACK match, + IO_DRIVER_BIND_CALLBACK bind, + void* context) { + if (!driverName || !match || !bind) + return STATUS_INVALID_PARAMETER; + if (gDriverMatcherCount >= MAX_DRIVER_MATCHERS) + return STATUS_NO_MEMORY; + + IO_DRIVER_MATCHER_ENTRY* entry = &gDriverMatchers[gDriverMatcherCount++]; + memset(entry, 0, sizeof(*entry)); + entry->Used = true; + entry->DriverName = driverName; + entry->BusType = busType; + entry->Match = match; + entry->Bind = bind; + entry->Context = context; + + DbgPrint("io: registered %s matcher %s\n", + IoBusTypeName(busType), driverName); + + for (uint32_t i = 0; i < gPublishedDeviceCount; i++) { + IoTryBindDevice(&gPublishedDevices[i], entry); + } + + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device) { + if (!device || device->BusType == IoBusUnknown) + return STATUS_INVALID_PARAMETER; + if (gPublishedDeviceCount >= MAX_PUBLISHED_DEVICES) + return STATUS_NO_MEMORY; + + IO_PUBLISHED_DEVICE* stored = &gPublishedDevices[gPublishedDeviceCount++]; + memcpy(stored, device, sizeof(*stored)); + stored->Bound = false; + stored->BoundDriverName = (const char*)0; + + + for (uint32_t i = 0; i < gDriverMatcherCount; i++) { + if (IoTryBindDevice(stored, &gDriverMatchers[i])) + break; + } + + return STATUS_SUCCESS; +} + +__declspec(dllexport) uint32_t IoPublishedDeviceCount(void) { + return gPublishedDeviceCount; +} #define MAX_DISK_DEVICES 8 +#define MAX_DISK_ARRIVAL_CALLBACKS 16 + +typedef struct IO_DISK_CALLBACK_ENTRY { + bool Used; + IO_DISK_ARRIVAL_CALLBACK Callback; + void* Context; +} IO_DISK_CALLBACK_ENTRY; static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES]; +static IO_DISK_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_ARRIVAL_CALLBACKS]; +static uint32_t gDiskArrivalCallbackCount; + +static void IoNotifyDiskArrived(uint8_t driveIndex, DEVICE_OBJECT* device) { + for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) { + IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i]; + if (entry->Used && entry->Callback) + entry->Callback(driveIndex, device, entry->Context); + } +} __declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver, size_t extensionSize, @@ -43,6 +170,41 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, device->Flags |= DEVICE_OBJECT_DISK; device->DiskIndex = driveIndex; gDiskDevices[driveIndex] = device; + IoNotifyDiskArrived(driveIndex, device); + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback, + void* context) { + if (!callback) + return STATUS_INVALID_PARAMETER; + if (gDiskArrivalCallbackCount >= MAX_DISK_ARRIVAL_CALLBACKS) + return STATUS_NO_MEMORY; + + IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++]; + memset(entry, 0, sizeof(*entry)); + entry->Used = true; + entry->Callback = callback; + entry->Context = context; + + for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) { + if (gDiskDevices[i]) + callback(i, gDiskDevices[i], context); + } + + return STATUS_SUCCESS; +} + +__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback, + void* context) { + if (!callback) + return STATUS_INVALID_PARAMETER; + + for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) { + if (gDiskDevices[i]) + callback(i, gDiskDevices[i], context); + } + return STATUS_SUCCESS; } diff --git a/code/kernel/io_file.c b/code/kernel/io_file.c index 2ec7647..da2029b 100644 --- a/code/kernel/io_file.c +++ b/code/kernel/io_file.c @@ -150,7 +150,7 @@ static void KiIoAsyncFileWorker(void* arg) { } if (req->CancelRequested) { - // The operation may already have reached the filesystem. The + // The operation may already have reached the filesystem. The // completion contract still reports cancellation so waiters/APCs do // not mistake a cancelled request for useful data. s = STATUS_CANCELLED; @@ -628,7 +628,7 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out } // --------------------------------------------------------------------------- -// Explicit-offset file I/O. Async I/O is built on these routines so it does +// Explicit-offset file I/O. Async I/O is built on these routines so it does // not race the sequential FILE_OBJECT.Offset used by legacy read/write calls. // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, diff --git a/code/kernel/ke_apc.c b/code/kernel/ke_apc.c index 77fc8b1..93a8a83 100644 --- a/code/kernel/ke_apc.c +++ b/code/kernel/ke_apc.c @@ -74,13 +74,13 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc, KeReleaseSpinlock(&thread->ApcLock, flags); // If the target opted into an alertable wait, make the APC itself the - // wait completion. The target thread will deliver the APC before + // wait completion. The target thread will deliver the APC before // KeWaitForEvent(..., true) returns STATUS_KERNEL_APC. if (wakeAlertable) KiWakeThreadForKernelApc(thread); // If the caller queued an APC to itself at PASSIVE_LEVEL, try to deliver - // immediately. Cross-thread APCs run the next time the target thread + // immediately. Cross-thread APCs run the next time the target thread // returns to PASSIVE_LEVEL. if (thread == KeGetCurrentThread()) KiDeliverKernelApcs(); @@ -165,7 +165,7 @@ void KiDeliverKernelApcs(void) { &arg2); // After the kernel routine has had a chance to adjust the callout, - // stop touching APC storage before invoking the normal routine. The + // stop touching APC storage before invoking the normal routine. The // normal routine is allowed to free the APC's containing allocation. apc->SystemArgument1 = (void*)0; apc->SystemArgument2 = (void*)0; diff --git a/code/kernel/ke_cpu.c b/code/kernel/ke_cpu.c index 2c5f4e4..85570ff 100644 --- a/code/kernel/ke_cpu.c +++ b/code/kernel/ke_cpu.c @@ -26,7 +26,7 @@ void KeLoadKernelGsBase(KCPU* cpu) { } void KeSetUserGsBase(uintptr_t userGsBase) { - // While running in the kernel, GS_BASE is the current KCPU. The user TEB + // While running in the kernel, GS_BASE is the current KCPU. The user TEB // base is parked in IA32_KERNEL_GS_BASE so swapgs can exchange it with the // KCPU pointer on return to ring 3. __writemsr(IA32_KERNEL_GS_BASE, (uint64_t)userGsBase); diff --git a/code/kernel/ke_debug.c b/code/kernel/ke_debug.c index 8d9e99d..50fa77a 100644 --- a/code/kernel/ke_debug.c +++ b/code/kernel/ke_debug.c @@ -5,6 +5,7 @@ #include #include #include +#pragma intrinsic(_disable) #include #include #include @@ -31,8 +32,8 @@ static int KiPanicIsActive(void) { return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0; } -// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the -// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely +// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the +// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely // unexpected NMI is not handled here and will fall through to the normal fault // path in the assembly stub. __declspec(dllexport) int KeHandleNmiIpi(void) { @@ -72,13 +73,13 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) { // Silence this CPU first. _disable(); - // Only one CPU owns the visible panic report. CPUs that fault while the + // Only one CPU owns the visible panic report. CPUs that fault while the // system is already panicking immediately join the stopped set. if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) != 0) { KiPanicSecondaryHalt(); } - // Stop every other CPU as early as possible. This uses an NMI IPI so it + // Stop every other CPU as early as possible. This uses an NMI IPI so it // still reaches CPUs with IF cleared inside spinlock/high-IRQL regions. HalApicSendPanicIpi(); @@ -98,7 +99,7 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) { (unsigned)cpu->CpuIndex, (unsigned)cpu->LapicId); if (thread) - printf(" thread=%p tid=%llu", + printf(" thread=0x%016llX tid=%llu", thread, (unsigned long long)thread->ThreadId); printf("\n"); @@ -192,7 +193,7 @@ static int KiFaultHasErrorCode(int vector) { // Walk the process module list to find which image owns an address. // Returns the module name string, or "unknown" if the address is outside all -// known image ranges. Base/size are optional and are useful for offset dumps. +// known image ranges. Base/size are optional and are useful for offset dumps. static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr, uintptr_t* baseOut, size_t* sizeOut) { if (baseOut) *baseOut = 0; @@ -230,7 +231,7 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) { const char* mod = KiResolveModuleEx(proc, addr, &base, &size); if (base) { - printf("%s%016llx <%s+0x%llx>\n", + printf("%s%016llx <%s+0x%016llX>\n", label, (unsigned long long)addr, mod, @@ -374,7 +375,7 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) { size_t size = 0; const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size); if (base) { - printf(" [%02d] %016llx : %016llx <%s+0x%llx>\n", + printf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n", i, (unsigned long long)slot, (unsigned long long)value, @@ -486,7 +487,7 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u } __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) { - // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode + // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode // the interrupted context on a secondary CPU once another CPU owns panic. if (KiPanicIsActive()) KiPanicSecondaryHalt(); @@ -503,10 +504,28 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr int vector = (int)frame->Vector; const char* name = KiFaultVectorName(vector); + DbgPrint("FAULT: kernel exception vector=%d name=%s rip=%016llx rsp=%016llx err=%04llx cr2=%016llx thread=%p pid=%u proc=%s\n", + vector, + name, + (unsigned long long)frame->Rip, + (unsigned long long)frame->Rsp, + (unsigned long long)frame->ErrorCode, + (unsigned long long)frame->Cr2, + thread, + proc ? proc->ProcessId : 0, + proc ? proc->Name : ""); + KiDumpFaultFrame(proc, frame); if (vector == 14) { - KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n", + MmDebugDescribeAddress((uintptr_t)frame->Cr2); + if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp); + MmDebugDumpUserRegions(proc); + } + + if (vector == 14) { + KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | RSP: %016llx | Code: %04llx | CR2: %016llx\n", name, (unsigned long long)frame->Rip, + (unsigned long long)frame->Rsp, (unsigned long long)frame->ErrorCode, (unsigned long long)frame->Cr2); } else if (KiFaultHasErrorCode(vector)) { @@ -547,7 +566,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr // KeFaultDispatch - single C entry point for all three IDT fault stubs. // // The asm stubs (ke_syscall.asm) collect the CPU exception frame values and -// call here. We check the CS RPL to decide: kernel fault → panic and halt, +// call here. We check the CS RPL to decide: kernel fault → panic and halt, // user fault → KeBugCheckProcess (kill process, resume scheduler). // --------------------------------------------------------------------------- __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip, diff --git a/code/kernel/ke_dpc.c b/code/kernel/ke_dpc.c index 242c310..d679e8e 100644 --- a/code/kernel/ke_dpc.c +++ b/code/kernel/ke_dpc.c @@ -46,7 +46,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc, KiEnsureDpcInitialized(); - // Coalesce repeated inserts of the same KDPC. The first insert owns the + // Coalesce repeated inserts of the same KDPC. The first insert owns the // argument pair until the DPC is popped by the dispatcher. if (_InterlockedCompareExchange(&dpc->Inserted, 1, 0) != 0) return false; diff --git a/code/kernel/ke_event.c b/code/kernel/ke_event.c index b8fcc12..537330f 100644 --- a/code/kernel/ke_event.c +++ b/code/kernel/ke_event.c @@ -38,7 +38,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) { } else { if (!LIST_IS_EMPTY(&event->WaitList)) { // Auto-reset handoff: transfer the signal directly to exactly one - // waiter. Do not leave Signaled=1, because a later waiter can steal + // waiter. Do not leave Signaled=1, because a later waiter can steal // the disk/APC completion before the selected thread resumes. event->Signaled = 0; LIST_ENTRY* node = ListPopHead(&event->WaitList); diff --git a/code/kernel/ke_idt.c b/code/kernel/ke_idt.c index 42e8433..b43d656 100644 --- a/code/kernel/ke_idt.c +++ b/code/kernel/ke_idt.c @@ -140,7 +140,7 @@ void KeLoadGdtForCurrentCpu(KCPU* cpu) { KeFlushGdt(&cpu->Gdtr, 0x08, 0x10, 0x28); // Reloading GS as a segment register during GDT reload can disturb the - // hidden GS base. Reinstall the kernel per-CPU base immediately after + // hidden GS base. Reinstall the kernel per-CPU base immediately after // the descriptor reload so KeGetCurrentCpu() remains GS-backed. KeLoadKernelGsBase(cpu); } diff --git a/code/kernel/ke_input.c b/code/kernel/ke_input.c new file mode 100644 index 0000000..6c50e38 --- /dev/null +++ b/code/kernel/ke_input.c @@ -0,0 +1,352 @@ +// ke_input.c - kernel-owned canonical input core +#include +#include +#include +#include + +#include +#include + +#define KEY_QUEUE_SIZE 256u +#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u) +#define MOUSE_QUEUE_SIZE 128u +#define MOUSE_QUEUE_MASK (MOUSE_QUEUE_SIZE - 1u) +#define MAX_INPUT_KEYS 256u +#define MAX_INPUT_DEVICES 16u +#define KEY_REPEAT_DELAY_US 500000ull +#define KEY_REPEAT_PERIOD_US 33000ull + +typedef struct KEY_CHAR_PAIR { + INPUT_KEY key; + char normal; + char shifted; +} KEY_CHAR_PAIR; + +static const HAL_API* gInputHal; + +static volatile uint8_t gKeyDown[MAX_INPUT_KEYS]; +static volatile uint32_t gModifiers; + +typedef struct INPUT_KEY_QUEUE_ENTRY { + INPUT_KEY key; + char ch; +} INPUT_KEY_QUEUE_ENTRY; + +static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE]; +static volatile int gKeyHead; +static volatile int gKeyTail; +static volatile INPUT_KEY gRepeatKey; +static volatile uint64_t gRepeatNextTick; + +static INPUT_MOUSE_EVENT gMouseQueue[MOUSE_QUEUE_SIZE]; +static volatile int gMouseHead; +static volatile int gMouseTail; +static volatile int gMouseX; +static volatile int gMouseY; +static volatile int gMouseBoundsW = 1920; +static volatile int gMouseBoundsH = 1080; +static volatile uint8_t gMouseButtons; + +static volatile uint32_t gNextInputDeviceId = 1; + +static char KiInputKeyToChar(INPUT_KEY key); + +static const KEY_CHAR_PAIR gCharMap[] = { + { INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' }, + { INPUT_KEY_3, '3', '#' }, { INPUT_KEY_4, '4', '$' }, + { INPUT_KEY_5, '5', '%' }, { INPUT_KEY_6, '6', '^' }, + { INPUT_KEY_7, '7', '&' }, { INPUT_KEY_8, '8', '*' }, + { INPUT_KEY_9, '9', '(' }, { INPUT_KEY_0, '0', ')' }, + { INPUT_KEY_MINUS, '-', '_' }, { INPUT_KEY_EQUAL, '=', '+' }, + { INPUT_KEY_LEFT_BRACKET, '[', '{' }, { INPUT_KEY_RIGHT_BRACKET, ']', '}' }, + { INPUT_KEY_BACKSLASH, '\\', '|' }, { INPUT_KEY_SEMICOLON, ';', ':' }, + { INPUT_KEY_APOSTROPHE, '\'', '"' }, { INPUT_KEY_GRAVE, '`', '~' }, + { INPUT_KEY_COMMA, ',', '<' }, { INPUT_KEY_PERIOD, '.', '>' }, + { INPUT_KEY_SLASH, '/', '?' }, + { INPUT_KEY_SPACE, ' ', ' ' }, { INPUT_KEY_TAB, '\t', '\t' }, + { INPUT_KEY_ENTER, '\n', '\n' }, { INPUT_KEY_BACKSPACE, '\b', '\b' }, + { INPUT_KEY_KP_DIVIDE, '/', '/' }, { INPUT_KEY_KP_MULTIPLY, '*', '*' }, + { INPUT_KEY_KP_MINUS, '-', '-' }, { INPUT_KEY_KP_PLUS, '+', '+' }, + { INPUT_KEY_KP_ENTER, '\n', '\n' }, { INPUT_KEY_KP_1, '1', '1' }, + { INPUT_KEY_KP_2, '2', '2' }, { INPUT_KEY_KP_3, '3', '3' }, + { INPUT_KEY_KP_4, '4', '4' }, { INPUT_KEY_KP_5, '5', '5' }, + { INPUT_KEY_KP_6, '6', '6' }, { INPUT_KEY_KP_7, '7', '7' }, + { INPUT_KEY_KP_8, '8', '8' }, { INPUT_KEY_KP_9, '9', '9' }, + { INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' }, +}; + +static void KiInputQueueKey(INPUT_KEY key, char ch) { + if (key == INPUT_KEY_NONE) return; + int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK; + if (next == gKeyHead) return; + gKeyQueue[gKeyTail].key = key; + gKeyQueue[gKeyTail].ch = ch; + gKeyTail = next; +} + +static bool KiInputQueueHasSpace(void) { + int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK; + return next != gKeyHead; +} + +static bool KiInputKeyIsRepeatable(INPUT_KEY key) { + if (key == INPUT_KEY_NONE) return false; + + switch (key) { + case INPUT_KEY_LEFT_SHIFT: + case INPUT_KEY_RIGHT_SHIFT: + case INPUT_KEY_LEFT_CTRL: + case INPUT_KEY_RIGHT_CTRL: + case INPUT_KEY_LEFT_ALT: + case INPUT_KEY_RIGHT_ALT: + case INPUT_KEY_LEFT_GUI: + case INPUT_KEY_RIGHT_GUI: + case INPUT_KEY_CAPS_LOCK: + case INPUT_KEY_NUM_LOCK: + return false; + default: + return true; + } +} + +static uint64_t KiInputNowTicks(void) { + if (gInputHal && gInputHal->Timer.QueryTicks) + return gInputHal->Timer.QueryTicks(); + return 0; +} + +static void KiInputStartRepeat(INPUT_KEY key, uint64_t now) { + if (!KiInputKeyIsRepeatable(key)) return; + gRepeatKey = key; + gRepeatNextTick = now + KEY_REPEAT_DELAY_US; +} + +static void KiInputStopRepeat(INPUT_KEY key) { + if (gRepeatKey == key) { + gRepeatKey = INPUT_KEY_NONE; + gRepeatNextTick = 0; + } +} + +static void KiInputServiceRepeat(void) { + INPUT_KEY key = gRepeatKey; + if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key]) { + gRepeatKey = INPUT_KEY_NONE; + gRepeatNextTick = 0; + return; + } + + uint64_t now = KiInputNowTicks(); + if (now == 0 || now < gRepeatNextTick) + return; + + if (!KiInputQueueHasSpace()) { + gRepeatNextTick = now + KEY_REPEAT_PERIOD_US; + return; + } + + KiInputQueueKey(key, KiInputKeyToChar(key)); + gRepeatNextTick = now + KEY_REPEAT_PERIOD_US; +} + +static void KiInputSetModifier(INPUT_KEY key, bool pressed) { + uint32_t bit = 0; + switch (key) { + case INPUT_KEY_LEFT_SHIFT: bit = INPUT_MOD_LSHIFT; break; + case INPUT_KEY_RIGHT_SHIFT: bit = INPUT_MOD_RSHIFT; break; + case INPUT_KEY_LEFT_CTRL: bit = INPUT_MOD_LCTRL; break; + case INPUT_KEY_RIGHT_CTRL: bit = INPUT_MOD_RCTRL; break; + case INPUT_KEY_LEFT_ALT: bit = INPUT_MOD_LALT; break; + case INPUT_KEY_RIGHT_ALT: bit = INPUT_MOD_RALT; break; + case INPUT_KEY_LEFT_GUI: bit = INPUT_MOD_LGUI; break; + case INPUT_KEY_RIGHT_GUI: bit = INPUT_MOD_RGUI; break; + default: break; + } + + if (bit) { + if (pressed) gModifiers |= bit; + else gModifiers &= ~bit; + } + + if (pressed && key == INPUT_KEY_CAPS_LOCK) gModifiers ^= INPUT_MOD_CAPS_LOCK; + if (pressed && key == INPUT_KEY_NUM_LOCK) gModifiers ^= INPUT_MOD_NUM_LOCK; +} + +static char KiInputKeyToChar(INPUT_KEY key) { + if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) { + bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0; + bool caps = (gModifiers & INPUT_MOD_CAPS_LOCK) != 0; + char c = (char)('a' + (key - INPUT_KEY_A)); + if (shifted ^ caps) c = (char)(c - 'a' + 'A'); + return c; + } + + bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0; + for (size_t i = 0; i < sizeof(gCharMap) / sizeof(gCharMap[0]); i++) { + if (gCharMap[i].key == key) + return shifted ? gCharMap[i].shifted : gCharMap[i].normal; + } + return 0; +} + +void KeInputInitialize(const HAL_API* hal) { + gInputHal = hal; + memset((void*)gKeyDown, 0, sizeof(gKeyDown)); + gModifiers = 0; + gKeyHead = gKeyTail = 0; + gRepeatKey = INPUT_KEY_NONE; + gRepeatNextTick = 0; + gMouseHead = gMouseTail = 0; + gMouseButtons = 0; + + int w = 1920, h = 1080; + if (hal && hal->Screen.GetSize) hal->Screen.GetSize(&w, &h); + KeInputMouseSetBounds(w, h); + KeInputMouseSetPosition(w / 2, h / 2); + + if (hal && hal->Screen.DrawCursor) + hal->Screen.DrawCursor(gMouseX, gMouseY); + + DbgPrint("Input: kernel input core initialized (%dx%d)\n", w, h); +} + +__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) { + (void)flags; + uint32_t id = gNextInputDeviceId++; + if (id >= MAX_INPUT_DEVICES) return STATUS_NO_MEMORY; + if (outDeviceId) *outDeviceId = id; + DbgPrint("Input: registered device %u (%s)\n", id, name ? name : "unnamed"); + return STATUS_SUCCESS; +} + +__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) { + if (!event) return; + INPUT_KEY key = event->Key; + if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return; + + bool wasDown = gKeyDown[key] != 0; + bool pressed = event->Pressed; + + if (pressed) { + // The input core owns typematic repeat. Treat repeated hardware make + // events as idempotent so PS/2 typematic and USB HID share one policy. + if (wasDown) + return; + + gKeyDown[key] = 1u; + KiInputSetModifier(key, true); + KiInputQueueKey(key, KiInputKeyToChar(key)); + KiInputStartRepeat(key, KiInputNowTicks()); + } else { + if (!wasDown) + return; + + gKeyDown[key] = 0u; + KiInputSetModifier(key, false); + KiInputStopRepeat(key); + } +} + +__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) { + if (!event) return; + + int nx = gMouseX + event->dx; + int ny = gMouseY + event->dy; + int bw = gMouseBoundsW; + int bh = gMouseBoundsH; + + if (bw < 1) bw = 1; + if (bh < 1) bh = 1; + if (nx < 0) nx = 0; + if (ny < 0) ny = 0; + if (nx >= bw) nx = bw - 1; + if (ny >= bh) ny = bh - 1; + + if (gInputHal && gInputHal->Screen.EraseCursor) + gInputHal->Screen.EraseCursor(); + + gMouseX = nx; + gMouseY = ny; + gMouseButtons = event->buttons; + + if (gInputHal && gInputHal->Screen.DrawCursor) + gInputHal->Screen.DrawCursor(nx, ny); + + int next = (gMouseTail + 1) & (int)MOUSE_QUEUE_MASK; + if (next != gMouseHead) { + gMouseQueue[gMouseTail] = *event; + gMouseTail = next; + } +} + +INPUT_KEY KeInputKbPoll(void) { + if (gKeyHead == gKeyTail) KiInputServiceRepeat(); + return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key; +} + +bool KeInputIsKeyDown(INPUT_KEY key) { + if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false; + return gKeyDown[key] != 0; +} + +INPUT_KEY KeInputReadKey(void) { + if (gKeyHead == gKeyTail) KiInputServiceRepeat(); + if (gKeyHead == gKeyTail) return INPUT_KEY_NONE; + INPUT_KEY key = gKeyQueue[gKeyHead].key; + gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK; + return key; +} + +char KeInputReadChar(void) { + if (gKeyHead == gKeyTail) KiInputServiceRepeat(); + if (gKeyHead == gKeyTail) return 0; + char ch = gKeyQueue[gKeyHead].ch; + gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK; + return ch; +} + +uint32_t KeInputGetModifiers(void) { + return gModifiers; +} + +bool KeInputMousePoll(void) { + return gMouseHead != gMouseTail; +} + +bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) { + if (!dx || !dy || !buttons) return false; + if (gMouseHead == gMouseTail) return false; + + INPUT_MOUSE_EVENT ev = gMouseQueue[gMouseHead]; + gMouseHead = (gMouseHead + 1) & (int)MOUSE_QUEUE_MASK; + *dx = ev.dx; + *dy = ev.dy; + *buttons = ev.buttons; + return true; +} + +void KeInputMouseGetPosition(int* x, int* y) { + if (x) *x = gMouseX; + if (y) *y = gMouseY; +} + +void KeInputMouseSetPosition(int x, int y) { + int bw = gMouseBoundsW; + int bh = gMouseBoundsH; + if (bw < 1) bw = 1; + if (bh < 1) bh = 1; + if (x < 0) x = 0; + if (y < 0) y = 0; + if (x >= bw) x = bw - 1; + if (y >= bh) y = bh - 1; + gMouseX = x; + gMouseY = y; +} + +void KeInputMouseSetBounds(int w, int h) { + if (w < 1) w = 1; + if (h < 1) h = 1; + gMouseBoundsW = w; + gMouseBoundsH = h; + KeInputMouseSetPosition(gMouseX, gMouseY); +} diff --git a/code/kernel/ke_irq.c b/code/kernel/ke_irq.c index a535fc2..fe19393 100644 --- a/code/kernel/ke_irq.c +++ b/code/kernel/ke_irq.c @@ -10,6 +10,8 @@ #define MAX_IRQS 24 #define MAX_IRQ_HANDLERS 8 +#define MSI_IRQ_FIRST 16u +#define MSI_IRQ_LAST 23u typedef struct IRQ_HANDLER_LIST { void (*Handlers[MAX_IRQ_HANDLERS])(void); @@ -17,6 +19,7 @@ typedef struct IRQ_HANDLER_LIST { } IRQ_HANDLER_LIST; static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS]; +static bool gIrqSoftwareVector[MAX_IRQS]; static const HAL_API *gHal; static KIRQL KiIrqToIrql(uint8_t irq) { @@ -30,6 +33,23 @@ void KeIrqInitialize(const HAL_API *hal) { gHal = hal; } +__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) { + if (!outIrq) return STATUS_INVALID_PARAMETER; + + for (uint8_t irq = MSI_IRQ_FIRST; irq <= MSI_IRQ_LAST; irq++) { + IRQ_HANDLER_LIST* list = &gIrqHandlers[irq]; + if (!gIrqSoftwareVector[irq] && list->Count == 0) { + gIrqSoftwareVector[irq] = true; + *outIrq = irq; + DbgPrint("KeIrq: allocated MSI vector slot IRQ %u vector 0x%02x\n", + irq, 0x20u + irq); + return STATUS_SUCCESS; + } + } + + return STATUS_NO_MEMORY; +} + __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) { if (irq >= MAX_IRQS || !handler) return; @@ -45,7 +65,8 @@ __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) { } list->Handlers[list->Count++] = handler; - gHal->Intc.RouteIrq(irq); + if (!gIrqSoftwareVector[irq]) + gHal->Intc.RouteIrq(irq); } void KeIrqDispatch(uint8_t irq) { @@ -71,9 +92,9 @@ void KeIrqDispatch(uint8_t irq) { } } - // Handlers are no longer allowed to context-switch directly. Timer IRQs + // Handlers are no longer allowed to context-switch directly. Timer IRQs // only request rescheduling; KiExitInterrupt performs the switch after this - // function returns, so the EOI is guaranteed to be sent first. Keep the + // function returns, so the EOI is guaranteed to be sent first. Keep the // software level at DIRQL until after the controller has been acknowledged. gHal->Intc.SendEoi(); diff --git a/code/kernel/ke_mutex.c b/code/kernel/ke_mutex.c index 20696a5..a37cb45 100644 --- a/code/kernel/ke_mutex.c +++ b/code/kernel/ke_mutex.c @@ -61,7 +61,7 @@ void KeReleaseMutex(MUTEX* mutex) { LIST_ENTRY* node = ListPopHead(&mutex->WaitList); THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); // Direct handoff: the selected waiter owns the mutex when its wait - // returns. Do not drop Owner to NULL, or another thread can steal it. + // returns. Do not drop Owner to NULL, or another thread can steal it. mutex->Owner = waiter; KeReleaseSpinlock(&mutex->Lock, flags); KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS); diff --git a/code/kernel/ke_process.c b/code/kernel/ke_process.c index b33bc1f..ac20ea8 100644 --- a/code/kernel/ke_process.c +++ b/code/kernel/ke_process.c @@ -96,45 +96,21 @@ static void KiProcessDestroy(OBJECT* obj) { } } - // Release the main executable image. - if (proc->ImageBase && proc->ImageSize) { - MmFreeUserImage((void *)proc->ImageBase, proc->ImageSize); - proc->ImageBase = 0; - proc->ImageSize = 0; - } - - // Release per-process modules. - for (uint32_t i = 0; i < proc->ModuleCount; i++) { - if (proc->Modules[i].Base && proc->Modules[i].Size) { - MmFreeUserImage((void *)proc->Modules[i].Base, proc->Modules[i].Size); - } - proc->Modules[i].Base = 0; - proc->Modules[i].Size = 0; - proc->Modules[i].Name[0] = '\0'; - } + // Release every user VA region owned by this process. This includes the + // main image, DLLs, TEBs/stacks that survived thread teardown, process + // environment strings, and leaked user allocations. + MmDestroyAllUserRegions(proc); + proc->ImageBase = 0; + proc->ImageSize = 0; proc->ModuleCount = 0; - - // Release the user-mode process environment. - if (proc->UserPebBase && proc->UserPebSize) { - MmFreeUserMemory((void *)proc->UserPebBase); - proc->UserPebBase = 0; - proc->UserPebSize = 0; - } - if (proc->UserImagePathBase && proc->UserImagePathSize) { - MmFreeUserMemory((void *)proc->UserImagePathBase); - proc->UserImagePathBase = 0; - proc->UserImagePathSize = 0; - } - if (proc->UserCmdlineBase && proc->UserCmdlineSize) { - MmFreeUserMemory((void *)proc->UserCmdlineBase); - proc->UserCmdlineBase = 0; - proc->UserCmdlineSize = 0; - } - if (proc->UserCwdBase && proc->UserCwdSize) { - MmFreeUserMemory((void *)proc->UserCwdBase); - proc->UserCwdBase = 0; - proc->UserCwdSize = 0; - } + proc->UserPebBase = 0; + proc->UserPebSize = 0; + proc->UserImagePathBase = 0; + proc->UserImagePathSize = 0; + proc->UserCmdlineBase = 0; + proc->UserCmdlineSize = 0; + proc->UserCwdBase = 0; + proc->UserCwdSize = 0; // Release the exit event. if (proc->ExitEvent) { @@ -189,6 +165,8 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) { proc->CurrentDirectoryLength = 0; proc->CurrentDrive = 0; proc->CwdLock = (SPINLOCK)SPINLOCK_INIT; + LIST_INIT(&proc->UserRegions); + proc->UserRegionLock = (SPINLOCK)SPINLOCK_INIT; // Slot 0 is the invalid handle - leave it NULL. memset(proc->Handles, 0, sizeof(proc->Handles)); @@ -213,7 +191,8 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) { // --------------------------------------------------------------------------- // KeSetupUserProcessEnvironment // --------------------------------------------------------------------------- -static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src, +static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc, + SYS_STRING_VIEW src, uintptr_t* baseOut, size_t* sizeOut) { if (!baseOut || !sizeOut) return STATUS_INVALID_PARAMETER; @@ -225,7 +204,7 @@ static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src, if (src.Length == (size_t)-1) return STATUS_BUFFER_TOO_SMALL; size_t len = src.Length + 1; - char* dst = (char*)MmAllocateUserMemory(len); + char* dst = (char*)MmAllocateUserMemoryForProcess(proc, len); if (!dst) return STATUS_NO_MEMORY; if (src.Length) memcpy(dst, src.Data, src.Length); @@ -236,7 +215,8 @@ static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src, return STATUS_SUCCESS; } -static KSTATUS KiCopyEnvironmentStringViewFixed(SYS_STRING_VIEW src, +static KSTATUS KiCopyEnvironmentStringViewFixed(PROCESS* proc, + SYS_STRING_VIEW src, size_t capacity, uintptr_t* baseOut, size_t* sizeOut) { @@ -248,7 +228,7 @@ static KSTATUS KiCopyEnvironmentStringViewFixed(SYS_STRING_VIEW src, if (SysSvContainsByte(src, '\0')) return STATUS_INVALID_PARAMETER; if (src.Length + 1 > capacity) return STATUS_BUFFER_TOO_SMALL; - char* dst = (char*)MmAllocateUserMemory(capacity); + char* dst = (char*)MmAllocateUserMemoryForProcess(proc, capacity); if (!dst) return STATUS_NO_MEMORY; memset(dst, 0, capacity); @@ -264,22 +244,22 @@ static void KiFreePartialProcessEnvironment(PROCESS* proc) { if (!proc) return; if (proc->UserPebBase && proc->UserPebSize) { - MmFreeUserMemory((void*)proc->UserPebBase); + MmFreeUserMemoryForProcess(proc, (void*)proc->UserPebBase); proc->UserPebBase = 0; proc->UserPebSize = 0; } if (proc->UserImagePathBase && proc->UserImagePathSize) { - MmFreeUserMemory((void*)proc->UserImagePathBase); + MmFreeUserMemoryForProcess(proc, (void*)proc->UserImagePathBase); proc->UserImagePathBase = 0; proc->UserImagePathSize = 0; } if (proc->UserCmdlineBase && proc->UserCmdlineSize) { - MmFreeUserMemory((void*)proc->UserCmdlineBase); + MmFreeUserMemoryForProcess(proc, (void*)proc->UserCmdlineBase); proc->UserCmdlineBase = 0; proc->UserCmdlineSize = 0; } if (proc->UserCwdBase && proc->UserCwdSize) { - MmFreeUserMemory((void*)proc->UserCwdBase); + MmFreeUserMemoryForProcess(proc, (void*)proc->UserCwdBase); proc->UserCwdBase = 0; proc->UserCwdSize = 0; } @@ -337,19 +317,20 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc, KiFreePartialProcessEnvironment(proc); - s = KiCopyEnvironmentStringView(imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize); + s = KiCopyEnvironmentStringView(proc, imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize); if (!KSUCCESS(s)) goto fail; - s = KiCopyEnvironmentStringView(finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize); + s = KiCopyEnvironmentStringView(proc, finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize); if (!KSUCCESS(s)) goto fail; - s = KiCopyEnvironmentStringViewFixed(SysSvFromParts(finalCwd, finalCwdLength), + s = KiCopyEnvironmentStringViewFixed(proc, + SysSvFromParts(finalCwd, finalCwdLength), SYS_CWD_MAX, &proc->UserCwdBase, &proc->UserCwdSize); if (!KSUCCESS(s)) goto fail; - SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemory(sizeof(SYS_PEB)); + SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemoryForProcess(proc, sizeof(SYS_PEB)); if (!peb) { s = STATUS_NO_MEMORY; goto fail; } memset(peb, 0, sizeof(SYS_PEB)); @@ -389,7 +370,7 @@ void KeExitProcess(PROCESS* process, int exitCode) { process->ExitCode = exitCode; _InterlockedExchange(&process->Terminating, 1); - // The calling thread exits immediately after this in SysExitProcess. Do + // The calling thread exits immediately after this in SysExitProcess. Do // not drop the process object here: each thread owns one process reference, // and KiThreadDestroy releases that reference after unlinking the thread. KiCompleteProcessExitIfLastThread(process); @@ -402,7 +383,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) { _InterlockedExchange(&process->Terminating, 1); // Wake alertable waiters so they can notice Terminating at the syscall/ - // wait boundary. This is intentionally conservative: fully asynchronous + // wait boundary. This is intentionally conservative: fully asynchronous // forced thread teardown needs object-specific wait-list locking and is a // larger process-lifecycle project. uintptr_t flags = KeAcquireSpinlock(&process->HandleLock); diff --git a/code/kernel/ke_sched.c b/code/kernel/ke_sched.c index ab525ab..09f998b 100644 --- a/code/kernel/ke_sched.c +++ b/code/kernel/ke_sched.c @@ -2,6 +2,7 @@ #include #include #include +#pragma intrinsic(_enable) #include #include #include @@ -84,7 +85,7 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) { uint32_t quantum = KiBaseQuantumForPriority(thread->Priority); // Low-priority CPU-bound threads get longer batches once they have proven - // they mostly burn CPU. Higher-priority work still preempts them as soon + // they mostly burn CPU. Higher-priority work still preempts them as soon // as it becomes ready, so this reduces switch overhead without hurting // interactive wake latency. if (thread->Priority == PriorityLow && thread->CpuBurstTicks >= @@ -113,7 +114,7 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) { static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) { - // The scheduler lock is intentionally held across KeSwitchContext(). The + // The scheduler lock is intentionally held across KeSwitchContext(). The // unlock cookie belongs to the thread continuation that will resume after // the switch, not to the physical CPU. Another dispatch may resume this // thread from an unrelated interrupt, so using a KCPU slot here would @@ -129,7 +130,7 @@ static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) { static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) { KePanic(KE_PANIC_CODE_SCHEDULER, - "scheduler invariant failed: %s: thread=0x%llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n", + "scheduler invariant failed: %s: thread=0x%016llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n", where, (uint64_t)thread, thread ? thread->ThreadId : 0, @@ -201,7 +202,7 @@ static void KiIdleLoop(void) { continue; } - // HLT only wakes for maskable IPIs when IF=1. A CPU that reaches + // HLT only wakes for maskable IPIs when IF=1. A CPU that reaches // the idle thread through an unusual switch path must not sleep with // interrupts disabled, otherwise reschedule/TLB shootdown IPIs can // be delayed forever. @@ -282,7 +283,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { KeInitThreadFpuContext(idle); // The hand-built CPU idle thread is not created through KeCreateThread(), - // so register it manually with the idle process. This keeps PID 0 visible + // so register it manually with the idle process. This keeps PID 0 visible // as the real idle process while kernel worker threads live under PID 1. ObReferenceObject(&gIdleProcess->Header); uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock); @@ -418,8 +419,8 @@ static void KiSchedule(void) { uintptr_t flags = KeAcquireSpinlock(&gSchedLock); KiArmPostSwitchUnlock(cur, flags); - // The currently running thread is not present in a run queue. If it is - // still runnable and nobody else is ready, keep executing it. Otherwise a + // The currently running thread is not present in a run queue. If it is + // still runnable and nobody else is ready, keep executing it. Otherwise a // lone CPU-bound thread gets alternated with the idle thread on every timer // tick, effectively turning a periodic preemption check into a 50% throttle. if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableForCpuLocked(cpu)) { @@ -508,7 +509,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - // Complete the wait while holding the scheduler lock. On SMP a wake can + // Complete the wait while holding the scheduler lock. On SMP a wake can // race the prepare->commit window on another CPU: the target thread is // already on the object's wait list and has State=ThreadWaiting, but it is // still executing with RunningCpu set until it actually calls KeYield(). @@ -529,7 +530,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { thread->QuantumRemaining = 0; if (thread->RunningCpu != KE_INVALID_CPU) { - // Prepare->commit race: the waiter is still running somewhere. Its + // Prepare->commit race: the waiter is still running somewhere. Its // KeCommitBlockThread() will see State != ThreadWaiting and return the // completed wait status without ever entering the ready queue. thread->State = ThreadRunning; @@ -598,7 +599,7 @@ void KeSchedulerTick(void) { } // Periodic aging pass to prevent indefinite starvation of PriorityLow and - // PriorityNormal threads. Ready threads promoted by aging get a fresh CPU + // PriorityNormal threads. Ready threads promoted by aging get a fresh CPU // burst history; they earned the boost by waiting, not by burning CPU. gAgingCounter++; if (gAgingCounter >= SCHED_AGING_INTERVAL_TICKS) { diff --git a/code/kernel/ke_selftest.c b/code/kernel/ke_selftest.c index ed20938..2acfa14 100644 --- a/code/kernel/ke_selftest.c +++ b/code/kernel/ke_selftest.c @@ -42,7 +42,7 @@ static void KiApcSelfTestKernelRoutine(KAPC* apc, void** normalContext, void** systemArgument1, void** systemArgument2) { - DbgPrint("APC selftest: kernel routine apc=0x%llX normal=0x%llX ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n", + DbgPrint("APC selftest: kernel routine apc=0x%016llX normal=0x%016llX ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n", (uint64_t)apc, (uint64_t)*normalRoutine, (uint64_t)*normalContext, @@ -56,7 +56,7 @@ static void KiApcSelfTestNormalRoutine(void* normalContext, void* systemArgument2) { volatile int* counter = (volatile int*)normalContext; - DbgPrint("APC selftest: normal routine ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n", + DbgPrint("APC selftest: normal routine ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n", (uint64_t)normalContext, (uint64_t)systemArgument1, (uint64_t)systemArgument2, @@ -73,7 +73,7 @@ static void KiApcSelfTestCrossThreadRoutine(void* normalContext, void* systemArgument2) { KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext; - DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n", + DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n", (uint64_t)systemArgument1, (uint64_t)systemArgument2, KeGetCurrentIrql()); @@ -124,7 +124,7 @@ static void KiApcAlertableSelfTestNormalRoutine(void* normalContext, KI_APC_ALERTABLE_SELFTEST_STATE* st = (KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext; - DbgPrint("APC selftest: alertable APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n", + DbgPrint("APC selftest: alertable APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n", (uint64_t)systemArgument1, (uint64_t)systemArgument2, KeGetCurrentIrql()); @@ -366,7 +366,7 @@ static void KiIrpApcSelfTestKernelRoutine(KAPC* apc, void** normalContext, void** systemArgument1, void** systemArgument2) { - DbgPrint("IRP/APC selftest: kernel routine apc=0x%llX irp=0x%llX status=%u irql=%u\n", + DbgPrint("IRP/APC selftest: kernel routine apc=0x%016llX irp=0x%016llX status=%u irql=%u\n", (uint64_t)apc, (uint64_t)*systemArgument1, (uint32_t)(uintptr_t)*systemArgument2, @@ -383,7 +383,7 @@ static void KiIrpApcSelfTestNormalRoutine(void* normalContext, IRP* irp = (IRP*)systemArgument1; KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2; - DbgPrint("IRP/APC selftest: APC delivered irp=0x%llX status=%u info=%llu irql=%u\n", + DbgPrint("IRP/APC selftest: APC delivered irp=0x%016llX status=%u info=%llu irql=%u\n", (uint64_t)irp, status, (uint64_t)(irp ? irp->Information : 0), @@ -553,7 +553,7 @@ static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc, void** normalContext, void** systemArgument1, void** systemArgument2) { - DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%llX iosb=0x%llX status=%u irql=%u\n", + DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%016llX iosb=0x%016llX status=%u irql=%u\n", (uint64_t)apc, (uint64_t)*systemArgument1, (uint32_t)(uintptr_t)*systemArgument2, @@ -570,7 +570,7 @@ static void KiFileAsyncSelfTestNormalRoutine(void* normalContext, (KI_FILE_ASYNC_SELFTEST_STATE*)normalContext; KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2; - DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%llX status=%u iosb=%u/%llu irql=%u\n", + DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%016llX status=%u iosb=%u/%llu irql=%u\n", (uint64_t)systemArgument1, status, st->Iosb.Status, @@ -759,10 +759,10 @@ void KiRunCoreHardeningSelfTests(void) { KCPU* cpu = KeGetCurrentCpu(); if (cpu != &gCpus[0]) - KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%llX cpu0=0x%llX\n", + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%016llX cpu0=0x%016llX\n", (uint64_t)cpu, (uint64_t)&gCpus[0]); if (cpu->Self != cpu) - KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%llX cpu=0x%llX\n", + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%016llX cpu=0x%016llX\n", (uint64_t)cpu->Self, (uint64_t)cpu); if (KeGetCurrentCpuIndex() != 0) KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n"); @@ -776,15 +776,17 @@ void KiRunCoreHardeningSelfTests(void) { if (KeGetCurrentTss() != &cpu->Tss) KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n"); if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64)) - KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%llX iomap=%u\n", + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%016llX iomap=%u\n", cpu->Tss.RSP0, cpu->Tss.IoMapBase); if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt) - KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%llX gdt=0x%llX\n", + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%016llX gdt=0x%016llX\n", cpu->Gdtr.Base, (uint64_t)cpu->Gdt); for (uint32_t i = 1; i < gCpuCount; i++) { KCPU* ap = &gCpus[i]; - if (!ap->Present || !ap->Started || ap->Parked || !ap->Online || ap->IsBsp) + if (!ap->Present) + continue; + if (!ap->Started || ap->Parked || !ap->Online || ap->IsBsp) KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n", i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp); @@ -795,7 +797,7 @@ void KiRunCoreHardeningSelfTests(void) { if (!ap->KernelStack || !ap->KernelStackTop) KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i); if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64)) - KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%llX iomap=%u\n", + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%016llX iomap=%u\n", i, ap->Tss.RSP0, ap->Tss.IoMapBase); if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt) KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i); diff --git a/code/kernel/ke_semaphore.c b/code/kernel/ke_semaphore.c index 25c1c67..42e1d98 100644 --- a/code/kernel/ke_semaphore.c +++ b/code/kernel/ke_semaphore.c @@ -57,7 +57,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) { if (!sem) return; uintptr_t flags = KeAcquireSpinlock(&sem->Lock); if (!LIST_IS_EMPTY(&sem->WaitList)) { - // Transfer this semaphore release directly to the selected waiter. The + // Transfer this semaphore release directly to the selected waiter. The // count stays unchanged; the waiter returns success without looping and // consuming a count later. LIST_ENTRY* node = ListPopHead(&sem->WaitList); diff --git a/code/kernel/ke_smp.c b/code/kernel/ke_smp.c index eede9d5..7951889 100644 --- a/code/kernel/ke_smp.c +++ b/code/kernel/ke_smp.c @@ -8,6 +8,8 @@ #include #include #include +#pragma intrinsic(_disable) +#pragma intrinsic(_enable) #include #include #include @@ -22,6 +24,8 @@ #include #define AP_TRAMPOLINE_MAX_PHYS 0x000A0000ULL +#define AP_TRAMPOLINE_PREFERRED_MIN 0x00070000ULL +#define AP_TRAMPOLINE_FALLBACK_MIN 0x00010000ULL #define AP_TRAMPOLINE_BOOT_OFF 0x0800u #define AP_STACK_PAGES 8u #define AP_START_TIMEOUT_SPINS 20000000u @@ -74,6 +78,7 @@ _Static_assert(offsetof(AP_BOOT_BLOCK, State) == 48, "AP_BOOT_BLOCK.State offset _Static_assert(offsetof(AP_BOOT_BLOCK, Stage) == 52, "AP_BOOT_BLOCK.Stage offset"); static PHYSICAL_ADDRESS gApTrampolinePhys; +static PHYSICAL_ADDRESS gReservedApTrampolinePhys; static AP_BOOT_BLOCK* gApBootBlock; static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; } @@ -94,13 +99,55 @@ static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, u ((uint64_t)((base >> 24) & 0xFF) << 56); } +void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys) { + if (trampolinePhys >= PAGE_SIZE && + trampolinePhys < AP_TRAMPOLINE_MAX_PHYS && + !(trampolinePhys & (PAGE_SIZE - 1))) { + gReservedApTrampolinePhys = (PHYSICAL_ADDRESS)trampolinePhys; + } else { + gReservedApTrampolinePhys = 0; + } +} + static KSTATUS KiBuildApTrampoline(void) { if (gApTrampolinePhys) return STATUS_SUCCESS; - PHYSICAL_ADDRESS frame = MmAllocateFrameBelow(AP_TRAMPOLINE_MAX_PHYS); - if (!frame || frame < PAGE_SIZE || (frame & (PAGE_SIZE - 1))) + PHYSICAL_ADDRESS frame = gReservedApTrampolinePhys; + if (frame) { + if (frame < AP_TRAMPOLINE_FALLBACK_MIN || + frame >= AP_TRAMPOLINE_MAX_PHYS || + (frame & (PAGE_SIZE - 1))) { + DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n", + (uint64_t)frame); + frame = 0; + } else { + DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n", + (uint64_t)frame); + } + } + + if (!frame) { + DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n", + (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN, + (uint64_t)AP_TRAMPOLINE_MAX_PHYS); + frame = MmAllocateFrameInRange(AP_TRAMPOLINE_PREFERRED_MIN, + AP_TRAMPOLINE_MAX_PHYS); + } + if (!frame) { + DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n", + (uint64_t)AP_TRAMPOLINE_FALLBACK_MIN, + (uint64_t)AP_TRAMPOLINE_PREFERRED_MIN); + frame = MmAllocateFrameInRange(AP_TRAMPOLINE_FALLBACK_MIN, + AP_TRAMPOLINE_PREFERRED_MIN); + } + if (!frame) { + DbgPrint("SMP: no free PMM frame available below 0x%016llX for AP trampoline\n", + (uint64_t)AP_TRAMPOLINE_MAX_PHYS); return STATUS_NO_MEMORY; + } + + DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame); uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame); memset(page, 0, PAGE_SIZE); @@ -150,7 +197,7 @@ static KSTATUS KiBuildApTrampoline(void) { while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90); uint32_t long64Off = (uint32_t)(p - page); - // --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). --- + // --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). --- KiEmit8(&p, 0x48); KiEmit8(&p, 0xBB); size_t bootBlockPatch = (size_t)(p - page); KiEmit64(&p, 0); // mov rbx, bootblock KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Stage)); KiEmit32(&p, AP_STAGE_LONG64); KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, State)); KiEmit32(&p, AP_STATE_LONG_MODE); @@ -201,7 +248,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) { } // APs start from the SIPI reset state, so do not assume the firmware/BSP - // CR4 feature bits are present. The scheduler and context-switch paths + // CR4 feature bits are present. The scheduler and context-switch paths // use FXSAVE/FXRSTOR on every CPU; without OSFXSR, those instructions // raise #UD as soon as an AP enters its idle thread. uintptr_t cr4 = __readcr4(); @@ -282,12 +329,20 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) { } static KSTATUS KiAllocateApStack(KCPU* cpu) { - PHYSICAL_ADDRESS stackPhys = MmAllocateContiguous(AP_STACK_PAGES); - if (!stackPhys) return STATUS_NO_MEMORY; + // AP kernel stacks do not need physically contiguous memory. Requiring + // MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being + // available, which is fragile on real machines and unrelated to AP startup. + size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE; + void* stack = MmAllocatePool(stackBytes); + if (!stack) { + DbgPrint("SMP: AP%u stack allocation failed bytes=%llu\n", + cpu ? cpu->CpuIndex : 0xFFFFFFFFu, (uint64_t)stackBytes); + return STATUS_NO_MEMORY; + } - cpu->KernelStack = (void*)(PHYS_MAP_BASE + stackPhys); - cpu->KernelStackTop = (uintptr_t)cpu->KernelStack + AP_STACK_PAGES * PAGE_SIZE; - memset(cpu->KernelStack, 0, AP_STACK_PAGES * PAGE_SIZE); + cpu->KernelStack = stack; + cpu->KernelStackTop = (uintptr_t)stack + stackBytes; + memset(cpu->KernelStack, 0, stackBytes); memset(&cpu->Tss, 0, sizeof(cpu->Tss)); cpu->Tss.IoMapBase = sizeof(TSS64); cpu->Tss.RSP0 = cpu->KernelStackTop; @@ -295,6 +350,20 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) { return STATUS_SUCCESS; } + +static const char* KiApStageName(uint32_t stage) { + switch (stage) { + case AP_STAGE_NONE: return "none"; + case AP_STAGE_LONG64: return "long64"; + case AP_STAGE_ENTRY: return "entry"; + case AP_STAGE_GS: return "gs"; + case AP_STAGE_GDT: return "gdt"; + case AP_STAGE_IDT: return "idt"; + case AP_STAGE_PARK: return "park"; + default: return "unknown"; + } +} + static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) { memset(gApBootBlock, 0, sizeof(*gApBootBlock)); gApBootBlock->Magic = AP_BOOT_MAGIC; @@ -326,30 +395,53 @@ KSTATUS KeStartApplicationProcessors(void) { bsp->SchedulerCommand = 0; bsp->SchedulerActive = 1; + DbgPrint("SMP: APIC mode=%s BSP LAPIC=%u MADT CPUs=%u\n", + HalApicUsingX2Apic() ? "x2APIC" : "xAPIC", + bspLapic, topo.CpuCount); + uint32_t count = 1; for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) { HAL_CPU_INFO* info = &topo.Cpus[i]; + DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n", + i, info->ProcessorUid, info->LapicId, info->Flags, + info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId)); + if (!info->Enabled) continue; if (info->LapicId == bspLapic) { bsp->AcpiProcessorId = info->ProcessorUid; continue; } + if (!HalApicCanAddressId(info->LapicId)) { + DbgPrint("SMP: skipping AP uid=%u lapic=%u; current APIC mode cannot address it\n", + info->ProcessorUid, info->LapicId); + continue; + } KiPrepareCpuRecord(count, info); count++; } gCpuCount = count; - DbgPrint("SMP: discovered %u CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic); + DbgPrint("SMP: discovered %u addressable CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic); if (gCpuCount <= 1) { DbgPrint("SMP: no application processors present\n"); return STATUS_SUCCESS; } - if ((__readcr3() >> 32) != 0) - return STATUS_NOT_SUPPORTED; + if ((__readcr3() >> 32) != 0) { + DbgPrint("SMP: disabled; bootstrap CR3 is above 4GiB (cr3=0x%016llX), AP trampoline cannot load it yet\n", + (uint64_t)__readcr3()); + gCpuCount = 1; + return STATUS_SUCCESS; + } + DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n", + (uint64_t)gReservedApTrampolinePhys); st = KiBuildApTrampoline(); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) { + DbgPrint("SMP: disabled; AP trampoline allocation/build failed status=0x%x\n", st); + gCpuCount = 1; + return STATUS_SUCCESS; + } uint8_t sipiVector = (uint8_t)(gApTrampolinePhys >> 12); uint32_t claimed = 0; @@ -359,13 +451,29 @@ KSTATUS KeStartApplicationProcessors(void) { if (!cpu->Present) continue; st = KiAllocateApStack(cpu); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) { + DbgPrint("SMP: AP%u LAPIC=%u stack allocation failed status=0x%x; disabling AP and continuing\n", + i, cpu->LapicId, st); + cpu->Present = 0; + cpu->Started = 0; + cpu->Parked = 0; + cpu->Online = 0; + continue; + } KiFillBootBlockForCpu(cpu, i); DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector); st = HalStartApplicationProcessor(cpu->LapicId, sipiVector); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) { + DbgPrint("SMP: AP%u LAPIC=%u startup IPI failed status=0x%x; disabling AP and continuing\n", + i, cpu->LapicId, st); + cpu->Present = 0; + cpu->Started = 0; + cpu->Parked = 0; + cpu->Online = 0; + continue; + } uint64_t lastHeartbeat = gApBootBlock->Heartbeat; int parked = 0; @@ -382,10 +490,14 @@ KSTATUS KeStartApplicationProcessors(void) { } if (!parked) { - KePanic(KE_PANIC_CODE_INIT, - "SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x heartbeat=%llu last=%llu\n", - i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage, - gApBootBlock->Heartbeat, lastHeartbeat); + DbgPrint("SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n", + i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage, + KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat); + cpu->Present = 0; + cpu->Started = 0; + cpu->Parked = 0; + cpu->Online = 0; + continue; } cpu->Started = 1; @@ -395,6 +507,9 @@ KSTATUS KeStartApplicationProcessors(void) { DbgPrint("SMP: AP%u reached kernel park\n", i); } + if (claimed == 0 && gCpuCount > 1) + DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n"); + DbgPrint("SMP: kernel claimed %u APs\n", claimed); return STATUS_SUCCESS; } @@ -482,7 +597,7 @@ struct SMP_AP_WORK_TEST { static void KiSmpApControlledWorker(void* context) { SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context; if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) { - KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%llX\n", (uint64_t)arg); + KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%016llX\n", (uint64_t)arg); } _InterlockedExchange(&arg->Ready, 1); diff --git a/code/kernel/ke_switch.asm b/code/kernel/ke_switch.asm index 3e2a60b..c611d9c 100644 --- a/code/kernel/ke_switch.asm +++ b/code/kernel/ke_switch.asm @@ -50,7 +50,7 @@ KeSwitchContext PROC ; rsp = to->SavedContext mov rsp, [rdx + THREAD_SavedContext] - ; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the + ; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the ; incoming thread's user TEB in IA32_KERNEL_GS_BASE; user-mode entry/exit ; swaps it with the kernel GS base using swapgs. mov rax, [rdx + THREAD_Teb] ; Fetch the incoming user GS/TEB pointer @@ -80,9 +80,9 @@ KeSwitchContext ENDP ; void KeStartInitialThread(THREAD* to) ; RCX = to ; -; Used by AP scheduler admission. The AP is already in kernel long mode on a +; Used by AP scheduler admission. The AP is already in kernel long mode on a ; AP bootstrap stack; this helper installs the idle thread's stack/TSS -; state and returns directly into that thread's saved context. There is no +; state and returns directly into that thread's saved context. There is no ; outgoing THREAD and no scheduler lock to release. ; ------------------------------------------------------------------------- KeStartInitialThread PROC @@ -94,7 +94,7 @@ KeStartInitialThread PROC call KeSetCurrentTssRsp0 add rsp, 40 - ; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel + ; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel ; threads have a NULL TEB, which is fine. mov rax, [rbx + THREAD_Teb] mov ecx, 0C0000102h diff --git a/code/kernel/ke_syscall.asm b/code/kernel/ke_syscall.asm index 54b5e37..168b0bd 100644 --- a/code/kernel/ke_syscall.asm +++ b/code/kernel/ke_syscall.asm @@ -1,7 +1,7 @@ ; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64) ; ; Syscall entry now builds an editable KTRAP_FRAME on the current kernel -; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET +; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET ; slots used by user APC return redirection. EXTERN KeSyscallDispatch:PROC @@ -147,8 +147,8 @@ KeSyscallEntry ENDP ; Fault stubs - snapshot a full user fault frame before entering C. ; ; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless -; by itself. The captured user RSP lets the C crash dumper read [RSP], which is -; usually the return address immediately after the bad CALL. The GPR snapshot +; by itself. The captured user RSP lets the C crash dumper read [RSP], which is +; usually the return address immediately after the bad CALL. The GPR snapshot ; also shows which register held the null target. ; ; KeFaultDispatchFrame signature (MS x64): @@ -239,7 +239,8 @@ KeFaultDE PROC jmp de_call de_kernel_frame: - mov qword ptr [rax + KF_Rsp], 0 + lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS + mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 de_call: @@ -304,7 +305,8 @@ KeFaultGP PROC jmp gp_call gp_kernel_frame: - mov qword ptr [rax + KF_Rsp], 0 + lea r10, [rdx+32] ; same-privilege fault: original RSP is above ERR/RIP/CS/RFLAGS + mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 gp_call: @@ -370,7 +372,8 @@ KeFaultPF PROC jmp pf_call pf_kernel_frame: - mov qword ptr [rax + KF_Rsp], 0 + lea r10, [rdx+32] ; same-privilege #PF frame has ERR/RIP/CS/RFLAGS + mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 pf_call: @@ -438,7 +441,8 @@ StubName PROC jmp noerr_call noerr_kernel_frame: - mov qword ptr [rax + KF_Rsp], 0 + lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS + mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 noerr_call: @@ -502,7 +506,8 @@ StubName PROC jmp err_call err_kernel_frame: - mov qword ptr [rax + KF_Rsp], 0 + lea r10, [rdx+32] ; same-privilege error-code frame has ERR/RIP/CS/RFLAGS + mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 err_call: @@ -516,8 +521,8 @@ ENDM EXTERN KeHandleNmiIpi:PROC PUBLIC KeNmiIpiStub KeNmiIpiStub PROC - ; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1, - ; the NMI was a panic/TLB-control event and is fully handled. If it + ; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1, + ; the NMI was a panic/TLB-control event and is fully handled. If it ; returns 0, restore the interrupted context and fall through to the normal ; #NMI fault stub so unexpected NMIs still panic with a useful frame. push rax diff --git a/code/kernel/ke_syscall.c b/code/kernel/ke_syscall.c index 9d10063..0927192 100644 --- a/code/kernel/ke_syscall.c +++ b/code/kernel/ke_syscall.c @@ -42,6 +42,16 @@ #include #include #include + +#ifndef PROCESS_LAUNCH_DEBUG +#define PROCESS_LAUNCH_DEBUG 0 +#endif + +#if PROCESS_LAUNCH_DEBUG +#define ProcDbg(...) DbgPrint(__VA_ARGS__) +#else +#define ProcDbg(...) ((void)0) +#endif #include #include #include @@ -49,6 +59,7 @@ #include #include #include +#include #include // --------------------------------------------------------------------------- @@ -121,7 +132,7 @@ static KSTATUS KiAllocHandleForCurrentWithRights(OBJECT* obj, uint32_t rights, u KSTATUS s = KeAllocateHandleWithRights(proc, obj, rights, handleOut); // Most syscall create/open paths receive a fresh creator reference from the - // subsystem. The handle table takes its own reference; this helper always + // subsystem. The handle table takes its own reference; this helper always // drops the creator reference so the object is owned only by the returned // handle (and any subsystem-internal refs). ObDereferenceObject(obj); @@ -752,7 +763,7 @@ static void SyscallUserAsyncKernelRoutine(KAPC* apc, (void)systemArgument2; SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc); - DbgPrint("USER/APC async: kernel routine ctx=0x%llX iosb=0x%llX status=%u info=%llu irql=%u\n", + DbgPrint("USER/APC async: kernel routine ctx=0x%016llX iosb=0x%016llX status=%u info=%llu irql=%u\n", (uint64_t)ctx, (uint64_t)(uintptr_t)apc->SystemArgument1, (uint32_t)(uintptr_t)apc->SystemArgument2, @@ -1150,81 +1161,72 @@ static uint64_t SyscallScreenPutChar(uint64_t a0, uint64_t a1, uint64_t a2, } // --------------------------------------------------------------------------- -// SYS_KB_POLL (40) -> 1 or 0 +// SYS_KB_POLL (64) -> next INPUT_KEY or INPUT_KEY_NONE // --------------------------------------------------------------------------- static uint64_t SyscallKbPoll(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (!gHal) return 0; - return gHal->Keyboard.Poll(); + return (uint64_t)KeInputKbPoll(); } // --------------------------------------------------------------------------- -// SYS_KB_IS_KEY_DOWN (41) a0=scancode a1=extended -> 1 or 0 +// SYS_KB_IS_KEY_DOWN (65) a0=INPUT_KEY -> 1 or 0 // --------------------------------------------------------------------------- static uint64_t SyscallKbIsKeyDown(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { - (void)a2; (void)a3; (void)a4; (void)a5; - - // Check gHal and ensure the HAL struct actually has the new function pointer - if (!gHal || !gHal->Keyboard.IsKeyDown) return 0; - - return gHal->Keyboard.IsKeyDown((uint8_t)a0, (bool)a1) ? 1u : 0u; + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KeInputIsKeyDown((INPUT_KEY)a0) ? 1u : 0u; } // --------------------------------------------------------------------------- -// SYS_KB_READ_CHAR (42) -> char (blocks, yields while waiting) +// SYS_KB_READ_CHAR (66) -> char (blocks, yields while waiting) // --------------------------------------------------------------------------- static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (!gHal) return 0; - while (!gHal->Keyboard.Poll()) KeYield(); - return (uint64_t)(uint8_t)gHal->Keyboard.ReadChar(); + while (!KeInputKbPoll()) KeYield(); + return (uint64_t)(uint8_t)KeInputReadChar(); } // --------------------------------------------------------------------------- -// SYS_KB_READ_SCANCODE (43) -> uint8_t (blocks, yields while waiting) +// SYS_KB_READ_KEY (67) -> INPUT_KEY (blocks, yields while waiting) // --------------------------------------------------------------------------- -static uint64_t SyscallKbReadScancode(uint64_t a0, uint64_t a1, uint64_t a2, - uint64_t a3, uint64_t a4, uint64_t a5) { +static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { (void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (!gHal) return 0; - while (!gHal->Keyboard.Poll()) KeYield(); - return (uint64_t)gHal->Keyboard.ReadScancode(); + while (!KeInputKbPoll()) KeYield(); + return (uint64_t)KeInputReadKey(); } // --------------------------------------------------------------------------- -// SYS_KB_GET_MODIFIERS (44) -> uint32_t +// SYS_KB_GET_MODIFIERS (68) -> uint32_t INPUT_MOD_* flags // --------------------------------------------------------------------------- static uint64_t SyscallKbGetModifiers(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (!gHal) return 0; - return (uint64_t)gHal->Keyboard.GetModifiers(); + return (uint64_t)KeInputGetModifiers(); } // --------------------------------------------------------------------------- -// SYS_MOUSE_POLL (45) -> 1 or 0 +// SYS_MOUSE_POLL (69) -> 1 or 0 // --------------------------------------------------------------------------- static uint64_t SyscallMousePoll(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; - if (!gHal) return 0; - return gHal->Mouse.Poll() ? 1u : 0u; + return KeInputMousePoll() ? 1u : 0u; } // --------------------------------------------------------------------------- -// SYS_MOUSE_READ (46) a0=*dx a1=*dy a2=*buttons -> 1 or 0 +// SYS_MOUSE_READ (70) a0=*dx a1=*dy a2=*buttons -> 1 or 0 // --------------------------------------------------------------------------- static uint64_t SyscallMouseRead(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a3; (void)a4; (void)a5; - if (!gHal || !a0 || !a1 || !a2) return 0; + if (!a0 || !a1 || !a2) return 0; int dx = 0, dy = 0; uint8_t buttons = 0; - if (!gHal->Mouse.Read(&dx, &dy, &buttons)) return 0; + if (!KeInputMouseRead(&dx, &dy, &buttons)) return 0; if (!KSUCCESS(CopyToUser((void*)(uintptr_t)a0, &dx, sizeof(dx)))) return 0; if (!KSUCCESS(CopyToUser((void*)(uintptr_t)a1, &dy, sizeof(dy)))) return 0; @@ -1233,15 +1235,15 @@ static uint64_t SyscallMouseRead(uint64_t a0, uint64_t a1, uint64_t a2, } // --------------------------------------------------------------------------- -// SYS_MOUSE_GET_POS (47) a0=*x a1=*y +// SYS_MOUSE_GET_POS (71) a0=*x a1=*y // --------------------------------------------------------------------------- static uint64_t SyscallMouseGetPos(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a2; (void)a3; (void)a4; (void)a5; - if (!gHal || !a0 || !a1) return STATUS_INVALID_PARAMETER; + if (!a0 || !a1) return STATUS_INVALID_PARAMETER; int x = 0, y = 0; - gHal->Mouse.GetPosition(&x, &y); + KeInputMouseGetPosition(&x, &y); KSTATUS s = KiCopyIntToUser(a0, x); if (!KSUCCESS(s)) return s; return KiCopyIntToUser(a1, y); @@ -1334,22 +1336,25 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, // Read the executable from disk. FILE_OBJECT* f; + ProcDbg("CreateProcess: path='%v' cmd='%v' callerPid=%u\n", path, cmdlineView, KeGetCurrentProcess() ? KeGetCurrentProcess()->ProcessId : 0); s = IoOpenFile(path, &f); - if (!KSUCCESS(s)) return s; + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: IoOpenFile failed st=%d\n", s); return s; } uint64_t size = IoGetFileSize(f); + ProcDbg("CreateProcess: fileSize=%llu\n", size); void* mem = MmAllocatePool(size); - if (!mem) { IoCloseFile(f); return STATUS_NO_MEMORY; } + if (!mem) { IoCloseFile(f); ProcDbg("CreateProcess: raw image pool alloc failed size=%llu\n", size); return STATUS_NO_MEMORY; } size_t got; s = IoReadFile(f, mem, size, &got); IoCloseFile(f); - if (!KSUCCESS(s)) { MmFreePool(mem); return s; } + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: IoReadFile failed st=%d got=%llu\n", s, (unsigned long long)got); MmFreePool(mem); return s; } // Create the process object. PROCESS* proc; s = KeCreateProcess(pathBuf, &proc); - if (!KSUCCESS(s)) { MmFreePool(mem); return s; } + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateProcess failed st=%d\n", s); MmFreePool(mem); return s; } + ProcDbg("CreateProcess: created pid=%u name=%s\n", proc->ProcessId, proc->Name); // Inherit CWD from parent PROCESS* parent = KeGetCurrentProcess(); @@ -1368,34 +1373,37 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, path, cmdlineView, SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength)); - if (!KSUCCESS(s)) { MmFreePool(mem); ObDereferenceObject(&proc->Header); return s; } + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: env setup failed st=%d\n", s); MmFreePool(mem); ObDereferenceObject(&proc->Header); return s; } // Map the PE image into memory. uintptr_t imageBase = 0; - s = LdrLoadImage(mem, size, &imageBase, true); + s = LdrLoadImageForProcess(proc, mem, size, &imageBase); MmFreePool(mem); - if (!KSUCCESS(s)) { ObDereferenceObject(&proc->Header); return s; } + if (!KSUCCESS(s)) { ProcDbg("CreateProcess: LdrLoadImage failed st=%d\n", s); ObDereferenceObject(&proc->Header); return s; } + ProcDbg("CreateProcess: image mapped pid=%u base=%016llx size=%llx\n", proc->ProcessId, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(proc, imageBase)); s = LdrResolveImportsForProcess(proc, imageBase); if (!KSUCCESS(s)) { - MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase)); + ProcDbg("CreateProcess: resolve imports failed st=%d pid=%u image=%016llx\n", s, proc->ProcessId, (unsigned long long)imageBase); + MmDebugDumpUserRegions(proc); ObDereferenceObject(&proc->Header); return s; } + ProcDbg("CreateProcess: imports resolved pid=%u modules=%u\n", proc->ProcessId, proc->ModuleCount); - s = LdrProtectImage(imageBase, true); + s = LdrProtectImageForProcess(proc, imageBase, true); if (!KSUCCESS(s)) { - MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase)); + ProcDbg("CreateProcess: protect image failed st=%d pid=%u\n", s, proc->ProcessId); + MmDebugDumpUserRegions(proc); ObDereferenceObject(&proc->Header); return s; } proc->ImageBase = imageBase; - proc->ImageSize = LdrGetImageSize(imageBase); + proc->ImageSize = LdrGetMappedImageSize(proc, imageBase); PROCESS* caller = KeGetCurrentProcess(); if (!caller) { - MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase)); ObDereferenceObject(&proc->Header); return STATUS_FAILURE; } @@ -1405,7 +1413,6 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, if (!KSUCCESS(s)) { proc->ImageBase = 0; proc->ImageSize = 0; - MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase)); ObDereferenceObject(&proc->Header); return s; } @@ -1414,10 +1421,13 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, uintptr_t entryPoint = LdrGetEntryPoint(imageBase); s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread); if (!KSUCCESS(s)) { + ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint); + MmDebugDumpUserRegions(proc); KeCloseHandle(caller, handle); ObDereferenceObject(&proc->Header); return s; } + ProcDbg("CreateProcess: thread created pid=%u tid=%llu entry=%016llx handle=%u\n", proc->ProcessId, (unsigned long long)thread->ThreadId, (unsigned long long)entryPoint, handle); s = KiCopyHandleToUser((uint64_t)(uintptr_t)req.HandleOut, handle); if (!KSUCCESS(s)) { @@ -1427,7 +1437,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, return s; } - // Drop the creator's local reference. The returned handle and live thread + // Drop the creator's local reference. The returned handle and live thread // references now own the process lifetime. ObDereferenceObject(&proc->Header); return STATUS_SUCCESS; @@ -1666,7 +1676,7 @@ static uint64_t SyscallGetProcessPid(uint64_t a0, uint64_t a1, uint64_t a2, // Named Queue Registry // // A flat, fixed-size table of { name -> QUEUE* } pairs protected by a -// spinlock. The registry holds one reference to each published queue for +// spinlock. The registry holds one reference to each published queue for // the lifetime of the entry, so the queue object cannot be destroyed while // it is registered even if every process closes its own handle to it. // @@ -1747,7 +1757,7 @@ static uint64_t SyscallPublishQueue(uint64_t a0, uint64_t a1, uint64_t a2, // SYS_OPEN_NAMED_QUEUE (61) a0=*name a1=*handleOut // // Looks up 'name' in the registry and allocates a handle to the queue in the -// calling process's handle table. Returns STATUS_NOT_FOUND if no match. +// calling process's handle table. Returns STATUS_NOT_FOUND if no match. // Callers that race against the publisher should retry on NOT_FOUND. // --------------------------------------------------------------------------- static uint64_t SyscallOpenNamedQueue(uint64_t a0, uint64_t a1, uint64_t a2, @@ -2629,7 +2639,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { /* 64 SYS_KB_POLL */ SyscallKbPoll, /* 65 SYS_KB_IS_KEY_DOWN */ SyscallKbIsKeyDown, /* 66 SYS_KB_READ_CHAR */ SyscallKbReadChar, - /* 67 SYS_KB_READ_SCANCODE */ SyscallKbReadScancode, + /* 67 SYS_KB_READ_KEY */ SyscallKbReadKey, /* 68 SYS_KB_GET_MODIFIERS */ SyscallKbGetModifiers, /* 69 SYS_MOUSE_POLL */ SyscallMousePoll, /* 70 SYS_MOUSE_READ */ SyscallMouseRead, diff --git a/code/kernel/ke_thread.c b/code/kernel/ke_thread.c index 70a3eb2..6c3cd27 100644 --- a/code/kernel/ke_thread.c +++ b/code/kernel/ke_thread.c @@ -16,7 +16,7 @@ #define FPU_FCW_DEFAULT 0x037Fu #define FPU_MXCSR_DEFAULT 0x1F80u -// These offsets are consumed directly by ke_switch.asm. Keep the asserts +// These offsets are consumed directly by ke_switch.asm. Keep the asserts // here so THREAD layout edits fail at build time instead of corrupting // context switches at runtime. #define STATIC_ASSERT_OFFSET(type, field, expected) \ @@ -41,6 +41,20 @@ static void KiThreadDestroy(OBJECT* obj) { thread->ExitEvent = (EVENT*)0; } + PROCESS* ownerForUserRegions = thread->Owner; + if (ownerForUserRegions) { + if (thread->Teb && thread->TebSize) { + MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb); + thread->Teb = (void*)0; + thread->TebSize = 0; + } + + if (thread->UserStackBase) { + MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase); + thread->UserStackBase = (void *)0; + } + } + if (thread->Owner) { PROCESS* owner = thread->Owner; uint32_t remaining = 0; @@ -63,17 +77,6 @@ static void KiThreadDestroy(OBJECT* obj) { MmFreePool(thread->KernelStackBase); thread->KernelStackBase = (void *)0; } - - if (thread->Teb && thread->TebSize) { - MmFreeUserMemory(thread->Teb); - thread->Teb = (void*)0; - thread->TebSize = 0; - } - - if (thread->UserStackBase) { - MmFreeUserMemory(thread->UserStackBase); - thread->UserStackBase = (void *)0; - } } __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg, @@ -119,8 +122,14 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE; thread->TssRsp0 = thread->KernelStackTop; - void* ustack = MmAllocateUserMemory(USTACK_SIZE); - if (!ustack) { + KSTATUS s; + void* ustack = (void*)0; + s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH, + UserRegionStack, + &ustack); + if (!KSUCCESS(s) || !ustack) { MmFreePool(kstack); thread->KernelStackBase = (void*)0; ObDereferenceObject(&thread->Header); @@ -128,14 +137,14 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui } thread->UserStackBase = ustack; // Ring-3 starts at the image entry via IRET, not CALL, so there is no - // pushed return address. The x64 ABI expects RSP % 16 == 8 at function + // pushed return address. The x64 ABI expects RSP % 16 == 8 at function // entry; bias the initial stack as if a call had happened so compiler // generated aligned SSE spills (movaps) are safe. uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8; - SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemory(sizeof(SYS_TEB)); + SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB)); if (!teb) { - MmFreeUserMemory(ustack); + MmFreeUserMemoryForProcess(owner, ustack); thread->UserStackBase = (void*)0; MmFreePool(kstack); thread->KernelStackBase = (void*)0; @@ -177,12 +186,12 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui // Allocate the exit event before the thread becomes visible. Manual-reset // so multiple joiners (if any) all wake up when the thread exits. EVENT* exitEvent; - KSTATUS s = KeEventCreate(EventManualReset, 0, &exitEvent); + s = KeEventCreate(EventManualReset, 0, &exitEvent); if (!KSUCCESS(s)) { - if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb); + if (thread->Teb && thread->TebSize) MmFreeUserMemoryForProcess(owner, thread->Teb); thread->Teb = (void*)0; thread->TebSize = 0; - MmFreeUserMemory(ustack); + MmFreeUserMemoryForProcess(owner, ustack); thread->UserStackBase = (void*)0; MmFreePool(kstack); thread->KernelStackBase = (void*)0; @@ -238,7 +247,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, THREAD_PRIORITY priority, uint64_t affinityMask, THREAD** out) { - // Kernel worker threads are owned by the system process. The scheduler's + // Kernel worker threads are owned by the system process. The scheduler's // hand-built CPU idle thread stays attached to PID 0 (idle), while driver, // async I/O, and init worker threads are grouped under PID 1 (system). PROCESS* owner = KeGetSystemProcess(); @@ -331,7 +340,7 @@ __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uin void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) { // Called with the synchronization object's spinlock held, so the caller is - // normally at DISPATCH_LEVEL here. This function only marks intent to + // normally at DISPATCH_LEVEL here. This function only marks intent to // block; the actual scheduling point is KeCommitBlockThread at PASSIVE. KeAssertIrqlAtOrBelow(DISPATCH_LEVEL); if (!thread) return; diff --git a/code/kernel/ke_trap.c b/code/kernel/ke_trap.c index a429bac..f91b930 100644 --- a/code/kernel/ke_trap.c +++ b/code/kernel/ke_trap.c @@ -56,7 +56,7 @@ KTRAP_FRAME* KeGetCurrentTrapFrame(void) { void KiEnterTrapFrame(KTRAP_FRAME* frame) { if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) - KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame); + KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame); THREAD* t = KeGetCurrentThread(); if (!t) return; @@ -68,13 +68,13 @@ void KiEnterTrapFrame(KTRAP_FRAME* frame) { void KiExitTrapFrame(KTRAP_FRAME* frame) { if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) - KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame); + KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame); // Give alertable user-mode returns a chance to redirect through the // registered user APC dispatcher before committing IRET slots. KiCheckForUserApcDelivery(frame); - // Commit editable return state. Kernel-mode interrupts have no RSP/SS + // Commit editable return state. Kernel-mode interrupts have no RSP/SS // hardware slots, so those pointers are NULL in that case. if (frame->RipSlot) *frame->RipSlot = frame->Rip; if (frame->CsSlot) *frame->CsSlot = frame->Cs; @@ -85,7 +85,7 @@ void KiExitTrapFrame(KTRAP_FRAME* frame) { THREAD* t = KeGetCurrentThread(); if (t) { if (t->TrapFrame != frame) - KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: trap-frame stack mismatch top=0x%llX frame=0x%llX\n", + KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: trap-frame stack mismatch top=0x%016llX frame=0x%016llX\n", (uint64_t)t->TrapFrame, (uint64_t)frame); t->TrapFrame = frame->PreviousFrame; } @@ -98,11 +98,11 @@ void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1, if (!f) KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: no current trap frame for nr=%llu\n", nr); if (f->Magic != KTRAP_FRAME_MAGIC) - KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: bad magic 0x%llX\n", f->Magic); + KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: bad magic 0x%016llX\n", f->Magic); if (f->Kind != KTRAP_KIND_SYSCALL) KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: kind=%u, expected syscall\n", f->Kind); if (!KeTrapFrameFromUser(f) || f->PreviousMode != KTRAP_MODE_USER) - KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: syscall did not come from user cs=0x%llX mode=%u\n", + KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: syscall did not come from user cs=0x%016llX mode=%u\n", f->Cs, f->PreviousMode); if (f->Rax != nr || f->Rcx != a0 || f->Rdx != a1 || f->R8 != a2 || f->R9 != a3 || f->R10 != a4 || f->R11 != a5) { diff --git a/code/kernel/kmain.c b/code/kernel/kmain.c index 199a5de..72a7af1 100644 --- a/code/kernel/kmain.c +++ b/code/kernel/kmain.c @@ -2,6 +2,8 @@ #include #include #include +#pragma intrinsic(_disable) +#pragma intrinsic(_enable) #include #include #include @@ -22,6 +24,7 @@ #include #include #include +#include #include typedef struct { @@ -31,52 +34,9 @@ typedef struct { typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *); -// HAL pointer cached at boot so kernel threads can reach it without -// threading a parameter through every call chain. +// HAL pointer cached at boot for subsystems that still need platform services. static const HAL_API* gKmainHal; -// Kernel-owned DPC wrappers for HAL input devices. The HAL keeps the tiny -// hardware-specific ISR fast and exposes a bottom-half entry point; the kernel -// owns the actual DPC objects and scheduling policy. -static KDPC gKeyboardDpc; -static KDPC gMouseDpc; - -static void KeyboardDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { - (void)dpc; - (void)context; - (void)systemArgument1; - (void)systemArgument2; - - if (gKmainHal && gKmainHal->Keyboard.Dpc) - gKmainHal->Keyboard.Dpc(); -} - -static void KiMouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { - (void)dpc; - (void)context; - (void)systemArgument1; - (void)systemArgument2; - - if (gKmainHal && gKmainHal->Mouse.Dpc) - gKmainHal->Mouse.Dpc(); -} - -static void KeyboardIrqWrapper(void) { - if (gKmainHal && gKmainHal->Keyboard.Isr) - gKmainHal->Keyboard.Isr(); - - if (gKmainHal && gKmainHal->Keyboard.Dpc) - KeInsertQueueDpc(&gKeyboardDpc, (void*)0, (void*)0); -} - -static void KiMouseIrqWrapper(void) { - if (gKmainHal && gKmainHal->Mouse.Isr) - gKmainHal->Mouse.Isr(); - - if (gKmainHal && gKmainHal->Mouse.Dpc) - KeInsertQueueDpc(&gMouseDpc, (void*)0, (void*)0); -} - static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) { for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) { const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i]; @@ -288,17 +248,14 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { // Read the executable file into a raw buffer. s = IoOpenFile(SysSvFromCstr(filePath), &f); - DbgPrint("KiLaunchUserProcess: IoOpenFile: %d\n", s); if (s != STATUS_SUCCESS) { DbgPrint("Failed to load file: \"%s\"!\n", filePath); return 0; } uint64_t size = IoGetFileSize(f); - DbgPrint("KiLaunchUserProcess: IoGetFileSize: %llu\n", size); void* mem = MmAllocatePool(size); - DbgPrint("KiLaunchUserProcess: MmAllocatePool: 0x%llX\n", mem); if (!mem) { IoCloseFile(f); return 0; @@ -306,7 +263,6 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { size_t got = 0; s = IoReadFile(f, mem, size, &got); - DbgPrint("KiLaunchUserProcess: IoReadFile: %d\n", s); IoCloseFile(f); if (!KSUCCESS(s)) { MmFreePool(mem); @@ -338,7 +294,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { // Parse the raw PE file and map it into executable memory. uintptr_t imageBase = 0; - s = LdrLoadImage(mem, size, &imageBase, true); + s = LdrLoadImageForProcess(proc, mem, size, &imageBase); MmFreePool(mem); if (!KSUCCESS(s)) { DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s); @@ -349,16 +305,14 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { s = LdrResolveImportsForProcess(proc, imageBase); if (!KSUCCESS(s)) { DbgPrint("KiLaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s); - MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase)); ObDereferenceObject(&proc->Header); return 0; } proc->ImageBase = imageBase; - proc->ImageSize = LdrGetImageSize(imageBase); + proc->ImageSize = LdrGetMappedImageSize(proc, imageBase); uintptr_t entryPoint = LdrGetEntryPoint(imageBase); - DbgPrint("KiLaunchUserProcess: PE Image mapped! Base 0x%llx, Entry 0x%llX\n", imageBase, entryPoint); // Create the main thread last. KeCreateThread queues it before returning. s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread); @@ -366,7 +320,6 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s); proc->ImageBase = 0; proc->ImageSize = 0; - MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase)); ObDereferenceObject(&proc->Header); return 0; } @@ -388,19 +341,6 @@ void SystemInitThread(void* arg) { KiResolveSystemImports(args->bootInfo); KiReclaimBootloaderMemory(args->bootInfo); - args->hal->Screen.Clear(0x000000); - - // Print memory status now that the system is fully alive - DbgPrint( - "Memory Status:\n" - " -> Free (PMM): %.2f MB\n" - " -> Kernel Pool: %.2f MB\n" - " -> User Heap: %.2f MB\n", - (float)MmFreeFrameCount() * PAGE_SIZE / (1024.0f * 1024.0f), - (float)MmGetKernelPoolUsage() / (1024.0f * 1024.0f), - (float)MmGetUserHeapUsage() / (1024.0f * 1024.0f) - ); - if (args->bootInfo->Debug & 2) KiRunKernelSelfTests(args->hal); @@ -464,7 +404,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) { .FramebufferHeight = bootInfo->FramebufferHeight, .FramebufferStride = bootInfo->FramebufferStride, .RsdpPhysical = bootInfo->RsdpPhysical, - .Debug = bootInfo->Debug + .Debug = bootInfo->Debug, + .AllocatePool = MmAllocatePool }; KSTATUS s = HalInitialize(&halParams); @@ -482,6 +423,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) { // the kernel owns KCPU records, per-CPU stacks, GDT/TSS/IDT state, and the // AP park loop. External device IRQs remain routed to the BSP during this // step; APs only become runnable after scheduler admission below. + KeSetApplicationProcessorTrampoline((uint64_t)bootInfo->ApTrampolinePhys); KSTATUS smpStatus = KeStartApplicationProcessors(); if (!KSUCCESS(smpStatus)) { KePanic(KE_PANIC_CODE_INIT, "SMP: failed to start application processors, status=0x%x\n", smpStatus); @@ -490,14 +432,9 @@ extern void KernelMain(BOOT_INFO* bootInfo) { KeIrqInitialize(hal); KeInitializeDpcSystem(); - KeInitializeDpc(&gKeyboardDpc, KeyboardDpcRoutine, (void*)0); - KeInitializeDpc(&gMouseDpc, KiMouseDpcRoutine, (void*)0); + KeInputInitialize(hal); KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr); - KeIrqRegister(hal->Keyboard.IrqNumber, - hal->Keyboard.Dpc ? KeyboardIrqWrapper : hal->Keyboard.Isr); - KeIrqRegister(hal->Mouse.IrqNumber, - hal->Mouse.Dpc ? KiMouseIrqWrapper : hal->Mouse.Isr); HalRegisterScheduler(KeSchedulerTick); diff --git a/code/kernel/ldr_image.c b/code/kernel/ldr_image.c index da84d48..b455eeb 100644 --- a/code/kernel/ldr_image.c +++ b/code/kernel/ldr_image.c @@ -8,6 +8,16 @@ #include #include +#ifndef LDR_DEBUG_VERBOSE +#define LDR_DEBUG_VERBOSE 0 +#endif + +#if LDR_DEBUG_VERBOSE +#define LdrDbg(...) DbgPrint(__VA_ARGS__) +#else +#define LdrDbg(...) ((void)0) +#endif + int _stricmp(const char* a, const char* b); // --------------------------------------------------------------------------- @@ -78,6 +88,42 @@ typedef struct { uint32_t Characteristics; } SECTION_HEADER; +static size_t KiAlignPeSize(size_t value) { + return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); +} + +static KSTATUS KiComputeRequiredImageSize(const PE_HEADER* pe, size_t fileSize, size_t* outSize) { + if (!pe || !outSize) return STATUS_INVALID_PARAMETER; + + const OPT_HEADER64* opt = &pe->OptionalHeader; + size_t required = KiAlignPeSize(opt->SizeOfImage); + size_t headers = KiAlignPeSize(opt->SizeOfHeaders); + if (headers > required) required = headers; + + const SECTION_HEADER* sec = (const SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader); + for (int i = 0; i < pe->NumberOfSections; i++) { + size_t va = (size_t)sec[i].VirtualAddress; + size_t virt = (size_t)sec[i].VirtualSize; + size_t raw = (size_t)sec[i].SizeOfRawData; + size_t span = virt ? virt : raw; + if (raw > span) span = raw; + if (!span) continue; + + if (va > (size_t)-1 - span) return STATUS_NO_MEMORY; + size_t end = KiAlignPeSize(va + span); + if (end > required) required = end; + + if (raw) { + size_t rawBase = (size_t)sec[i].PointerToRawData; + if (rawBase > fileSize || raw > fileSize - rawBase) return STATUS_FAILURE; + } + } + + if (!required) return STATUS_FAILURE; + *outSize = required; + return STATUS_SUCCESS; +} + typedef struct { uint32_t Characteristics; uint32_t TimeDateStamp; @@ -246,33 +292,77 @@ void* LdrGetExportForProcess(PROCESS* process, const char* moduleName, const cha } size_t LdrGetImageSize(uintptr_t base) { + if (!base) return 0; uint32_t peOffset = *(uint32_t *)(base + 0x3C); PE_HEADER* pe = (PE_HEADER *)(base + peOffset); if (pe->Signature != PE_SIGNATURE) return 0; return (size_t)pe->OptionalHeader.SizeOfImage; } - -static void KiFreeLoadedImage(uintptr_t imageBase, bool user) { - if (!imageBase) return; - size_t imageSize = LdrGetImageSize(imageBase); - if (user) MmFreeUserImage((void*)imageBase, imageSize); - else MmFreePool((void*)imageBase); +size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) { + if (!base) return 0; + if (process) { + size_t regionSize = MmGetUserRegionSize(process, (const void*)base); + if (regionSize) return regionSize; + } + size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base); + if (globalRegionSize) return globalRegionSize; + return LdrGetImageSize(base); } -KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) { +static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) { + if (!imageBase) return; + size_t imageSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase); + if (user) MmFreeUserImageForProcess(process, (void*)imageBase, imageSize); + else MmFreePool((void*)imageBase); +} + +static void KiFreeLoadedImage(uintptr_t imageBase, bool user) { + KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user); +} + + +KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) { if (!imageBase) return STATUS_INVALID_PARAMETER; + size_t mappedSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase); + if (!mappedSize) return STATUS_FAILURE; + uint32_t peOffset = *(uint32_t *)(imageBase + 0x3C); + if ((size_t)peOffset > mappedSize || mappedSize - (size_t)peOffset < sizeof(uint32_t) + 20) + return STATUS_FAILURE; + PE_HEADER* pe = (PE_HEADER *)(imageBase + peOffset); if (pe->Signature != PE_SIGNATURE) return STATUS_FAILURE; - SECTION_HEADER* sec = (SECTION_HEADER *)((uint8_t *)&pe->OptionalHeader + pe->SizeOfOptionalHeader); + size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader; + size_t sectionTableSize = (size_t)pe->NumberOfSections * sizeof(SECTION_HEADER); + if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset) + return STATUS_FAILURE; + + SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset); for (int i = 0; i < pe->NumberOfSections; i++) { size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData; if (!protectSize) continue; + uintptr_t secBase = imageBase + (uintptr_t)sec[i].VirtualAddress; + uintptr_t secEnd = secBase + protectSize; + uintptr_t mapEnd = imageBase + mappedSize; + + if (secBase < imageBase || secBase >= mapEnd) { + LdrDbg("LDR: protect skipping out-of-range section %d base=%016llx size=%llx image=%016llx mapped=%llx\n", + i, (unsigned long long)secBase, (unsigned long long)protectSize, + (unsigned long long)imageBase, (unsigned long long)mappedSize); + continue; + } + if (secEnd < secBase) return STATUS_NO_MEMORY; + if (secEnd > mapEnd) { + LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n", + i, (unsigned long long)secEnd, (unsigned long long)mapEnd); + protectSize = (size_t)(mapEnd - secBase); + } + uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0); if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE) prot |= PAGE_WRITE; @@ -280,12 +370,27 @@ KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) { (sec[i].Characteristics & IMAGE_SCN_CNT_CODE))) prot |= PAGE_NX; - KSTATUS s = MmSetPageProtect(imageBase + sec[i].VirtualAddress, protectSize, prot); - if (!KSUCCESS(s)) return s; + KSTATUS s; + if (user) { + s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot); + } else { + s = MmSetPageProtect(secBase, protectSize, prot); + } + if (!KSUCCESS(s)) { + LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n", + i, (unsigned long long)secBase, (unsigned long long)protectSize, + (unsigned long long)prot, s, + (unsigned long long)imageBase, (unsigned long long)mappedSize); + return s; + } } return STATUS_SUCCESS; } +KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) { + return LdrProtectImageForProcess(user ? (PROCESS*)0 : (PROCESS*)0, imageBase, user); +} + // --------------------------------------------------------------------------- // Entry point lookup // --------------------------------------------------------------------------- @@ -368,6 +473,7 @@ static KSTATUS LdrResolveImportsCommon( //Compatibility remap. dllName = KiCanonicalModuleName(dllName); + LdrDbg("LDR: resolve import dll=%s image=%016llx pid=%u\n", dllName, (unsigned long long)imageBase, process ? process->ProcessId : 0); // // Ensure dependency is loaded into the process @@ -469,7 +575,7 @@ static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uint strncpy(mod->Name, name, 31); mod->Name[31] = '\0'; mod->Base = base; - mod->Size = LdrGetImageSize(base); + mod->Size = LdrGetMappedImageSize(process, base); return STATUS_SUCCESS; } @@ -500,6 +606,7 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_ } uint64_t fileSize = IoGetFileSize(f); + LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize); void* mem = MmAllocatePool((size_t)fileSize); if (!mem) { IoCloseFile(f); @@ -515,39 +622,45 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_ } uintptr_t imageBase = 0; - s = LdrLoadImage(mem, (size_t)fileSize, &imageBase, true); + s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase); MmFreePool(mem); if (!KSUCCESS(s)) { + DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s); return s; } + LdrDbg("LDR: module mapped pid=%u file=%s base=%016llx size=%llx\n", + process->ProcessId, filename, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(process, imageBase)); s = LdrResolveImportsForProcess(process, imageBase); if (!KSUCCESS(s)) { - KiFreeLoadedImage(imageBase, true); + KiFreeLoadedImageForProcess(process, imageBase, true); return s; } - s = LdrProtectImage(imageBase, true); + s = LdrProtectImageForProcess(process, imageBase, true); if (!KSUCCESS(s)) { - KiFreeLoadedImage(imageBase, true); + KiFreeLoadedImageForProcess(process, imageBase, true); return s; } s = LdrRegisterProcessModule(process, filename, imageBase); if (!KSUCCESS(s)) { - KiFreeLoadedImage(imageBase, true); + KiFreeLoadedImageForProcess(process, imageBase, true); return s; } + LdrDbg("LDR: module registered pid=%u name=%s base=%016llx modules=%u\n", + process->ProcessId, filename, (unsigned long long)imageBase, process->ModuleCount); if (baseOut) *baseOut = imageBase; return STATUS_SUCCESS; } // LdrLoadImage - load a PE from a raw buffer into virtual space. -KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) { +static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) { if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER; + if (user && !process) return STATUS_INVALID_PARAMETER; if (size < 0x40) return STATUS_FAILURE; const uint8_t* bytes = (const uint8_t*)buffer; @@ -579,10 +692,23 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u PE_HEADER* pe = (PE_HEADER *)(bytes + peOffset); OPT_HEADER64* opt = &pe->OptionalHeader; - size_t imageSize = opt->SizeOfImage; + size_t imageSize = 0; + KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize); + if (!KSUCCESS(sizeStatus)) return sizeStatus; - void* mem = user ? MmAllocateUserImage(imageSize) : MmAllocatePool(imageSize); - if (!mem) return STATUS_NO_MEMORY; + LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n", + opt->SizeOfImage, + (unsigned long long)imageSize, + (unsigned long long)opt->ImageBase, + pe->NumberOfSections, + user ? 1 : 0, + process ? process->ProcessId : 0); + void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize); + if (!mem) { + DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n", + (unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0); + return STATUS_NO_MEMORY; + } memset(mem, 0, imageSize); memcpy(mem, buffer, opt->SizeOfHeaders); @@ -591,7 +717,9 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u void* dst = (uint8_t *)mem + sec[i].VirtualAddress; const void* src = (const uint8_t *)buffer + sec[i].PointerToRawData; if (sec[i].SizeOfRawData) { - memcpy(dst, src, sec[i].SizeOfRawData < sec[i].VirtualSize ? sec[i].SizeOfRawData : sec[i].VirtualSize); + size_t copySize = sec[i].SizeOfRawData; + if (sec[i].VirtualSize && sec[i].VirtualSize < copySize) copySize = sec[i].VirtualSize; + memcpy(dst, src, copySize); } uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0); @@ -608,3 +736,11 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u *baseOut = (uintptr_t)mem; return STATUS_SUCCESS; } + +KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) { + return KiLdrLoadImageCommon(process, buffer, size, baseOut, true); +} + +KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) { + return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user); +} diff --git a/code/kernel/mm_phys.c b/code/kernel/mm_phys.c index c9ad2c2..8c70abb 100644 --- a/code/kernel/mm_phys.c +++ b/code/kernel/mm_phys.c @@ -101,27 +101,48 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) { // --------------------------------------------------------------------------- // Public API // --------------------------------------------------------------------------- -PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) { - if (maxExclusive <= PAGE_SIZE) return (PHYSICAL_ADDRESS)0; +static PHYSICAL_ADDRESS KiPopFrameAtIndexLocked(size_t idx) { + PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx]; + size_t tail = gFrameStack.Top - idx - 1; + for (size_t j = 0; j < tail; j++) + gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j]; + gFrameStack.Top--; + return frame; +} + +PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive) { + minInclusive = (minInclusive + PAGE_SIZE - 1) & PAGE_MASK; + maxExclusive &= PAGE_MASK; + + if (maxExclusive <= minInclusive) + return (PHYSICAL_ADDRESS)0; uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock); + + // Search high-to-low within the requested window. For low-memory users + // such as the AP trampoline this naturally prefers the traditional + // 0x70000-0x9f000 area and avoids consuming lower real-mode space first. for (size_t i = gFrameStack.Top; i > 0; i--) { size_t idx = i - 1; PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx]; - if (frame && frame < maxExclusive && !(frame & (PAGE_SIZE - 1))) { - size_t tail = gFrameStack.Top - idx - 1; - for (size_t j = 0; j < tail; j++) - gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j]; - gFrameStack.Top--; + if (frame >= minInclusive && frame < maxExclusive && + !(frame & (PAGE_SIZE - 1))) { + frame = KiPopFrameAtIndexLocked(idx); KeReleaseSpinlock(&gFrameStack.Lock, flags); return frame; } } + KeReleaseSpinlock(&gFrameStack.Lock, flags); return (PHYSICAL_ADDRESS)0; } -PHYSICAL_ADDRESS MmAllocateFrame(void) { +PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) { + return MmAllocateFrameInRange(PAGE_SIZE, maxExclusive); +} + +__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void) { uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock); if (gFrameStack.Top == 0) { KeReleaseSpinlock(&gFrameStack.Lock, flags); diff --git a/code/kernel/mm_virt.c b/code/kernel/mm_virt.c index 0269736..b514100 100644 --- a/code/kernel/mm_virt.c +++ b/code/kernel/mm_virt.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -139,7 +140,7 @@ void MmFlushTlbForKernelMapping(uintptr_t virt) { if (acks < targets) { KePanic(KE_PANIC_CODE_GENERIC, - "TLB shootdown timeout: va=0x%llX targets=%ld acks=%ld selfCpu=%u\n", + "TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n", (uint64_t)virt, targets, acks, selfIndex); } } @@ -506,6 +507,35 @@ typedef struct HEAP { SPINLOCK Lock; } HEAP; +static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) { + if (!h || !size) return 1; + if (start < h->Start) return 0; + if (start > (uintptr_t)-1 - (size - 1)) return 0; + + uintptr_t begin = start & PAGE_MASK; + uintptr_t end = (start + size + PAGE_SIZE - 1) & PAGE_MASK; + if (end < begin || end > h->Limit) return 0; + + for (uintptr_t va = begin; va < end; va += PAGE_SIZE) { + PHYSICAL_ADDRESS phys = 0; + uint64_t flags = 0; + KSTATUS qs = MmQueryPage(va, &phys, &flags); + if (KSUCCESS(qs) && (flags & PAGE_PRESENT)) + continue; + + PHYSICAL_ADDRESS frame = MmAllocateFrame(); + if (!frame) return 0; + KSTATUS ms = MmMapPage(va, frame, h->MapFlags); + if (!KSUCCESS(ms)) { + MmFreeFrame(frame); + return 0; + } + memset((void*)va, 0, PAGE_SIZE); + DbgPrint("MM: repaired missing heap page va=%016llx\n", (unsigned long long)va); + } + return 1; +} + // Extend heap by at least 'need' bytes (rounded up to page boundary). static int KiHeapGrow(HEAP* h, size_t need) { size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); @@ -521,6 +551,9 @@ static int KiHeapGrow(HEAP* h, size_t need) { memset((void *)va, 0, PAGE_SIZE); } + if (!KiHeapEnsureMapped(h, h->Brk, grow)) + return 0; + // Make the new pages a single free block. BLOCK_HDR* blk = (BLOCK_HDR *)h->Brk; blk->Magic = BLOCK_MAGIC; @@ -547,34 +580,18 @@ static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) { } static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) { + (void)h; if (!blk || blk->Free == 0) return; - while (h->Brk > h->Start) { - uintptr_t blockEnd = (uintptr_t)blk + HDR_SIZE + blk->Size + FTR_SIZE; - if (blockEnd != h->Brk) break; - - // Keep at least one page for the block header/footer and a small payload. - if (blk->Size < PAGE_SIZE + sizeof(void*)) break; - - uintptr_t pageStart = h->Brk - PAGE_SIZE; - if (pageStart <= (uintptr_t)blk) break; // would unmap the page containing the header - - PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(pageStart); - if (!phys) break; - if (!KSUCCESS(MmUnmapPage(pageStart))) { - break; - } - MmFreeFrame(phys); - - h->Brk -= PAGE_SIZE; - blk->Size -= PAGE_SIZE; - } - - if (blk->Free) { - KiHdrToFtr(blk)->Size = blk->Size; - KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; - KiHdrToFtr(blk)->Free = 1; - } + // Do not decommit kernel-pool tail pages for now. The pool allocator keeps + // free-list nodes and boundary tags inside the managed arena; shrinking the + // mapped tail while large launch-failure cleanup is walking/freeing many + // objects can leave stale metadata pointers into unmapped pages. Physical + // pages are still reclaimed for user regions by MmDestroyUserRegion(); pool + // pages remain committed until we have a dedicated, audited heap shrinker. + KiHdrToFtr(blk)->Size = blk->Size; + KiHdrToFtr(blk)->Magic = BLOCK_MAGIC; + KiHdrToFtr(blk)->Free = 1; } static void* KiHeapAlloc(HEAP* h, size_t size) { @@ -583,6 +600,11 @@ static void* KiHeapAlloc(HEAP* h, size_t size) { uintptr_t flags = KeAcquireSpinlock(&h->Lock); + if (h->Brk > h->Start && !KiHeapEnsureMapped(h, h->Start, h->Brk - h->Start)) { + KeReleaseSpinlock(&h->Lock, flags); + return (void*)0; + } + // Best-fit search. BLOCK_HDR* best = (BLOCK_HDR *)0; for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) { @@ -735,12 +757,29 @@ static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) { static HEAP gPoolHeap; // --------------------------------------------------------------------------- -// User heap +// User virtual region arena // --------------------------------------------------------------------------- -static HEAP gUserHeap; -__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size); -static SPINLOCK gUserImageLock = SPINLOCK_INIT; -static uintptr_t gUserImageNext = USER_LIMIT; +typedef struct USER_VA_REGION { + LIST_ENTRY ProcessLink; + LIST_ENTRY GlobalLink; + PROCESS* Owner; + + uintptr_t ReservationBase; + size_t ReservationSize; + uintptr_t Base; + size_t Size; + size_t CommittedSize; + + uint64_t PageFlags; + USER_REGION_TYPE Type; + uint32_t Flags; +} USER_VA_REGION; + +static LIST_ENTRY gUserRegionList; +static SPINLOCK gUserRegionLock = SPINLOCK_INIT; +static int gUserRegionListReady = 0; + +#define USER_REGION_ARENA_BASE 0x0000100000000000ULL // --------------------------------------------------------------------------- // Module init @@ -763,20 +802,11 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) { KiHeapGrow(&gPoolHeap, poolSize); } - // User heap. - gUserHeap.FreeList = (BLOCK_HDR *)0; - gUserHeap.Start = USER_BASE; - gUserHeap.Brk = USER_BASE; - gUserHeap.Limit = USER_LIMIT; - gUserHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX; - gUserHeap.Lock = (SPINLOCK)SPINLOCK_INIT; - - // Executable image mappings grow downward from the top of user space. - // The regular user heap grows upward from USER_BASE. Keeping them in - // separate page-aligned regions prevents W^X section protection from - // changing heap metadata or adjacent allocations. - gUserImageLock = (SPINLOCK)SPINLOCK_INIT; - gUserImageNext = USER_LIMIT; + // User VA arena. One global lower-half virtual-address bucket, with + // per-process ownership tracked by USER_VA_REGION descriptors. + LIST_INIT(&gUserRegionList); + gUserRegionLock = (SPINLOCK)SPINLOCK_INIT; + gUserRegionListReady = 1; } void MmActivate(void) { @@ -791,69 +821,86 @@ __declspec(dllexport) void MmFreePool(void* ptr) { KiHeap __declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); } // --------------------------------------------------------------------------- -// Public user heap API +// User virtual region API // --------------------------------------------------------------------------- -__declspec(dllexport) void* MmAllocateUserMemory(size_t size) { - KeAssertIrql(PASSIVE_LEVEL); - return KiHeapAlloc(&gUserHeap, size); +static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) { + if (align < PAGE_SIZE) align = PAGE_SIZE; + if (align & (align - 1)) return 0; + return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1); } -__declspec(dllexport) void MmFreeUserMemory(void* ptr) { - KeAssertIrql(PASSIVE_LEVEL); - KiHeapFree(&gUserHeap, ptr); +static inline size_t KiAlignUpSize(size_t v, size_t align) { + if (align < PAGE_SIZE) align = PAGE_SIZE; + if (align & (align - 1)) return 0; + return (v + align - 1) & ~(align - 1); } -__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { - KeAssertIrql(PASSIVE_LEVEL); - return KiHeapRealloc(&gUserHeap, ptr, newSize); +static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) { + uintptr_t aEnd = aBase + aSize; + uintptr_t bEnd = bBase + bSize; + return aBase < bEnd && bBase < aEnd; } +static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) { + if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER; + if (!gUserRegionListReady) return STATUS_FAILURE; -__declspec(dllexport) void* MmAllocateUserImage(size_t size) { - KeAssertIrql(PASSIVE_LEVEL); - if (!size) return (void*)0; + uintptr_t candidate = KiAlignUpPtr(USER_REGION_ARENA_BASE, alignment); + if (!candidate) return STATUS_INVALID_PARAMETER; - size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - if (pagesSize < size) return (void*)0; + for (;;) { + if (candidate < USER_REGION_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate)) + return STATUS_NO_MEMORY; - uintptr_t flags = KeAcquireSpinlock(&gUserImageLock); + uintptr_t next = candidate; + int collided = 0; - if (pagesSize > gUserImageNext || gUserImageNext - pagesSize < gUserHeap.Brk) { - KeReleaseSpinlock(&gUserImageLock, flags); - return (void*)0; - } - - uintptr_t base = gUserImageNext - pagesSize; - gUserImageNext = base; - KeReleaseSpinlock(&gUserImageLock, flags); - - for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) { - PHYSICAL_ADDRESS frame = MmAllocateFrame(); - if (!frame) { - MmFreeUserImage((void*)base, va - base); - return (void*)0; + for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); + if (KiRangesOverlap(candidate, reservationSize, r->ReservationBase, r->ReservationSize)) { + uintptr_t end = r->ReservationBase + r->ReservationSize; + next = KiAlignUpPtr(end, alignment); + if (!next || next <= candidate) return STATUS_NO_MEMORY; + collided = 1; + break; + } } - KSTATUS s = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX); - if (!KSUCCESS(s)) { - MmFreeFrame(frame); - MmFreeUserImage((void*)base, va - base); - return (void*)0; - } - } - memset((void*)base, 0, pagesSize); - return (void*)base; + if (!collided) { + *baseOut = candidate; + return STATUS_SUCCESS; + } + candidate = next; + } } -__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) { - KeAssertIrql(PASSIVE_LEVEL); - if (!ptr || !size) return; +static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) { + if (!process || !base) return (USER_VA_REGION*)0; + uintptr_t b = (uintptr_t)base; + for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); + if (r->Base == b) return r; + } + return (USER_VA_REGION*)0; +} - uintptr_t base = (uintptr_t)ptr & PAGE_MASK; - size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - if (pagesSize < size) return; +static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) { + if (!process || !base || !size) return (USER_VA_REGION*)0; + if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0; + uintptr_t end = base + size; - for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) { + for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); + uintptr_t regionEnd = r->Base + r->CommittedSize; + if (base >= r->Base && end <= regionEnd) return r; + } + return (USER_VA_REGION*)0; +} + +static void KiUnmapUserRegionPayload(USER_VA_REGION* r) { + if (!r) return; + uintptr_t end = r->Base + r->CommittedSize; + for (uintptr_t va = r->Base; va < end; va += PAGE_SIZE) { PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va); if (!phys) continue; if (KSUCCESS(MmUnmapPage(va))) { @@ -862,12 +909,499 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) { } } +KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment, + uint64_t pageFlags, uint32_t regionFlags, + USER_REGION_TYPE type, void** outBase) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process || !outBase || !size) return STATUS_INVALID_PARAMETER; + *outBase = (void*)0; + + if (!(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER; + if (!(pageFlags & PAGE_PRESENT)) pageFlags |= PAGE_PRESENT; + + size_t payloadSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (payloadSize < size) return STATUS_NO_MEMORY; + + size_t guardLow = (regionFlags & USER_REGION_GUARD_LOW) ? PAGE_SIZE : 0; + size_t guardHigh = (regionFlags & USER_REGION_GUARD_HIGH) ? PAGE_SIZE : 0; + size_t reservationSize = guardLow + payloadSize + guardHigh; + if (reservationSize < payloadSize) return STATUS_NO_MEMORY; + + if (alignment < PAGE_SIZE) alignment = PAGE_SIZE; + if (alignment & (alignment - 1)) return STATUS_INVALID_PARAMETER; + + USER_VA_REGION* r = (USER_VA_REGION*)MmAllocatePool(sizeof(USER_VA_REGION)); + if (!r) return STATUS_NO_MEMORY; + memset(r, 0, sizeof(*r)); + LIST_INIT(&r->ProcessLink); + LIST_INIT(&r->GlobalLink); + + uintptr_t lockFlags = KeAcquireSpinlock(&gUserRegionLock); + uintptr_t reservationBase = 0; + KSTATUS s = KiFindUserVaRange(reservationSize, alignment, &reservationBase); + if (KSUCCESS(s)) { + r->Owner = process; + r->ReservationBase = reservationBase; + r->ReservationSize = reservationSize; + r->Base = reservationBase + guardLow; + r->Size = size; + r->CommittedSize = payloadSize; + r->PageFlags = pageFlags; + r->Type = type; + r->Flags = regionFlags; + ListInsertTail(&gUserRegionList, &r->GlobalLink); + } + KeReleaseSpinlock(&gUserRegionLock, lockFlags); + + if (!KSUCCESS(s)) { + MmFreePool(r); + return s; + } + + uintptr_t va = r->Base; + for (; va < r->Base + payloadSize; va += PAGE_SIZE) { + PHYSICAL_ADDRESS frame = MmAllocateFrame(); + if (!frame) { + s = STATUS_NO_MEMORY; + break; + } + s = MmMapPage(va, frame, pageFlags); + if (!KSUCCESS(s)) { + MmFreeFrame(frame); + break; + } + } + + if (!KSUCCESS(s)) { + r->CommittedSize = (va > r->Base) ? (size_t)(va - r->Base) : 0; + KiUnmapUserRegionPayload(r); + lockFlags = KeAcquireSpinlock(&gUserRegionLock); + ListRemove(&r->GlobalLink); + KeReleaseSpinlock(&gUserRegionLock, lockFlags); + MmFreePool(r); + return s; + } + + if (regionFlags & USER_REGION_ZERO) { + memset((void*)r->Base, 0, payloadSize); + } + + uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); + ListInsertTail(&process->UserRegions, &r->ProcessLink); + KeReleaseSpinlock(&process->UserRegionLock, procFlags); + + *outBase = (void*)r->Base; + return STATUS_SUCCESS; +} + +KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process || !base || !size) return STATUS_INVALID_PARAMETER; + + uintptr_t start = (uintptr_t)base; + if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY; + + uintptr_t alignedStart = start & PAGE_MASK; + uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; + if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY; + + uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); + USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart); + uintptr_t regionBase = r ? r->Base : 0; + uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0; + uint64_t defaultFlags = r ? r->PageFlags : 0; + KeReleaseSpinlock(&process->UserRegionLock, procFlags); + + if (!r) { + DbgPrint("MM: ensure user mapping failed: pid=%u range=%016llx-%016llx not inside owned committed region\n", + process->ProcessId, + (unsigned long long)alignedStart, + (unsigned long long)alignedEnd); + return STATUS_NOT_FOUND; + } + + uint64_t flags = pageFlags ? pageFlags : defaultFlags; + flags |= PAGE_PRESENT | PAGE_USER; + + uint32_t committed = 0; + for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { + if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED; + + PHYSICAL_ADDRESS phys = 0; + uint64_t existingFlags = 0; + KSTATUS qs = MmQueryPage(va, &phys, &existingFlags); + if (KSUCCESS(qs) && (existingFlags & PAGE_PRESENT)) + continue; + + PHYSICAL_ADDRESS frame = MmAllocateFrame(); + if (!frame) return STATUS_NO_MEMORY; + + KSTATUS ms = MmMapPage(va, frame, flags); + if (!KSUCCESS(ms)) { + MmFreeFrame(frame); + return ms; + } + memset((void*)va, 0, PAGE_SIZE); + committed++; + } + + return STATUS_SUCCESS; +} + +KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process || !base || !size) return STATUS_INVALID_PARAMETER; + + uintptr_t start = (uintptr_t)base; + if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY; + + uintptr_t alignedStart = start & PAGE_MASK; + uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK; + if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY; + + uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); + USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart); + uintptr_t regionBase = r ? r->Base : 0; + uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0; + KeReleaseSpinlock(&process->UserRegionLock, procFlags); + + if (!r) { + DbgPrint("MM: protect user region failed: pid=%u range=%016llx-%016llx not owned/committed\n", + process->ProcessId, + (unsigned long long)alignedStart, + (unsigned long long)alignedEnd); + return STATUS_NOT_FOUND; + } + + uint64_t finalFlags = pageFlags | PAGE_PRESENT | PAGE_USER; + + for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) { + if (va < regionBase || va >= regionEnd) { + DbgPrint("MM: protect user region hit guard/outside page pid=%u va=%016llx region=%016llx-%016llx\n", + process->ProcessId, + (unsigned long long)va, + (unsigned long long)regionBase, + (unsigned long long)regionEnd); + return STATUS_ACCESS_DENIED; + } + + PHYSICAL_ADDRESS phys = 0; + uint64_t existingFlags = 0; + KSTATUS qs = MmQueryPage(va, &phys, &existingFlags); + if (!KSUCCESS(qs) || !(existingFlags & PAGE_PRESENT)) { + PHYSICAL_ADDRESS frame = MmAllocateFrame(); + if (!frame) return STATUS_NO_MEMORY; + KSTATUS ms = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX); + if (!KSUCCESS(ms)) { + MmFreeFrame(frame); + return ms; + } + memset((void*)va, 0, PAGE_SIZE); + DbgPrint("MM: committed missing user page during protect pid=%u va=%016llx\n", + process->ProcessId, (unsigned long long)va); + } + + KSTATUS ps = MmSetPageProtect(va, PAGE_SIZE, finalFlags); + if (!KSUCCESS(ps)) { + PHYSICAL_ADDRESS dbgPhys = 0; + uint64_t dbgFlags = 0; + KSTATUS dbg = MmQueryPage(va, &dbgPhys, &dbgFlags); + DbgPrint("MM: protect user page failed pid=%u va=%016llx st=%d query=%d phys=%016llx flags=%016llx final=%016llx\n", + process->ProcessId, + (unsigned long long)va, + ps, dbg, + (unsigned long long)dbgPhys, + (unsigned long long)dbgFlags, + (unsigned long long)finalFlags); + return ps; + } + } + + return STATUS_SUCCESS; +} + +KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process || !base) return STATUS_INVALID_PARAMETER; + + uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); + USER_VA_REGION* r = KiFindUserRegionLocked(process, base); + if (!r) { + KeReleaseSpinlock(&process->UserRegionLock, procFlags); + return STATUS_NOT_FOUND; + } + ListRemove(&r->ProcessLink); + KeReleaseSpinlock(&process->UserRegionLock, procFlags); + + uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock); + ListRemove(&r->GlobalLink); + KeReleaseSpinlock(&gUserRegionLock, globalFlags); + + KiUnmapUserRegionPayload(r); + MmFreePool(r); + return STATUS_SUCCESS; +} + +void MmDestroyAllUserRegions(PROCESS* process) { + KeAssertIrql(PASSIVE_LEVEL); + if (!process) return; + + for (;;) { + uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); + LIST_ENTRY* node = ListPopHead(&process->UserRegions); + KeReleaseSpinlock(&process->UserRegionLock, procFlags); + if (!node) break; + + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); + uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock); + ListRemove(&r->GlobalLink); + KeReleaseSpinlock(&gUserRegionLock, globalFlags); + KiUnmapUserRegionPayload(r); + MmFreePool(r); + } +} + +size_t MmGetUserRegionSize(PROCESS* process, const void* base) { + if (!process || !base) return 0; + uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock); + USER_VA_REGION* r = KiFindUserRegionLocked(process, base); + size_t size = r ? r->Size : 0; + KeReleaseSpinlock(&process->UserRegionLock, procFlags); + return size; +} + +size_t MmGetUserRegionSizeByBase(const void* base) { + if (!base) return 0; + uintptr_t b = (uintptr_t)base; + uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock); + for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); + if (r->Base == b) { + size_t size = r->Size; + KeReleaseSpinlock(&gUserRegionLock, flags); + return size; + } + } + KeReleaseSpinlock(&gUserRegionLock, flags); + return 0; +} + + +static const char* KiUserRegionTypeName(USER_REGION_TYPE type) { + switch (type) { + case UserRegionGeneric: return "generic"; + case UserRegionHeap: return "heap"; + case UserRegionImage: return "image"; + case UserRegionStack: return "stack"; + case UserRegionTeb: return "teb"; + case UserRegionPeb: return "peb"; + case UserRegionEnvironment: return "env"; + case UserRegionMappedFile: return "mapped"; + case UserRegionShared: return "shared"; + default: return "?"; + } +} + +void MmDebugDumpUserRegions(PROCESS* process) { + if (!process) { + DbgPrint("MM: user regions: \n"); + return; + } + + DbgPrint("MM: user regions for pid=%u name=%s:\n", process->ProcessId, process->Name); + uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock); + uint32_t count = 0; + for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) { + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink); + DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx flags=%x pte=%llx\n", + count, + KiUserRegionTypeName(r->Type), + (unsigned long long)r->ReservationBase, + (unsigned long long)(r->ReservationBase + r->ReservationSize), + (unsigned long long)r->Base, + (unsigned long long)r->Size, + (unsigned long long)r->CommittedSize, + r->Flags, + (unsigned long long)r->PageFlags); + count++; + } + KeReleaseSpinlock(&process->UserRegionLock, flags); + if (!count) DbgPrint(" \n"); +} + +static void KiDebugDescribePoolAddress(uintptr_t addr) { + DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n", + (unsigned long long)gPoolHeap.Start, + (unsigned long long)gPoolHeap.Brk, + (unsigned long long)gPoolHeap.Limit, + (unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024)); + + if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return; + if (addr >= gPoolHeap.Brk) { + DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n", + (unsigned long long)addr, + (unsigned long long)(addr - gPoolHeap.Brk)); + return; + } + + uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock); + for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start; + (uintptr_t)cur < gPoolHeap.Brk; + cur = KiNextBlock(cur)) { + if (cur->Magic != BLOCK_MAGIC) { + DbgPrint("MM: pool walk corrupt at hdr=%p magic=%08x\n", cur, cur->Magic); + break; + } + uintptr_t hdr = (uintptr_t)cur; + uintptr_t data = hdr + HDR_SIZE; + uintptr_t end = data + cur->Size; + uintptr_t ftr = end; + if (addr >= hdr && addr < ftr + FTR_SIZE) { + DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx free=%u\n", + (unsigned long long)addr, + (unsigned long long)hdr, + (unsigned long long)data, + (unsigned long long)end, + (unsigned long long)cur->Size, + cur->Free); + break; + } + if (KiNextBlock(cur) <= cur) { + DbgPrint("MM: pool walk stopped: non-advancing block hdr=%p size=%llx\n", cur, (unsigned long long)cur->Size); + break; + } + } + KeReleaseSpinlock(&gPoolHeap.Lock, flags); +} + +void MmDebugDescribeAddress(uintptr_t addr) { + PHYSICAL_ADDRESS phys = 0; + uint64_t flags = 0; + KSTATUS qs = MmQueryPage(addr, &phys, &flags); + DbgPrint("MM: describe va=%016llx query=%d phys=%016llx flags=%016llx\n", + (unsigned long long)addr, + qs, + (unsigned long long)phys, + (unsigned long long)flags); + + if (addr >= POOL_BASE && addr < POOL_LIMIT) { + KiDebugDescribePoolAddress(addr); + return; + } + + if (addr >= USER_REGION_ARENA_BASE && addr < USER_LIMIT) { + uintptr_t gl = KeAcquireSpinlock(&gUserRegionLock); + for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); + if (addr >= r->ReservationBase && addr < r->ReservationBase + r->ReservationSize) { + DbgPrint("MM: address belongs to pid=%u name=%s region=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n", + r->Owner ? r->Owner->ProcessId : 0, + r->Owner ? r->Owner->Name : "?", + KiUserRegionTypeName(r->Type), + (unsigned long long)r->ReservationBase, + (unsigned long long)(r->ReservationBase + r->ReservationSize), + (unsigned long long)r->Base, + (unsigned long long)r->Size, + r->Flags); + KeReleaseSpinlock(&gUserRegionLock, gl); + return; + } + } + KeReleaseSpinlock(&gUserRegionLock, gl); + DbgPrint("MM: address is in user arena but not reserved by any USER_VA_REGION\n"); + } +} + +__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) { + void* base = (void*)0; + if (!process || !size) return (void*)0; + KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + USER_REGION_ZERO, + UserRegionHeap, + &base); + return KSUCCESS(s) ? base : (void*)0; +} + +__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) { + if (!process || !ptr) return; + (void)MmDestroyUserRegion(process, ptr); +} + +__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) { + if (!process) return (void*)0; + if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize); + if (!newSize) { + MmFreeUserMemoryForProcess(process, ptr); + return (void*)0; + } + + size_t oldSize = MmGetUserRegionSize(process, ptr); + if (!oldSize) return (void*)0; + if (newSize <= oldSize) return ptr; + + void* newPtr = MmAllocateUserMemoryForProcess(process, newSize); + if (!newPtr) return (void*)0; + memcpy(newPtr, ptr, oldSize); + MmFreeUserMemoryForProcess(process, ptr); + return newPtr; +} + +__declspec(dllexport) void* MmAllocateUserMemory(size_t size) { + PROCESS* process = KeGetCurrentProcess(); + return MmAllocateUserMemoryForProcess(process, size); +} + +__declspec(dllexport) void MmFreeUserMemory(void* ptr) { + PROCESS* process = KeGetCurrentProcess(); + MmFreeUserMemoryForProcess(process, ptr); +} + +__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { + PROCESS* process = KeGetCurrentProcess(); + return MmReallocateUserMemoryForProcess(process, ptr, newSize); +} + +__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) { + void* base = (void*)0; + if (!process || !size) return (void*)0; + KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH, + UserRegionImage, + &base); + return KSUCCESS(s) ? base : (void*)0; +} + +__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) { + (void)size; + if (!process || !ptr) return; + (void)MmDestroyUserRegion(process, ptr); +} + +__declspec(dllexport) void* MmAllocateUserImage(size_t size) { + PROCESS* process = KeGetCurrentProcess(); + return MmAllocateUserImageForProcess(process, size); +} + +__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) { + PROCESS* process = KeGetCurrentProcess(); + MmFreeUserImageForProcess(process, ptr, size); +} + size_t MmGetKernelPoolUsage(void) { // Returns bytes committed to the kernel pool return (size_t)(gPoolHeap.Brk - gPoolHeap.Start); } size_t MmGetUserHeapUsage(void) { - // Returns bytes committed to the user heap - return (size_t)(gUserHeap.Brk - gUserHeap.Start); + size_t total = 0; + if (!gUserRegionListReady) return 0; + uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock); + for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) { + USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink); + total += r->CommittedSize; + } + KeReleaseSpinlock(&gUserRegionLock, flags); + return total; } diff --git a/code/public/ntlite/boot_info.h b/code/public/ntlite/boot_info.h index 31e0e51..7f390cb 100644 --- a/code/public/ntlite/boot_info.h +++ b/code/public/ntlite/boot_info.h @@ -39,6 +39,9 @@ typedef struct BOOT_INFO { /* Direct physical window base - always 0xFFFF880000000000. */ uintptr_t PhysicalMapBase; + /* Reserved low-memory page for the AP startup trampoline, or zero. */ + uintptr_t ApTrampolinePhys; + /* From boot.ini */ uint32_t Debug; } BOOT_INFO; diff --git a/code/public/ntlite/ddi/hal_api.h b/code/public/ntlite/ddi/hal_api.h index ed2c3b2..073c49f 100644 --- a/code/public/ntlite/ddi/hal_api.h +++ b/code/public/ntlite/ddi/hal_api.h @@ -20,42 +20,6 @@ typedef struct HAL_SCREEN { void (*EraseCursor)(void); } HAL_SCREEN; -typedef struct HAL_KEYBOARD { - uint8_t (*Poll)(void); - uint8_t (*ReadScancode)(void); - char (*ReadChar)(void); - uint32_t (*GetModifiers)(void); - void (*SetCallback)(void (*cb)(uint8_t scancode)); - bool (*IsKeyDown)(uint8_t scancode, bool extended); - void (*Isr)(void); - // Optional bottom half, queued by the kernel after IRQ dispatch. - void (*Dpc)(void); - uint8_t IrqNumber; -} HAL_KEYBOARD; - -// HAL_MOUSE – PS/2 auxiliary mouse on IRQ 12. -// Packet format: standard 3-byte PS/2 (buttons, deltaX, deltaY). -// Y is already screen-oriented (positive = down) after ISR inversion. -typedef struct HAL_MOUSE { - // Non-blocking: returns true if a packet is waiting in the ring buffer. - bool (*Poll)(void); - // Non-blocking dequeue. Returns false when buffer is empty. - // buttons: bit0=left, bit1=right, bit2=middle - bool (*Read)(int* dx, int* dy, uint8_t* buttons); - // Callback invoked from DPC context after every completed packet. - void (*SetCallback)(void (*cb)(int dx, int dy, uint8_t buttons)); - // Absolute cursor position, clamped to screen bounds. - void (*GetPosition)(int* x, int* y); - void (*SetPosition)(int x, int y); - // Update the clamping rectangle (call after mode change / resize). - void (*SetBounds)(int w, int h); - // Raw ISR – called by KeIrqDispatch for IRQ 12. - void (*Isr)(void); - // Optional bottom half, queued by the kernel after IRQ dispatch. - void (*Dpc)(void); - uint8_t IrqNumber; // always 12 -} HAL_MOUSE; - typedef struct HAL_TIMER { uint64_t (*QueryTicks)(void); void (*Stall)(uint32_t us); @@ -75,6 +39,10 @@ typedef struct HAL_PCI { // Returns the IRQ line from PCI config space (register 0x3C, byte 0). // Returns 0xFF if no interrupt is wired. uint8_t (*GetIrqLine)(uint8_t bus, uint8_t dev, uint8_t fn); + + // Program a PCI MSI capability to deliver to vector (0x20 + irqSlot). + // irqSlot must correspond to a kernel IRQ stub/vector slot. + KSTATUS (*EnableMsi)(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t irqSlot); } HAL_PCI; typedef struct HAL_INTC { @@ -94,8 +62,6 @@ typedef struct HAL_API { uint32_t Version; void *(*PhysToVirt)(uint64_t physAddr); HAL_SCREEN Screen; - HAL_KEYBOARD Keyboard; - HAL_MOUSE Mouse; HAL_TIMER Timer; HAL_PCI Pci; HAL_INTC Intc; @@ -117,9 +83,11 @@ typedef struct HAL_INIT_PARAMS { // Debug int Debug; + + void* (*AllocatePool)(size_t size); } HAL_INIT_PARAMS; -#define HAL_API_VERSION 2 +#define HAL_API_VERSION 3 __declspec(dllexport) const HAL_API* HalGetApi(void); __declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)); diff --git a/code/public/ntlite/ddi/input_device.h b/code/public/ntlite/ddi/input_device.h new file mode 100644 index 0000000..9b5e9ee --- /dev/null +++ b/code/public/ntlite/ddi/input_device.h @@ -0,0 +1,9 @@ +#pragma once +// input_device.h - driver-facing input submission DDI. + +#include +#include + +__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId); +__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event); +__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event); diff --git a/code/public/ntlite/ddi/io_driver.h b/code/public/ntlite/ddi/io_driver.h index 60c52a7..b1d401e 100644 --- a/code/public/ntlite/ddi/io_driver.h +++ b/code/public/ntlite/ddi/io_driver.h @@ -1,9 +1,9 @@ #pragma once // io_driver.h - definitive kernel driver/device I/O model // -// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through +// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through // IoCallDriver()/IoSubmitIrp(); drivers complete accepted IRPs by calling -// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table. +// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table. #include #include @@ -33,6 +33,75 @@ struct DRIVER_OBJECT { DRIVER_UNLOAD_FN DriverUnload; }; + + +typedef enum IO_BUS_TYPE { + IoBusUnknown = 0, + IoBusPci = 1, + IoBusUsb = 2, +} IO_BUS_TYPE; + +#define IO_PUBLISHED_USB_MAX_ENDPOINTS 8u + +typedef struct IO_PUBLISHED_USB_ENDPOINT { + uint8_t Address; + uint8_t Attributes; + uint16_t MaxPacketSize; + uint8_t Interval; +} IO_PUBLISHED_USB_ENDPOINT; + +typedef struct IO_PUBLISHED_USB_DEVICE { + uint16_t VendorId; + uint16_t ProductId; + uint16_t BcdUsb; + uint16_t BcdDevice; + + uint8_t DeviceClass; + uint8_t DeviceSubclass; + uint8_t DeviceProtocol; + + uint8_t SlotId; + uint8_t Port; + uint8_t Speed; + uint8_t ConfigurationValue; + + uint8_t InterfaceNumber; + uint8_t AlternateSetting; + uint8_t InterfaceClass; + uint8_t InterfaceSubclass; + uint8_t InterfaceProtocol; + + uint8_t EndpointCount; + IO_PUBLISHED_USB_ENDPOINT Endpoints[IO_PUBLISHED_USB_MAX_ENDPOINTS]; +} IO_PUBLISHED_USB_DEVICE; + +typedef struct IO_PUBLISHED_DEVICE { + IO_BUS_TYPE BusType; + const char* Name; + void* BusContext; + void* DeviceContext; + uint32_t Flags; + bool Bound; + const char* BoundDriverName; + + union { + IO_PUBLISHED_USB_DEVICE Usb; + } u; +} IO_PUBLISHED_DEVICE; + +typedef bool (*IO_DRIVER_MATCH_CALLBACK)(const IO_PUBLISHED_DEVICE* device, + void* context); +typedef KSTATUS (*IO_DRIVER_BIND_CALLBACK)(IO_PUBLISHED_DEVICE* device, + void* context); + +__declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName, + IO_BUS_TYPE busType, + IO_DRIVER_MATCH_CALLBACK match, + IO_DRIVER_BIND_CALLBACK bind, + void* context); +__declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device); +__declspec(dllexport) uint32_t IoPublishedDeviceCount(void); + struct DEVICE_OBJECT { DRIVER_OBJECT* DriverObject; void* DeviceExtension; @@ -56,6 +125,15 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex); __declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex); __declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex); +typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex, + DEVICE_OBJECT* device, + void* context); + +__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback, + void* context); +__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback, + void* context); + __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, struct IRP* irp); diff --git a/code/public/ntlite/input.h b/code/public/ntlite/input.h new file mode 100644 index 0000000..9989322 --- /dev/null +++ b/code/public/ntlite/input.h @@ -0,0 +1,142 @@ +#pragma once +// input.h - canonical keyboard/mouse input ABI shared by kernel, drivers, and user-mode. +// +// Key values intentionally follow USB HID Keyboard/Keypad usage IDs. PS/2, +// USB HID, and future input drivers all translate their hardware format into +// these values before entering the kernel input core. + +#include +#include + +typedef uint16_t INPUT_KEY; + +#define INPUT_KEY_NONE 0x0000u +#define INPUT_KEY_A 0x0004u +#define INPUT_KEY_B 0x0005u +#define INPUT_KEY_C 0x0006u +#define INPUT_KEY_D 0x0007u +#define INPUT_KEY_E 0x0008u +#define INPUT_KEY_F 0x0009u +#define INPUT_KEY_G 0x000Au +#define INPUT_KEY_H 0x000Bu +#define INPUT_KEY_I 0x000Cu +#define INPUT_KEY_J 0x000Du +#define INPUT_KEY_K 0x000Eu +#define INPUT_KEY_L 0x000Fu +#define INPUT_KEY_M 0x0010u +#define INPUT_KEY_N 0x0011u +#define INPUT_KEY_O 0x0012u +#define INPUT_KEY_P 0x0013u +#define INPUT_KEY_Q 0x0014u +#define INPUT_KEY_R 0x0015u +#define INPUT_KEY_S 0x0016u +#define INPUT_KEY_T 0x0017u +#define INPUT_KEY_U 0x0018u +#define INPUT_KEY_V 0x0019u +#define INPUT_KEY_W 0x001Au +#define INPUT_KEY_X 0x001Bu +#define INPUT_KEY_Y 0x001Cu +#define INPUT_KEY_Z 0x001Du +#define INPUT_KEY_1 0x001Eu +#define INPUT_KEY_2 0x001Fu +#define INPUT_KEY_3 0x0020u +#define INPUT_KEY_4 0x0021u +#define INPUT_KEY_5 0x0022u +#define INPUT_KEY_6 0x0023u +#define INPUT_KEY_7 0x0024u +#define INPUT_KEY_8 0x0025u +#define INPUT_KEY_9 0x0026u +#define INPUT_KEY_0 0x0027u +#define INPUT_KEY_ENTER 0x0028u +#define INPUT_KEY_ESCAPE 0x0029u +#define INPUT_KEY_BACKSPACE 0x002Au +#define INPUT_KEY_TAB 0x002Bu +#define INPUT_KEY_SPACE 0x002Cu +#define INPUT_KEY_MINUS 0x002Du +#define INPUT_KEY_EQUAL 0x002Eu +#define INPUT_KEY_LEFT_BRACKET 0x002Fu +#define INPUT_KEY_RIGHT_BRACKET 0x0030u +#define INPUT_KEY_BACKSLASH 0x0031u +#define INPUT_KEY_SEMICOLON 0x0033u +#define INPUT_KEY_APOSTROPHE 0x0034u +#define INPUT_KEY_GRAVE 0x0035u +#define INPUT_KEY_COMMA 0x0036u +#define INPUT_KEY_PERIOD 0x0037u +#define INPUT_KEY_SLASH 0x0038u +#define INPUT_KEY_CAPS_LOCK 0x0039u +#define INPUT_KEY_F1 0x003Au +#define INPUT_KEY_F2 0x003Bu +#define INPUT_KEY_F3 0x003Cu +#define INPUT_KEY_F4 0x003Du +#define INPUT_KEY_F5 0x003Eu +#define INPUT_KEY_F6 0x003Fu +#define INPUT_KEY_F7 0x0040u +#define INPUT_KEY_F8 0x0041u +#define INPUT_KEY_F9 0x0042u +#define INPUT_KEY_F10 0x0043u +#define INPUT_KEY_F11 0x0044u +#define INPUT_KEY_F12 0x0045u +#define INPUT_KEY_INSERT 0x0049u +#define INPUT_KEY_HOME 0x004Au +#define INPUT_KEY_PAGE_UP 0x004Bu +#define INPUT_KEY_DELETE 0x004Cu +#define INPUT_KEY_END 0x004Du +#define INPUT_KEY_PAGE_DOWN 0x004Eu +#define INPUT_KEY_RIGHT 0x004Fu +#define INPUT_KEY_LEFT 0x0050u +#define INPUT_KEY_DOWN 0x0051u +#define INPUT_KEY_UP 0x0052u +#define INPUT_KEY_NUM_LOCK 0x0053u +#define INPUT_KEY_KP_DIVIDE 0x0054u +#define INPUT_KEY_KP_MULTIPLY 0x0055u +#define INPUT_KEY_KP_MINUS 0x0056u +#define INPUT_KEY_KP_PLUS 0x0057u +#define INPUT_KEY_KP_ENTER 0x0058u +#define INPUT_KEY_KP_1 0x0059u +#define INPUT_KEY_KP_2 0x005Au +#define INPUT_KEY_KP_3 0x005Bu +#define INPUT_KEY_KP_4 0x005Cu +#define INPUT_KEY_KP_5 0x005Du +#define INPUT_KEY_KP_6 0x005Eu +#define INPUT_KEY_KP_7 0x005Fu +#define INPUT_KEY_KP_8 0x0060u +#define INPUT_KEY_KP_9 0x0061u +#define INPUT_KEY_KP_0 0x0062u +#define INPUT_KEY_KP_PERIOD 0x0063u + +#define INPUT_KEY_LEFT_CTRL 0x00E0u +#define INPUT_KEY_LEFT_SHIFT 0x00E1u +#define INPUT_KEY_LEFT_ALT 0x00E2u +#define INPUT_KEY_LEFT_GUI 0x00E3u +#define INPUT_KEY_RIGHT_CTRL 0x00E4u +#define INPUT_KEY_RIGHT_SHIFT 0x00E5u +#define INPUT_KEY_RIGHT_ALT 0x00E6u +#define INPUT_KEY_RIGHT_GUI 0x00E7u + +#define INPUT_MOD_LSHIFT (1u << 0) +#define INPUT_MOD_RSHIFT (1u << 1) +#define INPUT_MOD_LCTRL (1u << 2) +#define INPUT_MOD_RCTRL (1u << 3) +#define INPUT_MOD_LALT (1u << 4) +#define INPUT_MOD_RALT (1u << 5) +#define INPUT_MOD_CAPS_LOCK (1u << 6) +#define INPUT_MOD_NUM_LOCK (1u << 7) +#define INPUT_MOD_LGUI (1u << 8) +#define INPUT_MOD_RGUI (1u << 9) + +#define INPUT_MOD_SHIFT (INPUT_MOD_LSHIFT | INPUT_MOD_RSHIFT) +#define INPUT_MOD_CTRL (INPUT_MOD_LCTRL | INPUT_MOD_RCTRL) +#define INPUT_MOD_ALT (INPUT_MOD_LALT | INPUT_MOD_RALT) +#define INPUT_MOD_GUI (INPUT_MOD_LGUI | INPUT_MOD_RGUI) + +typedef struct INPUT_KEY_EVENT { + INPUT_KEY Key; + bool Pressed; +} INPUT_KEY_EVENT; + +typedef struct INPUT_MOUSE_EVENT { + int dx; + int dy; + int wheel; + uint8_t buttons; // bit0=left, bit1=right, bit2=middle +} INPUT_MOUSE_EVENT; diff --git a/code/public/ntlite/syscall.h b/code/public/ntlite/syscall.h index e2b55f2..bd51fc6 100644 --- a/code/public/ntlite/syscall.h +++ b/code/public/ntlite/syscall.h @@ -7,6 +7,7 @@ #include #include #include +#include #define SYS_STATUS_SUCCESS 0u #define SYS_STATUS_TIMEOUT 3u @@ -20,7 +21,7 @@ // --------------------------------------------------------------------------- // User process/thread environment // --------------------------------------------------------------------------- -// GS base points at SYS_TEB. GS:[0] is the process PEB pointer. +// GS base points at SYS_TEB. GS:[0] is the process PEB pointer. // This is this OS' ABI, not a Windows compatibility layout. #define SYS_PEB_MAGIC 0x30424550u /* "PEB0" */ #define SYS_TEB_MAGIC 0x30424554u /* "TEB0" */ @@ -48,7 +49,7 @@ typedef struct SYS_TEB { uint32_t ThreadId; uint32_t Magic; - // Native per-thread extension pointer. The kernel leaves this NULL; + // Native per-thread extension pointer. The kernel leaves this NULL; // user-mode personalities/libraries may hang their own thread-local // state here without baking subsystem semantics into SYS_TEB. void* SubsystemData; @@ -278,9 +279,9 @@ uint32_t SysScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg); // Input (62-69) // --------------------------------------------------------------------------- uint64_t SysKbPoll(void); -uint64_t SysKbIsKeyDown(uint8_t scancode, bool extended); +uint64_t SysKbIsKeyDown(INPUT_KEY key); char SysKbReadChar(void); -uint8_t SysKbReadScancode(void); +INPUT_KEY SysKbReadKey(void); uint32_t SysKbGetModifiers(void); uint64_t SysMousePoll(void); diff --git a/code/third_party/lwip/src/api/api_lib.c b/code/third_party/lwip/src/api/api_lib.c index 8931e94..b3b945d 100644 --- a/code/third_party/lwip/src/api/api_lib.c +++ b/code/third_party/lwip/src/api/api_lib.c @@ -665,7 +665,7 @@ netconn_recv_data(struct netconn *conn, void **new_buf, u8_t apiflags) /* Register event with callback */ API_EVENT(conn, NETCONN_EVT_RCVMINUS, len); - LWIP_DEBUGF(API_LIB_DEBUG, ("netconn_recv_data: received %p, len=%"U16_F"\n", buf, len)); + LWIP_DEBUGF(API_LIB_DEBUG, ("netconn_recv_data: received 0x%016llX, len=%"U16_F"\n", buf, len)); *new_buf = buf; /* don't set conn->last_err: it's only ERR_OK, anyway */ diff --git a/code/third_party/lwip/src/api/netdb.c b/code/third_party/lwip/src/api/netdb.c index 2730e17..5a84baf 100644 --- a/code/third_party/lwip/src/api/netdb.c +++ b/code/third_party/lwip/src/api/netdb.c @@ -126,11 +126,11 @@ lwip_gethostbyname(const char *name) #if DNS_DEBUG /* dump hostent */ LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_name == %s\n", s_hostent.h_name)); - LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_aliases == %p\n", (void *)s_hostent.h_aliases)); + LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_aliases == 0x%016llX\n", (void *)s_hostent.h_aliases)); /* h_aliases are always empty */ LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_addrtype == %d\n", s_hostent.h_addrtype)); LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_length == %d\n", s_hostent.h_length)); - LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_addr_list == %p\n", (void *)s_hostent.h_addr_list)); + LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_addr_list == 0x%016llX\n", (void *)s_hostent.h_addr_list)); if (s_hostent.h_addr_list != NULL) { u8_t idx; for (idx = 0; s_hostent.h_addr_list[idx]; idx++) { diff --git a/code/third_party/lwip/src/api/sockets.c b/code/third_party/lwip/src/api/sockets.c index aaab5fd..f2168a1 100644 --- a/code/third_party/lwip/src/api/sockets.c +++ b/code/third_party/lwip/src/api/sockets.c @@ -975,7 +975,7 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags) err_t err; u16_t copylen; - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: top while sock->lastdata=%p\n", (void *)sock->lastdata.pbuf)); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: top while sock->lastdata=0x%016llX\n", (void *)sock->lastdata.pbuf)); /* Check if there is data left from the last recv operation. */ if (sock->lastdata.pbuf) { p = sock->lastdata.pbuf; @@ -983,7 +983,7 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags) /* No data was left from the previous operation, so we try to get some from the network. */ err = netconn_recv_tcp_pbuf_flags(sock->conn, &p, apiflags); - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: netconn_recv err=%d, pbuf=%p\n", + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: netconn_recv err=%d, pbuf=0x%016llX\n", err, (void *)p)); if (err != ERR_OK) { @@ -1037,10 +1037,10 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags) /* If so, it should be saved in the sock structure for the next recv call. We store the pbuf but hide/free the consumed data: */ sock->lastdata.pbuf = pbuf_free_header(p, copylen); - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: lastdata now pbuf=%p\n", (void *)sock->lastdata.pbuf)); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: lastdata now pbuf=0x%016llX\n", (void *)sock->lastdata.pbuf)); } else { sock->lastdata.pbuf = NULL; - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: deleting pbuf=%p\n", (void *)p)); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: deleting pbuf=0x%016llX\n", (void *)p)); pbuf_free(p); } } @@ -1142,14 +1142,14 @@ lwip_recvfrom_udp_raw(struct lwip_sock *sock, int flags, struct msghdr *msg, u16 apiflags = 0; } - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: top sock->lastdata=%p\n", (void *)sock->lastdata.netbuf)); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: top sock->lastdata=0x%016llX\n", (void *)sock->lastdata.netbuf)); /* Check if there is data left from the last recv operation. */ buf = sock->lastdata.netbuf; if (buf == NULL) { /* No data was left from the previous operation, so we try to get some from the network. */ err = netconn_recv_udp_raw_netbuf_flags(sock->conn, &buf, apiflags); - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: netconn_recv err=%d, netbuf=%p\n", + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: netconn_recv err=%d, netbuf=0x%016llX\n", err, (void *)buf)); if (err != ERR_OK) { @@ -1242,7 +1242,7 @@ lwip_recvfrom(int s, void *mem, size_t len, int flags, struct lwip_sock *sock; ssize_t ret; - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom(%d, %p, %"SZT_F", 0x%x, ..)\n", s, mem, len, flags)); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom(%d, 0x%016llX, %"SZT_F", 0x%x, ..)\n", s, mem, len, flags)); sock = get_socket(s); if (!sock) { return -1; @@ -1324,7 +1324,7 @@ lwip_recvmsg(int s, struct msghdr *message, int flags) msg_iovlen_t i; ssize_t buflen; - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvmsg(%d, message=%p, flags=0x%x)\n", s, (void *)message, flags)); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvmsg(%d, message=0x%016llX, flags=0x%x)\n", s, (void *)message, flags)); LWIP_ERROR("lwip_recvmsg: invalid message pointer", message != NULL, return ERR_ARG;); LWIP_ERROR("lwip_recvmsg: unsupported flags", (flags & ~(MSG_PEEK|MSG_DONTWAIT)) == 0, set_errno(EOPNOTSUPP); return -1;); @@ -1426,7 +1426,7 @@ lwip_send(int s, const void *data, size_t size, int flags) u8_t write_flags; size_t written; - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_send(%d, data=%p, size=%"SZT_F", flags=0x%x)\n", + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_send(%d, data=0x%016llX, size=%"SZT_F", flags=0x%x)\n", s, data, size, flags)); sock = get_socket(s); @@ -1675,7 +1675,7 @@ lwip_sendto(int s, const void *data, size_t size, int flags, netbuf_fromport(&buf) = remote_port; - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_sendto(%d, data=%p, short_size=%"U16_F", flags=0x%x to=", + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_sendto(%d, data=0x%016llX, short_size=%"U16_F", flags=0x%x to=", s, data, short_size, flags)); ip_addr_debug_print_val(SOCKETS_DEBUG, buf.addr); LWIP_DEBUGF(SOCKETS_DEBUG, (" port=%"U16_F"\n", remote_port)); @@ -2018,7 +2018,7 @@ lwip_select(int maxfdp1, fd_set *readset, fd_set *writeset, fd_set *exceptset, #endif SYS_ARCH_DECL_PROTECT(lev); - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_select(%d, %p, %p, %p, tvsec=%"S32_F" tvusec=%"S32_F")\n", + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_select(%d, 0x%016llX, 0x%016llX, 0x%016llX, tvsec=%"S32_F" tvusec=%"S32_F")\n", maxfdp1, (void *)readset, (void *) writeset, (void *) exceptset, timeout ? (s32_t)timeout->tv_sec : (s32_t) - 1, timeout ? (s32_t)timeout->tv_usec : (s32_t) - 1)); @@ -2372,7 +2372,7 @@ lwip_poll(struct pollfd *fds, nfds_t nfds, int timeout) int waited = 0; #endif - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_poll(%p, %d, %d)\n", + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_poll(0x%016llX, %d, %d)\n", (void*)fds, (int)nfds, timeout)); LWIP_ERROR("lwip_poll: invalid fds", ((fds != NULL && nfds > 0) || (fds == NULL && nfds == 0)), set_errno(EINVAL); return -1;); @@ -3882,7 +3882,7 @@ lwip_ioctl(int s, long cmd, void *argp) } *((int *)argp) = recv_avail; - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, FIONREAD, %p) = %"U16_F"\n", s, argp, *((u16_t *)argp))); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, FIONREAD, 0x%016llX) = %"U16_F"\n", s, argp, *((u16_t *)argp))); set_errno(0); done_socket(sock); return 0; @@ -3905,7 +3905,7 @@ lwip_ioctl(int s, long cmd, void *argp) default: break; } /* switch (cmd) */ - LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, UNIMPL: 0x%lx, %p)\n", s, cmd, argp)); + LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, UNIMPL: 0x%lx, 0x%016llX)\n", s, cmd, argp)); set_errno(ENOSYS); /* not yet implemented */ done_socket(sock); return -1; diff --git a/code/third_party/lwip/src/api/tcpip.c b/code/third_party/lwip/src/api/tcpip.c index eda3e1a..df3c691 100644 --- a/code/third_party/lwip/src/api/tcpip.c +++ b/code/third_party/lwip/src/api/tcpip.c @@ -167,16 +167,16 @@ tcpip_thread_handle_msg(struct tcpip_msg *msg) switch (msg->type) { #if !LWIP_TCPIP_CORE_LOCKING case TCPIP_MSG_API: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API message %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API message 0x%016llX\n", (void *)msg)); msg->msg.api_msg.function(msg->msg.api_msg.msg); break; case TCPIP_MSG_API_CALL: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API CALL message %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API CALL message 0x%016llX\n", (void *)msg)); msg->msg.api_call.arg->err = msg->msg.api_call.function(msg->msg.api_call.arg); sys_sem_signal(msg->msg.api_call.sem); break; case TCPIP_MSG_CALLBACK_STATIC_WAIT: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK WAIT message %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK WAIT message 0x%016llX\n", (void *)msg)); msg->msg.cb_wait.function(msg->msg.cb_wait.ctx); sys_sem_signal(msg->msg.cb_wait.sem); break; @@ -184,7 +184,7 @@ tcpip_thread_handle_msg(struct tcpip_msg *msg) #if !LWIP_TCPIP_CORE_LOCKING_INPUT case TCPIP_MSG_INPKT: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: PACKET %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: PACKET 0x%016llX\n", (void *)msg)); if (msg->msg.inp.input_fn(msg->msg.inp.p, msg->msg.inp.netif) != ERR_OK) { pbuf_free(msg->msg.inp.p); } @@ -194,25 +194,25 @@ tcpip_thread_handle_msg(struct tcpip_msg *msg) #if LWIP_TCPIP_TIMEOUT && LWIP_TIMERS case TCPIP_MSG_TIMEOUT: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: TIMEOUT %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: TIMEOUT 0x%016llX\n", (void *)msg)); sys_timeout(msg->msg.tmo.msecs, msg->msg.tmo.h, msg->msg.tmo.arg); memp_free(MEMP_TCPIP_MSG_API, msg); break; case TCPIP_MSG_UNTIMEOUT: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: UNTIMEOUT %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: UNTIMEOUT 0x%016llX\n", (void *)msg)); sys_untimeout(msg->msg.tmo.h, msg->msg.tmo.arg); memp_free(MEMP_TCPIP_MSG_API, msg); break; #endif /* LWIP_TCPIP_TIMEOUT && LWIP_TIMERS */ case TCPIP_MSG_CALLBACK: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK 0x%016llX\n", (void *)msg)); msg->msg.cb.function(msg->msg.cb.ctx); memp_free(MEMP_TCPIP_MSG_API, msg); break; case TCPIP_MSG_CALLBACK_STATIC: - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK_STATIC %p\n", (void *)msg)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK_STATIC 0x%016llX\n", (void *)msg)); msg->msg.cb.function(msg->msg.cb.ctx); break; @@ -255,7 +255,7 @@ tcpip_inpkt(struct pbuf *p, struct netif *inp, netif_input_fn input_fn) { #if LWIP_TCPIP_CORE_LOCKING_INPUT err_t ret; - LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_inpkt: PACKET %p/%p\n", (void *)p, (void *)inp)); + LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_inpkt: PACKET 0x%016llX/0x%016llX\n", (void *)p, (void *)inp)); LOCK_TCPIP_CORE(); ret = input_fn(p, inp); UNLOCK_TCPIP_CORE(); diff --git a/code/third_party/lwip/src/apps/http/httpd.c b/code/third_party/lwip/src/apps/http/httpd.c index d4c52b1..14c40a7 100644 --- a/code/third_party/lwip/src/apps/http/httpd.c +++ b/code/third_party/lwip/src/apps/http/httpd.c @@ -623,7 +623,7 @@ static err_t http_close_or_abort_conn(struct altcp_pcb *pcb, struct http_state *hs, u8_t abort_conn) { err_t err; - LWIP_DEBUGF(HTTPD_DEBUG, ("Closing connection %p\n", (void *)pcb)); + LWIP_DEBUGF(HTTPD_DEBUG, ("Closing connection 0x%016llX\n", (void *)pcb)); http_state_close_post(hs); @@ -642,7 +642,7 @@ http_close_or_abort_conn(struct altcp_pcb *pcb, struct http_state *hs, u8_t abor } err = altcp_close(pcb); if (err != ERR_OK) { - LWIP_DEBUGF(HTTPD_DEBUG, ("Error %d closing %p\n", err, (void *)pcb)); + LWIP_DEBUGF(HTTPD_DEBUG, ("Error %d closing 0x%016llX\n", err, (void *)pcb)); /* error closing, try again later in poll */ altcp_poll(pcb, http_poll, HTTPD_POLL_INTERVAL); } @@ -1590,7 +1590,7 @@ http_send(struct altcp_pcb *pcb, struct http_state *hs) { u8_t data_to_send = HTTP_NO_DATA_TO_SEND; - LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_send: pcb=%p hs=%p left=%d\n", (void *)pcb, + LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_send: pcb=0x%016llX hs=0x%016llX left=%d\n", (void *)pcb, (void *)hs, hs != NULL ? (int)hs->left : 0)); #if LWIP_HTTPD_SUPPORT_POST && LWIP_HTTPD_POST_MANUAL_WND @@ -2480,7 +2480,7 @@ http_sent(void *arg, struct altcp_pcb *pcb, u16_t len) { struct http_state *hs = (struct http_state *)arg; - LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_sent %p\n", (void *)pcb)); + LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_sent 0x%016llX\n", (void *)pcb)); LWIP_UNUSED_ARG(len); @@ -2506,7 +2506,7 @@ static err_t http_poll(void *arg, struct altcp_pcb *pcb) { struct http_state *hs = (struct http_state *)arg; - LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_poll: pcb=%p hs=%p pcb_state=%s\n", + LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_poll: pcb=0x%016llX hs=0x%016llX pcb_state=%s\n", (void *)pcb, (void *)hs, tcp_debug_state_str(altcp_dbg_get_tcp_state(pcb)))); if (hs == NULL) { @@ -2554,7 +2554,7 @@ static err_t http_recv(void *arg, struct altcp_pcb *pcb, struct pbuf *p, err_t err) { struct http_state *hs = (struct http_state *)arg; - LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: pcb=%p pbuf=%p err=%s\n", (void *)pcb, + LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: pcb=0x%016llX pbuf=0x%016llX err=%s\n", (void *)pcb, (void *)p, lwip_strerr(err))); if ((err != ERR_OK) || (p == NULL) || (hs == NULL)) { @@ -2616,7 +2616,7 @@ http_recv(void *arg, struct altcp_pcb *pcb, struct pbuf *p, err_t err) if (hs->post_content_len_left == 0) #endif /* LWIP_HTTPD_SUPPORT_POST */ { - LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: data %p len %"S32_F"\n", (const void *)hs->file, hs->left)); + LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: data 0x%016llX len %"S32_F"\n", (const void *)hs->file, hs->left)); http_send(pcb, hs); } } else if (parsed == ERR_ARG) { @@ -2641,7 +2641,7 @@ http_accept(void *arg, struct altcp_pcb *pcb, err_t err) struct http_state *hs; LWIP_UNUSED_ARG(err); LWIP_UNUSED_ARG(arg); - LWIP_DEBUGF(HTTPD_DEBUG, ("http_accept %p / %p\n", (void *)pcb, arg)); + LWIP_DEBUGF(HTTPD_DEBUG, ("http_accept 0x%016llX / 0x%016llX\n", (void *)pcb, arg)); if ((err != ERR_OK) || (pcb == NULL)) { return ERR_VAL; diff --git a/code/third_party/lwip/src/apps/mqtt/mqtt.c b/code/third_party/lwip/src/apps/mqtt/mqtt.c index 7dd6a5a..c72efa9 100644 --- a/code/third_party/lwip/src/apps/mqtt/mqtt.c +++ b/code/third_party/lwip/src/apps/mqtt/mqtt.c @@ -1017,7 +1017,7 @@ mqtt_tcp_err_cb(void *arg, err_t err) { mqtt_client_t *client = (mqtt_client_t *)arg; LWIP_UNUSED_ARG(err); /* only used for debug output */ - LWIP_DEBUGF(MQTT_DEBUG_TRACE, ("mqtt_tcp_err_cb: TCP error callback: error %d, arg: %p\n", err, arg)); + LWIP_DEBUGF(MQTT_DEBUG_TRACE, ("mqtt_tcp_err_cb: TCP error callback: error %d, arg: 0x%016llX\n", err, arg)); LWIP_ASSERT("mqtt_tcp_err_cb: client != NULL", client != NULL); /* Set conn to null before calling close as pcb is already deallocated*/ client->conn = NULL; diff --git a/code/third_party/lwip/src/apps/smtp/smtp.c b/code/third_party/lwip/src/apps/smtp/smtp.c index 6347c11..aebfe7e 100644 --- a/code/third_party/lwip/src/apps/smtp/smtp.c +++ b/code/third_party/lwip/src/apps/smtp/smtp.c @@ -14,7 +14,7 @@ @code{.c} void my_smtp_result_fn(void *arg, u8_t smtp_result, u16_t srv_err, err_t err) { - printf("mail (%p) sent with results: 0x%02x, 0x%04x, 0x%08x\n", arg, + printf("mail (0x%016llX) sent with results: 0x%02x, 0x%04x, 0x%08x\n", arg, smtp_result, srv_err, err); } static void my_smtp_test(void) diff --git a/code/third_party/lwip/src/core/inet_chksum.c b/code/third_party/lwip/src/core/inet_chksum.c index 6f6d0be..3720606 100644 --- a/code/third_party/lwip/src/core/inet_chksum.c +++ b/code/third_party/lwip/src/core/inet_chksum.c @@ -264,7 +264,7 @@ inet_cksum_pseudo_base(struct pbuf *p, u8_t proto, u16_t proto_len, u32_t acc) /* iterate through all pbuf in chain */ for (q = p; q != NULL; q = q->next) { - LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf %p (has next %p) \n", + LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf 0x%016llX (has next 0x%016llX) \n", (void *)q, (void *)q->next)); acc += LWIP_CHKSUM(q->payload, q->len); /*LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): unwrapped lwip_chksum()=%"X32_F" \n", acc));*/ @@ -405,7 +405,7 @@ inet_cksum_pseudo_partial_base(struct pbuf *p, u8_t proto, u16_t proto_len, /* iterate through all pbuf in chain */ for (q = p; (q != NULL) && (chksum_len > 0); q = q->next) { - LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf %p (has next %p) \n", + LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf 0x%016llX (has next 0x%016llX) \n", (void *)q, (void *)q->next)); chklen = q->len; if (chklen > chksum_len) { diff --git a/code/third_party/lwip/src/core/ipv4/acd.c b/code/third_party/lwip/src/core/ipv4/acd.c index 6a2e7bb..0e145c3 100644 --- a/code/third_party/lwip/src/core/ipv4/acd.c +++ b/code/third_party/lwip/src/core/ipv4/acd.c @@ -185,7 +185,7 @@ acd_start(struct netif *netif, struct acd *acd, ip4_addr_t ipaddr) LWIP_UNUSED_ARG(netif); LWIP_DEBUGF(ACD_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, - ("acd_start(netif=%p) %c%c%"U16_F"\n", + ("acd_start(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); diff --git a/code/third_party/lwip/src/core/ipv4/autoip.c b/code/third_party/lwip/src/core/ipv4/autoip.c index c8a761f..4e2f42e 100644 --- a/code/third_party/lwip/src/core/ipv4/autoip.c +++ b/code/third_party/lwip/src/core/ipv4/autoip.c @@ -190,7 +190,7 @@ autoip_bind(struct netif *netif) autoip->state = AUTOIP_STATE_BOUND; LWIP_DEBUGF(AUTOIP_DEBUG | LWIP_DBG_TRACE, - ("autoip_bind(netif=%p) %c%c%"U16_F" %"U16_F".%"U16_F".%"U16_F".%"U16_F"\n", + ("autoip_bind(netif=0x%016llX) %c%c%"U16_F" %"U16_F".%"U16_F".%"U16_F".%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num, ip4_addr1_16(&autoip->llipaddr), ip4_addr2_16(&autoip->llipaddr), ip4_addr3_16(&autoip->llipaddr), ip4_addr4_16(&autoip->llipaddr))); @@ -265,7 +265,7 @@ autoip_start(struct netif *netif) if (autoip->state == AUTOIP_STATE_OFF) { LWIP_DEBUGF(AUTOIP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, - ("autoip_start(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], + ("autoip_start(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); /* add acd struct to list*/ @@ -282,7 +282,7 @@ autoip_start(struct netif *netif) acd_start(netif, &autoip->acd, autoip->llipaddr); } else { LWIP_DEBUGF(AUTOIP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, - ("autoip_start(): already started on netif=%p %c%c%"U16_F"\n", + ("autoip_start(): already started on netif=0x%016llX %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); } diff --git a/code/third_party/lwip/src/core/ipv4/dhcp.c b/code/third_party/lwip/src/core/ipv4/dhcp.c index 59b417a..dcdb82c 100644 --- a/code/third_party/lwip/src/core/ipv4/dhcp.c +++ b/code/third_party/lwip/src/core/ipv4/dhcp.c @@ -325,7 +325,7 @@ dhcp_handle_nak(struct netif *netif) { struct dhcp *dhcp = netif_dhcp_data(netif); - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_nak(netif=%p) %c%c%"U16_F"\n", + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_nak(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); /* Change to a defined state - set this before assigning the address to ensure the callback can use dhcp_supplied_address() */ @@ -394,7 +394,7 @@ dhcp_check(struct netif *netif) { struct dhcp *dhcp = netif_dhcp_data(netif); - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_check(netif=%p) %c%c\n", (void *)netif, (s16_t)netif->name[0], + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_check(netif=0x%016llX) %c%c\n", (void *)netif, (s16_t)netif->name[0], (s16_t)netif->name[1])); dhcp_set_state(dhcp, DHCP_STATE_CHECKING); @@ -413,7 +413,7 @@ dhcp_handle_offer(struct netif *netif, struct dhcp_msg *msg_in) { struct dhcp *dhcp = netif_dhcp_data(netif); - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_offer(netif=%p) %c%c%"U16_F"\n", + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_offer(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); /* obtain the server address */ if (dhcp_option_given(dhcp, DHCP_OPTION_IDX_SERVER_ID)) { @@ -430,7 +430,7 @@ dhcp_handle_offer(struct netif *netif, struct dhcp_msg *msg_in) dhcp_select(netif); } else { LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_LEVEL_SERIOUS, - ("dhcp_handle_offer(netif=%p) did not get server ID!\n", (void *)netif)); + ("dhcp_handle_offer(netif=0x%016llX) did not get server ID!\n", (void *)netif)); } } @@ -456,7 +456,7 @@ dhcp_select(struct netif *netif) dhcp = netif_dhcp_data(netif); LWIP_ERROR("dhcp_select: dhcp != NULL", (dhcp != NULL), return ERR_VAL;); - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_select(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_select(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); dhcp_set_state(dhcp, DHCP_STATE_REQUESTING); /* create and initialize the DHCP message header */ @@ -816,7 +816,7 @@ dhcp_start(struct netif *netif) LWIP_ERROR("netif != NULL", (netif != NULL), return ERR_ARG;); LWIP_ERROR("netif is not up, old style port?", netif_is_up(netif), return ERR_ARG;); dhcp = netif_dhcp_data(netif); - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp_start(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp_start(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); /* check MTU of the netif */ if (netif->mtu < DHCP_MAX_MSG_LEN_MIN_REQUIRED) { @@ -1087,7 +1087,7 @@ dhcp_bind(struct netif *netif) LWIP_ERROR("dhcp_bind: netif != NULL", (netif != NULL), return;); dhcp = netif_dhcp_data(netif); LWIP_ERROR("dhcp_bind: dhcp != NULL", (dhcp != NULL), return;); - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_bind(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_bind(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); /* reset time used of lease */ dhcp->lease_used = 0; @@ -1777,7 +1777,7 @@ dhcp_recv(void *arg, struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *addr, LWIP_ASSERT("invalid server address type", IP_IS_V4(addr)); - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_recv(pbuf = %p) from DHCP server %"U16_F".%"U16_F".%"U16_F".%"U16_F" port %"U16_F"\n", (void *)p, + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_recv(pbuf = 0x%016llX) from DHCP server %"U16_F".%"U16_F".%"U16_F".%"U16_F" port %"U16_F"\n", (void *)p, ip4_addr1_16(ip_2_ip4(addr)), ip4_addr2_16(ip_2_ip4(addr)), ip4_addr3_16(ip_2_ip4(addr)), ip4_addr4_16(ip_2_ip4(addr)), port)); LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("pbuf->len = %"U16_F"\n", p->len)); LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("pbuf->tot_len = %"U16_F"\n", p->tot_len)); diff --git a/code/third_party/lwip/src/core/ipv4/etharp.c b/code/third_party/lwip/src/core/ipv4/etharp.c index 5d78171..3412abd 100644 --- a/code/third_party/lwip/src/core/ipv4/etharp.c +++ b/code/third_party/lwip/src/core/ipv4/etharp.c @@ -173,7 +173,7 @@ etharp_free_entry(int i) /* and empty packet queue */ if (arp_table[i].q != NULL) { /* remove all queued packets */ - LWIP_DEBUGF(ETHARP_DEBUG, ("etharp_free_entry: freeing entry %"U16_F", packet queue %p.\n", (u16_t)i, (void *)(arp_table[i].q))); + LWIP_DEBUGF(ETHARP_DEBUG, ("etharp_free_entry: freeing entry %"U16_F", packet queue 0x%016llX.\n", (u16_t)i, (void *)(arp_table[i].q))); free_etharp_q(arp_table[i].q); arp_table[i].q = NULL; } @@ -373,7 +373,7 @@ etharp_find_entry(const ip4_addr_t *ipaddr, u8_t flags, struct netif *netif) } else if (old_queue < ARP_TABLE_SIZE) { /* recycle oldest pending (queued packets are free in etharp_free_entry) */ i = old_queue; - LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_find_entry: selecting oldest pending entry %d, freeing packet queue %p\n", (int)i, (void *)(arp_table[i].q))); + LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_find_entry: selecting oldest pending entry %d, freeing packet queue 0x%016llX\n", (int)i, (void *)(arp_table[i].q))); /* no empty or recyclable entries found */ } else { LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_find_entry: no empty or recyclable entries found\n")); @@ -1064,27 +1064,27 @@ etharp_query(struct netif *netif, const ip4_addr_t *ipaddr, struct pbuf *q) memp_free(MEMP_ARP_QUEUE, old); } #endif - LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: queued packet %p on ARP entry %"U16_F"\n", (void *)q, i)); + LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: queued packet 0x%016llX on ARP entry %"U16_F"\n", (void *)q, i)); result = ERR_OK; } else { /* the pool MEMP_ARP_QUEUE is empty */ pbuf_free(p); - LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: could not queue a copy of PBUF_REF packet %p (out of memory)\n", (void *)q)); + LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: could not queue a copy of PBUF_REF packet 0x%016llX (out of memory)\n", (void *)q)); result = ERR_MEM; } #else /* ARP_QUEUEING */ /* always queue one packet per ARP request only, freeing a previously queued packet */ if (arp_table[i].q != NULL) { - LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: dropped previously queued packet %p for ARP entry %"U16_F"\n", (void *)q, (u16_t)i)); + LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: dropped previously queued packet 0x%016llX for ARP entry %"U16_F"\n", (void *)q, (u16_t)i)); pbuf_free(arp_table[i].q); } arp_table[i].q = p; result = ERR_OK; - LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: queued packet %p on ARP entry %"U16_F"\n", (void *)q, (u16_t)i)); + LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: queued packet 0x%016llX on ARP entry %"U16_F"\n", (void *)q, (u16_t)i)); #endif /* ARP_QUEUEING */ } else { ETHARP_STATS_INC(etharp.memerr); - LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: could not queue a copy of PBUF_REF packet %p (out of memory)\n", (void *)q)); + LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: could not queue a copy of PBUF_REF packet 0x%016llX (out of memory)\n", (void *)q)); result = ERR_MEM; } } diff --git a/code/third_party/lwip/src/core/ipv4/igmp.c b/code/third_party/lwip/src/core/ipv4/igmp.c index 01538a4..c297934 100644 --- a/code/third_party/lwip/src/core/ipv4/igmp.c +++ b/code/third_party/lwip/src/core/ipv4/igmp.c @@ -129,7 +129,7 @@ igmp_start(struct netif *netif) { struct igmp_group *group; - LWIP_DEBUGF(IGMP_DEBUG, ("igmp_start: starting IGMP processing on if %p\n", (void *)netif)); + LWIP_DEBUGF(IGMP_DEBUG, ("igmp_start: starting IGMP processing on if 0x%016llX\n", (void *)netif)); group = igmp_lookup_group(netif, &allsystems); @@ -141,7 +141,7 @@ igmp_start(struct netif *netif) if (netif->igmp_mac_filter != NULL) { LWIP_DEBUGF(IGMP_DEBUG, ("igmp_start: igmp_mac_filter(ADD ")); ip4_addr_debug_print_val(IGMP_DEBUG, allsystems); - LWIP_DEBUGF(IGMP_DEBUG, (") on if %p\n", (void *)netif)); + LWIP_DEBUGF(IGMP_DEBUG, (") on if 0x%016llX\n", (void *)netif)); netif->igmp_mac_filter(netif, &allsystems, NETIF_ADD_MAC_FILTER); } @@ -170,7 +170,7 @@ igmp_stop(struct netif *netif) if (netif->igmp_mac_filter != NULL) { LWIP_DEBUGF(IGMP_DEBUG, ("igmp_stop: igmp_mac_filter(DEL ")); ip4_addr_debug_print_val(IGMP_DEBUG, group->group_address); - LWIP_DEBUGF(IGMP_DEBUG, (") on if %p\n", (void *)netif)); + LWIP_DEBUGF(IGMP_DEBUG, (") on if 0x%016llX\n", (void *)netif)); netif->igmp_mac_filter(netif, &(group->group_address), NETIF_DEL_MAC_FILTER); } @@ -193,7 +193,7 @@ igmp_report_groups(struct netif *netif) { struct igmp_group *group = netif_igmp_data(netif); - LWIP_DEBUGF(IGMP_DEBUG, ("igmp_report_groups: sending IGMP reports on if %p\n", (void *)netif)); + LWIP_DEBUGF(IGMP_DEBUG, ("igmp_report_groups: sending IGMP reports on if 0x%016llX\n", (void *)netif)); /* Skip the first group in the list, it is always the allsystems group added in igmp_start() */ if (group != NULL) { @@ -280,7 +280,7 @@ igmp_lookup_group(struct netif *ifp, const ip4_addr_t *addr) LWIP_DEBUGF(IGMP_DEBUG, ("igmp_lookup_group: %sallocated a new group with address ", (group ? "" : "impossible to "))); ip4_addr_debug_print(IGMP_DEBUG, addr); - LWIP_DEBUGF(IGMP_DEBUG, (" on if %p\n", (void *)ifp)); + LWIP_DEBUGF(IGMP_DEBUG, (" on if 0x%016llX\n", (void *)ifp)); return group; } @@ -340,7 +340,7 @@ igmp_input(struct pbuf *p, struct netif *inp, const ip4_addr_t *dest) ip4_addr_debug_print_val(IGMP_DEBUG, ip4_current_header()->src); LWIP_DEBUGF(IGMP_DEBUG, (" to address ")); ip4_addr_debug_print_val(IGMP_DEBUG, ip4_current_header()->dest); - LWIP_DEBUGF(IGMP_DEBUG, (" on if %p\n", (void *)inp)); + LWIP_DEBUGF(IGMP_DEBUG, (" on if 0x%016llX\n", (void *)inp)); /* Now calculate and check the checksum */ igmp = (struct igmp_msg *)p->payload; @@ -427,7 +427,7 @@ igmp_input(struct pbuf *p, struct netif *inp, const ip4_addr_t *dest) } break; default: - LWIP_DEBUGF(IGMP_DEBUG, ("igmp_input: unexpected msg %d in state %d on group %p on if %p\n", + LWIP_DEBUGF(IGMP_DEBUG, ("igmp_input: unexpected msg %d in state %d on group 0x%016llX on if 0x%016llX\n", igmp->igmp_msgtype, group->group_state, (void *)&group, (void *)inp)); IGMP_STATS_INC(igmp.proterr); break; @@ -512,7 +512,7 @@ igmp_joingroup_netif(struct netif *netif, const ip4_addr_t *groupaddr) if ((group->use == 0) && (netif->igmp_mac_filter != NULL)) { LWIP_DEBUGF(IGMP_DEBUG, ("igmp_joingroup_netif: igmp_mac_filter(ADD ")); ip4_addr_debug_print(IGMP_DEBUG, groupaddr); - LWIP_DEBUGF(IGMP_DEBUG, (") on if %p\n", (void *)netif)); + LWIP_DEBUGF(IGMP_DEBUG, (") on if 0x%016llX\n", (void *)netif)); netif->igmp_mac_filter(netif, groupaddr, NETIF_ADD_MAC_FILTER); } @@ -616,7 +616,7 @@ igmp_leavegroup_netif(struct netif *netif, const ip4_addr_t *groupaddr) if (netif->igmp_mac_filter != NULL) { LWIP_DEBUGF(IGMP_DEBUG, ("igmp_leavegroup_netif: igmp_mac_filter(DEL ")); ip4_addr_debug_print(IGMP_DEBUG, groupaddr); - LWIP_DEBUGF(IGMP_DEBUG, (") on if %p\n", (void *)netif)); + LWIP_DEBUGF(IGMP_DEBUG, (") on if 0x%016llX\n", (void *)netif)); netif->igmp_mac_filter(netif, groupaddr, NETIF_DEL_MAC_FILTER); } @@ -672,7 +672,7 @@ igmp_timeout(struct netif *netif, struct igmp_group *group) (!(ip4_addr_eq(&(group->group_address), &allsystems)))) { LWIP_DEBUGF(IGMP_DEBUG, ("igmp_timeout: report membership for group with address ")); ip4_addr_debug_print_val(IGMP_DEBUG, group->group_address); - LWIP_DEBUGF(IGMP_DEBUG, (" on if %p\n", (void *)netif)); + LWIP_DEBUGF(IGMP_DEBUG, (" on if 0x%016llX\n", (void *)netif)); group->group_state = IGMP_GROUP_IDLE_MEMBER; diff --git a/code/third_party/lwip/src/core/ipv6/dhcp6.c b/code/third_party/lwip/src/core/ipv6/dhcp6.c index 9252e2e..dda4cab 100644 --- a/code/third_party/lwip/src/core/ipv6/dhcp6.c +++ b/code/third_party/lwip/src/core/ipv6/dhcp6.c @@ -317,7 +317,7 @@ dhcp6_enable_stateless(struct netif *netif) { struct dhcp6 *dhcp6; - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp6_enable_stateless(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp6_enable_stateless(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); dhcp6 = dhcp6_get_struct(netif, "dhcp6_enable_stateless()"); if (dhcp6 == NULL) { @@ -347,7 +347,7 @@ dhcp6_disable(struct netif *netif) { struct dhcp6 *dhcp6; - LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp6_disable(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); + LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp6_disable(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num)); dhcp6 = netif_dhcp6_data(netif); if (dhcp6 != NULL) { @@ -712,7 +712,7 @@ dhcp6_recv(void *arg, struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *addr LWIP_ERROR("invalid server address type", IP_IS_V6(addr), goto free_pbuf_and_return;); - LWIP_DEBUGF(DHCP6_DEBUG | LWIP_DBG_TRACE, ("dhcp6_recv(pbuf = %p) from DHCPv6 server %s port %"U16_F"\n", (void *)p, + LWIP_DEBUGF(DHCP6_DEBUG | LWIP_DBG_TRACE, ("dhcp6_recv(pbuf = 0x%016llX) from DHCPv6 server %s port %"U16_F"\n", (void *)p, ipaddr_ntoa(addr), port)); LWIP_DEBUGF(DHCP6_DEBUG | LWIP_DBG_TRACE, ("pbuf->len = %"U16_F"\n", p->len)); LWIP_DEBUGF(DHCP6_DEBUG | LWIP_DBG_TRACE, ("pbuf->tot_len = %"U16_F"\n", p->tot_len)); diff --git a/code/third_party/lwip/src/core/ipv6/nd6.c b/code/third_party/lwip/src/core/ipv6/nd6.c index f0a321c..abc243a 100644 --- a/code/third_party/lwip/src/core/ipv6/nd6.c +++ b/code/third_party/lwip/src/core/ipv6/nd6.c @@ -2156,12 +2156,12 @@ nd6_queue_packet(s8_t neighbor_index, struct pbuf *q) /* queue did not exist, first item in queue */ neighbor_cache[neighbor_index].q = new_entry; } - LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: queued packet %p on neighbor entry %"S16_F"\n", (void *)p, (s16_t)neighbor_index)); + LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: queued packet 0x%016llX on neighbor entry %"S16_F"\n", (void *)p, (s16_t)neighbor_index)); result = ERR_OK; } else { /* the pool MEMP_ND6_QUEUE is empty */ pbuf_free(p); - LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: could not queue a copy of packet %p (out of memory)\n", (void *)p)); + LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: could not queue a copy of packet 0x%016llX (out of memory)\n", (void *)p)); /* { result == ERR_MEM } through initialization */ } #else /* LWIP_ND6_QUEUEING */ @@ -2170,11 +2170,11 @@ nd6_queue_packet(s8_t neighbor_index, struct pbuf *q) pbuf_free(neighbor_cache[neighbor_index].q); } neighbor_cache[neighbor_index].q = p; - LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: queued packet %p on neighbor entry %"S16_F"\n", (void *)p, (s16_t)neighbor_index)); + LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: queued packet 0x%016llX on neighbor entry %"S16_F"\n", (void *)p, (s16_t)neighbor_index)); result = ERR_OK; #endif /* LWIP_ND6_QUEUEING */ } else { - LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: could not queue a copy of packet %p (out of memory)\n", (void *)q)); + LWIP_DEBUGF(LWIP_DBG_TRACE, ("ipv6: could not queue a copy of packet 0x%016llX (out of memory)\n", (void *)q)); /* { result == ERR_MEM } through initialization */ } diff --git a/code/third_party/lwip/src/core/pbuf.c b/code/third_party/lwip/src/core/pbuf.c index 67c87ad..072529c 100644 --- a/code/third_party/lwip/src/core/pbuf.c +++ b/code/third_party/lwip/src/core/pbuf.c @@ -297,7 +297,7 @@ pbuf_alloc(pbuf_layer layer, u16_t length, pbuf_type type) LWIP_ASSERT("pbuf_alloc: erroneous type", 0); return NULL; } - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_alloc(length=%"U16_F") == %p\n", length, (void *)p)); + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_alloc(length=%"U16_F") == 0x%016llX\n", length, (void *)p)); return p; } @@ -504,7 +504,7 @@ pbuf_add_header_impl(struct pbuf *p, size_t header_size_increment, u8_t force) /* boundary check fails? */ if ((u8_t *)payload < (u8_t *)p + SIZEOF_STRUCT_PBUF) { LWIP_DEBUGF( PBUF_DEBUG | LWIP_DBG_TRACE, - ("pbuf_add_header: failed as %p < %p (not enough space for new header size)\n", + ("pbuf_add_header: failed as 0x%016llX < 0x%016llX (not enough space for new header size)\n", (void *)payload, (void *)((u8_t *)p + SIZEOF_STRUCT_PBUF))); /* bail out unsuccessfully */ return 1; @@ -520,7 +520,7 @@ pbuf_add_header_impl(struct pbuf *p, size_t header_size_increment, u8_t force) return 1; } } - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_add_header: old %p new %p (%"U16_F")\n", + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_add_header: old 0x%016llX new 0x%016llX (%"U16_F")\n", (void *)p->payload, (void *)payload, increment_magnitude)); /* modify pbuf fields */ @@ -611,7 +611,7 @@ pbuf_remove_header(struct pbuf *p, size_t header_size_decrement) p->len = (u16_t)(p->len - increment_magnitude); p->tot_len = (u16_t)(p->tot_len - increment_magnitude); - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_remove_header: old %p new %p (%"U16_F")\n", + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_remove_header: old 0x%016llX new 0x%016llX (%"U16_F")\n", (void *)payload, (void *)p->payload, increment_magnitude)); return 0; @@ -739,7 +739,7 @@ pbuf_free(struct pbuf *p) ("pbuf_free(p == NULL) was called.\n")); return 0; } - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_free(%p)\n", (void *)p)); + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_free(0x%016llX)\n", (void *)p)); PERF_START; @@ -762,7 +762,7 @@ pbuf_free(struct pbuf *p) if (ref == 0) { /* remember next pbuf in chain for next iteration */ q = p->next; - LWIP_DEBUGF( PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_free: deallocating %p\n", (void *)p)); + LWIP_DEBUGF( PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_free: deallocating 0x%016llX\n", (void *)p)); alloc_src = pbuf_get_allocsrc(p); #if LWIP_SUPPORT_CUSTOM_PBUF /* is this a custom pbuf? */ @@ -793,7 +793,7 @@ pbuf_free(struct pbuf *p) /* p->ref > 0, this pbuf is still referenced to */ /* (and so the remaining pbufs in chain as well) */ } else { - LWIP_DEBUGF( PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_free: %p has ref %"U16_F", ending here.\n", (void *)p, (u16_t)ref)); + LWIP_DEBUGF( PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_free: 0x%016llX has ref %"U16_F", ending here.\n", (void *)p, (u16_t)ref)); /* stop walking through the chain */ p = NULL; } @@ -902,7 +902,7 @@ pbuf_chain(struct pbuf *h, struct pbuf *t) pbuf_cat(h, t); /* t is now referenced by h */ pbuf_ref(t); - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_chain: %p references %p\n", (void *)h, (void *)t)); + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_chain: 0x%016llX references 0x%016llX\n", (void *)h, (void *)t)); } /** @@ -931,11 +931,11 @@ pbuf_dechain(struct pbuf *p) /* total length of pbuf p is its own length only */ p->tot_len = p->len; /* q is no longer referenced by p, free it */ - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_dechain: unreferencing %p\n", (void *)q)); + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_dechain: unreferencing 0x%016llX\n", (void *)q)); tail_gone = pbuf_free(q); if (tail_gone > 0) { LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, - ("pbuf_dechain: deallocated %p (as it is no longer referenced)\n", (void *)q)); + ("pbuf_dechain: deallocated 0x%016llX (as it is no longer referenced)\n", (void *)q)); } /* return remaining tail or NULL if deallocated */ } @@ -961,7 +961,7 @@ pbuf_dechain(struct pbuf *p) err_t pbuf_copy(struct pbuf *p_to, const struct pbuf *p_from) { - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_copy(%p, %p)\n", + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_copy(0x%016llX, 0x%016llX)\n", (const void *)p_to, (const void *)p_from)); LWIP_ERROR("pbuf_copy: invalid source", p_from != NULL, return ERR_ARG;); @@ -990,7 +990,7 @@ pbuf_copy_partial_pbuf(struct pbuf *p_to, const struct pbuf *p_from, u16_t copy_ { size_t offset_to = offset, offset_from = 0, len; - LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_copy_partial_pbuf(%p, %p, %"U16_F", %"U16_F")\n", + LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_copy_partial_pbuf(0x%016llX, 0x%016llX, %"U16_F", %"U16_F")\n", (const void *)p_to, (const void *)p_from, copy_len, offset)); /* is the copy_len in range? */ diff --git a/code/third_party/lwip/src/core/raw.c b/code/third_party/lwip/src/core/raw.c index da5936a..ca49f19 100644 --- a/code/third_party/lwip/src/core/raw.c +++ b/code/third_party/lwip/src/core/raw.c @@ -475,7 +475,7 @@ raw_sendto_if_src(struct raw_pcb *pcb, struct pbuf *p, const ip_addr_t *dst_ip, pbuf_chain(q, p); } /* { first pbuf q points to header pbuf } */ - LWIP_DEBUGF(RAW_DEBUG, ("raw_sendto: added header pbuf %p before given pbuf %p\n", (void *)q, (void *)p)); + LWIP_DEBUGF(RAW_DEBUG, ("raw_sendto: added header pbuf 0x%016llX before given pbuf 0x%016llX\n", (void *)q, (void *)p)); } else { /* first pbuf q equals given pbuf */ q = p; @@ -489,7 +489,7 @@ raw_sendto_if_src(struct raw_pcb *pcb, struct pbuf *p, const ip_addr_t *dst_ip, if (IP_IS_V4(dst_ip)) { /* broadcast filter? */ if (!ip_get_option(pcb, SOF_BROADCAST) && ip_addr_isbroadcast(dst_ip, netif)) { - LWIP_DEBUGF(RAW_DEBUG | LWIP_DBG_LEVEL_WARNING, ("raw_sendto: SOF_BROADCAST not enabled on pcb %p\n", (void *)pcb)); + LWIP_DEBUGF(RAW_DEBUG | LWIP_DBG_LEVEL_WARNING, ("raw_sendto: SOF_BROADCAST not enabled on pcb 0x%016llX\n", (void *)pcb)); /* free any temporary header pbuf allocated by pbuf_header() */ if (q != p) { pbuf_free(q); diff --git a/code/third_party/lwip/src/core/tcp.c b/code/third_party/lwip/src/core/tcp.c index 980229f..e47a453 100644 --- a/code/third_party/lwip/src/core/tcp.c +++ b/code/third_party/lwip/src/core/tcp.c @@ -1742,7 +1742,7 @@ tcp_kill_prio(u8_t prio) } } if (inactive != NULL) { - LWIP_DEBUGF(TCP_DEBUG, ("tcp_kill_prio: killing oldest PCB %p (%"S32_F")\n", + LWIP_DEBUGF(TCP_DEBUG, ("tcp_kill_prio: killing oldest PCB 0x%016llX (%"S32_F")\n", (void *)inactive, inactivity)); tcp_abort(inactive); } @@ -1773,7 +1773,7 @@ tcp_kill_state(enum tcp_state state) } } if (inactive != NULL) { - LWIP_DEBUGF(TCP_DEBUG, ("tcp_kill_closing: killing oldest %s PCB %p (%"S32_F")\n", + LWIP_DEBUGF(TCP_DEBUG, ("tcp_kill_closing: killing oldest %s PCB 0x%016llX (%"S32_F")\n", tcp_state_str[state], (void *)inactive, inactivity)); /* Don't send a RST, since no data is lost. */ tcp_abandon(inactive, 0); @@ -1800,7 +1800,7 @@ tcp_kill_timewait(void) } } if (inactive != NULL) { - LWIP_DEBUGF(TCP_DEBUG, ("tcp_kill_timewait: killing oldest TIME-WAIT PCB %p (%"S32_F")\n", + LWIP_DEBUGF(TCP_DEBUG, ("tcp_kill_timewait: killing oldest TIME-WAIT PCB 0x%016llX (%"S32_F")\n", (void *)inactive, inactivity)); tcp_abort(inactive); } @@ -2323,7 +2323,7 @@ tcp_netif_ip_addr_changed_pcblist(const ip_addr_t *old_addr, struct tcp_pcb *pcb ) { /* this connection must be aborted */ struct tcp_pcb *next = pcb->next; - LWIP_DEBUGF(NETIF_DEBUG | LWIP_DBG_STATE, ("netif_set_ipaddr: aborting TCP pcb %p\n", (void *)pcb)); + LWIP_DEBUGF(NETIF_DEBUG | LWIP_DBG_STATE, ("netif_set_ipaddr: aborting TCP pcb 0x%016llX\n", (void *)pcb)); tcp_abort(pcb); pcb = next; } else { diff --git a/code/third_party/lwip/src/core/tcp_out.c b/code/third_party/lwip/src/core/tcp_out.c index f7b8c99..d1e535f 100644 --- a/code/third_party/lwip/src/core/tcp_out.c +++ b/code/third_party/lwip/src/core/tcp_out.c @@ -427,7 +427,7 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags) apiflags |= TCP_WRITE_FLAG_COPY; #endif /* LWIP_NETIF_TX_SINGLE_PBUF */ - LWIP_DEBUGF(TCP_OUTPUT_DEBUG, ("tcp_write(pcb=%p, data=%p, len=%"U16_F", apiflags=%"U16_F")\n", + LWIP_DEBUGF(TCP_OUTPUT_DEBUG, ("tcp_write(pcb=0x%016llX, data=0x%016llX, len=%"U16_F", apiflags=%"U16_F")\n", (void *)pcb, arg, len, (u16_t)apiflags)); LWIP_ERROR("tcp_write: arg == NULL (programmer violates API)", arg != NULL, return ERR_ARG;); @@ -1268,7 +1268,7 @@ tcp_output(struct tcp_pcb *pcb) seg = pcb->unsent; if (seg == NULL) { - LWIP_DEBUGF(TCP_OUTPUT_DEBUG, ("tcp_output: nothing to send (%p)\n", + LWIP_DEBUGF(TCP_OUTPUT_DEBUG, ("tcp_output: nothing to send (0x%016llX)\n", (void *)pcb->unsent)); LWIP_DEBUGF(TCP_CWND_DEBUG, ("tcp_output: snd_wnd %"TCPWNDSIZE_F ", cwnd %"TCPWNDSIZE_F", wnd %"U32_F diff --git a/code/third_party/lwip/src/core/timeouts.c b/code/third_party/lwip/src/core/timeouts.c index a0b4cc4..44aaac0 100644 --- a/code/third_party/lwip/src/core/timeouts.c +++ b/code/third_party/lwip/src/core/timeouts.c @@ -198,7 +198,7 @@ sys_timeout_abs(u32_t abs_time, sys_timeout_handler handler, void *arg) #if LWIP_DEBUG_TIMERNAMES timeout->handler_name = handler_name; - LWIP_DEBUGF(TIMERS_DEBUG, ("sys_timeout: %p abs_time=%"U32_F" handler=%s arg=%p\n", + LWIP_DEBUGF(TIMERS_DEBUG, ("sys_timeout: 0x%016llX abs_time=%"U32_F" handler=%s arg=0x%016llX\n", (void *)timeout, abs_time, handler_name, (void *)arg)); #endif /* LWIP_DEBUG_TIMERNAMES */ @@ -381,7 +381,7 @@ sys_check_timeouts(void) current_timeout_due_time = tmptimeout->time; #if LWIP_DEBUG_TIMERNAMES if (handler != NULL) { - LWIP_DEBUGF(TIMERS_DEBUG, ("sct calling h=%s t=%"U32_F" arg=%p\n", + LWIP_DEBUGF(TIMERS_DEBUG, ("sct calling h=%s t=%"U32_F" arg=0x%016llX\n", tmptimeout->handler_name, sys_now() - tmptimeout->time, arg)); } #endif /* LWIP_DEBUG_TIMERNAMES */ diff --git a/code/third_party/lwip/src/core/udp.c b/code/third_party/lwip/src/core/udp.c index cdc850a..7ecf6d7 100644 --- a/code/third_party/lwip/src/core/udp.c +++ b/code/third_party/lwip/src/core/udp.c @@ -737,7 +737,7 @@ udp_sendto_if_src_chksum(struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *d #endif /* LWIP_IPV6 */ ip_addr_isbroadcast(dst_ip, netif)) { LWIP_DEBUGF(UDP_DEBUG | LWIP_DBG_LEVEL_SERIOUS, - ("udp_sendto_if: SOF_BROADCAST not enabled on pcb %p\n", (void *)pcb)); + ("udp_sendto_if: SOF_BROADCAST not enabled on pcb 0x%016llX\n", (void *)pcb)); return ERR_VAL; } #endif /* LWIP_IPV4 && IP_SOF_BROADCAST */ @@ -771,12 +771,12 @@ udp_sendto_if_src_chksum(struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *d } /* first pbuf q points to header pbuf */ LWIP_DEBUGF(UDP_DEBUG, - ("udp_send: added header pbuf %p before given pbuf %p\n", (void *)q, (void *)p)); + ("udp_send: added header pbuf 0x%016llX before given pbuf 0x%016llX\n", (void *)q, (void *)p)); } else { /* adding space for header within p succeeded */ /* first pbuf q equals given pbuf */ q = p; - LWIP_DEBUGF(UDP_DEBUG, ("udp_send: added header in given pbuf %p\n", (void *)p)); + LWIP_DEBUGF(UDP_DEBUG, ("udp_send: added header in given pbuf 0x%016llX\n", (void *)p)); } LWIP_ASSERT("check that first pbuf can hold struct udp_hdr", (q->len >= sizeof(struct udp_hdr))); diff --git a/code/third_party/lwip/src/include/lwip/priv/tcp_priv.h b/code/third_party/lwip/src/include/lwip/priv/tcp_priv.h index b896f5a..f38f5fe 100644 --- a/code/third_party/lwip/src/include/lwip/priv/tcp_priv.h +++ b/code/third_party/lwip/src/include/lwip/priv/tcp_priv.h @@ -354,7 +354,7 @@ extern struct tcp_pcb ** const tcp_pcb_lists[NUM_TCP_PCB_LISTS]; #if TCP_DEBUG_PCB_LISTS #define TCP_REG(pcbs, npcb) do {\ struct tcp_pcb *tcp_tmp_pcb; \ - LWIP_DEBUGF(TCP_DEBUG, ("TCP_REG %p local port %"U16_F"\n", (void *)(npcb), (npcb)->local_port)); \ + LWIP_DEBUGF(TCP_DEBUG, ("TCP_REG 0x%016llX local port %"U16_F"\n", (void *)(npcb), (npcb)->local_port)); \ for (tcp_tmp_pcb = *(pcbs); \ tcp_tmp_pcb != NULL; \ tcp_tmp_pcb = tcp_tmp_pcb->next) { \ @@ -370,7 +370,7 @@ extern struct tcp_pcb ** const tcp_pcb_lists[NUM_TCP_PCB_LISTS]; #define TCP_RMV(pcbs, npcb) do { \ struct tcp_pcb *tcp_tmp_pcb; \ LWIP_ASSERT("TCP_RMV: pcbs != NULL", *(pcbs) != NULL); \ - LWIP_DEBUGF(TCP_DEBUG, ("TCP_RMV: removing %p from %p\n", (void *)(npcb), (void *)(*(pcbs)))); \ + LWIP_DEBUGF(TCP_DEBUG, ("TCP_RMV: removing 0x%016llX from 0x%016llX\n", (void *)(npcb), (void *)(*(pcbs)))); \ if(*(pcbs) == (npcb)) { \ *(pcbs) = (*pcbs)->next; \ } else for (tcp_tmp_pcb = *(pcbs); tcp_tmp_pcb != NULL; tcp_tmp_pcb = tcp_tmp_pcb->next) { \ @@ -381,7 +381,7 @@ extern struct tcp_pcb ** const tcp_pcb_lists[NUM_TCP_PCB_LISTS]; } \ (npcb)->next = NULL; \ LWIP_ASSERT("TCP_RMV: tcp_pcbs sane", tcp_pcbs_sane()); \ - LWIP_DEBUGF(TCP_DEBUG, ("TCP_RMV: removed %p from %p\n", (void *)(npcb), (void *)(*(pcbs)))); \ + LWIP_DEBUGF(TCP_DEBUG, ("TCP_RMV: removed 0x%016llX from 0x%016llX\n", (void *)(npcb), (void *)(*(pcbs)))); \ } while(0) #else /* LWIP_DEBUG */ diff --git a/code/third_party/lwip/src/netif/bridgeif.c b/code/third_party/lwip/src/netif/bridgeif.c index d8fe4e3..9dc5bdb 100644 --- a/code/third_party/lwip/src/netif/bridgeif.c +++ b/code/third_party/lwip/src/netif/bridgeif.c @@ -256,7 +256,7 @@ bridgeif_send_to_port(bridgeif_private_t *br, struct pbuf *p, u8_t dstport_idx) /* prevent sending out to rx port */ if (netif_get_index(portif) != p->if_idx) { if (netif_is_link_up(portif)) { - LWIP_DEBUGF(BRIDGEIF_FW_DEBUG, ("br -> flood(%p:%d) -> %d\n", (void *)p, p->if_idx, netif_get_index(portif))); + LWIP_DEBUGF(BRIDGEIF_FW_DEBUG, ("br -> flood(0x%016llX:%d) -> %d\n", (void *)p, p->if_idx, netif_get_index(portif))); return portif->linkoutput(portif, p); } } @@ -357,7 +357,7 @@ bridgeif_input(struct pbuf *p, struct netif *netif) bridgeif_send_to_ports(br, p, dstports); if (dstports & (1 << BRIDGEIF_MAX_PORTS)) { /* we pass the reference to ->input or have to free it */ - LWIP_DEBUGF(BRIDGEIF_FW_DEBUG, ("br -> input(%p)\n", (void *)p)); + LWIP_DEBUGF(BRIDGEIF_FW_DEBUG, ("br -> input(0x%016llX)\n", (void *)p)); if (br->netif->input(p, br->netif) != ERR_OK) { pbuf_free(p); } @@ -371,7 +371,7 @@ bridgeif_input(struct pbuf *p, struct netif *netif) /* is this for one of the local ports? */ if (bridgeif_is_local_mac(br, dst)) { /* yes, send to cpu port only */ - LWIP_DEBUGF(BRIDGEIF_FW_DEBUG, ("br -> input(%p)\n", (void *)p)); + LWIP_DEBUGF(BRIDGEIF_FW_DEBUG, ("br -> input(0x%016llX)\n", (void *)p)); return br->netif->input(p, br->netif); } diff --git a/code/third_party/lwip/src/netif/ethernet.c b/code/third_party/lwip/src/netif/ethernet.c index e78dd64..3fbda10 100644 --- a/code/third_party/lwip/src/netif/ethernet.c +++ b/code/third_party/lwip/src/netif/ethernet.c @@ -314,7 +314,7 @@ ethernet_output(struct netif * netif, struct pbuf * p, LWIP_ASSERT("netif->hwaddr_len must be 6 for ethernet_output!", (netif->hwaddr_len == ETH_HWADDR_LEN)); LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, - ("ethernet_output: sending packet %p\n", (void *)p)); + ("ethernet_output: sending packet 0x%016llX\n", (void *)p)); /* send the packet */ return netif->linkoutput(netif, p); diff --git a/code/third_party/lwip/src/netif/lowpan6.c b/code/third_party/lwip/src/netif/lowpan6.c index 7ddb030..5eff32f 100644 --- a/code/third_party/lwip/src/netif/lowpan6.c +++ b/code/third_party/lwip/src/netif/lowpan6.c @@ -431,7 +431,7 @@ lowpan6_frag(struct netif *netif, struct pbuf *p, const struct lowpan6_link_addr /* send the packet */ MIB2_STATS_NETIF_ADD(netif, ifoutoctets, p_frag->tot_len); - LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG | LWIP_DBG_TRACE, ("lowpan6_send: sending packet %p\n", (void *)p)); + LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG | LWIP_DBG_TRACE, ("lowpan6_send: sending packet 0x%016llX\n", (void *)p)); err = netif->linkoutput(netif, p_frag); while ((remaining_len > 0) && (err == ERR_OK)) { @@ -462,7 +462,7 @@ lowpan6_frag(struct netif *netif, struct pbuf *p, const struct lowpan6_link_addr /* send the packet */ MIB2_STATS_NETIF_ADD(netif, ifoutoctets, p_frag->tot_len); - LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG | LWIP_DBG_TRACE, ("lowpan6_send: sending packet %p\n", (void *)p)); + LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG | LWIP_DBG_TRACE, ("lowpan6_send: sending packet 0x%016llX\n", (void *)p)); err = netif->linkoutput(netif, p_frag); } } else { @@ -483,7 +483,7 @@ lowpan6_frag(struct netif *netif, struct pbuf *p, const struct lowpan6_link_addr /* send the packet */ MIB2_STATS_NETIF_ADD(netif, ifoutoctets, p_frag->tot_len); - LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG | LWIP_DBG_TRACE, ("lowpan6_send: sending packet %p\n", (void *)p)); + LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG | LWIP_DBG_TRACE, ("lowpan6_send: sending packet 0x%016llX\n", (void *)p)); err = netif->linkoutput(netif, p_frag); } diff --git a/code/third_party/lwip/src/netif/lowpan6_ble.c b/code/third_party/lwip/src/netif/lowpan6_ble.c index 328d255..fed4c27 100644 --- a/code/third_party/lwip/src/netif/lowpan6_ble.c +++ b/code/third_party/lwip/src/netif/lowpan6_ble.c @@ -273,7 +273,7 @@ rfc7668_compress(struct netif *netif, struct pbuf *p) /* send the packet */ MIB2_STATS_NETIF_ADD(netif, ifoutoctets, p_frag->tot_len); - LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG|LWIP_DBG_TRACE, ("rfc7668_output: sending packet %p\n", (void *)p)); + LWIP_DEBUGF(LWIP_LOWPAN6_DEBUG|LWIP_DBG_TRACE, ("rfc7668_output: sending packet 0x%016llX\n", (void *)p)); err = netif->linkoutput(netif, p_frag); pbuf_free(p_frag); diff --git a/code/third_party/lwip/src/netif/ppp/demand.c b/code/third_party/lwip/src/netif/ppp/demand.c index af3d9e1..c0b966f 100644 --- a/code/third_party/lwip/src/netif/ppp/demand.c +++ b/code/third_party/lwip/src/netif/ppp/demand.c @@ -288,7 +288,7 @@ loop_frame(frame, len) { struct packet *pkt; - /* dbglog("from loop: %P", frame, len); */ + /* dbglog("from loop: 0x%016llX", frame, len); */ if (len < PPP_HDRLEN) return 0; if ((PPP_PROTOCOL(frame) & 0x8000) != 0) diff --git a/code/third_party/lwip/src/netif/ppp/fsm.c b/code/third_party/lwip/src/netif/ppp/fsm.c index fffd950..67ea77b 100644 --- a/code/third_party/lwip/src/netif/ppp/fsm.c +++ b/code/third_party/lwip/src/netif/ppp/fsm.c @@ -464,7 +464,7 @@ static void fsm_rconfack(fsm *f, int id, u_char *inp, int len) { if( !(f->callbacks->ackci? (*f->callbacks->ackci)(f, inp, len): (len == 0)) ){ /* Ack is bad - ignore it */ - ppp_error(("Received bad configure-ack: %P", inp, len)); + ppp_error(("Received bad configure-ack: 0x%016llX", inp, len)); return; } f->seen_ack = 1; @@ -524,14 +524,14 @@ static void fsm_rconfnakrej(fsm *f, int code, int id, u_char *inp, int len) { treat_as_reject = (f->rnakloops >= f->maxnakloops); if (f->callbacks->nakci == NULL || !(ret = f->callbacks->nakci(f, inp, len, treat_as_reject))) { - ppp_error(("Received bad configure-nak: %P", inp, len)); + ppp_error(("Received bad configure-nak: 0x%016llX", inp, len)); return; } } else { f->rnakloops = 0; if (f->callbacks->rejci == NULL || !(ret = f->callbacks->rejci(f, inp, len))) { - ppp_error(("Received bad configure-rej: %P", inp, len)); + ppp_error(("Received bad configure-rej: 0x%016llX", inp, len)); return; } } diff --git a/code/third_party/lwip/src/netif/ppp/magic.c b/code/third_party/lwip/src/netif/ppp/magic.c index ce9e920..5910bed 100644 --- a/code/third_party/lwip/src/netif/ppp/magic.c +++ b/code/third_party/lwip/src/netif/ppp/magic.c @@ -100,7 +100,7 @@ static u32_t magic_randomseed; /* Seed used for random number generation. */ static void magic_churnrand(char *rand_data, u32_t rand_len) { lwip_md5_context md5_ctx; - /* LWIP_DEBUGF(LOG_INFO, ("magic_churnrand: %u@%P\n", rand_len, rand_data)); */ + /* LWIP_DEBUGF(LOG_INFO, ("magic_churnrand: %u@0x%016llX\n", rand_len, rand_data)); */ lwip_md5_init(&md5_ctx); lwip_md5_starts(&md5_ctx); lwip_md5_update(&md5_ctx, (u_char *)magic_randpool, sizeof(magic_randpool)); diff --git a/code/third_party/lwip/src/netif/ppp/pppos.c b/code/third_party/lwip/src/netif/ppp/pppos.c index 408b9e5..e7db7cd 100644 --- a/code/third_party/lwip/src/netif/ppp/pppos.c +++ b/code/third_party/lwip/src/netif/ppp/pppos.c @@ -847,7 +847,7 @@ pppos_input_drop(pppos_pcb *pppos) #if 0 PPPDEBUG(LOG_INFO, ("pppos_input_drop: %d:%.*H\n", pppos->in_head->len, min(60, pppos->in_head->len * 2), pppos->in_head->payload)); #endif - PPPDEBUG(LOG_INFO, ("pppos_input_drop: pbuf len=%d, addr %p\n", pppos->in_head->len, (void*)pppos->in_head)); + PPPDEBUG(LOG_INFO, ("pppos_input_drop: pbuf len=%d, addr 0x%016llX\n", pppos->in_head->len, (void*)pppos->in_head)); } pppos_input_free_current_packet(pppos); #if VJ_SUPPORT diff --git a/code/third_party/lwip/src/netif/ppp/utils.c b/code/third_party/lwip/src/netif/ppp/utils.c index eed3ed4..84b4109 100644 --- a/code/third_party/lwip/src/netif/ppp/utils.c +++ b/code/third_party/lwip/src/netif/ppp/utils.c @@ -403,7 +403,7 @@ int ppp_vslprintf(char *buf, int buflen, const char *fmt, va_list args) { #if PRINTPKT_SUPPORT /* - * vslp_printer - used in processing a %P format + * vslp_printer - used in processing a 0x%016llX format */ static void ppp_vslp_printer(void *arg, const char *fmt, ...) { int n; @@ -724,7 +724,7 @@ void ppp_dump_packet(ppp_pcb *pcb, const char *tag, unsigned char *p, int len) { return; } - ppp_dbglog(("%s %P", tag, p, len)); + ppp_dbglog(("%s 0x%016llX", tag, p, len)); } #endif /* PRINTPKT_SUPPORT */ diff --git a/code/user/apps/doom/PureDOOM.h b/code/user/apps/doom/PureDOOM.h index 4d49eb4..7275eb5 100644 --- a/code/user/apps/doom/PureDOOM.h +++ b/code/user/apps/doom/PureDOOM.h @@ -48439,7 +48439,7 @@ void Z_DumpHeap(int lowtag, int hightag) { memblock_t* block; - //doom_print("zone size: %i location: %p\n", + //doom_print("zone size: %i location: 0x%016llX\n", // mainzone->size, mainzone); doom_print("zone size: "); doom_print(doom_itoa(mainzone->size, 10)); @@ -48459,7 +48459,7 @@ void Z_DumpHeap(int lowtag, int hightag) { if (block->tag >= lowtag && block->tag <= hightag) { - //doom_print("block:%p size:%7i user:%p tag:%3i\n", block, block->size, block->user, block->tag); + //doom_print("block:0x%016llX size:%7i user:0x%016llX tag:%3i\n", block, block->size, block->user, block->tag); doom_print("block"); doom_print(doom_ptoa(block)); doom_print(" size:"); @@ -48496,7 +48496,7 @@ void Z_FileDumpHeap(void* f) { memblock_t* block; - //fprintf(f, "zone size: %i location: %p\n", mainzone->size, mainzone); + //fprintf(f, "zone size: %i location: 0x%016llX\n", mainzone->size, mainzone); doom_fprint(f, "zone size: "); doom_fprint(f, doom_itoa(mainzone->size, 10)); doom_fprint(f, " location: "); @@ -48505,7 +48505,7 @@ void Z_FileDumpHeap(void* f) for (block = mainzone->blocklist.next; ; block = block->next) { - //fprintf(f, "block:%p size:%7i user:%p tag:%3i\n", + //fprintf(f, "block:0x%016llX size:%7i user:0x%016llX tag:%3i\n", // block, block->size, block->user, block->tag); doom_fprint(f, "block:"); doom_fprint(f, doom_ptoa(block)); diff --git a/code/user/apps/doom/doom.c b/code/user/apps/doom/doom.c index 41cd143..ed45a8f 100644 --- a/code/user/apps/doom/doom.c +++ b/code/user/apps/doom/doom.c @@ -180,67 +180,67 @@ static char* doom_getenv_wrapper(const char* name) } // --------------------------------------------------------------------------- -// Input — Maps Scancode Set 1 values to PureDOOM +// Input — Maps canonical input keys to PureDOOM // --------------------------------------------------------------------------- -typedef struct { uint8_t scancode; bool extended; doom_key_t key; } KeyMap; +typedef struct { INPUT_KEY inputKey; doom_key_t key; } KeyMap; static const KeyMap g_keymap[] = { // Letters - { 0x11, false, DOOM_KEY_W }, - { 0x1E, false, DOOM_KEY_A }, - { 0x1F, false, DOOM_KEY_S }, - { 0x20, false, DOOM_KEY_D }, - { 0x12, false, DOOM_KEY_E }, - { 0x10, false, DOOM_KEY_Q }, - { 0x15, false, DOOM_KEY_Y }, - { 0x31, false, DOOM_KEY_N }, + { INPUT_KEY_W, DOOM_KEY_W }, + { INPUT_KEY_A, DOOM_KEY_A }, + { INPUT_KEY_S, DOOM_KEY_S }, + { INPUT_KEY_D, DOOM_KEY_D }, + { INPUT_KEY_E, DOOM_KEY_E }, + { INPUT_KEY_Q, DOOM_KEY_Q }, + { INPUT_KEY_Y, DOOM_KEY_Y }, + { INPUT_KEY_N, DOOM_KEY_N }, // Arrow keys - { 0x48, true, DOOM_KEY_UP_ARROW }, - { 0x50, true, DOOM_KEY_DOWN_ARROW }, - { 0x4B, true, DOOM_KEY_LEFT_ARROW }, - { 0x4D, true, DOOM_KEY_RIGHT_ARROW }, + { INPUT_KEY_UP, DOOM_KEY_UP_ARROW }, + { INPUT_KEY_DOWN, DOOM_KEY_DOWN_ARROW }, + { INPUT_KEY_LEFT, DOOM_KEY_LEFT_ARROW }, + { INPUT_KEY_RIGHT, DOOM_KEY_RIGHT_ARROW }, // Action keys - { 0x1D, false, DOOM_KEY_CTRL }, - { 0x39, false, DOOM_KEY_SPACE }, - { 0x2A, false, DOOM_KEY_SHIFT }, - { 0x38, false, DOOM_KEY_ALT }, + { INPUT_KEY_LEFT_CTRL, DOOM_KEY_CTRL }, + { INPUT_KEY_SPACE, DOOM_KEY_SPACE }, + { INPUT_KEY_LEFT_SHIFT, DOOM_KEY_SHIFT }, + { INPUT_KEY_LEFT_ALT, DOOM_KEY_ALT }, // Navigation - { 0x01, false, DOOM_KEY_ESCAPE }, - { 0x1C, false, DOOM_KEY_ENTER }, - { 0x0F, false, DOOM_KEY_TAB }, - { 0x0E, false, DOOM_KEY_BACKSPACE }, + { INPUT_KEY_ESCAPE, DOOM_KEY_ESCAPE }, + { INPUT_KEY_ENTER, DOOM_KEY_ENTER }, + { INPUT_KEY_TAB, DOOM_KEY_TAB }, + { INPUT_KEY_BACKSPACE, DOOM_KEY_BACKSPACE }, // Weapon slots - { 0x02, false, DOOM_KEY_1 }, - { 0x03, false, DOOM_KEY_2 }, - { 0x04, false, DOOM_KEY_3 }, - { 0x05, false, DOOM_KEY_4 }, - { 0x06, false, DOOM_KEY_5 }, - { 0x07, false, DOOM_KEY_6 }, - { 0x08, false, DOOM_KEY_7 }, + { INPUT_KEY_1, DOOM_KEY_1 }, + { INPUT_KEY_2, DOOM_KEY_2 }, + { INPUT_KEY_3, DOOM_KEY_3 }, + { INPUT_KEY_4, DOOM_KEY_4 }, + { INPUT_KEY_5, DOOM_KEY_5 }, + { INPUT_KEY_6, DOOM_KEY_6 }, + { INPUT_KEY_7, DOOM_KEY_7 }, // Misc - { 0x0C, false, DOOM_KEY_MINUS }, - { 0x0D, false, DOOM_KEY_EQUALS }, - { 0x33, false, DOOM_KEY_COMMA }, - { 0x34, false, DOOM_KEY_PERIOD }, + { INPUT_KEY_MINUS, DOOM_KEY_MINUS }, + { INPUT_KEY_EQUAL, DOOM_KEY_EQUALS }, + { INPUT_KEY_COMMA, DOOM_KEY_COMMA }, + { INPUT_KEY_PERIOD, DOOM_KEY_PERIOD }, // Function keys - { 0x3B, false, DOOM_KEY_F1 }, - { 0x3C, false, DOOM_KEY_F2 }, - { 0x3D, false, DOOM_KEY_F3 }, - { 0x3E, false, DOOM_KEY_F4 }, - { 0x3F, false, DOOM_KEY_F5 }, - { 0x40, false, DOOM_KEY_F6 }, - { 0x41, false, DOOM_KEY_F7 }, - { 0x42, false, DOOM_KEY_F8 }, - { 0x43, false, DOOM_KEY_F9 }, - { 0x44, false, DOOM_KEY_F10 }, - { 0x57, false, DOOM_KEY_F11 }, - { 0x58, false, DOOM_KEY_F12 }, + { INPUT_KEY_F1, DOOM_KEY_F1 }, + { INPUT_KEY_F2, DOOM_KEY_F2 }, + { INPUT_KEY_F3, DOOM_KEY_F3 }, + { INPUT_KEY_F4, DOOM_KEY_F4 }, + { INPUT_KEY_F5, DOOM_KEY_F5 }, + { INPUT_KEY_F6, DOOM_KEY_F6 }, + { INPUT_KEY_F7, DOOM_KEY_F7 }, + { INPUT_KEY_F8, DOOM_KEY_F8 }, + { INPUT_KEY_F9, DOOM_KEY_F9 }, + { INPUT_KEY_F10, DOOM_KEY_F10 }, + { INPUT_KEY_F11, DOOM_KEY_F11 }, + { INPUT_KEY_F12, DOOM_KEY_F12 }, }; #define KEYMAP_COUNT (sizeof(g_keymap) / sizeof(g_keymap[0])) @@ -250,7 +250,7 @@ static bool g_prev_key[KEYMAP_COUNT]; static void poll_input(void) { for (int i = 0; i < (int)KEYMAP_COUNT; i++) { - bool cur_down = (SysKbIsKeyDown(g_keymap[i].scancode, g_keymap[i].extended) != 0); + bool cur_down = (SysKbIsKeyDown(g_keymap[i].inputKey) != 0); bool prev_down = g_prev_key[i]; if (cur_down && !prev_down) doom_key_down(g_keymap[i].key); diff --git a/code/user/apps/edit/edit.c b/code/user/apps/edit/edit.c index a39f2a0..b0f3d2d 100644 --- a/code/user/apps/edit/edit.c +++ b/code/user/apps/edit/edit.c @@ -27,29 +27,21 @@ #define COL_CURSOR 0xE0E0E0u #define COL_CURSORF 0x000000u -#define MOD_LSHIFT (1u << 0) -#define MOD_RSHIFT (1u << 1) -#define MOD_LCTRL (1u << 2) -#define MOD_RCTRL (1u << 3) -#define MOD_CAPS (1u << 6) -#define MOD_SHIFT (MOD_LSHIFT | MOD_RSHIFT) -#define MOD_CTRL (MOD_LCTRL | MOD_RCTRL) - -#define SC_ESC 0x01u -#define SC_BACKSPACE 0x0Eu -#define SC_TAB 0x0Fu -#define SC_Q 0x10u -#define SC_S 0x1Fu -#define SC_ENTER 0x1Cu -#define SC_HOME 0x47u -#define SC_UP 0x48u -#define SC_PGUP 0x49u -#define SC_LEFT 0x4Bu -#define SC_RIGHT 0x4Du -#define SC_END 0x4Fu -#define SC_DOWN 0x50u -#define SC_PGDN 0x51u -#define SC_DEL 0x53u +#define EDIT_KEY_ESC INPUT_KEY_ESCAPE +#define EDIT_KEY_BACKSPACE INPUT_KEY_BACKSPACE +#define EDIT_KEY_TAB INPUT_KEY_TAB +#define EDIT_KEY_Q INPUT_KEY_Q +#define EDIT_KEY_S INPUT_KEY_S +#define EDIT_KEY_ENTER INPUT_KEY_ENTER +#define EDIT_KEY_HOME INPUT_KEY_HOME +#define EDIT_KEY_UP INPUT_KEY_UP +#define EDIT_KEY_PGUP INPUT_KEY_PAGE_UP +#define EDIT_KEY_LEFT INPUT_KEY_LEFT +#define EDIT_KEY_RIGHT INPUT_KEY_RIGHT +#define EDIT_KEY_END INPUT_KEY_END +#define EDIT_KEY_DOWN INPUT_KEY_DOWN +#define EDIT_KEY_PGDN INPUT_KEY_PAGE_DOWN +#define EDIT_KEY_DEL INPUT_KEY_DELETE typedef struct LINE { int len; @@ -69,22 +61,6 @@ static bool gQuitArmed = false; static char gFilename[PATH_MAX2] = "untitled.txt"; static char gMessage[MSG_MAX] = "ready"; -static const char gUnshifted[128] = { - 0, 0, '1','2','3','4','5','6','7','8','9','0','-','=','\b','\t', - 'q','w','e','r','t','y','u','i','o','p','[',']','\n', 0, 'a','s', - 'd','f','g','h','j','k','l',';','\'','`', 0, '\\','z','x','c','v', - 'b','n','m',',','.','/', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.' -}; - -static const char gShifted[128] = { - 0, 0, '!','@','#','$','%','^','&','*','(',')','_','+','\b','\t', - 'Q','W','E','R','T','Y','U','I','O','P','{','}','\n', 0, 'A','S', - 'D','F','G','H','J','K','L',':','"','~', 0, '|','Z','X','C','V', - 'B','N','M','<','>','?', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.' -}; - static void put_cell(int col, int row, char c, uint32_t fg, uint32_t bg) { SysScreenPutChar(c ? c : ' ', col * GLYPH_W, row * GLYPH_H, fg, bg); } @@ -124,14 +100,33 @@ static void append_u32(char* b, size_t cap, size_t* p, uint32_t v) { while (n > 0) append_char(b, cap, p, tmp[--n]); } -static char key_to_char(uint8_t sc, uint32_t mods) { - if (sc >= 128) return 0; - bool shifted = (mods & MOD_SHIFT) != 0; - if (mods & MOD_CAPS) { - char base = gUnshifted[sc]; - if (base >= 'a' && base <= 'z') shifted = !shifted; +static char key_to_char(INPUT_KEY key, uint32_t mods) { + if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) { + bool shifted = (mods & INPUT_MOD_SHIFT) != 0; + bool caps = (mods & INPUT_MOD_CAPS_LOCK) != 0; + char c = (char)('a' + (key - INPUT_KEY_A)); + return (shifted ^ caps) ? (char)(c - 'a' + 'A') : c; } - return shifted ? gShifted[sc] : gUnshifted[sc]; + + typedef struct { INPUT_KEY key; char normal; char shifted; } PAIR; + static const PAIR map[] = { + { INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' }, + { INPUT_KEY_3, '3', '#' }, { INPUT_KEY_4, '4', '$' }, + { INPUT_KEY_5, '5', '%' }, { INPUT_KEY_6, '6', '^' }, + { INPUT_KEY_7, '7', '&' }, { INPUT_KEY_8, '8', '*' }, + { INPUT_KEY_9, '9', '(' }, { INPUT_KEY_0, '0', ')' }, + { INPUT_KEY_MINUS, '-', '_' }, { INPUT_KEY_EQUAL, '=', '+' }, + { INPUT_KEY_LEFT_BRACKET, '[', '{' }, { INPUT_KEY_RIGHT_BRACKET, ']', '}' }, + { INPUT_KEY_BACKSLASH, '\\', '|' }, { INPUT_KEY_SEMICOLON, ';', ':' }, + { INPUT_KEY_APOSTROPHE, '\'', '"' }, { INPUT_KEY_GRAVE, '`', '~' }, + { INPUT_KEY_COMMA, ',', '<' }, { INPUT_KEY_PERIOD, '.', '>' }, + { INPUT_KEY_SLASH, '/', '?' }, { INPUT_KEY_SPACE, ' ', ' ' }, + }; + bool shifted = (mods & INPUT_MOD_SHIFT) != 0; + for (size_t i = 0; i < sizeof(map) / sizeof(map[0]); i++) { + if (map[i].key == key) return shifted ? map[i].shifted : map[i].normal; + } + return 0; } static void clamp_cursor(void) { @@ -423,11 +418,11 @@ static void editor_loop(void) { while (true) { render(); - uint8_t sc = SysKbReadScancode(); + INPUT_KEY key = SysKbReadKey(); uint32_t mods = SysKbGetModifiers(); - bool ctrl = (mods & MOD_CTRL) != 0; + bool ctrl = (mods & INPUT_MOD_CTRL) != 0; - if (ctrl && sc == SC_Q) { + if (ctrl && key == EDIT_KEY_Q) { if (gDirty && !gQuitArmed) { gQuitArmed = true; msg_set("unsaved changes: press Ctrl+Q again to quit"); @@ -438,38 +433,38 @@ static void editor_loop(void) { gQuitArmed = false; - if (ctrl && sc == SC_S) { save_file(); continue; } - if (sc == SC_ESC) { msg_set("Ctrl+Q quits, Ctrl+S saves"); continue; } + if (ctrl && key == EDIT_KEY_S) { save_file(); continue; } + if (key == EDIT_KEY_ESC) { msg_set("Ctrl+Q quits, Ctrl+S saves"); continue; } - switch (sc) { - case SC_LEFT: move_left(); break; - case SC_RIGHT: move_right(); break; - case SC_UP: move_up(); break; - case SC_DOWN: move_down(); break; - case SC_HOME: gCx = 0; break; - case SC_END: gCx = gLines[gCy].len; break; - case SC_PGUP: { + switch (key) { + case EDIT_KEY_LEFT: move_left(); break; + case EDIT_KEY_RIGHT: move_right(); break; + case EDIT_KEY_UP: move_up(); break; + case EDIT_KEY_DOWN: move_down(); break; + case EDIT_KEY_HOME: gCx = 0; break; + case EDIT_KEY_END: gCx = gLines[gCy].len; break; + case EDIT_KEY_PGUP: { int n = gRows - 2; if (n < 1) n = 1; gCy -= n; clamp_cursor(); break; } - case SC_PGDN: { + case EDIT_KEY_PGDN: { int n = gRows - 2; if (n < 1) n = 1; gCy += n; clamp_cursor(); break; } - case SC_BACKSPACE: backspace_at_cursor(); break; - case SC_DEL: delete_at_cursor(); break; - case SC_ENTER: newline_at_cursor(); break; - case SC_TAB: + case EDIT_KEY_BACKSPACE: backspace_at_cursor(); break; + case EDIT_KEY_DEL: delete_at_cursor(); break; + case EDIT_KEY_ENTER: newline_at_cursor(); break; + case EDIT_KEY_TAB: for (int i = 0; i < 4; i++) insert_char_at_cursor(' '); break; default: { - char c = key_to_char(sc, mods); + char c = key_to_char(key, mods); if (!ctrl && c >= 0x20 && c <= 0x7E) insert_char_at_cursor(c); break; } diff --git a/code/user/apps/video/bink.h b/code/user/apps/video/bink.h index 62d4eb1..228eaf2 100644 --- a/code/user/apps/video/bink.h +++ b/code/user/apps/video/bink.h @@ -173,7 +173,7 @@ typedef struct BINKSND { // will do it's own threading) // 2 = No background thread processing at all, but setup // thread mutexes, so that BinkServiceSound() can be - // call from an external background thread. + // call from an external background thread. U32 SoundDroppedOut; U32 sndconvert8; // convert back to 8-bit sound at runtime diff --git a/code/user/personalities/posix/posix_thread.c b/code/user/personalities/posix/posix_thread.c index f25e2c3..1bc9aee 100644 --- a/code/user/personalities/posix/posix_thread.c +++ b/code/user/personalities/posix/posix_thread.c @@ -50,7 +50,7 @@ static void PthreadTrampoline(uintptr_t raw) { uint32_t pid = tr->Pid; uint32_t tid = SysGetCurrentTid(); - // Create the join slot from inside the new thread. KeCreateThread makes the + // Create the join slot from inside the new thread. KeCreateThread makes the // thread runnable before the creator regains control, so the old design // raced on tr->Tid and could observe zero/stale memory. PTHREAD_SLOT* slot = PthreadSlotFind(pid, tid); diff --git a/code/user/personalities/win32/kernel32.c b/code/user/personalities/win32/kernel32.c index e269697..06d6ede 100644 --- a/code/user/personalities/win32/kernel32.c +++ b/code/user/personalities/win32/kernel32.c @@ -107,7 +107,7 @@ typedef struct _WIN32_FIND_DATAA { static inline HANDLE KH_TO_HANDLE(uint32_t kh) { return (HANDLE)(uintptr_t)kh; } static inline uint32_t HANDLE_TO_KH(HANDLE h) { return (uint32_t)(uintptr_t)h; } -// Thread IDs are not kernel handles. Keep Win32 thread pseudo-handles out of +// Thread IDs are not kernel handles. Keep Win32 thread pseudo-handles out of // the low numeric handle namespace so CloseHandle/WaitForSingleObject cannot // accidentally operate on an unrelated file/process handle with the same value. #define WIN32_THREAD_HANDLE_TAG ((uintptr_t)1ULL << 63) @@ -133,7 +133,7 @@ static void TranslatePath(const char* input, char* output) { if (!input) { output[0] = '\0'; return; } if (input[0] != '\0' && input[1] == ':' && (input[2] == '\\' || input[2] == '/')) { - // Translate "C:\" -> "0:/". Native nt-lite paths such as + // Translate "C:\" -> "0:/". Native nt-lite paths such as // "0:/system/foo.dll" are already drive-indexed and pass through. char drive = input[0]; if (drive >= 'a' && drive <= 'z') drive -= ('a' - 'A'); // To uppercase @@ -440,61 +440,60 @@ __declspec(dllexport) size_t VirtualQuery(const void* lpAddress, // --- Win32 API --- -// Updated GetAsyncKeyState implementation +// GetAsyncKeyState implementation over the canonical input key namespace. __declspec(dllexport) SHORT GetAsyncKeyState(int vKey) { - static const struct { int vk; uint8_t sc; bool ext; } vk_to_sc[] = { + if (vKey == 0x11) { // VK_CONTROL + return (SysKbIsKeyDown(INPUT_KEY_LEFT_CTRL) || SysKbIsKeyDown(INPUT_KEY_RIGHT_CTRL)) ? (SHORT)0x8000 : 0; + } + if (vKey == 0x10) { // VK_SHIFT + return (SysKbIsKeyDown(INPUT_KEY_LEFT_SHIFT) || SysKbIsKeyDown(INPUT_KEY_RIGHT_SHIFT)) ? (SHORT)0x8000 : 0; + } + if (vKey == 0x12) { // VK_MENU / ALT + return (SysKbIsKeyDown(INPUT_KEY_LEFT_ALT) || SysKbIsKeyDown(INPUT_KEY_RIGHT_ALT)) ? (SHORT)0x8000 : 0; + } + + static const struct { int vk; INPUT_KEY key; } vk_to_key[] = { // Letters - { 0x41, 0x1E, false }, { 0x42, 0x30, false }, { 0x43, 0x2E, false }, - { 0x44, 0x20, false }, { 0x45, 0x12, false }, { 0x46, 0x21, false }, - { 0x47, 0x22, false }, { 0x48, 0x23, false }, { 0x49, 0x17, false }, - { 0x4A, 0x24, false }, { 0x4B, 0x25, false }, { 0x4C, 0x26, false }, - { 0x4D, 0x32, false }, { 0x4E, 0x31, false }, { 0x4F, 0x18, false }, - { 0x50, 0x19, false }, { 0x51, 0x10, false }, { 0x52, 0x13, false }, - { 0x53, 0x1F, false }, { 0x54, 0x14, false }, { 0x55, 0x16, false }, - { 0x56, 0x2F, false }, { 0x57, 0x11, false }, { 0x58, 0x2D, false }, - { 0x59, 0x15, false }, { 0x5A, 0x2C, false }, + { 0x41, INPUT_KEY_A }, { 0x42, INPUT_KEY_B }, { 0x43, INPUT_KEY_C }, + { 0x44, INPUT_KEY_D }, { 0x45, INPUT_KEY_E }, { 0x46, INPUT_KEY_F }, + { 0x47, INPUT_KEY_G }, { 0x48, INPUT_KEY_H }, { 0x49, INPUT_KEY_I }, + { 0x4A, INPUT_KEY_J }, { 0x4B, INPUT_KEY_K }, { 0x4C, INPUT_KEY_L }, + { 0x4D, INPUT_KEY_M }, { 0x4E, INPUT_KEY_N }, { 0x4F, INPUT_KEY_O }, + { 0x50, INPUT_KEY_P }, { 0x51, INPUT_KEY_Q }, { 0x52, INPUT_KEY_R }, + { 0x53, INPUT_KEY_S }, { 0x54, INPUT_KEY_T }, { 0x55, INPUT_KEY_U }, + { 0x56, INPUT_KEY_V }, { 0x57, INPUT_KEY_W }, { 0x58, INPUT_KEY_X }, + { 0x59, INPUT_KEY_Y }, { 0x5A, INPUT_KEY_Z }, // Numbers - { 0x31, 0x02, false }, { 0x32, 0x03, false }, { 0x33, 0x04, false }, - { 0x34, 0x05, false }, { 0x35, 0x06, false }, { 0x36, 0x07, false }, - { 0x37, 0x08, false }, { 0x38, 0x09, false }, { 0x39, 0x0A, false }, - { 0x30, 0x0B, false }, + { 0x31, INPUT_KEY_1 }, { 0x32, INPUT_KEY_2 }, { 0x33, INPUT_KEY_3 }, + { 0x34, INPUT_KEY_4 }, { 0x35, INPUT_KEY_5 }, { 0x36, INPUT_KEY_6 }, + { 0x37, INPUT_KEY_7 }, { 0x38, INPUT_KEY_8 }, { 0x39, INPUT_KEY_9 }, + { 0x30, INPUT_KEY_0 }, - // Arrows (Extended) - { 0x26, 0x48, true }, // VK_UP - { 0x28, 0x50, true }, // VK_DOWN - { 0x25, 0x4B, true }, // VK_LEFT - { 0x27, 0x4D, true }, // VK_RIGHT + // Arrows + { 0x26, INPUT_KEY_UP }, { 0x28, INPUT_KEY_DOWN }, + { 0x25, INPUT_KEY_LEFT }, { 0x27, INPUT_KEY_RIGHT }, // Controls - { 0x1B, 0x01, false }, // ESC - { 0x0D, 0x1C, false }, // ENTER - { 0x20, 0x39, false }, // SPACE - { 0x11, 0x1D, false }, // CTRL (Generic) - { 0x10, 0x2A, false }, // SHIFT (Generic) - { 0x12, 0x38, false }, // ALT (Generic) - { 0x08, 0x0E, false }, // BACKSPACE - { 0x09, 0x0F, false }, // TAB + { 0x1B, INPUT_KEY_ESCAPE }, { 0x0D, INPUT_KEY_ENTER }, + { 0x20, INPUT_KEY_SPACE }, + { 0x08, INPUT_KEY_BACKSPACE }, { 0x09, INPUT_KEY_TAB }, // Function Keys - { 0x70, 0x3B, false }, { 0x71, 0x3C, false }, { 0x72, 0x3D, false }, - { 0x73, 0x3E, false }, { 0x74, 0x3F, false }, { 0x75, 0x40, false }, - { 0x76, 0x41, false }, { 0x77, 0x42, false }, { 0x78, 0x43, false }, - { 0x79, 0x44, false }, { 0x7A, 0x57, false }, { 0x7B, 0x58, false }, + { 0x70, INPUT_KEY_F1 }, { 0x71, INPUT_KEY_F2 }, { 0x72, INPUT_KEY_F3 }, + { 0x73, INPUT_KEY_F4 }, { 0x74, INPUT_KEY_F5 }, { 0x75, INPUT_KEY_F6 }, + { 0x76, INPUT_KEY_F7 }, { 0x77, INPUT_KEY_F8 }, { 0x78, INPUT_KEY_F9 }, + { 0x79, INPUT_KEY_F10 }, { 0x7A, INPUT_KEY_F11 }, { 0x7B, INPUT_KEY_F12 }, // Symbols - { 0xBD, 0x0C, false }, // MINUS '-' - { 0xBB, 0x0D, false }, // EQUALS '=' - { 0xBC, 0x33, false }, // COMMA ',' - { 0xBE, 0x34, false }, // PERIOD '.' + { 0xBD, INPUT_KEY_MINUS }, { 0xBB, INPUT_KEY_EQUAL }, + { 0xBC, INPUT_KEY_COMMA }, { 0xBE, INPUT_KEY_PERIOD }, }; - for (size_t i = 0; i < sizeof(vk_to_sc)/sizeof(vk_to_sc[0]); ++i) { - if (vk_to_sc[i].vk == vKey) { - // Check the hardware state array via our HAL helper - if (SysKbIsKeyDown(vk_to_sc[i].sc, vk_to_sc[i].ext)) { + for (size_t i = 0; i < sizeof(vk_to_key) / sizeof(vk_to_key[0]); ++i) { + if (vk_to_key[i].vk == vKey) { + if (SysKbIsKeyDown(vk_to_key[i].key)) return (SHORT)0x8000; - } break; } } @@ -668,7 +667,7 @@ __declspec(dllexport) BOOL ReadFile(HANDLE hFile, void* buf, DWORD toRead, uint32_t s = SysReadFile(HANDLE_TO_KH(hFile), buf, (size_t)toRead, &got); if (bytesRead) *bytesRead = (DWORD)got; - // Harden callers that accidentally look past *bytesRead. EOF/short reads + // Harden callers that accidentally look past *bytesRead. EOF/short reads // are legal for files, but consuming stale stack/heap bytes after them makes // the failure mode look like random game-data corruption. if (got < (size_t)toRead) @@ -899,13 +898,13 @@ __declspec(dllexport) HANDLE LoadLibraryA(const char* lpLibFileName) { } // Minimal GetProcAddress for code that probes kernel32 functions already present -// in this win32.dll image. Full dynamic DLL loading remains a kernel-loader job. +// in this win32.dll image. Full dynamic DLL loading remains a kernel-loader job. __declspec(dllexport) BOOL FreeLibrary(HANDLE hLibModule); __declspec(dllexport) void Sleep(DWORD dwMilliseconds); __declspec(dllexport) void* GetProcAddress(HANDLE hModule, const char* lpProcName) { if (hModule != WIN32_HMODULE_WIN32 || !lpProcName) { gLastError = 127; return NULL; } - // Ordinal probes are represented as small integer pointers. Do not treat + // Ordinal probes are represented as small integer pointers. Do not treat // them as strings while debugging misses. if ((uintptr_t)lpProcName < 0x10000) { KiDbgWriteA("GetProcAddress ordinal miss\n"); @@ -1116,7 +1115,7 @@ __declspec(dllexport) BOOL CreateProcessA( } uint32_t kh = 0; - // Call the syscall.dll stub. The kernel snapshots the current CWD into + // Call the syscall.dll stub. The kernel snapshots the current CWD into // the child process, so the temporary lpCurrentDirectory switch can be // restored immediately after creation. uint32_t s = K32CreateProcessNative(translatedPath, &kh, cmdline); @@ -1469,11 +1468,11 @@ __declspec(dllexport) BOOL SetCurrentDirectoryA(const char* lpPathName) { __declspec(dllexport) DWORD GetCurrentDirectoryA(DWORD nBufferLength, char* lpBuffer) { if (!lpBuffer || nBufferLength == 0) { - // Return required size (including NUL). Max CWD is 512 bytes + "C:\\". + // Return required size (including NUL). Max CWD is 512 bytes + "C:\\". return 515; } - // Kernel returns native "N:/path". Present it as a Win32 drive letter. + // Kernel returns native "N:/path". Present it as a Win32 drive letter. char raw[512]; uint32_t s = K32GetCwdNative(raw, sizeof(raw), NULL); if (s != 0) { gLastError = 1; return 0; } diff --git a/code/user/syscall/syscall_nr.inc b/code/user/syscall/syscall_nr.inc index f4f61c5..78c5aa0 100644 --- a/code/user/syscall/syscall_nr.inc +++ b/code/user/syscall/syscall_nr.inc @@ -1,94 +1,94 @@ ; syscall_nr.inc -SYS_EXIT_PROCESS EQU 0 -SYS_EXIT_THREAD EQU 1 -SYS_CREATE_PROCESS EQU 2 -SYS_WAIT_FOR_PROCESS EQU 3 -SYS_GET_PROCESS_EXIT_CODE EQU 4 -SYS_KILL_PROCESS EQU 5 -SYS_GET_PROCESS_LIST EQU 6 -SYS_GET_PROCESS_PID EQU 7 -SYS_CREATE_THREAD EQU 8 -SYS_WAIT_THREAD EQU 9 -SYS_GET_CURRENT_TID EQU 10 -SYS_GET_CURRENT_PID EQU 11 -SYS_SET_THREAD_PRIORITY EQU 12 -SYS_YIELD EQU 13 -SYS_SLEEP EQU 14 -SYS_SET_USER_APC_DISPATCHER EQU 15 -SYS_APC_RETURN EQU 16 -SYS_ALLOC EQU 17 -SYS_FREE EQU 18 -SYS_REALLOC EQU 19 -SYS_CLOSE_HANDLE EQU 20 -SYS_MUTEX_CREATE EQU 21 -SYS_MUTEX_ACQUIRE EQU 22 -SYS_MUTEX_TRY_ACQUIRE EQU 23 -SYS_MUTEX_RELEASE EQU 24 -SYS_SEM_CREATE EQU 25 -SYS_SEM_WAIT EQU 26 -SYS_SEM_TRY_WAIT EQU 27 -SYS_SEM_SIGNAL EQU 28 -SYS_EVENT_CREATE EQU 29 -SYS_EVENT_SIGNAL EQU 30 -SYS_EVENT_CLEAR EQU 31 -SYS_EVENT_WAIT EQU 32 -SYS_QUEUE_CREATE EQU 33 -SYS_QUEUE_SEND EQU 34 -SYS_QUEUE_RECV EQU 35 -SYS_PUBLISH_QUEUE EQU 36 -SYS_OPEN_NAMED_QUEUE EQU 37 -SYS_QUEUE_DUPLICATE EQU 38 -SYS_NAMED_QUEUE_OWNER_PID EQU 39 -SYS_OPEN_FILE EQU 40 -SYS_CREATE_FILE EQU 41 -SYS_READ_FILE EQU 42 -SYS_WRITE_FILE EQU 43 -SYS_READ_FILE_ASYNC EQU 44 -SYS_WRITE_FILE_ASYNC EQU 45 -SYS_CANCEL_IO EQU 46 -SYS_SEEK_FILE EQU 47 -SYS_GET_FILE_POS EQU 48 -SYS_GET_FILE_SIZE EQU 49 -SYS_TRUNCATE_FILE EQU 50 -SYS_DELETE_FILE EQU 51 -SYS_RENAME_FILE EQU 52 -SYS_OPEN_DIR EQU 53 -SYS_READ_DIR EQU 54 -SYS_CREATE_DIR EQU 55 -SYS_REMOVE_DIR EQU 56 -SYS_SET_CWD EQU 57 -SYS_GET_CWD EQU 58 -SYS_WRITE_CONSOLE EQU 59 -SYS_SCREEN_BLIT EQU 60 -SYS_SCREEN_CLEAR EQU 61 -SYS_SCREEN_GET_SIZE EQU 62 -SYS_SCREEN_PUT_CHAR EQU 63 -SYS_KB_POLL EQU 64 -SYS_KB_IS_KEY_DOWN EQU 65 -SYS_KB_READ_CHAR EQU 66 -SYS_KB_READ_SCANCODE EQU 67 -SYS_KB_GET_MODIFIERS EQU 68 -SYS_MOUSE_POLL EQU 69 -SYS_MOUSE_READ EQU 70 -SYS_MOUSE_GET_POS EQU 71 -SYS_GET_TICK EQU 72 -SYS_DEBUG_PRINT EQU 73 -SYS_QUERY_TRAP_FRAME EQU 74 -SYS_SOCKET EQU 75 -SYS_BIND EQU 76 -SYS_LISTEN EQU 77 -SYS_ACCEPT EQU 78 -SYS_CONNECT EQU 79 -SYS_SEND EQU 80 -SYS_RECV EQU 81 -SYS_SHUTDOWN EQU 82 -SYS_SEND_TO EQU 83 -SYS_RECV_FROM EQU 84 -SYS_RESOLVE_HOST EQU 85 -SYS_SET_SOCKET_OPTION EQU 86 -SYS_GET_SOCKET_OPTION EQU 87 -SYS_POLL EQU 88 -SYS_GET_MEMORY_STATUS EQU 89 +SYS_EXIT_PROCESS EQU 0 +SYS_EXIT_THREAD EQU 1 +SYS_CREATE_PROCESS EQU 2 +SYS_WAIT_FOR_PROCESS EQU 3 +SYS_GET_PROCESS_EXIT_CODE EQU 4 +SYS_KILL_PROCESS EQU 5 +SYS_GET_PROCESS_LIST EQU 6 +SYS_GET_PROCESS_PID EQU 7 +SYS_CREATE_THREAD EQU 8 +SYS_WAIT_THREAD EQU 9 +SYS_GET_CURRENT_TID EQU 10 +SYS_GET_CURRENT_PID EQU 11 +SYS_SET_THREAD_PRIORITY EQU 12 +SYS_YIELD EQU 13 +SYS_SLEEP EQU 14 +SYS_SET_USER_APC_DISPATCHER EQU 15 +SYS_APC_RETURN EQU 16 +SYS_ALLOC EQU 17 +SYS_FREE EQU 18 +SYS_REALLOC EQU 19 +SYS_CLOSE_HANDLE EQU 20 +SYS_MUTEX_CREATE EQU 21 +SYS_MUTEX_ACQUIRE EQU 22 +SYS_MUTEX_TRY_ACQUIRE EQU 23 +SYS_MUTEX_RELEASE EQU 24 +SYS_SEM_CREATE EQU 25 +SYS_SEM_WAIT EQU 26 +SYS_SEM_TRY_WAIT EQU 27 +SYS_SEM_SIGNAL EQU 28 +SYS_EVENT_CREATE EQU 29 +SYS_EVENT_SIGNAL EQU 30 +SYS_EVENT_CLEAR EQU 31 +SYS_EVENT_WAIT EQU 32 +SYS_QUEUE_CREATE EQU 33 +SYS_QUEUE_SEND EQU 34 +SYS_QUEUE_RECV EQU 35 +SYS_PUBLISH_QUEUE EQU 36 +SYS_OPEN_NAMED_QUEUE EQU 37 +SYS_QUEUE_DUPLICATE EQU 38 +SYS_NAMED_QUEUE_OWNER_PID EQU 39 +SYS_OPEN_FILE EQU 40 +SYS_CREATE_FILE EQU 41 +SYS_READ_FILE EQU 42 +SYS_WRITE_FILE EQU 43 +SYS_READ_FILE_ASYNC EQU 44 +SYS_WRITE_FILE_ASYNC EQU 45 +SYS_CANCEL_IO EQU 46 +SYS_SEEK_FILE EQU 47 +SYS_GET_FILE_POS EQU 48 +SYS_GET_FILE_SIZE EQU 49 +SYS_TRUNCATE_FILE EQU 50 +SYS_DELETE_FILE EQU 51 +SYS_RENAME_FILE EQU 52 +SYS_OPEN_DIR EQU 53 +SYS_READ_DIR EQU 54 +SYS_CREATE_DIR EQU 55 +SYS_REMOVE_DIR EQU 56 +SYS_SET_CWD EQU 57 +SYS_GET_CWD EQU 58 +SYS_WRITE_CONSOLE EQU 59 +SYS_SCREEN_BLIT EQU 60 +SYS_SCREEN_CLEAR EQU 61 +SYS_SCREEN_GET_SIZE EQU 62 +SYS_SCREEN_PUT_CHAR EQU 63 +SYS_KB_POLL EQU 64 +SYS_KB_IS_KEY_DOWN EQU 65 +SYS_KB_READ_CHAR EQU 66 +SYS_KB_READ_KEY EQU 67 +SYS_KB_GET_MODIFIERS EQU 68 +SYS_MOUSE_POLL EQU 69 +SYS_MOUSE_READ EQU 70 +SYS_MOUSE_GET_POS EQU 71 +SYS_GET_TICK EQU 72 +SYS_DEBUG_PRINT EQU 73 +SYS_QUERY_TRAP_FRAME EQU 74 +SYS_SOCKET EQU 75 +SYS_BIND EQU 76 +SYS_LISTEN EQU 77 +SYS_ACCEPT EQU 78 +SYS_CONNECT EQU 79 +SYS_SEND EQU 80 +SYS_RECV EQU 81 +SYS_SHUTDOWN EQU 82 +SYS_SEND_TO EQU 83 +SYS_RECV_FROM EQU 84 +SYS_RESOLVE_HOST EQU 85 +SYS_SET_SOCKET_OPTION EQU 86 +SYS_GET_SOCKET_OPTION EQU 87 +SYS_POLL EQU 88 +SYS_GET_MEMORY_STATUS EQU 89 -SYS_MAX EQU 90 +SYS_MAX EQU 90 diff --git a/code/user/syscall/syscall_stubs.asm b/code/user/syscall/syscall_stubs.asm index 00825fb..f01ae90 100644 --- a/code/user/syscall/syscall_stubs.asm +++ b/code/user/syscall/syscall_stubs.asm @@ -83,8 +83,8 @@ SYSCALL_STUB2 ApcReturn, SYS_APC_RETURN ; --------------------------------------------------------------------------- PUBLIC SysUserApcDispatcher SysUserApcDispatcher PROC EXPORT - ; Kernel enters here with RSP pointing at SYS_USER_APC_FRAME. There is no - ; return address because this is an IRET redirection. Preserve RBX because + ; Kernel enters here with RSP pointing at SYS_USER_APC_FRAME. There is no + ; return address because this is an IRET redirection. Preserve RBX because ; the original user context resumes after SysApcReturn. push rbx lea rbx, [rsp + 8] ; original injected RSP = frame pointer @@ -183,7 +183,7 @@ SYSCALL_STUB6 ScreenPutChar, SYS_SCREEN_PUT_CHAR SYSCALL_STUB2 KbPoll, SYS_KB_POLL SYSCALL_STUB2 KbIsKeyDown, SYS_KB_IS_KEY_DOWN SYSCALL_STUB2 KbReadChar, SYS_KB_READ_CHAR -SYSCALL_STUB2 KbReadScancode, SYS_KB_READ_SCANCODE +SYSCALL_STUB2 KbReadKey, SYS_KB_READ_KEY SYSCALL_STUB2 KbGetModifiers, SYS_KB_GET_MODIFIERS SYSCALL_STUB2 MousePoll, SYS_MOUSE_POLL diff --git a/readme.md b/readme.md index 62e9297..6200951 100644 --- a/readme.md +++ b/readme.md @@ -10,7 +10,7 @@ NT-Lite is a 64-bit operating system kernel written from scratch. It has a UEFI It boots into a shell, launches user processes, survives application crashes, and can run things it probably should not be able to run yet. -The codebase is still small enough to understand, but large enough to hurt your feelings. The kernel is just 10k lines of code and the entire project is about 20k lines. +The codebase is still small enough to understand, but large enough to hurt your feelings. The entire project is about 40k lines. ## features @@ -39,8 +39,8 @@ The codebase is still small enough to understand, but large enough to hurt your - `IoCallDriver()` dispatch path - AHCI driver, interrupt-driven - IDE driver, PIO/DMA -- FAT32 filesystem driver -- VFAT long filename support for real dotfiles like `.doomrc` +- FAT32 filesystem driver with VFAT +- USB with bus, xHCI, mass storage and HID - drive-number paths such as `0:/apps/shell.exe` - lwIP network stack as a runtime driver (`lwip.sys`) - RTL8139 and RTL816x ethernet drivers @@ -207,14 +207,14 @@ Port forwarding is enabled by default in `runa.bat` — host port 8080 maps to g ```text "apps/shell.exe" -> current working directory -"0:/apps/shell.exe" -> volume 0 -"2:/data/config.ini" -> volume 2 -"C:\apps\doom.exe" -> volume 0, mapping is simple c->0, d->1, etc. -"/apps/video.exe" -> volume 0 +"0:/apps/shell.exe" -> volume 0, native path +"2:/data/config.ini" -> volume 2, native path +"C:\apps\doom.exe" -> volume 0, win32.dll translated path, mapping is simple c->0, d->1, etc. +"/apps/video.exe" -> volume 0, posix.dll translated path, mapping is simple: root is volume 0, then directories as volumes, so "/1/apps/video.exe" -> volume 1 ``` -Volumes `0`-`7` are available. A volume only works if a filesystem is registered on it (duh). +Volumes `0`-`7` are available. A volume only works if a filesystem is registered on it (duh). The Win32 personality translates drive letters to volume numbers, while the POSIX personality maps volumes onto a tree. Path parsing is length-aware internally. Embedded NUL bytes are rejected for text paths and names. @@ -284,18 +284,15 @@ A: Don't connect to the internet with it. Q: Who is this for? A: Me. -Q: How do I contribute? -A: maksw@maksw.pl - -Q: Where's the source? -A: Right here. - Q: What's next? A: todo Q: Does it run Doom? A: `doom.exe` +Q: Does USB work? +A: For me, my keyboard, mouse and thumbdrive. + Q: If I write a program now, will it run in the future? A: Not a chance. WE DO - in fact - BREAK USERSPACE. @@ -315,8 +312,8 @@ Q: Is the Win32 layer real? A: Depends how you frame it. Some apps will run, most will not. Q: Are processes isolated? -A: No. But IPC is free. Is this how a serious modern OS should isolate processes? No. -Is it very convenient while building? Absolutely. Do I care? Nope. +A: No. But IPC is free. Is this how a serious modern OS should isolate processes? No. +Are we a serious OS? No. Is it very convenient while building? Absolutely. Do I care? Nope. Q: How often does it crash? A: Like Windows 11. @@ -339,6 +336,12 @@ A: How else do I debug it? Q: What should I read first? A: This document. +Q: How do I contribute? +A: maksw@maksw.pl + +Q: Where's the source? +A: Right here. + Q: Can I use this in production? A: Don't. diff --git a/runa.bat b/runa.bat index 69203b0..771ead2 100644 --- a/runa.bat +++ b/runa.bat @@ -1 +1 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=vpc,file=disk.vhd,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -serial stdio -M q35 -m 256M -smp cpus=4,cores=4,threads=1,sockets=1 -netdev user,id=net0,hostfwd=tcp::8080-:8080 -device rtl8139,netdev=net0 +@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=vpc,file=disk.vhd,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -serial stdio -M q35 -m 256M -smp cpus=4,cores=4,threads=1,sockets=1 -netdev user,id=net0,hostfwd=tcp::8080-:8080 -device rtl8139,netdev=net0 -device qemu-xhci,id=usb -device usb-kbd,bus=usb.0 -device usb-mouse,bus=usb.0 -drive if=none,id=usbflash,format=vpc,file=usb.vhd -device usb-storage,drive=usbflash,bus=usb.0