diff --git a/.gitignore b/.gitignore index e86b1ac..bd8eb9a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ build.obj bin/ obj/ qemu.log -fatroot/ +disk.vhd usbfatroot/ code.zip code/build/build.exe diff --git a/OVMF/OVMF_VARS_4M.fd b/OVMF/OVMF_VARS_4M.fd index 4c58e5d..5f4d30b 100644 Binary files a/OVMF/OVMF_VARS_4M.fd and b/OVMF/OVMF_VARS_4M.fd differ diff --git a/OVMF/OVMF_VARS_4M_d.fd b/OVMF/OVMF_VARS_4M_d.fd index 143cbf2..3b482b3 100644 Binary files a/OVMF/OVMF_VARS_4M_d.fd and b/OVMF/OVMF_VARS_4M_d.fd differ diff --git a/code/boot.ini b/code/boot.ini index 19372a3..8fc189f 100644 --- a/code/boot.ini +++ b/code/boot.ini @@ -1,5 +1,6 @@ [kernel] ; bitmask: 1 = debug prints, 2 = self check, 4 = file log, 4096 = checked intrusive lists +; 0x8000 reports MTRRs. Add 0x10000 for one temporary framebuffer WC experiment. debug = 0xffff [init] commandline = apps/shell_native.exe diff --git a/code/boot/uefi/main.c b/code/boot/uefi/main.c index baa4e0f..4027824 100644 --- a/code/boot/uefi/main.c +++ b/code/boot/uefi/main.c @@ -26,6 +26,11 @@ int _fltused = 0; #define KERNEL_VIRT_BASE 0xFFFF800000000000ULL #define PHYS_MAP_BASE 0xFFFF880000000000ULL #define PAGE_SIZE 4096ULL +#define BOOT_PAGE_PRESENT 0x01ULL +#define BOOT_PAGE_WRITE 0x02ULL +#define BOOT_PAGE_CACHE_WB 0x00ULL +#define BOOT_PAGE_CACHE_WC 0x08ULL +#define BOOT_PAGE_CACHE_UC 0x10ULL // Implements the ascii Str To Unicode Str S operation. static int AsciiStrToUnicodeStrS(const char* Source, uint16_t* Dest, size_t DestMax) { @@ -168,6 +173,35 @@ static void MapRangeSmart(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags) { } } +// Returns whether a firmware descriptor describes memory that is safe to expose through the direct map. +static bool IsDirectMapRamType(EFI_MEMORY_TYPE type) { + switch (type) { + case EfiLoaderCode: + case EfiLoaderData: + case EfiBootServicesCode: + case EfiBootServicesData: + case EfiRuntimeServicesCode: + case EfiRuntimeServicesData: + case EfiConventionalMemory: + case EfiACPIReclaimMemory: + case EfiACPIMemoryNVS: + return true; + default: + return false; + } +} + +// Converts firmware cache attributes into the kernel's canonical PAT policy. +static UINT64 DirectMapCacheFlags(const EFI_MEMORY_DESCRIPTOR* desc) { + if (!desc) + return BOOT_PAGE_CACHE_UC; + if (desc->Attribute & (EFI_MEMORY_UC | EFI_MEMORY_UCE | EFI_MEMORY_WT)) + return BOOT_PAGE_CACHE_UC; + if (desc->Attribute & EFI_MEMORY_WC) + return BOOT_PAGE_CACHE_WC; + return BOOT_PAGE_CACHE_WB; +} + static void MapRangeSkipping(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags, const UINT64* holes, UINTN holeCount) { UINT64 curVirt = virt; @@ -790,8 +824,8 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable for (UINT64 i = 0; i < fbBytes; i += PAGE_SIZE) { UINT64 phys = gop->Mode->FrameBufferBase + i; - Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // P|W|PCD (write-combined MMIO) - Map4K(phys, phys, 0x13); + Map4K(PHYS_MAP_BASE + phys, phys, BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_WC); + Map4K(phys, phys, BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_WC); } } @@ -866,21 +900,16 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable for (UINTN i = 0; i < mapSize / descSz; i++) { EFI_MEMORY_DESCRIPTOR* desc = (EFI_MEMORY_DESCRIPTOR *)((UINT8 *)efiMap + i * descSz); - // Map normal firmware-described regions into the physical window. - // PCI/firmware MMIO is mapped explicitly by the owning driver. - if (desc->Type != EfiUnusableMemory && - desc->Type != EfiMemoryMappedIO && - desc->Type != EfiMemoryMappedIOPortSpace) { + // Only firmware-described RAM receives a direct-map alias. Reserved + // and MMIO descriptors stay unmapped until an owning subsystem maps + // them with an explicit PAT policy. + if (IsDirectMapRamType(desc->Type)) { UINT64 start = desc->PhysicalStart; - UINT64 end = start + (desc->NumberOfPages * PAGE_SIZE); + UINT64 size = desc->NumberOfPages * PAGE_SIZE; + UINT64 flags = BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | DirectMapCacheFlags(desc); - // Add 2 pages (8KB) of padding to the end of the region. - // This catches ACPI tables that cross page boundaries into unmapped - // memory because of buggy UEFI size reporting. - end += 2 * PAGE_SIZE; - - MapRangeSkipping(start, start, end - start, 0x03, idHoles, idHoleCount); - MapRangeSkipping(PHYS_MAP_BASE + start, start, end - start, 0x03, winHoles, winHoleCount); + MapRangeSkipping(start, start, size, flags, idHoles, idHoleCount); + MapRangeSkipping(PHYS_MAP_BASE + start, start, size, flags, winHoles, winHoleCount); } } } @@ -891,8 +920,9 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable // We explicitly map the region from 0xFEC00000 up through 0xFEE00FFF // to guarantee the HAL can access them via the physical window. for (UINT64 phys = 0xFEC00000; phys < 0xFEE01000; phys += PAGE_SIZE) { - Map4K(phys, phys, 0x13); // Identity Map, P|W|PCD - Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // Physical Window Map, P|W|PCD + UINT64 flags = BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_UC; + Map4K(phys, phys, flags); + Map4K(PHYS_MAP_BASE + phys, phys, flags); } Print(L"8\n"); diff --git a/code/build/build.c b/code/build/build.c index baaf443..91ae5f1 100644 --- a/code/build/build.c +++ b/code/build/build.c @@ -21,6 +21,10 @@ #define LWIP_CFLAGS_LITERAL DRIVER_CFLAGS_LITERAL " /Icode\\drivers\\net\\lwip\\include /Icode\\drivers\\net\\lwip\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706" #define LFLAGS_LITERAL "/nologo /NODEFAULTLIB /DEBUG" +// Partition offsets inside disk.vhd +#define VHD_PART1 "disk.vhd@@1M" // ESP / Partition 1 (Offset 1MB) +#define VHD_PART2 "disk.vhd@@65M" // Test/Data / Partition 2 (Offset 65MB) + const char *CommonCflags = COMMON_CFLAGS_LITERAL; const char *KernelCflags = KERNEL_CFLAGS_LITERAL; const char *UserCflags = USER_CFLAGS_LITERAL; @@ -46,6 +50,7 @@ static const Driver_Target PCI_BUS_DRIVER = static const Driver_Target STANDALONE_DRIVERS[] = { DRIVER_TARGET("ps2", "obj\\drivers\\ps2", "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"), DRIVER_TARGET("serial", "obj\\drivers\\serial", "code\\drivers\\input\\serial\\serial.c"), + DRIVER_TARGET_LIBS("hda", "obj\\drivers\\hda", "code\\drivers\\audio\\hda\\hda.c code\\drivers\\audio\\hda\\hda_transport.c code\\drivers\\audio\\hda\\hda_codec.c", "bin\\pci.lib"), DRIVER_TARGET_LIBS("ata", "obj\\drivers\\ata", "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"), DRIVER_TARGET_LIBS("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"), DRIVER_TARGET("fat", "obj\\drivers\\fat", "code\\drivers\\fs\\fat\\fat.c code\\drivers\\fs\\fat\\fat_mount.c"), @@ -64,7 +69,7 @@ void create_layout() { "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", "obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget", "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop", - "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", + "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread", "obj\\all_tests", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke" }; @@ -91,44 +96,190 @@ Nob_Proc run_async(const char *cmd_str) { return proc; } -// Cleans all object output by replacing obj/ as a whole. -// Only obj/ is disposable. fatroot/ and usbfatroot/ are developer-owned vvfat roots. -bool clean_build_object_directories(void) { - const char *cmd = "if exist obj rmdir /s /q obj"; - if (!run_sync(cmd)) return false; - return nob_mkdir_if_not_exists("obj"); +// Deletes stale object files from a target directory before compiling into it. +bool clean_object_directory(const char *obj_dir) { + const char *cmd = nob_temp_sprintf("if exist %s\\*.obj del /q %s\\*.obj", obj_dir, obj_dir); + return run_sync(cmd); } -// Creates the FAT root layout used by QEMU vvfat without deleting existing files. -// The vvfat root is also a convenient developer workspace, so packaging must be non-destructive. -bool create_fatroot_layout(void) { +// Cleans object directories that are linked through wildcards. +bool clean_build_object_directories(void) { const char *dirs[] = { - "fatroot\\EFI", "fatroot\\EFI\\BOOT", - "fatroot\\system", "fatroot\\drivers", "fatroot\\apps", "fatroot\\tests" + "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal", + "obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix", + "obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", + "obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget", + "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", + "obj\\doom", "obj\\video", "obj\\memtop", "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", + "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread", + "obj\\all_tests", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke" }; for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) { - if (!nob_mkdir_if_not_exists(dirs[i])) return false; + if (!clean_object_directory(dirs[i])) return false; + } + + if (!clean_object_directory(PCI_BUS_DRIVER.obj_dir)) return false; + for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) { + if (!clean_object_directory(STANDALONE_DRIVERS[i].obj_dir)) return false; } return true; } -// Creates the USB vvfat root once and leaves its contents alone between builds. +// Creates the USB fat root once and leaves its contents alone between builds. bool create_usbfatroot_if_missing(void) { return nob_mkdir_if_not_exists("usbfatroot"); } +// Utility to copy a local file into a FAT image using mcopy without admin rights. +bool mcopy_file(const char *src, const char *vhd_target, const char *dst_path) { + const char *cmd = nob_temp_sprintf("mcopy -o -i %s %s ::%s", vhd_target, src, dst_path); + return run_sync(cmd); +} + +// Utility to create a directory in a FAT image using mmd. +bool mmd_dir(const char *vhd_target, const char *dir_path) { + const char *cmd = nob_temp_sprintf("mmd -i %s ::%s 2>nul", vhd_target, dir_path); + run_sync(cmd); + return true; +} + +// Writes a 512-byte Connectix VHD Fixed Disk Footer at the end of the file. +static void write_vpc_footer(FILE *f, uint64_t disk_size_bytes) { + uint8_t footer[512] = {0}; + + memcpy(&footer[0], "conectix", 8); + + // Features: 0x00000002 (Big Endian) + footer[11] = 0x02; + + // File Format Version: 0x00010000 + footer[13] = 0x01; + + // Data Offset: 0xFFFFFFFFFFFFFFFF (Fixed disk) + memset(&footer[16], 0xFF, 8); + + // Creator Application: "qemu" + memcpy(&footer[28], "qemu", 4); + + // Creator Version: 0x00020000 + footer[33] = 0x02; + + // Creator Host OS: "Wi2k" + memcpy(&footer[36], "Wi2k", 4); + + // Original & Current Size (uint64_t Big Endian) + for (int i = 0; i < 8; i++) { + uint8_t b = (uint8_t)(disk_size_bytes >> ((7 - i) * 8)); + footer[40 + i] = b; + footer[48 + i] = b; + } + + // Geometry for 512MB: 1024 Cylinders (0x0400), 16 Heads, 63 Sectors/Track + footer[56] = 0x04; + footer[57] = 0x00; + footer[58] = 16; + footer[59] = 63; + + // Disk Type: 2 (Fixed Hard Disk) + footer[63] = 0x02; + + // Unique ID GUID + static const uint8_t guid[16] = { + 0xA1, 0xB2, 0xC3, 0xD4, 0xE5, 0xF6, 0x78, 0x90, + 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0 + }; + memcpy(&footer[68], guid, 16); + + // Checksum: 1's complement sum of all bytes in footer with checksum cleared + uint32_t sum = 0; + for (int i = 0; i < 512; i++) { + sum += footer[i]; + } + uint32_t checksum = ~sum; + + footer[64] = (uint8_t)(checksum >> 24); + footer[65] = (uint8_t)(checksum >> 16); + footer[66] = (uint8_t)(checksum >> 8); + footer[67] = (uint8_t)(checksum & 0xFF); + + fseek(f, (long)disk_size_bytes, SEEK_SET); + fwrite(footer, 1, 512, f); +} + +// Creates a 512 MB partitioned disk.vhd image with a valid VPC footer. +bool create_disk_vhd_if_missing(void) { + if (nob_file_exists("disk.vhd")) return true; + + nob_log(NOB_INFO, "disk.vhd missing, creating new 512 MB fixed VHD image..."); + + const uint64_t disk_size = (uint64_t)512 * 1024 * 1024; // 512 MB Payload + FILE *f = fopen("disk.vhd", "wb"); + if (!f) { + nob_log(NOB_ERROR, "Failed to create disk.vhd file!"); + return false; + } + + // Expand file to 512MB payload + if (fseek(f, (long)(disk_size - 1), SEEK_SET) != 0) { + fclose(f); + return false; + } + fputc(0, f); + + // Build standard MBR partition table (Sector 0) + uint8_t mbr[512] = {0}; + mbr[510] = 0x55; + mbr[511] = 0xAA; + + // Partition 1 Entry: 64MB FAT32 @ LBA 2048 (1MB offset) + uint8_t *p1 = &mbr[0x01BE]; + p1[0] = 0x80; // Active / Bootable + p1[1] = 0x00; p1[2] = 0x02; p1[3] = 0x00; // CHS Start + p1[4] = 0x0C; // FAT32 LBA + p1[5] = 0xFE; p1[6] = 0xFF; p1[7] = 0xFF; // CHS End (LBA boundary) + uint32_t p1_lba = 2048; + uint32_t p1_sectors = 131072; // 64 MB + memcpy(&p1[8], &p1_lba, 4); + memcpy(&p1[12], &p1_sectors, 4); + + // Partition 2 Entry: 256MB FAT32 @ LBA 133120 (65MB offset) + uint8_t *p2 = &mbr[0x01CE]; + p2[0] = 0x00; // Inactive + p2[1] = 0x00; p2[2] = 0x01; p2[3] = 0x08; // CHS Start + p2[4] = 0x0C; // FAT32 LBA + p2[5] = 0xFE; p2[6] = 0xFF; p2[7] = 0xFF; // CHS End (LBA boundary) + uint32_t p2_lba = 133120; + uint32_t p2_sectors = 524288; // 256 MB + memcpy(&p2[8], &p2_lba, 4); + memcpy(&p2[12], &p2_sectors, 4); + + // Write Sector 0 + fseek(f, 0, SEEK_SET); + fwrite(mbr, 1, 512, f); + + // Append 512-byte Connectix VPC Footer + write_vpc_footer(f, disk_size); + + fclose(f); + + nob_log(NOB_INFO, "Formatting disk.vhd partitions with mformat..."); + if (!run_sync("mformat -i " VHD_PART1 " -F")) return false; + if (!run_sync("mformat -i " VHD_PART2 " -F")) return false; + + return true; +} + // Runs the requested operation. int main(int argc, char **argv) { - // The build tool rebuilds itself before doing any real work so the graph below matches the source. NOB_GO_REBUILD_URSELF(argc, argv); - if (!clean_build_object_directories()) return 1; - create_layout(); + if (!create_disk_vhd_if_missing()) return 1; if (!create_usbfatroot_if_missing()) return 1; + create_layout(); + if (!clean_build_object_directories()) return 1; - // Core images build sequentially because later images link against earlier import libraries. nob_log(NOB_INFO, "--- Building Core Subsystems ---"); const char *core_tasks[] = { @@ -165,7 +316,6 @@ int main(int argc, char **argv) { if (!run_sync(core_tasks[i])) return 1; } - // PCI builds before PCI function drivers because it exports the bus-driver import library. nob_log(NOB_INFO, "--- Building Bus Drivers ---"); const char *pci_cmd = nob_temp_sprintf( "cl %s /MP /c /Fo%s\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /IMPLIB:bin\\%s.lib /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\printf.obj obj\\libc.obj %s\\*.obj", @@ -174,12 +324,9 @@ int main(int argc, char **argv) { ); if (!run_sync(pci_cmd)) return 1; - // 4. Parallel Phase (Drivers and Applications). nob_log(NOB_INFO, "--- Compiling Drivers and Apps in Parallel ---"); Nob_Procs procs = {0}; - // Queue standalone drivers. Each driver is now a target with one or more - // translation units, but still links to one .sys image with DriverEntry. for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) { const Driver_Target *driver = &STANDALONE_DRIVERS[i]; const char *cmd = nob_temp_sprintf( @@ -190,7 +337,6 @@ int main(int argc, char **argv) { nob_da_append(&procs, run_async(cmd)); } - // Parse lwIP Sources and Queue it. Nob_String_Builder sb = {0}, lwip_files = {0}; if (nob_read_entire_file("code\\drivers\\net\\lwip\\lwip_sources.txt", &sb)) { Nob_String_View sv = nob_sv_from_parts(sb.items, sb.count); @@ -206,9 +352,7 @@ int main(int argc, char **argv) { nob_da_append(&procs, run_async(nob_temp_sprintf("cl %s /MP /c /Foobj\\lwip\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\lwip.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj obj\\lwip\\*.obj", LwipCflags, lwip_files.items))); } - // Queue Subsystem Applications. const char *apps[][4] = { - // Folder, Libs, Name, Optional Extra Link Args. {"system\\init", NATIVE_INIT_LIBS_LITERAL, "init", "/ENTRY:_start"}, {"tests\\all_tests", NATIVE_EXE_LIBS_LITERAL, "all_tests", NATIVE_EXE_ENTRY_LITERAL}, {"apps\\shell_win32", WIN32_EXE_LIBS_LITERAL, "shell_win32", NATIVE_EXE_ENTRY_LITERAL}, @@ -239,6 +383,12 @@ int main(int argc, char **argv) { {"tests\\win32_seh_smoke", WIN32_EXE_LIBS_LITERAL, "win32_seh_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\syscall_contract_smoke", NATIVE_EXE_LIBS_LITERAL, "syscall_contract_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\would_block_smoke", NATIVE_EXE_LIBS_LITERAL, "would_block_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\audio_stream_smoke", NATIVE_EXE_LIBS_LITERAL, "audio_stream_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\winmm_waveout_smoke", WIN32_EXE_LIBS_LITERAL, "winmm_waveout_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_condition_variable_smoke", WIN32_EXE_LIBS_LITERAL, "win32_condition_variable_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_mutex_smoke", WIN32_EXE_LIBS_LITERAL, "win32_mutex_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_thread_suspend_smoke", WIN32_EXE_LIBS_LITERAL, "win32_thread_suspend_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_module_lookup_smoke", WIN32_EXE_LIBS_LITERAL, "win32_module_lookup_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\cwd_peb_smoke", NATIVE_EXE_LIBS_LITERAL, "cwd_peb_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\mmap_smoke", NATIVE_EXE_LIBS_LITERAL, "mmap_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\pcache_smoke", NATIVE_EXE_LIBS_LITERAL, "pcache_smoke", NATIVE_EXE_ENTRY_LITERAL}, @@ -270,98 +420,118 @@ int main(int argc, char **argv) { nob_da_append(&procs, run_async(cmd)); } - // Wait for everything to link simultaneously. if (!nob_procs_wait(procs)) { nob_log(NOB_ERROR, "One or more parallel build jobs failed!"); return 1; } - // 4. Packing vvfat root. - nob_log(NOB_INFO, "--- Assembling fatroot ---"); - if (!create_fatroot_layout()) return 1; + // 4. Formatting and Packing VHD via mtools (No Admin, Fast). + nob_log(NOB_INFO, "--- Formatting & Packing disk.vhd with mtools ---"); + + run_sync("mformat -i " VHD_PART1 " -F"); + run_sync("mformat -i " VHD_PART2 " -F"); - const char *copies[][2] = { - {"bin\\bootx64.efi", "fatroot\\EFI\\BOOT\\bootx64.efi"}, - {"bin\\kernel.exe", "fatroot\\system\\colonel.exe"}, - {"bin\\hal.dll", "fatroot\\system\\hal.dll"}, - {"bin\\syscall.dll", "fatroot\\system\\syscall.dll"}, - {"bin\\win32.dll", "fatroot\\system\\win32.dll"}, - {"bin\\posix.dll", "fatroot\\system\\posix.dll"}, - {"code\\settings.ini", "fatroot\\system\\settings.ini"}, - {"bin\\init.exe", "fatroot\\system\\init.exe"}, + mmd_dir(VHD_PART1, "apps"); + mmd_dir(VHD_PART1, "system"); + mmd_dir(VHD_PART1, "drivers"); + mmd_dir(VHD_PART1, "EFI"); + mmd_dir(VHD_PART1, "EFI/BOOT"); + mmd_dir(VHD_PART2, "tests"); - {"bin\\pci.sys", "fatroot\\drivers\\pci.sys"}, - {"bin\\ps2.sys", "fatroot\\drivers\\ps2.sys"}, - {"bin\\serial.sys", "fatroot\\drivers\\serial.sys"}, - {"bin\\ata.sys", "fatroot\\drivers\\ata.sys"}, - {"bin\\ahci.sys", "fatroot\\drivers\\ahci.sys"}, - {"bin\\fat.sys", "fatroot\\drivers\\fat.sys"}, - {"bin\\ntfs.sys", "fatroot\\drivers\\ntfs.sys"}, - {"bin\\vmwaresvga.sys", "fatroot\\drivers\\vmwaresvga.sys"}, - {"bin\\lwip.sys", "fatroot\\drivers\\lwip.sys"}, - {"bin\\usb.sys", "fatroot\\drivers\\usb.sys"}, - {"bin\\rtl816x.sys", "fatroot\\drivers\\rtl816x.sys"}, - {"bin\\rtl8139.sys", "fatroot\\drivers\\rtl8139.sys"}, - {"code\\drivers\\drivers.ini", "fatroot\\drivers\\drivers.ini"}, - {"code\\boot.ini", "fatroot\\boot.ini"}, + const char *copies[][3] = { + {"bin\\bootx64.efi", VHD_PART1, "EFI/BOOT/bootx64.efi"}, + {"bin\\kernel.exe", VHD_PART1, "system/colonel.exe"}, + {"bin\\hal.dll", VHD_PART1, "system/hal.dll"}, + {"bin\\syscall.dll", VHD_PART1, "system/syscall.dll"}, + {"bin\\win32.dll", VHD_PART1, "system/win32.dll"}, + {"bin\\posix.dll", VHD_PART1, "system/posix.dll"}, + {"code\\settings.ini", VHD_PART1, "system/settings.ini"}, + {"bin\\init.exe", VHD_PART1, "system/init.exe"}, + + {"bin\\pci.sys", VHD_PART1, "drivers/pci.sys"}, + {"bin\\ps2.sys", VHD_PART1, "drivers/ps2.sys"}, + {"bin\\serial.sys", VHD_PART1, "drivers/serial.sys"}, + {"bin\\hda.sys", VHD_PART1, "drivers/hda.sys"}, + {"bin\\ata.sys", VHD_PART1, "drivers/ata.sys"}, + {"bin\\ahci.sys", VHD_PART1, "drivers/ahci.sys"}, + {"bin\\fat.sys", VHD_PART1, "drivers/fat.sys"}, + {"bin\\ntfs.sys", VHD_PART1, "drivers/ntfs.sys"}, + {"bin\\vmwaresvga.sys", VHD_PART1, "drivers/vmwaresvga.sys"}, + {"bin\\lwip.sys", VHD_PART1, "drivers/lwip.sys"}, + {"bin\\usb.sys", VHD_PART1, "drivers/usb.sys"}, + {"bin\\rtl816x.sys", VHD_PART1, "drivers/rtl816x.sys"}, + {"bin\\rtl8139.sys", VHD_PART1, "drivers/rtl8139.sys"}, + {"code\\drivers\\drivers.ini", VHD_PART1, "drivers/drivers.ini"}, + {"code\\boot.ini", VHD_PART1, "boot.ini"}, - {"bin\\shell_win32.exe", "fatroot\\apps\\shell_win32.exe"}, - {"bin\\shell_posix.exe", "fatroot\\apps\\shell_posix.exe"}, - {"bin\\shell_native.exe", "fatroot\\apps\\shell_native.exe"}, - {"bin\\edit.exe", "fatroot\\apps\\edit.exe"}, - {"bin\\httpget.exe", "fatroot\\apps\\httpget.exe"}, - {"bin\\tcpserve.exe", "fatroot\\apps\\tcpserve.exe"}, - {"bin\\udp.exe", "fatroot\\apps\\udp.exe"}, - {"bin\\resolve.exe", "fatroot\\apps\\resolve.exe"}, - {"bin\\ping.exe", "fatroot\\apps\\ping.exe"}, - {"bin\\doom.exe", "fatroot\\apps\\doom.exe"}, - {"bin\\video.exe", "fatroot\\apps\\video.exe"}, - {"bin\\memtop.exe", "fatroot\\apps\\memtop.exe"}, + {"bin\\shell_win32.exe", VHD_PART1, "apps/shell_win32.exe"}, + {"bin\\shell_posix.exe", VHD_PART1, "apps/shell_posix.exe"}, + {"bin\\shell_native.exe", VHD_PART1, "apps/shell_native.exe"}, + {"bin\\edit.exe", VHD_PART1, "apps/edit.exe"}, + {"bin\\httpget.exe", VHD_PART1, "apps/httpget.exe"}, + {"bin\\tcpserve.exe", VHD_PART1, "apps/tcpserve.exe"}, + {"bin\\udp.exe", VHD_PART1, "apps/udp.exe"}, + {"bin\\resolve.exe", VHD_PART1, "apps/resolve.exe"}, + {"bin\\ping.exe", VHD_PART1, "apps/ping.exe"}, + {"bin\\doom.exe", VHD_PART1, "apps/doom.exe"}, + {"bin\\video.exe", VHD_PART1, "apps/video.exe"}, + {"bin\\memtop.exe", VHD_PART1, "apps/memtop.exe"}, - {"bin\\all_tests.exe", "fatroot\\tests\\all_tests.exe"}, - {"bin\\async_apc.exe", "fatroot\\tests\\async_apc.exe"}, - {"bin\\async_cancel.exe", "fatroot\\tests\\async_cancel.exe"}, - {"bin\\apc_return_smoke.exe", "fatroot\\tests\\apc_return_smoke.exe"}, - {"bin\\file_stat_smoke.exe", "fatroot\\tests\\file_stat_smoke.exe"}, - {"bin\\handle_dup_smoke.exe", "fatroot\\tests\\handle_dup_smoke.exe"}, - {"bin\\named_queue_lifetime_smoke.exe", "fatroot\\tests\\named_queue_lifetime_smoke.exe"}, - {"bin\\input_api_smoke.exe", "fatroot\\tests\\input_api_smoke.exe"}, - {"bin\\input_model_smoke.exe", "fatroot\\tests\\input_model_smoke.exe"}, - {"bin\\timer_smoke.exe", "fatroot\\tests\\timer_smoke.exe"}, - {"bin\\stats_abi_smoke.exe", "fatroot\\tests\\stats_abi_smoke.exe"}, - {"bin\\realpath_smoke.exe", "fatroot\\tests\\realpath_smoke.exe"}, - {"bin\\file_lock_smoke.exe", "fatroot\\tests\\file_lock_smoke.exe"}, - {"bin\\process_protection_smoke.exe", "fatroot\\tests\\process_protection_smoke.exe"}, - {"bin\\socket_addr_smoke.exe", "fatroot\\tests\\socket_addr_smoke.exe"}, - {"bin\\net_irp_smoke.exe", "fatroot\\tests\\net_irp_smoke.exe"}, - {"bin\\module_load_smoke.exe", "fatroot\\tests\\module_load_smoke.exe"}, - {"bin\\posix_module_smoke.exe", "fatroot\\tests\\posix_module_smoke.exe"}, - {"bin\\exception_dispatcher_smoke.exe", "fatroot\\tests\\exception_dispatcher_smoke.exe"}, - {"bin\\native_exception_runtime_smoke.exe", "fatroot\\tests\\native_exception_runtime_smoke.exe"}, - {"bin\\native_raise_exception_smoke.exe", "fatroot\\tests\\native_raise_exception_smoke.exe"}, - {"bin\\exception_smoke.exe", "fatroot\\tests\\exception_smoke.exe"}, - {"bin\\win32_exception_smoke.exe", "fatroot\\tests\\win32_exception_smoke.exe"}, - {"bin\\win32_seh_smoke.exe", "fatroot\\tests\\win32_seh_smoke.exe"}, - {"bin\\syscall_contract_smoke.exe", "fatroot\\tests\\syscall_contract_smoke.exe"}, - {"bin\\would_block_smoke.exe", "fatroot\\tests\\would_block_smoke.exe"}, - {"bin\\cwd_peb_smoke.exe", "fatroot\\tests\\cwd_peb_smoke.exe"}, - {"bin\\mmap_smoke.exe", "fatroot\\tests\\mmap_smoke.exe"}, - {"bin\\pcache_smoke.exe", "fatroot\\tests\\pcache_smoke.exe"}, - {"bin\\pcache_pressure.exe", "fatroot\\tests\\pcache_pressure.exe"}, - {"bin\\dirty_writeback.exe", "fatroot\\tests\\dirty_writeback.exe"}, - {"bin\\shared_mmap.exe", "fatroot\\tests\\shared_mmap.exe"}, - {"bin\\pcache_invalidate.exe", "fatroot\\tests\\pcache_invalidate.exe"}, - {"bin\\pcache_mmap_read_reuse.exe", "fatroot\\tests\\pcache_mmap_read_reuse.exe"}, - {"bin\\pcache_multithread.exe", "fatroot\\tests\\pcache_multithread.exe"}, - {"bin\\surface_smoke.exe", "fatroot\\tests\\surface_smoke.exe"}, - {"bin\\surface_invalid_rect.exe", "fatroot\\tests\\surface_invalid_rect.exe"}, - {"bin\\surface_multithread_present.exe", "fatroot\\tests\\surface_multithread_present.exe"} + // Partition 2 Apps + {"bin\\all_tests.exe", VHD_PART2, "tests/all_tests.exe"}, + {"bin\\async_apc.exe", VHD_PART2, "tests/async_apc.exe"}, + {"bin\\async_cancel.exe", VHD_PART2, "tests/async_cancel.exe"}, + {"bin\\apc_return_smoke.exe", VHD_PART2, "tests/apc_return_smoke.exe"}, + {"bin\\file_stat_smoke.exe", VHD_PART2, "tests/file_stat_smoke.exe"}, + {"bin\\handle_dup_smoke.exe", VHD_PART2, "tests/handle_dup_smoke.exe"}, + {"bin\\named_queue_lifetime_smoke.exe", VHD_PART2, "tests/named_queue_lifetime_smoke.exe"}, + {"bin\\input_api_smoke.exe", VHD_PART2, "tests/input_api_smoke.exe"}, + {"bin\\input_model_smoke.exe", VHD_PART2, "tests/input_model_smoke.exe"}, + {"bin\\timer_smoke.exe", VHD_PART2, "tests/timer_smoke.exe"}, + {"bin\\stats_abi_smoke.exe", VHD_PART2, "tests/stats_abi_smoke.exe"}, + {"bin\\realpath_smoke.exe", VHD_PART2, "tests/realpath_smoke.exe"}, + {"bin\\file_lock_smoke.exe", VHD_PART2, "tests/file_lock_smoke.exe"}, + {"bin\\process_protection_smoke.exe", VHD_PART2, "tests/process_protection_smoke.exe"}, + {"bin\\socket_addr_smoke.exe", VHD_PART2, "tests/socket_addr_smoke.exe"}, + {"bin\\net_irp_smoke.exe", VHD_PART2, "tests/net_irp_smoke.exe"}, + {"bin\\module_load_smoke.exe", VHD_PART2, "tests/module_load_smoke.exe"}, + {"bin\\posix_module_smoke.exe", VHD_PART2, "tests/posix_module_smoke.exe"}, + {"bin\\exception_dispatcher_smoke.exe", VHD_PART2, "tests/exception_dispatcher_smoke.exe"}, + {"bin\\native_exception_runtime_smoke.exe", VHD_PART2, "tests/native_exception_runtime_smoke.exe"}, + {"bin\\native_raise_exception_smoke.exe", VHD_PART2, "tests/native_raise_exception_smoke.exe"}, + {"bin\\exception_smoke.exe", VHD_PART2, "tests/exception_smoke.exe"}, + {"bin\\win32_exception_smoke.exe", VHD_PART2, "tests/win32_exception_smoke.exe"}, + {"bin\\win32_seh_smoke.exe", VHD_PART2, "tests/win32_seh_smoke.exe"}, + {"bin\\syscall_contract_smoke.exe", VHD_PART2, "tests/syscall_contract_smoke.exe"}, + {"bin\\would_block_smoke.exe", VHD_PART2, "tests/would_block_smoke.exe"}, + {"bin\\audio_stream_smoke.exe", VHD_PART2, "tests/audio_stream_smoke.exe"}, + {"bin\\winmm_waveout_smoke.exe", VHD_PART2, "tests/winmm_waveout_smoke.exe"}, + {"bin\\win32_condition_variable_smoke.exe", VHD_PART2, "tests/win32_condition_variable_smoke.exe"}, + {"bin\\win32_mutex_smoke.exe", VHD_PART2, "tests/win32_mutex_smoke.exe"}, + {"bin\\win32_thread_suspend_smoke.exe", VHD_PART2, "tests/win32_thread_suspend_smoke.exe"}, + {"bin\\win32_module_lookup_smoke.exe", VHD_PART2, "tests/win32_module_lookup_smoke.exe"}, + {"bin\\cwd_peb_smoke.exe", VHD_PART2, "tests/cwd_peb_smoke.exe"}, + {"bin\\mmap_smoke.exe", VHD_PART2, "tests/mmap_smoke.exe"}, + {"bin\\pcache_smoke.exe", VHD_PART2, "tests/pcache_smoke.exe"}, + {"bin\\pcache_pressure.exe", VHD_PART2, "tests/pcache_pressure.exe"}, + {"bin\\dirty_writeback.exe", VHD_PART2, "tests/dirty_writeback.exe"}, + {"bin\\shared_mmap.exe", VHD_PART2, "tests/shared_mmap.exe"}, + {"bin\\pcache_invalidate.exe", VHD_PART2, "tests/pcache_invalidate.exe"}, + {"bin\\pcache_mmap_read_reuse.exe", VHD_PART2, "tests/pcache_mmap_read_reuse.exe"}, + {"bin\\pcache_multithread.exe", VHD_PART2, "tests/pcache_multithread.exe"}, + {"bin\\surface_smoke.exe", VHD_PART2, "tests/surface_smoke.exe"}, + {"bin\\surface_invalid_rect.exe", VHD_PART2, "tests/surface_invalid_rect.exe"}, + {"bin\\surface_multithread_present.exe", VHD_PART2, "tests/surface_multithread_present.exe"} }; for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) { - if (!nob_copy_file(copies[i][0], copies[i][1])) return 1; + mcopy_file(copies[i][0], copies[i][1], copies[i][2]); } + if (nob_file_exists("code\\user\\apps\\video\\bink2w64.dll")) mcopy_file("code\\user\\apps\\video\\bink2w64.dll", VHD_PART1, "apps/bink2w64.dll"); + if (nob_file_exists("code\\user\\apps\\video\\video.bk2")) mcopy_file("code\\user\\apps\\video\\video.bk2", VHD_PART1, "apps/video.bk2"); + if (nob_file_exists("code\\user\\apps\\doom\\doom1.wad")) mcopy_file("code\\user\\apps\\doom\\doom1.wad", VHD_PART1, "apps/doom1.wad"); + nob_log(NOB_INFO, "Build complete!"); return 0; } diff --git a/code/common/libc.c b/code/common/libc.c index 89dc672..05e1697 100644 --- a/code/common/libc.c +++ b/code/common/libc.c @@ -20,6 +20,31 @@ char* strncpy(char* dest, const char* src, size_t n) { return dest; } +char *strstr(const char *haystack, const char *needle) { + // An empty needle matches the beginning of haystack + if (!*needle) { + return (char *)haystack; + } + + for (; *haystack != '\0'; haystack++) { + const char *h = haystack; + const char *n = needle; + + // Match as many characters as possible + while (*h != '\0' && *n != '\0' && *h == *n) { + h++; + n++; + } + + // If we reached the end of needle, we found a match + if (*n == '\0') { + return (char *)haystack; + } + } + + return NULL; +} + // Implements the strcat operation. char* strcat(char* dest, const char* src) { char* ptr = dest; diff --git a/code/drivers/audio/hda/hda.c b/code/drivers/audio/hda/hda.c new file mode 100644 index 0000000..809e758 --- /dev/null +++ b/code/drivers/audio/hda/hda.c @@ -0,0 +1,403 @@ +// Intel High Definition Audio PCI attachment and controller lifetime management. + +#include "hda.h" + +#include + +const NTL_DDK* gHdaDdk; +static DRIVER_OBJECT gHdaDriver; +static HDA_CONTROLLER* gHdaControllers[HDA_MAX_CONTROLLERS]; +static uint32_t gHdaControllerCount; +static SPINLOCK gHdaRegistryLock = SPINLOCK_INIT; + +static const IO_DEVICE_ID gHdaIds[] = { + { IoBusPci, { "PCI\\VEN_8086&DEV_2668", sizeof("PCI\\VEN_8086&DEV_2668") - 1u } }, + { IoBusPci, { "PCI\\VEN_15AD&DEV_1977", sizeof("PCI\\VEN_15AD&DEV_1977") - 1u } }, + { IoBusPci, { "PCI\\CC_040300", sizeof("PCI\\CC_040300") - 1u } }, +}; + +static const IO_DRIVER_ID_TABLE gHdaIdTable = { + gHdaIds, + sizeof(gHdaIds) / sizeof(gHdaIds[0]), +}; + +// Registers a controller in the shared interrupt scan set. +static KSTATUS HdaRegisterController(HDA_CONTROLLER* controller) { + if (!controller) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock); + for (uint32_t index = 0; index < gHdaControllerCount; index++) { + if (gHdaControllers[index] == controller) { + KeReleaseSpinlock(&gHdaRegistryLock, flags); + return NTLSTATUS_SUCCESS; + } + } + if (gHdaControllerCount == HDA_MAX_CONTROLLERS) { + KeReleaseSpinlock(&gHdaRegistryLock, flags); + return NTLSTATUS_NO_MEMORY; + } + gHdaControllers[gHdaControllerCount++] = controller; + KeReleaseSpinlock(&gHdaRegistryLock, flags); + return NTLSTATUS_SUCCESS; +} + +// Removes a controller from the shared interrupt scan set. +static void HdaUnregisterController(HDA_CONTROLLER* controller) { + if (!controller) + return; + + uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock); + for (uint32_t index = 0; index < gHdaControllerCount; index++) { + if (gHdaControllers[index] != controller) + continue; + gHdaControllers[index] = gHdaControllers[gHdaControllerCount - 1u]; + gHdaControllers[gHdaControllerCount - 1u] = (HDA_CONTROLLER*)0; + gHdaControllerCount--; + break; + } + KeReleaseSpinlock(&gHdaRegistryLock, flags); +} + +// Dispatches a hardware interrupt to every live HDA controller. +static void HdaIsr(void) { + DEVICE_OBJECT* devices[HDA_MAX_CONTROLLERS]; + uint32_t count = 0; + + uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock); + count = gHdaControllerCount; + if (count > HDA_MAX_CONTROLLERS) + count = HDA_MAX_CONTROLLERS; + for (uint32_t index = 0; index < count; index++) { + HDA_CONTROLLER* controller = gHdaControllers[index]; + devices[index] = controller ? controller->Fdo : (DEVICE_OBJECT*)0; + if (devices[index]) + IoReferenceDeviceObject(devices[index]); + } + KeReleaseSpinlock(&gHdaRegistryLock, flags); + + for (uint32_t index = 0; index < count; index++) { + DEVICE_OBJECT* device = devices[index]; + if (!device) + continue; + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension; + if (controller && controller->Magic == HDA_CONTROLLER_MAGIC) + HdaHandleInterrupt(controller); + IoDereferenceDeviceObject(device); + } +} + +// Registers the global ISR once for each kernel IRQ slot used by HDA. +static KSTATUS HdaRegisterIrq(uint8_t irq) { + if (irq >= 24u) + return NTLSTATUS_NOT_SUPPORTED; + KeIrqRegister(irq, HdaIsr); + return NTLSTATUS_SUCCESS; +} + +// Maps BAR0 and validates the stream-descriptor aperture described by GCAP. +static KSTATUS HdaMapRegisters(HDA_CONTROLLER* controller) { + PCI_BAR bar; + KSTATUS status = PciGetBar(controller->Pdo, 0, &bar); + if (!KSUCCESS(status)) + return status; + if (bar.Type != PciBarMemory32 && bar.Type != PciBarMemory64) + return NTLSTATUS_NOT_SUPPORTED; + + status = PciMapBar(controller->Pdo, + 0, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, + (void**)&controller->Registers, + &controller->RegisterBytes); + if (!KSUCCESS(status)) + return status; + if (controller->RegisterBytes < HDA_REG_STREAM_BASE) + return NTLSTATUS_BUFFER_TOO_SMALL; + + controller->Gcap = HdaRead16(controller, HDA_REG_GCAP); + controller->InputStreamCount = + (uint8_t)((controller->Gcap & HDA_GCAP_ISS_MASK) >> HDA_GCAP_ISS_SHIFT); + controller->OutputStreamCount = + (uint8_t)((controller->Gcap & HDA_GCAP_OSS_MASK) >> HDA_GCAP_OSS_SHIFT); + controller->BidirectionalStreamCount = + (uint8_t)((controller->Gcap & HDA_GCAP_BSS_MASK) >> HDA_GCAP_BSS_SHIFT); + controller->TotalStreamCount = + (uint8_t)(controller->InputStreamCount + + controller->OutputStreamCount + + controller->BidirectionalStreamCount); + if ((controller->OutputStreamCount == 0 && + controller->BidirectionalStreamCount == 0) || + controller->TotalStreamCount == 0 || + controller->TotalStreamCount > 30u) + return NTLSTATUS_NOT_SUPPORTED; + + uint64_t requiredBytes = HDA_REG_STREAM_BASE + + (uint64_t)controller->TotalStreamCount * + HDA_REG_STREAM_STRIDE; + if (controller->RegisterBytes < requiredBytes) + return NTLSTATUS_BUFFER_TOO_SMALL; + + // Dedicated output descriptors precede bidirectional descriptors. Use the + // first bidirectional descriptor when no output-only descriptor exists. + uint8_t outputDescriptorBase = controller->InputStreamCount; + uint8_t bidirectionalDescriptorBase = + (uint8_t)(controller->InputStreamCount + controller->OutputStreamCount); + controller->Output.DescriptorIndex = controller->OutputStreamCount + ? outputDescriptorBase + : bidirectionalDescriptorBase; + controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG; + return NTLSTATUS_SUCCESS; +} + +// Configures MSI first and enables INTx only after MSI setup reports failure. +static KSTATUS HdaConfigureInterrupt(HDA_CONTROLLER* controller) { + KSTATUS status = PciEnableMsiAuto(controller->Pdo, &controller->Irq); + if (KSUCCESS(status)) { + controller->UsingMsi = true; + return NTLSTATUS_SUCCESS; + } + + gHdaDdk->Debug->Print( + "HDA: MSI setup failed status=0x%x; attempting explicit INTx fallback\n", + status); + controller->UsingMsi = false; + status = PciGetInterrupt(controller->Pdo, &controller->Irq); + if (!KSUCCESS(status)) + return status; + if (controller->Irq == 0xFFu || controller->Irq >= 24u) + return NTLSTATUS_NOT_SUPPORTED; + + status = PciEnableDevice(controller->Pdo, + PCI_ENABLE_MEMORY | + PCI_ENABLE_BUSMASTER | + PCI_ENABLE_INTX); + return status; +} + +// Waits until queued or active interrupt work no longer references the controller. +static KSTATUS HdaWaitForControllerInterruptWorkIdle(HDA_CONTROLLER* controller) { + if (!controller) + return NTLSTATUS_INVALID_PARAMETER; + for (uint32_t elapsed = 0; elapsed < 100000u; elapsed += 10u) { + if (controller->IsrActive == 0 && + controller->Dpc.Inserted == 0 && controller->DpcActive == 0) + return NTLSTATUS_SUCCESS; + gHdaDdk->Timer->Stall(10); + } + return NTLSTATUS_TIMEOUT; +} + +// Starts one PCI HDA controller and leaves endpoint registration to codec discovery. +static KSTATUS HdaStartDevice(HDA_CONTROLLER* controller) { + if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || + !controller->Pdo || !controller->Fdo) + return NTLSTATUS_INVALID_PARAMETER; + if (controller->Started) + return NTLSTATUS_SUCCESS; + + KSTATUS status = PciGetDeviceInfo(controller->Pdo, &controller->PciInfo); + if (!KSUCCESS(status)) + return status; + if (controller->PciInfo.ClassCode != 0x04u || + controller->PciInfo.Subclass != 0x03u || + controller->PciInfo.ProgIf != 0x00u) + return NTLSTATUS_NOT_SUPPORTED; + + status = PciEnableDevice(controller->Pdo, + PCI_ENABLE_MEMORY | + PCI_ENABLE_BUSMASTER | + PCI_DISABLE_INTX); + if (!KSUCCESS(status)) + return status; + + status = HdaMapRegisters(controller); + if (!KSUCCESS(status)) + return status; + + status = HdaConfigureInterrupt(controller); + if (!KSUCCESS(status)) + return status; + + status = HdaAllocateDma(controller); + if (!KSUCCESS(status)) + return status; + KeInitializeDpc(&controller->Dpc, HdaDpcRoutine, controller); + + status = HdaStartRecoveryWorker(controller); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + status = HdaInitializeController(controller); + if (!KSUCCESS(status)) { + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + + status = HdaRegisterController(controller); + if (!KSUCCESS(status)) { + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + status = HdaRegisterIrq(controller->Irq); + if (!KSUCCESS(status)) { + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + + controller->Started = true; + controller->Fdo->PnpState = DevicePnpStarted; + + status = HdaCodecInitialize(controller); + if (!KSUCCESS(status) && status != NTLSTATUS_NOT_SUPPORTED && + status != NTLSTATUS_NOT_FOUND) { + controller->Started = false; + controller->Fdo->PnpState = DevicePnpNotStarted; + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + if (!KSUCCESS(status)) { + gHdaDdk->Debug->Print( + "HDA: no analog endpoint registered status=0x%x\n", + status); + } + + HdaArmInterrupts(controller); + HdaHandleInterrupt(controller); + + return NTLSTATUS_SUCCESS; +} + +// Handles PnP lifetime transitions for one HDA function device object. +static KSTATUS HdaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp || !device->DeviceExtension) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension; + if (controller->Magic != HDA_CONTROLLER_MAGIC) + return NTLSTATUS_INVALID_PARAMETER; + + IO_STACK_LOCATION* stack = IoGetCurrentIrpStackLocation(irp); + if (!stack || stack->MajorFunction != IRP_MJ_PNP) + return NTLSTATUS_INVALID_PARAMETER; + + switch ((IO_PNP_MINOR_FUNCTION)stack->MinorFunction) { + case IRP_MN_START_DEVICE: + return HdaStartDevice(controller); + + case IRP_MN_QUERY_REMOVE_DEVICE: + case IRP_MN_CANCEL_REMOVE_DEVICE: + return NTLSTATUS_SUCCESS; + + case IRP_MN_SURPRISE_REMOVAL: + controller->Started = false; + controller->SurpriseRemoved = true; + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + KSTATUS surpriseDpcStatus = + HdaWaitForControllerInterruptWorkIdle(controller); + if (!KSUCCESS(surpriseDpcStatus)) { + KePanic(KE_PANIC_CODE_DRIVER, + "HDA: interrupt work did not quiesce during surprise removal status=0x%x\n", + surpriseDpcStatus); + } + HdaCodecStop(controller, IoAudioSurpriseRemoval); + HdaCodecDestroy(controller); + device->PnpState = DevicePnpSurpriseRemoved; + return NTLSTATUS_SUCCESS; + + case IRP_MN_REMOVE_DEVICE: + if (!controller->SurpriseRemoved && controller->Registers) + HdaWrite32(controller, HDA_REG_INTCTL, 0); + controller->Started = false; + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + KSTATUS dpcStatus = HdaWaitForControllerInterruptWorkIdle(controller); + if (!KSUCCESS(dpcStatus)) { + KePanic(KE_PANIC_CODE_DRIVER, + "HDA: interrupt work did not quiesce during removal status=0x%x\n", + dpcStatus); + } + HdaCodecStop(controller, + controller->SurpriseRemoved + ? IoAudioSurpriseRemoval + : IoAudioOrderlyRemoval); + HdaCodecDestroy(controller); + if (!controller->SurpriseRemoved) + HdaQuiesceController(controller); + controller->Removed = true; + HdaReleaseDma(controller); + if (!controller->SurpriseRemoved) + (void)PciEnableDevice(controller->Pdo, PCI_DISABLE_INTX); + device->PnpState = DevicePnpDeleted; + IoDetachDevice(device); + IoDeleteDevice(device); + return NTLSTATUS_SUCCESS; + + default: + return IoForwardIrpToLower(device, irp); + } +} + +// Attaches one HDA function device object to a matching PCI PDO. +static KSTATUS HdaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) { + if (!driver || !physicalDeviceObject || + physicalDeviceObject->Type != DeviceObjectPhysical) + return NTLSTATUS_INVALID_PARAMETER; + + DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0; + KSTATUS status = IoCreateDevice(driver, sizeof(HDA_CONTROLLER), &fdo); + if (!KSUCCESS(status)) + return status; + (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("hda")); + + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)fdo->DeviceExtension; + memset(controller, 0, sizeof(*controller)); + controller->Magic = HDA_CONTROLLER_MAGIC; + controller->Fdo = fdo; + controller->Pdo = physicalDeviceObject; + controller->Irq = 0xFFu; + controller->StateLock = (SPINLOCK)SPINLOCK_INIT; + controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG; + + controller->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject); + if (!controller->LowerDevice) { + IoDeleteDevice(fdo); + return NTLSTATUS_FAILURE; + } + return NTLSTATUS_SUCCESS; +} + +// Registers the Intel HDA transport driver with the I/O manager. +KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { + (void)init; + if (!ddk || !ddk->Debug || !ddk->Debug->Print || !ddk->Dma || !ddk->Timer || + !ddk->Timer->QueryTicks || !ddk->Timer->Stall || !ddk->Audio || + !ddk->Scheduler || !ddk->Scheduler->DelayThread) + return NTLSTATUS_INVALID_PARAMETER; + + gHdaDdk = ddk; + gHdaControllerCount = 0; + memset(gHdaControllers, 0, sizeof(gHdaControllers)); + + memset(&gHdaDriver, 0, sizeof(gHdaDriver)); + IoInitializeDriverObject(&gHdaDriver, SYS_SV_LITERAL("hda")); + gHdaDriver.IdTable = &gHdaIdTable; + gHdaDriver.DriverAddDevice = HdaAddDevice; + gHdaDriver.MajorFunction[IRP_MJ_PNP] = HdaDispatchPnp; + + KSTATUS status = IoRegisterDriverObject(&gHdaDriver); + if (!KSUCCESS(status)) + gHdaDdk->Debug->Print("HDA: IoRegisterDriverObject failed status=0x%x\n", status); + return status; +} diff --git a/code/drivers/audio/hda/hda.h b/code/drivers/audio/hda/hda.h new file mode 100644 index 0000000..b307394 --- /dev/null +++ b/code/drivers/audio/hda/hda.h @@ -0,0 +1,389 @@ +// Intel High Definition Audio transport declarations shared by the controller and stream implementation. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#define HDA_CONTROLLER_MAGIC 0x4144484Eu +#define HDA_MAX_CONTROLLERS 8u +#define HDA_MAX_CODECS 15u +#define HDA_MAX_NODE_ID 0xFFu +#define HDA_MAX_UNSOLICITED_RESPONSES 32u + +#define HDA_OUTPUT_SAMPLE_RATE 48000u +#define HDA_OUTPUT_CHANNELS 2u +#define HDA_OUTPUT_VALID_BITS 16u +#define HDA_OUTPUT_BYTES_PER_SAMPLE 2u +#define HDA_OUTPUT_BYTES_PER_FRAME 4u +#define HDA_OUTPUT_PERIOD_FRAMES 512u +#define HDA_OUTPUT_PERIOD_COUNT 4u +#define HDA_OUTPUT_PERIOD_BYTES (HDA_OUTPUT_PERIOD_FRAMES * HDA_OUTPUT_BYTES_PER_FRAME) +#define HDA_OUTPUT_BUFFER_BYTES (HDA_OUTPUT_PERIOD_COUNT * HDA_OUTPUT_PERIOD_BYTES) +#define HDA_OUTPUT_STREAM_FORMAT 0x0011u +#define HDA_OUTPUT_STREAM_TAG 1u +#define HDA_MAX_LOGICAL_ENDPOINTS 3u + +#define HDA_REG_GCAP 0x00u +#define HDA_REG_VMIN 0x02u +#define HDA_REG_VMAJ 0x03u +#define HDA_REG_GCTL 0x08u +#define HDA_REG_STATESTS 0x0Eu +#define HDA_REG_INTCTL 0x20u +#define HDA_REG_INTSTS 0x24u +#define HDA_REG_WALLCLK 0x30u +#define HDA_REG_CORBLBASE 0x40u +#define HDA_REG_CORBUBASE 0x44u +#define HDA_REG_CORBWP 0x48u +#define HDA_REG_CORBRP 0x4Au +#define HDA_REG_CORBCTL 0x4Cu +#define HDA_REG_CORBSTS 0x4Du +#define HDA_REG_CORBSIZE 0x4Eu +#define HDA_REG_RIRBLBASE 0x50u +#define HDA_REG_RIRBUBASE 0x54u +#define HDA_REG_RIRBWP 0x58u +#define HDA_REG_RINTCNT 0x5Au +#define HDA_REG_RIRBCTL 0x5Cu +#define HDA_REG_RIRBSTS 0x5Du +#define HDA_REG_RIRBSIZE 0x5Eu +#define HDA_REG_DPLBASE 0x70u +#define HDA_REG_DPUBASE 0x74u +#define HDA_REG_STREAM_BASE 0x80u +#define HDA_REG_STREAM_STRIDE 0x20u + +#define HDA_GCAP_64OK 0x0001u +#define HDA_GCAP_BSS_MASK 0x00F8u +#define HDA_GCAP_BSS_SHIFT 3u +#define HDA_GCAP_ISS_MASK 0x0F00u +#define HDA_GCAP_ISS_SHIFT 8u +#define HDA_GCAP_OSS_MASK 0xF000u +#define HDA_GCAP_OSS_SHIFT 12u + +#define HDA_GCTL_RESET 0x00000001u +#define HDA_GCTL_UNSOLICITED_ENABLE 0x00000100u + +#define HDA_INT_CONTROLLER_ENABLE 0x40000000u +#define HDA_INT_GLOBAL_ENABLE 0x80000000u +#define HDA_INT_CONTROLLER_STATUS 0x40000000u +#define HDA_INT_GLOBAL_STATUS 0x80000000u + +#define HDA_CORBRP_POINTER_MASK 0x00FFu +#define HDA_CORBRP_RESET 0x8000u +#define HDA_CORBCTL_MEMORY_ERROR_INTERRUPT 0x01u +#define HDA_CORBCTL_RUN 0x02u +#define HDA_CORBSTS_MEMORY_ERROR 0x01u + +#define HDA_RIRBWP_RESET 0x8000u +#define HDA_RIRBCTL_RESPONSE_INTERRUPT 0x01u +#define HDA_RIRBCTL_RUN 0x02u +#define HDA_RIRBCTL_OVERRUN_INTERRUPT 0x04u +#define HDA_RIRBSTS_RESPONSE 0x01u +#define HDA_RIRBSTS_OVERRUN 0x04u +#define HDA_RIRB_RESPONSE_CODEC_MASK 0x0000000Fu +#define HDA_RIRB_RESPONSE_UNSOLICITED 0x00000010u + +#define HDA_RING_SIZE_CAP_2 0x10u +#define HDA_RING_SIZE_CAP_16 0x20u +#define HDA_RING_SIZE_CAP_256 0x40u +#define HDA_RING_SIZE_SELECT_2 0x00u +#define HDA_RING_SIZE_SELECT_16 0x01u +#define HDA_RING_SIZE_SELECT_256 0x02u + +#define HDA_DPLBASE_ENABLE 0x00000001u + +#define HDA_STREAM_REG_CTL 0x00u +#define HDA_STREAM_REG_STS 0x03u +#define HDA_STREAM_REG_LPIB 0x04u +#define HDA_STREAM_REG_CBL 0x08u +#define HDA_STREAM_REG_LVI 0x0Cu +#define HDA_STREAM_REG_FORMAT 0x12u +#define HDA_STREAM_REG_BDLPL 0x18u +#define HDA_STREAM_REG_BDLPU 0x1Cu + +#define HDA_STREAM_CTL_RESET 0x000001u +#define HDA_STREAM_CTL_RUN 0x000002u +#define HDA_STREAM_CTL_INTERRUPT_COMPLETE 0x000004u +#define HDA_STREAM_CTL_INTERRUPT_FIFO_ERROR 0x000008u +#define HDA_STREAM_CTL_INTERRUPT_DESCRIPTOR_ERROR 0x000010u +#define HDA_STREAM_CTL_INTERRUPT_MASK 0x00001Cu +#define HDA_STREAM_CTL_TAG_SHIFT 20u +#define HDA_STREAM_CTL_TAG_MASK 0x00F00000u + +#define HDA_STREAM_STS_COMPLETE 0x04u +#define HDA_STREAM_STS_FIFO_ERROR 0x08u +#define HDA_STREAM_STS_DESCRIPTOR_ERROR 0x10u +#define HDA_STREAM_STS_ERROR_MASK 0x18u +#define HDA_STREAM_STS_INTERRUPT_MASK 0x1Cu + +#define HDA_BDL_FLAG_INTERRUPT_ON_COMPLETION 0x00000001u + +#define HDA_VERB_GET_PARAMETER(parameterId) (0xF0000u | ((uint32_t)(parameterId) & 0xFFu)) +#define HDA_PARAMETER_VENDOR_ID 0x00u + +#define HDA_CONTROLLER_RESET_TIMEOUT_US 100000u +#define HDA_STREAM_RESET_TIMEOUT_US 10000u +#define HDA_RING_RESET_TIMEOUT_US 10000u +#define HDA_CODEC_COMMAND_TIMEOUT_US 100000u +#define HDA_CODEC_COMMAND_ATTEMPTS 2u +#define HDA_CODEC_RESET_SETTLE_US 1000u +#define HDA_STREAM_ERROR_WINDOW_US 1000000ull +#define HDA_STREAM_ERRORS_BEFORE_CONTROLLER_RESET 3u + +#define HDA_RECOVERY_NONE 0u +#define HDA_RECOVERY_STREAM 1u +#define HDA_RECOVERY_CONTROLLER 2u + +#pragma pack(push, 1) +typedef struct HDA_BDL_ENTRY { + uint64_t Address; + uint32_t Length; + uint32_t Flags; +} HDA_BDL_ENTRY; + +typedef struct HDA_RIRB_ENTRY { + uint32_t Response; + uint32_t ResponseExtra; +} HDA_RIRB_ENTRY; +#pragma pack(pop) + +_Static_assert(sizeof(HDA_BDL_ENTRY) == 16u, "HDA BDL entries must be 16 bytes"); +_Static_assert(sizeof(HDA_RIRB_ENTRY) == 8u, "HDA RIRB entries must be 8 bytes"); + +typedef struct HDA_CONTROLLER HDA_CONTROLLER; +typedef struct HDA_CODEC_STATE HDA_CODEC_STATE; + +typedef struct HDA_OUTPUT_DESCRIPTION { + void* PeriodBuffer; + size_t PeriodBufferBytes; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + size_t PeriodStrideBytes; + IO_AUDIO_FORMAT HardwareFormat; + uint8_t StreamTag; +} HDA_OUTPUT_DESCRIPTION; + +typedef struct HDA_TRANSPORT_CALLBACKS { + // Reports one BDL-owned period after completion interrupt processing. + void (*PeriodComplete)(void* context, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t finePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks); + // Delivers one unsolicited codec response outside the controller lock. + void (*UnsolicitedResponse)(void* context, + uint8_t codecAddress, + uint32_t response, + uint32_t responseExtra); + // Restores codec programming in a muted state while transport start and stop are serialized. + KSTATUS (*RestoreAfterControllerReset)(void* context); + // Unmutes the restored route only after controller recovery has restarted DMA. + KSTATUS (*ResumeAfterControllerReset)(void* context); + // Reports a recovery failure that leaves the transport stopped. + void (*TransportFailure)(void* context, KSTATUS status); +} HDA_TRANSPORT_CALLBACKS; + +typedef struct HDA_UNSOLICITED_RESPONSE { + uint32_t Response; + uint32_t ResponseExtra; +} HDA_UNSOLICITED_RESPONSE; + +typedef struct HDA_OUTPUT_RING { + IO_DMA_BUFFER* Dma; + uint8_t* Buffer; + uint64_t DeviceAddress; + uint8_t NextLogicalPeriod; + uint64_t PresentedFrames; +} HDA_OUTPUT_RING; + +typedef struct HDA_OUTPUT_STREAM { + uint8_t DescriptorIndex; + uint8_t StreamTag; + uint8_t BdlPeriodMap[HDA_OUTPUT_PERIOD_COUNT]; + uint8_t NextCompletionBdl; + bool Prepared; + bool Running; + volatile long PendingPeriodInterrupts; + volatile long PendingDmaErrors; + HDA_OUTPUT_RING Rings[HDA_MAX_LOGICAL_ENDPOINTS]; + uint8_t ActiveRingIndex; + IO_DMA_BUFFER* BdlDma; + HDA_BDL_ENTRY* Bdl; + uint64_t BdlDeviceAddress; + uint64_t PositionRunBaseFrames; + uint64_t LastPositionFrames; + uint64_t PositionWrapFrames; + uint32_t LastHardwarePositionBytes; + bool PositionInitialized; + uint64_t InterruptCount; + uint64_t PeriodCompletionCount; + uint64_t DmaErrorCount; + uint64_t StreamResetCount; + uint64_t PositionBufferReadCount; + uint64_t LpibReadCount; + uint64_t PositionDiscontinuityCount; +} HDA_OUTPUT_STREAM; + +struct HDA_CONTROLLER { + uint32_t Magic; + DEVICE_OBJECT* Fdo; + DEVICE_OBJECT* Pdo; + DEVICE_OBJECT* LowerDevice; + PCI_DEVICE_INFO PciInfo; + volatile uint8_t* Registers; + uint64_t RegisterBytes; + uint16_t Gcap; + uint8_t InputStreamCount; + uint8_t OutputStreamCount; + uint8_t BidirectionalStreamCount; + uint8_t TotalStreamCount; + uint16_t CodecMask; + uint8_t Irq; + bool UsingMsi; + bool Started; + bool Removed; + bool SurpriseRemoved; + SPINLOCK StateLock; + KDPC Dpc; + volatile long IsrActive; + volatile long DpcActive; + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* CorbDma; + IO_DMA_BUFFER* RirbDma; + IO_DMA_BUFFER* PositionDma; + uint32_t* Corb; + HDA_RIRB_ENTRY* Rirb; + volatile uint64_t* PositionBuffer; + uint64_t CorbDeviceAddress; + uint64_t RirbDeviceAddress; + uint64_t PositionDeviceAddress; + uint16_t CorbEntryCount; + uint16_t RirbEntryCount; + uint16_t CorbWritePointer; + uint16_t RirbReadPointer; + MUTEX* CommandMutex; + MUTEX* StreamMutex; + bool CommandWaiting; + bool CommandResponseReady; + uint8_t CommandCodecAddress; + uint32_t CommandResponse; + uint32_t CommandResponseExtra; + uint64_t CodecCommandCount; + uint64_t CodecCommandRetryCount; + uint64_t CodecCommandFailureCount; + uint64_t RirbResponseCount; + uint64_t RirbUnsolicitedCount; + uint64_t RirbOverrunCount; + uint64_t CorbMemoryErrorCount; + HDA_UNSOLICITED_RESPONSE Unsolicited[HDA_MAX_UNSOLICITED_RESPONSES]; + uint8_t UnsolicitedRead; + uint8_t UnsolicitedWrite; + uint8_t UnsolicitedCount; + HDA_OUTPUT_STREAM Output; + HDA_TRANSPORT_CALLBACKS Callbacks; + void* CallbackContext; + SEMAPHORE* RecoverySemaphore; + EVENT* RecoveryFinished; + THREAD* RecoveryThread; + bool RecoveryWorkerRunning; + bool RecoveryInProgress; + uint8_t PendingRecovery; + uint64_t StreamErrorWindowStart; + uint32_t StreamErrorsInWindow; + uint64_t ControllerResetCount; + HDA_CODEC_STATE* Codec; + uint64_t InterruptCount; + uint64_t ControllerInterruptCount; +}; + +extern const NTL_DDK* gHdaDdk; + +// Returns one byte from the controller register aperture. +uint8_t HdaRead8(const HDA_CONTROLLER* controller, uint32_t offset); + +// Returns one word from the controller register aperture. +uint16_t HdaRead16(const HDA_CONTROLLER* controller, uint32_t offset); + +// Returns one dword from the controller register aperture. +uint32_t HdaRead32(const HDA_CONTROLLER* controller, uint32_t offset); + +// Writes one byte to the controller register aperture. +void HdaWrite8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t value); + +// Writes one word to the controller register aperture. +void HdaWrite16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t value); + +// Writes one dword to the controller register aperture. +void HdaWrite32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t value); + +// Returns the register offset of the controller's one output stream descriptor. +uint32_t HdaOutputStreamBase(const HDA_CONTROLLER* controller); + +// Allocates the transport and output DMA resources for one controller. +KSTATUS HdaAllocateDma(HDA_CONTROLLER* controller); + +// Releases all DMA and synchronization resources owned by one controller. +void HdaReleaseDma(HDA_CONTROLLER* controller); + +// Resets and initializes the controller, command rings, position buffer, and output descriptor. +KSTATUS HdaInitializeController(HDA_CONTROLLER* controller); + +// Arms controller and output-stream interrupts after ISR registration. +void HdaArmInterrupts(HDA_CONTROLLER* controller); + +// Stops hardware access without releasing software resources. +void HdaQuiesceController(HDA_CONTROLLER* controller); + +// Sends one 20-bit codec verb payload through CORB/RIRB with one retry. +KSTATUS HdaCodecVerb(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut); + +// Copies one logical endpoint ring contract for codec and endpoint setup. +KSTATUS HdaDescribeOutput(HDA_CONTROLLER* controller, + uint8_t ringIndex, + HDA_OUTPUT_DESCRIPTION* descriptionOut); + +// Selects one logical endpoint ring while the stream is stopped. +KSTATUS HdaSelectOutputRing(HDA_CONTROLLER* controller, uint8_t ringIndex); + +// Replaces the transport callbacks while the output stream is stopped. +KSTATUS HdaSetTransportCallbacks(HDA_CONTROLLER* controller, + const HDA_TRANSPORT_CALLBACKS* callbacks, + void* context); + +// Starts the prepared fixed-format output stream. +KSTATUS HdaStartOutput(HDA_CONTROLLER* controller); + +// Stops the output stream while retaining its DMA ring. +void HdaStopOutput(HDA_CONTROLLER* controller); + +// Returns the monotonic fine position derived from the position buffer or LPIB. +uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller); + +// Captures one controller interrupt and queues deferred processing. +void HdaHandleInterrupt(HDA_CONTROLLER* controller); + +// Runs period delivery, unsolicited callbacks, and recovery scheduling at DPC level. +void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2); + +// Starts the passive recovery worker for one controller. +KSTATUS HdaStartRecoveryWorker(HDA_CONTROLLER* controller); + +// Stops and joins the passive recovery worker for one controller. +void HdaStopRecoveryWorker(HDA_CONTROLLER* controller); + +// Discovers codec topology and registers usable analog render endpoints. +KSTATUS HdaCodecInitialize(HDA_CONTROLLER* controller); + +// Stops jack workers and unregisters codec endpoints. +void HdaCodecStop(HDA_CONTROLLER* controller, IO_AUDIO_REMOVAL_MODE mode); + +// Releases codec topology storage after interrupt work is quiescent. +void HdaCodecDestroy(HDA_CONTROLLER* controller); diff --git a/code/drivers/audio/hda/hda_codec.c b/code/drivers/audio/hda/hda_codec.c new file mode 100644 index 0000000..ad341de --- /dev/null +++ b/code/drivers/audio/hda/hda_codec.c @@ -0,0 +1,1928 @@ +// Generic Intel HDA codec discovery and analog render routing. + +#include "hda.h" + +#include +#include + +#define HDA_CODEC_POOL_TAG MM_POOL_TAG('h', 'd', 'a', 'c') +#define HDA_MAX_WIDGETS (HDA_MAX_NODE_ID + 1u) +#define HDA_MAX_AUDIO_FUNCTIONS 8u +#define HDA_MAX_CONNECTIONS HDA_MAX_WIDGETS +#define HDA_MAX_ROUTE_NODES 32u +#define HDA_MAX_ROUTES 32u +#define HDA_MAX_ENDPOINT_ROUTES 16u +#define HDA_INVALID_INDEX 0xFFu +#define HDA_JACK_POLL_INTERVAL_MS 500u +#define HDA_POWER_TIMEOUT_US 10000u + +#define HDA_PARAMETER_NODE_COUNT 0x04u +#define HDA_PARAMETER_FUNCTION_TYPE 0x05u +#define HDA_PARAMETER_AUDIO_WIDGET_CAPS 0x09u +#define HDA_PARAMETER_PCM 0x0Au +#define HDA_PARAMETER_STREAM_FORMATS 0x0Bu +#define HDA_PARAMETER_PIN_CAPS 0x0Cu +#define HDA_PARAMETER_INPUT_AMP_CAPS 0x0Du +#define HDA_PARAMETER_CONNECTION_LIST_LENGTH 0x0Eu +#define HDA_PARAMETER_POWER_STATES 0x0Fu +#define HDA_PARAMETER_OUTPUT_AMP_CAPS 0x12u + +#define HDA_FUNCTION_TYPE_AUDIO 0x01u + +#define HDA_WIDGET_TYPE_OUTPUT 0x00u +#define HDA_WIDGET_TYPE_INPUT 0x01u +#define HDA_WIDGET_TYPE_MIXER 0x02u +#define HDA_WIDGET_TYPE_SELECTOR 0x03u +#define HDA_WIDGET_TYPE_PIN 0x04u +#define HDA_WIDGET_TYPE_POWER 0x05u +#define HDA_WIDGET_TYPE_VOLUME_KNOB 0x06u +#define HDA_WIDGET_TYPE_BEEP 0x07u +#define HDA_WIDGET_TYPE_VENDOR 0x0Fu + +#define HDA_WIDGET_CAP_STEREO 0x00000001u +#define HDA_WIDGET_CAP_INPUT_AMP 0x00000002u +#define HDA_WIDGET_CAP_OUTPUT_AMP 0x00000004u +#define HDA_WIDGET_CAP_AMP_OVERRIDE 0x00000008u +#define HDA_WIDGET_CAP_FORMAT_OVERRIDE 0x00000010u +#define HDA_WIDGET_CAP_UNSOLICITED 0x00000080u +#define HDA_WIDGET_CAP_CONNECTION_LIST 0x00000100u +#define HDA_WIDGET_CAP_DIGITAL 0x00000200u +#define HDA_WIDGET_CAP_POWER_CONTROL 0x00000400u +#define HDA_WIDGET_CAP_TYPE_SHIFT 20u +#define HDA_WIDGET_CAP_TYPE_MASK 0x00F00000u + +#define HDA_PCM_RATE_48000 (1u << 6) +#define HDA_PCM_BITS_16 (1u << 17) +#define HDA_STREAM_FORMAT_PCM (1u << 0) + +#define HDA_PIN_CAP_TRIGGER_REQUIRED (1u << 1) +#define HDA_PIN_CAP_PRESENCE_DETECT (1u << 2) +#define HDA_PIN_CAP_HEADPHONE_DRIVE (1u << 3) +#define HDA_PIN_CAP_OUTPUT (1u << 4) +#define HDA_PIN_CAP_HDMI (1u << 7) +#define HDA_PIN_CAP_EAPD (1u << 16) + +#define HDA_DEFAULT_DEVICE_LINE_OUT 0x0u +#define HDA_DEFAULT_DEVICE_SPEAKER 0x1u +#define HDA_DEFAULT_DEVICE_HEADPHONE 0x2u +#define HDA_DEFAULT_DEVICE_SPDIF_OUT 0x4u +#define HDA_DEFAULT_DEVICE_DIGITAL_OTHER_OUT 0x5u +#define HDA_DEFAULT_DEVICE_SHIFT 20u +#define HDA_DEFAULT_DEVICE_MASK 0x00F00000u +#define HDA_DEFAULT_LOCATION_SHIFT 24u +#define HDA_DEFAULT_LOCATION_MASK 0x3F000000u +#define HDA_DEFAULT_ASSOCIATION_SHIFT 4u +#define HDA_DEFAULT_ASSOCIATION_MASK 0x000000F0u +#define HDA_DEFAULT_SEQUENCE_MASK 0x0000000Fu +#define HDA_DEFAULT_PORT_SHIFT 30u +#define HDA_DEFAULT_PORT_MASK 0xC0000000u +#define HDA_DEFAULT_PORT_NONE 0x3u +#define HDA_LOCATION_GROSS_INTERNAL 0x1u + +#define HDA_VERB_GET_CONNECTION_LIST_ENTRY(index) (0xF0200u | ((uint32_t)(index) & 0xFFu)) +#define HDA_VERB_GET_POWER_STATE 0xF0500u +#define HDA_VERB_GET_PIN_SENSE 0xF0900u +#define HDA_VERB_GET_CONFIG_DEFAULT 0xF1C00u +#define HDA_VERB_SET_CONNECTION_SELECT(index) (0x70100u | ((uint32_t)(index) & 0xFFu)) +#define HDA_VERB_SET_POWER_STATE(state) (0x70500u | ((uint32_t)(state) & 0x0Fu)) +#define HDA_VERB_SET_CONVERTER_STREAM(tag, channel) \ + (0x70600u | (((uint32_t)(tag) & 0x0Fu) << 4) | ((uint32_t)(channel) & 0x0Fu)) +#define HDA_VERB_SET_PIN_CONTROL(value) (0x70700u | ((uint32_t)(value) & 0xFFu)) +#define HDA_VERB_SET_UNSOLICITED(value) (0x70800u | ((uint32_t)(value) & 0xFFu)) +#define HDA_VERB_EXECUTE_PIN_SENSE 0x70900u +#define HDA_VERB_SET_EAPD(value) (0x70C00u | ((uint32_t)(value) & 0xFFu)) +#define HDA_VERB_SET_CONVERTER_FORMAT(format) (0x20000u | ((uint32_t)(format) & 0xFFFFu)) +#define HDA_VERB_SET_AMP(payload) (0x30000u | ((uint32_t)(payload) & 0xFFFFu)) + +#define HDA_PIN_CONTROL_OUTPUT_ENABLE 0x40u +#define HDA_PIN_CONTROL_HEADPHONE_ENABLE 0x80u +#define HDA_EAPD_ENABLE 0x02u +#define HDA_UNSOLICITED_ENABLE 0x80u +#define HDA_UNSOLICITED_TAG_MASK 0x3Fu +#define HDA_UNSOLICITED_TAG_SHIFT 26u + +#define HDA_AMP_CAP_OFFSET_MASK 0x0000007Fu +#define HDA_AMP_CAP_STEPS_SHIFT 8u +#define HDA_AMP_CAP_STEPS_MASK 0x00007F00u +#define HDA_AMP_CAP_STEP_SIZE_SHIFT 16u +#define HDA_AMP_CAP_STEP_SIZE_MASK 0x007F0000u +#define HDA_AMP_CAP_MUTE 0x80000000u +#define HDA_AMP_SET_OUTPUT (1u << 15) +#define HDA_AMP_SET_INPUT (1u << 14) +#define HDA_AMP_SET_LEFT (1u << 13) +#define HDA_AMP_SET_RIGHT (1u << 12) +#define HDA_AMP_SET_INDEX_SHIFT 8u +#define HDA_AMP_SET_MUTE (1u << 7) +#define HDA_AMP_SET_GAIN_MASK 0x7Fu + +#define HDA_PIN_SENSE_PRESENT 0x80000000u + +#define HDA_POWER_STATE_D0 0u +#define HDA_POWER_ACTUAL_SHIFT 4u +#define HDA_POWER_ACTUAL_MASK 0xF0u + +typedef enum HDA_ROUTE_CLASS { + HdaRouteLineOut = 1, + HdaRouteSpeaker, + HdaRouteHeadphone, + HdaRouteOtherAnalog +} HDA_ROUTE_CLASS; + +typedef struct HDA_WIDGET { + bool Present; + uint8_t NodeId; + uint8_t Type; + uint8_t FunctionIndex; + uint32_t Capabilities; + uint32_t PcmCapabilities; + uint32_t StreamFormats; + uint32_t PinCapabilities; + uint32_t ConfigurationDefault; + uint32_t InputAmpCapabilities; + uint32_t OutputAmpCapabilities; + uint16_t ConnectionCount; + uint8_t Connections[HDA_MAX_CONNECTIONS]; + bool PresenceKnown; + bool PresentAtJack; + bool PollPresence; + bool UnsolicitedEnabled; + uint8_t UnsolicitedTag; +} HDA_WIDGET; + +typedef struct HDA_AUDIO_FUNCTION { + uint8_t NodeId; + uint32_t PcmCapabilities; + uint32_t StreamFormats; + uint32_t InputAmpCapabilities; + uint32_t OutputAmpCapabilities; +} HDA_AUDIO_FUNCTION; + +typedef struct HDA_CODEC_TOPOLOGY { + uint8_t Address; + uint32_t VendorId; + uint8_t FunctionCount; + HDA_AUDIO_FUNCTION Functions[HDA_MAX_AUDIO_FUNCTIONS]; + HDA_WIDGET Widgets[HDA_MAX_WIDGETS]; +} HDA_CODEC_TOPOLOGY; + +typedef struct HDA_ROUTE { + HDA_CODEC_TOPOLOGY* Codec; + HDA_ROUTE_CLASS Class; + uint8_t PinNode; + uint8_t ConverterNode; + uint8_t NodeCount; + uint8_t Nodes[HDA_MAX_ROUTE_NODES]; + uint8_t ConnectionIndices[HDA_MAX_ROUTE_NODES]; + uint32_t SortKey; + uint8_t MasterAmpNode; + uint32_t MasterAmpCapabilities; +} HDA_ROUTE; + +typedef struct HDA_LOGICAL_ENDPOINT { + struct HDA_CODEC_STATE* State; + IO_AUDIO_ENDPOINT* KernelEndpoint; + IO_AUDIO_ENDPOINT_KIND Kind; + uint8_t RingIndex; + uint8_t RouteCount; + uint8_t RouteIndices[HDA_MAX_ENDPOINT_ROUTES]; + uint8_t CurrentRouteIndex; + bool Running; + bool Starting; + bool Available; + bool HasHardwareGain; + uint32_t HardwareAmpCapabilities; + int32_t RequestedGainMilliDb; + bool RequestedMute; +} HDA_LOGICAL_ENDPOINT; + +struct HDA_CODEC_STATE { + HDA_CONTROLLER* Controller; + MUTEX* Lock; + SPINLOCK CallbackLock; + HDA_CODEC_TOPOLOGY* Codecs[HDA_MAX_CODECS]; + HDA_ROUTE Routes[HDA_MAX_ROUTES]; + uint8_t RouteCount; + HDA_LOGICAL_ENDPOINT Endpoints[HDA_MAX_LOGICAL_ENDPOINTS]; + uint8_t EndpointCount; + HDA_LOGICAL_ENDPOINT* ActiveEndpoint; + uint16_t UnsolicitedNodes[64]; + SEMAPHORE* WorkSemaphore; + EVENT* WorkFinished; + EVENT* PollFinished; + THREAD* WorkThread; + THREAD* PollThread; + volatile long WorkRunning; + volatile long PollRunning; + volatile long ShuttingDown; + uint64_t PendingUnsolicitedTags; + bool PollRequested; + bool HavePollingPins; + uint64_t JackEventCount; + uint64_t JackPollCount; + uint64_t JackChangeCount; + uint64_t RouteChangeCount; + uint64_t RouteFailureCount; +}; + +// Returns a widget only when the node was enumerated in this audio function group. +static HDA_WIDGET* HdaCodecWidget(HDA_CODEC_TOPOLOGY* codec, uint8_t nodeId) { + if (!codec || !codec->Widgets[nodeId].Present) + return (HDA_WIDGET*)0; + return &codec->Widgets[nodeId]; +} + +// Sends one codec verb through the transport owned by this codec state. +static KSTATUS HdaCodecCommand(HDA_CODEC_STATE* state, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t payload, + uint32_t* responseOut) { + return HdaCodecVerb(state->Controller, + codecAddress, + nodeId, + payload, + responseOut); +} + +// Reads one architectural codec parameter. +static KSTATUS HdaCodecGetParameter(HDA_CODEC_STATE* state, + uint8_t codecAddress, + uint8_t nodeId, + uint8_t parameter, + uint32_t* responseOut) { + return HdaCodecCommand(state, + codecAddress, + nodeId, + HDA_VERB_GET_PARAMETER(parameter), + responseOut); +} + +// Powers one function group or widget to D0 and waits for the actual state. +static KSTATUS HdaCodecPowerD0(HDA_CODEC_STATE* state, + uint8_t codecAddress, + uint8_t nodeId) { + KSTATUS status = HdaCodecCommand(state, + codecAddress, + nodeId, + HDA_VERB_SET_POWER_STATE(HDA_POWER_STATE_D0), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + for (uint32_t elapsed = 0; elapsed < HDA_POWER_TIMEOUT_US; elapsed += 10u) { + uint32_t powerState = 0; + status = HdaCodecCommand(state, + codecAddress, + nodeId, + HDA_VERB_GET_POWER_STATE, + &powerState); + if (!KSUCCESS(status)) + return status; + if ((powerState & HDA_POWER_ACTUAL_MASK) == + (HDA_POWER_STATE_D0 << HDA_POWER_ACTUAL_SHIFT)) + return NTLSTATUS_SUCCESS; + gHdaDdk->Timer->Stall(10); + } + return NTLSTATUS_TIMEOUT; +} + +// Expands one short- or long-form connection list into selectable node IDs. +static KSTATUS HdaCodecReadConnections(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + HDA_WIDGET* widget) { + uint32_t lengthParameter = 0; + KSTATUS status = HdaCodecGetParameter(state, + codec->Address, + widget->NodeId, + HDA_PARAMETER_CONNECTION_LIST_LENGTH, + &lengthParameter); + if (!KSUCCESS(status)) + return status; + uint8_t encodedCount = (uint8_t)(lengthParameter & 0x7Fu); + bool longForm = (lengthParameter & 0x80u) != 0; + uint8_t entriesPerResponse = longForm ? 2u : 4u; + uint16_t nodeMask = longForm ? 0x7FFFu : 0x007Fu; + uint16_t rangeMask = longForm ? 0x8000u : 0x0080u; + uint16_t previousNode = 0; + bool havePrevious = false; + + widget->ConnectionCount = 0; + for (uint8_t offset = 0; offset < encodedCount; offset += entriesPerResponse) { + uint32_t response = 0; + status = HdaCodecCommand(state, + codec->Address, + widget->NodeId, + HDA_VERB_GET_CONNECTION_LIST_ENTRY(offset), + &response); + if (!KSUCCESS(status)) + return status; + for (uint8_t slot = 0; slot < entriesPerResponse && + (uint8_t)(offset + slot) < encodedCount; slot++) { + uint16_t raw = longForm + ? (uint16_t)(response >> (slot * 16u)) + : (uint16_t)((response >> (slot * 8u)) & 0xFFu); + uint16_t node = raw & nodeMask; + bool range = (raw & rangeMask) != 0; + uint16_t first = node; + if (range) { + if (!havePrevious || node <= previousNode) + return NTLSTATUS_DEVICE_ERROR; + first = (uint16_t)(previousNode + 1u); + } + for (uint16_t expanded = first; expanded <= node; expanded++) { + if (expanded > HDA_MAX_NODE_ID || + widget->ConnectionCount == HDA_MAX_CONNECTIONS) + return NTLSTATUS_NOT_SUPPORTED; + widget->Connections[widget->ConnectionCount++] = (uint8_t)expanded; + } + previousNode = node; + havePrevious = true; + } + } + return NTLSTATUS_SUCCESS; +} + +// Enumerates one widget and inherits function defaults when the widget has no override. +static KSTATUS HdaCodecEnumerateWidget(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t functionIndex, + uint8_t nodeId) { + if (functionIndex >= codec->FunctionCount) + return NTLSTATUS_INVALID_PARAMETER; + HDA_AUDIO_FUNCTION* function = &codec->Functions[functionIndex]; + HDA_WIDGET* widget = &codec->Widgets[nodeId]; + uint32_t capabilities = 0; + KSTATUS status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_AUDIO_WIDGET_CAPS, + &capabilities); + if (!KSUCCESS(status)) + return status; + + memset(widget, 0, sizeof(*widget)); + widget->Present = true; + widget->NodeId = nodeId; + widget->FunctionIndex = functionIndex; + widget->Capabilities = capabilities; + widget->Type = (uint8_t)((capabilities & HDA_WIDGET_CAP_TYPE_MASK) >> + HDA_WIDGET_CAP_TYPE_SHIFT); + widget->PcmCapabilities = function->PcmCapabilities; + widget->StreamFormats = function->StreamFormats; + widget->InputAmpCapabilities = function->InputAmpCapabilities; + widget->OutputAmpCapabilities = function->OutputAmpCapabilities; + + if ((capabilities & HDA_WIDGET_CAP_FORMAT_OVERRIDE) != 0 && + (widget->Type == HDA_WIDGET_TYPE_OUTPUT || + widget->Type == HDA_WIDGET_TYPE_INPUT)) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_PCM, + &widget->PcmCapabilities); + if (!KSUCCESS(status)) + return status; + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_STREAM_FORMATS, + &widget->StreamFormats); + if (!KSUCCESS(status)) + return status; + } + + if ((capabilities & HDA_WIDGET_CAP_AMP_OVERRIDE) != 0) { + if ((capabilities & HDA_WIDGET_CAP_INPUT_AMP) != 0) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_INPUT_AMP_CAPS, + &widget->InputAmpCapabilities); + if (!KSUCCESS(status)) + return status; + } + if ((capabilities & HDA_WIDGET_CAP_OUTPUT_AMP) != 0) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_OUTPUT_AMP_CAPS, + &widget->OutputAmpCapabilities); + if (!KSUCCESS(status)) + return status; + } + } + + if ((capabilities & HDA_WIDGET_CAP_CONNECTION_LIST) != 0) { + status = HdaCodecReadConnections(state, codec, widget); + if (!KSUCCESS(status)) + return status; + } + + if (widget->Type == HDA_WIDGET_TYPE_PIN) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_PIN_CAPS, + &widget->PinCapabilities); + if (!KSUCCESS(status)) + return status; + status = HdaCodecCommand(state, + codec->Address, + nodeId, + HDA_VERB_GET_CONFIG_DEFAULT, + &widget->ConfigurationDefault); + if (!KSUCCESS(status)) + return status; + } + + return NTLSTATUS_SUCCESS; +} + +// Enumerates one audio function group and all of its child widgets. +static KSTATUS HdaCodecEnumerateAudioFunction(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t functionNode) { + if (codec->FunctionCount == HDA_MAX_AUDIO_FUNCTIONS) + return NTLSTATUS_NOT_SUPPORTED; + uint8_t functionIndex = codec->FunctionCount++; + HDA_AUDIO_FUNCTION* function = &codec->Functions[functionIndex]; + memset(function, 0, sizeof(*function)); + function->NodeId = functionNode; + + KSTATUS status = HdaCodecPowerD0(state, codec->Address, functionNode); + if (!KSUCCESS(status)) + return status; + + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_PCM, + &function->PcmCapabilities); + if (!KSUCCESS(status)) + return status; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_STREAM_FORMATS, + &function->StreamFormats); + if (!KSUCCESS(status)) + return status; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_INPUT_AMP_CAPS, + &function->InputAmpCapabilities); + if (!KSUCCESS(status)) + function->InputAmpCapabilities = 0; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_OUTPUT_AMP_CAPS, + &function->OutputAmpCapabilities); + if (!KSUCCESS(status)) + function->OutputAmpCapabilities = 0; + + uint32_t nodeCount = 0; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_NODE_COUNT, + &nodeCount); + if (!KSUCCESS(status)) + return status; + uint8_t startNode = (uint8_t)((nodeCount >> 16) & 0xFFu); + uint8_t count = (uint8_t)(nodeCount & 0xFFu); + if (count == 0 || (uint16_t)startNode + count > HDA_MAX_WIDGETS) + return NTLSTATUS_NOT_SUPPORTED; + + for (uint16_t node = startNode; node < (uint16_t)startNode + count; node++) { + if (codec->Widgets[node].Present) + return NTLSTATUS_DEVICE_ERROR; + status = HdaCodecEnumerateWidget(state, + codec, + functionIndex, + (uint8_t)node); + if (!KSUCCESS(status)) + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Enumerates all audio function groups exposed by one responding codec. +static KSTATUS HdaCodecEnumerateOne(HDA_CODEC_STATE* state, uint8_t codecAddress) { + HDA_CODEC_TOPOLOGY* codec = (HDA_CODEC_TOPOLOGY*)MmAllocatePoolTag( + sizeof(HDA_CODEC_TOPOLOGY), + HDA_CODEC_POOL_TAG); + if (!codec) + return NTLSTATUS_NO_MEMORY; + memset(codec, 0, sizeof(*codec)); + codec->Address = codecAddress; + + KSTATUS status = HdaCodecGetParameter(state, + codecAddress, + 0, + HDA_PARAMETER_VENDOR_ID, + &codec->VendorId); + if (!KSUCCESS(status)) { + MmFreePool(codec); + return status; + } + uint32_t functionCount = 0; + status = HdaCodecGetParameter(state, + codecAddress, + 0, + HDA_PARAMETER_NODE_COUNT, + &functionCount); + if (!KSUCCESS(status)) { + MmFreePool(codec); + return status; + } + uint8_t startNode = (uint8_t)((functionCount >> 16) & 0xFFu); + uint8_t count = (uint8_t)(functionCount & 0xFFu); + if (count == 0 || (uint16_t)startNode + count > HDA_MAX_WIDGETS) { + MmFreePool(codec); + return NTLSTATUS_NOT_SUPPORTED; + } + + for (uint16_t node = startNode; node < (uint16_t)startNode + count; node++) { + uint32_t functionType = 0; + status = HdaCodecGetParameter(state, + codecAddress, + (uint8_t)node, + HDA_PARAMETER_FUNCTION_TYPE, + &functionType); + if (!KSUCCESS(status)) + continue; + if ((functionType & 0xFFu) != HDA_FUNCTION_TYPE_AUDIO) + continue; + status = HdaCodecEnumerateAudioFunction(state, codec, (uint8_t)node); + if (!KSUCCESS(status)) { + MmFreePool(codec); + return status; + } + } + if (codec->FunctionCount == 0) { + MmFreePool(codec); + return NTLSTATUS_NOT_FOUND; + } + + state->Codecs[codecAddress] = codec; + return NTLSTATUS_SUCCESS; +} + +// Enumerates every responding codec without assuming any codec node IDs. +static KSTATUS HdaCodecEnumerateAll(HDA_CODEC_STATE* state) { + KSTATUS lastStatus = NTLSTATUS_NOT_FOUND; + bool found = false; + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + if ((state->Controller->CodecMask & (uint16_t)(1u << codecAddress)) == 0) + continue; + KSTATUS status = HdaCodecEnumerateOne(state, codecAddress); + if (KSUCCESS(status)) { + found = true; + continue; + } + lastStatus = status; + gHdaDdk->Debug->Print( + "HDA: codec %u topology discovery failed status=0x%x\n", + codecAddress, + status); + } + return found ? NTLSTATUS_SUCCESS : lastStatus; +} + +// Returns whether a converter supports the fixed 48 kHz, stereo, 16-bit PCM contract. +static bool HdaCodecConverterSupportsOutput(const HDA_WIDGET* widget) { + return widget && widget->Type == HDA_WIDGET_TYPE_OUTPUT && + (widget->Capabilities & HDA_WIDGET_CAP_DIGITAL) == 0 && + (widget->Capabilities & HDA_WIDGET_CAP_STEREO) != 0 && + (widget->StreamFormats & HDA_STREAM_FORMAT_PCM) != 0 && + (widget->PcmCapabilities & HDA_PCM_RATE_48000) != 0 && + (widget->PcmCapabilities & HDA_PCM_BITS_16) != 0; +} + +// Classifies one analog output pin from its configuration default. +static HDA_ROUTE_CLASS HdaCodecClassifyPin(const HDA_WIDGET* pin) { + uint8_t device = (uint8_t)((pin->ConfigurationDefault & HDA_DEFAULT_DEVICE_MASK) >> + HDA_DEFAULT_DEVICE_SHIFT); + if (device == HDA_DEFAULT_DEVICE_LINE_OUT) + return HdaRouteLineOut; + if (device == HDA_DEFAULT_DEVICE_SPEAKER) + return HdaRouteSpeaker; + if (device == HDA_DEFAULT_DEVICE_HEADPHONE) + return HdaRouteHeadphone; + return HdaRouteOtherAnalog; +} + +// Returns whether a pin is a connected analog render destination. +static bool HdaCodecPinIsAnalogOutput(const HDA_WIDGET* pin) { + if (!pin || pin->Type != HDA_WIDGET_TYPE_PIN || + (pin->Capabilities & HDA_WIDGET_CAP_DIGITAL) != 0 || + (pin->PinCapabilities & HDA_PIN_CAP_OUTPUT) == 0 || + (pin->PinCapabilities & HDA_PIN_CAP_HDMI) != 0) + return false; + uint8_t port = (uint8_t)((pin->ConfigurationDefault & HDA_DEFAULT_PORT_MASK) >> + HDA_DEFAULT_PORT_SHIFT); + uint8_t device = (uint8_t)((pin->ConfigurationDefault & HDA_DEFAULT_DEVICE_MASK) >> + HDA_DEFAULT_DEVICE_SHIFT); + if (port == HDA_DEFAULT_PORT_NONE || + device == HDA_DEFAULT_DEVICE_SPDIF_OUT || + device == HDA_DEFAULT_DEVICE_DIGITAL_OTHER_OUT) + return false; + return true; +} + +// Computes a stable route ordering from association, sequence, and node ID. +static uint32_t HdaCodecRouteSortKey(const HDA_WIDGET* pin) { + uint32_t association = (pin->ConfigurationDefault & HDA_DEFAULT_ASSOCIATION_MASK) >> + HDA_DEFAULT_ASSOCIATION_SHIFT; + uint32_t sequence = pin->ConfigurationDefault & HDA_DEFAULT_SEQUENCE_MASK; + if (association == 0) + association = 0x10u; + return (association << 16) | (sequence << 8) | pin->NodeId; +} + +// Selects the first controllable output amplifier on a route, preferring the converter. +static void HdaCodecSelectMasterAmp(HDA_ROUTE* route) { + route->MasterAmpNode = HDA_INVALID_INDEX; + route->MasterAmpCapabilities = 0; + for (uint8_t reverse = route->NodeCount; reverse > 0; reverse--) { + uint8_t nodeId = route->Nodes[reverse - 1u]; + HDA_WIDGET* widget = HdaCodecWidget(route->Codec, nodeId); + if (!widget || (widget->Capabilities & HDA_WIDGET_CAP_OUTPUT_AMP) == 0) + continue; + route->MasterAmpNode = nodeId; + route->MasterAmpCapabilities = widget->OutputAmpCapabilities; + return; + } +} + +// Records one converter-to-pin route discovered by the reverse graph walk. +static KSTATUS HdaCodecRecordRoute(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + const uint8_t* nodes, + const uint8_t* connectionIndices, + uint8_t nodeCount) { + if (state->RouteCount == HDA_MAX_ROUTES || nodeCount < 2) + return NTLSTATUS_NOT_SUPPORTED; + HDA_ROUTE* route = &state->Routes[state->RouteCount]; + memset(route, 0, sizeof(*route)); + route->Codec = codec; + route->PinNode = nodes[0]; + route->ConverterNode = nodes[nodeCount - 1u]; + route->NodeCount = nodeCount; + memcpy(route->Nodes, nodes, nodeCount); + memcpy(route->ConnectionIndices, connectionIndices, nodeCount); + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + route->Class = HdaCodecClassifyPin(pin); + route->SortKey = HdaCodecRouteSortKey(pin); + HdaCodecSelectMasterAmp(route); + state->RouteCount++; + + return NTLSTATUS_SUCCESS; +} + +// Walks upstream connection lists from a pin until a valid output converter is reached. +static KSTATUS HdaCodecWalkRoute(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t functionIndex, + uint8_t nodeId, + uint8_t* nodes, + uint8_t* connectionIndices, + uint8_t depth, + bool* visited) { + if (depth == HDA_MAX_ROUTE_NODES) + return NTLSTATUS_NOT_SUPPORTED; + HDA_WIDGET* widget = HdaCodecWidget(codec, nodeId); + if (!widget || widget->FunctionIndex != functionIndex || visited[nodeId] || + (widget->Capabilities & HDA_WIDGET_CAP_DIGITAL) != 0) + return NTLSTATUS_NOT_FOUND; + + visited[nodeId] = true; + nodes[depth] = nodeId; + connectionIndices[depth] = HDA_INVALID_INDEX; + if (HdaCodecConverterSupportsOutput(widget)) { + KSTATUS status = HdaCodecRecordRoute(state, + codec, + nodes, + connectionIndices, + (uint8_t)(depth + 1u)); + visited[nodeId] = false; + return status; + } + if (widget->Type == HDA_WIDGET_TYPE_INPUT || widget->ConnectionCount == 0) { + visited[nodeId] = false; + return NTLSTATUS_NOT_FOUND; + } + + KSTATUS result = NTLSTATUS_NOT_FOUND; + for (uint16_t index = 0; index < widget->ConnectionCount; index++) { + connectionIndices[depth] = (uint8_t)index; + KSTATUS status = HdaCodecWalkRoute(state, + codec, + functionIndex, + widget->Connections[index], + nodes, + connectionIndices, + (uint8_t)(depth + 1u), + visited); + if (KSUCCESS(status)) { + visited[nodeId] = false; + return NTLSTATUS_SUCCESS; + } + if (status != NTLSTATUS_NOT_FOUND && result == NTLSTATUS_NOT_FOUND) + result = status; + } + visited[nodeId] = false; + return result; +} + +// Finds every usable analog path from an output converter to an output pin. +static KSTATUS HdaCodecBuildRoutes(HDA_CODEC_STATE* state) { + uint8_t nodes[HDA_MAX_ROUTE_NODES]; + uint8_t connectionIndices[HDA_MAX_ROUTE_NODES]; + bool visited[HDA_MAX_WIDGETS]; + memset(visited, 0, sizeof(visited)); + + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!HdaCodecPinIsAnalogOutput(pin)) + continue; + KSTATUS status = HdaCodecWalkRoute(state, + codec, + pin->FunctionIndex, + (uint8_t)node, + nodes, + connectionIndices, + 0, + visited); + if (status != NTLSTATUS_NOT_FOUND && !KSUCCESS(status)) + return status; + } + } + return state->RouteCount ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_SUPPORTED; +} + +// Reads the current presence state of one presence-capable analog pin. +static KSTATUS HdaCodecReadPinPresence(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + HDA_WIDGET* pin, + bool* changedOut) { + if (changedOut) + *changedOut = false; + if ((pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) == 0) { + pin->PresenceKnown = true; + pin->PresentAtJack = true; + return NTLSTATUS_SUCCESS; + } + if ((pin->PinCapabilities & HDA_PIN_CAP_TRIGGER_REQUIRED) != 0) { + KSTATUS status = HdaCodecCommand(state, + codec->Address, + pin->NodeId, + HDA_VERB_EXECUTE_PIN_SENSE, + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + uint32_t sense = 0; + KSTATUS status = HdaCodecCommand(state, + codec->Address, + pin->NodeId, + HDA_VERB_GET_PIN_SENSE, + &sense); + if (!KSUCCESS(status)) + return status; + bool present = (sense & HDA_PIN_SENSE_PRESENT) != 0; + bool changed = pin->PresenceKnown && pin->PresentAtJack != present; + pin->PresenceKnown = true; + pin->PresentAtJack = present; + if (changedOut) + *changedOut = changed; + return NTLSTATUS_SUCCESS; +} + +// Returns whether the physical pin behind one route is currently usable. +static bool HdaCodecRouteAvailable(const HDA_ROUTE* route) { + HDA_WIDGET* pin = HdaCodecWidget(route->Codec, route->PinNode); + if (!pin) + return false; + if ((pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) == 0) + return true; + return pin->PresenceKnown && pin->PresentAtJack; +} + +// Adds one route to a logical endpoint without duplicating route indices. +static KSTATUS HdaCodecEndpointAddRoute(HDA_LOGICAL_ENDPOINT* endpoint, + uint8_t routeIndex) { + if (endpoint->RouteCount == HDA_MAX_ENDPOINT_ROUTES) + return NTLSTATUS_NOT_SUPPORTED; + endpoint->RouteIndices[endpoint->RouteCount++] = routeIndex; + return NTLSTATUS_SUCCESS; +} + +// Returns whether an amplifier range fits the kernel endpoint gain contract. +static bool HdaCodecAmpRangeIsRegisterable(uint32_t capabilities) { + uint32_t offset = capabilities & HDA_AMP_CAP_OFFSET_MASK; + uint32_t steps = (capabilities & HDA_AMP_CAP_STEPS_MASK) >> + HDA_AMP_CAP_STEPS_SHIFT; + uint32_t step = (((capabilities & HDA_AMP_CAP_STEP_SIZE_MASK) >> + HDA_AMP_CAP_STEP_SIZE_SHIFT) + 1u) * 250u; + int64_t minimum = -(int64_t)offset * step; + int64_t maximum = ((int64_t)steps - offset) * step; + return minimum >= -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB && + maximum <= IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB; +} + +// Returns whether every route in one endpoint exposes the same mute-capable output amplifier. +static bool HdaCodecEndpointHasCommonGain(HDA_CODEC_STATE* state, + HDA_LOGICAL_ENDPOINT* endpoint, + uint32_t* capabilitiesOut) { + uint32_t capabilities = 0; + for (uint8_t index = 0; index < endpoint->RouteCount; index++) { + HDA_ROUTE* route = &state->Routes[endpoint->RouteIndices[index]]; + if (route->MasterAmpNode == HDA_INVALID_INDEX || + (route->MasterAmpCapabilities & HDA_AMP_CAP_MUTE) == 0) + return false; + if (index == 0) + capabilities = route->MasterAmpCapabilities; + else if (route->MasterAmpCapabilities != capabilities) + return false; + } + if (endpoint->RouteCount == 0 || !HdaCodecAmpRangeIsRegisterable(capabilities)) + return false; + if (capabilitiesOut) + *capabilitiesOut = capabilities; + return true; +} + +// Builds the line-out, speaker/headphone, and other-analog logical endpoint groups. +static KSTATUS HdaCodecBuildEndpoints(HDA_CODEC_STATE* state) { + HDA_LOGICAL_ENDPOINT groups[HDA_MAX_LOGICAL_ENDPOINTS]; + memset(groups, 0, sizeof(groups)); + groups[0].Kind = IoAudioEndpointLineOut; + groups[1].Kind = IoAudioEndpointSpeakerHeadphone; + groups[2].Kind = IoAudioEndpointOtherAnalog; + + for (uint8_t routeIndex = 0; routeIndex < state->RouteCount; routeIndex++) { + HDA_ROUTE* route = &state->Routes[routeIndex]; + uint8_t groupIndex = 2; + if (route->Class == HdaRouteLineOut) + groupIndex = 0; + else if (route->Class == HdaRouteSpeaker || route->Class == HdaRouteHeadphone) + groupIndex = 1; + KSTATUS status = HdaCodecEndpointAddRoute(&groups[groupIndex], routeIndex); + if (!KSUCCESS(status)) + return status; + } + + for (uint8_t groupIndex = 0; groupIndex < HDA_MAX_LOGICAL_ENDPOINTS; groupIndex++) { + if (groups[groupIndex].RouteCount == 0) + continue; + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[state->EndpointCount]; + *endpoint = groups[groupIndex]; + endpoint->State = state; + endpoint->RingIndex = state->EndpointCount; + endpoint->CurrentRouteIndex = HDA_INVALID_INDEX; + endpoint->RequestedGainMilliDb = 0; + endpoint->HardwareAmpCapabilities = 0; + endpoint->HasHardwareGain = HdaCodecEndpointHasCommonGain( + state, + endpoint, + &endpoint->HardwareAmpCapabilities); + state->EndpointCount++; + } + return state->EndpointCount ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_SUPPORTED; +} + +// Selects the best currently available route for one logical endpoint. +static uint8_t HdaCodecChooseRoute(HDA_LOGICAL_ENDPOINT* endpoint) { + HDA_CODEC_STATE* state = endpoint->State; + uint8_t selected = HDA_INVALID_INDEX; + uint32_t selectedKey = UINT32_MAX; + bool selectedHeadphone = false; + for (uint8_t index = 0; index < endpoint->RouteCount; index++) { + uint8_t routeIndex = endpoint->RouteIndices[index]; + HDA_ROUTE* route = &state->Routes[routeIndex]; + if (!HdaCodecRouteAvailable(route)) + continue; + bool headphone = route->Class == HdaRouteHeadphone; + if (endpoint->Kind == IoAudioEndpointSpeakerHeadphone) { + if (headphone && !selectedHeadphone) { + selected = routeIndex; + selectedKey = route->SortKey; + selectedHeadphone = true; + continue; + } + if (!headphone && selectedHeadphone) + continue; + } + if (selected == HDA_INVALID_INDEX || route->SortKey < selectedKey) { + selected = routeIndex; + selectedKey = route->SortKey; + selectedHeadphone = headphone; + } + } + return selected; +} + +// Converts amplifier capabilities into a millidecibel range and step. +static void HdaCodecAmpRange(uint32_t capabilities, + int32_t* minimumOut, + int32_t* maximumOut, + uint32_t* stepOut) { + uint32_t offset = capabilities & HDA_AMP_CAP_OFFSET_MASK; + uint32_t steps = (capabilities & HDA_AMP_CAP_STEPS_MASK) >> + HDA_AMP_CAP_STEPS_SHIFT; + uint32_t step = (((capabilities & HDA_AMP_CAP_STEP_SIZE_MASK) >> + HDA_AMP_CAP_STEP_SIZE_SHIFT) + 1u) * 250u; + *minimumOut = -(int32_t)(offset * step); + *maximumOut = (int32_t)((int64_t)steps * step - (int64_t)offset * step); + *stepOut = step; +} + +// Maps one millidecibel value to the closest supported amplifier gain index. +static uint8_t HdaCodecAmpGainForMilliDb(uint32_t capabilities, int32_t gainMilliDb) { + uint32_t offset = capabilities & HDA_AMP_CAP_OFFSET_MASK; + uint32_t steps = (capabilities & HDA_AMP_CAP_STEPS_MASK) >> + HDA_AMP_CAP_STEPS_SHIFT; + uint32_t step = (((capabilities & HDA_AMP_CAP_STEP_SIZE_MASK) >> + HDA_AMP_CAP_STEP_SIZE_SHIFT) + 1u) * 250u; + int64_t numerator = (int64_t)gainMilliDb + (int64_t)offset * step; + if (numerator <= 0) + return 0; + uint64_t rounded = ((uint64_t)numerator + step / 2u) / step; + if (rounded > steps) + rounded = steps; + return (uint8_t)rounded; +} + +// Programs both channels of one input or output amplifier. +static KSTATUS HdaCodecSetAmp(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t nodeId, + bool output, + uint8_t connectionIndex, + uint8_t gain, + bool muted) { + uint16_t payload = (uint16_t)(HDA_AMP_SET_LEFT | HDA_AMP_SET_RIGHT | + ((uint16_t)connectionIndex << HDA_AMP_SET_INDEX_SHIFT) | + (gain & HDA_AMP_SET_GAIN_MASK)); + payload |= output ? HDA_AMP_SET_OUTPUT : HDA_AMP_SET_INPUT; + if (muted) + payload |= HDA_AMP_SET_MUTE; + return HdaCodecCommand(state, + codec->Address, + nodeId, + HDA_VERB_SET_AMP(payload), + (uint32_t*)0); +} + +// Programs selectors and amplifiers along one route while its pin remains disabled. +static KSTATUS HdaCodecPrepareRouteLocked(HDA_CODEC_STATE* state, + HDA_ROUTE* route, + int32_t gainMilliDb, + bool muted) { + HDA_CODEC_TOPOLOGY* codec = route->Codec; + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + if (!pin || pin->FunctionIndex >= codec->FunctionCount) + return NTLSTATUS_DEVICE_ERROR; + HDA_AUDIO_FUNCTION* function = &codec->Functions[pin->FunctionIndex]; + KSTATUS status = HdaCodecPowerD0(state, codec->Address, function->NodeId); + if (!KSUCCESS(status)) + return status; + + status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_PIN_CONTROL(0), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + if ((pin->PinCapabilities & HDA_PIN_CAP_EAPD) != 0) { + status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_EAPD(0), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + + for (uint8_t pathIndex = 0; pathIndex < route->NodeCount; pathIndex++) { + HDA_WIDGET* widget = HdaCodecWidget(codec, route->Nodes[pathIndex]); + if (!widget || widget->FunctionIndex != pin->FunctionIndex) + return NTLSTATUS_DEVICE_ERROR; + if ((widget->Capabilities & HDA_WIDGET_CAP_POWER_CONTROL) != 0) { + status = HdaCodecPowerD0(state, codec->Address, widget->NodeId); + if (!KSUCCESS(status)) + return status; + } + if (pathIndex + 1u < route->NodeCount && + widget->ConnectionCount > 1u && + widget->Type != HDA_WIDGET_TYPE_MIXER) { + status = HdaCodecCommand( + state, + codec->Address, + widget->NodeId, + HDA_VERB_SET_CONNECTION_SELECT(route->ConnectionIndices[pathIndex]), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + + if ((widget->Capabilities & HDA_WIDGET_CAP_INPUT_AMP) != 0 && + pathIndex + 1u < route->NodeCount) { + uint32_t caps = widget->InputAmpCapabilities; + uint8_t zeroDb = (uint8_t)(caps & HDA_AMP_CAP_OFFSET_MASK); + for (uint16_t input = 0; input < widget->ConnectionCount; input++) { + bool inputMuted = input != route->ConnectionIndices[pathIndex] && + (caps & HDA_AMP_CAP_MUTE) != 0; + status = HdaCodecSetAmp(state, + codec, + widget->NodeId, + false, + (uint8_t)input, + zeroDb, + inputMuted); + if (!KSUCCESS(status)) + return status; + } + } + + if ((widget->Capabilities & HDA_WIDGET_CAP_OUTPUT_AMP) != 0) { + uint32_t caps = widget->OutputAmpCapabilities; + uint8_t ampGain = (uint8_t)(caps & HDA_AMP_CAP_OFFSET_MASK); + bool ampMuted = false; + if (widget->NodeId == route->MasterAmpNode) { + ampGain = HdaCodecAmpGainForMilliDb(caps, gainMilliDb); + ampMuted = muted && (caps & HDA_AMP_CAP_MUTE) != 0; + } + status = HdaCodecSetAmp(state, + codec, + widget->NodeId, + true, + 0, + ampGain, + ampMuted); + if (!KSUCCESS(status)) + return status; + } + } + + status = HdaCodecCommand(state, + codec->Address, + route->ConverterNode, + HDA_VERB_SET_CONVERTER_FORMAT(HDA_OUTPUT_STREAM_FORMAT), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + return HdaCodecCommand( + state, + codec->Address, + route->ConverterNode, + HDA_VERB_SET_CONVERTER_STREAM(state->Controller->Output.StreamTag, 0), + (uint32_t*)0); +} + +// Enables the selected pin and applies the endpoint's requested gain and mute. +static KSTATUS HdaCodecActivateRouteLocked(HDA_CODEC_STATE* state, + HDA_ROUTE* route, + int32_t gainMilliDb, + bool muted) { + HDA_CODEC_TOPOLOGY* codec = route->Codec; + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + if (!pin) + return NTLSTATUS_DEVICE_ERROR; + + if ((pin->PinCapabilities & HDA_PIN_CAP_EAPD) != 0) { + KSTATUS status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_EAPD(HDA_EAPD_ENABLE), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + uint8_t pinControl = HDA_PIN_CONTROL_OUTPUT_ENABLE; + if (route->Class == HdaRouteHeadphone || + (pin->PinCapabilities & HDA_PIN_CAP_HEADPHONE_DRIVE) != 0) + pinControl |= HDA_PIN_CONTROL_HEADPHONE_ENABLE; + KSTATUS status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_PIN_CONTROL(pinControl), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + + if (route->MasterAmpNode != HDA_INVALID_INDEX) { + uint8_t gain = HdaCodecAmpGainForMilliDb(route->MasterAmpCapabilities, + gainMilliDb); + status = HdaCodecSetAmp(state, + codec, + route->MasterAmpNode, + true, + 0, + gain, + muted); + } + return status; +} + +// Mutes and disconnects one route without stopping the controller DMA stream. +static void HdaCodecDeactivateRouteLocked(HDA_CODEC_STATE* state, HDA_ROUTE* route) { + HDA_CODEC_TOPOLOGY* codec = route->Codec; + if (route->MasterAmpNode != HDA_INVALID_INDEX && + (route->MasterAmpCapabilities & HDA_AMP_CAP_MUTE) != 0) { + uint8_t gain = HdaCodecAmpGainForMilliDb(route->MasterAmpCapabilities, 0); + (void)HdaCodecSetAmp(state, + codec, + route->MasterAmpNode, + true, + 0, + gain, + true); + } + (void)HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_PIN_CONTROL(0), + (uint32_t*)0); + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + if (pin && (pin->PinCapabilities & HDA_PIN_CAP_EAPD) != 0) { + (void)HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_EAPD(0), + (uint32_t*)0); + } + (void)HdaCodecCommand(state, + codec->Address, + route->ConverterNode, + HDA_VERB_SET_CONVERTER_STREAM(0, 0), + (uint32_t*)0); +} + +// Recomputes endpoint availability and reroutes a running combined endpoint in place. +static void HdaCodecRefreshEndpointsLocked(HDA_CODEC_STATE* state) { + for (uint8_t endpointIndex = 0; endpointIndex < state->EndpointCount; endpointIndex++) { + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[endpointIndex]; + uint8_t selectedRoute = HdaCodecChooseRoute(endpoint); + bool available = selectedRoute != HDA_INVALID_INDEX; + if (endpoint->Available != available) { + endpoint->Available = available; + if (endpoint->KernelEndpoint) + gHdaDdk->Audio->SetEndpointAvailable(endpoint->KernelEndpoint, + available ? 1u : 0u); + } + if (!endpoint->Running || endpoint->Starting || + endpoint != state->ActiveEndpoint || + endpoint->CurrentRouteIndex == selectedRoute) + continue; + + if (endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + endpoint->CurrentRouteIndex = selectedRoute; + KSTATUS status = NTLSTATUS_SUCCESS; + if (selectedRoute != HDA_INVALID_INDEX) { + HDA_ROUTE* route = &state->Routes[selectedRoute]; + status = HdaCodecPrepareRouteLocked(state, + route, + endpoint->RequestedGainMilliDb, + true); + if (KSUCCESS(status)) { + status = HdaCodecActivateRouteLocked(state, + route, + endpoint->RequestedGainMilliDb, + endpoint->RequestedMute); + } + } + if (KSUCCESS(status)) { + state->RouteChangeCount++; + } else { + state->RouteFailureCount++; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = false; + KeReleaseSpinlock(&state->CallbackLock, flags); + gHdaDdk->Debug->Print( + "HDA: jack reroute failed kind=%u route=%u status=0x%x\n", + endpoint->Kind, + selectedRoute, + status); + if (endpoint->KernelEndpoint) { + gHdaDdk->Audio->ReportEndpointFailure(endpoint->KernelEndpoint, + status, + 0); + } + } + } +} + +// Updates one pin after an event or poll and applies any logical routing change. +static void HdaCodecRefreshPinLocked(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + HDA_WIDGET* pin, + bool fromPoll) { + bool changed = false; + KSTATUS status = HdaCodecReadPinPresence(state, codec, pin, &changed); + if (!KSUCCESS(status)) { + return; + } + if (fromPoll) + state->JackPollCount++; + else + state->JackEventCount++; + if (!changed) + return; + state->JackChangeCount++; + HdaCodecRefreshEndpointsLocked(state); +} + +// Processes pending unsolicited tags and one requested polling pass. +static void HdaCodecProcessJackWork(HDA_CODEC_STATE* state) { + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return; + + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + uint64_t tags = state->PendingUnsolicitedTags; + bool poll = state->PollRequested; + state->PendingUnsolicitedTags = 0; + state->PollRequested = false; + KeReleaseSpinlock(&state->CallbackLock, flags); + + while (tags) { + uint8_t tag = 0; + while (tag < 64u && (tags & (1ull << tag)) == 0) + tag++; + if (tag == 64u) + break; + tags &= ~(1ull << tag); + uint16_t encoded = state->UnsolicitedNodes[tag]; + uint8_t codecAddress = (uint8_t)(encoded >> 8); + uint8_t nodeId = (uint8_t)encoded; + HDA_CODEC_TOPOLOGY* codec = codecAddress < HDA_MAX_CODECS + ? state->Codecs[codecAddress] + : (HDA_CODEC_TOPOLOGY*)0; + HDA_WIDGET* pin = HdaCodecWidget(codec, nodeId); + if (pin) + HdaCodecRefreshPinLocked(state, codec, pin, false); + } + + if (poll) { + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (pin && pin->PollPresence) + HdaCodecRefreshPinLocked(state, codec, pin, true); + } + } + } + KeReleaseMutex(state->Lock); +} + +// Runs immediate jack-event work at passive level. +static void HdaCodecWorkThread(uintptr_t argument) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)argument; + while (state->WorkRunning) { + KSTATUS status = KeWaitSemaphore(state->WorkSemaphore, KE_WAIT_INFINITE); + if (!KSUCCESS(status) || !state->WorkRunning) + break; + HdaCodecProcessJackWork(state); + } + KeSignalEvent(state->WorkFinished); +} + +// Requests a fallback presence polling pass every 500 milliseconds. +static void HdaCodecPollThread(uintptr_t argument) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)argument; + while (state->PollRunning) { + (void)gHdaDdk->Scheduler->DelayThread(HDA_JACK_POLL_INTERVAL_MS); + if (!state->PollRunning) + break; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + state->PollRequested = true; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeSignalSemaphore(state->WorkSemaphore); + } + KeSignalEvent(state->PollFinished); +} + +// Queues an unsolicited pin event for passive-level sense and route processing. +static void HdaCodecUnsolicitedResponse(void* context, + uint8_t codecAddress, + uint32_t response, + uint32_t responseExtra) { + (void)responseExtra; + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return; + uint8_t tag = (uint8_t)((response >> HDA_UNSOLICITED_TAG_SHIFT) & + HDA_UNSOLICITED_TAG_MASK); + if (tag == 0 || state->UnsolicitedNodes[tag] == 0) + return; + uint16_t encoded = state->UnsolicitedNodes[tag]; + if ((uint8_t)(encoded >> 8) != codecAddress) + return; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + state->PendingUnsolicitedTags |= 1ull << tag; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeSignalSemaphore(state->WorkSemaphore); +} + +// Forwards one hardware period completion to the currently active logical endpoint. +static void HdaCodecPeriodComplete(void* context, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t finePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return; + IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)0; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + if (state->ActiveEndpoint && state->ActiveEndpoint->Running) + endpoint = state->ActiveEndpoint->KernelEndpoint; + KeReleaseSpinlock(&state->CallbackLock, flags); + if (endpoint) { + gHdaDdk->Audio->NotifyPeriod(endpoint, + completedPeriodIndex, + presentedFrames, + finePositionFrames, + interruptCount, + timestampTicks); + } +} + +// Restores power, jack events, and muted route programming after controller reset. +static KSTATUS HdaCodecRestoreAfterControllerReset(void* context) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint8_t functionIndex = 0; + functionIndex < codec->FunctionCount; + functionIndex++) { + status = HdaCodecPowerD0( + state, + codecAddress, + codec->Functions[functionIndex].NodeId); + if (!KSUCCESS(status)) + break; + } + if (!KSUCCESS(status)) + break; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!pin || !pin->UnsolicitedEnabled) + continue; + status = HdaCodecCommand( + state, + codecAddress, + pin->NodeId, + HDA_VERB_SET_UNSOLICITED(HDA_UNSOLICITED_ENABLE | + pin->UnsolicitedTag), + (uint32_t*)0); + if (!KSUCCESS(status)) + break; + } + if (!KSUCCESS(status)) + break; + } + + if (KSUCCESS(status) && state->ActiveEndpoint && + state->ActiveEndpoint->Running && + state->ActiveEndpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HDA_LOGICAL_ENDPOINT* endpoint = state->ActiveEndpoint; + status = HdaCodecPrepareRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex], + endpoint->RequestedGainMilliDb, + true); + } + KeReleaseMutex(state->Lock); + return status; +} + +// Re-enables the physical route after controller recovery has restarted DMA. +static KSTATUS HdaCodecResumeAfterControllerReset(void* context) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (state->ActiveEndpoint && state->ActiveEndpoint->Running && + state->ActiveEndpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HDA_LOGICAL_ENDPOINT* endpoint = state->ActiveEndpoint; + status = HdaCodecActivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex], + endpoint->RequestedGainMilliDb, + endpoint->RequestedMute); + } + KeReleaseMutex(state->Lock); + return status; +} + +// Fails every logical endpoint after unrecoverable transport failure. +static void HdaCodecTransportFailure(void* context, KSTATUS failureStatus) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + if (state->ActiveEndpoint) + state->ActiveEndpoint->Running = false; + KeReleaseSpinlock(&state->CallbackLock, flags); + for (uint8_t index = 0; index < state->EndpointCount; index++) { + if (state->Endpoints[index].KernelEndpoint) { + gHdaDdk->Audio->ReportEndpointFailure( + state->Endpoints[index].KernelEndpoint, + failureStatus, + 0); + } + } +} + +// Starts DMA for one logical endpoint after programming its selected analog route. +static KSTATUS HdaCodecEndpointStart(void* providerContext) { + HDA_LOGICAL_ENDPOINT* endpoint = (HDA_LOGICAL_ENDPOINT*)providerContext; + if (!endpoint || !endpoint->State) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CODEC_STATE* state = endpoint->State; + if (state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + + KSTATUS status = HdaSelectOutputRing(state->Controller, endpoint->RingIndex); + if (!KSUCCESS(status)) + return status; + status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (state->ActiveEndpoint && state->ActiveEndpoint != endpoint) { + KeReleaseMutex(state->Lock); + return NTLSTATUS_WOULD_BLOCK; + } + + uint8_t routeIndex = HdaCodecChooseRoute(endpoint); + if (routeIndex == HDA_INVALID_INDEX) { + KeReleaseMutex(state->Lock); + return NTLSTATUS_NOT_FOUND; + } + endpoint->Starting = true; + endpoint->CurrentRouteIndex = routeIndex; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + state->ActiveEndpoint = endpoint; + KeReleaseSpinlock(&state->CallbackLock, flags); + status = HdaCodecPrepareRouteLocked(state, + &state->Routes[routeIndex], + endpoint->RequestedGainMilliDb, + true); + KeReleaseMutex(state->Lock); + if (!KSUCCESS(status)) + goto failed; + + status = HdaStartOutput(state->Controller); + if (!KSUCCESS(status)) + goto failed; + + status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + goto failed_after_start; + endpoint->Starting = false; + routeIndex = HdaCodecChooseRoute(endpoint); + if (routeIndex != endpoint->CurrentRouteIndex) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + endpoint->CurrentRouteIndex = routeIndex; + if (routeIndex != HDA_INVALID_INDEX) { + status = HdaCodecPrepareRouteLocked(state, + &state->Routes[routeIndex], + endpoint->RequestedGainMilliDb, + true); + } + } + if (KSUCCESS(status) && routeIndex == HDA_INVALID_INDEX) + status = NTLSTATUS_NOT_FOUND; + if (KSUCCESS(status)) { + status = HdaCodecActivateRouteLocked(state, + &state->Routes[routeIndex], + endpoint->RequestedGainMilliDb, + endpoint->RequestedMute); + } + flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = KSUCCESS(status); + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + if (KSUCCESS(status)) + return NTLSTATUS_SUCCESS; + +failed_after_start: + HdaStopOutput(state->Controller); +failed: + if (KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(state->Lock, KE_WAIT_INFINITE))) { + if (endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + } + endpoint->Starting = false; + endpoint->CurrentRouteIndex = HDA_INVALID_INDEX; + flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = false; + if (state->ActiveEndpoint == endpoint) + state->ActiveEndpoint = (HDA_LOGICAL_ENDPOINT*)0; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + } + return status; +} + +// Mutes one logical endpoint and stops the shared output stream. +static void HdaCodecEndpointStop(void* providerContext) { + HDA_LOGICAL_ENDPOINT* endpoint = (HDA_LOGICAL_ENDPOINT*)providerContext; + if (!endpoint || !endpoint->State) + return; + HDA_CODEC_STATE* state = endpoint->State; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + if (!state->ShuttingDown && + state->ActiveEndpoint == endpoint && + endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + } + endpoint->Starting = false; + endpoint->CurrentRouteIndex = HDA_INVALID_INDEX; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = false; + if (state->ActiveEndpoint == endpoint) + state->ActiveEndpoint = (HDA_LOGICAL_ENDPOINT*)0; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + } + if (!state->ShuttingDown || !state->Controller->SurpriseRemoved) + HdaStopOutput(state->Controller); +} + +// Stores or applies one endpoint hardware gain and mute request. +static KSTATUS HdaCodecEndpointSetGain(void* providerContext, + int32_t gainMilliDb, + uint8_t muted) { + HDA_LOGICAL_ENDPOINT* endpoint = (HDA_LOGICAL_ENDPOINT*)providerContext; + if (!endpoint || !endpoint->State || !endpoint->HasHardwareGain) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CODEC_STATE* state = endpoint->State; + if (state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + endpoint->RequestedGainMilliDb = gainMilliDb; + endpoint->RequestedMute = muted != 0; + if (endpoint->Running && state->ActiveEndpoint == endpoint && + endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HDA_ROUTE* route = &state->Routes[endpoint->CurrentRouteIndex]; + uint8_t gain = HdaCodecAmpGainForMilliDb(route->MasterAmpCapabilities, + gainMilliDb); + status = HdaCodecSetAmp(state, + route->Codec, + route->MasterAmpNode, + true, + 0, + gain, + muted != 0); + } + KeReleaseMutex(state->Lock); + return status; +} + +// Configures initial jack state and chooses event or polling delivery per pin. +static KSTATUS HdaCodecInitializeJacks(HDA_CODEC_STATE* state) { + uint8_t nextTag = 1; + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!HdaCodecPinIsAnalogOutput(pin)) + continue; + bool changed = false; + KSTATUS status = HdaCodecReadPinPresence(state, codec, pin, &changed); + if (!KSUCCESS(status)) + return status; + if ((pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) == 0) + continue; + + bool eventCapable = (pin->Capabilities & HDA_WIDGET_CAP_UNSOLICITED) != 0 && + nextTag <= HDA_UNSOLICITED_TAG_MASK; + if (eventCapable) { + status = HdaCodecCommand( + state, + codecAddress, + pin->NodeId, + HDA_VERB_SET_UNSOLICITED(HDA_UNSOLICITED_ENABLE | nextTag), + (uint32_t*)0); + if (KSUCCESS(status)) { + pin->UnsolicitedEnabled = true; + pin->UnsolicitedTag = nextTag; + state->UnsolicitedNodes[nextTag] = + (uint16_t)(((uint16_t)codecAddress << 8) | pin->NodeId); + nextTag++; + } + } + // Polling remains active as a watchdog because event-capable pins can lose notifications. + pin->PollPresence = true; + state->HavePollingPins = true; + } + } + HdaCodecRefreshEndpointsLocked(state); + return NTLSTATUS_SUCCESS; +} + +// Starts the passive jack worker and optional 500 millisecond polling ticker. +static KSTATUS HdaCodecStartWorkers(HDA_CODEC_STATE* state) { + KSTATUS status = KeSemaphoreCreate(0, 1, &state->WorkSemaphore); + if (!KSUCCESS(status)) + return status; + status = KeEventCreate(EventManualReset, 0, &state->WorkFinished); + if (!KSUCCESS(status)) + return status; + status = KeEventCreate(EventManualReset, 0, &state->PollFinished); + if (!KSUCCESS(status)) + return status; + + state->WorkRunning = 1; + KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + status = KeCreateKernelThread((uintptr_t)HdaCodecWorkThread, + (uintptr_t)state, + ¶meters, + &state->WorkThread); + if (!KSUCCESS(status)) { + state->WorkRunning = 0; + return status; + } + if (!state->HavePollingPins) + return NTLSTATUS_SUCCESS; + + state->PollRunning = 1; + status = KeCreateKernelThread((uintptr_t)HdaCodecPollThread, + (uintptr_t)state, + ¶meters, + &state->PollThread); + if (!KSUCCESS(status)) { + state->PollRunning = 0; + state->WorkRunning = 0; + KeSignalSemaphore(state->WorkSemaphore); + (void)KeWaitForEvent(state->WorkFinished, 0); + ObDereferenceObject((OBJECT*)state->WorkThread); + state->WorkThread = (THREAD*)0; + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Registers every discovered logical endpoint against its dedicated DMA ring. +static KSTATUS HdaCodecRegisterEndpoints(HDA_CODEC_STATE* state) { + for (uint8_t index = 0; index < state->EndpointCount; index++) { + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[index]; + HDA_OUTPUT_DESCRIPTION output; + KSTATUS status = HdaDescribeOutput(state->Controller, + endpoint->RingIndex, + &output); + if (!KSUCCESS(status)) + return status; + + IO_AUDIO_ENDPOINT_REGISTRATION registration; + memset(®istration, 0, sizeof(registration)); + registration.Device = state->Controller->Fdo; + registration.Kind = endpoint->Kind; + registration.InitiallyAvailable = endpoint->Available ? 1u : 0u; + registration.ProviderContext = endpoint; + registration.Operations.Start = HdaCodecEndpointStart; + registration.Operations.Stop = HdaCodecEndpointStop; + registration.PeriodBuffer = output.PeriodBuffer; + registration.PeriodBufferBytes = output.PeriodBufferBytes; + registration.PeriodCount = output.PeriodCount; + registration.FramesPerPeriod = output.FramesPerPeriod; + registration.PeriodStrideBytes = output.PeriodStrideBytes; + registration.HardwareFormat = output.HardwareFormat; + if (endpoint->HasHardwareGain) { + registration.Capabilities |= IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN; + registration.Operations.SetGain = HdaCodecEndpointSetGain; + HdaCodecAmpRange(endpoint->HardwareAmpCapabilities, + ®istration.HardwareGainMinimumMilliDb, + ®istration.HardwareGainMaximumMilliDb, + ®istration.HardwareGainStepMilliDb); + } + for (uint8_t routeOffset = 0; routeOffset < endpoint->RouteCount; routeOffset++) { + HDA_ROUTE* route = &state->Routes[endpoint->RouteIndices[routeOffset]]; + HDA_WIDGET* pin = HdaCodecWidget(route->Codec, route->PinNode); + if (pin && (pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) != 0) { + registration.Capabilities |= IO_AUDIO_ENDPOINT_CAP_JACK_SENSE; + break; + } + } + + status = gHdaDdk->Audio->RegisterEndpoint(®istration, + &endpoint->KernelEndpoint); + if (!KSUCCESS(status)) + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Stops worker threads and releases their thread references. +static void HdaCodecStopWorkers(HDA_CODEC_STATE* state) { + if (!state) + return; + state->PollRunning = 0; + state->WorkRunning = 0; + if (state->WorkSemaphore) + KeSignalSemaphore(state->WorkSemaphore); + if (state->PollThread && state->PollFinished) + (void)KeWaitForEvent(state->PollFinished, 0); + if (state->WorkThread && state->WorkFinished) + (void)KeWaitForEvent(state->WorkFinished, 0); + if (state->PollThread) { + ObDereferenceObject((OBJECT*)state->PollThread); + state->PollThread = (THREAD*)0; + } + if (state->WorkThread) { + ObDereferenceObject((OBJECT*)state->WorkThread); + state->WorkThread = (THREAD*)0; + } +} + +// Discovers codec topology and registers usable analog render endpoints. +KSTATUS HdaCodecInitialize(HDA_CONTROLLER* controller) { + if (!controller || !gHdaDdk || !gHdaDdk->Audio || !gHdaDdk->Scheduler) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)MmAllocatePoolTag( + sizeof(HDA_CODEC_STATE), + HDA_CODEC_POOL_TAG); + if (!state) + return NTLSTATUS_NO_MEMORY; + memset(state, 0, sizeof(*state)); + state->Controller = controller; + state->CallbackLock = (SPINLOCK)SPINLOCK_INIT; + controller->Codec = state; + for (uint8_t index = 0; index < HDA_MAX_LOGICAL_ENDPOINTS; index++) + state->Endpoints[index].CurrentRouteIndex = HDA_INVALID_INDEX; + + KSTATUS status = KeMutexCreate(&state->Lock); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecEnumerateAll(state); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecBuildRoutes(state); + if (!KSUCCESS(status)) { + gHdaDdk->Debug->Print( + "HDA: no usable 48000 Hz, stereo, 16-bit analog converter-to-pin route\n"); + goto failed; + } + status = HdaCodecBuildEndpoints(state); + if (!KSUCCESS(status)) + goto failed; + + status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + goto failed; + status = HdaCodecInitializeJacks(state); + KeReleaseMutex(state->Lock); + if (!KSUCCESS(status)) + goto failed; + + HDA_TRANSPORT_CALLBACKS callbacks; + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.PeriodComplete = HdaCodecPeriodComplete; + callbacks.UnsolicitedResponse = HdaCodecUnsolicitedResponse; + callbacks.RestoreAfterControllerReset = HdaCodecRestoreAfterControllerReset; + callbacks.ResumeAfterControllerReset = HdaCodecResumeAfterControllerReset; + callbacks.TransportFailure = HdaCodecTransportFailure; + status = HdaSetTransportCallbacks(controller, &callbacks, state); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecStartWorkers(state); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecRegisterEndpoints(state); + if (!KSUCCESS(status)) + goto failed; + + return NTLSTATUS_SUCCESS; + +failed: + HdaCodecStop(controller, IoAudioOrderlyRemoval); + HdaCodecDestroy(controller); + return status; +} + +// Stops jack workers and unregisters codec endpoints. +void HdaCodecStop(HDA_CONTROLLER* controller, IO_AUDIO_REMOVAL_MODE mode) { + if (!controller || !controller->Codec) + return; + HDA_CODEC_STATE* state = controller->Codec; + state->ShuttingDown = 1; + HdaCodecStopWorkers(state); + + if (mode == IoAudioOrderlyRemoval && state->Lock) { + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + HDA_LOGICAL_ENDPOINT* active = state->ActiveEndpoint; + if (active && active->CurrentRouteIndex != HDA_INVALID_INDEX) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[active->CurrentRouteIndex]); + active->CurrentRouteIndex = HDA_INVALID_INDEX; + } + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + if (active) + active->Running = false; + state->ActiveEndpoint = (HDA_LOGICAL_ENDPOINT*)0; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + } + } + + for (uint8_t index = state->EndpointCount; index > 0; index--) { + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[index - 1u]; + if (!endpoint->KernelEndpoint) + continue; + gHdaDdk->Audio->UnregisterEndpoint(endpoint->KernelEndpoint, mode); + endpoint->KernelEndpoint = (IO_AUDIO_ENDPOINT*)0; + } + + if (mode == IoAudioOrderlyRemoval && state->Lock) { + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!pin || !pin->UnsolicitedEnabled) + continue; + (void)HdaCodecCommand(state, + codecAddress, + pin->NodeId, + HDA_VERB_SET_UNSOLICITED(0), + (uint32_t*)0); + pin->UnsolicitedEnabled = false; + } + } + KeReleaseMutex(state->Lock); + } + (void)HdaSetTransportCallbacks(controller, + (const HDA_TRANSPORT_CALLBACKS*)0, + (void*)0); + } +} + +// Releases codec topology storage after interrupt work is quiescent. +void HdaCodecDestroy(HDA_CONTROLLER* controller) { + if (!controller || !controller->Codec) + return; + HDA_CODEC_STATE* state = controller->Codec; + controller->Codec = (HDA_CODEC_STATE*)0; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + if (controller->CallbackContext == state) { + memset(&controller->Callbacks, 0, sizeof(controller->Callbacks)); + controller->CallbackContext = (void*)0; + } + KeReleaseSpinlock(&controller->StateLock, flags); + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + MmFreePool(state->Codecs[codecAddress]); + state->Codecs[codecAddress] = (HDA_CODEC_TOPOLOGY*)0; + } + if (state->WorkSemaphore) + ObDereferenceObject(&state->WorkSemaphore->Header); + if (state->WorkFinished) + ObDereferenceObject(&state->WorkFinished->Header); + if (state->PollFinished) + ObDereferenceObject(&state->PollFinished->Header); + if (state->Lock) + ObDereferenceObject(&state->Lock->Header); + MmFreePool(state); +} diff --git a/code/drivers/audio/hda/hda_transport.c b/code/drivers/audio/hda/hda_transport.c new file mode 100644 index 0000000..d1af379 --- /dev/null +++ b/code/drivers/audio/hda/hda_transport.c @@ -0,0 +1,1421 @@ +// Intel High Definition Audio controller transport, command rings, output DMA, and recovery. + +#include "hda.h" + +#include +#include + +// Orders CPU stores before a controller ownership transition. +static __forceinline void HdaMemoryBarrier(void) { + _ReadWriteBarrier(); + _mm_mfence(); + _ReadWriteBarrier(); +} + +// Returns the controller's monotonic microsecond counter. +static uint64_t HdaQueryTicks(void) { + if (!gHdaDdk || !gHdaDdk->Timer || !gHdaDdk->Timer->QueryTicks) + return 0; + return gHdaDdk->Timer->QueryTicks(); +} + +// Delays for a bounded controller settling interval. +static void HdaStall(uint32_t microseconds) { + if (gHdaDdk && gHdaDdk->Timer && gHdaDdk->Timer->Stall) + gHdaDdk->Timer->Stall(microseconds); +} + +// Returns one byte from the controller register aperture. +uint8_t HdaRead8(const HDA_CONTROLLER* controller, uint32_t offset) { + return *(volatile const uint8_t*)(controller->Registers + offset); +} + +// Returns one word from the controller register aperture. +uint16_t HdaRead16(const HDA_CONTROLLER* controller, uint32_t offset) { + return *(volatile const uint16_t*)(controller->Registers + offset); +} + +// Returns one dword from the controller register aperture. +uint32_t HdaRead32(const HDA_CONTROLLER* controller, uint32_t offset) { + return *(volatile const uint32_t*)(controller->Registers + offset); +} + +// Writes one byte to the controller register aperture. +void HdaWrite8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t value) { + *(volatile uint8_t*)(controller->Registers + offset) = value; +} + +// Writes one word to the controller register aperture. +void HdaWrite16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t value) { + *(volatile uint16_t*)(controller->Registers + offset) = value; +} + +// Writes one dword to the controller register aperture. +void HdaWrite32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t value) { + *(volatile uint32_t*)(controller->Registers + offset) = value; +} + +// Returns the register offset of the controller's one output stream descriptor. +uint32_t HdaOutputStreamBase(const HDA_CONTROLLER* controller) { + return HDA_REG_STREAM_BASE + + (uint32_t)controller->Output.DescriptorIndex * HDA_REG_STREAM_STRIDE; +} + +// Reads the 24-bit stream control register without crossing into status. +static uint32_t HdaReadStreamControl(const HDA_CONTROLLER* controller) { + uint32_t base = HdaOutputStreamBase(controller); + return (uint32_t)HdaRead16(controller, base + HDA_STREAM_REG_CTL) | + ((uint32_t)HdaRead8(controller, base + HDA_STREAM_REG_CTL + 2u) << 16); +} + +// Writes the 24-bit stream control register without modifying status. +static void HdaWriteStreamControl(HDA_CONTROLLER* controller, uint32_t value) { + uint32_t base = HdaOutputStreamBase(controller); + HdaWrite16(controller, base + HDA_STREAM_REG_CTL, (uint16_t)value); + HdaWrite8(controller, base + HDA_STREAM_REG_CTL + 2u, (uint8_t)(value >> 16)); +} + +// Waits until selected bits in one dword register match the requested state. +static KSTATUS HdaWait32(HDA_CONTROLLER* controller, + uint32_t offset, + uint32_t mask, + uint32_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaRead32(controller, offset) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Waits until selected bits in one byte register match the requested state. +static KSTATUS HdaWait8(HDA_CONTROLLER* controller, + uint32_t offset, + uint8_t mask, + uint8_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaRead8(controller, offset) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Waits until selected bits in one word register match the requested state. +static KSTATUS HdaWait16(HDA_CONTROLLER* controller, + uint32_t offset, + uint16_t mask, + uint16_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaRead16(controller, offset) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Waits until selected bits in the output stream control register match. +static KSTATUS HdaWaitStreamControl(HDA_CONTROLLER* controller, + uint32_t mask, + uint32_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaReadStreamControl(controller) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Selects the largest command-ring size advertised by the controller. +static KSTATUS HdaSelectRingSize(uint8_t capability, uint16_t* countOut) { + if (!countOut) + return NTLSTATUS_INVALID_PARAMETER; + if (capability & HDA_RING_SIZE_CAP_256) { + *countOut = 256u; + return NTLSTATUS_SUCCESS; + } + if (capability & HDA_RING_SIZE_CAP_16) { + *countOut = 16u; + return NTLSTATUS_SUCCESS; + } + if (capability & HDA_RING_SIZE_CAP_2) { + *countOut = 2u; + return NTLSTATUS_SUCCESS; + } + return NTLSTATUS_NOT_SUPPORTED; +} + +// Returns the hardware selector encoding for a supported ring entry count. +static uint8_t HdaRingSizeSelector(uint16_t count) { + if (count == 256u) + return HDA_RING_SIZE_SELECT_256; + if (count == 16u) + return HDA_RING_SIZE_SELECT_16; + return HDA_RING_SIZE_SELECT_2; +} + +// Releases all DMA and synchronization resources owned by one controller. +void HdaReleaseDma(HDA_CONTROLLER* controller) { + if (!controller || !gHdaDdk || !gHdaDdk->Dma) + return; + + if (controller->StreamMutex) { + ObDereferenceObject(&controller->StreamMutex->Header); + controller->StreamMutex = (MUTEX*)0; + } + if (controller->CommandMutex) { + ObDereferenceObject(&controller->CommandMutex->Header); + controller->CommandMutex = (MUTEX*)0; + } + if (controller->RecoverySemaphore) { + ObDereferenceObject(&controller->RecoverySemaphore->Header); + controller->RecoverySemaphore = (SEMAPHORE*)0; + } + if (controller->RecoveryFinished) { + ObDereferenceObject(&controller->RecoveryFinished->Header); + controller->RecoveryFinished = (EVENT*)0; + } + + gHdaDdk->Dma->FreeCommonBuffer(controller->Output.BdlDma); + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) + gHdaDdk->Dma->FreeCommonBuffer(controller->Output.Rings[ringIndex].Dma); + gHdaDdk->Dma->FreeCommonBuffer(controller->PositionDma); + gHdaDdk->Dma->FreeCommonBuffer(controller->RirbDma); + gHdaDdk->Dma->FreeCommonBuffer(controller->CorbDma); + controller->Output.BdlDma = (IO_DMA_BUFFER*)0; + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { + controller->Output.Rings[ringIndex].Dma = (IO_DMA_BUFFER*)0; + controller->Output.Rings[ringIndex].Buffer = (uint8_t*)0; + controller->Output.Rings[ringIndex].DeviceAddress = 0; + } + controller->PositionDma = (IO_DMA_BUFFER*)0; + controller->RirbDma = (IO_DMA_BUFFER*)0; + controller->CorbDma = (IO_DMA_BUFFER*)0; + controller->Output.Bdl = (HDA_BDL_ENTRY*)0; + controller->PositionBuffer = (volatile uint64_t*)0; + controller->Rirb = (HDA_RIRB_ENTRY*)0; + controller->Corb = (uint32_t*)0; + + if (controller->DmaAdapter) { + gHdaDdk->Dma->PutAdapter(controller->DmaAdapter); + controller->DmaAdapter = (IO_DMA_ADAPTER*)0; + } +} + +// Allocates the transport and output DMA resources for one controller. +KSTATUS HdaAllocateDma(HDA_CONTROLLER* controller) { + if (!controller || !controller->Fdo || !gHdaDdk || !gHdaDdk->Dma) + return NTLSTATUS_INVALID_PARAMETER; + + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = (controller->Gcap & HDA_GCAP_64OK) + ? UINT64_MAX + : 0xFFFFFFFFull; + requirements.Alignment = 128u; + requirements.MaximumSegments = 1u; + + KSTATUS status = gHdaDdk->Dma->GetAdapter(controller->Fdo, + &requirements, + &controller->DmaAdapter); + if (!KSUCCESS(status)) + return status; + + status = HdaSelectRingSize(HdaRead8(controller, HDA_REG_CORBSIZE), + &controller->CorbEntryCount); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = HdaSelectRingSize(HdaRead8(controller, HDA_REG_RIRBSIZE), + &controller->RirbEntryCount); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + (size_t)controller->CorbEntryCount * sizeof(uint32_t), + 128u, + &controller->CorbDma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + (size_t)controller->RirbEntryCount * sizeof(HDA_RIRB_ENTRY), + 128u, + &controller->RirbDma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + // The position buffer is optional because LPIB is the supported fallback. + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + (size_t)controller->TotalStreamCount * sizeof(uint64_t), + 128u, + &controller->PositionDma); + if (!KSUCCESS(status)) + controller->PositionDma = (IO_DMA_BUFFER*)0; + + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { + status = gHdaDdk->Dma->AllocateCommonBuffer( + controller->DmaAdapter, + HDA_OUTPUT_BUFFER_BYTES, + 128u, + &controller->Output.Rings[ringIndex].Dma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + } + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + HDA_OUTPUT_PERIOD_COUNT * sizeof(HDA_BDL_ENTRY), + 128u, + &controller->Output.BdlDma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + controller->Corb = (uint32_t*)gHdaDdk->Dma->GetCommonBufferAddress(controller->CorbDma); + controller->Rirb = (HDA_RIRB_ENTRY*)gHdaDdk->Dma->GetCommonBufferAddress(controller->RirbDma); + controller->CorbDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->CorbDma); + controller->RirbDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->RirbDma); + if (controller->PositionDma) { + controller->PositionBuffer = (volatile uint64_t*) + gHdaDdk->Dma->GetCommonBufferAddress(controller->PositionDma); + controller->PositionDeviceAddress = + gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->PositionDma); + } + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { + controller->Output.Rings[ringIndex].Buffer = (uint8_t*) + gHdaDdk->Dma->GetCommonBufferAddress(controller->Output.Rings[ringIndex].Dma); + controller->Output.Rings[ringIndex].DeviceAddress = + gHdaDdk->Dma->GetCommonBufferDeviceAddress( + controller->Output.Rings[ringIndex].Dma); + } + controller->Output.Bdl = (HDA_BDL_ENTRY*) + gHdaDdk->Dma->GetCommonBufferAddress(controller->Output.BdlDma); + controller->Output.BdlDeviceAddress = + gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->Output.BdlDma); + + memset(controller->Corb, 0, (size_t)controller->CorbEntryCount * sizeof(uint32_t)); + memset(controller->Rirb, 0, (size_t)controller->RirbEntryCount * sizeof(HDA_RIRB_ENTRY)); + if (controller->PositionBuffer) + memset((void*)controller->PositionBuffer, 0, + (size_t)controller->TotalStreamCount * sizeof(uint64_t)); + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) + memset(controller->Output.Rings[ringIndex].Buffer, 0, HDA_OUTPUT_BUFFER_BYTES); + memset(controller->Output.Bdl, 0, + HDA_OUTPUT_PERIOD_COUNT * sizeof(HDA_BDL_ENTRY)); + + status = KeMutexCreate(&controller->CommandMutex); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = KeMutexCreate(&controller->StreamMutex); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = KeSemaphoreCreate(0, 1, &controller->RecoverySemaphore); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = KeEventCreate(EventManualReset, 0, &controller->RecoveryFinished); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + return NTLSTATUS_SUCCESS; +} + +// Stops the output stream and waits for the run bit to clear. +static KSTATUS HdaStopOutputHardware(HDA_CONTROLLER* controller) { + uint32_t control = HdaReadStreamControl(controller); + control &= ~(HDA_STREAM_CTL_RUN | HDA_STREAM_CTL_INTERRUPT_MASK); + HdaWriteStreamControl(controller, control); + return HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RUN, + 0, + HDA_STREAM_RESET_TIMEOUT_US); +} + +// Resets the one output stream descriptor to its architected idle state. +static KSTATUS HdaResetOutputDescriptor(HDA_CONTROLLER* controller) { + KSTATUS status = HdaStopOutputHardware(controller); + if (!KSUCCESS(status)) + return status; + + uint32_t control = HdaReadStreamControl(controller) | HDA_STREAM_CTL_RESET; + HdaWriteStreamControl(controller, control); + status = HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RESET, + HDA_STREAM_CTL_RESET, + HDA_STREAM_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + + control &= ~HDA_STREAM_CTL_RESET; + HdaWriteStreamControl(controller, control); + status = HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RESET, + 0, + HDA_STREAM_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + + HdaWrite8(controller, + HdaOutputStreamBase(controller) + HDA_STREAM_REG_STS, + HDA_STREAM_STS_INTERRUPT_MASK); + return NTLSTATUS_SUCCESS; +} + +// Builds a four-entry BDL beginning with the active ring period owned by software. +static void HdaBuildOutputBdl(HDA_CONTROLLER* controller) { + HDA_OUTPUT_RING* ring = + &controller->Output.Rings[controller->Output.ActiveRingIndex]; + for (uint32_t bdlIndex = 0; bdlIndex < HDA_OUTPUT_PERIOD_COUNT; bdlIndex++) { + uint8_t period = (uint8_t)((ring->NextLogicalPeriod + bdlIndex) % + HDA_OUTPUT_PERIOD_COUNT); + controller->Output.BdlPeriodMap[bdlIndex] = period; + controller->Output.Bdl[bdlIndex].Address = + ring->DeviceAddress + + (uint64_t)period * HDA_OUTPUT_PERIOD_BYTES; + controller->Output.Bdl[bdlIndex].Length = HDA_OUTPUT_PERIOD_BYTES; + controller->Output.Bdl[bdlIndex].Flags = HDA_BDL_FLAG_INTERRUPT_ON_COMPLETION; + } + controller->Output.NextCompletionBdl = 0; + HdaMemoryBarrier(); +} + +// Programs the fixed-format output descriptor without starting DMA. +static KSTATUS HdaProgramOutputDescriptor(HDA_CONTROLLER* controller) { + KSTATUS status = HdaResetOutputDescriptor(controller); + if (!KSUCCESS(status)) + return status; + + if (controller->PositionBuffer) + controller->PositionBuffer[controller->Output.DescriptorIndex] = 0; + HdaBuildOutputBdl(controller); + uint32_t base = HdaOutputStreamBase(controller); + HdaWrite32(controller, base + HDA_STREAM_REG_CBL, HDA_OUTPUT_BUFFER_BYTES); + HdaWrite16(controller, base + HDA_STREAM_REG_LVI, + (uint16_t)(HDA_OUTPUT_PERIOD_COUNT - 1u)); + HdaWrite16(controller, base + HDA_STREAM_REG_FORMAT, HDA_OUTPUT_STREAM_FORMAT); + HdaWrite32(controller, base + HDA_STREAM_REG_BDLPL, + (uint32_t)controller->Output.BdlDeviceAddress); + HdaWrite32(controller, base + HDA_STREAM_REG_BDLPU, + (uint32_t)(controller->Output.BdlDeviceAddress >> 32)); + + uint32_t control = ((uint32_t)controller->Output.StreamTag << HDA_STREAM_CTL_TAG_SHIFT) | + HDA_STREAM_CTL_INTERRUPT_COMPLETE | + HDA_STREAM_CTL_INTERRUPT_FIFO_ERROR | + HDA_STREAM_CTL_INTERRUPT_DESCRIPTOR_ERROR; + HdaWriteStreamControl(controller, control); + controller->Output.Prepared = true; + return NTLSTATUS_SUCCESS; +} + +// Resets the controller global state after all DMA engines are stopped. +static KSTATUS HdaResetControllerHardware(HDA_CONTROLLER* controller) { + HdaWrite32(controller, HDA_REG_INTCTL, 0); + (void)HdaStopOutputHardware(controller); + HdaWrite8(controller, HDA_REG_CORBCTL, 0); + HdaWrite8(controller, HDA_REG_RIRBCTL, 0); + + uint32_t gctl = HdaRead32(controller, HDA_REG_GCTL); + HdaWrite32(controller, HDA_REG_GCTL, gctl & ~HDA_GCTL_RESET); + KSTATUS status = HdaWait32(controller, + HDA_REG_GCTL, + HDA_GCTL_RESET, + 0, + HDA_CONTROLLER_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + + HdaStall(100); + HdaWrite32(controller, HDA_REG_GCTL, HDA_GCTL_RESET); + status = HdaWait32(controller, + HDA_REG_GCTL, + HDA_GCTL_RESET, + HDA_GCTL_RESET, + HDA_CONTROLLER_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + HdaStall(HDA_CODEC_RESET_SETTLE_US); + return NTLSTATUS_SUCCESS; +} + +// Resets the CORB read pointer while accepting controllers that self-clear the reset bit. +static KSTATUS HdaResetCorbReadPointer(HDA_CONTROLLER* controller) { + HdaWrite16(controller, HDA_REG_CORBRP, HDA_CORBRP_RESET); + + for (uint32_t elapsed = 0; elapsed < HDA_RING_RESET_TIMEOUT_US; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + + uint16_t value = HdaRead16(controller, HDA_REG_CORBRP); + if ((value & HDA_CORBRP_RESET) != 0) { + HdaWrite16(controller, HDA_REG_CORBRP, 0); + return HdaWait16(controller, + HDA_REG_CORBRP, + HDA_CORBRP_RESET, + 0, + HDA_RING_RESET_TIMEOUT_US); + } + + // Some controllers complete the reset before software can observe the + // reset bit. A zero read pointer proves that the requested state was + // reached, so requiring an intermediate asserted state is incorrect. + if ((value & HDA_CORBRP_POINTER_MASK) == 0) { + HdaWrite16(controller, HDA_REG_CORBRP, 0); + return NTLSTATUS_SUCCESS; + } + + HdaStall(1); + } + + return NTLSTATUS_TIMEOUT; +} + +// Reports the register state that owns a controller initialization failure. +static KSTATUS HdaReportInitializationFailure(HDA_CONTROLLER* controller, + const char* stage, + KSTATUS status) { + gHdaDdk->Debug->Print( + "HDA: init stage=%s failed status=0x%x GCTL=0x%08x STATESTS=0x%04x " + "CORB=%02x/%02x/%04x/%04x RIRB=%02x/%02x/%04x SDCTL=0x%06x\n", + stage, + status, + HdaRead32(controller, HDA_REG_GCTL), + HdaRead16(controller, HDA_REG_STATESTS), + HdaRead8(controller, HDA_REG_CORBCTL), + HdaRead8(controller, HDA_REG_CORBSTS), + HdaRead16(controller, HDA_REG_CORBWP), + HdaRead16(controller, HDA_REG_CORBRP), + HdaRead8(controller, HDA_REG_RIRBCTL), + HdaRead8(controller, HDA_REG_RIRBSTS), + HdaRead16(controller, HDA_REG_RIRBWP), + HdaReadStreamControl(controller)); + return status; +} + +// Initializes CORB and RIRB with software pointers synchronized to hardware. +static KSTATUS HdaInitializeCommandRings(HDA_CONTROLLER* controller) { + HdaWrite8(controller, HDA_REG_CORBCTL, 0); + HdaWrite8(controller, HDA_REG_RIRBCTL, 0); + KSTATUS status = HdaWait8(controller, + HDA_REG_CORBCTL, + HDA_CORBCTL_RUN, + 0, + HDA_RING_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + status = HdaWait8(controller, + HDA_REG_RIRBCTL, + HDA_RIRBCTL_RUN, + 0, + HDA_RING_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + HdaWrite8(controller, HDA_REG_CORBSTS, HDA_CORBSTS_MEMORY_ERROR); + HdaWrite8(controller, HDA_REG_RIRBSTS, + HDA_RIRBSTS_RESPONSE | HDA_RIRBSTS_OVERRUN); + + HdaWrite32(controller, HDA_REG_CORBLBASE, (uint32_t)controller->CorbDeviceAddress); + HdaWrite32(controller, HDA_REG_CORBUBASE, + (uint32_t)(controller->CorbDeviceAddress >> 32)); + HdaWrite8(controller, HDA_REG_CORBSIZE, + HdaRingSizeSelector(controller->CorbEntryCount)); + HdaWrite16(controller, HDA_REG_CORBWP, 0); + status = HdaResetCorbReadPointer(controller); + if (!KSUCCESS(status)) + return status; + + HdaWrite32(controller, HDA_REG_RIRBLBASE, (uint32_t)controller->RirbDeviceAddress); + HdaWrite32(controller, HDA_REG_RIRBUBASE, + (uint32_t)(controller->RirbDeviceAddress >> 32)); + HdaWrite8(controller, HDA_REG_RIRBSIZE, + HdaRingSizeSelector(controller->RirbEntryCount)); + HdaWrite16(controller, HDA_REG_RIRBWP, HDA_RIRBWP_RESET); + HdaWrite16(controller, HDA_REG_RINTCNT, 1); + + controller->CorbWritePointer = 0; + controller->RirbReadPointer = 0; + controller->CommandWaiting = false; + controller->CommandResponseReady = false; + HdaMemoryBarrier(); + HdaWrite8(controller, HDA_REG_CORBCTL, + HDA_CORBCTL_MEMORY_ERROR_INTERRUPT | HDA_CORBCTL_RUN); + HdaWrite8(controller, HDA_REG_RIRBCTL, + HDA_RIRBCTL_RESPONSE_INTERRUPT | + HDA_RIRBCTL_OVERRUN_INTERRUPT | + HDA_RIRBCTL_RUN); + status = HdaWait8(controller, + HDA_REG_CORBCTL, + HDA_CORBCTL_RUN, + HDA_CORBCTL_RUN, + HDA_RING_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + return HdaWait8(controller, + HDA_REG_RIRBCTL, + HDA_RIRBCTL_RUN, + HDA_RIRBCTL_RUN, + HDA_RING_RESET_TIMEOUT_US); +} + +// Queues one unsolicited response for DPC-level topology handling. +static void HdaQueueUnsolicitedLocked(HDA_CONTROLLER* controller, + uint32_t response, + uint32_t responseExtra) { + if (controller->UnsolicitedCount == HDA_MAX_UNSOLICITED_RESPONSES) { + controller->UnsolicitedRead = (uint8_t)((controller->UnsolicitedRead + 1u) % + HDA_MAX_UNSOLICITED_RESPONSES); + controller->UnsolicitedCount--; + } + controller->Unsolicited[controller->UnsolicitedWrite].Response = response; + controller->Unsolicited[controller->UnsolicitedWrite].ResponseExtra = responseExtra; + controller->UnsolicitedWrite = (uint8_t)((controller->UnsolicitedWrite + 1u) % + HDA_MAX_UNSOLICITED_RESPONSES); + controller->UnsolicitedCount++; + controller->RirbUnsolicitedCount++; +} + +// Drains every hardware-owned RIRB entry while the controller lock is held. +static void HdaDrainRirbLocked(HDA_CONTROLLER* controller) { + uint16_t hardwareWritePointer = (uint16_t)(HdaRead16(controller, HDA_REG_RIRBWP) & 0x00FFu); + while (controller->RirbReadPointer != hardwareWritePointer) { + controller->RirbReadPointer = + (uint16_t)((controller->RirbReadPointer + 1u) % controller->RirbEntryCount); + HDA_RIRB_ENTRY entry = controller->Rirb[controller->RirbReadPointer]; + controller->RirbResponseCount++; + + if (entry.ResponseExtra & HDA_RIRB_RESPONSE_UNSOLICITED) { + HdaQueueUnsolicitedLocked(controller, entry.Response, entry.ResponseExtra); + continue; + } + + uint8_t codecAddress = (uint8_t)(entry.ResponseExtra & HDA_RIRB_RESPONSE_CODEC_MASK); + if (controller->CommandWaiting && + codecAddress == controller->CommandCodecAddress && + !controller->CommandResponseReady) { + controller->CommandResponse = entry.Response; + controller->CommandResponseExtra = entry.ResponseExtra; + controller->CommandResponseReady = true; + } + } +} + +// Drains RIRB responses and acknowledges response or overrun status. +static uint8_t HdaServiceRirbLocked(HDA_CONTROLLER* controller) { + uint8_t status = HdaRead8(controller, HDA_REG_RIRBSTS); + if (status & HDA_RIRBSTS_RESPONSE) + HdaDrainRirbLocked(controller); + if (status & HDA_RIRBSTS_OVERRUN) { + controller->RirbOverrunCount++; + if (controller->Started) + controller->PendingRecovery = HDA_RECOVERY_CONTROLLER; + } + if (status) + HdaWrite8(controller, HDA_REG_RIRBSTS, status); + return status; +} + +// Sends one codec verb attempt while the command mutex is held. +static KSTATUS HdaCodecVerbAttemptLocked(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut) { + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + (void)HdaServiceRirbLocked(controller); + uint64_t startingOverruns = controller->RirbOverrunCount; + + uint16_t next = (uint16_t)((controller->CorbWritePointer + 1u) % + controller->CorbEntryCount); + uint16_t hardwareReadPointer = (uint16_t)(HdaRead16(controller, HDA_REG_CORBRP) & 0x00FFu); + if (next == hardwareReadPointer) { + KeReleaseSpinlock(&controller->StateLock, flags); + return NTLSTATUS_WOULD_BLOCK; + } + + controller->CommandWaiting = true; + controller->CommandResponseReady = false; + controller->CommandCodecAddress = codecAddress; + controller->Corb[next] = ((uint32_t)codecAddress << 28) | + ((uint32_t)nodeId << 20) | + (verbPayload & 0x000FFFFFu); + HdaMemoryBarrier(); + controller->CorbWritePointer = next; + HdaWrite16(controller, HDA_REG_CORBWP, next); + KeReleaseSpinlock(&controller->StateLock, flags); + + for (uint32_t elapsed = 0; elapsed < HDA_CODEC_COMMAND_TIMEOUT_US; elapsed += 10u) { + flags = KeAcquireSpinlock(&controller->StateLock); + (void)HdaServiceRirbLocked(controller); + if (controller->CommandResponseReady) { + uint32_t response = controller->CommandResponse; + controller->CommandWaiting = false; + controller->CommandResponseReady = false; + KeReleaseSpinlock(&controller->StateLock, flags); + if (responseOut) + *responseOut = response; + return NTLSTATUS_SUCCESS; + } + bool overrun = controller->RirbOverrunCount != startingOverruns; + bool removed = controller->Removed || controller->SurpriseRemoved; + KeReleaseSpinlock(&controller->StateLock, flags); + if (overrun || removed) + break; + HdaStall(10); + } + + flags = KeAcquireSpinlock(&controller->StateLock); + controller->CommandWaiting = false; + controller->CommandResponseReady = false; + KeReleaseSpinlock(&controller->StateLock, flags); + return NTLSTATUS_TIMEOUT; +} + +// Sends one codec verb with the required retry while the command mutex is held. +static KSTATUS HdaCodecVerbSerialized(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut) { + KSTATUS finalStatus = NTLSTATUS_TIMEOUT; + for (uint32_t attempt = 0; attempt < HDA_CODEC_COMMAND_ATTEMPTS; attempt++) { + controller->CodecCommandCount++; + finalStatus = HdaCodecVerbAttemptLocked(controller, + codecAddress, + nodeId, + verbPayload, + responseOut); + if (KSUCCESS(finalStatus)) + break; + if (attempt + 1u < HDA_CODEC_COMMAND_ATTEMPTS) { + controller->CodecCommandRetryCount++; + HdaStall(100); + } + } + if (!KSUCCESS(finalStatus)) + controller->CodecCommandFailureCount++; + return finalStatus; +} + +// Sends one 20-bit codec verb payload through CORB/RIRB with one retry. +KSTATUS HdaCodecVerb(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut) { + if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || + codecAddress >= HDA_MAX_CODECS || + (verbPayload & 0xFFF00000u) != 0 || !controller->CommandMutex) + return NTLSTATUS_INVALID_PARAMETER; + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + + KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + status = HdaCodecVerbSerialized(controller, + codecAddress, + nodeId, + verbPayload, + responseOut); + KeReleaseMutex(controller->CommandMutex); + + bool queueDpc = false; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + queueDpc = controller->Started && !controller->RecoveryInProgress && + (controller->UnsolicitedCount != 0 || + controller->PendingRecovery != HDA_RECOVERY_NONE); + KeReleaseSpinlock(&controller->StateLock, flags); + if (queueDpc) + (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); + return status; +} + +// Verifies responding codec addresses through the architectural root node. +static KSTATUS HdaProbeCodecs(HDA_CONTROLLER* controller) { + uint16_t present = controller->CodecMask; + controller->CodecMask = 0; + for (uint8_t codec = 0; codec < HDA_MAX_CODECS; codec++) { + if ((present & (uint16_t)(1u << codec)) == 0) + continue; + uint32_t vendorId = 0; + KSTATUS status = HdaCodecVerbSerialized( + controller, + codec, + 0, + HDA_VERB_GET_PARAMETER(HDA_PARAMETER_VENDOR_ID), + &vendorId); + if (!KSUCCESS(status)) + continue; + controller->CodecMask |= (uint16_t)(1u << codec); + } + return controller->CodecMask ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND; +} + +// Initializes the controller while no other command producer is active. +static KSTATUS HdaInitializeControllerLocked(HDA_CONTROLLER* controller) { + KSTATUS status = HdaResetControllerHardware(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "controller-reset", + status); + + uint16_t state = (uint16_t)(HdaRead16(controller, HDA_REG_STATESTS) & 0x7FFFu); + controller->CodecMask = state; + if (state) + HdaWrite16(controller, HDA_REG_STATESTS, state); + + status = HdaInitializeCommandRings(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "command-rings", + status); + + if (controller->PositionBuffer) { + HdaWrite32(controller, HDA_REG_DPLBASE, + (uint32_t)controller->PositionDeviceAddress | HDA_DPLBASE_ENABLE); + HdaWrite32(controller, HDA_REG_DPUBASE, + (uint32_t)(controller->PositionDeviceAddress >> 32)); + } else { + HdaWrite32(controller, HDA_REG_DPLBASE, 0); + HdaWrite32(controller, HDA_REG_DPUBASE, 0); + } + + status = HdaProgramOutputDescriptor(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "output-descriptor", + status); + + status = HdaProbeCodecs(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "codec-probe", + status); + + HdaWrite32(controller, + HDA_REG_GCTL, + HdaRead32(controller, HDA_REG_GCTL) | + HDA_GCTL_RESET | + HDA_GCTL_UNSOLICITED_ENABLE); + HdaWrite32(controller, HDA_REG_INTCTL, 0); + return NTLSTATUS_SUCCESS; +} + +// Resets and initializes the controller, command rings, position buffer, and output descriptor. +KSTATUS HdaInitializeController(HDA_CONTROLLER* controller) { + if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || + !controller->Registers || !controller->CommandMutex) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + status = HdaInitializeControllerLocked(controller); + KeReleaseMutex(controller->CommandMutex); + return status; +} + +// Arms controller and output-stream interrupts after ISR registration. +void HdaArmInterrupts(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers || !controller->Started || + controller->Removed || controller->SurpriseRemoved) + return; + uint32_t streamInterrupt = 1u << controller->Output.DescriptorIndex; + HdaWrite32(controller, + HDA_REG_INTCTL, + HDA_INT_GLOBAL_ENABLE | + HDA_INT_CONTROLLER_ENABLE | + streamInterrupt); +} + +// Stops hardware access without releasing software resources. +void HdaQuiesceController(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers || controller->SurpriseRemoved) + return; + + bool streamLocked = false; + bool commandLocked = false; + if (controller->StreamMutex) { + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return; + streamLocked = true; + } + if (controller->CommandMutex) { + KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + if (streamLocked) + KeReleaseMutex(controller->StreamMutex); + return; + } + commandLocked = true; + } + HdaWrite32(controller, HDA_REG_INTCTL, 0); + (void)HdaStopOutputHardware(controller); + HdaWrite8(controller, HDA_REG_CORBCTL, 0); + HdaWrite8(controller, HDA_REG_RIRBCTL, 0); + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, flags); + if (commandLocked) + KeReleaseMutex(controller->CommandMutex); + if (streamLocked) + KeReleaseMutex(controller->StreamMutex); +} + +// Copies one logical endpoint ring contract for codec and endpoint setup. +KSTATUS HdaDescribeOutput(HDA_CONTROLLER* controller, + uint8_t ringIndex, + HDA_OUTPUT_DESCRIPTION* descriptionOut) { + if (!controller || !descriptionOut || ringIndex >= HDA_MAX_LOGICAL_ENDPOINTS || + !controller->Output.Rings[ringIndex].Buffer) + return NTLSTATUS_INVALID_PARAMETER; + descriptionOut->PeriodBuffer = controller->Output.Rings[ringIndex].Buffer; + descriptionOut->PeriodBufferBytes = HDA_OUTPUT_BUFFER_BYTES; + descriptionOut->PeriodCount = HDA_OUTPUT_PERIOD_COUNT; + descriptionOut->FramesPerPeriod = HDA_OUTPUT_PERIOD_FRAMES; + descriptionOut->PeriodStrideBytes = HDA_OUTPUT_PERIOD_BYTES; + descriptionOut->HardwareFormat.SampleRate = HDA_OUTPUT_SAMPLE_RATE; + descriptionOut->HardwareFormat.ChannelCount = HDA_OUTPUT_CHANNELS; + descriptionOut->HardwareFormat.SampleFormat = + IoAudioSampleS16; + descriptionOut->StreamTag = controller->Output.StreamTag; + return NTLSTATUS_SUCCESS; +} + +// Selects one logical endpoint ring while the stream is stopped. +KSTATUS HdaSelectOutputRing(HDA_CONTROLLER* controller, uint8_t ringIndex) { + if (!controller || !controller->StreamMutex || + ringIndex >= HDA_MAX_LOGICAL_ENDPOINTS || + !controller->Output.Rings[ringIndex].Buffer) + return NTLSTATUS_INVALID_PARAMETER; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (controller->Output.Running) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_WOULD_BLOCK; + } + controller->Output.ActiveRingIndex = ringIndex; + status = HdaProgramOutputDescriptor(controller); + KeReleaseMutex(controller->StreamMutex); + return status; +} + +// Replaces the transport callbacks while the output stream is stopped. +KSTATUS HdaSetTransportCallbacks(HDA_CONTROLLER* controller, + const HDA_TRANSPORT_CALLBACKS* callbacks, + void* context) { + if (!controller || !controller->StreamMutex) + return NTLSTATUS_INVALID_PARAMETER; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (controller->Output.Running) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_INVALID_PARAMETER; + } + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + if (callbacks) + controller->Callbacks = *callbacks; + else + memset(&controller->Callbacks, 0, sizeof(controller->Callbacks)); + controller->CallbackContext = context; + KeReleaseSpinlock(&controller->StateLock, flags); + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_SUCCESS; +} + +// Starts one already programmed output descriptor without rebuilding its BDL. +static KSTATUS HdaRunPreparedOutput(HDA_CONTROLLER* controller) { + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.PositionRunBaseFrames = controller->Output.LastPositionFrames; + controller->Output.PositionWrapFrames = 0; + controller->Output.LastHardwarePositionBytes = 0; + controller->Output.PositionInitialized = false; + KeReleaseSpinlock(&controller->StateLock, flags); + HdaMemoryBarrier(); + uint32_t control = HdaReadStreamControl(controller) | HDA_STREAM_CTL_RUN; + HdaWriteStreamControl(controller, control); + KSTATUS status = HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RUN, + HDA_STREAM_CTL_RUN, + HDA_STREAM_RESET_TIMEOUT_US); + if (KSUCCESS(status)) { + flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = true; + KeReleaseSpinlock(&controller->StateLock, flags); + } + return status; +} + +// Starts the prepared fixed-format output stream. +KSTATUS HdaStartOutput(HDA_CONTROLLER* controller) { + if (!controller || !controller->StreamMutex) + return NTLSTATUS_INVALID_PARAMETER; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (!controller->Started || controller->Removed || controller->SurpriseRemoved || + !controller->Output.Prepared) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_INVALID_PARAMETER; + } + if (controller->Output.Running) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_SUCCESS; + } + + status = HdaProgramOutputDescriptor(controller); + if (KSUCCESS(status)) { + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.PendingPeriodInterrupts = 0; + controller->Output.PendingDmaErrors = 0; + KeReleaseSpinlock(&controller->StateLock, flags); + status = HdaRunPreparedOutput(controller); + } + KeReleaseMutex(controller->StreamMutex); + return status; +} + +// Stops the output stream while retaining its DMA ring. +void HdaStopOutput(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers || controller->SurpriseRemoved || + !controller->StreamMutex) + return; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return; + (void)HdaQueryOutputPositionFrames(controller); + (void)HdaStopOutputHardware(controller); + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, flags); + KeReleaseMutex(controller->StreamMutex); +} + +// Returns the current byte position from the position buffer or LPIB. +static uint32_t HdaQueryOutputPositionBytesLocked(HDA_CONTROLLER* controller) { + uint32_t position; + if (controller->PositionBuffer) { + position = (uint32_t)controller->PositionBuffer[controller->Output.DescriptorIndex]; + controller->Output.PositionBufferReadCount++; + } else { + position = HdaRead32(controller, + HdaOutputStreamBase(controller) + HDA_STREAM_REG_LPIB); + controller->Output.LpibReadCount++; + } + if (position > HDA_OUTPUT_BUFFER_BYTES) { + controller->Output.PositionDiscontinuityCount++; + position %= HDA_OUTPUT_BUFFER_BYTES; + } else if (position == HDA_OUTPUT_BUFFER_BYTES) { + position = 0; + } + return position; +} + +// Advances the monotonic fine position while the state lock is held. +static uint64_t HdaQueryOutputPositionFramesLocked(HDA_CONTROLLER* controller) { + if (!controller->Output.Running) + return controller->Output.LastPositionFrames; + + uint32_t positionBytes = HdaQueryOutputPositionBytesLocked(controller); + if (controller->Output.PositionInitialized && + positionBytes < controller->Output.LastHardwarePositionBytes) { + controller->Output.PositionWrapFrames += + HDA_OUTPUT_PERIOD_COUNT * HDA_OUTPUT_PERIOD_FRAMES; + } + controller->Output.PositionInitialized = true; + controller->Output.LastHardwarePositionBytes = positionBytes; + + uint64_t candidate = controller->Output.PositionRunBaseFrames + + controller->Output.PositionWrapFrames + + positionBytes / HDA_OUTPUT_BYTES_PER_FRAME; + if (candidate < controller->Output.LastPositionFrames) { + controller->Output.PositionDiscontinuityCount++; + candidate = controller->Output.LastPositionFrames; + } + controller->Output.LastPositionFrames = candidate; + return candidate; +} + +// Returns the monotonic fine position derived from the position buffer or LPIB. +uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers) + return 0; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + uint64_t position = HdaQueryOutputPositionFramesLocked(controller); + KeReleaseSpinlock(&controller->StateLock, flags); + return position; +} + +// Captures one controller interrupt and queues deferred processing. +void HdaHandleInterrupt(HDA_CONTROLLER* controller) { + if (!controller) + return; + _InterlockedIncrement(&controller->IsrActive); + if (!controller->Started || controller->Removed || controller->SurpriseRemoved) + goto finished; + + uint32_t interruptStatus = HdaRead32(controller, HDA_REG_INTSTS); + if (interruptStatus == 0 || interruptStatus == UINT32_MAX) + goto finished; + + bool queueDpc = false; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + bool recovering = controller->RecoveryInProgress; + controller->InterruptCount++; + + uint32_t streamBit = 1u << controller->Output.DescriptorIndex; + if (interruptStatus & streamBit) { + uint32_t base = HdaOutputStreamBase(controller); + uint8_t streamStatus = HdaRead8(controller, base + HDA_STREAM_REG_STS); + streamStatus &= HDA_STREAM_STS_INTERRUPT_MASK; + if (streamStatus) { + HdaWrite8(controller, base + HDA_STREAM_REG_STS, streamStatus); + controller->Output.InterruptCount++; + if ((streamStatus & HDA_STREAM_STS_COMPLETE) && !recovering) { + controller->Output.PendingPeriodInterrupts++; + queueDpc = true; + } + if (streamStatus & HDA_STREAM_STS_ERROR_MASK) { + controller->Output.DmaErrorCount++; + if (!recovering) { + controller->Output.PendingDmaErrors++; + queueDpc = true; + } + } + } + } + + if (interruptStatus & HDA_INT_CONTROLLER_STATUS) { + controller->ControllerInterruptCount++; + (void)HdaServiceRirbLocked(controller); + if (!recovering) { + queueDpc = queueDpc || controller->UnsolicitedCount != 0 || + controller->PendingRecovery != HDA_RECOVERY_NONE; + } + + uint8_t corbStatus = HdaRead8(controller, HDA_REG_CORBSTS); + if (corbStatus & HDA_CORBSTS_MEMORY_ERROR) { + controller->CorbMemoryErrorCount++; + HdaWrite8(controller, HDA_REG_CORBSTS, corbStatus); + controller->PendingRecovery = HDA_RECOVERY_CONTROLLER; + if (!recovering) + queueDpc = true; + } + } + KeReleaseSpinlock(&controller->StateLock, flags); + + if (queueDpc) + (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); + +finished: + _InterlockedDecrement(&controller->IsrActive); +} + +// Promotes an error to stream or controller recovery using the one-second threshold. +static void HdaRecordDmaErrorsLocked(HDA_CONTROLLER* controller, + uint32_t errorCount) { + uint64_t now = HdaQueryTicks(); + if (controller->StreamErrorsInWindow == 0 || + now - controller->StreamErrorWindowStart > HDA_STREAM_ERROR_WINDOW_US) { + controller->StreamErrorWindowStart = now; + controller->StreamErrorsInWindow = 0; + } + controller->StreamErrorsInWindow += errorCount; + uint8_t requested = controller->StreamErrorsInWindow >= + HDA_STREAM_ERRORS_BEFORE_CONTROLLER_RESET + ? HDA_RECOVERY_CONTROLLER + : HDA_RECOVERY_STREAM; + if (requested > controller->PendingRecovery) + controller->PendingRecovery = requested; +} + +// Runs period delivery, unsolicited callbacks, and recovery scheduling at DPC level. +void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)systemArgument1; + (void)systemArgument2; + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)context; + if (!controller) + return; + _InterlockedIncrement(&controller->DpcActive); + if (controller->Removed || controller->SurpriseRemoved) + goto finished; + + while (true) { + uint32_t periodCount = 0; + uint32_t errorCount = 0; + HDA_UNSOLICITED_RESPONSE unsolicited; + bool haveUnsolicited = false; + HDA_TRANSPORT_CALLBACKS callbacks; + void* callbackContext; + + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + if (controller->Output.PendingPeriodInterrupts > 0) { + periodCount = (uint32_t)controller->Output.PendingPeriodInterrupts; + controller->Output.PendingPeriodInterrupts = 0; + } + if (controller->Output.PendingDmaErrors > 0) { + errorCount = (uint32_t)controller->Output.PendingDmaErrors; + controller->Output.PendingDmaErrors = 0; + } + if (controller->UnsolicitedCount) { + unsolicited = controller->Unsolicited[controller->UnsolicitedRead]; + controller->UnsolicitedRead = + (uint8_t)((controller->UnsolicitedRead + 1u) % + HDA_MAX_UNSOLICITED_RESPONSES); + controller->UnsolicitedCount--; + haveUnsolicited = true; + } + callbacks = controller->Callbacks; + callbackContext = controller->CallbackContext; + KeReleaseSpinlock(&controller->StateLock, flags); + + for (uint32_t index = 0; index < periodCount; index++) { + flags = KeAcquireSpinlock(&controller->StateLock); + uint8_t bdlIndex = controller->Output.NextCompletionBdl; + uint32_t completedPeriod = controller->Output.BdlPeriodMap[bdlIndex]; + controller->Output.NextCompletionBdl = + (uint8_t)((bdlIndex + 1u) % HDA_OUTPUT_PERIOD_COUNT); + HDA_OUTPUT_RING* ring = + &controller->Output.Rings[controller->Output.ActiveRingIndex]; + ring->NextLogicalPeriod = + (uint8_t)((completedPeriod + 1u) % HDA_OUTPUT_PERIOD_COUNT); + ring->PresentedFrames += HDA_OUTPUT_PERIOD_FRAMES; + controller->Output.PeriodCompletionCount++; + uint64_t presentedFrames = ring->PresentedFrames; + uint64_t finePosition = HdaQueryOutputPositionFramesLocked(controller); + uint64_t interruptCount = controller->Output.InterruptCount; + KeReleaseSpinlock(&controller->StateLock, flags); + + uint64_t timestamp = HdaQueryTicks(); + if (callbacks.PeriodComplete) { + callbacks.PeriodComplete(callbackContext, + completedPeriod, + presentedFrames, + finePosition, + interruptCount, + timestamp); + } + } + + if (haveUnsolicited && callbacks.UnsolicitedResponse) { + callbacks.UnsolicitedResponse( + callbackContext, + (uint8_t)(unsolicited.ResponseExtra & HDA_RIRB_RESPONSE_CODEC_MASK), + unsolicited.Response, + unsolicited.ResponseExtra); + } + + if (errorCount) { + flags = KeAcquireSpinlock(&controller->StateLock); + HdaRecordDmaErrorsLocked(controller, errorCount); + KeReleaseSpinlock(&controller->StateLock, flags); + } + + bool signalRecovery = false; + flags = KeAcquireSpinlock(&controller->StateLock); + signalRecovery = controller->PendingRecovery != HDA_RECOVERY_NONE; + KeReleaseSpinlock(&controller->StateLock, flags); + if (signalRecovery && controller->RecoverySemaphore) + KeSignalSemaphore(controller->RecoverySemaphore); + + if (periodCount == 0 && errorCount == 0 && !haveUnsolicited) + break; + } + +finished: + _InterlockedDecrement(&controller->DpcActive); +} + +// Recovers one isolated stream error without disturbing the codec topology. +static KSTATUS HdaRecoverOutputStream(HDA_CONTROLLER* controller) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + (void)HdaQueryOutputPositionFrames(controller); + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + bool restart = controller->Output.Running; + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, flags); + KSTATUS status = HdaProgramOutputDescriptor(controller); + if (!KSUCCESS(status)) + return status; + controller->Output.StreamResetCount++; + HdaArmInterrupts(controller); + if (restart) + status = HdaRunPreparedOutput(controller); + return status; +} + +// Recovers command and DMA engines, then asks the codec layer to restore routing. +static KSTATUS HdaRecoverController(HDA_CONTROLLER* controller) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + (void)HdaQueryOutputPositionFrames(controller); + uintptr_t stateFlags = KeAcquireSpinlock(&controller->StateLock); + bool restart = controller->Output.Running; + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, stateFlags); + KSTATUS status = HdaInitializeController(controller); + if (!KSUCCESS(status)) + return status; + controller->ControllerResetCount++; + controller->StreamErrorWindowStart = 0; + controller->StreamErrorsInWindow = 0; + + HDA_TRANSPORT_CALLBACKS callbacks; + void* callbackContext; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + callbacks = controller->Callbacks; + callbackContext = controller->CallbackContext; + KeReleaseSpinlock(&controller->StateLock, flags); + if (callbacks.RestoreAfterControllerReset) { + status = callbacks.RestoreAfterControllerReset(callbackContext); + if (!KSUCCESS(status)) + return status; + } + HdaArmInterrupts(controller); + if (restart) { + status = HdaRunPreparedOutput(controller); + if (KSUCCESS(status) && callbacks.ResumeAfterControllerReset) + status = callbacks.ResumeAfterControllerReset(callbackContext); + if (!KSUCCESS(status)) { + (void)HdaStopOutputHardware(controller); + uintptr_t stopFlags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, stopFlags); + } + } + return status; +} + +// Waits until every pre-recovery ISR and DPC has left the controller object. +static KSTATUS HdaWaitForInterruptWorkIdle(HDA_CONTROLLER* controller) { + for (uint32_t elapsed = 0; elapsed < 100000u; elapsed += 10u) { + if (controller->IsrActive == 0 && + controller->Dpc.Inserted == 0 && controller->DpcActive == 0) + return NTLSTATUS_SUCCESS; + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + HdaStall(10); + } + return NTLSTATUS_TIMEOUT; +} + +// Runs passive-level reset work that cannot safely execute in an ISR or DPC. +static void HdaRecoveryWorker(uintptr_t argument) { + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)argument; + while (controller->RecoveryWorkerRunning) { + KSTATUS waitStatus = KeWaitSemaphore(controller->RecoverySemaphore, KE_WAIT_INFINITE); + if (!KSUCCESS(waitStatus) || !controller->RecoveryWorkerRunning) + break; + + uint8_t recovery; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + recovery = controller->PendingRecovery; + controller->PendingRecovery = HDA_RECOVERY_NONE; + controller->RecoveryInProgress = recovery != HDA_RECOVERY_NONE; + KeReleaseSpinlock(&controller->StateLock, flags); + if (recovery == HDA_RECOVERY_NONE) + continue; + + if (!controller->Removed && !controller->SurpriseRemoved) + HdaWrite32(controller, HDA_REG_INTCTL, 0); + KSTATUS status = HdaWaitForInterruptWorkIdle(controller); + bool streamLocked = false; + if (KSUCCESS(status)) { + status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + streamLocked = KE_MUTEX_ACQUIRE_SUCCEEDED(status); + } + if (streamLocked) { + status = recovery == HDA_RECOVERY_CONTROLLER + ? HdaRecoverController(controller) + : HdaRecoverOutputStream(controller); + KeReleaseMutex(controller->StreamMutex); + } + + HDA_TRANSPORT_CALLBACKS callbacks; + void* callbackContext; + bool queueDpc; + bool repeatRecovery; + flags = KeAcquireSpinlock(&controller->StateLock); + controller->RecoveryInProgress = false; + callbacks = controller->Callbacks; + callbackContext = controller->CallbackContext; + queueDpc = controller->UnsolicitedCount != 0 || + controller->Output.PendingPeriodInterrupts != 0 || + controller->Output.PendingDmaErrors != 0; + repeatRecovery = controller->PendingRecovery != HDA_RECOVERY_NONE; + KeReleaseSpinlock(&controller->StateLock, flags); + + if (!KSUCCESS(status) && status != NTLSTATUS_DEVICE_REMOVED) { + if (callbacks.TransportFailure) + callbacks.TransportFailure(callbackContext, status); + gHdaDdk->Debug->Print("HDA: recovery failed status=0x%x\n", status); + } + if (queueDpc) + (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); + if (repeatRecovery) + KeSignalSemaphore(controller->RecoverySemaphore); + } + if (controller->RecoveryFinished) + KeSignalEvent(controller->RecoveryFinished); +} + +// Starts the passive recovery worker for one controller. +KSTATUS HdaStartRecoveryWorker(HDA_CONTROLLER* controller) { + if (!controller || !controller->RecoverySemaphore || !controller->RecoveryFinished) + return NTLSTATUS_INVALID_PARAMETER; + controller->RecoveryWorkerRunning = true; + KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityHigh); + KSTATUS status = KeCreateKernelThread((uintptr_t)HdaRecoveryWorker, + (uintptr_t)controller, + ¶meters, + &controller->RecoveryThread); + if (!KSUCCESS(status)) + controller->RecoveryWorkerRunning = false; + return status; +} + +// Stops and joins the passive recovery worker for one controller. +void HdaStopRecoveryWorker(HDA_CONTROLLER* controller) { + if (!controller) + return; + controller->RecoveryWorkerRunning = false; + if (controller->RecoverySemaphore) + KeSignalSemaphore(controller->RecoverySemaphore); + if (controller->RecoveryThread && controller->RecoveryFinished) + (void)KeWaitForEvent(controller->RecoveryFinished, 0); + if (controller->RecoveryThread) { + ObDereferenceObject((OBJECT*)controller->RecoveryThread); + controller->RecoveryThread = (THREAD*)0; + } +} diff --git a/code/drivers/bus/pci/pci.c b/code/drivers/bus/pci/pci.c index 90dae80..a72a8f1 100644 --- a/code/drivers/bus/pci/pci.c +++ b/code/drivers/bus/pci/pci.c @@ -33,7 +33,7 @@ static uint64_t PciAlignUp64(uint64_t value, uint64_t alignment) { // Implements the default Mmio Flags operation. static uint64_t PciDefaultMmioFlags(void) { - return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_WRITETHROUGH | NTL_DDK_PAGE_NOCACHE | NTL_DDK_PAGE_NX; + return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_CACHE_UC | NTL_DDK_PAGE_NX; } // Creates mmio Va. diff --git a/code/drivers/bus/usb/usb.c b/code/drivers/bus/usb/usb.c index ba1419b..5439b52 100644 --- a/code/drivers/bus/usb/usb.c +++ b/code/drivers/bus/usb/usb.c @@ -37,7 +37,7 @@ static KSTATUS UsbInitializeXhciDriver(void) { // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; + if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("usb: driver loading\n"); gUsbDdk = ddk; diff --git a/code/drivers/bus/usb/usb.h b/code/drivers/bus/usb/usb.h index ba6930c..427f003 100644 --- a/code/drivers/bus/usb/usb.h +++ b/code/drivers/bus/usb/usb.h @@ -124,6 +124,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define XHCI_TRB_SETUP_TRT_IN (3u << 16) #define XHCI_TRB_STATUS_DIR_IN (1u << 16) +#define XHCI_HCC1_AC64 (1u << 0) #define XHCI_HCC1_CSZ (1u << 2) #define XHCI_CONTEXT_ENTRIES(n) ((uint32_t)(n) << 27) @@ -199,6 +200,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define SCSI_INQUIRY 0x12u #define SCSI_READ_10 0x28u #define SCSI_WRITE_10 0x2Au +#define SCSI_SYNCHRONIZE_CACHE_10 0x35u #define SCSI_READ_CAPACITY_10 0x25u #define XHCI_RING_TRBS 256u @@ -393,7 +395,11 @@ typedef struct USB_HCD_OPERATIONS { KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb); KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed); KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port); - void* (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys); + void (*DestroyPipe)(USB_HCD* hcd, USB_PIPE* pipe); + void* (*AllocateDma)(USB_HCD* hcd, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer); } USB_HCD_OPERATIONS; struct USB_HCD { @@ -454,7 +460,7 @@ struct USB_URB { USB_PIPE* Pipe; USB_SETUP_PACKET Setup; void* Buffer; - PHYSICAL_ADDRESS BufferPhys; + uint64_t BufferDeviceAddress; uint32_t Length; uint32_t ActualLength; uint32_t HcdCompletionCode; @@ -521,8 +527,9 @@ typedef struct USB_HUB_EXTENSION { uint32_t PendingHubPorts[1]; SPINLOCK PendingLock; SEMAPHORE* WorkerSemaphore; + EVENT* WorkerFinished; THREAD* WorkerThread; - bool WorkerRunning; + volatile bool WorkerRunning; bool InitialScanPending; bool InitialScanComplete; bool InitialScanInProgress; @@ -533,7 +540,8 @@ typedef struct USB_HUB_EXTENSION { uint8_t InterruptInterval; USB_PIPE* InterruptPipe; uint8_t* ChangeBitmap; - PHYSICAL_ADDRESS ChangeBitmapPhys; + uint64_t ChangeBitmapDeviceAddress; + IO_DMA_BUFFER* ChangeBitmapDma; USB_URB ChangeUrb; bool ChangeUrbPending; uint8_t PowerOnToPowerGoodMs; @@ -573,7 +581,8 @@ typedef struct USB_HID_EXTENSION { USB_PIPE* InterruptPipe; uint8_t* ReportBuffer; - PHYSICAL_ADDRESS ReportBufferPhys; + uint64_t ReportBufferDeviceAddress; + IO_DMA_BUFFER* ReportBufferDma; uint16_t ReportLength; USB_URB InterruptUrb; bool InterruptPending; @@ -584,6 +593,12 @@ typedef struct USB_HID_EXTENSION { uint32_t InputDeviceId; } USB_HID_EXTENSION; +typedef enum USB_STORAGE_FLUSH_CAPABILITY { + UsbStorageFlushUnknown = 0, + UsbStorageFlushSupported, + UsbStorageFlushUnsupported, +} USB_STORAGE_FLUSH_CAPABILITY; + typedef enum USB_STORAGE_EXTENSION_KIND { UsbStorageExtensionInterface = 0x55534946u, UsbStorageExtensionDisk = 0x5553444Bu, @@ -620,6 +635,7 @@ struct USB_STORAGE_EXTENSION { uint32_t Tag; uint32_t BlockSize; uint64_t BlockCount; + USB_STORAGE_FLUSH_CAPABILITY FlushCapability; volatile long IoBusy; }; @@ -630,14 +646,15 @@ struct USB_STORAGE_DISK_EXTENSION { typedef struct XHCI_PIPE_CONTEXT { XHCI_TRB* Ring; - PHYSICAL_ADDRESS RingPhys; + uint64_t RingDeviceAddress; + IO_DMA_BUFFER* RingDma; uint32_t Enqueue; uint8_t Cycle; uint8_t EndpointId; USB_PIPE_TYPE Type; bool Configured; bool ActiveTransfer; - uint64_t ActiveTrbPhys; + uint64_t ActiveTrbDeviceAddress; USB_URB* ActiveUrb; } XHCI_PIPE_CONTEXT; @@ -683,7 +700,7 @@ typedef struct XHCI_DEVICE { uint8_t HidLastModifiers; uint8_t HidLastKeys[6]; uint8_t HidLastButtons; - uint64_t HidLastTrbPhys; + uint64_t HidLastTrbDeviceAddress; bool MassCandidate; bool MassActive; @@ -708,15 +725,25 @@ typedef struct XHCI_DEVICE { bool MassRemoved; volatile long MassIoBusy; - PHYSICAL_ADDRESS InputContextPhys; - PHYSICAL_ADDRESS DeviceContextPhys; - PHYSICAL_ADDRESS Ep0RingPhys; - PHYSICAL_ADDRESS HidRingPhys; - PHYSICAL_ADDRESS MassInRingPhys; - PHYSICAL_ADDRESS MassOutRingPhys; - PHYSICAL_ADDRESS DescriptorPhys; - PHYSICAL_ADDRESS HidBufferPhys; - PHYSICAL_ADDRESS MassBufferPhys; + uint64_t InputContextDeviceAddress; + uint64_t DeviceContextDeviceAddress; + uint64_t Ep0RingDeviceAddress; + uint64_t HidRingDeviceAddress; + uint64_t MassInRingDeviceAddress; + uint64_t MassOutRingDeviceAddress; + uint64_t DescriptorDeviceAddress; + uint64_t HidBufferDeviceAddress; + uint64_t MassBufferDeviceAddress; + + IO_DMA_BUFFER* InputContextDma; + IO_DMA_BUFFER* DeviceContextDma; + IO_DMA_BUFFER* Ep0RingDma; + IO_DMA_BUFFER* HidRingDma; + IO_DMA_BUFFER* MassInRingDma; + IO_DMA_BUFFER* MassOutRingDma; + IO_DMA_BUFFER* DescriptorDma; + IO_DMA_BUFFER* HidBufferDma; + IO_DMA_BUFFER* MassBufferDma; uint8_t* InputContext; uint8_t* DeviceContext; @@ -746,8 +773,8 @@ typedef struct XHCI_TRANSFER_RESULT { typedef struct XHCI_WAIT_TARGET { struct XHCI_WAIT_TARGET* Next; - uint64_t CommandPhys; - uint64_t TransferPhys; + uint64_t CommandDeviceAddress; + uint64_t TransferDeviceAddress; uint8_t TransferSlotId; uint8_t TransferEndpointId; volatile bool CommandMatched; @@ -793,12 +820,20 @@ typedef struct XHCI_CONTROLLER { uint8_t PortProtocolMajor[XHCI_MAX_ROOT_PORTS + 1u]; uint8_t PortProtocolMinor[XHCI_MAX_ROOT_PORTS + 1u]; - PHYSICAL_ADDRESS DcbaaPhys; - PHYSICAL_ADDRESS ScratchpadArrayPhys; - PHYSICAL_ADDRESS ScratchpadPhys[XHCI_MAX_SCRATCHPADS]; - PHYSICAL_ADDRESS CommandRingPhys; - PHYSICAL_ADDRESS EventRingPhys; - PHYSICAL_ADDRESS ErstPhys; + uint64_t DcbaaDeviceAddress; + uint64_t ScratchpadArrayDeviceAddress; + uint64_t ScratchpadDeviceAddress[XHCI_MAX_SCRATCHPADS]; + uint64_t CommandRingDeviceAddress; + uint64_t EventRingDeviceAddress; + uint64_t ErstDeviceAddress; + + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* DcbaaDma; + IO_DMA_BUFFER* ScratchpadArrayDma; + IO_DMA_BUFFER* ScratchpadDma[XHCI_MAX_SCRATCHPADS]; + IO_DMA_BUFFER* CommandRingDma; + IO_DMA_BUFFER* EventRingDma; + IO_DMA_BUFFER* ErstDma; uint64_t* Dcbaa; uint64_t* ScratchpadArray; @@ -825,6 +860,7 @@ typedef struct XHCI_CONTROLLER { XHCI_WAIT_TARGET* WaiterList; volatile bool Running; + bool SurpriseRemoved; USB_HCD Hcd; } XHCI_CONTROLLER; @@ -867,7 +903,11 @@ void UsbDereferencePipe(USB_PIPE* pipe); KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb); KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb); KSTATUS UsbCancelUrb(USB_URB* urb); -void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys); +void* UsbAllocateDmaBuffer(USB_DEVICE* dev, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer); +void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer); KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp); KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp); @@ -882,7 +922,11 @@ void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value); void UsbDelayUs(const NTL_DDK* ddk, uint32_t us); bool UsbWaitSet32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs); bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs); -void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys); +void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer); +void XhciReleaseDeviceDma(XHCI_DEVICE* dev); uint16_t XhciScratchpadCount(uint32_t hcs2); uint32_t XhciTrbType(const XHCI_TRB* trb); uint32_t XhciCompletionCode(const XHCI_TRB* trb); @@ -918,7 +962,7 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci); KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci); KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci); KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci); -uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index); +uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index); KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, XHCI_COMMAND_RESULT* outResult); KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci); KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId); @@ -931,17 +975,17 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed); KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId); KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); -uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index); -KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys); -KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs); -KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length); +uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index); +KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress); +KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length, uint32_t timeoutUs); +KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length); KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup); KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs); KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc); KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength); uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress); uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval); -KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys); +KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress); bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key); INPUT_KEY XhciHidModifierKey(uint8_t bit); void XhciSubmitHidKey(INPUT_KEY key, bool pressed); @@ -949,24 +993,23 @@ void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len); KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); void XhciArmAllHid(XHCI_CONTROLLER* xhci); -void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue); +void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue); KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci); KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject); KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp); -void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle); +void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint8_t cycle); KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe); -KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn); +KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t* enqueue, uint8_t* cycle, uint64_t bufferDeviceAddress, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, uint64_t dataDeviceAddress, uint32_t dataLength, bool dataIn); uint32_t UsbBe32(const uint8_t* p); void UsbStoreBe32(uint8_t* p, uint32_t v); void UsbStoreBe16(uint8_t* p, uint16_t v); KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage); -KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage); KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage); KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage); KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer); diff --git a/code/drivers/bus/usb/usb_bus.c b/code/drivers/bus/usb/usb_bus.c index 617736b..188dafd 100644 --- a/code/drivers/bus/usb/usb_bus.c +++ b/code/drivers/bus/usb/usb_bus.c @@ -24,9 +24,12 @@ static void UsbDeviceDestroy(OBJECT* object) { } } -// Implements the pipe Destroy operation. +// Destroys a pipe after the host controller releases its private context. static void UsbPipeDestroy(OBJECT* object) { USB_PIPE* pipe = (USB_PIPE*)object; + if (pipe->Device && pipe->Device->Hcd && pipe->Device->Hcd->Operations && + pipe->Device->Hcd->Operations->DestroyPipe) + pipe->Device->Hcd->Operations->DestroyPipe(pipe->Device->Hcd, pipe); if (pipe->Device) { UsbDereferenceDevice(pipe->Device); pipe->Device = (USB_DEVICE*)0; @@ -53,11 +56,21 @@ void UsbDereferencePipe(USB_PIPE* pipe) { if (pipe) ObDereferenceObject(&pipe->Header); } -// Creates dma Buffer. -void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) { - if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma) +// Allocates a DMA buffer through the device host-controller adapter. +void* UsbAllocateDmaBuffer(USB_DEVICE* dev, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer) { + if (!dev || !dev->Hcd || !dev->Hcd->Operations || + !dev->Hcd->Operations->AllocateDma || !outDeviceAddress || !outBuffer) return (void*)0; - return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys); + return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outDeviceAddress, outBuffer); +} + +// Frees a DMA buffer allocated for a USB class driver. +void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer) { + if (buffer && gUsbDdk && gUsbDdk->Dma) + gUsbDdk->Dma->FreeCommonBuffer(buffer); } // Creates device. @@ -464,7 +477,7 @@ KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) { if (!parent) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { DEVICE_OBJECT* child = UsbReferenceFirstChild(parent); if (!child) break; diff --git a/code/drivers/bus/usb/usb_core.c b/code/drivers/bus/usb/usb_core.c index fb0e562..af656c1 100644 --- a/code/drivers/bus/usb/usb_core.c +++ b/code/drivers/bus/usb/usb_core.c @@ -258,6 +258,9 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) { XHCI_DEVICE* dev = &xhci->Devices[slotId]; + if (dev->Allocated && xhci->Running) + (void)XhciDisableSlot(xhci, slotId); + if (dev->UsbPdo) { (void)UsbRemovePdoSubtree(dev->UsbPdo, true); dev->UsbPdo = (DEVICE_OBJECT*)0; @@ -269,9 +272,7 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) { dev->UsbDevice = (USB_DEVICE*)0; } - if (dev->Allocated) - (void)XhciDisableSlot(xhci, slotId); - + XhciReleaseDeviceDma(dev); memset(dev, 0, sizeof(*dev)); } diff --git a/code/drivers/bus/usb/usb_hid.c b/code/drivers/bus/usb/usb_hid.c index 37bb02f..509909b 100644 --- a/code/drivers/bus/usb/usb_hid.c +++ b/code/drivers/bus/usb/usb_hid.c @@ -2,6 +2,16 @@ #include "usb.h" +// Releases the HID class DMA buffer after all pending transfers are cancelled. +static void UsbHidReleaseDma(USB_HID_EXTENSION* ext) { + if (!ext) + return; + UsbFreeDmaBuffer(ext->ReportBufferDma); + ext->ReportBufferDma = (IO_DMA_BUFFER*)0; + ext->ReportBuffer = (uint8_t*)0; + ext->ReportBufferDeviceAddress = 0; +} + // Implements the hid Key In Report operation. bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) { if (key == 0 || key == 1) return false; @@ -91,21 +101,21 @@ KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { memset(dev->HidBuffer, 0, dev->HidReportLength); - uint64_t trbPhys = 0; + uint64_t trbDeviceAddress = 0; KSTATUS st = XhciRingAppend(dev->HidRing, - dev->HidRingPhys, + dev->HidRingDeviceAddress, XHCI_HID_RING_TRBS, &dev->HidEnqueue, &dev->HidCycle, - dev->HidBufferPhys, + dev->HidBufferDeviceAddress, dev->HidReportLength, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, - &trbPhys); + &trbDeviceAddress); if (!KSUCCESS(st)) return st; - dev->HidLastTrbPhys = trbPhys; + dev->HidLastTrbDeviceAddress = trbDeviceAddress; dev->HidArmed = true; XhciBarrier(); XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, dev->HidEndpointId); @@ -125,13 +135,13 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) { } } -static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, +static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue) { if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return; XHCI_DEVICE* dev = &xhci->Devices[slotId]; - if (!dev->HidActive || endpointId != dev->HidEndpointId || trbPhys != dev->HidLastTrbPhys) + if (!dev->HidActive || endpointId != dev->HidEndpointId || trbDeviceAddress != dev->HidLastTrbDeviceAddress) return; dev->HidArmed = false; @@ -200,17 +210,26 @@ KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { if (dev->HidRing && dev->HidBuffer) return NTLSTATUS_SUCCESS; - dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingPhys); - dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferPhys); - if (!dev->HidRing || !dev->HidBuffer) + dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingDeviceAddress, &dev->HidRingDma); + dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferDeviceAddress, &dev->HidBufferDma); + if (!dev->HidRing || !dev->HidBuffer) { + if (gUsbDdk && gUsbDdk->Dma) { + gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma); + } + dev->HidBufferDma = (IO_DMA_BUFFER*)0; + dev->HidRingDma = (IO_DMA_BUFFER*)0; + dev->HidBuffer = (uint8_t*)0; + dev->HidRing = (XHCI_TRB*)0; return NTLSTATUS_NO_MEMORY; + } dev->HidEnqueue = 0; dev->HidCycle = 1; XHCI_TRB* link = &dev->HidRing[XHCI_HID_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)dev->HidRingPhys; - link->ParameterHi = (uint32_t)(dev->HidRingPhys >> 32); + link->ParameterLo = (uint32_t)dev->HidRingDeviceAddress; + link->ParameterHi = (uint32_t)(dev->HidRingDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | @@ -253,15 +272,15 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { ep[0] = (uint32_t)interval << 16; ep[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_INT_IN | ((uint32_t)maxPacket << 16); - ep[2] = (uint32_t)(dev->HidRingPhys | XHCI_EP_DCS); - ep[3] = (uint32_t)((dev->HidRingPhys | XHCI_EP_DCS) >> 32); + ep[2] = (uint32_t)(dev->HidRingDeviceAddress | XHCI_EP_DCS); + ep[3] = (uint32_t)((dev->HidRingDeviceAddress | XHCI_EP_DCS) >> 32); ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16); XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -499,7 +518,7 @@ static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) { static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) { if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed) return NTLSTATUS_OBJECT_DEAD; - if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys) + if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferDeviceAddress) return NTLSTATUS_INVALID_PARAMETER; if (ext->InterruptPending) return NTLSTATUS_SUCCESS; @@ -511,7 +530,7 @@ static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) { ext->InterruptUrb.Device = ext->Device; ext->InterruptUrb.Pipe = ext->InterruptPipe; ext->InterruptUrb.Buffer = ext->ReportBuffer; - ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys; + ext->InterruptUrb.BufferDeviceAddress = ext->ReportBufferDeviceAddress; ext->InterruptUrb.Length = ext->ReportLength; ext->InterruptUrb.Complete = UsbHidInterruptComplete; ext->InterruptUrb.CompleteContext = ext; @@ -565,7 +584,7 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT; ext->ReportLength = reportLength; - ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys); + ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferDeviceAddress, &ext->ReportBufferDma); if (!ext->ReportBuffer) return NTLSTATUS_NO_MEMORY; @@ -575,13 +594,22 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) ext->MaxPacketSize, ext->Interval, &ext->InterruptPipe); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + UsbHidReleaseDma(ext); return st; + } if (!ext->AbsoluteMouse) { st = UsbHidSendBootProtocol(ext); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + if (ext->InterruptPipe) { + ext->InterruptPipe->Open = false; + UsbDereferencePipe(ext->InterruptPipe); + ext->InterruptPipe = (USB_PIPE*)0; + } + UsbHidReleaseDma(ext); return st; + } } memset(ext->LastKeys, 0, sizeof(ext->LastKeys)); @@ -678,6 +706,7 @@ KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { IoDestroyRemoveLock(&ext->RemoveLock); ext->RemoveLockReady = false; } + UsbHidReleaseDma(ext); } KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS; diff --git a/code/drivers/bus/usb/usb_hub.c b/code/drivers/bus/usb/usb_hub.c index 534f57d..f4d14dc 100644 --- a/code/drivers/bus/usb/usb_hub.c +++ b/code/drivers/bus/usb/usb_hub.c @@ -8,6 +8,16 @@ DRIVER_OBJECT gUsbHubDriver; +// Releases the external-hub change bitmap after transfer rundown. +static void UsbHubReleaseDma(USB_HUB_EXTENSION* hub) { + if (!hub) + return; + UsbFreeDmaBuffer(hub->ChangeBitmapDma); + hub->ChangeBitmapDma = (IO_DMA_BUFFER*)0; + hub->ChangeBitmap = (uint8_t*)0; + hub->ChangeBitmapDeviceAddress = 0; +} + static const IO_DEVICE_ID gUsbHubIds[] = { { IoBusUsb, { "USB\\ROOT_HUB", sizeof("USB\\ROOT_HUB") - 1 } }, { IoBusUsb, { "USB\\CLASS_09", sizeof("USB\\CLASS_09") - 1 } }, @@ -61,8 +71,10 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub, if (maxAttempts == 0) maxAttempts = 1; for (uint8_t attempt = 1; attempt <= maxAttempts; attempt++) { - if (!UsbHubRootPortConnected(xhci, port)) - return NTLSTATUS_NOT_FOUND; + if (!UsbHubRootPortConnected(xhci, port)) { + last = NTLSTATUS_NOT_FOUND; + break; + } uint8_t slot = 0; last = XhciEnumeratePortEx(xhci, port, &slot); @@ -100,6 +112,40 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub, return last; } +// Enumerates one connected root port through the same debounce path at boot and hotplug. +static KSTATUS UsbHubEnumerateConnectedRootPort(USB_HUB_EXTENSION* hub, + XHCI_CONTROLLER* xhci, + uint8_t port, + uint8_t* outSlot) { + if (outSlot) + *outSlot = 0; + if (!hub || !xhci || port == 0 || port > hub->PortCount || + port > USB_ROOT_HUB_MAX_PORTS) + return NTLSTATUS_INVALID_PARAMETER; + + hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; + hub->RootPortState[port] = UsbHubPortConnectedDebounce; + + UsbDelayUs(xhci->Ddk, 30000u); + if (!UsbHubRootPortConnected(xhci, port)) { + UsbHubClearRootPortFailure(hub, port); + hub->RootPortSlot[port] = 0; + hub->RootPortState[port] = UsbHubPortDisconnected; + return NTLSTATUS_NOT_FOUND; + } + + hub->RootPortState[port] = UsbHubPortEnumerating; + uint8_t slot = 0; + KSTATUS status = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + if (KSUCCESS(status)) { + hub->RootPortSlot[port] = slot; + hub->RootPortState[port] = UsbHubPortChildPresent; + if (outSlot) + *outSlot = slot; + } + return status; +} + // Implements the hub Pop Pending Root Port operation. static bool UsbHubPopPendingRootPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) { if (!hub || !outPort) return false; @@ -184,17 +230,19 @@ static void UsbHubRemoveSlotChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t DbgPrint("usb-hub: %s port %u removing slot=%u\n", hub->Kind == UsbHubKindRoot ? "root" : "hub", port, slot); + if (xdev->Allocated && xhci->Running) + (void)XhciDisableSlot(xhci, slot); + if (xdev->UsbPdo) (void)UsbRemovePdoSubtree(xdev->UsbPdo, true); - if (xdev->Allocated) - (void)XhciDisableSlot(xhci, slot); - if (xdev->UsbDevice) { xdev->UsbDevice->State = UsbDeviceRemoved; UsbDereferenceDevice(xdev->UsbDevice); + xdev->UsbDevice = (USB_DEVICE*)0; } + XhciReleaseDeviceDma(xdev); memset(xdev, 0, sizeof(*xdev)); if (hub->Kind == UsbHubKindRoot && port <= USB_ROOT_HUB_MAX_PORTS) { hub->RootPortSlot[port] = 0; @@ -216,9 +264,9 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub, uint16_t value, uint16_t index, void* buffer, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint16_t length) { - if (!hub || !hub->LowerDevice || !buffer || !bufferPhys || !length) + if (!hub || !hub->LowerDevice || !buffer || !bufferDeviceAddress || !length) return NTLSTATUS_INVALID_PARAMETER; USB_URB urb; @@ -227,7 +275,7 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub, urb.Direction = UsbTransferIn; urb.Device = hub->Device; urb.Buffer = buffer; - urb.BufferPhys = bufferPhys; + urb.BufferDeviceAddress = bufferDeviceAddress; urb.Length = length; urb.Setup.bmRequestType = requestType; urb.Setup.bRequest = request; @@ -260,7 +308,7 @@ static KSTATUS UsbHubControlNoData(USB_HUB_EXTENSION* hub, // Implements the hub Read Descriptor operation. static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* outDesc) { - if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapPhys) + if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapDeviceAddress) return NTLSTATUS_INVALID_PARAMETER; memset(hub->ChangeBitmap, 0, USB_HUB_CHANGE_BITMAP_BYTES); @@ -270,7 +318,7 @@ static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* (uint16_t)(USB_DESC_HUB << 8), 0, hub->ChangeBitmap, - hub->ChangeBitmapPhys, + hub->ChangeBitmapDeviceAddress, USB_HUB_CHANGE_BITMAP_BYTES); if (!KSUCCESS(st)) return st; @@ -293,7 +341,7 @@ static KSTATUS UsbHubGetPortStatus(USB_HUB_EXTENSION* hub, uint8_t port, USB_HUB 0, port, hub->ChangeBitmap, - hub->ChangeBitmapPhys, + hub->ChangeBitmapDeviceAddress, sizeof(USB_HUB_PORT_STATUS)); if (!KSUCCESS(st)) return st; @@ -501,19 +549,8 @@ static void UsbHubHandleRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) { return; } - hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; - hub->RootPortState[port] = UsbHubPortConnectedDebounce; - - UsbDelayUs(xhci->Ddk, 30000); - portsc = XhciRead32(xhci->Op, XhciPortReg(port)); - if (!(portsc & XHCI_PORTSC_CCS)) { - UsbHubClearRootPortFailure(hub, port); - return; - } - - hub->RootPortState[port] = UsbHubPortEnumerating; uint8_t slot = 0; - KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot); if (KSUCCESS(st)) { hub->RootPortSlot[port] = slot; hub->RootPortState[port] = UsbHubPortChildPresent; @@ -544,9 +581,11 @@ static bool UsbHubTakeInitialRootScan(USB_HUB_EXTENSION* hub) { static void UsbHubWorker(void* arg) { USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)arg; while (hub && hub->WorkerRunning) { - KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore); + KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(st)) continue; + if (!hub->WorkerRunning) + break; if (hub->Kind == UsbHubKindRoot) { if (UsbHubTakeInitialRootScan(hub)) @@ -563,6 +602,61 @@ static void UsbHubWorker(void* arg) { UsbHubArmExternalInterrupt(hub); } } + if (hub && hub->WorkerFinished) + KeSignalEvent(hub->WorkerFinished); +} + +// Stops the hub worker and releases its synchronization objects. +static void UsbHubStopWorker(USB_HUB_EXTENSION* hub) { + if (!hub) + return; + + hub->WorkerRunning = false; + if (hub->WorkerSemaphore) + KeSignalSemaphore(hub->WorkerSemaphore); + if (hub->WorkerThread && hub->WorkerFinished) + (void)KeWaitForEvent(hub->WorkerFinished, 0); + + if (hub->WorkerThread) { + ObDereferenceObject((OBJECT*)hub->WorkerThread); + hub->WorkerThread = (THREAD*)0; + } + if (hub->WorkerFinished) { + ObDereferenceObject(&hub->WorkerFinished->Header); + hub->WorkerFinished = (EVENT*)0; + } + if (hub->WorkerSemaphore) { + ObDereferenceObject(&hub->WorkerSemaphore->Header); + hub->WorkerSemaphore = (SEMAPHORE*)0; + } +} + +// Creates the synchronization objects and thread for one hub worker. +static KSTATUS UsbHubStartWorker(USB_HUB_EXTENSION* hub) { + if (!hub) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); + if (!KSUCCESS(status)) + return status; + status = KeEventCreate(EventManualReset, 0, &hub->WorkerFinished); + if (!KSUCCESS(status)) { + UsbHubStopWorker(hub); + return status; + } + + hub->WorkerRunning = true; + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + status = KeCreateKernelThread((uintptr_t)UsbHubWorker, + (uintptr_t)hub, + &workerParameters, + &hub->WorkerThread); + if (!KSUCCESS(status)) { + hub->WorkerRunning = false; + UsbHubStopWorker(hub); + return status; + } + return NTLSTATUS_SUCCESS; } // Implements the hub Do Root Port Scan operation. @@ -592,10 +686,8 @@ static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd) { } attempted++; - hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; - hub->RootPortState[port] = UsbHubPortEnumerating; uint8_t slot = 0; - KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot); if (KSUCCESS(st)) { hub->RootPortSlot[port] = slot; hub->RootPortState[port] = UsbHubPortChildPresent; @@ -734,7 +826,7 @@ static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub) { hub->ChangeUrb.Device = hub->Device; hub->ChangeUrb.Pipe = hub->InterruptPipe; hub->ChangeUrb.Buffer = hub->ChangeBitmap; - hub->ChangeUrb.BufferPhys = hub->ChangeBitmapPhys; + hub->ChangeUrb.BufferDeviceAddress = hub->ChangeBitmapDeviceAddress; hub->ChangeUrb.Length = hub->InterruptMaxPacket; if (hub->ChangeUrb.Length > USB_HUB_CHANGE_BITMAP_BYTES) hub->ChangeUrb.Length = USB_HUB_CHANGE_BITMAP_BYTES; @@ -756,20 +848,10 @@ static KSTATUS UsbHubStartRootDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* h if (hub->PortCount > USB_ROOT_HUB_MAX_PORTS) hub->PortCount = USB_ROOT_HUB_MAX_PORTS; - KSTATUS st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); + KSTATUS st = UsbHubStartWorker(hub); if (!KSUCCESS(st)) return st; - hub->WorkerRunning = true; - st = KeCreateKernelThread((uintptr_t)UsbHubWorker, - (uintptr_t)hub, - PriorityNormal, - &hub->WorkerThread); - if (!KSUCCESS(st)) { - hub->WorkerRunning = false; - return st; - } - hub->Started = true; device->PnpState = DevicePnpStarted; if (hub->HubPdo) @@ -784,7 +866,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO if (!hub || !hub->Device) return NTLSTATUS_INVALID_PARAMETER; - hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapPhys); + hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapDeviceAddress, &hub->ChangeBitmapDma); if (!hub->ChangeBitmap) return NTLSTATUS_NO_MEMORY; @@ -793,6 +875,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO KSTATUS st = UsbHubReadDescriptor(hub, &desc); if (!KSUCCESS(st)) { DbgPrint("usb-hub: slot=%u descriptor read failed st=0x%x\n", hub->Device->SlotId, st); + UsbHubReleaseDma(hub); return st; } @@ -809,17 +892,9 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO UsbDelayUs(((XHCI_CONTROLLER*)hub->Hcd->ControllerContext)->Ddk, (uint32_t)hub->PowerOnToPowerGoodMs * 1000u); - st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); - if (!KSUCCESS(st)) - return st; - - hub->WorkerRunning = true; - st = KeCreateKernelThread((uintptr_t)UsbHubWorker, - (uintptr_t)hub, - PriorityNormal, - &hub->WorkerThread); + st = UsbHubStartWorker(hub); if (!KSUCCESS(st)) { - hub->WorkerRunning = false; + UsbHubReleaseDma(hub); return st; } @@ -829,8 +904,11 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO hub->InterruptMaxPacket, hub->InterruptInterval, &hub->InterruptPipe); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + UsbHubStopWorker(hub); + UsbHubReleaseDma(hub); return st; + } hub->Started = true; device->PnpState = DevicePnpStarted; @@ -897,6 +975,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { (void)UsbCancelUrb(&hub->ChangeUrb); if (hub->WorkerSemaphore) KeSignalSemaphore(hub->WorkerSemaphore); + UsbHubStopWorker(hub); // Remove downstream PDOs owned by this hub FDO. The explicit // slot arrays are still used as the hardware lookup table; the @@ -929,8 +1008,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { UsbDereferencePipe(hub->InterruptPipe); hub->InterruptPipe = (USB_PIPE*)0; } - if (hub->WorkerSemaphore) - KeSignalSemaphore(hub->WorkerSemaphore); + UsbHubStopWorker(hub); if (hub->Kind == UsbHubKindRoot && hub->Hcd && hub->Hcd->RootHubContext == hub) hub->Hcd->RootHubContext = (void*)0; if (hub->RemoveLockReady) { @@ -938,6 +1016,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { IoDestroyRemoveLock(&hub->RemoveLock); hub->RemoveLockReady = false; } + UsbHubReleaseDma(hub); } KSTATUS lower = hub && hub->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS; diff --git a/code/drivers/bus/usb/usb_storage.c b/code/drivers/bus/usb/usb_storage.c index b5677d8..cd2c890 100644 --- a/code/drivers/bus/usb/usb_storage.c +++ b/code/drivers/bus/usb/usb_storage.c @@ -56,10 +56,10 @@ static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_E static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage, bool dataIn, - PHYSICAL_ADDRESS phys, + uint64_t deviceAddress, uint32_t length, uint32_t* outActual) { - if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0) + if (!storage || !storage->LowerDevice || !storage->Device || !deviceAddress || length == 0) return NTLSTATUS_INVALID_PARAMETER; USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe; @@ -73,7 +73,7 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage, urb.Device = storage->Device; urb.Pipe = pipe; urb.Buffer = (void*)0; - urb.BufferPhys = phys; + urb.BufferDeviceAddress = deviceAddress; urb.Length = length; KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb); @@ -83,29 +83,42 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage, } KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, - PHYSICAL_ADDRESS phys, + uint64_t deviceAddress, uint32_t length, uint32_t* outActual) { - return UsbStorageSubmitBulk(storage, true, phys, length, outActual); + return UsbStorageSubmitBulk(storage, true, deviceAddress, length, outActual); } KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, - PHYSICAL_ADDRESS phys, + uint64_t deviceAddress, uint32_t length, uint32_t* outActual) { - return UsbStorageSubmitBulk(storage, false, phys, length, outActual); + return UsbStorageSubmitBulk(storage, false, deviceAddress, length, outActual); } -KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, - const uint8_t* cdb, - uint8_t cdbLen, - void* data, - PHYSICAL_ADDRESS dataPhys, - uint32_t dataLength, - bool dataIn) { +typedef struct USB_STORAGE_SENSE_DATA { + bool Valid; + uint8_t ResponseCode; + uint8_t Key; + uint8_t Asc; + uint8_t Ascq; +} USB_STORAGE_SENSE_DATA; + +// Executes one BOT command without issuing follow-up commands. +static KSTATUS UsbStorageBotCommandRaw(USB_STORAGE_EXTENSION* storage, + const uint8_t* cdb, + uint8_t cdbLen, + void* data, + uint64_t dataDeviceAddress, + uint32_t dataLength, + bool dataIn, + uint8_t* outCswStatus, + uint32_t* outResidue) { + if (outCswStatus) *outCswStatus = 0xFFu; + if (outResidue) *outResidue = 0; if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16) return NTLSTATUS_INVALID_PARAMETER; - if (dataLength && (!data || !dataPhys)) + if (dataLength && (!data || !dataDeviceAddress)) return NTLSTATUS_INVALID_PARAMETER; XHCI_DEVICE* dev = storage->XhciDevice; @@ -120,14 +133,14 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, memcpy(cbw->CommandBlock, cdb, cdbLen); uint32_t actual = 0; - KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual); + KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorDeviceAddress, sizeof(*cbw), &actual); if (!KSUCCESS(st) || actual != sizeof(*cbw)) return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; if (dataLength) { actual = 0; - st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual) - : UsbStorageBulkOut(storage, dataPhys, dataLength, &actual); + st = dataIn ? UsbStorageBulkIn(storage, dataDeviceAddress, dataLength, &actual) + : UsbStorageBulkOut(storage, dataDeviceAddress, dataLength, &actual); if (!KSUCCESS(st)) return st; } @@ -135,7 +148,7 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer; memset(csw, 0, sizeof(*csw)); actual = 0; - st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual); + st = UsbStorageBulkIn(storage, dev->DescriptorDeviceAddress, sizeof(*csw), &actual); if (!KSUCCESS(st) || actual != sizeof(*csw)) return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; @@ -144,15 +157,136 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, dev->SlotId, csw->Signature, csw->Tag, storage->Tag); return NTLSTATUS_DEVICE_ERROR; } - if (csw->Status != USB_MASS_BOT_CSW_PASSED) { - DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u\n", - cdb[0], dev->SlotId, csw->Status, csw->DataResidue); - return NTLSTATUS_DEVICE_ERROR; + + if (outCswStatus) *outCswStatus = csw->Status; + if (outResidue) *outResidue = csw->DataResidue; + return csw->Status == USB_MASS_BOT_CSW_PASSED + ? NTLSTATUS_SUCCESS + : NTLSTATUS_DEVICE_ERROR; +} + +// Returns a readable SCSI sense-key name. +static const char* UsbStorageSenseKeyName(uint8_t key) { + switch (key) { + case 0x00u: return "NO SENSE"; + case 0x01u: return "RECOVERED ERROR"; + case 0x02u: return "NOT READY"; + case 0x03u: return "MEDIUM ERROR"; + case 0x04u: return "HARDWARE ERROR"; + case 0x05u: return "ILLEGAL REQUEST"; + case 0x06u: return "UNIT ATTENTION"; + case 0x07u: return "DATA PROTECT"; + case 0x08u: return "BLANK CHECK"; + case 0x0Bu: return "ABORTED COMMAND"; + case 0x0Eu: return "MISCOMPARE"; + default: return "UNKNOWN"; + } +} + +// Decodes fixed or descriptor-format REQUEST SENSE data. +static void UsbStorageDecodeSense(const uint8_t* bytes, + uint32_t length, + USB_STORAGE_SENSE_DATA* sense) { + if (!sense) return; + memset(sense, 0, sizeof(*sense)); + if (!bytes || length == 0) return; + + sense->ResponseCode = bytes[0] & 0x7Fu; + if ((sense->ResponseCode == 0x70u || sense->ResponseCode == 0x71u) && length >= 14u) { + sense->Key = bytes[2] & 0x0Fu; + sense->Asc = bytes[12]; + sense->Ascq = bytes[13]; + sense->Valid = true; + return; + } + if ((sense->ResponseCode == 0x72u || sense->ResponseCode == 0x73u) && length >= 4u) { + sense->Key = bytes[1] & 0x0Fu; + sense->Asc = bytes[2]; + sense->Ascq = bytes[3]; + sense->Valid = true; + } +} + +// Fetches and decodes sense data after a command failure. +static KSTATUS UsbStorageFetchSense(USB_STORAGE_EXTENSION* storage, + USB_STORAGE_SENSE_DATA* sense) { + if (!storage || !storage->XhciDevice || !sense) + return NTLSTATUS_INVALID_PARAMETER; + + XHCI_DEVICE* dev = storage->XhciDevice; + uint8_t cdb[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 }; + memset(dev->MassBuffer, 0, 18); + + uint8_t cswStatus = 0xFFu; + uint32_t residue = 0; + KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, sizeof(cdb), + dev->MassBuffer, dev->MassBufferDeviceAddress, + 18, true, &cswStatus, &residue); + if (!KSUCCESS(st)) { + DbgPrint("usb-storage: REQUEST SENSE failed slot=%u st=0x%x csw_status=%u residue=%u\n", + dev->SlotId, st, cswStatus, residue); + memset(sense, 0, sizeof(*sense)); + return st; } + uint32_t returned = residue < 18u ? 18u - residue : 0u; + UsbStorageDecodeSense(dev->MassBuffer, returned, sense); return NTLSTATUS_SUCCESS; } +// Executes one BOT command and reports SCSI sense data on command failure. +static KSTATUS UsbStorageExecuteCommand(USB_STORAGE_EXTENSION* storage, + const uint8_t* cdb, + uint8_t cdbLen, + void* data, + uint64_t dataDeviceAddress, + uint32_t dataLength, + bool dataIn, + USB_STORAGE_SENSE_DATA* outSense) { + if (outSense) memset(outSense, 0, sizeof(*outSense)); + + uint8_t cswStatus = 0xFFu; + uint32_t residue = 0; + KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, cdbLen, data, + dataDeviceAddress, dataLength, dataIn, + &cswStatus, &residue); + if (KSUCCESS(st)) return st; + + XHCI_DEVICE* dev = storage ? storage->XhciDevice : (XHCI_DEVICE*)0; + uint32_t slot = dev ? dev->SlotId : 0; + if (cswStatus != USB_MASS_BOT_CSW_FAILED || !cdb || cdb[0] == SCSI_REQUEST_SENSE) { + DbgPrint("usb-storage: command %02x failed slot=%u st=0x%x csw_status=%u residue=%u\n", + cdb ? cdb[0] : 0, slot, st, cswStatus, residue); + return st; + } + + USB_STORAGE_SENSE_DATA sense; + KSTATUS senseStatus = UsbStorageFetchSense(storage, &sense); + if (KSUCCESS(senseStatus) && sense.Valid) { + DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=%s key=%02x asc=%02x ascq=%02x\n", + cdb[0], slot, cswStatus, residue, + UsbStorageSenseKeyName(sense.Key), sense.Key, sense.Asc, sense.Ascq); + if (outSense) *outSense = sense; + } else { + DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=unavailable\n", + cdb[0], slot, cswStatus, residue); + } + return st; +} + +// Executes one BOT command through the sense-reporting path. +KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, + const uint8_t* cdb, + uint8_t cdbLen, + void* data, + uint64_t dataDeviceAddress, + uint32_t dataLength, + bool dataIn) { + return UsbStorageExecuteCommand(storage, cdb, cdbLen, data, + dataDeviceAddress, dataLength, dataIn, + (USB_STORAGE_SENSE_DATA*)0); +} + // Implements the be32 operation. uint32_t UsbBe32(const uint8_t* p) { return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | @@ -182,7 +316,7 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) { uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 }; memset(dev->MassBuffer, 0, 36); KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), - dev->MassBuffer, dev->MassBufferPhys, + dev->MassBuffer, dev->MassBufferDeviceAddress, 36, true); if (!KSUCCESS(st)) return st; @@ -195,18 +329,6 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) { return NTLSTATUS_SUCCESS; } -// Implements the storage Request Sense operation. -KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) { - if (!storage || !storage->XhciDevice) - return NTLSTATUS_INVALID_PARAMETER; - - XHCI_DEVICE* dev = storage->XhciDevice; - uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 }; - memset(dev->MassBuffer, 0, 18); - return UsbStorageBotCommand(storage, rs, sizeof(rs), - dev->MassBuffer, dev->MassBufferPhys, 18, true); -} - // Implements the storage Test Unit Ready operation. KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) { if (!storage || !storage->XhciDevice) @@ -222,7 +344,6 @@ KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) { if (KSUCCESS(st)) return NTLSTATUS_SUCCESS; last = st; - (void)UsbStorageRequestSense(storage); UsbDelayUs(xhci->Ddk, 10000u); } @@ -238,7 +359,7 @@ KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) { uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; memset(dev->MassBuffer, 0, 8); KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), - dev->MassBuffer, dev->MassBufferPhys, + dev->MassBuffer, dev->MassBufferDeviceAddress, 8, true); if (!KSUCCESS(st)) return st; @@ -288,7 +409,7 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, memcpy(dev->MassBuffer, buffer, bytes); KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), - dev->MassBuffer, dev->MassBufferPhys, + dev->MassBuffer, dev->MassBufferDeviceAddress, bytes, !write); if (!KSUCCESS(st)) return st; @@ -297,6 +418,48 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, return NTLSTATUS_SUCCESS; } +// Returns whether sense data means SYNCHRONIZE CACHE is unsupported. +static bool UsbStorageSenseRejectsSynchronizeCache(const USB_STORAGE_SENSE_DATA* sense) { + if (!sense || !sense->Valid || sense->Key != 0x05u) + return false; + return sense->Asc == 0x20u || sense->Asc == 0x24u; +} + +// Flushes device write caches through SCSI SYNCHRONIZE CACHE(10). +static KSTATUS UsbStorageSynchronizeCache(USB_STORAGE_EXTENSION* storage) { + if (!storage || !storage->XhciDevice) + return NTLSTATUS_INVALID_PARAMETER; + if (storage->FlushCapability == UsbStorageFlushUnsupported) + return NTLSTATUS_SUCCESS; + + uint8_t cdb[10]; + memset(cdb, 0, sizeof(cdb)); + cdb[0] = SCSI_SYNCHRONIZE_CACHE_10; + + for (uint32_t attempt = 0; attempt < 2; attempt++) { + USB_STORAGE_SENSE_DATA sense; + KSTATUS st = UsbStorageExecuteCommand(storage, cdb, sizeof(cdb), + (void*)0, 0, 0, false, &sense); + if (KSUCCESS(st)) { + storage->FlushCapability = UsbStorageFlushSupported; + return NTLSTATUS_SUCCESS; + } + + if (UsbStorageSenseRejectsSynchronizeCache(&sense)) { + storage->FlushCapability = UsbStorageFlushUnsupported; + DbgPrint("usb-storage: slot=%u does not support SYNCHRONIZE CACHE; flushes use BOT command-completion semantics only\n", + storage->XhciDevice->SlotId); + return NTLSTATUS_SUCCESS; + } + + if (sense.Valid && sense.Key == 0x06u && attempt == 0) + continue; + return st; + } + + return NTLSTATUS_DEVICE_ERROR; +} + // Implements the storage Acquire Io operation. void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) { if (!storage) return; @@ -375,24 +538,70 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) { return NTLSTATUS_PENDING; } +// Flushes one USB mass-storage disk synchronously. +static KSTATUS UsbStorageDispatchFlush(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) + return NTLSTATUS_INVALID_PARAMETER; + + USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device); + if (!storage || !storage->RemoveLockReady) + return NTLSTATUS_OBJECT_DEAD; + + KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp); + if (!KSUCCESS(st)) + return st; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + return NTLSTATUS_INVALID_PARAMETER; + } + + if (!storage->Started || storage->SurpriseRemoved || storage->Removed) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + IoCompleteRequest(irp, NTLSTATUS_OBJECT_DEAD, 0); + return NTLSTATUS_PENDING; + } + + UsbStorageAcquireIo(storage); + st = UsbStorageSynchronizeCache(storage); + UsbStorageReleaseIo(storage); + + IoReleaseRemoveLock(&storage->RemoveLock, irp); + IoCompleteRequest(irp, st, 0); + return NTLSTATUS_PENDING; +} + // Creates mass Storage Memory. KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer) return NTLSTATUS_SUCCESS; - dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingPhys); - dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingPhys); - dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferPhys); - if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer) + dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingDeviceAddress, &dev->MassInRingDma); + dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingDeviceAddress, &dev->MassOutRingDma); + dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferDeviceAddress, &dev->MassBufferDma); + if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer) { + if (gUsbDdk && gUsbDdk->Dma) { + gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma); + } + dev->MassBufferDma = (IO_DMA_BUFFER*)0; + dev->MassOutRingDma = (IO_DMA_BUFFER*)0; + dev->MassInRingDma = (IO_DMA_BUFFER*)0; + dev->MassBuffer = (uint8_t*)0; + dev->MassOutRing = (XHCI_TRB*)0; + dev->MassInRing = (XHCI_TRB*)0; return NTLSTATUS_NO_MEMORY; + } dev->MassInEnqueue = 0; dev->MassInCycle = 1; dev->MassOutEnqueue = 0; dev->MassOutCycle = 1; dev->MassTag = 0x1000u + dev->SlotId; - XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingPhys, XHCI_BULK_RING_TRBS, dev->MassInCycle); - XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingPhys, XHCI_BULK_RING_TRBS, dev->MassOutCycle); + XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassInCycle); + XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassOutCycle); return NTLSTATUS_SUCCESS; } @@ -431,21 +640,21 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de epIn[0] = 0; epIn[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_IN | ((uint32_t)inMaxPacket << 16); - epIn[2] = (uint32_t)(dev->MassInRingPhys | XHCI_EP_DCS); - epIn[3] = (uint32_t)((dev->MassInRingPhys | XHCI_EP_DCS) >> 32); + epIn[2] = (uint32_t)(dev->MassInRingDeviceAddress | XHCI_EP_DCS); + epIn[3] = (uint32_t)((dev->MassInRingDeviceAddress | XHCI_EP_DCS) >> 32); epIn[4] = inMaxPacket; epOut[0] = 0; epOut[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_OUT | ((uint32_t)outMaxPacket << 16); - epOut[2] = (uint32_t)(dev->MassOutRingPhys | XHCI_EP_DCS); - epOut[3] = (uint32_t)((dev->MassOutRingPhys | XHCI_EP_DCS) >> 32); + epOut[2] = (uint32_t)(dev->MassOutRingDeviceAddress | XHCI_EP_DCS); + epOut[3] = (uint32_t)((dev->MassOutRingDeviceAddress | XHCI_EP_DCS) >> 32); epOut[4] = outMaxPacket; XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -549,6 +758,7 @@ static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSIO storage->BulkInMaxPacket = bulkIn->wMaxPacketSize; storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize; storage->Tag = 0x1000u + storage->Device->SlotId; + storage->FlushCapability = UsbStorageFlushUnknown; xdev->MassCandidate = true; xdev->MassInterface = storage->InterfaceNumber; @@ -844,6 +1054,7 @@ KSTATUS UsbInitializeStorageDriver(void) { gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice; gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite; gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite; + gUsbStorageDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = UsbStorageDispatchFlush; gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp; return IoRegisterDriverObject(&gUsbStorageDriver); } diff --git a/code/drivers/bus/usb/usb_xhci.c b/code/drivers/bus/usb/usb_xhci.c index 842b319..a62ca5a 100644 --- a/code/drivers/bus/usb/usb_xhci.c +++ b/code/drivers/bus/usb/usb_xhci.c @@ -73,32 +73,105 @@ bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, return false; } -void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, - PHYSICAL_ADDRESS* outPhys) { - if (!outPhys || pages == 0) - return NULL; +// Allocates page-aligned common-buffer memory from the controller adapter. +void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer) { + if (!xhci || !xhci->DmaAdapter || !outDeviceAddress || !outBuffer || pages == 0 || + pages > ((size_t)-1) / PAGE_SIZE) + return (void*)0; - // Most xHCI runtime objects are one page. Do not route those through the - // physically-contiguous allocator: on real machines that allocator is a - // scarce DMA primitive, while a single frame is already contiguous. QEMU - // hid this because it had simple low memory; laptop firmware does not. - PHYSICAL_ADDRESS phys = 0; - if (pages == 1) { - phys = MmPfnAllocate(MmPfnUseDma); - } else { - phys = MmPfnAllocateContiguous(pages, MmPfnUseDma); + IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0; + KSTATUS status = xhci->Ddk->Dma->AllocateCommonBuffer(xhci->DmaAdapter, + pages * (size_t)PAGE_SIZE, + PAGE_SIZE, + &buffer); + if (!KSUCCESS(status)) { + DbgPrint("usb: DMA allocation failed pages=%llu st=0x%x\n", + (unsigned long long)pages, status); + return (void*)0; } - if (!phys) { - DbgPrint("usb: DMA allocation failed pages=%llu\n", - (unsigned long long)pages); - return NULL; + *outDeviceAddress = xhci->Ddk->Dma->GetCommonBufferDeviceAddress(buffer); + *outBuffer = buffer; + return xhci->Ddk->Dma->GetCommonBufferAddress(buffer); +} + +// Releases all DMA allocations owned by one xHCI slot. +void XhciReleaseDeviceDma(XHCI_DEVICE* dev) { + if (!dev || !gUsbDdk || !gUsbDdk->Dma) + return; + + for (uint32_t endpoint = 1; endpoint < 32; endpoint++) { + USB_PIPE* pipe = dev->EndpointPipes[endpoint]; + dev->EndpointPipes[endpoint] = (USB_PIPE*)0; + if (pipe) + UsbDereferencePipe(pipe); } - void* virt = xhci->Ddk->Memory->PhysToVirt(phys); - memset(virt, 0, pages * (size_t)PAGE_SIZE); - *outPhys = phys; - return virt; + gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->DescriptorDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->Ep0RingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->DeviceContextDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->InputContextDma); + + dev->MassBufferDma = (IO_DMA_BUFFER*)0; + dev->HidBufferDma = (IO_DMA_BUFFER*)0; + dev->DescriptorDma = (IO_DMA_BUFFER*)0; + dev->MassOutRingDma = (IO_DMA_BUFFER*)0; + dev->MassInRingDma = (IO_DMA_BUFFER*)0; + dev->HidRingDma = (IO_DMA_BUFFER*)0; + dev->Ep0RingDma = (IO_DMA_BUFFER*)0; + dev->DeviceContextDma = (IO_DMA_BUFFER*)0; + dev->InputContextDma = (IO_DMA_BUFFER*)0; + + dev->MassBuffer = (uint8_t*)0; + dev->HidBuffer = (uint8_t*)0; + dev->DescriptorBuffer = (uint8_t*)0; + dev->MassOutRing = (XHCI_TRB*)0; + dev->MassInRing = (XHCI_TRB*)0; + dev->HidRing = (XHCI_TRB*)0; + dev->Ep0Ring = (XHCI_TRB*)0; + dev->DeviceContext = (uint8_t*)0; + dev->InputContext = (uint8_t*)0; +} + +// Releases controller-wide xHCI DMA allocations. +static void XhciReleaseControllerDma(XHCI_CONTROLLER* xhci) { + if (!xhci || !xhci->Ddk || !xhci->Ddk->Dma) + return; + + for (uint32_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++) + XhciReleaseDeviceDma(&xhci->Devices[slot]); + for (uint16_t i = 0; i < XHCI_MAX_SCRATCHPADS; i++) { + xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadDma[i]); + xhci->ScratchpadDma[i] = (IO_DMA_BUFFER*)0; + xhci->ScratchpadDeviceAddress[i] = 0; + } + + xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadArrayDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->ErstDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->EventRingDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->CommandRingDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->DcbaaDma); + xhci->ScratchpadArrayDma = (IO_DMA_BUFFER*)0; + xhci->ErstDma = (IO_DMA_BUFFER*)0; + xhci->EventRingDma = (IO_DMA_BUFFER*)0; + xhci->CommandRingDma = (IO_DMA_BUFFER*)0; + xhci->DcbaaDma = (IO_DMA_BUFFER*)0; + xhci->ScratchpadArray = (uint64_t*)0; + xhci->Erst = (XHCI_ERST_ENTRY*)0; + xhci->EventRing = (XHCI_TRB*)0; + xhci->CommandRing = (XHCI_TRB*)0; + xhci->Dcbaa = (uint64_t*)0; + + xhci->Ddk->Dma->PutAdapter(xhci->DmaAdapter); + xhci->DmaAdapter = (IO_DMA_ADAPTER*)0; } // Implements the scratchpad Count operation. @@ -328,26 +401,26 @@ void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) { // Updates erdp. void XhciUpdateErdp(XHCI_CONTROLLER* xhci) { volatile uint8_t* ir0 = xhci->Runtime + 0x20u; - uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB); + uint64_t erdp = xhci->EventRingDeviceAddress + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB); XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear } // Returns command Waiter Locked. -static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) { +static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandDeviceAddress) { for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { - if (w->CommandPhys && w->CommandPhys == commandPhys) + if (w->CommandDeviceAddress && w->CommandDeviceAddress == commandDeviceAddress) return w; } return (XHCI_WAIT_TARGET*)0; } static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci, - uint64_t transferPhys, + uint64_t transferDeviceAddress, uint8_t slotId, uint8_t endpointId) { for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { - if (w->TransferPhys && - w->TransferPhys == transferPhys && + if (w->TransferDeviceAddress && + w->TransferDeviceAddress == transferDeviceAddress && w->TransferSlotId == slotId && w->TransferEndpointId == endpointId) { return w; @@ -357,11 +430,11 @@ static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci, } static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci, - uint64_t commandPhys, + uint64_t commandDeviceAddress, uint32_t completionCode, uint8_t slotId) { uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); - XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys); + XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandDeviceAddress); if (w) { w->Command.CompletionCode = completionCode; w->Command.SlotId = slotId; @@ -373,13 +446,13 @@ static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci, } static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci, - uint64_t transferPhys, + uint64_t transferDeviceAddress, uint8_t slotId, uint8_t endpointId, uint32_t completionCode, uint32_t residue) { uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); - XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId); + XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferDeviceAddress, slotId, endpointId); if (w) { w->Transfer.CompletionCode = completionCode; w->Transfer.SlotId = slotId; @@ -397,7 +470,7 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { (void)target; bool processed = false; - for (;;) { + while (true) { XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue]; uint32_t control = ev->Control; uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE); @@ -460,8 +533,8 @@ bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { // Waits for target Complete. bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) { if (!target) return false; - if (target->CommandPhys && !target->CommandMatched) return false; - if (target->TransferPhys && !target->TransferMatched) return false; + if (target->CommandDeviceAddress && !target->CommandMatched) return false; + if (target->TransferDeviceAddress && !target->TransferMatched) return false; return true; } @@ -568,6 +641,21 @@ static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) { return NTLSTATUS_SUCCESS; } +// Removes a controller from the interrupt-dispatch registry. +static void XhciUnregisterController(XHCI_CONTROLLER* xhci) { + if (!xhci) + return; + for (uint32_t i = 0; i < gXhciControllerCount; i++) { + if (gXhciControllers[i] != xhci) + continue; + for (uint32_t next = i + 1; next < gXhciControllerCount; next++) + gXhciControllers[next - 1] = gXhciControllers[next]; + gXhciControllerCount--; + gXhciControllers[gXhciControllerCount] = (XHCI_CONTROLLER*)0; + return; + } +} + // Implements the isr operation. void XhciIsr(void) { for (uint32_t i = 0; i < gXhciControllerCount; i++) { @@ -663,6 +751,34 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) { return NTLSTATUS_SUCCESS; } +// Halts the controller before any device-visible memory can be released. +static void XhciHaltForDmaRelease(XHCI_CONTROLLER* xhci) { + if (!xhci || !xhci->DmaAdapter) + return; + + xhci->InterruptsEnabled = false; + xhci->Running = false; + if (xhci->SurpriseRemoved || !xhci->Op) + return; + + if (xhci->Runtime) { + volatile uint8_t* interrupter = xhci->Runtime + 0x20u; + XhciWrite32(interrupter, 0x00u, XHCI_IMAN_IP); + } + + uint32_t command = XhciRead32(xhci->Op, XHCI_OP_USBCMD); + command &= ~(XHCI_USBCMD_RS | XHCI_USBCMD_INTE); + XhciWrite32(xhci->Op, XHCI_OP_USBCMD, command); + if (!UsbWaitSet32(xhci->Ddk, + xhci->Op, + XHCI_OP_USBSTS, + XHCI_USBSTS_HCH, + 1000000u)) + KePanic(KE_PANIC_CODE_DRIVER, + "usb: xHCI failed to halt before DMA release usbsts=%08x\n", + XhciRead32(xhci->Op, XHCI_OP_USBSTS)); +} + // Creates scratchpads. KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { if (xhci->ScratchpadCount == 0) return NTLSTATUS_SUCCESS; @@ -672,7 +788,7 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { return NTLSTATUS_NOT_SUPPORTED; } - xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayPhys); + xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayDeviceAddress, &xhci->ScratchpadArrayDma); if (!xhci->ScratchpadArray) { DbgPrint("usb: xHCI scratchpad pointer-array allocation failed count=%u\n", xhci->ScratchpadCount); @@ -680,29 +796,29 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { } for (uint16_t i = 0; i < xhci->ScratchpadCount; i++) { - void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadPhys[i]); + void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadDeviceAddress[i], &xhci->ScratchpadDma[i]); if (!page) { DbgPrint("usb: xHCI scratchpad buffer allocation failed index=%u/%u\n", i, xhci->ScratchpadCount); return NTLSTATUS_NO_MEMORY; } (void)page; - xhci->ScratchpadArray[i] = xhci->ScratchpadPhys[i]; + xhci->ScratchpadArray[i] = xhci->ScratchpadDeviceAddress[i]; } - xhci->Dcbaa[0] = xhci->ScratchpadArrayPhys; + xhci->Dcbaa[0] = xhci->ScratchpadArrayDeviceAddress; UsbTrace("usb: xHCI allocated %u scratchpad buffer(s) array=0x%016llX\n", xhci->ScratchpadCount, - (unsigned long long)xhci->ScratchpadArrayPhys); + (unsigned long long)xhci->ScratchpadArrayDeviceAddress); return NTLSTATUS_SUCCESS; } // Creates runtime Memory. KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { - xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys); - xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys); - xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingPhys); - xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstPhys); + xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaDeviceAddress, &xhci->DcbaaDma); + xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingDeviceAddress, &xhci->CommandRingDma); + xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingDeviceAddress, &xhci->EventRingDma); + xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstDeviceAddress, &xhci->ErstDma); if (!xhci->Dcbaa || !xhci->CommandRing || !xhci->EventRing || !xhci->Erst) { DbgPrint("usb: xHCI runtime DMA allocation failed dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", @@ -716,14 +832,14 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { xhci->EventCycle = 1; XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)xhci->CommandRingPhys; - link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); + link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress; + link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | xhci->CommandCycle; - xhci->Erst[0].RingSegmentBase = xhci->EventRingPhys; + xhci->Erst[0].RingSegmentBase = xhci->EventRingDeviceAddress; xhci->Erst[0].RingSegmentSize = XHCI_EVENT_TRBS; xhci->Erst[0].Reserved = 0; @@ -731,10 +847,10 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { if (!KSUCCESS(st)) return st; UsbTrace("usb: xHCI dma dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", - (unsigned long long)xhci->DcbaaPhys, - (unsigned long long)xhci->CommandRingPhys, - (unsigned long long)xhci->EventRingPhys, - (unsigned long long)xhci->ErstPhys); + (unsigned long long)xhci->DcbaaDeviceAddress, + (unsigned long long)xhci->CommandRingDeviceAddress, + (unsigned long long)xhci->EventRingDeviceAddress, + (unsigned long long)xhci->ErstDeviceAddress); return NTLSTATUS_SUCCESS; } @@ -743,8 +859,8 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) { uint8_t slots = xhci->MaxSlots; if (slots == 0) return NTLSTATUS_DEVICE_ERROR; - XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaPhys); - XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingPhys | xhci->CommandCycle); + XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaDeviceAddress); + XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingDeviceAddress | xhci->CommandCycle); XhciWrite32(xhci->Op, XHCI_OP_DNCTRL, 0); XhciWrite32(xhci->Op, XHCI_OP_CONFIG, slots); @@ -752,30 +868,30 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) { XhciWrite32(ir0, 0x00u, XHCI_IMAN_IP); // clear pending, leave disabled until smoke test passes XhciWrite32(ir0, 0x04u, 0); // no interrupt moderation yet XhciWrite32(ir0, 0x08u, 1); // one ERST entry - XhciWrite64(ir0, 0x10u, xhci->ErstPhys); - XhciWrite64(ir0, 0x18u, xhci->EventRingPhys); + XhciWrite64(ir0, 0x10u, xhci->ErstDeviceAddress); + XhciWrite64(ir0, 0x18u, xhci->EventRingDeviceAddress); XhciBarrier(); return NTLSTATUS_SUCCESS; } -// Implements the command Trb Phys operation. -uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) { - return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB); +// Returns the device address of one command TRB. +uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index) { + return xhci->CommandRingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB); } static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, - uint64_t* outPhys) { + uint64_t* outDeviceAddress) { if (!xhci || !xhci->CommandRing) return NTLSTATUS_INVALID_PARAMETER; if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) { XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)xhci->CommandRingPhys; - link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); + link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress; + link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | @@ -787,7 +903,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, uint32_t index = xhci->CommandEnqueue++; XHCI_TRB* trb = &xhci->CommandRing[index]; - uint64_t trbPhys = XhciCommandTrbPhys(xhci, index); + uint64_t trbDeviceAddress = XhciCommandTrbDeviceAddress(xhci, index); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); @@ -795,7 +911,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, XhciBarrier(); trb->Control = controlWithoutCycle | xhci->CommandCycle; - if (outPhys) *outPhys = trbPhys; + if (outDeviceAddress) *outDeviceAddress = trbDeviceAddress; return NTLSTATUS_SUCCESS; } @@ -811,14 +927,14 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, memset(&target, 0, sizeof(target)); uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock); - uint64_t trbPhys = 0; - KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys); + uint64_t trbDeviceAddress = 0; + KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbDeviceAddress); if (!KSUCCESS(st)) { KeReleaseSpinlock(&xhci->CommandLock, cmdFlags); return st; } - target.CommandPhys = trbPhys; + target.CommandDeviceAddress = trbDeviceAddress; XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); @@ -912,7 +1028,7 @@ KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { memset(&result, 0, sizeof(result)); KSTATUS st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_ADDRESS_DEVICE << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -1059,20 +1175,22 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) { // Creates device Memory. KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { - dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys); - dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys); - dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingPhys); - dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorPhys); + dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextDeviceAddress, &dev->InputContextDma); + dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextDeviceAddress, &dev->DeviceContextDma); + dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingDeviceAddress, &dev->Ep0RingDma); + dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorDeviceAddress, &dev->DescriptorDma); - if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) + if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) { + XhciReleaseDeviceDma(dev); return NTLSTATUS_NO_MEMORY; + } dev->Ep0Enqueue = 0; dev->Ep0Cycle = 1; XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)dev->Ep0RingPhys; - link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); + link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress; + link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | @@ -1116,31 +1234,31 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { ep0[0] = 0; ep0[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_CONTROL | (maxPacket << 16); - ep0[2] = (uint32_t)(dev->Ep0RingPhys | XHCI_EP_DCS); - ep0[3] = (uint32_t)((dev->Ep0RingPhys | XHCI_EP_DCS) >> 32); + ep0[2] = (uint32_t)(dev->Ep0RingDeviceAddress | XHCI_EP_DCS); + ep0[3] = (uint32_t)((dev->Ep0RingDeviceAddress | XHCI_EP_DCS) >> 32); ep0[4] = 8; - xhci->Dcbaa[dev->SlotId] = dev->DeviceContextPhys; + xhci->Dcbaa[dev->SlotId] = dev->DeviceContextDeviceAddress; XhciBarrier(); } -// Implements the ep0 Trb Phys operation. -uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) { - return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB); +// Returns the device address of one endpoint-zero TRB. +uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index) { + return dev->Ep0RingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB); } KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, - uint64_t* outPhys) { + uint64_t* outDeviceAddress) { if (!dev || !dev->Ep0Ring) return NTLSTATUS_INVALID_PARAMETER; if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) { XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)dev->Ep0RingPhys; - link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); + link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress; + link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | @@ -1152,7 +1270,7 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint32_t index = dev->Ep0Enqueue++; XHCI_TRB* trb = &dev->Ep0Ring[index]; - uint64_t phys = XhciEp0TrbPhys(dev, index); + uint64_t deviceAddress = XhciEp0TrbDeviceAddress(dev, index); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); @@ -1160,14 +1278,14 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, XhciBarrier(); trb->Control = controlWithoutCycle | dev->Ep0Cycle; - if (outPhys) *outPhys = phys; + if (outDeviceAddress) *outDeviceAddress = deviceAddress; return NTLSTATUS_SUCCESS; } KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint16_t length, uint32_t timeoutUs) { if (!buffer || length == 0 || length > PAGE_SIZE) @@ -1188,26 +1306,26 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, st = XhciEp0AppendTrb( dev, - bufferPhys, + bufferDeviceAddress, length, (XHCI_TRB_TYPE_DATA << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_DATA_DIR_IN, NULL); if (!KSUCCESS(st)) return st; - uint64_t statusTrbPhys = 0; + uint64_t statusTrbDeviceAddress = 0; st = XhciEp0AppendTrb( dev, 0, 0, (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_IOC, - &statusTrbPhys); + &statusTrbDeviceAddress); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); - target.TransferPhys = statusTrbPhys; + target.TransferDeviceAddress = statusTrbDeviceAddress; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; XhciRegisterWaitTarget(xhci, &target); @@ -1235,9 +1353,9 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint16_t length) { - return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length, + return XhciControlInTimeout(xhci, dev, setup, buffer, bufferDeviceAddress, length, USB_XHCI_CONTROL_TIMEOUT_US); } @@ -1254,7 +1372,7 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, NULL); if (!KSUCCESS(st)) return st; - uint64_t statusTrbPhys = 0; + uint64_t statusTrbDeviceAddress = 0; st = XhciEp0AppendTrb( dev, 0, @@ -1262,12 +1380,12 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_STATUS_DIR_IN | XHCI_TRB_IOC, - &statusTrbPhys); + &statusTrbDeviceAddress); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); - target.TransferPhys = statusTrbPhys; + target.TransferDeviceAddress = statusTrbDeviceAddress; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; XhciRegisterWaitTarget(xhci, &target); @@ -1304,7 +1422,7 @@ KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US; KSTATUS st = XhciControlInTimeout(xhci, dev, &setup, dev->DescriptorBuffer, - dev->DescriptorPhys, + dev->DescriptorDeviceAddress, sizeof(USB_DEVICE_DESCRIPTOR), timeoutUs); if (!KSUCCESS(st)) return st; @@ -1339,7 +1457,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, KSTATUS st = XhciControlIn(xhci, dev, &setup, dev->DescriptorBuffer, - dev->DescriptorPhys, + dev->DescriptorDeviceAddress, sizeof(USB_CONFIGURATION_DESCRIPTOR)); if (!KSUCCESS(st)) return st; @@ -1366,7 +1484,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, setup.wLength = total; st = XhciControlIn(xhci, dev, &setup, dev->DescriptorBuffer, - dev->DescriptorPhys, + dev->DescriptorDeviceAddress, total); if (!KSUCCESS(st)) return st; @@ -1415,17 +1533,17 @@ uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) { return interval; } -KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, +KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, - uint32_t controlWithoutCycle, uint64_t* outPhys) { + uint32_t controlWithoutCycle, uint64_t* outDeviceAddress) { if (!ring || !enqueue || !cycle || trbCount < 2) return NTLSTATUS_INVALID_PARAMETER; if (*enqueue >= trbCount - 1u) { XHCI_TRB* link = &ring[trbCount - 1u]; - link->ParameterLo = (uint32_t)ringPhys; - link->ParameterHi = (uint32_t)(ringPhys >> 32); + link->ParameterLo = (uint32_t)ringDeviceAddress; + link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | @@ -1437,7 +1555,7 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t index = (*enqueue)++; XHCI_TRB* trb = &ring[index]; - uint64_t phys = ringPhys + (uint64_t)index * sizeof(XHCI_TRB); + uint64_t deviceAddress = ringDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); @@ -1445,16 +1563,16 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, XhciBarrier(); trb->Control = controlWithoutCycle | *cycle; - if (outPhys) *outPhys = phys; + if (outDeviceAddress) *outDeviceAddress = deviceAddress; return NTLSTATUS_SUCCESS; } -void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, +void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint8_t cycle) { memset(ring, 0, trbCount * sizeof(XHCI_TRB)); XHCI_TRB* link = &ring[trbCount - 1u]; - link->ParameterLo = (uint32_t)ringPhys; - link->ParameterHi = (uint32_t)(ringPhys >> 32); + link->ParameterLo = (uint32_t)ringDeviceAddress; + link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | @@ -1464,34 +1582,34 @@ void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, - PHYSICAL_ADDRESS ringPhys, + uint64_t ringDeviceAddress, uint32_t* enqueue, uint8_t* cycle, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint32_t length, uint32_t* outActual) { - if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferPhys || length == 0) + if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferDeviceAddress || length == 0) return NTLSTATUS_INVALID_PARAMETER; if (length > USB_STORAGE_MAX_BYTES) return NTLSTATUS_INVALID_PARAMETER; - uint64_t trbPhys = 0; + uint64_t trbDeviceAddress = 0; KSTATUS st = XhciRingAppend(ring, - ringPhys, + ringDeviceAddress, XHCI_BULK_RING_TRBS, enqueue, cycle, - bufferPhys, + bufferDeviceAddress, length, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, - &trbPhys); + &trbDeviceAddress); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); - target.TransferPhys = trbPhys; + target.TransferDeviceAddress = trbDeviceAddress; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = endpointId; @@ -1537,7 +1655,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P if (!ctx) return NTLSTATUS_NO_MEMORY; memset(ctx, 0, sizeof(*ctx)); - ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys); + ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingDeviceAddress, &ctx->RingDma); if (!ctx->Ring) { MmFreePool(ctx); return NTLSTATUS_NO_MEMORY; @@ -1546,7 +1664,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P ctx->Cycle = 1; ctx->EndpointId = dci; ctx->Type = pipe->Type; - XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle); + XhciInitialiseTransferRing(ctx->Ring, ctx->RingDeviceAddress, XHCI_PIPE_RING_TRBS, ctx->Cycle); memset(dev->InputContext, 0, PAGE_SIZE); uint32_t* icc = XhciInputControlContext(xhci, dev); @@ -1574,21 +1692,22 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P } else if (pipe->Type == UsbPipeBulk) { endpointType = XHCI_EP_TYPE_BULK_OUT; } else { + xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma); MmFreePool(ctx); return NTLSTATUS_NOT_SUPPORTED; } ep[0] = (uint32_t)interval << 16; ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16); - ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS); - ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32); + ep[2] = (uint32_t)(ctx->RingDeviceAddress | XHCI_EP_DCS); + ep[3] = (uint32_t)((ctx->RingDeviceAddress | XHCI_EP_DCS) >> 32); ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16); XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); KSTATUS st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -1596,6 +1715,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) { DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n", dev->SlotId, dci, pipe->Type, st, result.CompletionCode); + xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma); MmFreePool(ctx); return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; } @@ -1609,7 +1729,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P // Implements the hcd Submit Interrupt Urb operation. static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { - if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) + if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0) return NTLSTATUS_INVALID_PARAMETER; if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) return NTLSTATUS_NOT_SUPPORTED; @@ -1623,22 +1743,22 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev if (ctx->ActiveTransfer) return NTLSTATUS_QUEUE_FULL; - uint64_t trbPhys = 0; + uint64_t trbDeviceAddress = 0; st = XhciRingAppend(ctx->Ring, - ctx->RingPhys, + ctx->RingDeviceAddress, XHCI_PIPE_RING_TRBS, &ctx->Enqueue, &ctx->Cycle, - urb->BufferPhys, + urb->BufferDeviceAddress, urb->Length, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, - &trbPhys); + &trbDeviceAddress); if (!KSUCCESS(st)) return st; ctx->ActiveTransfer = true; - ctx->ActiveTrbPhys = trbPhys; + ctx->ActiveTrbDeviceAddress = trbDeviceAddress; ctx->ActiveUrb = urb; urb->Pending = true; urb->Cancelled = false; @@ -1651,7 +1771,7 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev return NTLSTATUS_PENDING; } -void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, +void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue) { if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32) @@ -1663,12 +1783,12 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, return; XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; - if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys) + if (!ctx->ActiveTransfer || ctx->ActiveTrbDeviceAddress != trbDeviceAddress) return; USB_URB* urb = ctx->ActiveUrb; ctx->ActiveTransfer = false; - ctx->ActiveTrbPhys = 0; + ctx->ActiveTrbDeviceAddress = 0; ctx->ActiveUrb = (USB_URB*)0; if (!urb) @@ -1704,23 +1824,23 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, // Implements the hcd Submit Bulk Urb operation. static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { - if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) + if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0) return NTLSTATUS_INVALID_PARAMETER; uint8_t endpointId = urb->Pipe->EndpointId; XHCI_TRB* ring = (XHCI_TRB*)0; - PHYSICAL_ADDRESS ringPhys = 0; + uint64_t ringDeviceAddress = 0; uint32_t* enqueue = (uint32_t*)0; uint8_t* cycle = (uint8_t*)0; if (endpointId == dev->MassBulkInEndpointId) { ring = dev->MassInRing; - ringPhys = dev->MassInRingPhys; + ringDeviceAddress = dev->MassInRingDeviceAddress; enqueue = &dev->MassInEnqueue; cycle = &dev->MassInCycle; } else if (endpointId == dev->MassBulkOutEndpointId) { ring = dev->MassOutRing; - ringPhys = dev->MassOutRingPhys; + ringDeviceAddress = dev->MassOutRingDeviceAddress; enqueue = &dev->MassOutEnqueue; cycle = &dev->MassOutCycle; } else { @@ -1732,10 +1852,10 @@ static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB dev, endpointId, ring, - ringPhys, + ringDeviceAddress, enqueue, cycle, - urb->BufferPhys, + urb->BufferDeviceAddress, urb->Length, &actual); urb->ActualLength = KSUCCESS(st) ? actual : 0; @@ -1759,7 +1879,7 @@ static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) { st = XhciControlNoData(xhci, dev, &urb->Setup); urb->ActualLength = 0; } else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) { - st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length); + st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferDeviceAddress, (uint16_t)urb->Length); urb->ActualLength = KSUCCESS(st) ? urb->Length : 0; } else { st = NTLSTATUS_NOT_SUPPORTED; @@ -1796,9 +1916,29 @@ static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) { } // Implements the hcd Allocate Dma operation. -static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) { +static void* XhciHcdAllocateDma(USB_HCD* hcd, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer) { if (!hcd) return (void*)0; - return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys); + return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, + pages, + outDeviceAddress, + outBuffer); +} + +// Releases the host-controller context attached to a USB pipe. +static void XhciHcdDestroyPipe(USB_HCD* hcd, USB_PIPE* pipe) { + (void)hcd; + if (!pipe || !pipe->HcdPipeContext) + return; + XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; + if (gUsbDdk && gUsbDdk->Dma) + gUsbDdk->Dma->FreeCommonBuffer(ctx->RingDma); + ctx->RingDma = (IO_DMA_BUFFER*)0; + ctx->Ring = (XHCI_TRB*)0; + pipe->HcdPipeContext = (void*)0; + MmFreePool(ctx); } // Implements the hcd Reset Root Port operation. @@ -1819,6 +1959,7 @@ const USB_HCD_OPERATIONS gXhciHcdOperations = { XhciHcdCancelUrb, XhciHcdResetRootPort, XhciHcdClearRootPortChanges, + XhciHcdDestroyPipe, XhciHcdAllocateDma, }; @@ -1886,7 +2027,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { uint64_t mappedLength = 0; st = PciMapBar(xhci->Pdo, 0, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&xhci->Mmio, &mappedLength); if (!KSUCCESS(st)) { @@ -1960,17 +2101,37 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { return NTLSTATUS_NOT_SUPPORTED; } + IO_DMA_REQUIREMENTS dmaRequirements; + memset(&dmaRequirements, 0, sizeof(dmaRequirements)); + dmaRequirements.MaximumDeviceAddress = + (xhci->Hcc1 & XHCI_HCC1_AC64) ? UINT64_MAX : UINT32_MAX; + dmaRequirements.Alignment = 1; + dmaRequirements.Boundary = 0x10000u; + dmaRequirements.MaximumSegmentLength = 0x10000u; + dmaRequirements.MaximumSegments = XHCI_PIPE_RING_TRBS - 1u; + st = xhci->Ddk->Dma->GetAdapter(xhci->Fdo, + &dmaRequirements, + &xhci->DmaAdapter); + if (!KSUCCESS(st)) { + DbgPrint("usb: xHCI DMA adapter creation failed st=0x%x\n", st); + return st; + } + st = XhciAllocateRuntimeMemory(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; st = XhciProgramRuntime(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; st = XhciStart(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; st = XhciSubmitNoopCommand(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; XhciPowerRootPorts(xhci); XhciLogPortStatus(xhci); @@ -1978,13 +2139,13 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { st = UsbCreateRootHubPdo(&xhci->Hcd); if (!KSUCCESS(st) && st != NTLSTATUS_ALREADY_EXISTS) { DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st); - return st; + goto failDma; } st = XhciRegisterController(xhci); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI controller registry full st=0x%x\n", st); - return st; + goto failDma; } if (xhci->Irq != 0xFFu) { @@ -2009,6 +2170,16 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n", xhci->HidDeviceCount); return NTLSTATUS_SUCCESS; + +failDma: + XhciHaltForDmaRelease(xhci); + XhciUnregisterController(xhci); + if (xhci->Hcd.RootHubPdo) { + (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); + xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0; + } + XhciReleaseControllerDma(xhci); + return st; } // Dispatches pnp. @@ -2037,22 +2208,22 @@ KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: device->PnpState = DevicePnpSurpriseRemoved; + xhci->SurpriseRemoved = true; xhci->Running = false; xhci->InterruptsEnabled = false; - if (xhci->Op) { - uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); - XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS); - } if (xhci->Hcd.RootHubPdo) (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: device->PnpState = DevicePnpDeleted; - xhci->Running = false; - xhci->InterruptsEnabled = false; - if (xhci->Hcd.RootHubPdo) + XhciHaltForDmaRelease(xhci); + if (xhci->Hcd.RootHubPdo) { (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); + xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0; + } + XhciUnregisterController(xhci); + XhciReleaseControllerDma(xhci); IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; diff --git a/code/drivers/display/vmwaresvga/vmwaresvga.c b/code/drivers/display/vmwaresvga/vmwaresvga.c index 5fc0bae..1d21eb6 100644 --- a/code/drivers/display/vmwaresvga/vmwaresvga.c +++ b/code/drivers/display/vmwaresvga/vmwaresvga.c @@ -748,10 +748,103 @@ static KSTATUS VmwSvgaDisplayGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMo return NTLSTATUS_SUCCESS; } -// Implements the display Blit operation. -static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) { +// Streams one pixel row into the write-combined SVGA framebuffer. +static void VmwSvgaCopyToFramebuffer(uint32_t* destination, + const uint32_t* source, + size_t pixelCount) { + if (!destination || !source || pixelCount == 0) + return; + + while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) { + *destination++ = *source++; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_stream_si128((__m128i*)(destination + 0), v0); + _mm_stream_si128((__m128i*)(destination + 4), v1); + _mm_stream_si128((__m128i*)(destination + 8), v2); + _mm_stream_si128((__m128i*)(destination + 12), v3); + source += 16; + destination += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_stream_si128((__m128i*)destination, value); + source += 4; + destination += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + *destination++ = *source++; + pixelCount--; + } +} + +// Copies one row into the retained SVGA shadow and framebuffer in one read pass. +static void VmwSvgaCopyToShadowAndFramebuffer(uint32_t* shadow, + uint32_t* framebuffer, + const uint32_t* source, + size_t pixelCount) { + if (!shadow || !framebuffer || !source || pixelCount == 0) + return; + + while (pixelCount != 0 && ((uintptr_t)framebuffer & 15u) != 0) { + uint32_t value = *source++; + *shadow++ = value; + *framebuffer++ = value; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_storeu_si128((__m128i*)(shadow + 0), v0); + _mm_storeu_si128((__m128i*)(shadow + 4), v1); + _mm_storeu_si128((__m128i*)(shadow + 8), v2); + _mm_storeu_si128((__m128i*)(shadow + 12), v3); + _mm_stream_si128((__m128i*)(framebuffer + 0), v0); + _mm_stream_si128((__m128i*)(framebuffer + 4), v1); + _mm_stream_si128((__m128i*)(framebuffer + 8), v2); + _mm_stream_si128((__m128i*)(framebuffer + 12), v3); + source += 16; + shadow += 16; + framebuffer += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_storeu_si128((__m128i*)shadow, value); + _mm_stream_si128((__m128i*)framebuffer, value); + source += 4; + shadow += 4; + framebuffer += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + uint32_t value = *source++; + *shadow++ = value; + *framebuffer++ = value; + pixelCount--; + } +} + +// Implements the display pitched blit operation. +static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) { VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device); - if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0) + if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; DISPLAY_RECT r = { x, y, w, h }; @@ -766,12 +859,60 @@ static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, i size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t); for (int row = 0; row < r.H; row++) { uint32_t* dst = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X; - const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX; + const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX; memcpy(dst, src, bytesPerRow); } return NTLSTATUS_SUCCESS; } +// Presents one pitched rectangle without a shadow-to-framebuffer second pass. +static KSTATUS VmwSvgaDisplayPresentPitched(DISPLAY_DEVICE* device, + int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device); + if (!v || !v->Framebuffer || !v->Backbuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) + return NTLSTATUS_INVALID_PARAMETER; + + DISPLAY_RECT r = { x, y, w, h }; + if (!VmwSvgaClipRect(v, &r)) + return NTLSTATUS_SUCCESS; + + int srcOffsetX = r.X - x; + int srcOffsetY = r.Y - y; + size_t pixelCount = (size_t)r.W; + uint64_t bytes = (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t); + v->Stats.BlitCount++; + v->Stats.PresentCount++; + v->Stats.PresentRectCount++; + v->Stats.DirectPresentCount++; + v->Stats.ScanoutBytes += bytes; + v->Stats.BytesCopied += bytes * 2u; + + for (int row = 0; row < r.H; row++) { + const uint8_t* sourceRow = (const uint8_t*)pixels + + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX; + uint32_t* shadow = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X; + uint32_t* framebuffer = (uint32_t*)(uintptr_t)(v->Framebuffer + + ((size_t)(r.Y + row) * v->BytesPerLine) + + ((size_t)r.X * sizeof(uint32_t))); + VmwSvgaCopyToShadowAndFramebuffer(shadow, framebuffer, source, pixelCount); + } + + _mm_sfence(); + _ReadWriteBarrier(); + KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H); + if (!KSUCCESS(st)) + return st; + return VmwSvgaSyncDevice(v); +} + // Implements the display Fill Rect operation. static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) { VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device); @@ -869,13 +1010,19 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* DISPLAY_RECT r = rects[i]; if (!VmwSvgaClipRect(v, &r)) continue; - v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t); - size_t rowBytes = (size_t)r.W * sizeof(uint32_t); + uint64_t bytes = (uint64_t)(uint32_t)r.W * + (uint64_t)(uint32_t)r.H * sizeof(uint32_t); + v->Stats.BytesCopied += bytes; + v->Stats.ScanoutBytes += bytes; + size_t pixelCount = (size_t)r.W; for (int row = 0; row < r.H; row++) { - volatile uint8_t* dstBytes = v->Framebuffer + ((size_t)(r.Y + row) * v->BytesPerLine) + ((size_t)r.X * sizeof(uint32_t)); + uint32_t* dst = (uint32_t*)(uintptr_t)(v->Framebuffer + + ((size_t)(r.Y + row) * v->BytesPerLine) + + ((size_t)r.X * sizeof(uint32_t))); uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X; - memcpy((void*)dstBytes, src, rowBytes); + VmwSvgaCopyToFramebuffer(dst, src, pixelCount); } + _mm_sfence(); _ReadWriteBarrier(); KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H); @@ -1019,6 +1166,7 @@ static KSTATUS VmwSvgaDisplayQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* o static const DISPLAY_DEVICE_OPS gVmwSvgaDisplayOps = { .GetMode = VmwSvgaDisplayGetMode, .Blit = VmwSvgaDisplayBlit, + .PresentPitched = VmwSvgaDisplayPresentPitched, .FillRect = VmwSvgaDisplayFillRect, .CopyRect = VmwSvgaDisplayCopyRect, .ReadRect = VmwSvgaDisplayReadRect, @@ -1060,15 +1208,19 @@ static KSTATUS VmwSvgaSelectBars(VMWSVGA_DEVICE* v) { return NTLSTATUS_SUCCESS; } -// Implements the mmio Page Flags operation. -static uint64_t VmwSvgaMmioPageFlags(void) { - return PAGE_PRESENT | PAGE_WRITE | PAGE_WRITETHROUGH | PAGE_NOCACHE | PAGE_NX; +// Returns the write-combined mapping flags used for the framebuffer aperture. +static uint64_t VmwSvgaFramebufferPageFlags(void) { + return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_WC | PAGE_NX; +} + +// Returns the uncached mapping flags used for device-control apertures. +static uint64_t VmwSvgaRegisterPageFlags(void) { + return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC | PAGE_NX; } // Maps bars. static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) { - uint64_t mmioFlags = VmwSvgaMmioPageFlags(); - KSTATUS st = PciMapBar(v->Pdo, 1, mmioFlags, + KSTATUS st = PciMapBar(v->Pdo, 1, VmwSvgaFramebufferPageFlags(), (void**)&v->FramebufferMapping, &v->FramebufferBytes); if (!KSUCCESS(st) || !v->FramebufferMapping) { DbgPrint("VMWSVGA: PciMapBar framebuffer failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n", @@ -1079,7 +1231,7 @@ static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) { return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; } - st = PciMapBar(v->Pdo, 2, mmioFlags, + st = PciMapBar(v->Pdo, 2, VmwSvgaRegisterPageFlags(), (void**)&v->Fifo, &v->FifoBytes); if (!KSUCCESS(st) || !v->Fifo) { DbgPrint("VMWSVGA: PciMapBar FIFO failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n", diff --git a/code/drivers/drivers.ini b/code/drivers/drivers.ini index 8fe39f7..6d147b5 100644 --- a/code/drivers/drivers.ini +++ b/code/drivers/drivers.ini @@ -1,5 +1,6 @@ [boot] pci.sys +hda.sys ps2.sys serial.sys ahci.sys diff --git a/code/drivers/fs/fat/fat.c b/code/drivers/fs/fat/fat.c index 948112a..da108f8 100644 --- a/code/drivers/fs/fat/fat.c +++ b/code/drivers/fs/fat/fat.c @@ -80,6 +80,11 @@ static void FatCacheEnsureMutex(void) { gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT; gFatCacheMutex.Owner = (struct THREAD*)0; LIST_INIT(&gFatCacheMutex.WaitList); + LIST_INIT(&gFatCacheMutex.OwnerLink); + gFatCacheMutex.DonationClass = KeSchedulingNormal; + gFatCacheMutex.DonationPriority = PriorityLow; + gFatCacheMutex.HasDonation = 0; + gFatCacheMutex.Abandoned = 0; gFatCacheMutexReady = 1; } KeReleaseSpinlock(&gFatCacheMutexInitLock, flags); @@ -89,7 +94,7 @@ static int FatCacheAcquire(void) { if (!KeCanWait()) return 0; FatCacheEnsureMutex(); - return KSUCCESS(KeAcquireMutex(&gFatCacheMutex)); + return KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatCacheMutex, KE_WAIT_INFINITE)); } static void FatCacheRelease(int held) { @@ -1913,6 +1918,17 @@ KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { return NTLSTATUS_SUCCESS; } +// Flushes FAT metadata and the backing device write cache. +KSTATUS FatFlush(IO_FILESYSTEM* fs, void* fileCtx) { + (void)fs; + FAT_FILE* f = (FAT_FILE*)fileCtx; + if (!f || !f->Vol) + return NTLSTATUS_INVALID_PARAMETER; + + FlushFatCache(f->Vol); + return IoFlushDisk(f->Vol->DriveIndex); +} + // Releases the requested object. void FatClose(IO_FILESYSTEM *fs, void *fileCtx) { (void)fs; @@ -2304,11 +2320,14 @@ KSTATUS FatDelete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) { if (!KSUCCESS(s)) return s; if (!direntLba) return NTLSTATUS_ACCESS_DENIED; - uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector); - if (!sec) return NTLSTATUS_NO_MEMORY; - s = ReadSectors(v, direntLba, 1, sec); - if (KSUCCESS(s)) { sec[direntOffset] = 0xE5; s = WriteSectors(v, direntLba, 1, sec); } - MmFreePool(sec); + uint32_t parentCluster = 0; + SYS_STRING_VIEW filenameView; + s = FatResolveParentForMutation(v, path, &parentCluster, &filenameView); + if (!KSUCCESS(s)) return s; + + // Delete the complete entry run so long-name metadata cannot survive the + // short entry and keep writable vvfat mirrors visibly present. + s = FatDeleteDirentRunV(v, parentCluster, filenameView); if (!KSUCCESS(s)) return s; if (cluster >= 2) FreeClusterChain(v, cluster); @@ -2583,7 +2602,7 @@ KSTATUS FatReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) { FAT_DIR_CTX *ctx = (FAT_DIR_CTX *)dirCtx; FAT_VOL *v = ctx->Vol; - for (;;) { + while (true) { if (ctx->Cursor != ctx->LoadedCursor) { if (!KSUCCESS(ReadSectors(v, ctx->LoadedLba, ctx->LoadedSectors, ctx->ClusterBuf))) return NTLSTATUS_NO_MORE_FILES; @@ -2918,8 +2937,8 @@ static void FatAsyncWriteSubmitNext(FAT_ASYNC_RW* ctx); static void FatAsyncWorker(void* arg) { (void)arg; - for (;;) { - KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore); + while (true) { + KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(s)) { KeYield(); continue; @@ -2959,8 +2978,9 @@ static KSTATUS FatEnsureAsyncWorkerStarted(void) { KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gFatAsyncSemaphore); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)FatAsyncWorker, 0, - PriorityNormal, &worker); + ¶meters, &worker); (void)worker; } @@ -3355,6 +3375,11 @@ static void FatMountEnsureMutex(void) { gFatMountMutex.Lock = (SPINLOCK)SPINLOCK_INIT; gFatMountMutex.Owner = (struct THREAD*)0; LIST_INIT(&gFatMountMutex.WaitList); + LIST_INIT(&gFatMountMutex.OwnerLink); + gFatMountMutex.DonationClass = KeSchedulingNormal; + gFatMountMutex.DonationPriority = PriorityLow; + gFatMountMutex.HasDonation = 0; + gFatMountMutex.Abandoned = 0; gFatMountMutexReady = 1; } KeReleaseSpinlock(&gFatMountMutexInitLock, flags); @@ -3375,7 +3400,7 @@ static uint32_t FatTryMountDisk(uint8_t disk) { return 0; FatMountEnsureMutex(); - if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex))) + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE))) return 0; if (gDiskScanned[disk]) { @@ -3509,7 +3534,7 @@ static void FatUnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) { uint32_t removed = 0; FatMountEnsureMutex(); - if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex))) + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE))) return; for (uint8_t i = 0; i < FAT_MAX_VOLUMES; i++) { diff --git a/code/drivers/fs/fat/fat.h b/code/drivers/fs/fat/fat.h index 3e6b7ab..f55d6cb 100644 --- a/code/drivers/fs/fat/fat.h +++ b/code/drivers/fs/fat/fat.h @@ -22,6 +22,7 @@ typedef struct FAT_VOL_S FAT_VOL; KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx); +KSTATUS FatFlush(IO_FILESYSTEM *fs, void *fileCtx); void FatClose(IO_FILESYSTEM *fs, void *fileCtx); KSTATUS FatRead(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead); KSTATUS FatWrite(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten); diff --git a/code/drivers/fs/fat/fat_mount.c b/code/drivers/fs/fat/fat_mount.c index 88e1289..d7ba9b1 100644 --- a/code/drivers/fs/fat/fat_mount.c +++ b/code/drivers/fs/fat/fat_mount.c @@ -19,6 +19,7 @@ void FatInitVtable(IO_FILESYSTEM* fs, FAT_VOL* vol) { fs->QueryInfo = FatQueryInfo; fs->Stat = FatStat; fs->QueryIdentity = FatQueryIdentity; + fs->Flush = FatFlush; fs->Close = FatClose; fs->DriverCtx = vol; fs->OpenDir = FatOpenDir; diff --git a/code/drivers/net/lwip/lwip_stack.c b/code/drivers/net/lwip/lwip_stack.c index 921646c..f86e5c2 100644 --- a/code/drivers/net/lwip/lwip_stack.c +++ b/code/drivers/net/lwip/lwip_stack.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -74,6 +75,15 @@ typedef struct LWIP_ATTACH_CTX { err_t Result; } LWIP_ATTACH_CTX; +typedef struct LWIP_RESOLVE_CONTEXT { + SEMAPHORE* Done; + volatile long References; + err_t Result; + u8_t DnsType; + ip_addr_t Address; + char Name[DNS_MAX_NAME_LENGTH]; +} LWIP_RESOLVE_CONTEXT; + typedef struct LWIP_PROVIDER_SOCKET { SEMAPHORE* Lock; struct netconn* Conn; @@ -118,6 +128,9 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how); static KSTATUS LwipProviderSetOption(void* socket, uint32_t option, const void* value, size_t valueSize); static KSTATUS LwipProviderGetOption(void* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut); static KSTATUS LwipProviderPoll(void* socket, uint32_t events, uint32_t* returnedEventsOut); +static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context); +static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument); +static void LwipResolveStartCallback(void* argument); static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut); static KSTATUS LwipDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp); @@ -355,7 +368,7 @@ static KSTATUS LwipLoadNetifConfig(const NET_DEVICE* device, LWIP_NETIF_CONFIG* static KSTATUS LwipSocketLock(LWIP_PROVIDER_SOCKET* s) { if (!s || !s->Lock) return NTLSTATUS_INVALID_PARAMETER; - return KeWaitSemaphore(s->Lock); + return KeWaitSemaphore(s->Lock, KE_WAIT_INFINITE); } // Implements the socket Unlock operation. @@ -464,7 +477,7 @@ static KSTATUS LwipSocketWaitRecvNetbuf(LWIP_PROVIDER_SOCKET* s, uint32_t originalTimeout = s->ReceiveTimeoutMs; uint32_t elapsed = 0; - for (;;) { + while (true) { if (LwipSocketCancelRequested(cancelRequested)) { netconn_set_recvtimeout(s->Conn, originalTimeout); return NTLSTATUS_CANCELLED; @@ -510,7 +523,7 @@ static KSTATUS LwipSocketWaitAccept(LWIP_PROVIDER_SOCKET* s, uint32_t originalTimeout = s->ReceiveTimeoutMs; uint32_t elapsed = 0; - for (;;) { + while (true) { if (LwipSocketCancelRequested(cancelRequested)) { netconn_set_recvtimeout(s->Conn, originalTimeout); return NTLSTATUS_CANCELLED; @@ -778,7 +791,7 @@ static KSTATUS LwipStackInitialize(const NTL_DDK* ddk) { return st; tcpip_init(LwipInitDone, done); - st = KeWaitSemaphore(done); + st = KeWaitSemaphore(done, KE_WAIT_INFINITE); ObDereferenceObject(&done->Header); if (!KSUCCESS(st)) return st; @@ -826,7 +839,7 @@ static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device) { err_t err = tcpip_callback(LwipAttachCallback, &ctx); if (err == ERR_OK) - st = KeWaitSemaphore(done); + st = KeWaitSemaphore(done, KE_WAIT_INFINITE); else st = NTLSTATUS_FAILURE; @@ -1135,7 +1148,7 @@ static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_K* add return LwipErrToStatus(err); } - for (;;) { + while (true) { if (LwipSocketCancelRequested(cancelRequested)) { (void)netconn_close(s->Conn); s->ShutdownRead = 1; @@ -1595,6 +1608,51 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how) { return LwipErrToStatus(err); } +// Releases one resolver owner and destroys the request after the final callback or waiter leaves. +static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context) { + if (!context) + return; + + if (_InterlockedDecrement(&context->References) != 0) + return; + + if (context->Done) + ObDereferenceObject(&context->Done->Header); + MmFreePool(context); +} + +// Completes an asynchronous DNS lookup without retaining the caller's kernel stack. +static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument) { + LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument; + (void)name; + + if (address) { + context->Address = *address; + context->Result = ERR_OK; + } else { + context->Result = ERR_VAL; + } + + KeSignalSemaphore(context->Done); + LwipResolveRelease(context); +} + +// Starts one DNS lookup in the tcpip thread and transfers its resolver reference to the callback. +static void LwipResolveStartCallback(void* argument) { + LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument; + err_t err = dns_gethostbyname_addrtype(context->Name, + &context->Address, + LwipResolveDnsDoneCallback, + context, + context->DnsType); + context->Result = err; + if (err == ERR_INPROGRESS) + return; + + KeSignalSemaphore(context->Done); + LwipResolveRelease(context); +} + // Implements the provider Resolve Host operation. static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut) { if (!name || !addressOut) @@ -1610,10 +1668,48 @@ static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SO else return NTLSTATUS_NOT_SUPPORTED; - ip_addr_t resolved; - err_t err = netconn_gethostbyname_addrtype(name, &resolved, dnsType); - if (err != ERR_OK) - return LwipErrToStatus(err); + size_t nameLength = strlen(name); + if (nameLength == 0 || nameLength >= DNS_MAX_NAME_LENGTH) + return NTLSTATUS_INVALID_PARAMETER; + + LWIP_RESOLVE_CONTEXT* context = + (LWIP_RESOLVE_CONTEXT*)MmAllocatePool(sizeof(*context)); + if (!context) + return NTLSTATUS_NO_MEMORY; + memset(context, 0, sizeof(*context)); + memcpy(context->Name, name, nameLength + 1); + context->References = 2; + context->Result = ERR_INPROGRESS; + context->DnsType = dnsType; + + KSTATUS status = KeSemaphoreCreate(0, 1, &context->Done); + if (!KSUCCESS(status)) { + MmFreePool(context); + return status; + } + + err_t queueError = tcpip_try_callback(LwipResolveStartCallback, context); + if (queueError != ERR_OK) { + LwipResolveRelease(context); + LwipResolveRelease(context); + return LwipErrToStatus(queueError); + } + + status = KeWaitSemaphore(context->Done, KE_WAIT_INFINITE); + if (!KSUCCESS(status)) { + DbgPrint("lwIP: resolve waiter left name=%s status=0x%x; DNS request remains active\n", + context->Name, + status); + LwipResolveRelease(context); + return status; + } + + err_t result = context->Result; + ip_addr_t resolved = context->Address; + LwipResolveRelease(context); + + if (result != ERR_OK) + return LwipErrToStatus(result); if ((family == SYS_AF_INET && !IP_IS_V4(&resolved)) || (family == SYS_AF_INET6 && !IP_IS_V6(&resolved))) return NTLSTATUS_NOT_FOUND; diff --git a/code/drivers/net/lwip/lwip_sys_arch.c b/code/drivers/net/lwip/lwip_sys_arch.c index 1d70545..b30f635 100644 --- a/code/drivers/net/lwip/lwip_sys_arch.c +++ b/code/drivers/net/lwip/lwip_sys_arch.c @@ -117,19 +117,14 @@ u32_t sys_arch_sem_wait(sys_sem_t* sem, u32_t timeout) { u32_t start = sys_now(); if (timeout == 0) { - KSTATUS st = KeWaitSemaphore((*sem)->Sem); + KSTATUS st = KeWaitSemaphore((*sem)->Sem, KE_WAIT_INFINITE); return KSUCCESS(st) ? (u32_t)(sys_now() - start) : SYS_ARCH_TIMEOUT; } - for (;;) { - if (KSUCCESS(KeTrySemaphore((*sem)->Sem))) - return (u32_t)(sys_now() - start); - - if ((u32_t)(sys_now() - start) >= timeout) - return SYS_ARCH_TIMEOUT; - - KeYield(); - } + KSTATUS status = KeWaitSemaphore((*sem)->Sem, timeout); + return KSUCCESS(status) + ? (u32_t)(sys_now() - start) + : SYS_ARCH_TIMEOUT; } // Implements the sys sem free operation. @@ -165,7 +160,7 @@ err_t sys_mutex_new(sys_mutex_t* mutex) { // Implements the sys mutex lock operation. void sys_mutex_lock(sys_mutex_t* mutex) { if (mutex && *mutex && (*mutex)->Sem) - (void)KeWaitSemaphore((*mutex)->Sem); + (void)KeWaitSemaphore((*mutex)->Sem, KE_WAIT_INFINITE); } // Implements the sys mutex unlock operation. @@ -279,28 +274,23 @@ u32_t sys_arch_mbox_fetch(sys_mbox_t* mbox, void** msg, u32_t timeout) { u32_t start = sys_now(); if (timeout == 0) { - KSTATUS st = KeWaitSemaphore((*mbox)->Items); + KSTATUS st = KeWaitSemaphore((*mbox)->Items, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return SYS_ARCH_TIMEOUT; return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start)); } - for (;;) { - if (KSUCCESS(KeTrySemaphore((*mbox)->Items))) - return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start)); - - if ((u32_t)(sys_now() - start) >= timeout) - return SYS_ARCH_TIMEOUT; - - KeYield(); - } + KSTATUS status = KeWaitSemaphore((*mbox)->Items, timeout); + if (!KSUCCESS(status)) + return SYS_ARCH_TIMEOUT; + return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start)); } u32_t sys_arch_mbox_tryfetch(sys_mbox_t* mbox, void** msg) { if (!mbox || !*mbox || !(*mbox)->Valid) return SYS_MBOX_EMPTY; - if (!KSUCCESS(KeTrySemaphore((*mbox)->Items))) + if (!KSUCCESS(KeWaitSemaphore((*mbox)->Items, 0))) return SYS_MBOX_EMPTY; return LwipMboxPopAfterItem(mbox, msg, 0); @@ -347,9 +337,10 @@ sys_thread_t sys_thread_new(const char* name, lwip_thread_fn thread, void* arg, // function pointer that lwIP supplied, with exactly the argument it supplied. THREAD* t = 0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(kp); KSTATUS st = KeCreateKernelThread((uintptr_t)thread, (uintptr_t)arg, - kp, + ¶meters, &t); if (!KSUCCESS(st)) KePanic(KE_PANIC_CODE_DRIVER, "lwIP: failed to create kernel thread: 0x%x\n", st); diff --git a/code/drivers/net/rtl8139/rtl8139.c b/code/drivers/net/rtl8139/rtl8139.c index e3d241e..5d58811 100644 --- a/code/drivers/net/rtl8139/rtl8139.c +++ b/code/drivers/net/rtl8139/rtl8139.c @@ -82,29 +82,79 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) { r->NetDevice.HardwareAddressLength = 6; } -// Creates dma. -static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) { - r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma); - r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma); - if (!r->RxPhys || !r->TxPhys) - return NTLSTATUS_NO_MEMORY; +// Stops bus mastering before DMA memory can be released. +static void Rtl8139StopDma(RTL8139_DEVICE* r) { + if (!r || !r->Started) + return; + r->Started = false; + if (r->SurpriseRemoved) + return; + Rtl8139Write16(r, RTL8139_REG_IMR, 0); + Rtl8139Write8(r, RTL8139_REG_CMD, 0); + KSTATUS status = Rtl8139Reset(r); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_DRIVER, + "RTL8139: failed to stop DMA before release status=0x%x\n", + status); +} - // RTL8139 has 32-bit DMA address registers. This is fine for typical QEMU - // and low-memory hobby OS configs, but make failure explicit. - if ((r->RxPhys + RTL8139_RX_ALLOC_SIZE - 1) > 0xFFFFFFFFULL || - (r->TxPhys + RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE - 1) > 0xFFFFFFFFULL) { - DbgPrint("RTL8139: DMA allocation above 4GiB is not supported by this NIC\n"); - return NTLSTATUS_NOT_SUPPORTED; +// Releases the common buffers and adapter owned by one RTL8139 device. +static void Rtl8139ReleaseDma(RTL8139_DEVICE* r) { + if (!r || !gRtl8139Ddk || !gRtl8139Ddk->Dma) + return; + gRtl8139Ddk->Dma->FreeCommonBuffer(r->TxDma); + gRtl8139Ddk->Dma->FreeCommonBuffer(r->RxDma); + r->TxDma = (IO_DMA_BUFFER*)0; + r->RxDma = (IO_DMA_BUFFER*)0; + r->TxBuffers = (uint8_t*)0; + r->RxBuffer = (uint8_t*)0; + r->TxDeviceAddress = 0; + r->RxDeviceAddress = 0; + gRtl8139Ddk->Dma->PutAdapter(r->DmaAdapter); + r->DmaAdapter = (IO_DMA_ADAPTER*)0; +} + +// Allocates the receive ring and transmit slots through the device DMA adapter. +static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) { + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = 0xFFFFFFFFULL; + requirements.Alignment = 4; + requirements.MaximumSegmentLength = RTL8139_TX_BUF_SIZE; + requirements.MaximumSegments = 1; + + KSTATUS status = gRtl8139Ddk->Dma->GetAdapter(r->Fdo, + &requirements, + &r->DmaAdapter); + if (!KSUCCESS(status)) + return status; + + status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter, + RTL8139_RX_ALLOC_SIZE, + PAGE_SIZE, + &r->RxDma); + if (!KSUCCESS(status)) { + Rtl8139ReleaseDma(r); + return status; } - r->RxBuffer = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->RxPhys); - r->TxBuffers = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->TxPhys); - memset(r->RxBuffer, 0, PAGE_SIZE * RTL8139_RX_PAGES); - memset(r->TxBuffers, 0, PAGE_SIZE * RTL8139_TX_PAGES); + status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter, + RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE, + PAGE_SIZE, + &r->TxDma); + if (!KSUCCESS(status)) { + Rtl8139ReleaseDma(r); + return status; + } + + r->RxBuffer = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->RxDma); + r->TxBuffers = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->TxDma); + r->RxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->RxDma); + r->TxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->TxDma); DbgPrint("RTL8139: rx=0x%016llX tx=0x%016llX\n", - (unsigned long long)r->RxPhys, - (unsigned long long)r->TxPhys); + (unsigned long long)r->RxDeviceAddress, + (unsigned long long)r->TxDeviceAddress); return NTLSTATUS_SUCCESS; } @@ -136,13 +186,13 @@ static void Rtl8139ProgramHardware(RTL8139_DEVICE* r) { Rtl8139Write16(r, RTL8139_REG_IMR, 0); Rtl8139Write16(r, RTL8139_REG_ISR, 0xFFFFu); - Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxPhys); + Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxDeviceAddress); r->RxOffset = 0; Rtl8139Write16(r, RTL8139_REG_CAPR, 0xFFF0u); // CAPR is rxOffset - 16. for (uint32_t i = 0; i < RTL8139_TX_SLOTS; i++) { - PHYSICAL_ADDRESS p = r->TxPhys + (PHYSICAL_ADDRESS)i * RTL8139_TX_BUF_SIZE; - Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)p); + uint64_t address = r->TxDeviceAddress + (uint64_t)i * RTL8139_TX_BUF_SIZE; + Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)address); } Rtl8139Write32(r, RTL8139_REG_TCR, @@ -360,7 +410,7 @@ static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) { if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) { KSTATUS st = PciMapBar(r->Pdo, best.Index, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&r->Mmio, &r->RegisterBytes); if (!KSUCCESS(st)) @@ -460,8 +510,10 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) { r->NetDevice.SendFrame = Rtl8139SendFrame; st = Rtl8139RegisterDevice(r); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + Rtl8139ReleaseDma(r); return st; + } KeIrqRegister(r->Irq, Rtl8139Isr); r->Started = true; @@ -470,9 +522,9 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) { st = NetRegisterDevice(&r->NetDevice); if (!KSUCCESS(st)) { - Rtl8139Write16(r, RTL8139_REG_IMR, 0); - r->Started = false; + Rtl8139StopDma(r); Rtl8139UnregisterDevice(r); + Rtl8139ReleaseDma(r); return st; } @@ -503,17 +555,16 @@ KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: r->SurpriseRemoved = true; + r->Started = false; device->PnpState = DevicePnpSurpriseRemoved; - if (r->Started) - Rtl8139Write16(r, RTL8139_REG_IMR, 0); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: r->Removed = true; device->PnpState = DevicePnpDeleted; - if (r->Started) - Rtl8139Write16(r, RTL8139_REG_IMR, 0); + Rtl8139StopDma(r); Rtl8139UnregisterDevice(r); + Rtl8139ReleaseDma(r); IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; @@ -555,7 +606,7 @@ KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) + if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("RTL8139: driver loading...\n"); diff --git a/code/drivers/net/rtl8139/rtl8139.h b/code/drivers/net/rtl8139/rtl8139.h index ccd2140..a6f8d86 100644 --- a/code/drivers/net/rtl8139/rtl8139.h +++ b/code/drivers/net/rtl8139/rtl8139.h @@ -115,8 +115,11 @@ typedef struct RTL8139_DEVICE { volatile uint8_t* Mmio; uint16_t IoBase; - PHYSICAL_ADDRESS RxPhys; - PHYSICAL_ADDRESS TxPhys; + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* RxDma; + IO_DMA_BUFFER* TxDma; + uint64_t RxDeviceAddress; + uint64_t TxDeviceAddress; uint8_t* RxBuffer; uint8_t* TxBuffers; uint8_t RxScratch[RTL8139_TX_BUF_SIZE]; diff --git a/code/drivers/net/rtl816x/rtl816x.c b/code/drivers/net/rtl816x/rtl816x.c index 2732b29..c912323 100644 --- a/code/drivers/net/rtl816x/rtl816x.c +++ b/code/drivers/net/rtl816x/rtl816x.c @@ -83,48 +83,117 @@ static void RtlReadMac(RTL816X_DEVICE* r) { r->NetDevice.HardwareAddressLength = 6; } -// Implements the rtl Allocate Dma operation. +// Stops bus mastering before DMA memory can be released. +static void RtlStopDma(RTL816X_DEVICE* r) { + if (!r || !r->Started) + return; + r->Started = false; + if (r->SurpriseRemoved) + return; + RtlWrite16(r, RTL_REG_INTR_MASK, 0); + RtlWrite8(r, RTL_REG_CHIP_CMD, 0); + KSTATUS status = RtlReset(r); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_DRIVER, + "RTL816X: failed to stop DMA before release status=0x%x\n", + status); +} + +// Releases the common buffers and adapter owned by one RTL816x device. +static void RtlReleaseDma(RTL816X_DEVICE* r) { + if (!r || !gRtl816xDdk || !gRtl816xDdk->Dma) + return; + gRtl816xDdk->Dma->FreeCommonBuffer(r->TxBuffersDma); + gRtl816xDdk->Dma->FreeCommonBuffer(r->RxBuffersDma); + gRtl816xDdk->Dma->FreeCommonBuffer(r->TxRingDma); + gRtl816xDdk->Dma->FreeCommonBuffer(r->RxRingDma); + r->TxBuffersDma = (IO_DMA_BUFFER*)0; + r->RxBuffersDma = (IO_DMA_BUFFER*)0; + r->TxRingDma = (IO_DMA_BUFFER*)0; + r->RxRingDma = (IO_DMA_BUFFER*)0; + r->TxBuffers = (uint8_t*)0; + r->RxBuffers = (uint8_t*)0; + r->TxRing = (RTL_DESC*)0; + r->RxRing = (RTL_DESC*)0; + r->TxBuffersDeviceAddress = 0; + r->RxBuffersDeviceAddress = 0; + r->TxRingDeviceAddress = 0; + r->RxRingDeviceAddress = 0; + gRtl816xDdk->Dma->PutAdapter(r->DmaAdapter); + r->DmaAdapter = (IO_DMA_ADAPTER*)0; +} + +// Allocates descriptor rings and packet buffers through the device DMA adapter. static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) { - r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma); - r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma); - r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma); - r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma); + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = (uint64_t)-1; + requirements.Alignment = 1; + requirements.MaximumSegmentLength = RTL_BUF_SIZE; + requirements.MaximumSegments = RTL_TX_COUNT; - if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys) - return NTLSTATUS_NO_MEMORY; + KSTATUS status = gRtl816xDdk->Dma->GetAdapter(r->Fdo, + &requirements, + &r->DmaAdapter); + if (!KSUCCESS(status)) + return status; - r->RxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->RxRingPhys); - r->TxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->TxRingPhys); - r->RxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->RxBuffersPhys); - r->TxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->TxBuffersPhys); + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_RING_PAGES, + PAGE_SIZE, + &r->RxRingDma); + if (!KSUCCESS(status)) goto fail; + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_RING_PAGES, + PAGE_SIZE, + &r->TxRingDma); + if (!KSUCCESS(status)) goto fail; + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_RXBUF_PAGES, + PAGE_SIZE, + &r->RxBuffersDma); + if (!KSUCCESS(status)) goto fail; + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_TXBUF_PAGES, + PAGE_SIZE, + &r->TxBuffersDma); + if (!KSUCCESS(status)) goto fail; - memset(r->RxRing, 0, PAGE_SIZE * RTL_RING_PAGES); - memset(r->TxRing, 0, PAGE_SIZE * RTL_RING_PAGES); - memset(r->RxBuffers, 0, PAGE_SIZE * RTL_RXBUF_PAGES); - memset(r->TxBuffers, 0, PAGE_SIZE * RTL_TXBUF_PAGES); + r->RxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxRingDma); + r->TxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxRingDma); + r->RxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxBuffersDma); + r->TxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxBuffersDma); + r->RxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxRingDma); + r->TxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxRingDma); + r->RxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxBuffersDma); + r->TxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxBuffersDma); DbgPrint("RTL816X: rxRing=0x%016llX txRing=0x%016llX rxBuf=0x%016llX txBuf=0x%016llX\n", - (unsigned long long)r->RxRingPhys, - (unsigned long long)r->TxRingPhys, - (unsigned long long)r->RxBuffersPhys, - (unsigned long long)r->TxBuffersPhys); + (unsigned long long)r->RxRingDeviceAddress, + (unsigned long long)r->TxRingDeviceAddress, + (unsigned long long)r->RxBuffersDeviceAddress, + (unsigned long long)r->TxBuffersDeviceAddress); return NTLSTATUS_SUCCESS; + +fail: + RtlReleaseDma(r); + return status; } // Implements the rtl Init Descriptors operation. static void RtlInitDescriptors(RTL816X_DEVICE* r) { for (uint32_t i = 0; i < RTL_RX_COUNT; i++) { - PHYSICAL_ADDRESS p = r->RxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE; - r->RxRing[i].addrLo = (uint32_t)p; - r->RxRing[i].addrHi = (uint32_t)(p >> 32); + uint64_t address = r->RxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE; + r->RxRing[i].addrLo = (uint32_t)address; + r->RxRing[i].addrHi = (uint32_t)(address >> 32); r->RxRing[i].opts2 = 0; r->RxRing[i].opts1 = RTL_DESC_OWN | RTL_BUF_SIZE | ((i == RTL_RX_COUNT - 1) ? RTL_DESC_EOR : 0); } for (uint32_t i = 0; i < RTL_TX_COUNT; i++) { - PHYSICAL_ADDRESS p = r->TxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE; - r->TxRing[i].addrLo = (uint32_t)p; - r->TxRing[i].addrHi = (uint32_t)(p >> 32); + uint64_t address = r->TxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE; + r->TxRing[i].addrLo = (uint32_t)address; + r->TxRing[i].addrHi = (uint32_t)(address >> 32); r->TxRing[i].opts2 = 0; r->TxRing[i].opts1 = (i == RTL_TX_COUNT - 1) ? RTL_DESC_EOR : 0; } @@ -141,12 +210,12 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) { RtlWrite16(r, RTL_REG_INTR_MASK, 0); RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu); - RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingPhys); - RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingPhys >> 32)); - RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingPhys); - RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingPhys >> 32)); - RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingPhys); - RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingPhys >> 32)); + RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingDeviceAddress); + RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32)); + RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingDeviceAddress); + RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32)); + RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingDeviceAddress); + RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingDeviceAddress >> 32)); RtlWrite16(r, RTL_REG_CPLUS_CMD, RtlRead16(r, RTL_REG_CPLUS_CMD) & ~(uint16_t)0x00E0u); RtlWrite16(r, RTL_REG_RX_MAXSIZE, RTL_BUF_SIZE); @@ -366,7 +435,7 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) { if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) { KSTATUS st = PciMapBar(r->Pdo, best.Index, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&r->Mmio, &r->RegisterBytes); if (!KSUCCESS(st)) @@ -400,6 +469,41 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) { return NTLSTATUS_NOT_FOUND; } +// Configures a dedicated MSI vector and falls back to a routed INTx line. +static KSTATUS RtlConfigureInterrupt(RTL816X_DEVICE* r) { + if (!r || !r->Pdo) + return NTLSTATUS_INVALID_PARAMETER; + + r->Irq = 0xFFu; + r->UsingMsi = false; + + KSTATUS msiStatus = PciEnableMsiAuto(r->Pdo, &r->Irq); + if (KSUCCESS(msiStatus)) { + r->UsingMsi = true; + DbgPrint("RTL816X: using MSI IRQ slot %u\n", r->Irq); + return NTLSTATUS_SUCCESS; + } + + DbgPrint("RTL816X: MSI setup failed status=0x%x; attempting explicit INTx fallback\n", + msiStatus); + + KSTATUS intxStatus = PciGetInterrupt(r->Pdo, &r->Irq); + if (!KSUCCESS(intxStatus)) + return intxStatus; + if (r->Irq == 0xFFu || r->Irq >= 24u) + return NTLSTATUS_NOT_SUPPORTED; + + intxStatus = PciEnableDevice(r->Pdo, + PCI_ENABLE_BUSMASTER | + PCI_ENABLE_INTX | + (r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO)); + if (!KSUCCESS(intxStatus)) + return intxStatus; + + DbgPrint("RTL816X: using legacy INTx IRQ %u\n", r->Irq); + return NTLSTATUS_SUCCESS; +} + // Implements the rtl Start Device operation. static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { if (!r || r->Magic != RTL816X_DEVICE_MAGIC || !r->Pdo || !r->Fdo) @@ -428,18 +532,15 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { st = PciEnableDevice(r->Pdo, PCI_ENABLE_BUSMASTER | - PCI_ENABLE_INTX | + PCI_DISABLE_INTX | (r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO)); if (!KSUCCESS(st)) { DbgPrint("RTL816X: PciEnableDevice failed: 0x%x\n", st); return st; } - st = PciGetInterrupt(r->Pdo, &r->Irq); - if (!KSUCCESS(st) || r->Irq == 0xFFu) { - DbgPrint("RTL816X: no legacy PCI IRQ line wired\n"); - return KSUCCESS(st) ? NTLSTATUS_NOT_SUPPORTED : st; - } + RtlWrite16(r, RTL_REG_INTR_MASK, 0); + RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu); st = RtlReset(r); if (!KSUCCESS(st)) { @@ -447,12 +548,18 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { return st; } + st = RtlConfigureInterrupt(r); + if (!KSUCCESS(st)) { + DbgPrint("RTL816X: no usable MSI or INTx interrupt status=0x%x\n", st); + return st; + } + RtlReadMac(r); - DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u\n", + DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u mode=%s\n", r->NetDevice.HardwareAddress[0], r->NetDevice.HardwareAddress[1], r->NetDevice.HardwareAddress[2], r->NetDevice.HardwareAddress[3], r->NetDevice.HardwareAddress[4], r->NetDevice.HardwareAddress[5], - r->Irq); + r->Irq, r->UsingMsi ? "MSI" : "INTx"); st = RtlAllocateDma(r); if (!KSUCCESS(st)) @@ -466,8 +573,10 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { r->NetDevice.SendFrame = RtlSendFrame; st = RtlRegisterDevice(r); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + RtlReleaseDma(r); return st; + } KeIrqRegister(r->Irq, RtlIsr); r->Started = true; @@ -476,13 +585,14 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { st = NetRegisterDevice(&r->NetDevice); if (!KSUCCESS(st)) { - RtlWrite16(r, RTL_REG_INTR_MASK, 0); - r->Started = false; + RtlStopDma(r); RtlUnregisterDevice(r); + RtlReleaseDma(r); return st; } - DbgPrint("RTL816X: initialized; RX/TX enabled on IRQ %u\n", r->Irq); + DbgPrint("RTL816X: initialized; RX/TX enabled on %s IRQ %u\n", + r->UsingMsi ? "MSI" : "INTx", r->Irq); return NTLSTATUS_SUCCESS; } @@ -509,17 +619,16 @@ KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: r->SurpriseRemoved = true; + r->Started = false; device->PnpState = DevicePnpSurpriseRemoved; - if (r->Started) - RtlWrite16(r, RTL_REG_INTR_MASK, 0); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: r->Removed = true; device->PnpState = DevicePnpDeleted; - if (r->Started) - RtlWrite16(r, RTL_REG_INTR_MASK, 0); + RtlStopDma(r); RtlUnregisterDevice(r); + RtlReleaseDma(r); IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; @@ -561,7 +670,7 @@ KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) + if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("RTL816X: driver loading...\n"); diff --git a/code/drivers/net/rtl816x/rtl816x.h b/code/drivers/net/rtl816x/rtl816x.h index cfa8d00..f84c30e 100644 --- a/code/drivers/net/rtl816x/rtl816x.h +++ b/code/drivers/net/rtl816x/rtl816x.h @@ -117,6 +117,7 @@ typedef struct RTL816X_DEVICE { PCI_BAR RegisterBar; uint64_t RegisterBytes; uint8_t Irq; + bool UsingMsi; bool Started; bool Removed; @@ -126,15 +127,20 @@ typedef struct RTL816X_DEVICE { volatile uint8_t* Mmio; uint16_t IoBase; + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* RxRingDma; + IO_DMA_BUFFER* TxRingDma; + IO_DMA_BUFFER* RxBuffersDma; + IO_DMA_BUFFER* TxBuffersDma; RTL_DESC* RxRing; RTL_DESC* TxRing; - PHYSICAL_ADDRESS RxRingPhys; - PHYSICAL_ADDRESS TxRingPhys; + uint64_t RxRingDeviceAddress; + uint64_t TxRingDeviceAddress; uint8_t* RxBuffers; uint8_t* TxBuffers; - PHYSICAL_ADDRESS RxBuffersPhys; - PHYSICAL_ADDRESS TxBuffersPhys; + uint64_t RxBuffersDeviceAddress; + uint64_t TxBuffersDeviceAddress; uint32_t RxIndex; uint32_t TxIndex; diff --git a/code/drivers/storage/ahci/ahci.c b/code/drivers/storage/ahci/ahci.c index 7e18152..55d1b28 100644 --- a/code/drivers/storage/ahci/ahci.c +++ b/code/drivers/storage/ahci/ahci.c @@ -9,6 +9,11 @@ static AHCI_CONTROLLER* g_controllers[MAX_AHCI_CONTROLLERS]; static int g_controllerCount; static uint8_t g_nextDiskIndex; +#define AHCI_COMMAND_TIMEOUT_MS 10000ull +#define AHCI_PORT_ERROR_MASK \ + (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS) +#define AHCI_PORT_TERMINAL_MASK (PORT_IS_DHRS | AHCI_PORT_ERROR_MASK) + static const IO_DEVICE_ID g_ahciIds[] = { { IoBusPci, { "PCI\\CC_010601", sizeof("PCI\\CC_010601") - 1 } }, }; @@ -21,20 +26,21 @@ static const IO_DRIVER_ID_TABLE g_ahciIdTable = { // Helpers. static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl); +static void AhciUnregisterController(AHCI_CONTROLLER* ctl); -// Accessors for port structure memory at known offsets from structVirt/Phys. +// Accessors for port structure memory at known CPU and device-address offsets. static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) { return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST); } -// Implements the cmd list phys operation. -static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) { - return drv->structPhys + PORT_OFF_CMDLIST; +// Returns the command-list device address. +static inline uint64_t cmd_list_device_address(AHCI_DRIVE *drv) { + return drv->structDeviceAddress + PORT_OFF_CMDLIST; } -// Implements the fis phys operation. -static inline uint64_t fis_phys(AHCI_DRIVE *drv) { - return drv->structPhys + PORT_OFF_FIS; +// Returns the received-FIS device address. +static inline uint64_t fis_device_address(AHCI_DRIVE *drv) { + return drv->structDeviceAddress + PORT_OFF_FIS; } static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) { @@ -43,9 +49,9 @@ static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) { + (uint32_t)slot * PORT_CMDTBL_SIZE); } -// Implements the cmd tbl phys operation. -static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { - return drv->structPhys + PORT_OFF_CMDTBLS +// Returns one command-table device address. +static inline uint64_t cmd_tbl_device_address(AHCI_DRIVE *drv, int slot) { + return drv->structDeviceAddress + PORT_OFF_CMDTBLS + (uint64_t)slot * PORT_CMDTBL_SIZE; } @@ -53,53 +59,108 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { // Port memory initialisation. +// Stops one drive worker before its queue and DMA storage are released. +static void AhciStopWorker(AHCI_DRIVE* drv) { + if (!drv) + return; + + drv->workerRunning = false; + if (drv->irpQueue.Semaphore) + KeSignalSemaphore(drv->irpQueue.Semaphore); + if (drv->workerThread && drv->workerFinished) + (void)KeWaitForEvent(drv->workerFinished, 0); + + if (drv->workerThread) { + ObDereferenceObject((OBJECT*)drv->workerThread); + drv->workerThread = (THREAD*)0; + } + if (drv->workerFinished) { + ObDereferenceObject(&drv->workerFinished->Header); + drv->workerFinished = (EVENT*)0; + } +} + +// Releases the non-DMA synchronization objects owned by one AHCI drive. +static void AhciReleaseDriveRuntime(AHCI_DRIVE* drv) { + if (!drv) + return; + + AhciStopWorker(drv); + if (drv->removeLockReady) { + IoDestroyRemoveLock(&drv->removeLock); + drv->removeLockReady = false; + } + if (drv->irpQueue.Semaphore) { + ObDereferenceObject(&drv->irpQueue.Semaphore->Header); + drv->irpQueue.Semaphore = (SEMAPHORE*)0; + } + if (drv->completionEvent) { + ObDereferenceObject(&drv->completionEvent->Header); + drv->completionEvent = (EVENT*)0; + } +} + +// Releases the DMA buffers owned by one AHCI drive. +static void AhciReleaseDriveDma(AHCI_DRIVE* drv) { + if (!drv || !gAhciDdk || !gAhciDdk->Dma) + return; + if (drv->port && drv->controller && + !drv->surpriseRemoved && !drv->controller->surpriseRemoved) + port_stop(drv->port); + gAhciDdk->Dma->FreeCommonBuffer(drv->bounceDma); + gAhciDdk->Dma->FreeCommonBuffer(drv->structDma); + drv->bounceDma = (IO_DMA_BUFFER*)0; + drv->structDma = (IO_DMA_BUFFER*)0; + drv->bounceDeviceAddress = 0; + drv->structDeviceAddress = 0; + drv->bounceVirt = (void*)0; + drv->structVirt = (void*)0; +} + +// Allocates and programs one port's command, FIS, and bounce buffers. static KSTATUS port_init_memory(AHCI_DRIVE *drv) { - uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; + if (!drv || !drv->controller || !drv->controller->dmaAdapter) + return NTLSTATUS_INVALID_PARAMETER; - drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma); - if (!drv->structPhys) return NTLSTATUS_NO_MEMORY; - drv->structVirt = gAhciDdk->Memory->PhysToVirt(drv->structPhys); - // Force NOCACHE on DMA descriptor memory. - MmPteProtectRange((uintptr_t)drv->structVirt, - PORT_PAGES_STRUCT * 4096, flags); - memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096); + KSTATUS status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter, + PORT_PAGES_STRUCT * PAGE_SIZE, + PAGE_SIZE, + &drv->structDma); + if (!KSUCCESS(status)) + return status; + drv->structVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->structDma); + drv->structDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->structDma); - drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma); - if (!drv->bouncePhys) return NTLSTATUS_NO_MEMORY; - drv->bounceVirt = gAhciDdk->Memory->PhysToVirt(drv->bouncePhys); - // Bounce buffer: NOCACHE to ensure CPU writes reach physical memory - // before the HBA's DMA engine reads them. - MmPteProtectRange((uintptr_t)drv->bounceVirt, - BOUNCE_PAGES * 4096, flags); - memset(drv->bounceVirt, 0, BOUNCE_BYTES); + status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter, + BOUNCE_BYTES, + PAGE_SIZE, + &drv->bounceDma); + if (!KSUCCESS(status)) { + AhciReleaseDriveDma(drv); + return status; + } + drv->bounceVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->bounceDma); + drv->bounceDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->bounceDma); HBA_PORT *port = drv->port; - port_stop(port); - // Set command list base. - port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF); - port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32); + port->clb = (uint32_t)(cmd_list_device_address(drv) & 0xFFFFFFFF); + port->clbu = (uint32_t)(cmd_list_device_address(drv) >> 32); + port->fb = (uint32_t)(fis_device_address(drv) & 0xFFFFFFFF); + port->fbu = (uint32_t)(fis_device_address(drv) >> 32); - // Set FIS base. - port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF); - port->fbu = (uint32_t)(fis_phys(drv) >> 32); - - // Initialise all 32 command headers, pointing each to its command table. HBA_CMD_HEADER *hdrs = cmd_list(drv); for (int i = 0; i < NUM_CMD_SLOTS; i++) { hdrs[i].prdtl = PORT_PRDT_COUNT; - uint64_t tbl = cmd_tbl_phys(drv, i); - hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF); - hdrs[i].ctbau = (uint32_t)(tbl >> 32); + uint64_t table = cmd_tbl_device_address(drv, i); + hdrs[i].ctba = (uint32_t)(table & 0xFFFFFFFF); + hdrs[i].ctbau = (uint32_t)(table >> 32); } - // Clear any stale interrupts then start the port engine. port->serr = (uint32_t)-1; port->is = (uint32_t)-1; - port_start(port); - return NTLSTATUS_SUCCESS; } @@ -125,16 +186,22 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) { // Issue slot 0. port->ci = (1u << 0); - // Poll until slot 0 clears. - while (true) { - if (!(port->ci & (1u << 0))) break; - if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { + // Poll until slot 0 clears, but never let early-init I/O hang forever. + spin = 0; + while ((port->ci & (1u << 0)) && spin < POLL_TIMEOUT) { + if (port->is & AHCI_PORT_ERROR_MASK) { DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", port->is, port->tfd, port->serr); port->serr = (uint32_t)-1; port->is = (uint32_t)-1; return NTLSTATUS_DEVICE_ERROR; } + spin++; + } + if (port->ci & (1u << 0)) { + DbgPrint("AHCI: command timed out while polling CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, port->is, port->serr); + return NTLSTATUS_TIMEOUT; } if (port->is & PORT_IS_TFES) { @@ -146,6 +213,30 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) { return NTLSTATUS_SUCCESS; } +// Recovers a port after a command that cannot be trusted to have retired. +static void AhciRecoverPort(AHCI_DRIVE *drv) { + if (!drv || !drv->port) + return; + + HBA_PORT *port = drv->port; + uint32_t savedIe = port->ie; + + // Keep reset-generated FIS traffic from satisfying a command waiter. + port->ie = 0; + KSTATUS status = port_comreset(drv); + port->serr = (uint32_t)-1; + port->is = (uint32_t)-1; + drv->lastIs = 0; + if (drv->completionEvent) + KeClearEvent(drv->completionEvent); + port->ie = savedIe; + + if (!KSUCCESS(status)) { + DbgPrint("AHCI: port recovery failed st=0x%x CI=0x%x TFD=0x%x SSTS=0x%x\n", + status, port->ci, port->tfd, port->ssts); + } +} + // Interrupt bottom half. The ISR owns hardware acknowledgement and status // capture; this DPC wakes the worker thread after the hard IRQ has unwound. @@ -191,7 +282,11 @@ static void AhciIsr(void) { // Clear the HBA-level bit. ctl->hba->is = portBit; - KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0); + // DMA Setup, PIO Setup, and other progress interrupts are not + // command completion. Only wake the worker for D2H completion or + // an error that terminates the command. + if (portIs & AHCI_PORT_TERMINAL_MASK) + KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0); } } } @@ -206,8 +301,19 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { HBA_PORT *port = drv->port; - // Clear stale IS and snapshot before issuing so we don't mis-read a - // leftover value from a prior command. + // A serialized slot must be idle before reuse. A stale active bit means a + // prior command escaped normal completion and must be recovered first. + if (port->ci & (1u << 0)) { + DbgPrint("AHCI: slot 0 still active before issue CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, port->is, port->serr); + AhciRecoverPort(drv); + if (port->ci & (1u << 0)) + return NTLSTATUS_DEVICE_ERROR; + } + + // Clear stale software and hardware completion state before submission. + // The event can otherwise retain a late DPC signal from an older command. + KeClearEvent(drv->completionEvent); port->is = (uint32_t)-1; drv->lastIs = 0; @@ -216,20 +322,54 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) spin++; if (spin == POLL_TIMEOUT) { - DbgPrint("AHCI: port hung (pre-issue)\n"); + DbgPrint("AHCI: port hung before issue TFD=0x%x IS=0x%x SERR=0x%x\n", + port->tfd, port->is, port->serr); + AhciRecoverPort(drv); return NTLSTATUS_TIMEOUT; } - // Issue slot 0, then block on the event. + // Issue slot 0, then block with a finite deadline. port->ci = (1u << 0); - KeWaitForEvent(drv->completionEvent, 0); + EVENT *events[1] = { drv->completionEvent }; + KSTATUS waitStatus = KeWaitForMultipleEvents(events, 1, 0, + AHCI_COMMAND_TIMEOUT_MS, + (uint32_t *)0); - // The ISR has cleared IS - check for errors in the snapshot. - uint32_t is = drv->lastIs; - if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { - DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", - is, port->tfd, port->serr); - port->serr = (uint32_t)-1; + // Merge any live status that was not consumed by the ISR. This lets a + // missed IRQ still report the hardware error that accompanied it. + uint32_t liveIs = port->is; + uint32_t is = drv->lastIs | liveIs; + if (liveIs) + port->is = liveIs; + + if (is & AHCI_PORT_ERROR_MASK) { + DbgPrint("AHCI: command error IS=0x%x CI=0x%x TFD=0x%x SERR=0x%x\n", + is, port->ci, port->tfd, port->serr); + AhciRecoverPort(drv); + return NTLSTATUS_DEVICE_ERROR; + } + + if (waitStatus == NTLSTATUS_TIMEOUT) { + if (!(port->ci & (1u << 0))) { + DbgPrint("AHCI: completion IRQ missed; command retired IS=0x%x TFD=0x%x SERR=0x%x\n", + is, port->tfd, port->serr); + return NTLSTATUS_SUCCESS; + } + + DbgPrint("AHCI: command timeout CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, is, port->serr); + AhciRecoverPort(drv); + return NTLSTATUS_TIMEOUT; + } + + if (waitStatus != NTLSTATUS_SUCCESS) + return waitStatus; + + // An interrupt is only completion if hardware has also retired slot 0. + if (port->ci & (1u << 0)) { + DbgPrint("AHCI: premature completion IRQ CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, is, port->serr); + AhciRecoverPort(drv); return NTLSTATUS_DEVICE_ERROR; } @@ -239,7 +379,7 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { // Build a standard DMA read or write command in slot 0. static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, - uint64_t bufPhys, bool write) { + uint64_t bufferDeviceAddress, bool write) { HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0 HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); @@ -256,7 +396,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, // Fill PRDT entries. uint32_t remaining = sectors; - uint64_t buf = bufPhys; + uint64_t buf = bufferDeviceAddress; for (uint32_t i = 0; i < prdtl; i++) { uint32_t sects = (remaining > 16) ? 16 : remaining; uint32_t bytes = sects * 512; @@ -289,6 +429,31 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, fis->counth = (uint8_t)(sectors >> 8); } +// Builds a non-data ATA FLUSH CACHE EXT command in slot 0. +static void build_flush_cmd(AHCI_DRIVE* drv) { + HBA_CMD_HEADER* hdr = cmd_list(drv); + HBA_CMD_TBL* tbl = cmd_tbl(drv, 0); + + memset(tbl, 0, PORT_CMDTBL_SIZE); + hdr->cfl = sizeof(FIS_REG_H2D) / 4; + hdr->w = 0; + hdr->prdtl = 0; + hdr->prdbc = 0; + + FIS_REG_H2D* fis = (FIS_REG_H2D*)tbl->cfis; + memset(fis, 0, sizeof(*fis)); + fis->fis_type = FIS_TYPE_REG_H2D; + fis->c = 1; + fis->command = ATA_CMD_FLUSH_EXT; +} + +// Flushes the drive write cache through the serialized command worker. +static KSTATUS do_flush(AHCI_DRIVE* drv) { + if (!drv) return NTLSTATUS_INVALID_PARAMETER; + build_flush_cmd(drv); + return port_issue_cmd(drv); +} + // IDENTIFY command - fills a 512-byte buffer and returns it. static KSTATUS port_identify(AHCI_DRIVE *drv) { @@ -305,8 +470,8 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) { hdr->prdtl = 1; hdr->prdbc = 0; - tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF); - tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32); + tbl->prdt_entry[0].dba = (uint32_t)(drv->bounceDeviceAddress & 0xFFFFFFFF); + tbl->prdt_entry[0].dbau = (uint32_t)(drv->bounceDeviceAddress >> 32); tbl->prdt_entry[0].dbc = 512 - 1; tbl->prdt_entry[0].i = 1; @@ -347,7 +512,7 @@ static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba, while (count > 0) { uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; - build_rw_cmd(drv, lba, batch, drv->bouncePhys, false); + build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, false); KSTATUS st = port_issue_cmd(drv); if (!KSUCCESS(st)) return st; @@ -368,7 +533,7 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba, uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; memcpy(drv->bounceVirt, in, (size_t)batch * 512); - build_rw_cmd(drv, lba, batch, drv->bouncePhys, true); + build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, true); KSTATUS st = port_issue_cmd(drv); if (!KSUCCESS(st)) return st; @@ -463,13 +628,16 @@ static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: drv->surpriseRemoved = true; device->PnpState = DevicePnpSurpriseRemoved; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: drv->removed = true; device->PnpState = DevicePnpDeleted; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); - IoReleaseRemoveLockAndWait(&drv->removeLock, irp); + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + if (drv->removeLockReady) + IoReleaseRemoveLockAndWait(&drv->removeLock, irp); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); IoDeleteDevice(device); drv->deviceObject = (DEVICE_OBJECT*)0; return NTLSTATUS_SUCCESS; @@ -511,7 +679,10 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) { for (int i = 0; i < ctl->driveCount; i++) { AHCI_DRIVE* drv = &ctl->drives[i]; drv->surpriseRemoved = true; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + if (drv->deviceObject) + (void)IoNotifyDiskSurpriseRemoved(drv->deviceObject->u.Disk.DiskIndex); + else + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); } return NTLSTATUS_SUCCESS; @@ -520,9 +691,18 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) { device->PnpState = DevicePnpDeleted; for (int i = 0; i < ctl->driveCount; i++) { AHCI_DRIVE* drv = &ctl->drives[i]; - drv->removed = true; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + if (drv->deviceObject) { + (void)IoUnregisterDiskDevice(drv->deviceObject->u.Disk.DiskIndex); + } else { + drv->removed = true; + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); + } } + AhciUnregisterController(ctl); + gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter); + ctl->dmaAdapter = (IO_DMA_ADAPTER*)0; IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; @@ -532,6 +712,29 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) { } } +// Queues one write-cache flush on the per-drive worker. +static KSTATUS AhciDispatchFlush(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER; + AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension; + if (!drv || drv->magic != AHCI_DRIVE_MAGIC) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp); + if (!KSUCCESS(st)) + return st; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) { + IoReleaseRemoveLock(&drv->removeLock, irp); + return NTLSTATUS_INVALID_PARAMETER; + } + + irp->Status = NTLSTATUS_FAILURE; + irp->Information = 0; + IrpQueueEnqueue(&drv->irpQueue, irp); + return NTLSTATUS_PENDING; +} + // Dispatches pnp. static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { if (!device || !device->DeviceExtension) @@ -552,7 +755,7 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { // ever having blocked in a driver call. // NCQ remains serialized until the driver grows a multi-slot dispatcher: -// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP. +// 1. Wait for a free slot and a pending IRP through the queue semaphore. // 2. Pick the oldest IRP from the queue. // 3. Grab a free bit in drv->slotMask (under the slot spinlock). // 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA), @@ -561,30 +764,45 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { static void AhciWorker(uintptr_t arg) { AHCI_DRIVE *drv = (AHCI_DRIVE *)arg; - while (true) { + while (drv && drv->workerRunning) { // Block until AHCI dispatch signals a new IRP. - KeWaitSemaphore(drv->irpQueue.Semaphore); + KeWaitSemaphore(drv->irpQueue.Semaphore, KE_WAIT_INFINITE); + if (!drv->workerRunning) + break; IRP *irp = IrpQueueDequeue(&drv->irpQueue); if (!irp) continue; - bool write = false; - uint64_t lba = 0; - uint32_t sectors = 0; - void* buffer = (void*)0; - KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); - if (KSUCCESS(st)) { - if (write) - st = do_write(drv, lba, sectors, buffer); - else - st = do_read (drv, lba, sectors, buffer); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + KSTATUS st = NTLSTATUS_INVALID_PARAMETER; + size_t information = 0; + + if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) { + st = do_flush(drv); + } else { + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + if (KSUCCESS(st)) { + if (write) + st = do_write(drv, lba, sectors, buffer); + else + st = do_read(drv, lba, sectors, buffer); + if (KSUCCESS(st)) + information = (size_t)sectors * 512u; + } } // Publish result then fire completion hooks through the common I/O path. IoReleaseRemoveLock(&drv->removeLock, irp); - IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0); + IoCompleteRequest(irp, st, information); } + + if (drv && drv->workerFinished) + KeSignalEvent(drv->workerFinished); } // AHCI controller initialisation. @@ -597,18 +815,8 @@ static void AhciWorker(uintptr_t arg) { // FRE must already be set (port_init_memory does this) so the HBA can // capture the signature FIS the device sends after the reset. -// PORT_IE bits we care about: -// DHRS (bit 0) - D2H Register FIS received - normal completion -// PSS (bit 1) - PIO Setup FIS received -// DSS (bit 2) - DMA Setup FIS received -// SDBS (bit 3) - Set Device Bits FIS received -// TFES (bit 30) - Task File Error Status - error -// HBFS (bit 29) - Host Bus Fatal error -// HBDS (bit 28) - Host Bus Data error -// IFS (bit 27) - Interface Fatal error. -#define PORT_IE_NORMAL (0x0000000Fu) -#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS) -#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS) +// Runtime commands are non-NCQ DMA commands. DHRS is their completion signal; +// setup/progress FIS interrupts must not wake the serialized command waiter. // Registers controller. static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) { @@ -623,6 +831,20 @@ static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) { return NTLSTATUS_SUCCESS; } +// Removes a controller from the interrupt registry. +static void AhciUnregisterController(AHCI_CONTROLLER* ctl) { + if (!ctl) + return; + for (int i = 0; i < g_controllerCount; i++) { + if (g_controllers[i] != ctl) + continue; + g_controllerCount--; + g_controllers[i] = g_controllers[g_controllerCount]; + g_controllers[g_controllerCount] = (AHCI_CONTROLLER*)0; + return; + } +} + // Starts controller. static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC || !ctl->pdo || !ctl->fdo) @@ -676,7 +898,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = PciMapBar(ctl->pdo, AHCI_PCI_BAR_ABAR, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&ctl->hba, &ctl->abarLength); if (!KSUCCESS(st)) { @@ -712,9 +934,23 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n", cap, nPorts, nSlots, (int)s64a); + IO_DMA_REQUIREMENTS dmaRequirements; + memset(&dmaRequirements, 0, sizeof(dmaRequirements)); + dmaRequirements.MaximumDeviceAddress = s64a ? (uint64_t)-1 : 0xFFFFFFFFULL; + dmaRequirements.Alignment = 2; + dmaRequirements.MaximumSegmentLength = 1u << 22; + dmaRequirements.MaximumSegments = PORT_PRDT_COUNT; + st = gAhciDdk->Dma->GetAdapter(ctl->fdo, &dmaRequirements, &ctl->dmaAdapter); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: DMA adapter creation failed (0x%x)\n", st); + return st; + } + st = AhciRegisterController(ctl); if (!KSUCCESS(st)) { DbgPrint("AHCI: controller registry full (0x%x)\n", st); + gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter); + ctl->dmaAdapter = (IO_DMA_ADAPTER*)0; return st; } @@ -751,6 +987,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = port_comreset(drv); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: COMRESET failed\n", portno); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } @@ -758,6 +996,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { if (!port_is_sata_disk(port)) { DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n", portno, port->sig); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } @@ -765,6 +1005,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } KeInitializeDpc(&drv->completionDpc, AhciCompletionDpc, drv); @@ -775,19 +1017,26 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } IrpQueueInit(&drv->irpQueue, irpSem); st = IoInitializeRemoveLock(&drv->removeLock); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } + drv->removeLockReady = true; // Phase 6: IDENTIFY (polled - scheduler not yet running). st = port_identify(drv); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } @@ -799,12 +1048,14 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { // Phase 7: enable per-port interrupts. // Clear any stale IS first so we don't get a spurious signal immediately. port->is = (uint32_t)-1; - port->ie = PORT_IE_ALL; + port->ie = AHCI_PORT_TERMINAL_MASK; // Phase 8: create and register a disk DEVICE_OBJECT. st = IoCreateDevice(&g_driver, 0, &drv->deviceObject); if (!KSUCCESS(st)) { DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", g_nextDiskIndex, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } drv->deviceObject->DeviceExtension = drv; @@ -822,19 +1073,29 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st); IoDeleteDevice(drv->deviceObject); drv->deviceObject = (DEVICE_OBJECT*)0; + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } // Phase 9: spawn the per-drive IRP worker thread. // The worker owns do_read / do_write exclusively once the scheduler // is live; it is the only thread that may call port_issue_cmd(). - st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv, - PriorityHigh, &drv->workerThread); + st = KeEventCreate(EventManualReset, 0, &drv->workerFinished); + if (KSUCCESS(st)) { + drv->workerRunning = true; + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh); + st = KeCreateKernelThread((uintptr_t)AhciWorker, + (uintptr_t)drv, + &workerParameters, + &drv->workerThread); + } if (!KSUCCESS(st)) { - DbgPrint("AHCI: drive %u: worker thread create failed (0x%x)\n", - diskIndex, st); - // Fatal for this device: all disk access is IRP-based now, so - // leave the device registered but clearly fail during testing. + drv->workerRunning = false; + DbgPrint("AHCI: drive %u: worker creation failed (0x%x)\n", diskIndex, st); + (void)IoUnregisterDiskDevice(diskIndex); + ctl->driveCount--; + continue; } } @@ -889,7 +1150,7 @@ static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevic // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt) + if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Dma) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("AHCI: driver loading...\n"); @@ -903,6 +1164,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { g_driver.DriverAddDevice = AhciAddDevice; g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite; g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite; + g_driver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AhciDispatchFlush; g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp; KSTATUS st = IoRegisterDriverObject(&g_driver); diff --git a/code/drivers/storage/ahci/ahci.h b/code/drivers/storage/ahci/ahci.h index 1547f32..f80dea8 100644 --- a/code/drivers/storage/ahci/ahci.h +++ b/code/drivers/storage/ahci/ahci.h @@ -250,6 +250,7 @@ typedef volatile struct { #define PORT_CMD_ICC_ACTIVE (0x1u << 28) // Port IS register bits (interrupt status). +#define PORT_IS_DHRS (1u << 0) // D2H Register FIS Received. #define PORT_IS_TFES (1u << 30) // Task File Error Status #define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error #define PORT_IS_HBDS (1u << 28) // Host Bus Data Error @@ -321,11 +322,13 @@ typedef struct { uint8_t portNo; // Physical port index (for ISR dispatch) // Structures block (command list + FIS + cmd tables). - uint64_t structPhys; // Physical base - void* structVirt; // Virtual base + IO_DMA_BUFFER* structDma; + uint64_t structDeviceAddress; // Device base. + void* structVirt; // CPU base. // Bounce buffer for DMA data. - uint64_t bouncePhys; + IO_DMA_BUFFER* bounceDma; + uint64_t bounceDeviceAddress; void* bounceVirt; // Signaled by the completion DPC when the port raises an interrupt. @@ -345,13 +348,16 @@ typedef struct { IO_REMOVE_LOCK removeLock; bool surpriseRemoved; bool removed; + bool removeLockReady; + volatile bool workerRunning; THREAD* workerThread; + EVENT* workerFinished; DEVICE_OBJECT* deviceObject; // Multi-slot NCQ needs per-slot ownership state in addition to the serialized queue: // uint32_t slotMask; // bitmask of in-flight slots // IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP - // uint64_t slotBouncePhys[NUM_CMD_SLOTS]; + // uint64_t slotBounceDeviceAddress[NUM_CMD_SLOTS]; // void* slotBounceVirt[NUM_CMD_SLOTS]; // volatile uint32_t prevSact; // previous port->sact snapshot. } AHCI_DRIVE; @@ -361,6 +367,7 @@ struct AHCI_CONTROLLER { DEVICE_OBJECT* fdo; DEVICE_OBJECT* pdo; DEVICE_OBJECT* lowerDevice; + IO_DMA_ADAPTER* dmaAdapter; PCI_DEVICE_INFO pciInfo; PCI_BAR abar; diff --git a/code/drivers/storage/ahci/ahci_port.c b/code/drivers/storage/ahci/ahci_port.c index 6012db0..6d82b81 100644 --- a/code/drivers/storage/ahci/ahci_port.c +++ b/code/drivers/storage/ahci/ahci_port.c @@ -11,16 +11,13 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) { int pageIndex = 0; for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) { uintptr_t virt = (uintptr_t)gAhciDdk->Memory->PhysToVirt(phys); - uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; - KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent); - // Ignore NTLSTATUS_ALREADY_EXISTS - a prior mapping with different flags - // is possible for the first page if the bootloader was overly generous. - // Use MmPteProtectRange to force NOCACHE in that case. - if (st == NTLSTATUS_ALREADY_EXISTS) { - st = MmPteProtectRange(virt, 4096, flags); - } + uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC; + KSTATUS st = gAhciDdk->Memory->MapPage(virt, + phys, + flags, + NtlDdkMapReplaceExisting); if (!KSUCCESS(st)) { - DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n", + DbgPrint("AHCI: MMIO map failed for page %d (phys=0x%016llX, st=0x%x)\n", pageIndex, (unsigned long long)phys, st); return st; } diff --git a/code/drivers/storage/ata/ata.c b/code/drivers/storage/ata/ata.c index adb76db..af04614 100644 --- a/code/drivers/storage/ata/ata.c +++ b/code/drivers/storage/ata/ata.c @@ -92,7 +92,7 @@ static KSTATUS AtaIdentify(ATA_DEVICE* dev) { uint16_t ident[256]; memset(ident, 0, sizeof(ident)); - KSTATUS st = KeWaitSemaphore(ch->Lock); + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return st; @@ -193,7 +193,7 @@ static KSTATUS AtaReadPio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, void* b ATA_CHANNEL* ch = dev->Channel; uint16_t* out = (uint16_t*)buffer; - KSTATUS st = KeWaitSemaphore(ch->Lock); + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return st; @@ -234,6 +234,22 @@ static KSTATUS AtaFlush(ATA_DEVICE* dev) { return AtaWaitNotBusy(ch); } +// Flushes one device while holding the shared channel lock. +static KSTATUS AtaFlushSerialized(ATA_DEVICE* dev) { + if (!dev || !dev->Channel) + return NTLSTATUS_INVALID_PARAMETER; + + ATA_CHANNEL* ch = dev->Channel; + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); + if (!KSUCCESS(st)) + return st; + + AtaSelectDrive(dev); + st = AtaFlush(dev); + KeSignalSemaphore(ch->Lock); + return st; +} + // Updates pio. static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const void* buffer) { if (!dev || !buffer) @@ -246,7 +262,7 @@ static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const ATA_CHANNEL* ch = dev->Channel; const uint16_t* in = (const uint16_t*)buffer; - KSTATUS st = KeWaitSemaphore(ch->Lock); + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return st; @@ -308,27 +324,38 @@ static void AtaWorker(uintptr_t arg) { return; while (true) { - KeWaitSemaphore(dev->IrpQueue.Semaphore); + KeWaitSemaphore(dev->IrpQueue.Semaphore, KE_WAIT_INFINITE); IRP* irp = IrpQueueDequeue(&dev->IrpQueue); if (!irp) continue; - bool write = false; - uint64_t lba = 0; - uint32_t sectors = 0; - void* buffer = (void*)0; - KSTATUS st = AtaGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); - if (KSUCCESS(st)) { - if (lba + sectors > dev->SectorCount) - st = NTLSTATUS_INVALID_PARAMETER; - else - st = write ? AtaWritePio(dev, lba, sectors, buffer) - : AtaReadPio(dev, lba, sectors, buffer); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + KSTATUS st = NTLSTATUS_INVALID_PARAMETER; + size_t information = 0; + + if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) { + st = AtaFlushSerialized(dev); + } else { + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + st = AtaGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + if (KSUCCESS(st)) { + if (lba + sectors > dev->SectorCount) { + st = NTLSTATUS_INVALID_PARAMETER; + } else { + st = write ? AtaWritePio(dev, lba, sectors, buffer) + : AtaReadPio(dev, lba, sectors, buffer); + if (KSUCCESS(st)) + information = (size_t)sectors * ATA_SECTOR_SIZE; + } + } } IoReleaseRemoveLock(&dev->RemoveLock, irp); - IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * ATA_SECTOR_SIZE) : 0); + IoCompleteRequest(irp, st, information); } } @@ -363,6 +390,31 @@ KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) { return NTLSTATUS_PENDING; } +// Queues one cache flush on the device worker. +static KSTATUS AtaDispatchFlush(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp || !device->DeviceExtension) + return NTLSTATUS_INVALID_PARAMETER; + + ATA_DEVICE* dev = (ATA_DEVICE*)device->DeviceExtension; + if (dev->Magic != ATA_DEVICE_MAGIC || !dev->Present) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS st = IoAcquireRemoveLock(&dev->RemoveLock, irp); + if (!KSUCCESS(st)) + return st; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) { + IoReleaseRemoveLock(&dev->RemoveLock, irp); + return NTLSTATUS_INVALID_PARAMETER; + } + + irp->Status = NTLSTATUS_FAILURE; + irp->Information = 0; + IrpQueueEnqueue(&dev->IrpQueue, irp); + return NTLSTATUS_PENDING; +} + // Returns free Disk Index. static uint8_t AtaFindFreeDiskIndex(void) { for (uint8_t i = 0; i < 8; i++) { @@ -486,8 +538,9 @@ static KSTATUS AtaProbeDrive(ATA_CONTROLLER* ctl, ATA_CHANNEL* ch, uint8_t drive return st; } + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh); st = KeCreateKernelThread((uintptr_t)AtaWorker, (uintptr_t)dev, - PriorityHigh, &dev->WorkerThread); + &workerParameters, &dev->WorkerThread); if (!KSUCCESS(st)) DbgPrint("ATA: drive %u worker thread create failed (0x%x)\n", dev->DiskIndex, st); return NTLSTATUS_SUCCESS; @@ -580,7 +633,7 @@ static void AtaRemoveControllerDevices(ATA_CONTROLLER* ctl, IO_DEVICE_CHANGE cha if (!dev->Present) continue; - IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); + IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); if (dev->DiskIndex != 0xFFu) { if (change == IoDeviceSurpriseRemoved) IoNotifyDiskSurpriseRemoved(dev->DiskIndex); @@ -647,12 +700,12 @@ KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: device->PnpState = DevicePnpSurpriseRemoved; dev->Present = false; - IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); + IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: device->PnpState = DevicePnpDeleted; dev->Present = false; - IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); + IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); IoReleaseRemoveLockAndWait(&dev->RemoveLock, irp); IoDeleteDevice(device); dev->DeviceObject = (DEVICE_OBJECT*)0; @@ -708,6 +761,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { gAtaDriver.DriverAddDevice = AtaAddDevice; gAtaDriver.MajorFunction[IRP_MJ_READ] = AtaDispatchReadWrite; gAtaDriver.MajorFunction[IRP_MJ_WRITE] = AtaDispatchReadWrite; + gAtaDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AtaDispatchFlush; gAtaDriver.MajorFunction[IRP_MJ_PNP] = AtaDispatchPnp; KSTATUS st = IoRegisterDriverObject(&gAtaDriver); diff --git a/code/kernel/hal/hal_processor.c b/code/kernel/hal/hal_processor.c index 860ba78..4ac4efe 100644 --- a/code/kernel/hal/hal_processor.c +++ b/code/kernel/hal/hal_processor.c @@ -37,6 +37,6 @@ __declspec(dllexport) void HalHalt(void) { // Masks interrupts and permanently stops the current processor. __declspec(dllexport) _Noreturn void HalStopProcessor(void) { _disable(); - for (;;) + while (true) __halt(); } diff --git a/code/kernel/include/kernel/io_dma.h b/code/kernel/include/kernel/io_dma.h new file mode 100644 index 0000000..4556a5b --- /dev/null +++ b/code/kernel/include/kernel/io_dma.h @@ -0,0 +1,13 @@ +// Kernel DMA declarations shared by the I/O manager and DDK provider. + +#pragma once + +#include + +struct DEVICE_OBJECT; + +// Returns the kernel-owned DDK DMA table. +const NTL_DDK_DMA* IoGetDmaDdk(void); + +// Detaches the adapter owned by a device before the device object can die. +void IoDmaDetachDevice(struct DEVICE_OBJECT* device); diff --git a/code/kernel/include/kernel/io_file.h b/code/kernel/include/kernel/io_file.h index 928adf8..34982d6 100644 --- a/code/kernel/include/kernel/io_file.h +++ b/code/kernel/include/kernel/io_file.h @@ -12,6 +12,16 @@ struct PROCESS; +// Initializes filesystem namespace change notification before drivers register volumes. +KSTATUS IoInitializeVolumeNamespace(void); + +// Returns the generation of the mounted-volume namespace. +uint64_t IoGetVolumeNamespaceGeneration(void); + +// Waits until the mounted-volume namespace changes or the timeout expires. +KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration, + uint64_t timeoutMilliseconds); + __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf, size_t size, diff --git a/code/kernel/include/kernel/ke_audio.h b/code/kernel/include/kernel/ke_audio.h new file mode 100644 index 0000000..eea1974 --- /dev/null +++ b/code/kernel/include/kernel/ke_audio.h @@ -0,0 +1,183 @@ +// Kernel audio-core declarations shared by the mixer, syscall layer, and selftests. + +#pragma once + +#include +#include +#include +#include +#include + +#define KE_AUDIO_MAX_ENDPOINTS 32u +#define KE_AUDIO_MAX_STREAMS_PER_ENDPOINT 64u +#define KE_AUDIO_STREAM_RING_FRAMES 4096u +#define KE_AUDIO_OUTPUT_SAMPLE_RATE 48000u +#define KE_AUDIO_OUTPUT_CHANNELS 2u +#define KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD 512u +#define KE_AUDIO_OUTPUT_PERIOD_COUNT 4u +#define KE_AUDIO_OUTPUT_BYTES_PER_FRAME 4u +#define KE_AUDIO_OUTPUT_PERIOD_BYTES 2048u +#define KE_AUDIO_GAIN_RAMP_FRAMES 480u +#define KE_AUDIO_IDLE_STOP_MILLISECONDS 250u +#define KE_AUDIO_MIXER_REALTIME_PRIORITY 16u + +#define KE_AUDIO_WAIT_WRITABLE 0x00000001u +#define KE_AUDIO_WAIT_CONSUMED 0x00000002u +#define KE_AUDIO_WAIT_PRESENTED 0x00000004u +#define KE_AUDIO_WAIT_STOPPED 0x00000008u +#define KE_AUDIO_WAIT_DRAINED 0x00000010u +#define KE_AUDIO_WAIT_FAILED 0x00000020u +#define KE_AUDIO_WAIT_DISCONTINUITY 0x00000040u +#define KE_AUDIO_WAIT_VALID_MASK 0x0000007Fu + +typedef enum KE_AUDIO_ENDPOINT_STATE { + KeAudioEndpointStarting = 1, + KeAudioEndpointReady, + KeAudioEndpointStopping, + KeAudioEndpointFailed, + KeAudioEndpointRemoved +} KE_AUDIO_ENDPOINT_STATE; + +typedef enum KE_AUDIO_STREAM_STATE { + KeAudioStreamCreated = 1, + KeAudioStreamRunning, + KeAudioStreamPaused, + KeAudioStreamDraining, + KeAudioStreamStopped, + KeAudioStreamFailed, + KeAudioStreamClosed +} KE_AUDIO_STREAM_STATE; + +typedef struct KE_AUDIO_STREAM KE_AUDIO_STREAM; + +// Reports one endpoint snapshot without exposing provider-owned pointers. +typedef struct KE_AUDIO_ENDPOINT_INFO { + uint64_t EndpointId; + IO_AUDIO_ENDPOINT_KIND Kind; + KE_AUDIO_ENDPOINT_STATE State; + uint32_t Capabilities; + IO_AUDIO_FORMAT HardwareFormat; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + uint8_t Available; + uint8_t IsDefault; + uint8_t HardwareRunning; + uint8_t MasterMuted; + int32_t MasterHardwareGainMilliDb; + float MasterRequestedVolume; + uint32_t StreamCount; + uint32_t ActiveStreamCount; + uint64_t PeriodsMixed; + uint64_t PeriodsCompleted; + uint64_t FramesPresented; + uint64_t HardwarePositionFrames; + uint64_t InterruptCount; + uint64_t LastPresentationTimestampTicks; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t DiscontinuityGeneration; + KSTATUS FailureStatus; +} KE_AUDIO_ENDPOINT_INFO; + +// Selects the next condition that should signal one stream's wait event. +typedef struct KE_AUDIO_STREAM_WAIT_CONDITION { + uint32_t Mask; + uint32_t WritableFrames; + uint64_t ConsumedFrames; + uint64_t PresentedFrames; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; +} KE_AUDIO_STREAM_WAIT_CONDITION; + +// Reports one logical stream snapshot. +typedef struct KE_AUDIO_STREAM_INFO { + KE_AUDIO_STREAM_STATE State; + IO_AUDIO_FORMAT Format; + uint32_t CapacityFrames; + uint32_t BufferedFrames; + uint32_t WritableFrames; + uint64_t FramesSubmitted; + uint64_t FramesConsumed; + uint64_t FramesPresented; + uint64_t FramesDiscarded; + uint64_t OutputFramesMixed; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; + KSTATUS FailureStatus; + float Volume; + float Pan; + uint8_t Muted; +} KE_AUDIO_STREAM_INFO; + +// Initializes endpoint registry and mixer ownership before drivers load. +void KeAudioInitialize(void); + +// Returns the provider-facing audio DDK table. +const NTL_DDK_AUDIO* KeAudioGetDdk(void); + +// Copies current endpoint snapshots and reports the full available count. +KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t capacity, + uint32_t* countOut); + +// Returns the currently selected endpoint identifier, or zero when none is usable. +uint64_t KeAudioGetDefaultEndpointId(void); + +// Creates one fixed-format logical render stream and returns a caller-owned reference. +KSTATUS KeAudioCreateStream(uint64_t endpointId, + const IO_AUDIO_FORMAT* format, + KE_AUDIO_STREAM** outStream); + +// Records one process handle that owns the stream. +void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream); + +// Releases one process-handle owner without blocking the handle-table path. +void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream); + +// Returns a borrowed passive event valid while the caller retains the stream. +EVENT* KeAudioStreamWaitEvent(KE_AUDIO_STREAM* stream); + +// Arms the stream event for one or more progress conditions without a lost-wakeup window. +KSTATUS KeAudioArmStreamWait(KE_AUDIO_STREAM* stream, + const KE_AUDIO_STREAM_WAIT_CONDITION* condition); + +// Copies as many complete source frames as fit without blocking. +KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream, + const void* frames, + uint32_t requestedFrames, + uint32_t* writtenFramesOut); + +// Starts one created or stopped stream. +KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream); + +// Pauses future mixing while preserving source frames not yet consumed. +KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream); + +// Resumes one paused stream. +KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream); + +// Rejects new writes and stops after every submitted frame is presented or discarded. +KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream); + +// Stops future mixing and discards source frames not yet committed to endpoint periods. +KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream); + +// Invalidates queued presentation accounting and returns an empty stream to position zero. +KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream); + +// Updates per-stream software gain with a ten-millisecond ramp. +KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream, + float volume, + float pan, + bool muted); + +// Updates endpoint master software gain and separate logical mute state. +KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted); + +// Copies one coherent stream snapshot. +KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut); + +// Runs the final render path selftest behind KE_DEBUG_AUDIO_SELFTEST. +void KiRunAudioFinalPathSelfTest(void); diff --git a/code/kernel/include/kernel/ke_cpu.h b/code/kernel/include/kernel/ke_cpu.h index a1987b7..7e373a4 100644 --- a/code/kernel/include/kernel/ke_cpu.h +++ b/code/kernel/include/kernel/ke_cpu.h @@ -7,6 +7,7 @@ // owns CPU-local scheduler, IRQL, GDT, and TSS state. #include +#include #include #include @@ -83,6 +84,8 @@ extern KCPU gCpus[KE_MAX_CPUS]; extern uint32_t gCpuCount; void KeInitializeBootstrapCpu(void); +void KeInitializeMemoryTypes(void); +bool KeWriteCombiningAvailable(void); void KeLoadKernelGsBase(KCPU* cpu); void KeSetUserGsBase(uintptr_t userGsBase); uintptr_t KeGetUserGsBase(void); diff --git a/code/kernel/include/kernel/ke_debug.h b/code/kernel/include/kernel/ke_debug.h index 86e2d69..946a061 100644 --- a/code/kernel/include/kernel/ke_debug.h +++ b/code/kernel/include/kernel/ke_debug.h @@ -10,21 +10,6 @@ #include #include -#define KE_DEBUG_CONSOLE 0x00000001u -#define KE_DEBUG_SELFTEST 0x00000002u -#define KE_DEBUG_FILE_LOG 0x00000004u - -#define KE_DEBUG_POOL_CANARY 0x00000008u -#define KE_DEBUG_POOL_POISON 0x00000010u -#define KE_DEBUG_POOL_QUARANTINE 0x00000020u -#define KE_DEBUG_POOL_CALLERS 0x00000040u -#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u -#define KE_DEBUG_DISPLAY_TRACE 0x00000100u -#define KE_DEBUG_INPUT_TRACE 0x00000200u -#define KE_DEBUG_VM_ASSERTS 0x00000400u -#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u -#define KE_DEBUG_LIST_CHECKS 0x00001000u - void KeInitDebug(const HAL_API* hal, uint32_t flags); uint32_t KeGetDebugFlags(void); __declspec(dllexport) void DbgPrint(const char* fmt, ...); diff --git a/code/kernel/include/kernel/ke_display.h b/code/kernel/include/kernel/ke_display.h index 733e006..66204b0 100644 --- a/code/kernel/include/kernel/ke_display.h +++ b/code/kernel/include/kernel/ke_display.h @@ -28,6 +28,9 @@ typedef struct DISPLAY_CLOCK_OBJECT { } DISPLAY_CLOCK_OBJECT; void KeDisplayInitialize(const HAL_API* hal); + +// Completes every console glyph queued before the call. +KSTATUS KeDisplayFlushConsoleOutput(void); DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock); DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock); DISPLAY_API void KeDisplayTick(void); diff --git a/code/kernel/include/kernel/ke_mutex.h b/code/kernel/include/kernel/ke_mutex.h index 6a88659..52b3aad 100644 --- a/code/kernel/include/kernel/ke_mutex.h +++ b/code/kernel/include/kernel/ke_mutex.h @@ -1,20 +1,37 @@ -// Kernel ke mutex declarations shared by implementation files and kernel clients. +// Kernel mutex declarations shared by implementation files and kernel clients. #pragma once -// ke_mutex.h - binary ownership mutex. + #include #include #include +#include #include +struct THREAD; + typedef struct MUTEX { - OBJECT Header; - SPINLOCK Lock; - struct THREAD* Owner; // NULL if unowned - LIST_ENTRY WaitList; + OBJECT Header; + SPINLOCK Lock; + struct THREAD* Owner; + LIST_ENTRY WaitList; + LIST_ENTRY OwnerLink; + KE_SCHEDULING_CLASS DonationClass; + uint32_t DonationPriority; + uint8_t HasDonation; + uint8_t Abandoned; } MUTEX; -__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out); -__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex); -__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex); -__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex); +// Tests whether a mutex acquisition transferred ownership to the caller. +#define KE_MUTEX_ACQUIRE_SUCCEEDED(status) \ + ((status) == NTLSTATUS_SUCCESS || (status) == NTLSTATUS_ABANDONED) + +__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out); +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds); +__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex); + +// Removes one blocked waiter while preserving priority-inheritance state. +int KiMutexAbortWait(MUTEX* mutex, struct THREAD* thread); + +// Releases every mutex owned by a terminating thread as abandoned. +void KiMutexRundownThread(struct THREAD* thread); diff --git a/code/kernel/include/kernel/ke_process.h b/code/kernel/include/kernel/ke_process.h index 495df82..b33d805 100644 --- a/code/kernel/include/kernel/ke_process.h +++ b/code/kernel/include/kernel/ke_process.h @@ -30,6 +30,7 @@ #define HANDLE_RIGHT_QUERY 0x00000020u #define HANDLE_RIGHT_TERMINATE 0x00000040u #define HANDLE_RIGHT_LOCK 0x00000080u +#define HANDLE_RIGHT_RESUME 0x00000100u #define HANDLE_RIGHT_ALL 0xFFFFFFFFu struct SYS_EXCEPTION_STATS; @@ -62,7 +63,8 @@ typedef struct PROCESS { uint8_t CurrentDrive; SPINLOCK CwdLock; - // Thread list. + // Live thread set. Dead THREAD objects may outlive this membership while + // retained handles remain waitable and queryable. LIST_ENTRY ThreadList; // links THREAD.ProcessLink uint32_t ThreadCount; diff --git a/code/kernel/include/kernel/ke_sched.h b/code/kernel/include/kernel/ke_sched.h index 972a8bb..e9bb39a 100644 --- a/code/kernel/include/kernel/ke_sched.h +++ b/code/kernel/include/kernel/ke_sched.h @@ -1,18 +1,20 @@ // Kernel ke sched declarations shared by implementation files and kernel clients. #pragma once -// ke_sched.h - Multi-Level Feedback Queue scheduler -// Three run queues (Low / Normal / High). LAPIC timer IRQs drive KeSchedulerTick. +// The scheduler combines dynamic normal queues with static real-time queues. +// LAPIC timer IRQs drive KeSchedulerTick. #include struct KCPU; #define SCHED_TICK_MS 10u +#define SCHED_REALTIME_QUANTUM_TICKS 1u #define SCHED_HIGH_QUANTUM_TICKS 2u #define SCHED_NORMAL_QUANTUM_TICKS 4u #define SCHED_LOW_QUANTUM_TICKS 8u #define SCHED_DEMOTE_AFTER_BURST_TICKS 16u #define SCHED_AGING_INTERVAL_TICKS 100u +#define KE_PRIORITY_INHERITANCE_MAX_DEPTH 64u typedef struct KE_SCHED_STATS { uint64_t Ticks; @@ -27,8 +29,61 @@ typedef struct KE_SCHED_STATS { uint64_t CpuBurstDemotions; uint64_t SmtSiblingDeferrals; uint64_t SmtPrimaryPokes; + uint64_t RealtimeDispatches; + uint64_t RealtimePreemptions; + uint64_t PriorityInheritanceBoosts; + uint64_t PriorityInheritanceDrops; + uint64_t PriorityInheritancePropagations; + uint64_t PriorityInheritanceMaxDepth; } KE_SCHED_STATS; + +struct MUTEX; + +typedef struct KE_THREAD_SCHEDULER_SNAPSHOT { + THREAD_STATE State; + KE_SCHEDULING_CLASS BaseSchedulingClass; + uint32_t BasePriority; + KE_SCHEDULING_CLASS EffectiveSchedulingClass; + uint32_t EffectivePriority; + uint32_t RunningCpu; + uint32_t LastCpu; + uint32_t QuantumRemaining; + uint32_t QuantumTicks; + uint32_t CpuBurstTicks; + uint32_t LastRunTicks; + uint32_t WaitObjectType; + KSTATUS WaitStatus; + struct MUTEX* WaitingMutex; + uint8_t Queued; + uint8_t AlertableWait; +} KE_THREAD_SCHEDULER_SNAPSHOT; + +// Copies scheduler-owned thread state into a stable diagnostic snapshot. +void KiSchedulerSnapshotThread(THREAD* thread, KE_THREAD_SCHEDULER_SNAPSHOT* snapshot); + +// Selects waiters and refreshes donations while the mutex lock is held. +struct THREAD* KiSchedulerSelectMutexWaiter(struct MUTEX* mutex); +int KiSchedulerRefreshMutexDonation(struct MUTEX* mutex); + +// Updates one thread's normal-class base priority. +KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority); + +// Publishes one mutex dependency atomically with cycle detection. +KSTATUS KiSchedulerPrepareMutexWait(THREAD* thread, struct MUTEX* mutex, THREAD* owner); +void KiSchedulerCancelMutexWait(THREAD* thread, struct MUTEX* mutex); + +// Associates mutex ownership with priority-inheritance bookkeeping. +void KiSchedulerAcquireMutexOwnership(THREAD* thread, struct MUTEX* mutex); +void KiSchedulerTransferMutexOwnership(THREAD* oldOwner, THREAD* newOwner, + struct MUTEX* mutex); +void KiSchedulerReleaseMutexOwnership(THREAD* thread, struct MUTEX* mutex); +struct MUTEX* KiSchedulerFirstOwnedMutex(THREAD* thread); +void KiSchedulerNotePriorityPropagation(uint32_t depth); + +// Recomputes effective rank and optionally returns a referenced upstream mutex. +int KiSchedulerRefreshEffectivePriority(THREAD* thread, struct MUTEX** waitingMutexOut); + // Called once by KernelMain before any threads are created. void KeInitializeScheduler(void); diff --git a/code/kernel/include/kernel/ke_semaphore.h b/code/kernel/include/kernel/ke_semaphore.h index 36b302b..a9be982 100644 --- a/code/kernel/include/kernel/ke_semaphore.h +++ b/code/kernel/include/kernel/ke_semaphore.h @@ -17,6 +17,5 @@ typedef struct SEMAPHORE { } SEMAPHORE; __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out); -__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore); -__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* semaphore); +__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore, uint64_t timeoutMilliseconds); __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* semaphore); diff --git a/code/kernel/include/kernel/ke_thread.h b/code/kernel/include/kernel/ke_thread.h index 6f6e397..201d446 100644 --- a/code/kernel/include/kernel/ke_thread.h +++ b/code/kernel/include/kernel/ke_thread.h @@ -18,6 +18,7 @@ #define KE_USER_EXCEPTION_MAX_DEPTH 8u typedef struct THREAD THREAD; +struct MUTEX; #ifndef NTL_THREAD_PRIORITY_DEFINED typedef enum THREAD_PRIORITY { @@ -34,6 +35,7 @@ typedef enum THREAD_STATE { ThreadRunning = 1, ThreadWaiting = 2, ThreadDead = 3, + ThreadSuspended = 4, } THREAD_STATE; typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread, @@ -51,17 +53,20 @@ typedef struct KCONTEXT { struct THREAD { OBJECT Header; // must be first - uint32_t ThreadId; - THREAD_STATE State; - THREAD_PRIORITY Priority; + uint32_t ThreadId; + THREAD_STATE State; + KE_SCHEDULING_CLASS BaseSchedulingClass; + uint32_t BasePriority; + KE_SCHEDULING_CLASS EffectiveSchedulingClass; + uint32_t EffectivePriority; // CPU-burst mirror kept here for the assembly-stable THREAD layout. // Dynamic quantum state lives later in the structure. - uint32_t QuantaConsumed; + uint32_t QuantaConsumed; PROCESS* Owner; LIST_ENTRY ProcessLink; // node in Owner->ThreadList - LIST_ENTRY SchedulerLink; // node in run queue or wait list + LIST_ENTRY SchedulerLink; // node in run queue or object wait list // Kernel stack - never PAGE_USER, pointed to by TssRsp0. void* KernelStackBase; @@ -108,6 +113,12 @@ struct THREAD { uint32_t CpuBurstTicks; uint32_t LastRunTicks; uint32_t RunningCpu; + uint32_t SuspendCount; + + // Priority inheritance ownership is scheduler-owned. Mutex wait-list + // membership remains protected by the mutex spinlock. + LIST_ENTRY OwnedMutexList; + struct MUTEX* WaitingMutex; // Scheduler hardening state. Queued means membership in a scheduler ready // queue only; wait-object membership is tracked by WaitListHead. Idle @@ -125,7 +136,9 @@ struct THREAD { KSTATUS WaitStatus; LIST_ENTRY* WaitListHead; uint8_t AlertableWait; - uint64_t SleepUntilTick; + LIST_ENTRY TimeoutLink; + uint8_t TimeoutArmed; + uint64_t TimeoutTick; KSTATUS TimeoutStatus; // Optional cleanup hook for waits that use auxiliary wait blocks. @@ -167,6 +180,11 @@ struct THREAD { uint32_t UserExceptionNoncontinuableMask; uintptr_t UserExceptionFrameUserAddress; KSTATUS UserExceptionExitStatus[KE_USER_EXCEPTION_MAX_DEPTH]; + + // Synchronous kernel work can temporarily retain caller-owned stack or + // process memory. Process termination must wait until that ownership has + // returned instead of freeing the thread underneath an active driver. + volatile long TerminationDeferralCount; }; // Creates and queues a user thread. On success, *out receives a referenced @@ -174,24 +192,31 @@ struct THREAD { // after copying any needed IDs/state. The live scheduler/thread lifetime is // separate and remains valid until the thread exits and is reaped. __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out); + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out); -// Create a kernel-mode-only thread. fn is called directly in ring 0 with +// Cancels a thread that has never left its initial suspended state. This drops +// the scheduler-owned lifetime reference but leaves the caller's reference +// intact. +KSTATUS KeCancelSuspendedThread(THREAD* thread); + +// Creates a kernel-mode-only thread. fn is called directly in ring 0 with // arg; when fn returns the thread exits automatically. No user stack or TEB -// is allocated. The thread is owned by the system process and runs at the -// given priority. On success, non-NULL out receives a referenced THREAD -// pointer owned by the caller; release it with ObDereferenceObject() after -// copying any needed IDs/state or after joining/waiting on caller-owned -// completion state. The scheduler's live thread reference is separate. +// is allocated. The thread is owned by the system process. On success, +// non-NULL out receives a referenced THREAD pointer owned by the caller; +// release it with ObDereferenceObject() after copying any needed IDs/state or +// after joining caller-owned completion state. The scheduler's live thread +// reference is separate. __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out); -__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, uint64_t affinityMask, + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out); // Mark a thread dead and deschedule it permanently. __declspec(dllexport) void KeExitThread(THREAD* thread, int code); +// Publishes completion, releases per-thread user memory, and removes one dead +// thread from its process live-thread set. Retained handles remain waitable. +void KeFinalizeThreadExit(THREAD* thread); + // Block / unblock primitives used by sync objects. // Correct wait pattern: @@ -213,12 +238,17 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable); KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable); void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status); void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status); -void KeAbortThreadWait(THREAD* thread, KSTATUS status); +int KeAbortThreadWait(THREAD* thread, KSTATUS status); void KeUnblockThread(THREAD* thread); KSTATUS KeAlertThread(THREAD* thread); void KiWakeThreadForKernelApc(THREAD* thread); void KiWakeThreadForUserApc(THREAD* thread); +// Defers process termination while kernel work owns caller-backed storage. +void KeEnterTerminationDeferral(void); +void KeLeaveTerminationDeferral(void); +int KeThreadTerminationDeferred(const THREAD* thread); + // Assembly stub - implemented in ke_switch.asm. void KeSwitchContext(THREAD* from, THREAD* to); __declspec(noreturn) void KeStartInitialThread(THREAD* thread); diff --git a/code/kernel/include/kernel/ke_time.h b/code/kernel/include/kernel/ke_time.h index 813db9a..8870f7a 100644 --- a/code/kernel/include/kernel/ke_time.h +++ b/code/kernel/include/kernel/ke_time.h @@ -11,3 +11,6 @@ void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal); KSTATUS KeTimeReloadSettings(void); __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime); __declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void); + +// Returns the monotonic platform counter in microseconds. +uint64_t KeQueryMonotonicTicks(void); diff --git a/code/kernel/include/kernel/mm_file_cache.h b/code/kernel/include/kernel/mm_file_cache.h index 603bb7c..a31e13e 100644 --- a/code/kernel/include/kernel/mm_file_cache.h +++ b/code/kernel/include/kernel/mm_file_cache.h @@ -43,19 +43,6 @@ typedef struct MM_FILE_CACHE_STATS { uint64_t DropFailures; uint64_t StandbyTransitions; uint64_t StandbyHits; - uint64_t ReadAheadRequests; - uint64_t ReadAheadPagesRequested; - uint64_t ReadAheadPagesQueued; - uint64_t ReadAheadPagesLoaded; - uint64_t ReadAheadPagesSkipped; - uint64_t ReadAheadFailures; - uint64_t ReadAheadPagesThrottled; - uint64_t MaxReadAheadPagesPerRequest; - uint64_t BatchReads; - uint64_t BatchPagesRequested; - uint64_t BatchPagesLoaded; - uint64_t MaxBatchPagesRequested; - uint64_t MaxBatchPagesLoaded; uint64_t ResidentPages; uint64_t ActivePages; uint64_t StandbyPages; @@ -92,8 +79,4 @@ void MmFileCacheInvalidateAll(FILE_OBJECT* file); // it never writes back because v1 stores only clean file data. KSTATUS MmFileCacheDropOnePageForPressure(void); -// Queues best-effort read-ahead for clean file pages. The queue owns a file -// reference until the worker has consumed the request. -KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages); - void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out); diff --git a/code/kernel/include/kernel/mm_pfn.h b/code/kernel/include/kernel/mm_pfn.h index 1ee0843..2a559b8 100644 --- a/code/kernel/include/kernel/mm_pfn.h +++ b/code/kernel/include/kernel/mm_pfn.h @@ -124,6 +124,12 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage); __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage); +PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count, + PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive, + size_t alignment, + size_t boundary, + MM_PFN_USAGE usage); __declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame); __declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame); __declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame); diff --git a/code/kernel/include/kernel/mm_pte.h b/code/kernel/include/kernel/mm_pte.h index 6ef7520..83c8497 100644 --- a/code/kernel/include/kernel/mm_pte.h +++ b/code/kernel/include/kernel/mm_pte.h @@ -12,10 +12,11 @@ #define PAGE_PRESENT (1ULL << 0) #define PAGE_WRITE (1ULL << 1) #define PAGE_USER (1ULL << 2) -#ifndef PAGE_WRITETHROUGH -#define PAGE_WRITETHROUGH (1ULL << 3) -#endif -#define PAGE_NOCACHE (1ULL << 4) +// Cache policy is selected through the canonical PAT layout. +#define PAGE_CACHE_WB 0ULL +#define PAGE_CACHE_WC (1ULL << 3) +#define PAGE_CACHE_UC (1ULL << 4) +#define PAGE_CACHE_MASK (PAGE_CACHE_WC | PAGE_CACHE_UC) #define PAGE_ACCESSED (1ULL << 5) #define PAGE_DIRTY (1ULL << 6) #define PAGE_NX (1ULL << 63) diff --git a/code/kernel/include/kernel/mm_virt.h b/code/kernel/include/kernel/mm_virt.h index c3c4ef4..6691d3c 100644 --- a/code/kernel/include/kernel/mm_virt.h +++ b/code/kernel/include/kernel/mm_virt.h @@ -66,6 +66,15 @@ typedef struct MM_PINNED_USER_PAGE { } MM_PINNED_USER_PAGE; typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER; +typedef struct MM_KERNEL_VIEW MM_KERNEL_VIEW; + +// Maps existing frames into a stable kernel-only virtual range. The view owns +// virtual address space and PTEs, but it does not own the physical frames. +KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames, + size_t pageCount, + MM_KERNEL_VIEW** outView); +void* MmKernelViewBase(const MM_KERNEL_VIEW* view); +void MmDestroyKernelView(MM_KERNEL_VIEW* view); // Pins one user page and returns a kernel direct-map alias for the requested // address. Callers may copy through KernelAddress + PageOffset while the page is @@ -80,6 +89,8 @@ __declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page); // unlocked. WriteAccess describes kernel writes to the user buffer. __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out); __declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer); +size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer); +PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index); // Virtual address descriptors. // The single shared address space still has per-process VAD ownership so process diff --git a/code/kernel/include/kernel/syscall_nr.h b/code/kernel/include/kernel/syscall_nr.h index 97ecb5f..4be8df4 100644 --- a/code/kernel/include/kernel/syscall_nr.h +++ b/code/kernel/include/kernel/syscall_nr.h @@ -17,138 +17,154 @@ #define SYS_GET_PROCESS_LIST 6 #define SYS_GET_PROCESS_PID 7 #define SYS_CREATE_THREAD 8 -#define SYS_GET_THREAD_LIST 9 -#define SYS_LOAD_MODULE 10 -#define SYS_UNLOAD_MODULE 11 -#define SYS_GET_MODULE_INFO 12 -#define SYS_GET_MODULE_EXPORT 13 -#define SYS_GET_MODULE_LIST 14 -#define SYS_GET_CURRENT_TID 15 -#define SYS_GET_CURRENT_PID 16 -#define SYS_SET_THREAD_PRIORITY 17 -#define SYS_YIELD 18 -#define SYS_SLEEP 19 -#define SYS_WAIT_SINGLE 20 -#define SYS_WAIT_MANY 21 -#define SYS_SET_USER_APC_DISPATCHER 22 -#define SYS_APC_RETURN 23 -#define SYS_SET_EXCEPTION_DISPATCHER 24 -#define SYS_GET_EXCEPTION_DISPATCHER 25 -#define SYS_CONTINUE_EXCEPTION 26 -#define SYS_RAISE_EXCEPTION 27 +#define SYS_RESUME_THREAD 9 +#define SYS_GET_THREAD_LIST 10 +#define SYS_LOAD_MODULE 11 +#define SYS_UNLOAD_MODULE 12 +#define SYS_GET_MODULE_INFO 13 +#define SYS_GET_MODULE_EXPORT 14 +#define SYS_GET_MODULE_LIST 15 +#define SYS_GET_CURRENT_TID 16 +#define SYS_GET_CURRENT_PID 17 +#define SYS_SET_THREAD_PRIORITY 18 +#define SYS_YIELD 19 +#define SYS_SLEEP 20 +#define SYS_WAIT_SINGLE 21 +#define SYS_WAIT_MANY 22 +#define SYS_SET_USER_APC_DISPATCHER 23 +#define SYS_APC_RETURN 24 +#define SYS_SET_EXCEPTION_DISPATCHER 25 +#define SYS_GET_EXCEPTION_DISPATCHER 26 +#define SYS_CONTINUE_EXCEPTION 27 +#define SYS_RAISE_EXCEPTION 28 // Virtual memory, mappings, and handles. -#define SYS_ALLOC_VM 28 -#define SYS_FREE_VM 29 -#define SYS_PROTECT_VM 30 -#define SYS_QUERY_VM 31 -#define SYS_CLOSE_HANDLE 32 -#define SYS_DUPLICATE_HANDLE 33 -#define SYS_CREATE_FILE_MAPPING 34 -#define SYS_MAP_FILE 35 +#define SYS_ALLOC_VM 29 +#define SYS_FREE_VM 30 +#define SYS_PROTECT_VM 31 +#define SYS_QUERY_VM 32 +#define SYS_CLOSE_HANDLE 33 +#define SYS_DUPLICATE_HANDLE 34 +#define SYS_CREATE_FILE_MAPPING 35 +#define SYS_MAP_FILE 36 // Synchronization objects. -#define SYS_CREATE_MUTEX 36 -#define SYS_TRY_ACQUIRE_MUTEX 37 -#define SYS_RELEASE_MUTEX 38 -#define SYS_CREATE_SEM 39 -#define SYS_TRY_WAIT_SEM 40 -#define SYS_SIGNAL_SEM 41 -#define SYS_CREATE_EVENT 42 -#define SYS_SIGNAL_EVENT 43 -#define SYS_CLEAR_EVENT 44 +#define SYS_CREATE_MUTEX 37 +#define SYS_TRY_ACQUIRE_MUTEX 38 +#define SYS_RELEASE_MUTEX 39 +#define SYS_CREATE_SEM 40 +#define SYS_TRY_WAIT_SEM 41 +#define SYS_SIGNAL_SEM 42 +#define SYS_CREATE_EVENT 43 +#define SYS_SIGNAL_EVENT 44 +#define SYS_CLEAR_EVENT 45 // Queues and named queues. -#define SYS_CREATE_QUEUE 45 -#define SYS_SEND_QUEUE 46 -#define SYS_RECV_QUEUE 47 -#define SYS_PUBLISH_QUEUE 48 -#define SYS_OPEN_NAMED_QUEUE 49 -#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 50 +#define SYS_CREATE_QUEUE 46 +#define SYS_SEND_QUEUE 47 +#define SYS_RECV_QUEUE 48 +#define SYS_PUBLISH_QUEUE 49 +#define SYS_OPEN_NAMED_QUEUE 50 +#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 51 // Files, directories, and current directory. -#define SYS_OPEN_FILE 51 -#define SYS_READ_FILE 52 -#define SYS_WRITE_FILE 53 -#define SYS_READ_FILE_ASYNC 54 -#define SYS_WRITE_FILE_ASYNC 55 -#define SYS_CANCEL_IO 56 -#define SYS_SEEK_FILE 57 -#define SYS_GET_FILE_POS 58 -#define SYS_GET_FILE_SIZE 59 -#define SYS_GET_FILE_STAT 60 -#define SYS_STAT_FILE 61 -#define SYS_TRUNCATE_FILE 62 -#define SYS_FLUSH_FILE 63 -#define SYS_DELETE_FILE 64 -#define SYS_RENAME_FILE 65 -#define SYS_LOCK_FILE 66 -#define SYS_UNLOCK_FILE 67 -#define SYS_OPEN_DIR 68 -#define SYS_READ_DIR 69 -#define SYS_CREATE_DIR 70 -#define SYS_REMOVE_DIR 71 -#define SYS_SET_CWD 72 -#define SYS_REAL_PATH 73 +#define SYS_OPEN_FILE 52 +#define SYS_READ_FILE 53 +#define SYS_WRITE_FILE 54 +#define SYS_READ_FILE_ASYNC 55 +#define SYS_WRITE_FILE_ASYNC 56 +#define SYS_CANCEL_IO 57 +#define SYS_SEEK_FILE 58 +#define SYS_GET_FILE_POS 59 +#define SYS_GET_FILE_SIZE 60 +#define SYS_GET_FILE_STAT 61 +#define SYS_STAT_FILE 62 +#define SYS_TRUNCATE_FILE 63 +#define SYS_FLUSH_FILE 64 +#define SYS_DELETE_FILE 65 +#define SYS_RENAME_FILE 66 +#define SYS_LOCK_FILE 67 +#define SYS_UNLOCK_FILE 68 +#define SYS_OPEN_DIR 69 +#define SYS_READ_DIR 70 +#define SYS_CREATE_DIR 71 +#define SYS_REMOVE_DIR 72 +#define SYS_SET_CWD 73 +#define SYS_REAL_PATH 74 // Console, display, and surfaces. -#define SYS_WRITE_CONSOLE 74 -#define SYS_GET_CONSOLE_INFO 75 -#define SYS_SET_CONSOLE_CURSOR 76 -#define SYS_FILL_CONSOLE_CHARACTER 77 -#define SYS_FILL_CONSOLE_ATTRIBUTE 78 -#define SYS_PUT_SCREEN_CHAR 79 -#define SYS_CLEAR_SCREEN 80 -#define SYS_GET_SCREEN_SIZE 81 -#define SYS_QUERY_DISPLAY_INFO 82 -#define SYS_OPEN_DISPLAY_CLOCK 83 -#define SYS_CREATE_SURFACE 84 -#define SYS_MAP_SURFACE 85 -#define SYS_QUERY_SURFACE 86 -#define SYS_PRESENT_SURFACE 87 +#define SYS_WRITE_CONSOLE 75 +#define SYS_GET_CONSOLE_INFO 76 +#define SYS_SET_CONSOLE_CURSOR 77 +#define SYS_FILL_CONSOLE_CHARACTER 78 +#define SYS_FILL_CONSOLE_ATTRIBUTE 79 +#define SYS_PUT_SCREEN_CHAR 80 +#define SYS_CLEAR_SCREEN 81 +#define SYS_GET_SCREEN_SIZE 82 +#define SYS_QUERY_DISPLAY_INFO 83 +#define SYS_OPEN_DISPLAY_CLOCK 84 +#define SYS_CREATE_SURFACE 85 +#define SYS_MAP_SURFACE 86 +#define SYS_QUERY_SURFACE 87 +#define SYS_PRESENT_SURFACE 88 + +// Native render audio. +#define SYS_AUDIO_ENUMERATE_ENDPOINTS 89 +#define SYS_AUDIO_OPEN_STREAM 90 +#define SYS_AUDIO_WRITE 91 +#define SYS_AUDIO_ARM_WAIT 92 +#define SYS_AUDIO_QUERY_STREAM 93 +#define SYS_AUDIO_START 94 +#define SYS_AUDIO_PAUSE 95 +#define SYS_AUDIO_RESUME 96 +#define SYS_AUDIO_DRAIN 97 +#define SYS_AUDIO_STOP 98 +#define SYS_AUDIO_RESET 99 +#define SYS_AUDIO_SET_VOLUME 100 +#define SYS_AUDIO_SET_MASTER_VOLUME 101 // Input. -#define SYS_POLL_KEYBOARD 88 -#define SYS_IS_KEY_DOWN 89 -#define SYS_READ_KEYBOARD_CHAR 90 -#define SYS_READ_KEY 91 -#define SYS_GET_KEY_MODIFIERS 92 -#define SYS_QUERY_GLOBAL_INPUT_STATE 93 -#define SYS_OPEN_INPUT_QUEUE 94 -#define SYS_READ_INPUT_EVENT 95 -#define SYS_PEEK_INPUT_EVENT 96 -#define SYS_PEEK_MOUSE 97 -#define SYS_READ_MOUSE 98 -#define SYS_GET_MOUSE_POS 99 +#define SYS_POLL_KEYBOARD 102 +#define SYS_IS_KEY_DOWN 103 +#define SYS_READ_KEYBOARD_CHAR 104 +#define SYS_READ_KEY 105 +#define SYS_GET_KEY_MODIFIERS 106 +#define SYS_QUERY_GLOBAL_INPUT_STATE 107 +#define SYS_OPEN_INPUT_QUEUE 108 +#define SYS_READ_INPUT_EVENT 109 +#define SYS_PEEK_INPUT_EVENT 110 +#define SYS_PEEK_MOUSE 111 +#define SYS_READ_MOUSE 112 +#define SYS_GET_MOUSE_POS 113 // Time, debug, and system information. -#define SYS_CREATE_TIMER 100 -#define SYS_SET_TIMER 101 -#define SYS_CANCEL_TIMER 102 -#define SYS_GET_TICK 103 -#define SYS_GET_TICK_FREQUENCY 104 -#define SYS_GET_SYSTEM_TIME 105 -#define SYS_PRINT_DEBUG 106 -#define SYS_QUERY_TRAP_FRAME 107 -#define SYS_GET_MEMORY_STATUS 108 -#define SYS_QUERY_KERNEL_STATS 109 +#define SYS_CREATE_TIMER 114 +#define SYS_SET_TIMER 115 +#define SYS_CANCEL_TIMER 116 +#define SYS_GET_TICK 117 +#define SYS_GET_TICK_FREQUENCY 118 +#define SYS_GET_SYSTEM_TIME 119 +#define SYS_PRINT_DEBUG 120 +#define SYS_QUERY_TRAP_FRAME 121 +#define SYS_GET_MEMORY_STATUS 122 +#define SYS_QUERY_KERNEL_STATS 123 // Native sockets and polling. -#define SYS_CREATE_SOCKET 110 -#define SYS_SOCKET_IO_ASYNC 111 -#define SYS_BIND_SOCKET 112 -#define SYS_LISTEN_SOCKET 113 -#define SYS_ACCEPT_SOCKET 114 -#define SYS_CONNECT_SOCKET 115 -#define SYS_SEND_SOCKET 116 -#define SYS_RECV_SOCKET 117 -#define SYS_SHUTDOWN_SOCKET 118 -#define SYS_SEND_TO_SOCKET 119 -#define SYS_RECV_FROM_SOCKET 120 -#define SYS_RESOLVE_HOST 121 -#define SYS_SET_SOCKET_OPTION 122 -#define SYS_GET_SOCKET_OPTION 123 -#define SYS_POLL 124 -#define SYS_UNPUBLISH_QUEUE 125 +#define SYS_CREATE_SOCKET 124 +#define SYS_SOCKET_IO_ASYNC 125 +#define SYS_BIND_SOCKET 126 +#define SYS_LISTEN_SOCKET 127 +#define SYS_ACCEPT_SOCKET 128 +#define SYS_CONNECT_SOCKET 129 +#define SYS_SEND_SOCKET 130 +#define SYS_RECV_SOCKET 131 +#define SYS_SHUTDOWN_SOCKET 132 +#define SYS_SEND_TO_SOCKET 133 +#define SYS_RECV_FROM_SOCKET 134 +#define SYS_RESOLVE_HOST 135 +#define SYS_SET_SOCKET_OPTION 136 +#define SYS_GET_SOCKET_OPTION 137 +#define SYS_POLL 138 +#define SYS_UNPUBLISH_QUEUE 139 -#define SYS_MAX 126 +#define SYS_MAX 140 diff --git a/code/kernel/io_ddk.c b/code/kernel/io_ddk.c index 595abc9..d769d78 100644 --- a/code/kernel/io_ddk.c +++ b/code/kernel/io_ddk.c @@ -5,8 +5,12 @@ #include #include #include +#include +#include +#include #include #include +#include static const HAL_API* gIoDdkHal; @@ -68,6 +72,7 @@ static KSTATUS IoDdkAllocateMsiVector(uint8_t* outIrq) { static const NTL_DDK_DEBUG gIoDdkDebug = { .Print = DbgPrint, + .GetFlags = KeGetDebugFlags, }; static const NTL_DDK_INTERRUPT gIoDdkInterrupt = { @@ -99,6 +104,8 @@ static NTL_DDK_TIMER gIoDdkTimer = { static NTL_DDK gIoDdk = { .Version = NTL_DDK_VERSION, .Debug = &gIoDdkDebug, + .Audio = (const NTL_DDK_AUDIO*)0, + .Dma = (const NTL_DDK_DMA*)0, .Interrupt = &gIoDdkInterrupt, .Memory = &gIoDdkMemory, .Platform = &gIoDdkPlatform, @@ -109,6 +116,8 @@ static NTL_DDK gIoDdk = { // Initializes the kernel-owned DDK table from platform services. void IoInitializeDdk(const HAL_API* hal) { gIoDdkHal = hal; + gIoDdk.Audio = KeAudioGetDdk(); + gIoDdk.Dma = IoGetDmaDdk(); gIoDdkTimer.QueryTicks = hal ? hal->Timer.QueryTicks : 0; gIoDdkTimer.Stall = hal ? hal->Timer.Stall : 0; } diff --git a/code/kernel/io_device.c b/code/kernel/io_device.c index 4ff8247..500b9b4 100644 --- a/code/kernel/io_device.c +++ b/code/kernel/io_device.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,7 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) { static void IoDeviceDestroy(OBJECT* object) { DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object; + IoDmaDetachDevice(dev); for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) { if (gPhysicalDevices[i] == dev) { @@ -316,12 +318,19 @@ static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) { // Releases device. __declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) { - if (!device) return; - if (device->DeletePending) + if (!device) return; + uintptr_t flags = KeAcquireSpinlock(&device->Lock); + if (device->DeletePending) { + KeReleaseSpinlock(&device->Lock, flags); + return; + } device->DeletePending = true; device->PnpState = DevicePnpDeleted; + KeReleaseSpinlock(&device->Lock, flags); + + IoDmaDetachDevice(device); IoUnlinkDeviceFromParent(device); IoRemovePhysicalDeviceRegistryEntry(device); IoDereferenceDeviceObject(device); @@ -441,167 +450,50 @@ static uint32_t gDiskChangeCallbackCount; static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS]; static uint32_t gDiskArrivalCallbackCount; -#define IO_DISK_READAHEAD_MAX_SECTORS 128u -#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u -#define IO_DISK_READAHEAD_MIN_SECTORS 8u #define IO_DISK_SECTOR_BYTES 512u -typedef struct IO_DISK_READAHEAD_STATE { +typedef struct IO_DISK_STATS_STATE { SPINLOCK Lock; - uint8_t* Buffer; - uint8_t Valid; - uint8_t Loading; - uint64_t Generation; - uint64_t BaseLba; - uint32_t Count; - uint64_t NextLba; - uint32_t SequentialScore; IO_DISK_STATS Stats; -} IO_DISK_READAHEAD_STATE; +} IO_DISK_STATS_STATE; -static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES]; +static IO_DISK_STATS_STATE gDiskStats[MAX_DISK_DEVICES]; -// Updates a maximum statistic while holding the disk read-ahead lock. -static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) { - if (field && *field < value) *field = value; -} - -// Prepares the per-disk read-ahead buffer. -static void IoDiskReadAheadPrepare(uint8_t driveIndex) { +// Initializes per-disk I/O statistics. +static void IoDiskStatsPrepare(uint8_t driveIndex) { if (driveIndex >= MAX_DISK_DEVICES) return; - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex]; state->Lock = (SPINLOCK)SPINLOCK_INIT; - state->Valid = 0; - state->Loading = 0; - state->Generation = 0; - state->BaseLba = 0; - state->Count = 0; - state->NextLba = 0; - state->SequentialScore = 0; memset(&state->Stats, 0, sizeof(state->Stats)); - - if (!state->Buffer) { - state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES, - MM_POOL_TAG('d', 'r', 'a', 'h')); - if (!state->Buffer) - state->Stats.ReadaheadAllocFailures++; - } } -// Releases the per-disk read-ahead buffer. -static void IoDiskReadAheadRelease(uint8_t driveIndex) { +// Records one disk read request. +static void IoDiskStatsNoteRead(uint8_t driveIndex, uint32_t count) { if (driveIndex >= MAX_DISK_DEVICES) return; - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - uint8_t* buffer = (uint8_t*)0; - for (;;) { - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - if (!state->Loading) { - buffer = state->Buffer; - state->Buffer = (uint8_t*)0; - state->Valid = 0; - state->Generation++; - state->Count = 0; - state->SequentialScore = 0; - KeReleaseSpinlock(&state->Lock, flags); - break; - } - KeReleaseSpinlock(&state->Lock, flags); - KeYield(); - } - - if (buffer) - MmFreePool(buffer); -} - -// Invalidates cached disk read-ahead data for one disk. -static void IoDiskReadAheadInvalidate(uint8_t driveIndex) { - if (driveIndex >= MAX_DISK_DEVICES) - return; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - state->Valid = 0; - state->Generation++; - state->Count = 0; - state->Stats.ReadaheadInvalidations++; - KeReleaseSpinlock(&state->Lock, flags); -} - -// Returns true when one range fully contains another range. -static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount, - uint64_t innerBase, uint32_t innerCount) { - uint64_t outerEnd = outerBase + outerCount; - uint64_t innerEnd = innerBase + innerCount; - if (outerEnd < outerBase || innerEnd < innerBase) - return false; - return innerBase >= outerBase && innerEnd <= outerEnd; -} - -// Attempts to satisfy a disk read from the per-disk read-ahead buffer. -static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) { - if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0) - return false; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - bool hit = state->Buffer && state->Valid && !state->Loading && - IoDiskRangeContains(state->BaseLba, state->Count, lba, count); - if (hit) { - size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES; - size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES; - memcpy(buffer, state->Buffer + offset, bytes); - state->Stats.ReadaheadHits++; - state->Stats.ReadaheadBytesServed += bytes; - } else { - state->Stats.ReadaheadMisses++; - } - KeReleaseSpinlock(&state->Lock, flags); - return hit; -} - -// Records request size and sequential access state for one disk read. -static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) { - if (driveIndex >= MAX_DISK_DEVICES || count == 0) - return 0; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex]; uintptr_t flags = KeAcquireSpinlock(&state->Lock); state->Stats.ReadRequests++; state->Stats.ReadSectors += count; - IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count); - - if (lba == state->NextLba) { - if (state->SequentialScore < 32) - state->SequentialScore++; - } else { - state->SequentialScore = 1; - } - state->NextLba = lba + count; - uint32_t score = state->SequentialScore; - if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE) - state->Stats.ReadaheadSequentialDetections++; + if (state->Stats.MaxReadSectors < count) + state->Stats.MaxReadSectors = count; KeReleaseSpinlock(&state->Lock, flags); - return score; } -// Records one disk write and invalidates stale read-ahead data. -static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) { +// Records one successful disk write request. +static void IoDiskStatsNoteWrite(uint8_t driveIndex, uint32_t count) { if (driveIndex >= MAX_DISK_DEVICES) return; - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex]; uintptr_t flags = KeAcquireSpinlock(&state->Lock); state->Stats.WriteRequests++; state->Stats.WriteSectors += count; - IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count); - state->Valid = 0; - state->Generation++; - state->Count = 0; - state->Stats.ReadaheadInvalidations++; + if (state->Stats.MaxWriteSectors < count) + state->Stats.MaxWriteSectors = count; KeReleaseSpinlock(&state->Lock, flags); } @@ -653,7 +545,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, if (pnpStatus == NTLSTATUS_NOT_SUPPORTED) device->PnpState = DevicePnpStarted; - IoDiskReadAheadPrepare(driveIndex); + IoDiskStatsPrepare(driveIndex); IoReferenceDeviceObject(device); gDiskDevices[driveIndex] = device; @@ -668,8 +560,6 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) { DEVICE_OBJECT* dev = gDiskDevices[driveIndex]; if (!dev) return NTLSTATUS_NOT_FOUND; - IoDiskReadAheadInvalidate(driveIndex); - dev->PnpState = DevicePnpSurpriseRemoved; IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved); (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL); @@ -698,7 +588,6 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) { (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE); gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0; - IoDiskReadAheadRelease(driveIndex); IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved); IoDereferenceDeviceObject(dev); return NTLSTATUS_SUCCESS; @@ -846,83 +735,6 @@ static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba, return IoSubmitIrpSync(dev, &irp); } -// Chooses a conservative read-ahead span from the current sequential score. -static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) { - uint32_t sectors = currentCount; - if (sectors < IO_DISK_READAHEAD_MIN_SECTORS) - sectors = IO_DISK_READAHEAD_MIN_SECTORS; - if (score >= 8 && sectors < 128u) - sectors = 128u; - else if (score >= 4 && sectors < 64u) - sectors = 64u; - else if (score >= 2 && sectors < 32u) - sectors = 32u; - if (sectors > IO_DISK_READAHEAD_MAX_SECTORS) - sectors = IO_DISK_READAHEAD_MAX_SECTORS; - return sectors; -} - -// Loads the next sequential disk range into the read-ahead buffer. -static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev, - uint64_t nextLba, uint32_t currentCount, uint32_t score) { - if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE) - return; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - if (!state->Buffer) { - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - state->Stats.ReadaheadSuppressed++; - KeReleaseSpinlock(&state->Lock, flags); - return; - } - - uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score); - uint64_t diskSectors = dev->u.Disk.SectorCount; - if (nextLba >= diskSectors) - return; - if ((uint64_t)sectors > diskSectors - nextLba) - sectors = (uint32_t)(diskSectors - nextLba); - if (sectors == 0) - return; - - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - bool covered = state->Valid && !state->Loading && - IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors); - if (covered) { - KeReleaseSpinlock(&state->Lock, flags); - return; - } - if (state->Loading) { - state->Stats.ReadaheadSuppressed++; - KeReleaseSpinlock(&state->Lock, flags); - return; - } - uint64_t generation = state->Generation; - state->Loading = 1; - state->Valid = 0; - state->Count = 0; - KeReleaseSpinlock(&state->Lock, flags); - - KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer); - - flags = KeAcquireSpinlock(&state->Lock); - state->Loading = 0; - if (KSUCCESS(st) && generation == state->Generation) { - state->BaseLba = nextLba; - state->Count = sectors; - state->Valid = 1; - state->Stats.ReadaheadStores++; - state->Stats.ReadaheadSectorsLoaded += sectors; - IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors); - } else { - state->Valid = 0; - state->Count = 0; - if (!KSUCCESS(st)) - state->Stats.ReadaheadReadFailures++; - } - KeReleaseSpinlock(&state->Lock, flags); -} - __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, uint64_t lba, uint32_t count, @@ -938,17 +750,8 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, return rangeStatus; } - if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) { - uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count); - IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score); - IoDereferenceDeviceObject(dev); - return NTLSTATUS_SUCCESS; - } - KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer); - uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count); - if (KSUCCESS(st)) - IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score); + IoDiskStatsNoteRead(driveIndex, count); IoDereferenceDeviceObject(dev); return st; @@ -971,19 +774,39 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer); if (KSUCCESS(st)) - IoDiskReadAheadNoteWrite(driveIndex, count); + IoDiskStatsNoteWrite(driveIndex, count); IoDereferenceDeviceObject(dev); return st; } +// Flushes volatile write caches for one disk device. +__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex) { + DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex); + if (!dev) return NTLSTATUS_NOT_FOUND; + + IRP irp; + IoInitializeIrp(&irp, 1); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(&irp); + if (!ios) { + IoDereferenceDeviceObject(dev); + return NTLSTATUS_FAILURE; + } + ios->MajorFunction = IRP_MJ_FLUSH_BUFFERS; + ios->MinorFunction = 0; + + KSTATUS st = IoSubmitIrpSync(dev, &irp); + IoDereferenceDeviceObject(dev); + return st; +} + // Queries aggregate disk I/O statistics. __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) { if (!out) return; memset(out, 0, sizeof(*out)); for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) { - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i]; + IO_DISK_STATS_STATE* state = &gDiskStats[i]; uintptr_t flags = KeAcquireSpinlock(&state->Lock); if (gDiskDevices[i]) out->RegisteredDisks++; @@ -995,18 +818,6 @@ __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) { out->MaxReadSectors = state->Stats.MaxReadSectors; if (out->MaxWriteSectors < state->Stats.MaxWriteSectors) out->MaxWriteSectors = state->Stats.MaxWriteSectors; - out->ReadaheadHits += state->Stats.ReadaheadHits; - out->ReadaheadMisses += state->Stats.ReadaheadMisses; - out->ReadaheadStores += state->Stats.ReadaheadStores; - out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations; - out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed; - out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded; - out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections; - out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed; - out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures; - out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures; - if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors) - out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors; KeReleaseSpinlock(&state->Lock, flags); } } diff --git a/code/kernel/io_dma.c b/code/kernel/io_dma.c new file mode 100644 index 0000000..32f6065 --- /dev/null +++ b/code/kernel/io_dma.c @@ -0,0 +1,1056 @@ +// Kernel DMA core. This file owns device adapters, common buffers, pinned memory, streaming mappings, and bounce copies. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IO_DMA_ADAPTER_TAG MM_POOL_TAG('d', 'm', 'a', 'a') +#define IO_DMA_BUFFER_TAG MM_POOL_TAG('d', 'm', 'a', 'b') +#define IO_DMA_PIN_TAG MM_POOL_TAG('d', 'm', 'a', 'p') +#define IO_DMA_MAP_TAG MM_POOL_TAG('d', 'm', 'a', 'm') + +struct IO_DMA_ADAPTER { + volatile long RefCount; + volatile long Removed; + SPINLOCK Lock; + DEVICE_OBJECT* Device; + IO_DMA_REQUIREMENTS Requirements; +}; + +struct IO_DMA_BUFFER { + IO_DMA_ADAPTER* Adapter; + void* CpuAddress; + PHYSICAL_ADDRESS PhysicalAddress; + uint64_t DeviceAddress; + size_t Size; + size_t AllocationSize; + size_t PageCount; +}; + +typedef enum MM_PINNED_MEMORY_ORIGIN { + MmPinnedKernel = 1, + MmPinnedUser +} MM_PINNED_MEMORY_ORIGIN; + +struct MM_PINNED_MEMORY { + volatile long RefCount; + MM_PINNED_MEMORY_ORIGIN Origin; + IO_DMA_DIRECTION Direction; + void* Address; + size_t Size; + size_t PageCount; + size_t FirstPageOffset; + MM_LOCKED_USER_BUFFER* UserBuffer; + PHYSICAL_ADDRESS Frames[1]; +}; + +struct IO_DMA_MAPPING { + IO_DMA_ADAPTER* Adapter; + MM_PINNED_MEMORY* Memory; + IO_DMA_BUFFER* BounceBuffer; + size_t MemoryOffset; + size_t Length; + IO_DMA_DIRECTION Direction; + uint32_t SegmentCount; + int DeviceOwnsMapping; + IO_DMA_SEGMENT Segments[1]; +}; + +typedef struct IO_DMA_INTERNAL_STATS { + volatile long long AdaptersCreated; + volatile long long AdaptersDestroyed; + volatile long long AdaptersInUse; + volatile long long CommonBuffersAllocated; + volatile long long CommonBuffersFreed; + volatile long long CommonBuffersInUse; + volatile long long CommonBufferBytesInUse; + volatile long long PinnedKernelRanges; + volatile long long PinnedUserRanges; + volatile long long PinnedRangesReleased; + volatile long long PinnedRangesInUse; + volatile long long PinnedPagesInUse; + volatile long long StreamingMappings; + volatile long long StreamingMappingsUnmapped; + volatile long long StreamingMappingsInUse; + volatile long long DirectMappings; + volatile long long BounceMappings; + volatile long long BounceBytesCopiedToDevice; + volatile long long BounceBytesCopiedToCpu; + volatile long long FailedAllocations; + volatile long long FailedMappings; +} IO_DMA_INTERNAL_STATS; + +static IO_DMA_INTERNAL_STATS gIoDmaStats; + +// Returns whether a size is a nonzero power of two. +static int IoDmaIsPowerOfTwo(size_t value) { + return value && ((value & (value - 1)) == 0); +} + +// Normalizes and validates one adapter requirement set. +static KSTATUS IoDmaNormalizeRequirements(const IO_DMA_REQUIREMENTS* source, + IO_DMA_REQUIREMENTS* destination) { + if (!destination) + return NTLSTATUS_INVALID_PARAMETER; + + memset(destination, 0, sizeof(*destination)); + if (source) + *destination = *source; + + if (!destination->MaximumDeviceAddress) + destination->MaximumDeviceAddress = (uint64_t)-1; + if (!destination->Alignment) + destination->Alignment = 1; + if (!destination->MaximumSegmentLength) + destination->MaximumSegmentLength = (size_t)-1; + if (!destination->MaximumSegments) + destination->MaximumSegments = UINT32_MAX; + + if (!IoDmaIsPowerOfTwo(destination->Alignment)) + return NTLSTATUS_INVALID_PARAMETER; + if (destination->Boundary && !IoDmaIsPowerOfTwo(destination->Boundary)) + return NTLSTATUS_INVALID_PARAMETER; + return NTLSTATUS_SUCCESS; +} + +// Returns whether two normalized requirement sets are identical. +static int IoDmaRequirementsEqual(const IO_DMA_REQUIREMENTS* left, + const IO_DMA_REQUIREMENTS* right) { + return left && right && + left->MaximumDeviceAddress == right->MaximumDeviceAddress && + left->Alignment == right->Alignment && + left->Boundary == right->Boundary && + left->MaximumSegmentLength == right->MaximumSegmentLength && + left->MaximumSegments == right->MaximumSegments; +} + +// Returns whether device removal has closed the adapter to new ownership. +static int IoDmaAdapterRemoved(const IO_DMA_ADAPTER* adapter) { + if (!adapter) + return 1; + return _InterlockedCompareExchange((volatile long*)&adapter->Removed, 0, 0) != 0; +} + +// Adds one reference to an adapter that is already stabilized by its owner. +static void IoDmaReferenceAdapter(IO_DMA_ADAPTER* adapter) { + if (adapter) + _InterlockedIncrement(&adapter->RefCount); +} + +// Adds a reference only while the adapter still accepts new DMA ownership. +static int IoDmaTryReferenceAdapter(IO_DMA_ADAPTER* adapter) { + if (!adapter) + return 0; + uintptr_t flags = KeAcquireSpinlock(&adapter->Lock); + if (adapter->Removed) { + KeReleaseSpinlock(&adapter->Lock, flags); + return 0; + } + _InterlockedIncrement(&adapter->RefCount); + KeReleaseSpinlock(&adapter->Lock, flags); + return 1; +} + +// Releases one adapter reference and destroys the detached adapter at zero. +static void IoDmaPutAdapterImpl(IO_DMA_ADAPTER* adapter) { + if (!adapter) + return; + long remaining = _InterlockedDecrement(&adapter->RefCount); + if (remaining > 0) + return; + if (remaining < 0) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: adapter reference underflow adapter=0x%016llX\n", + (unsigned long long)(uintptr_t)adapter); + if (!IoDmaAdapterRemoved(adapter)) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: attached adapter reached zero references adapter=0x%016llX\n", + (unsigned long long)(uintptr_t)adapter); + _InterlockedIncrement64(&gIoDmaStats.AdaptersDestroyed); + _InterlockedDecrement64(&gIoDmaStats.AdaptersInUse); + MmFreePool(adapter); +} + +// Returns the one canonical DMA adapter owned by a device. +static KSTATUS IoDmaGetAdapterImpl(DEVICE_OBJECT* device, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_ADAPTER** outAdapter) { + if (!device || !outAdapter) + return NTLSTATUS_INVALID_PARAMETER; + *outAdapter = (IO_DMA_ADAPTER*)0; + + IO_DMA_REQUIREMENTS normalized; + KSTATUS status = IoDmaNormalizeRequirements(requirements, &normalized); + if (!KSUCCESS(status)) + return status; + + uintptr_t flags = KeAcquireSpinlock(&device->Lock); + if (device->DeletePending || device->PnpState == DevicePnpDeleted) { + KeReleaseSpinlock(&device->Lock, flags); + return NTLSTATUS_OBJECT_DEAD; + } + + IO_DMA_ADAPTER* adapter = device->DmaAdapter; + if (adapter) { + if (!IoDmaRequirementsEqual(&adapter->Requirements, &normalized)) { + KeReleaseSpinlock(&device->Lock, flags); + return NTLSTATUS_ALREADY_EXISTS; + } + IoDmaReferenceAdapter(adapter); + KeReleaseSpinlock(&device->Lock, flags); + *outAdapter = adapter; + return NTLSTATUS_SUCCESS; + } + KeReleaseSpinlock(&device->Lock, flags); + + IO_DMA_ADAPTER* candidate = (IO_DMA_ADAPTER*)MmAllocatePoolTag(sizeof(*candidate), + IO_DMA_ADAPTER_TAG); + if (!candidate) { + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + memset(candidate, 0, sizeof(*candidate)); + candidate->RefCount = 2; + candidate->Device = device; + candidate->Requirements = normalized; + + flags = KeAcquireSpinlock(&device->Lock); + if (device->DeletePending || device->PnpState == DevicePnpDeleted) { + KeReleaseSpinlock(&device->Lock, flags); + MmFreePool(candidate); + return NTLSTATUS_OBJECT_DEAD; + } + + adapter = device->DmaAdapter; + if (adapter) { + if (!IoDmaRequirementsEqual(&adapter->Requirements, &normalized)) { + KeReleaseSpinlock(&device->Lock, flags); + MmFreePool(candidate); + return NTLSTATUS_ALREADY_EXISTS; + } + IoDmaReferenceAdapter(adapter); + KeReleaseSpinlock(&device->Lock, flags); + MmFreePool(candidate); + *outAdapter = adapter; + return NTLSTATUS_SUCCESS; + } + + device->DmaAdapter = candidate; + KeReleaseSpinlock(&device->Lock, flags); + + _InterlockedIncrement64(&gIoDmaStats.AdaptersCreated); + _InterlockedIncrement64(&gIoDmaStats.AdaptersInUse); + *outAdapter = candidate; + return NTLSTATUS_SUCCESS; +} + +// Detaches a device-owned adapter so no new mappings can be created. +void IoDmaDetachDevice(DEVICE_OBJECT* device) { + if (!device) + return; + + uintptr_t flags = KeAcquireSpinlock(&device->Lock); + IO_DMA_ADAPTER* adapter = device->DmaAdapter; + device->DmaAdapter = (IO_DMA_ADAPTER*)0; + if (adapter) { + uintptr_t adapterFlags = KeAcquireSpinlock(&adapter->Lock); + adapter->Removed = 1; + adapter->Device = (DEVICE_OBJECT*)0; + KeReleaseSpinlock(&adapter->Lock, adapterFlags); + } + KeReleaseSpinlock(&device->Lock, flags); + + if (adapter) + IoDmaPutAdapterImpl(adapter); +} + +// Converts an inclusive device-address limit to the allocator's exclusive limit. +static PHYSICAL_ADDRESS IoDmaMaximumExclusive(const IO_DMA_ADAPTER* adapter) { + if (!adapter || adapter->Requirements.MaximumDeviceAddress == (uint64_t)-1) + return (PHYSICAL_ADDRESS)-1; + return adapter->Requirements.MaximumDeviceAddress + 1; +} + +// Allocates one contiguous backing run while applying address and alignment requirements. +static KSTATUS IoDmaAllocateCommonBufferInternal(IO_DMA_ADAPTER* adapter, + size_t size, + size_t alignment, + int enforceAllocationBoundary, + IO_DMA_BUFFER** outBuffer) { + if (!adapter || !outBuffer || !size) + return NTLSTATUS_INVALID_PARAMETER; + *outBuffer = (IO_DMA_BUFFER*)0; + if (IoDmaAdapterRemoved(adapter)) + return NTLSTATUS_OBJECT_DEAD; + + if (!alignment) + alignment = 1; + if (!IoDmaIsPowerOfTwo(alignment)) + return NTLSTATUS_INVALID_PARAMETER; + if (alignment < adapter->Requirements.Alignment) + alignment = adapter->Requirements.Alignment; + if (alignment < PAGE_SIZE) + alignment = PAGE_SIZE; + + if (size > ((size_t)-1) - (PAGE_SIZE - 1)) + return NTLSTATUS_NO_MEMORY; + size_t allocationSize = (size + PAGE_SIZE - 1) & PAGE_MASK; + size_t pageCount = allocationSize / PAGE_SIZE; + + size_t allocationBoundary = 0; + if (enforceAllocationBoundary && adapter->Requirements.Boundary) { + if (size > adapter->Requirements.Boundary) + return NTLSTATUS_NOT_SUPPORTED; + if (adapter->Requirements.Boundary >= PAGE_SIZE) + allocationBoundary = adapter->Requirements.Boundary; + } + + PHYSICAL_ADDRESS physical = MmPfnAllocateContiguousInRange( + pageCount, + PAGE_SIZE, + IoDmaMaximumExclusive(adapter), + alignment, + allocationBoundary, + MmPfnUseDma); + if (!physical) { + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)MmAllocatePoolTag(sizeof(*buffer), IO_DMA_BUFFER_TAG); + if (!buffer) { + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + memset(buffer, 0, sizeof(*buffer)); + buffer->Adapter = adapter; + buffer->CpuAddress = MmPfnToVirtual(physical); + buffer->PhysicalAddress = physical; + buffer->DeviceAddress = physical; + buffer->Size = size; + buffer->AllocationSize = allocationSize; + buffer->PageCount = pageCount; + if (!buffer->CpuAddress) { + MmFreePool(buffer); + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_FAILURE; + } + + memset(buffer->CpuAddress, 0, allocationSize); + if (!IoDmaTryReferenceAdapter(adapter)) { + MmFreePool(buffer); + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + return NTLSTATUS_OBJECT_DEAD; + } + _InterlockedIncrement64(&gIoDmaStats.CommonBuffersAllocated); + _InterlockedIncrement64(&gIoDmaStats.CommonBuffersInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, (long long)allocationSize); + *outBuffer = buffer; + return NTLSTATUS_SUCCESS; +} + +// Allocates one coherent common buffer represented by one device segment. +static KSTATUS IoDmaAllocateCommonBufferImpl(IO_DMA_ADAPTER* adapter, + size_t size, + size_t alignment, + IO_DMA_BUFFER** outBuffer) { + return IoDmaAllocateCommonBufferInternal(adapter, size, alignment, 1, outBuffer); +} + +// Frees one common buffer and its owned physical pages. +static void IoDmaFreeCommonBufferImpl(IO_DMA_BUFFER* buffer) { + if (!buffer) + return; + + IO_DMA_ADAPTER* adapter = buffer->Adapter; + PHYSICAL_ADDRESS physical = buffer->PhysicalAddress; + size_t pageCount = buffer->PageCount; + size_t allocationSize = buffer->AllocationSize; + memset(buffer, 0, sizeof(*buffer)); + MmFreePool(buffer); + + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + + _InterlockedIncrement64(&gIoDmaStats.CommonBuffersFreed); + _InterlockedDecrement64(&gIoDmaStats.CommonBuffersInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, -(long long)allocationSize); + IoDmaPutAdapterImpl(adapter); +} + +// Returns the CPU address of a common buffer. +static void* IoDmaGetCommonBufferAddressImpl(const IO_DMA_BUFFER* buffer) { + return buffer ? buffer->CpuAddress : (void*)0; +} + +// Returns the first device address of a common buffer. +static uint64_t IoDmaGetCommonBufferDeviceAddressImpl(const IO_DMA_BUFFER* buffer) { + return buffer ? buffer->DeviceAddress : 0; +} + +// Returns the caller-requested size of a common buffer. +static size_t IoDmaGetCommonBufferSizeImpl(const IO_DMA_BUFFER* buffer) { + return buffer ? buffer->Size : 0; +} + +// Returns whether a pinned access mode can satisfy a mapping direction. +static int IoDmaDirectionAllows(IO_DMA_DIRECTION pinned, IO_DMA_DIRECTION mapped) { + if (pinned == IoDmaBidirectional) + return mapped == IoDmaToDevice || mapped == IoDmaFromDevice || mapped == IoDmaBidirectional; + return pinned == mapped; +} + +// Allocates a pinned-memory descriptor with room for every backing frame. +static MM_PINNED_MEMORY* IoDmaAllocatePinnedMemory(size_t pageCount) { + if (!pageCount || pageCount > (((size_t)-1) - sizeof(MM_PINNED_MEMORY)) / sizeof(PHYSICAL_ADDRESS)) + return (MM_PINNED_MEMORY*)0; + size_t bytes = sizeof(MM_PINNED_MEMORY) + (pageCount - 1) * sizeof(PHYSICAL_ADDRESS); + MM_PINNED_MEMORY* memory = (MM_PINNED_MEMORY*)MmAllocatePoolTag(bytes, IO_DMA_PIN_TAG); + if (memory) + memset(memory, 0, bytes); + return memory; +} + +// Releases a pinned-memory descriptor after its last mapping leaves. +static void IoDmaReleasePinnedMemory(MM_PINNED_MEMORY* memory) { + if (!memory) + return; + long remaining = _InterlockedDecrement(&memory->RefCount); + if (remaining > 0) + return; + if (remaining < 0) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: pinned-memory reference underflow memory=0x%016llX\n", + (unsigned long long)(uintptr_t)memory); + + if (memory->Origin == MmPinnedUser) { + MmUnlockUserBuffer(memory->UserBuffer); + } else { + for (size_t i = 0; i < memory->PageCount; ++i) + (void)MmPfnUnpin(memory->Frames[i]); + } + _InterlockedIncrement64(&gIoDmaStats.PinnedRangesReleased); + _InterlockedDecrement64(&gIoDmaStats.PinnedRangesInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, -(long long)memory->PageCount); + MmFreePool(memory); +} + +// Adds one mapping reference to pinned memory. +static void IoDmaReferencePinnedMemory(MM_PINNED_MEMORY* memory) { + if (memory) + _InterlockedIncrement(&memory->RefCount); +} + +// Pins a kernel virtual range for later DMA mapping. +static KSTATUS IoDmaPinKernelMemoryImpl(void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory) { + if (!outMemory) + return NTLSTATUS_INVALID_PARAMETER; + *outMemory = (MM_PINNED_MEMORY*)0; + if (!address || !size) + return NTLSTATUS_INVALID_PARAMETER; + if (direction < IoDmaToDevice || direction > IoDmaBidirectional) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t start = (uintptr_t)address; + if (start < KERNEL_BASE || start + size < start) + return NTLSTATUS_INVALID_PARAMETER; + uintptr_t firstPage = start & PAGE_MASK; + uintptr_t lastByte = start + size - 1; + size_t pageCount = ((lastByte & PAGE_MASK) - firstPage) / PAGE_SIZE + 1; + + MM_PINNED_MEMORY* memory = IoDmaAllocatePinnedMemory(pageCount); + if (!memory) { + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + memory->RefCount = 1; + memory->Origin = MmPinnedKernel; + memory->Direction = direction; + memory->Address = address; + memory->Size = size; + memory->PageCount = pageCount; + memory->FirstPageOffset = start & (PAGE_SIZE - 1); + + size_t pinned = 0; + KSTATUS status = NTLSTATUS_SUCCESS; + for (size_t i = 0; i < pageCount; ++i) { + PHYSICAL_ADDRESS physical = MmPteTranslate(firstPage + i * PAGE_SIZE); + PHYSICAL_ADDRESS frame = physical & PAGE_MASK; + if (!physical || !MmPfnGet(frame)) { + status = NTLSTATUS_NOT_FOUND; + break; + } + status = MmPfnPin(frame); + if (!KSUCCESS(status)) + break; + memory->Frames[i] = frame; + pinned++; + } + + if (!KSUCCESS(status)) { + for (size_t i = 0; i < pinned; ++i) + (void)MmPfnUnpin(memory->Frames[i]); + MmFreePool(memory); + return status; + } + + _InterlockedIncrement64(&gIoDmaStats.PinnedKernelRanges); + _InterlockedIncrement64(&gIoDmaStats.PinnedRangesInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (long long)pageCount); + *outMemory = memory; + return NTLSTATUS_SUCCESS; +} + +// Pins a current-process user range through the VAD locking contract. +static KSTATUS IoDmaPinUserMemoryImpl(struct PROCESS* process, + void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory) { + if (!outMemory) + return NTLSTATUS_INVALID_PARAMETER; + *outMemory = (MM_PINNED_MEMORY*)0; + if (!address || !size) + return NTLSTATUS_INVALID_PARAMETER; + if (direction < IoDmaToDevice || direction > IoDmaBidirectional) + return NTLSTATUS_INVALID_PARAMETER; + + if (!process) + return NTLSTATUS_ACCESS_DENIED; + + MM_LOCKED_USER_BUFFER* locked = (MM_LOCKED_USER_BUFFER*)0; + int writeAccess = direction == IoDmaFromDevice || direction == IoDmaBidirectional; + KSTATUS status = MmLockUserBuffer(process, address, size, writeAccess, &locked); + if (!KSUCCESS(status)) + return status; + + size_t pageCount = MmLockedUserBufferPageCount(locked); + MM_PINNED_MEMORY* memory = IoDmaAllocatePinnedMemory(pageCount); + if (!memory) { + MmUnlockUserBuffer(locked); + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + memory->RefCount = 1; + memory->Origin = MmPinnedUser; + memory->Direction = direction; + memory->Address = address; + memory->Size = size; + memory->PageCount = pageCount; + memory->FirstPageOffset = (uintptr_t)address & (PAGE_SIZE - 1); + memory->UserBuffer = locked; + for (size_t i = 0; i < pageCount; ++i) + memory->Frames[i] = MmLockedUserBufferPageFrame(locked, i); + + _InterlockedIncrement64(&gIoDmaStats.PinnedUserRanges); + _InterlockedIncrement64(&gIoDmaStats.PinnedRangesInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (long long)pageCount); + *outMemory = memory; + return NTLSTATUS_SUCCESS; +} + +// Releases the caller's pinned-memory reference. +static void IoDmaUnpinMemoryImpl(MM_PINNED_MEMORY* memory) { + IoDmaReleasePinnedMemory(memory); +} + +// Returns the physical address and bytes remaining in one pinned page span. +static KSTATUS IoDmaGetPinnedSpan(const MM_PINNED_MEMORY* memory, + size_t offset, + PHYSICAL_ADDRESS* address, + size_t* length) { + if (!memory || !address || !length || offset >= memory->Size) + return NTLSTATUS_INVALID_PARAMETER; + + size_t absolute = memory->FirstPageOffset + offset; + size_t pageIndex = absolute / PAGE_SIZE; + size_t pageOffset = absolute & (PAGE_SIZE - 1); + if (pageIndex >= memory->PageCount || !memory->Frames[pageIndex]) + return NTLSTATUS_NOT_FOUND; + + *address = memory->Frames[pageIndex] + pageOffset; + *length = PAGE_SIZE - pageOffset; + size_t remaining = memory->Size - offset; + if (*length > remaining) + *length = remaining; + return NTLSTATUS_SUCCESS; +} + +// Copies a pinned range into one linear kernel buffer. +static KSTATUS IoDmaCopyFromPinned(const MM_PINNED_MEMORY* memory, + size_t offset, + void* destination, + size_t length) { + if (!memory || (!destination && length) || offset > memory->Size || length > memory->Size - offset) + return NTLSTATUS_INVALID_PARAMETER; + + uint8_t* output = (uint8_t*)destination; + size_t copied = 0; + while (copied < length) { + size_t absolute = memory->FirstPageOffset + offset + copied; + size_t pageIndex = absolute / PAGE_SIZE; + size_t pageOffset = absolute & (PAGE_SIZE - 1); + size_t chunk = PAGE_SIZE - pageOffset; + if (chunk > length - copied) + chunk = length - copied; + void* page = MmPfnToVirtual(memory->Frames[pageIndex]); + if (!page) + return NTLSTATUS_FAILURE; + memcpy(output + copied, (uint8_t*)page + pageOffset, chunk); + copied += chunk; + } + return NTLSTATUS_SUCCESS; +} + +// Copies one linear kernel buffer back into a pinned range. +static KSTATUS IoDmaCopyToPinned(MM_PINNED_MEMORY* memory, + size_t offset, + const void* source, + size_t length) { + if (!memory || (!source && length) || offset > memory->Size || length > memory->Size - offset) + return NTLSTATUS_INVALID_PARAMETER; + + const uint8_t* input = (const uint8_t*)source; + size_t copied = 0; + while (copied < length) { + size_t absolute = memory->FirstPageOffset + offset + copied; + size_t pageIndex = absolute / PAGE_SIZE; + size_t pageOffset = absolute & (PAGE_SIZE - 1); + size_t chunk = PAGE_SIZE - pageOffset; + if (chunk > length - copied) + chunk = length - copied; + void* page = MmPfnToVirtual(memory->Frames[pageIndex]); + if (!page) + return NTLSTATUS_FAILURE; + memcpy((uint8_t*)page + pageOffset, input + copied, chunk); + copied += chunk; + } + return NTLSTATUS_SUCCESS; +} + +// Returns the maximum bytes one segment may cover from a given address. +static size_t IoDmaSegmentCapacity(const IO_DMA_REQUIREMENTS* requirements, + uint64_t address, + size_t currentLength) { + size_t capacity = requirements->MaximumSegmentLength - currentLength; + if (requirements->Boundary) { + size_t boundaryRemaining = requirements->Boundary - + (size_t)(address & (requirements->Boundary - 1)); + if (boundaryRemaining > currentLength) + boundaryRemaining -= currentLength; + else + boundaryRemaining = 0; + if (capacity > boundaryRemaining) + capacity = boundaryRemaining; + } + return capacity; +} + +// Adds physical spans to a constrained scatter/gather list. +static KSTATUS IoDmaBuildSegments(const MM_PINNED_MEMORY* memory, + size_t offset, + size_t length, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_SEGMENT* segments, + uint32_t* segmentCount) { + if (!memory || !requirements || !segmentCount || !length) + return NTLSTATUS_INVALID_PARAMETER; + + uint32_t count = 0; + uint64_t currentAddress = 0; + size_t currentLength = 0; + size_t consumed = 0; + + while (consumed < length) { + PHYSICAL_ADDRESS address = 0; + size_t spanLength = 0; + KSTATUS status = IoDmaGetPinnedSpan(memory, offset + consumed, &address, &spanLength); + if (!KSUCCESS(status)) + return status; + if (spanLength > length - consumed) + spanLength = length - consumed; + + if (address > requirements->MaximumDeviceAddress || + spanLength - 1 > requirements->MaximumDeviceAddress - address) + return NTLSTATUS_NOT_SUPPORTED; + + size_t spanConsumed = 0; + while (spanConsumed < spanLength) { + uint64_t pieceAddress = address + spanConsumed; + if (currentLength && currentAddress + currentLength == pieceAddress) { + size_t capacity = IoDmaSegmentCapacity(requirements, currentAddress, currentLength); + if (capacity) { + size_t take = spanLength - spanConsumed; + if (take > capacity) + take = capacity; + currentLength += take; + if (segments) + segments[count - 1].Length = currentLength; + spanConsumed += take; + continue; + } + } + + if ((pieceAddress & (requirements->Alignment - 1)) != 0) + return NTLSTATUS_NOT_SUPPORTED; + if (count == requirements->MaximumSegments) + return NTLSTATUS_NOT_SUPPORTED; + + size_t capacity = requirements->MaximumSegmentLength; + if (requirements->Boundary) { + size_t boundaryRemaining = requirements->Boundary - + (size_t)(pieceAddress & (requirements->Boundary - 1)); + if (capacity > boundaryRemaining) + capacity = boundaryRemaining; + } + if (!capacity) + return NTLSTATUS_NOT_SUPPORTED; + + size_t take = spanLength - spanConsumed; + if (take > capacity) + take = capacity; + currentAddress = pieceAddress; + currentLength = take; + if (segments) { + segments[count].DeviceAddress = currentAddress; + segments[count].Length = currentLength; + } + count++; + spanConsumed += take; + } + consumed += spanLength; + } + + *segmentCount = count; + return NTLSTATUS_SUCCESS; +} + +// Builds a constrained scatter/gather list for one contiguous bounce allocation. +static KSTATUS IoDmaBuildContiguousSegments(uint64_t address, + size_t length, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_SEGMENT* segments, + uint32_t* segmentCount) { + if (!requirements || !segmentCount || !address || !length) + return NTLSTATUS_INVALID_PARAMETER; + if (address > requirements->MaximumDeviceAddress || + length - 1 > requirements->MaximumDeviceAddress - address) + return NTLSTATUS_NOT_SUPPORTED; + + uint32_t count = 0; + size_t consumed = 0; + while (consumed < length) { + uint64_t pieceAddress = address + consumed; + if ((pieceAddress & (requirements->Alignment - 1)) != 0) + return NTLSTATUS_NOT_SUPPORTED; + if (count == requirements->MaximumSegments) + return NTLSTATUS_NOT_SUPPORTED; + + size_t capacity = requirements->MaximumSegmentLength; + if (requirements->Boundary) { + size_t boundaryRemaining = requirements->Boundary - + (size_t)(pieceAddress & (requirements->Boundary - 1)); + if (capacity > boundaryRemaining) + capacity = boundaryRemaining; + } + if (!capacity) + return NTLSTATUS_NOT_SUPPORTED; + + size_t take = length - consumed; + if (take > capacity) + take = capacity; + if (segments) { + segments[count].DeviceAddress = pieceAddress; + segments[count].Length = take; + } + count++; + consumed += take; + } + + *segmentCount = count; + return NTLSTATUS_SUCCESS; +} + +// Allocates one mapping object with a fixed segment array. +static IO_DMA_MAPPING* IoDmaAllocateMapping(uint32_t segmentCount) { + if (!segmentCount) + return (IO_DMA_MAPPING*)0; + size_t extraBytes = ((size_t)segmentCount - 1) * sizeof(IO_DMA_SEGMENT); + if (extraBytes / sizeof(IO_DMA_SEGMENT) != (size_t)segmentCount - 1) + return (IO_DMA_MAPPING*)0; + size_t bytes = sizeof(IO_DMA_MAPPING) + extraBytes; + if (bytes < extraBytes) + return (IO_DMA_MAPPING*)0; + IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)MmAllocatePoolTag(bytes, IO_DMA_MAP_TAG); + if (mapping) + memset(mapping, 0, bytes); + return mapping; +} + +// Maps pinned memory directly or through an automatic contiguous bounce buffer. +static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter, + MM_PINNED_MEMORY* memory, + size_t offset, + size_t length, + IO_DMA_DIRECTION direction, + uint32_t flags, + IO_DMA_MAPPING** outMapping) { + if (!outMapping) + return NTLSTATUS_INVALID_PARAMETER; + *outMapping = (IO_DMA_MAPPING*)0; + if (!adapter || !memory || !length || offset > memory->Size || length > memory->Size - offset) + return NTLSTATUS_INVALID_PARAMETER; + if (IoDmaAdapterRemoved(adapter)) + return NTLSTATUS_OBJECT_DEAD; + if (direction < IoDmaToDevice || direction > IoDmaBidirectional || + !IoDmaDirectionAllows(memory->Direction, direction)) + return NTLSTATUS_ACCESS_DENIED; + if (flags & ~IO_DMA_MAP_NO_BOUNCE) + return NTLSTATUS_INVALID_PARAMETER; + + uint32_t segmentCount = 0; + KSTATUS directStatus = IoDmaBuildSegments(memory, + offset, + length, + &adapter->Requirements, + (IO_DMA_SEGMENT*)0, + &segmentCount); + IO_DMA_BUFFER* bounce = (IO_DMA_BUFFER*)0; + KSTATUS status = directStatus; + + if (!KSUCCESS(status)) { + if (status != NTLSTATUS_NOT_SUPPORTED || (flags & IO_DMA_MAP_NO_BOUNCE)) { + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + + status = IoDmaAllocateCommonBufferInternal(adapter, + length, + adapter->Requirements.Alignment, + 0, + &bounce); + if (!KSUCCESS(status)) { + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + status = IoDmaBuildContiguousSegments(bounce->DeviceAddress, + length, + &adapter->Requirements, + (IO_DMA_SEGMENT*)0, + &segmentCount); + if (!KSUCCESS(status)) { + IoDmaFreeCommonBufferImpl(bounce); + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + } + + IO_DMA_MAPPING* mapping = IoDmaAllocateMapping(segmentCount); + if (!mapping) { + IoDmaFreeCommonBufferImpl(bounce); + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return NTLSTATUS_NO_MEMORY; + } + + if (bounce) { + status = IoDmaBuildContiguousSegments(bounce->DeviceAddress, + length, + &adapter->Requirements, + mapping->Segments, + &segmentCount); + } else { + status = IoDmaBuildSegments(memory, + offset, + length, + &adapter->Requirements, + mapping->Segments, + &segmentCount); + } + if (!KSUCCESS(status)) { + MmFreePool(mapping); + IoDmaFreeCommonBufferImpl(bounce); + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + + mapping->Adapter = adapter; + mapping->Memory = memory; + mapping->BounceBuffer = bounce; + mapping->MemoryOffset = offset; + mapping->Length = length; + mapping->Direction = direction; + mapping->SegmentCount = segmentCount; + if (!IoDmaTryReferenceAdapter(adapter)) { + MmFreePool(mapping); + IoDmaFreeCommonBufferImpl(bounce); + return NTLSTATUS_OBJECT_DEAD; + } + IoDmaReferencePinnedMemory(memory); + + _InterlockedIncrement64(&gIoDmaStats.StreamingMappings); + _InterlockedIncrement64(&gIoDmaStats.StreamingMappingsInUse); + if (bounce) + _InterlockedIncrement64(&gIoDmaStats.BounceMappings); + else + _InterlockedIncrement64(&gIoDmaStats.DirectMappings); + *outMapping = mapping; + return NTLSTATUS_SUCCESS; +} + +// Synchronizes CPU-written data before the device consumes a mapping. +static KSTATUS IoDmaSynchronizeForDeviceImpl(IO_DMA_MAPPING* mapping) { + if (!mapping) + return NTLSTATUS_INVALID_PARAMETER; + + if (mapping->DeviceOwnsMapping) + return NTLSTATUS_FAILURE; + + if (mapping->Direction != IoDmaFromDevice && mapping->BounceBuffer) { + KSTATUS status = IoDmaCopyFromPinned(mapping->Memory, + mapping->MemoryOffset, + mapping->BounceBuffer->CpuAddress, + mapping->Length); + if (!KSUCCESS(status)) + return status; + _InterlockedExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToDevice, + (long long)mapping->Length); + } + _ReadWriteBarrier(); + _mm_mfence(); + _ReadWriteBarrier(); + mapping->DeviceOwnsMapping = 1; + return NTLSTATUS_SUCCESS; +} + +// Synchronizes device-written data before the CPU consumes a mapping. +static KSTATUS IoDmaSynchronizeForCpuImpl(IO_DMA_MAPPING* mapping) { + if (!mapping) + return NTLSTATUS_INVALID_PARAMETER; + if (!mapping->DeviceOwnsMapping) + return NTLSTATUS_SUCCESS; + + _ReadWriteBarrier(); + _mm_mfence(); + _ReadWriteBarrier(); + if (mapping->Direction != IoDmaToDevice && mapping->BounceBuffer) { + KSTATUS status = IoDmaCopyToPinned(mapping->Memory, + mapping->MemoryOffset, + mapping->BounceBuffer->CpuAddress, + mapping->Length); + if (!KSUCCESS(status)) + return status; + _InterlockedExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToCpu, + (long long)mapping->Length); + } + mapping->DeviceOwnsMapping = 0; + return NTLSTATUS_SUCCESS; +} + +// Releases a streaming mapping and performs the final receive-side bounce copy. +static void IoDmaUnmapMemoryImpl(IO_DMA_MAPPING* mapping) { + if (!mapping) + return; + + if (mapping->DeviceOwnsMapping) { + KSTATUS status = IoDmaSynchronizeForCpuImpl(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: final receive synchronization failed mapping=0x%016llX status=%u\n", + (unsigned long long)(uintptr_t)mapping, + status); + } + IO_DMA_BUFFER* bounce = mapping->BounceBuffer; + MM_PINNED_MEMORY* memory = mapping->Memory; + IO_DMA_ADAPTER* adapter = mapping->Adapter; + memset(mapping, 0, sizeof(*mapping)); + MmFreePool(mapping); + IoDmaFreeCommonBufferImpl(bounce); + IoDmaReleasePinnedMemory(memory); + IoDmaPutAdapterImpl(adapter); + _InterlockedIncrement64(&gIoDmaStats.StreamingMappingsUnmapped); + _InterlockedDecrement64(&gIoDmaStats.StreamingMappingsInUse); +} + +// Returns the number of device segments in a streaming mapping. +static uint32_t IoDmaGetSegmentCountImpl(const IO_DMA_MAPPING* mapping) { + return mapping ? mapping->SegmentCount : 0; +} + +// Returns the immutable device-segment array owned by a mapping. +static const IO_DMA_SEGMENT* IoDmaGetSegmentsImpl(const IO_DMA_MAPPING* mapping) { + return mapping ? mapping->Segments : (const IO_DMA_SEGMENT*)0; +} + +// Copies the current DMA counters into caller-owned storage. +static void IoDmaQueryStatsImpl(IO_DMA_STATS* outStats) { + if (!outStats) + return; + outStats->AdaptersCreated = (uint64_t)gIoDmaStats.AdaptersCreated; + outStats->AdaptersDestroyed = (uint64_t)gIoDmaStats.AdaptersDestroyed; + outStats->AdaptersInUse = (uint64_t)gIoDmaStats.AdaptersInUse; + outStats->CommonBuffersAllocated = (uint64_t)gIoDmaStats.CommonBuffersAllocated; + outStats->CommonBuffersFreed = (uint64_t)gIoDmaStats.CommonBuffersFreed; + outStats->CommonBuffersInUse = (uint64_t)gIoDmaStats.CommonBuffersInUse; + outStats->CommonBufferBytesInUse = (uint64_t)gIoDmaStats.CommonBufferBytesInUse; + outStats->PinnedKernelRanges = (uint64_t)gIoDmaStats.PinnedKernelRanges; + outStats->PinnedUserRanges = (uint64_t)gIoDmaStats.PinnedUserRanges; + outStats->PinnedRangesReleased = (uint64_t)gIoDmaStats.PinnedRangesReleased; + outStats->PinnedRangesInUse = (uint64_t)gIoDmaStats.PinnedRangesInUse; + outStats->PinnedPagesInUse = (uint64_t)gIoDmaStats.PinnedPagesInUse; + outStats->StreamingMappings = (uint64_t)gIoDmaStats.StreamingMappings; + outStats->StreamingMappingsUnmapped = (uint64_t)gIoDmaStats.StreamingMappingsUnmapped; + outStats->StreamingMappingsInUse = (uint64_t)gIoDmaStats.StreamingMappingsInUse; + outStats->DirectMappings = (uint64_t)gIoDmaStats.DirectMappings; + outStats->BounceMappings = (uint64_t)gIoDmaStats.BounceMappings; + outStats->BounceBytesCopiedToDevice = (uint64_t)gIoDmaStats.BounceBytesCopiedToDevice; + outStats->BounceBytesCopiedToCpu = (uint64_t)gIoDmaStats.BounceBytesCopiedToCpu; + outStats->FailedAllocations = (uint64_t)gIoDmaStats.FailedAllocations; + outStats->FailedMappings = (uint64_t)gIoDmaStats.FailedMappings; +} + +static const NTL_DDK_DMA gIoDmaDdk = { + .GetAdapter = IoDmaGetAdapterImpl, + .PutAdapter = IoDmaPutAdapterImpl, + .AllocateCommonBuffer = IoDmaAllocateCommonBufferImpl, + .FreeCommonBuffer = IoDmaFreeCommonBufferImpl, + .GetCommonBufferAddress = IoDmaGetCommonBufferAddressImpl, + .GetCommonBufferDeviceAddress = IoDmaGetCommonBufferDeviceAddressImpl, + .GetCommonBufferSize = IoDmaGetCommonBufferSizeImpl, + .PinKernelMemory = IoDmaPinKernelMemoryImpl, + .PinUserMemory = IoDmaPinUserMemoryImpl, + .UnpinMemory = IoDmaUnpinMemoryImpl, + .MapPinnedMemory = IoDmaMapPinnedMemoryImpl, + .UnmapMemory = IoDmaUnmapMemoryImpl, + .SynchronizeForDevice = IoDmaSynchronizeForDeviceImpl, + .SynchronizeForCpu = IoDmaSynchronizeForCpuImpl, + .GetSegmentCount = IoDmaGetSegmentCountImpl, + .GetSegments = IoDmaGetSegmentsImpl, + .QueryStats = IoDmaQueryStatsImpl, +}; + +// Returns the kernel-owned DDK DMA table. +const NTL_DDK_DMA* IoGetDmaDdk(void) { + return &gIoDmaDdk; +} diff --git a/code/kernel/io_file.c b/code/kernel/io_file.c index 9e22e98..ad32018 100644 --- a/code/kernel/io_file.c +++ b/code/kernel/io_file.c @@ -26,6 +26,47 @@ typedef struct { static VOLUME_ENTRY gVolumes[MAX_DRIVES]; +static EVENT* gVolumeChangeEvent; +static SPINLOCK gVolumeNamespaceLock = SPINLOCK_INIT; +static uint64_t gVolumeNamespaceGeneration; + +// Publishes a mounted-volume namespace change to boot discovery and other waiters. +static void KiIoNotifyVolumeNamespaceChange(void) { + uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock); + gVolumeNamespaceGeneration++; + KeReleaseSpinlock(&gVolumeNamespaceLock, flags); + if (gVolumeChangeEvent) + KeSignalEvent(gVolumeChangeEvent); +} + +// Initializes the waitable mounted-volume namespace notification. +KSTATUS IoInitializeVolumeNamespace(void) { + if (gVolumeChangeEvent) + return NTLSTATUS_SUCCESS; + return KeEventCreate(EventAutoReset, 0, &gVolumeChangeEvent); +} + +// Returns the current mounted-volume namespace generation. +uint64_t IoGetVolumeNamespaceGeneration(void) { + uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock); + uint64_t generation = gVolumeNamespaceGeneration; + KeReleaseSpinlock(&gVolumeNamespaceLock, flags); + return generation; +} + +// Waits for a mounted-volume namespace change without polling driver discovery. +KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration, + uint64_t timeoutMilliseconds) { + if (IoGetVolumeNamespaceGeneration() != observedGeneration) + return NTLSTATUS_SUCCESS; + if (!gVolumeChangeEvent) + return NTLSTATUS_OBJECT_DEAD; + + EVENT* events[1] = { gVolumeChangeEvent }; + uint32_t selected = 0; + return KeWaitForMultipleEvents(events, 1, 0, timeoutMilliseconds, &selected); +} + // Async file I/O worker. typedef enum IO_ASYNC_FILE_OP { IoAsyncFileRead = 0, @@ -275,8 +316,8 @@ static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) { static void KiIoAsyncFileWorker(void* arg) { (void)arg; - for (;;) { - KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore); + while (true) { + KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(s)) { KeYield(); continue; @@ -335,9 +376,10 @@ static KSTATUS KiIoEnsureAsyncFileIoStarted(void) { KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker, 0, - PriorityNormal, + ¶meters, &worker); // Keep the creator reference: this worker is intentionally persistent // for the lifetime of the kernel. @@ -751,6 +793,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex, return NTLSTATUS_ALREADY_EXISTS; gVolumes[volumeIndex].Fs = fs; gVolumes[volumeIndex].Vol = *vol; + KiIoNotifyVolumeNamespaceChange(); return NTLSTATUS_SUCCESS; } @@ -762,18 +805,23 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) { return NTLSTATUS_NOT_FOUND; memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex])); + KiIoNotifyVolumeNamespaceChange(); return NTLSTATUS_SUCCESS; } // Implements the unregister Filesystems For Disk operation. __declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) { KSTATUS last = NTLSTATUS_NOT_FOUND; + bool changed = false; for (uint8_t i = 0; i < MAX_DRIVES; i++) { if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) { memset(&gVolumes[i], 0, sizeof(gVolumes[i])); last = NTLSTATUS_SUCCESS; + changed = true; } } + if (changed) + KiIoNotifyVolumeNamespaceChange(); return last; } @@ -819,7 +867,11 @@ __declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) { static void KiFileDestroy(OBJECT* obj) { FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header); if (f->Fs && f->FileCtx) { - (void)IoFlushFile(f); + // Close drains kernel-owned dirty pages, but stable-media semantics + // remain explicit through IoFlushFile so every close does not issue a + // whole-device cache flush. + (void)MmVadSyncFileMappings(f); + (void)MmFileCacheFlushFile(f); MmFileCacheInvalidateAll(f); f->Fs->Close(f->Fs, f->FileCtx); } @@ -853,71 +905,11 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out f->Size = fs->GetSize(fs, fileCtx); f->Offset = 0; f->Flags = 0; - f->ReadaheadScore = 0; - f->ReadaheadNextOffset = 0; - f->ReadaheadIssuedUntil = 0; - f->ReadaheadSuppressedUntil = 0; *out = f; return NTLSTATUS_SUCCESS; } -static uint64_t KiAlignUpPage64(uint64_t value) { - return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u); -} - -static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) { - if (!file || bytesRead < 512) - return; - if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0) - return; - - uint64_t end = offset + bytesRead; - uint64_t start = 0; - uint32_t pages = 0; - - uintptr_t flags = KeAcquireSpinlock(&file->Lock); - if (offset == file->ReadaheadNextOffset) { - if (file->ReadaheadScore < 32) - file->ReadaheadScore++; - } else { - file->ReadaheadScore = 1; - file->ReadaheadIssuedUntil = end; - } - file->ReadaheadNextOffset = end; - - if (end < file->ReadaheadSuppressedUntil) { - pages = 0; - } else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) { - pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u); - start = KiAlignUpPage64(end); - if (start < file->ReadaheadIssuedUntil) - start = KiAlignUpPage64(file->ReadaheadIssuedUntil); - if (start >= fileSize) { - pages = 0; - } else { - uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE); - if (maxEnd > fileSize) - pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE); - file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE); - } - } - KeReleaseSpinlock(&file->Lock, flags); - - if (pages) { - KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages); - if (ras == NTLSTATUS_QUEUE_FULL) { - flags = KeAcquireSpinlock(&file->Lock); - uint64_t suppressUntil = end + (1024u * 1024u); - if (suppressUntil < end || suppressUntil > fileSize) - suppressUntil = fileSize; - if (file->ReadaheadSuppressedUntil < suppressUntil) - file->ReadaheadSuppressedUntil = suppressUntil; - KeReleaseSpinlock(&file->Lock, flags); - } - } -} - // Explicit-offset file I/O. Async I/O is built on these routines so it does // not race the sequential FILE_OBJECT.Offset used by legacy read/write calls. __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, @@ -979,7 +971,6 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, } *bytesRead = total; - KiIoMaybeScheduleReadAhead(file, offset, total, fileSize); return NTLSTATUS_SUCCESS; } @@ -1022,8 +1013,13 @@ __declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) { if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; KSTATUS sync = MmVadSyncFileMappings(file); - KSTATUS flush = MmFileCacheFlushFile(file); - return KSUCCESS(sync) ? flush : sync; + KSTATUS cache = MmFileCacheFlushFile(file); + KSTATUS media = NTLSTATUS_SUCCESS; + if (KSUCCESS(sync) && KSUCCESS(cache) && file->Fs && file->Fs->Flush) + media = file->Fs->Flush(file->Fs, file->FileCtx); + if (!KSUCCESS(sync)) return sync; + if (!KSUCCESS(cache)) return cache; + return media; } // IoReadFile - sequential read at current offset. @@ -1091,10 +1087,6 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o f->Size = 0; f->Offset = 0; f->Flags = 0; - f->ReadaheadScore = 0; - f->ReadaheadNextOffset = 0; - f->ReadaheadIssuedUntil = 0; - f->ReadaheadSuppressedUntil = 0; *out = f; return NTLSTATUS_SUCCESS; @@ -1278,7 +1270,7 @@ __declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint IO_FILE_LOCK_WAITER waiter; memset(&waiter, 0, sizeof(waiter)); - for (;;) { + while (true) { IO_FILE_LOCK_ENTRY* entry = (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k')); if (!entry) { if (waitEvent) ObDereferenceObject(&waitEvent->Header); diff --git a/code/kernel/ke_apc.c b/code/kernel/ke_apc.c index 9c561f8..85014f2 100644 --- a/code/kernel/ke_apc.c +++ b/code/kernel/ke_apc.c @@ -200,7 +200,7 @@ void KiDeliverKernelApcs(void) { thread->ApcDeliveryActive = 1; - for (;;) { + while (true) { if (KeGetCurrentIrql() != PASSIVE_LEVEL || thread->KernelApcDisable != 0) break; @@ -375,7 +375,7 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) { void KiRundownApcState(THREAD* thread) { if (!thread) return; - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock); LIST_ENTRY* node = ListPopHead(&thread->ApcQueue); if (LIST_IS_EMPTY(&thread->ApcQueue)) @@ -389,7 +389,7 @@ void KiRundownApcState(THREAD* thread) { apc->RundownRoutine(apc); } - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock); LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue); if (LIST_IS_EMPTY(&thread->UserApcQueue)) diff --git a/code/kernel/ke_audio.c b/code/kernel/ke_audio.c new file mode 100644 index 0000000..0699cbc --- /dev/null +++ b/code/kernel/ke_audio.c @@ -0,0 +1,2460 @@ +// Kernel audio core. This file owns endpoint identity, logical streams, mixing, and provider timing handoff. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KE_AUDIO_ENDPOINT_TAG MM_POOL_TAG('a', 'e', 'p', 't') +#define KE_AUDIO_RING_TAG MM_POOL_TAG('a', 'r', 'n', 'g') +#define KE_AUDIO_SELFTEST_TAG MM_POOL_TAG('a', 't', 's', 't') + +#define KE_AUDIO_PERIOD_STATE_FREE 0u +#define KE_AUDIO_PERIOD_STATE_FILLING 1u +#define KE_AUDIO_PERIOD_STATE_READY 2u +#define KE_AUDIO_PERIOD_STATE_COMPLETED 3u +#define KE_AUDIO_PERIOD_STATE_REAPING 4u + +#define KE_AUDIO_TICKS_PER_MILLISECOND 1000ull +#define KE_AUDIO_SELFTEST_WAIT_MILLISECONDS 5000ull + +typedef struct KE_AUDIO_PERIOD_CONTRIBUTION { + KE_AUDIO_STREAM* Stream; + uint64_t StreamGeneration; + uint32_t SourceFrames; +} KE_AUDIO_PERIOD_CONTRIBUTION; + +typedef struct KE_AUDIO_PERIOD_STATE { + uint64_t Generation; + uint32_t ContributionCount; + uint8_t State; + KE_AUDIO_PERIOD_CONTRIBUTION Contributions[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; +} KE_AUDIO_PERIOD_STATE; + +struct KE_AUDIO_STREAM { + OBJECT Header; + MUTEX* Lock; + EVENT* WaitEvent; + LIST_ENTRY EndpointLink; + IO_AUDIO_ENDPOINT* Endpoint; + KE_AUDIO_STREAM_STATE State; + volatile long ProcessHandleCount; + volatile long CloseRequested; + IO_AUDIO_FORMAT Format; + uint8_t* Ring; + uint32_t FrameBytes; + uint32_t ReadFrame; + uint32_t WriteFrame; + uint32_t BufferedFrames; + uint64_t ResamplePositionQ32; + uint64_t ResampleStepQ32; + uint64_t FramesSubmitted; + uint64_t FramesConsumed; + uint64_t FramesPresented; + uint64_t FramesDiscarded; + uint64_t OutputFramesMixed; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t ConsecutiveUnderrunPeriods; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; + uint64_t BufferGeneration; + KSTATUS FailureStatus; + uint32_t WaitMask; + uint32_t WaitWritableFrames; + uint64_t WaitConsumedFrames; + uint64_t WaitPresentedFrames; + uint64_t WaitDrainedGeneration; + uint64_t WaitDiscontinuityGeneration; + float Volume; + float Pan; + uint8_t Muted; + float CurrentLeftGain; + float CurrentRightGain; + float TargetLeftGain; + float TargetRightGain; + float LeftGainStep; + float RightGainStep; + uint32_t GainRampRemaining; +}; + +struct IO_AUDIO_ENDPOINT { + OBJECT Header; + LIST_ENTRY RegistryLink; + LIST_ENTRY StreamList; + SPINLOCK StreamListLock; + MUTEX* ControlLock; + EVENT* WorkEvent; + EVENT* MixerExitEvent; + THREAD* MixerThread; + SPINLOCK NotifyLock; + uint64_t EndpointId; + IO_AUDIO_ENDPOINT_KIND Kind; + KE_AUDIO_ENDPOINT_STATE State; + uint32_t Capabilities; + uint8_t Available; + uint8_t Registered; + uint8_t HardwareRunning; + uint8_t SkipProviderStop; + IO_AUDIO_REMOVAL_MODE RemovalMode; + DEVICE_OBJECT* Device; + void* ProviderContext; + IO_AUDIO_PROVIDER_OPERATIONS Operations; + uint8_t* PeriodBuffer; + size_t PeriodBufferBytes; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + size_t PeriodStrideBytes; + IO_AUDIO_FORMAT HardwareFormat; + int32_t HardwareGainMinimumMilliDb; + int32_t HardwareGainMaximumMilliDb; + uint32_t HardwareGainStepMilliDb; + KE_AUDIO_PERIOD_STATE Periods[KE_AUDIO_OUTPUT_PERIOD_COUNT]; + uint32_t ActiveStreamCount; + uint64_t IdleDeadlineTicks; + float MasterCurrentGain; + float MasterTargetGain; + float MasterRequestedVolume; + int32_t MasterHardwareGainMilliDb; + float MasterGainStep; + uint32_t MasterRampRemaining; + uint64_t MasterGainGeneration; + uint8_t MasterMuted; + KSTATUS FailureStatus; + uint64_t PeriodsMixed; + uint64_t PeriodsCompleted; + uint64_t FramesPresented; + uint64_t HardwarePositionFrames; + uint64_t InterruptCount; + uint64_t LastPresentationTimestampTicks; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t DiscontinuityGeneration; + uint64_t HardwareStarts; + uint64_t HardwareStops; + uint64_t SpuriousNotifications; + uint64_t PendingDiscontinuities; +}; + +typedef struct KE_AUDIO_CORE_STATS { + volatile long long EndpointsRegistered; + volatile long long EndpointsUnregistered; + volatile long long StreamsCreated; + volatile long long StreamsDestroyed; + volatile long long Starts; + volatile long long Stops; + volatile long long Pauses; + volatile long long Resumes; + volatile long long Drains; + volatile long long Resets; +} KE_AUDIO_CORE_STATS; + +typedef struct KE_AUDIO_LIFECYCLE_SNAPSHOT { + KE_AUDIO_ENDPOINT_STATE State; + KSTATUS FailureStatus; + IO_AUDIO_REMOVAL_MODE RemovalMode; + uint8_t SkipProviderStop; +} KE_AUDIO_LIFECYCLE_SNAPSHOT; + +static SPINLOCK gAudioRegistryLock = SPINLOCK_INIT; +static LIST_ENTRY gAudioEndpointList; +static uint32_t gAudioEndpointCount; +static uint64_t gAudioNextEndpointId = 1; +static uint64_t gAudioDefaultEndpointId; +static uint8_t gAudioInitialized; +static KE_AUDIO_CORE_STATS gAudioCoreStats; + +// Stops on one violated internal audio invariant. +static void KiAudioBug(const char* where, const char* reason, uint64_t value) { + KePanic(KE_PANIC_CODE_GENERIC, + "audio invariant failed: %s: %s value=0x%016llX\n", + where ? where : "?", + reason ? reason : "?", + (unsigned long long)value); +} + +// Acquires an internal mutex and rejects abandoned ownership as kernel corruption. +static void KiAudioAcquireMutex(MUTEX* mutex, const char* where) { + KSTATUS status = KeAcquireMutex(mutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KiAudioBug(where, "internal mutex acquisition failed", status); +} + +// Releases one internal mutex. +static void KiAudioReleaseMutex(MUTEX* mutex) { + KeReleaseMutex(mutex); +} + +// Returns the provider timer in microseconds when available. +static uint64_t KiAudioQueryTicks(void) { + return KeQueryMonotonicTicks(); +} + +// Returns whether one float is finite without depending on a runtime math library. +static int KiAudioFloatIsFinite(float value) { + union { + float Float; + uint32_t Bits; + } bits; + bits.Float = value; + return (bits.Bits & 0x7F800000u) != 0x7F800000u; +} + +// Clamps one mixer sample to the signed unit interval. +static float KiAudioClampSample(float value) { + const float maximum = 0.99999988079071044921875f; + if (value > maximum) return maximum; + if (value < -1.0f) return -1.0f; + return value; +} + +// Converts one normalized mixer sample to signed 16-bit PCM. +static int16_t KiAudioFloatToOutputSample(float value) { + float scaled = KiAudioClampSample(value) * 32768.0f; + int32_t sample = scaled >= 0.0f ? (int32_t)(scaled + 0.5f) + : (int32_t)(scaled - 0.5f); + if (sample > 32767) sample = 32767; + if (sample < -32768) sample = -32768; + return (int16_t)sample; +} + +// Approximates one linear gain as milli-decibels without a runtime math dependency. +static int32_t KiAudioMilliDbFromLinearGain(float gain) { + if (gain <= 0.0f) + return INT32_MIN; + + union { + float Float; + uint32_t Bits; + } bits; + bits.Float = gain; + uint32_t encodedExponent = (bits.Bits >> 23) & 0xFFu; + uint32_t fraction = bits.Bits & 0x007FFFFFu; + int32_t exponent = 0; + if (encodedExponent) { + exponent = (int32_t)encodedExponent - 127; + } else { + exponent = -126; + while ((fraction & 0x00800000u) == 0) { + fraction <<= 1; + exponent--; + } + fraction &= 0x007FFFFFu; + } + bits.Bits = fraction | 0x3F800000u; + float mantissa = bits.Float; + float y = (mantissa - 1.0f) / (mantissa + 1.0f); + float y2 = y * y; + float logarithm = 2.0f * y * + (1.0f + y2 * (0.3333333333333333f + + y2 * (0.2f + y2 * (0.1428571428571429f + y2 * 0.1111111111111111f)))); + float log2Gain = (float)exponent + logarithm * 1.4426950408889634f; + float milliDb = log2Gain * 6020.599913279624f; + return milliDb >= 0.0f ? (int32_t)(milliDb + 0.5f) + : (int32_t)(milliDb - 0.5f); +} + +// Approximates one nonpositive milli-decibel residual as a linear gain. +static float KiAudioLinearGainFromMilliDb(int32_t milliDb) { + if (milliDb >= 0) + return 1.0f; + float exponent = (float)milliDb * (1.0f / 6020.599913279624f); + int32_t whole = (int32_t)exponent; + if ((float)whole > exponent) + whole--; + float fraction = exponent - (float)whole; + float z = fraction * 0.6931471805599453f; + float z2 = z * z; + float result = 1.0f + z + z2 * 0.5f + z2 * z * (1.0f / 6.0f) + + z2 * z2 * (1.0f / 24.0f) + z2 * z2 * z * (1.0f / 120.0f); + while (whole < 0) { + result *= 0.5f; + whole++; + } + while (whole > 0) { + result *= 2.0f; + whole--; + } + return result; +} + +// Chooses the highest supported nonpositive hardware gain. +static int32_t KiAudioHighestSafeHardwareGain(const IO_AUDIO_ENDPOINT* endpoint) { + if (!(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) || + !endpoint->Operations.SetGain) + return 0; + + int32_t minimum = endpoint->HardwareGainMinimumMilliDb; + int32_t maximum = endpoint->HardwareGainMaximumMilliDb; + int64_t step = endpoint->HardwareGainStepMilliDb; + if (maximum <= 0) + return maximum; + int64_t steps = ((int64_t)0 - minimum) / step; + return (int32_t)((int64_t)minimum + steps * step); +} + +// Converts one requested master volume into software gain for the stable hardware gain. +static float KiAudioSoftwareGainForVolume(const IO_AUDIO_ENDPOINT* endpoint, + float requestedVolume) { + if (requestedVolume <= 0.0f) + return 0.0f; + if (!(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) || + !endpoint->Operations.SetGain) + return requestedVolume; + int32_t requestedMilliDb = KiAudioMilliDbFromLinearGain(requestedVolume); + int64_t residualMilliDb = (int64_t)requestedMilliDb - + endpoint->MasterHardwareGainMilliDb; + if (residualMilliDb > 0) + residualMilliDb = 0; + if (residualMilliDb < -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB) + return 0.0f; + return KiAudioLinearGainFromMilliDb((int32_t)residualMilliDb); +} + +// Returns the byte width of one source frame. +static uint32_t KiAudioFrameBytes(const IO_AUDIO_FORMAT* format) { + if (!format) return 0; + if (format->SampleFormat != IoAudioSampleS16) return 0; + return 2u * format->ChannelCount; +} + +// Validates one logical source format accepted by the first mixer contract. +static int KiAudioValidateSourceFormat(const IO_AUDIO_FORMAT* format) { + if (!format) return 0; + if (format->ChannelCount != 1 && format->ChannelCount != 2) + return 0; + if (format->SampleFormat != IoAudioSampleS16) + return 0; + return format->SampleRate == 11025u || + format->SampleRate == 22050u || + format->SampleRate == 44100u || + format->SampleRate == 48000u; +} + +// Validates the fixed endpoint ring contract used by the HDA provider series. +static KSTATUS KiAudioValidateRegistration( + const IO_AUDIO_ENDPOINT_REGISTRATION* registration) { + if (!registration || !registration->PeriodBuffer) + return NTLSTATUS_INVALID_PARAMETER; + if (!registration->Device) + return NTLSTATUS_INVALID_PARAMETER; + if (registration->Kind < IoAudioEndpointLineOut || + registration->Kind > IoAudioEndpointOtherAnalog || + (registration->Capabilities & ~IO_AUDIO_ENDPOINT_CAP_VALID_MASK) != 0) + return NTLSTATUS_INVALID_PARAMETER; + if (!registration->Operations.Start || !registration->Operations.Stop) + return NTLSTATUS_INVALID_PARAMETER; + if (registration->HardwareFormat.SampleRate != KE_AUDIO_OUTPUT_SAMPLE_RATE || + registration->HardwareFormat.ChannelCount != KE_AUDIO_OUTPUT_CHANNELS || + registration->HardwareFormat.SampleFormat != IoAudioSampleS16) + return NTLSTATUS_NOT_SUPPORTED; + if (registration->PeriodCount != KE_AUDIO_OUTPUT_PERIOD_COUNT || + registration->FramesPerPeriod != KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD || + registration->PeriodStrideBytes != KE_AUDIO_OUTPUT_PERIOD_BYTES) + return NTLSTATUS_NOT_SUPPORTED; + if (registration->PeriodBufferBytes < + registration->PeriodStrideBytes * registration->PeriodCount) + return NTLSTATUS_BUFFER_TOO_SMALL; + if (((uintptr_t)registration->PeriodBuffer & (sizeof(uint32_t) - 1u)) != 0) + return NTLSTATUS_INVALID_PARAMETER; + if ((registration->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) != 0) { + if (!registration->Operations.SetGain || !registration->HardwareGainStepMilliDb || + registration->HardwareGainStepMilliDb > INT32_MAX || + registration->HardwareGainMinimumMilliDb > registration->HardwareGainMaximumMilliDb || + registration->HardwareGainMinimumMilliDb > 0 || + registration->HardwareGainMinimumMilliDb < -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB || + registration->HardwareGainMaximumMilliDb > IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB || + ((int64_t)registration->HardwareGainMaximumMilliDb - + registration->HardwareGainMinimumMilliDb) % + registration->HardwareGainStepMilliDb != 0) + return NTLSTATUS_INVALID_PARAMETER; + } else if (registration->Operations.SetGain || + registration->HardwareGainMinimumMilliDb || + registration->HardwareGainMaximumMilliDb || + registration->HardwareGainStepMilliDb) { + return NTLSTATUS_INVALID_PARAMETER; + } + return NTLSTATUS_SUCCESS; +} + +// Returns whether one endpoint lifecycle transition is valid. +static int KiAudioEndpointTransitionAllowed(KE_AUDIO_ENDPOINT_STATE oldState, + KE_AUDIO_ENDPOINT_STATE newState) { + if (oldState == newState) + return 1; + switch (oldState) { + case KeAudioEndpointStarting: + return newState == KeAudioEndpointReady || + newState == KeAudioEndpointStopping; + case KeAudioEndpointReady: + return newState == KeAudioEndpointStopping || + newState == KeAudioEndpointFailed; + case KeAudioEndpointFailed: + return newState == KeAudioEndpointStopping; + case KeAudioEndpointStopping: + return newState == KeAudioEndpointRemoved; + case KeAudioEndpointRemoved: + return 0; + default: + return 0; + } +} + +// Changes one endpoint state while the registry lock serializes lifecycle ownership. +static void KiAudioSetEndpointStateLocked(IO_AUDIO_ENDPOINT* endpoint, + KE_AUDIO_ENDPOINT_STATE newState) { + if (!KiAudioEndpointTransitionAllowed(endpoint->State, newState)) + KiAudioBug("KiAudioSetEndpointStateLocked", "invalid endpoint transition", + ((uint64_t)endpoint->State << 32) | (uint32_t)newState); + endpoint->State = newState; +} + +// Returns the fixed endpoint preference rank used until policy moves to a service. +static uint32_t KiAudioEndpointRank(IO_AUDIO_ENDPOINT_KIND kind) { + if (kind == IoAudioEndpointLineOut) return 0; + if (kind == IoAudioEndpointSpeakerHeadphone) return 1; + return 2; +} + +// Recomputes the default endpoint while the registry lock is held. +static void KiAudioSelectDefaultEndpointLocked(void) { + uint64_t selectedId = 0; + uint32_t selectedRank = UINT32_MAX; + for (LIST_ENTRY* link = gAudioEndpointList.Fwd; + link != &gAudioEndpointList; + link = link->Fwd) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); + if (endpoint->State != KeAudioEndpointReady || !endpoint->Available) + continue; + uint32_t rank = KiAudioEndpointRank(endpoint->Kind); + if (!selectedId || rank < selectedRank || + (rank == selectedRank && endpoint->EndpointId < selectedId)) { + selectedId = endpoint->EndpointId; + selectedRank = rank; + } + } + gAudioDefaultEndpointId = selectedId; +} + +// References one ready endpoint by stable identifier. +static IO_AUDIO_ENDPOINT* KiAudioReferenceEndpointById(uint64_t endpointId) { + IO_AUDIO_ENDPOINT* result = (IO_AUDIO_ENDPOINT*)0; + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + for (LIST_ENTRY* link = gAudioEndpointList.Fwd; + link != &gAudioEndpointList; + link = link->Fwd) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); + if (endpoint->EndpointId != endpointId) + continue; + if (endpoint->State == KeAudioEndpointReady && ObTryReferenceObject(&endpoint->Header)) + result = endpoint; + break; + } + KeReleaseSpinlock(&gAudioRegistryLock, flags); + return result; +} + +// Signals one stream when an armed progress condition or terminal state is present. +static void KiAudioRefreshStreamEventLocked(KE_AUDIO_STREAM* stream) { + uint32_t writableFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; + int signaled = stream->State == KeAudioStreamFailed || + stream->State == KeAudioStreamClosed; + if ((stream->WaitMask & KE_AUDIO_WAIT_WRITABLE) && + writableFrames >= stream->WaitWritableFrames) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_CONSUMED) && + stream->FramesConsumed >= stream->WaitConsumedFrames) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_PRESENTED) && + stream->FramesPresented >= stream->WaitPresentedFrames) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_STOPPED) && + stream->State == KeAudioStreamStopped) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_DRAINED) && + stream->DrainedGeneration != stream->WaitDrainedGeneration) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_FAILED) && + stream->State == KeAudioStreamFailed) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_DISCONTINUITY) && + stream->DiscontinuityGeneration != stream->WaitDiscontinuityGeneration) + signaled = 1; + if (signaled) + KeSignalEvent(stream->WaitEvent); + else + KeClearEvent(stream->WaitEvent); +} + +// Updates active-stream timing under the endpoint notification lock. +static void KiAudioAdjustActiveStreams(IO_AUDIO_ENDPOINT* endpoint, int delta) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + uint32_t oldCount = endpoint->ActiveStreamCount; + if (delta > 0) { + endpoint->ActiveStreamCount += (uint32_t)delta; + endpoint->IdleDeadlineTicks = 0; + } else { + uint32_t amount = (uint32_t)(-delta); + if (oldCount < amount) { + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioBug("KiAudioAdjustActiveStreams", "active stream count underflow", oldCount); + } + endpoint->ActiveStreamCount -= amount; + if (endpoint->ActiveStreamCount == 0) { + uint64_t now = KiAudioQueryTicks(); + endpoint->IdleDeadlineTicks = now + + (uint64_t)KE_AUDIO_IDLE_STOP_MILLISECONDS * KE_AUDIO_TICKS_PER_MILLISECOND; + } + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KeSignalEvent(endpoint->WorkEvent); +} + +// Returns whether one stream state contributes to the active hardware clock. +static int KiAudioStreamStateIsActive(KE_AUDIO_STREAM_STATE state) { + return state == KeAudioStreamRunning || state == KeAudioStreamDraining; +} + +// Returns whether every frame targeted by one drain has reached a terminal disposition. +static int KiAudioDrainIsCompleteLocked(const KE_AUDIO_STREAM* stream) { + return stream->BufferedFrames == 0 && + stream->FramesPresented + stream->FramesDiscarded >= stream->FramesSubmitted; +} + +// Returns whether one logical stream transition is valid. +static int KiAudioStreamTransitionAllowed(KE_AUDIO_STREAM_STATE oldState, + KE_AUDIO_STREAM_STATE newState) { + if (oldState == newState) + return 1; + if (newState == KeAudioStreamFailed || newState == KeAudioStreamClosed) + return oldState != KeAudioStreamClosed; + switch (oldState) { + case KeAudioStreamCreated: + return newState == KeAudioStreamRunning || + newState == KeAudioStreamStopped; + case KeAudioStreamRunning: + return newState == KeAudioStreamPaused || + newState == KeAudioStreamDraining || + newState == KeAudioStreamStopped; + case KeAudioStreamPaused: + return newState == KeAudioStreamRunning || + newState == KeAudioStreamDraining || + newState == KeAudioStreamStopped; + case KeAudioStreamDraining: + return newState == KeAudioStreamStopped; + case KeAudioStreamStopped: + return newState == KeAudioStreamRunning; + case KeAudioStreamFailed: + case KeAudioStreamClosed: + default: + return 0; + } +} + +// Changes one stream state and updates endpoint activity in the same locked transaction. +static void KiAudioSetStreamStateLocked(KE_AUDIO_STREAM* stream, + KE_AUDIO_STREAM_STATE newState) { + if (!KiAudioStreamTransitionAllowed(stream->State, newState)) + KiAudioBug("KiAudioSetStreamStateLocked", "invalid stream transition", + ((uint64_t)stream->State << 32) | (uint32_t)newState); + int oldActive = KiAudioStreamStateIsActive(stream->State); + int newActive = KiAudioStreamStateIsActive(newState); + stream->State = newState; + if (oldActive != newActive) + KiAudioAdjustActiveStreams(stream->Endpoint, newActive ? 1 : -1); +} + +// Records one completed drain before publishing the stopped state. +static void KiAudioCompleteDrainLocked(KE_AUDIO_STREAM* stream) { + if (stream->State != KeAudioStreamDraining || !KiAudioDrainIsCompleteLocked(stream)) + KiAudioBug("KiAudioCompleteDrainLocked", "incomplete drain reached completion", stream->State); + stream->DrainedGeneration++; + KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); +} + +// Computes the target left and right gains for one volume, pan, and mute state. +static void KiAudioComputeStreamTargets(float volume, + float pan, + uint8_t muted, + float* leftOut, + float* rightOut) { + float left = volume; + float right = volume; + if (pan > 0.0f) + left *= 1.0f - pan; + else if (pan < 0.0f) + right *= 1.0f + pan; + if (muted) { + left = 0.0f; + right = 0.0f; + } + *leftOut = left; + *rightOut = right; +} + +// Starts a ten-millisecond linear gain ramp while the stream mutex is held. +static void KiAudioBeginStreamGainRampLocked(KE_AUDIO_STREAM* stream) { + KiAudioComputeStreamTargets(stream->Volume, + stream->Pan, + stream->Muted, + &stream->TargetLeftGain, + &stream->TargetRightGain); + stream->GainRampRemaining = KE_AUDIO_GAIN_RAMP_FRAMES; + stream->LeftGainStep = + (stream->TargetLeftGain - stream->CurrentLeftGain) / + (float)KE_AUDIO_GAIN_RAMP_FRAMES; + stream->RightGainStep = + (stream->TargetRightGain - stream->CurrentRightGain) / + (float)KE_AUDIO_GAIN_RAMP_FRAMES; +} + +// Advances one stream gain ramp by one output frame. +static void KiAudioAdvanceStreamGainLocked(KE_AUDIO_STREAM* stream) { + if (!stream->GainRampRemaining) + return; + stream->CurrentLeftGain += stream->LeftGainStep; + stream->CurrentRightGain += stream->RightGainStep; + stream->GainRampRemaining--; + if (!stream->GainRampRemaining) { + stream->CurrentLeftGain = stream->TargetLeftGain; + stream->CurrentRightGain = stream->TargetRightGain; + } +} + +// Decodes one interleaved source frame into normalized stereo samples. +static void KiAudioDecodeSourceFrame(const KE_AUDIO_STREAM* stream, + uint32_t offset, + float* leftOut, + float* rightOut) { + uint32_t index = (stream->ReadFrame + offset) % KE_AUDIO_STREAM_RING_FRAMES; + const uint8_t* frame = stream->Ring + (size_t)index * stream->FrameBytes; + float left = 0.0f; + float right = 0.0f; + + const int16_t* samples = (const int16_t*)frame; + left = (float)samples[0] * (1.0f / 32768.0f); + right = stream->Format.ChannelCount == 2 + ? (float)samples[1] * (1.0f / 32768.0f) + : left; + + *leftOut = left; + *rightOut = right; +} + +// Applies one final-handle close request while the stream mutex serializes state. +static int KiAudioApplyCloseRequestLocked(KE_AUDIO_STREAM* stream) { + if (!_InterlockedCompareExchange(&stream->CloseRequested, 0, 0) || + stream->State == KeAudioStreamClosed) + return stream->State == KeAudioStreamClosed; + + uint64_t terminalFrames = stream->FramesPresented + stream->FramesDiscarded; + if (terminalFrames < stream->FramesSubmitted) + stream->FramesDiscarded += stream->FramesSubmitted - terminalFrames; + stream->BufferGeneration++; + KiAudioSetStreamStateLocked(stream, KeAudioStreamClosed); + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->BufferedFrames = 0; + stream->ResamplePositionQ32 = 0; + stream->ConsecutiveUnderrunPeriods = 0; + stream->DiscontinuityGeneration++; + KiAudioRefreshStreamEventLocked(stream); + return 1; +} + +// Mixes one logical stream into one endpoint period. +static uint32_t KiAudioMixStreamPeriod(KE_AUDIO_STREAM* stream, + float* mix, + uint32_t frames, + uint64_t* streamGenerationOut) { + KiAudioAcquireMutex(stream->Lock, "KiAudioMixStreamPeriod"); + (void)KiAudioApplyCloseRequestLocked(stream); + if (!KiAudioStreamStateIsActive(stream->State)) { + KiAudioReleaseMutex(stream->Lock); + return 0; + } + + uint64_t position = stream->ResamplePositionQ32; + uint32_t silentFrames = 0; + for (uint32_t outputFrame = 0; outputFrame < frames; outputFrame++) { + uint32_t sourceOffset = (uint32_t)(position >> 32); + if (sourceOffset >= stream->BufferedFrames) { + silentFrames += frames - outputFrame; + for (; outputFrame < frames; outputFrame++) + KiAudioAdvanceStreamGainLocked(stream); + break; + } + + float left0 = 0.0f; + float right0 = 0.0f; + float left1 = 0.0f; + float right1 = 0.0f; + KiAudioDecodeSourceFrame(stream, sourceOffset, &left0, &right0); + if (sourceOffset + 1u < stream->BufferedFrames) + KiAudioDecodeSourceFrame(stream, sourceOffset + 1u, &left1, &right1); + else { + left1 = left0; + right1 = right0; + } + + float fraction = (float)(uint32_t)position * (1.0f / 4294967296.0f); + float left = left0 + (left1 - left0) * fraction; + float right = right0 + (right1 - right0) * fraction; + mix[outputFrame * 2u] += left * stream->CurrentLeftGain; + mix[outputFrame * 2u + 1u] += right * stream->CurrentRightGain; + KiAudioAdvanceStreamGainLocked(stream); + position += stream->ResampleStepQ32; + } + + uint32_t consumed = (uint32_t)(position >> 32); + if (consumed > stream->BufferedFrames) + consumed = stream->BufferedFrames; + stream->ReadFrame = (stream->ReadFrame + consumed) % KE_AUDIO_STREAM_RING_FRAMES; + stream->BufferedFrames -= consumed; + stream->FramesConsumed += consumed; + stream->ResamplePositionQ32 = position - ((uint64_t)consumed << 32); + stream->OutputFramesMixed += frames - silentFrames; + + if (silentFrames && stream->State == KeAudioStreamRunning) { + stream->SilenceFramesInserted += silentFrames; + stream->Underruns++; + stream->ConsecutiveUnderrunPeriods++; + } else if (!silentFrames && stream->ConsecutiveUnderrunPeriods) { + stream->ConsecutiveUnderrunPeriods = 0; + } + + *streamGenerationOut = stream->BufferGeneration; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + return consumed; +} + +// Snapshots referenced streams under the nonblocking membership lock. +static uint32_t KiAudioSnapshotStreams(IO_AUDIO_ENDPOINT* endpoint, + KE_AUDIO_STREAM** streams) { + uint32_t count = 0; + uintptr_t flags = KeAcquireSpinlock(&endpoint->StreamListLock); + for (LIST_ENTRY* link = endpoint->StreamList.Fwd; + link != &endpoint->StreamList && count < KE_AUDIO_MAX_STREAMS_PER_ENDPOINT; + link = link->Fwd) { + KE_AUDIO_STREAM* stream = CONTAINING_RECORD(link, KE_AUDIO_STREAM, EndpointLink); + if (ObTryReferenceObject(&stream->Header)) + streams[count++] = stream; + } + KeReleaseSpinlock(&endpoint->StreamListLock, flags); + return count; +} + +// Applies the endpoint master ramp and writes one interleaved HDA period. +static void KiAudioWriteOutputPeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t periodIndex, + float* mix) { + KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioWriteOutputPeriod"); + float current = 1.0f; + float target = 1.0f; + float step = 0.0f; + uint32_t remaining = 0; + uint64_t generation = 0; + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + current = endpoint->MasterCurrentGain; + target = endpoint->MasterTargetGain; + step = endpoint->MasterGainStep; + remaining = endpoint->MasterRampRemaining; + generation = endpoint->MasterGainGeneration; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + int16_t* output = (int16_t*)(endpoint->PeriodBuffer + + (size_t)periodIndex * endpoint->PeriodStrideBytes); + for (uint32_t frame = 0; frame < endpoint->FramesPerPeriod; frame++) { + if (remaining) { + current += step; + remaining--; + if (!remaining) + current = target; + } + output[frame * 2u] = KiAudioFloatToOutputSample(mix[frame * 2u] * current); + output[frame * 2u + 1u] = KiAudioFloatToOutputSample(mix[frame * 2u + 1u] * current); + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (endpoint->MasterGainGeneration == generation) { + endpoint->MasterCurrentGain = current; + endpoint->MasterRampRemaining = remaining; + if (!remaining) + endpoint->MasterGainStep = 0.0f; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); +} + +// Reserves one free hardware period for passive-level mixing. +static int KiAudioReservePeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t* periodIndexOut, + uint64_t* generationOut) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + if (endpoint->Periods[i].State != KE_AUDIO_PERIOD_STATE_FREE) + continue; + if (endpoint->Periods[i].ContributionCount) + KiAudioBug("KiAudioReservePeriod", "free period retains contributions", i); + endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FILLING; + *periodIndexOut = i; + *generationOut = endpoint->Periods[i].Generation; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return 1; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return 0; +} + +// Discards one contribution array after its mixed source frames cannot be presented. +static void KiAudioDiscardContributions(KE_AUDIO_PERIOD_CONTRIBUTION* contributions, + uint32_t contributionCount) { + for (uint32_t i = 0; i < contributionCount; i++) { + KE_AUDIO_PERIOD_CONTRIBUTION* contribution = &contributions[i]; + KE_AUDIO_STREAM* stream = contribution->Stream; + KiAudioAcquireMutex(stream->Lock, "KiAudioDiscardContributions"); + if (stream->BufferGeneration == contribution->StreamGeneration) { + stream->FramesDiscarded += contribution->SourceFrames; + if (stream->State == KeAudioStreamDraining && + KiAudioDrainIsCompleteLocked(stream)) + KiAudioCompleteDrainLocked(stream); + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } +} + +// Commits one mixed period only if no missed deadline invalidated the reservation. +static int KiAudioCommitPeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t periodIndex, + uint64_t generation, + KE_AUDIO_PERIOD_CONTRIBUTION* contributions, + uint32_t contributionCount) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + KE_AUDIO_PERIOD_STATE* period = &endpoint->Periods[periodIndex]; + int committed = period->State == KE_AUDIO_PERIOD_STATE_FILLING && + period->Generation == generation; + if (committed) { + if (period->ContributionCount) + KiAudioBug("KiAudioCommitPeriod", "stale period contributions", periodIndex); + for (uint32_t i = 0; i < contributionCount; i++) + period->Contributions[i] = contributions[i]; + period->ContributionCount = contributionCount; + period->State = KE_AUDIO_PERIOD_STATE_READY; + endpoint->PeriodsMixed++; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (!committed) { + memset(endpoint->PeriodBuffer + (size_t)periodIndex * endpoint->PeriodStrideBytes, + 0, + endpoint->PeriodStrideBytes); + KiAudioDiscardContributions(contributions, contributionCount); + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (period->State == KE_AUDIO_PERIOD_STATE_FILLING || + period->State == KE_AUDIO_PERIOD_STATE_COMPLETED) + period->State = KE_AUDIO_PERIOD_STATE_FREE; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + } + return committed; +} + +// Mixes one free endpoint period from all currently referenced streams. +static int KiAudioMixOnePeriod(IO_AUDIO_ENDPOINT* endpoint) { + uint32_t periodIndex = 0; + uint64_t generation = 0; + if (!KiAudioReservePeriod(endpoint, &periodIndex, &generation)) + return 0; + + float mix[KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD * KE_AUDIO_OUTPUT_CHANNELS]; + memset(mix, 0, sizeof(mix)); + + KE_AUDIO_PERIOD_CONTRIBUTION contributions[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t contributionCount = 0; + KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); + for (uint32_t i = 0; i < streamCount; i++) { + uint64_t streamGeneration = 0; + uint32_t consumed = KiAudioMixStreamPeriod(streams[i], + mix, + endpoint->FramesPerPeriod, + &streamGeneration); + if (consumed) { + contributions[contributionCount].Stream = streams[i]; + contributions[contributionCount].StreamGeneration = streamGeneration; + contributions[contributionCount].SourceFrames = consumed; + contributionCount++; + } else { + ObDereferenceObject(&streams[i]->Header); + } + } + + KiAudioWriteOutputPeriod(endpoint, periodIndex, mix); + return KiAudioCommitPeriod(endpoint, + periodIndex, + generation, + contributions, + contributionCount); +} + +// Applies one hardware-completed period to per-stream presentation counters. +static int KiAudioReapOneCompletedPeriod(IO_AUDIO_ENDPOINT* endpoint) { + uint32_t periodIndex = 0; + KE_AUDIO_PERIOD_STATE* period = (KE_AUDIO_PERIOD_STATE*)0; + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + if (endpoint->Periods[i].State == KE_AUDIO_PERIOD_STATE_COMPLETED) { + periodIndex = i; + period = &endpoint->Periods[i]; + period->State = KE_AUDIO_PERIOD_STATE_REAPING; + break; + } + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (!period) + return 0; + + for (uint32_t i = 0; i < period->ContributionCount; i++) { + KE_AUDIO_PERIOD_CONTRIBUTION* contribution = &period->Contributions[i]; + KE_AUDIO_STREAM* stream = contribution->Stream; + KiAudioAcquireMutex(stream->Lock, "KiAudioReapOneCompletedPeriod"); + if (stream->BufferGeneration == contribution->StreamGeneration) { + stream->FramesPresented += contribution->SourceFrames; + if (stream->State == KeAudioStreamDraining && + KiAudioDrainIsCompleteLocked(stream)) + KiAudioCompleteDrainLocked(stream); + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } + + memset(endpoint->PeriodBuffer + (size_t)periodIndex * endpoint->PeriodStrideBytes, + 0, + endpoint->PeriodStrideBytes); + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + period->ContributionCount = 0; + period->State = KE_AUDIO_PERIOD_STATE_FREE; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return 1; +} + +// Counts periods ready for hardware consumption. +static uint32_t KiAudioReadyPeriodCount(IO_AUDIO_ENDPOINT* endpoint) { + uint32_t ready = 0; + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + if (endpoint->Periods[i].State == KE_AUDIO_PERIOD_STATE_READY) + ready++; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return ready; +} + +// Starts provider DMA after at least two periods are prepared. +static KSTATUS KiAudioStartHardware(IO_AUDIO_ENDPOINT* endpoint) { + KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioStartHardware"); + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + if (endpoint->State != KeAudioEndpointReady) { + KSTATUS status = KFAILED(endpoint->FailureStatus) + ? endpoint->FailureStatus + : NTLSTATUS_DEVICE_REMOVED; + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KiAudioReleaseMutex(endpoint->ControlLock); + return status; + } + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (endpoint->HardwareRunning || endpoint->ActiveStreamCount == 0) { + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); + return NTLSTATUS_SUCCESS; + } + endpoint->HardwareRunning = 1; + endpoint->IdleDeadlineTicks = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + KSTATUS status = NTLSTATUS_SUCCESS; + int startAttempted = 0; + if (endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) { + status = endpoint->Operations.SetGain(endpoint->ProviderContext, + endpoint->MasterHardwareGainMilliDb, + 0); + } + if (KSUCCESS(status)) { + startAttempted = 1; + status = endpoint->Operations.Start(endpoint->ProviderContext); + } + if (!KSUCCESS(status) && startAttempted) + endpoint->Operations.Stop(endpoint->ProviderContext); + if (!KSUCCESS(status)) { + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->HardwareRunning = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); + return status; + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->HardwareStarts++; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); + return NTLSTATUS_SUCCESS; +} + +// Discards one period's unpresented stream contributions during hardware teardown. +static void KiAudioDiscardPeriodContributions(KE_AUDIO_PERIOD_STATE* period) { + KiAudioDiscardContributions(period->Contributions, period->ContributionCount); + period->ContributionCount = 0; +} + +// Stops provider DMA and returns every period to a zeroed free state. +static void KiAudioStopHardware(IO_AUDIO_ENDPOINT* endpoint, int hardwareAccessible) { + KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioStopHardware"); + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + int wasRunning = endpoint->HardwareRunning; + endpoint->HardwareRunning = 0; + endpoint->IdleDeadlineTicks = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + if (wasRunning && hardwareAccessible) { + if (endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) { + KSTATUS muteStatus = endpoint->Operations.SetGain( + endpoint->ProviderContext, + endpoint->MasterHardwareGainMilliDb, + 1); + if (!KSUCCESS(muteStatus)) { + DbgPrint("audio: endpoint %llu hardware mute failed status=0x%x\n", + (unsigned long long)endpoint->EndpointId, + muteStatus); + } + } + endpoint->Operations.Stop(endpoint->ProviderContext); + } + + while (KiAudioReapOneCompletedPeriod(endpoint)) { + } + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) + KiAudioDiscardPeriodContributions(&endpoint->Periods[i]); + memset(endpoint->PeriodBuffer, 0, endpoint->PeriodBufferBytes); + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FREE; + endpoint->Periods[i].Generation++; + } + endpoint->MasterCurrentGain = endpoint->MasterTargetGain; + endpoint->MasterGainStep = 0.0f; + endpoint->MasterRampRemaining = 0; + endpoint->MasterGainGeneration++; + if (wasRunning) + endpoint->HardwareStops++; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); +} + +// Fails every logical stream while preserving final counters for later queries. +static void KiAudioFailAllStreams(IO_AUDIO_ENDPOINT* endpoint, KSTATUS status) { + KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); + for (uint32_t i = 0; i < streamCount; i++) { + KE_AUDIO_STREAM* stream = streams[i]; + KiAudioAcquireMutex(stream->Lock, "KiAudioFailAllStreams"); + if (stream->State != KeAudioStreamClosed) { + stream->FramesDiscarded += stream->BufferedFrames; + stream->BufferedFrames = 0; + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->ResamplePositionQ32 = 0; + KiAudioSetStreamStateLocked(stream, KeAudioStreamFailed); + stream->FailureStatus = status; + stream->BufferGeneration++; + stream->DiscontinuityGeneration++; + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (endpoint->ActiveStreamCount != 0) { + uint32_t active = endpoint->ActiveStreamCount; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioBug("KiAudioFailAllStreams", "active streams remained after failure", active); + } + endpoint->IdleDeadlineTicks = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); +} + +// Applies hardware deadline discontinuities to every live logical stream at passive level. +static void KiAudioApplyPendingDiscontinuities(IO_AUDIO_ENDPOINT* endpoint) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + uint64_t pending = endpoint->PendingDiscontinuities; + endpoint->PendingDiscontinuities = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (!pending) + return; + + KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); + for (uint32_t i = 0; i < streamCount; i++) { + KE_AUDIO_STREAM* stream = streams[i]; + KiAudioAcquireMutex(stream->Lock, "KiAudioApplyPendingDiscontinuities"); + if (KiAudioStreamStateIsActive(stream->State)) { + stream->Underruns += pending; + stream->SilenceFramesInserted += pending * endpoint->FramesPerPeriod; + stream->DiscontinuityGeneration += pending; + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } +} + +// Returns one coherent endpoint lifecycle snapshot. +static void KiAudioSnapshotLifecycle(IO_AUDIO_ENDPOINT* endpoint, + KE_AUDIO_LIFECYCLE_SNAPSHOT* snapshot) { + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + snapshot->State = endpoint->State; + snapshot->FailureStatus = endpoint->FailureStatus; + snapshot->RemovalMode = endpoint->RemovalMode; + snapshot->SkipProviderStop = endpoint->SkipProviderStop; + KeReleaseSpinlock(&gAudioRegistryLock, flags); +} + +// Processes all currently available period and lifecycle work. +static int KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint, + uint64_t* waitMillisecondsOut) { + *waitMillisecondsOut = (uint64_t)-1; + while (KiAudioReapOneCompletedPeriod(endpoint)) { + } + KiAudioApplyPendingDiscontinuities(endpoint); + KE_AUDIO_LIFECYCLE_SNAPSHOT lifecycle; + KiAudioSnapshotLifecycle(endpoint, &lifecycle); + KSTATUS failure = lifecycle.FailureStatus; + if (lifecycle.State == KeAudioEndpointStopping || + lifecycle.State == KeAudioEndpointFailed || + lifecycle.State == KeAudioEndpointRemoved) { + int accessible = !lifecycle.SkipProviderStop && + lifecycle.RemovalMode != IoAudioSurpriseRemoval; + KiAudioStopHardware(endpoint, accessible); + if (lifecycle.State == KeAudioEndpointRemoved && KSUCCESS(failure)) + failure = NTLSTATUS_DEVICE_REMOVED; + if (KSUCCESS(failure)) + failure = NTLSTATUS_DEVICE_ERROR; + KiAudioFailAllStreams(endpoint, failure); + return 0; + } + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + uint32_t active = endpoint->ActiveStreamCount; + int running = endpoint->HardwareRunning; + uint64_t idleDeadline = endpoint->IdleDeadlineTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + if (active && !running) { + while (KiAudioReadyPeriodCount(endpoint) < 2u && + KiAudioMixOnePeriod(endpoint)) { + } + if (KiAudioReadyPeriodCount(endpoint) >= 2u) { + KSTATUS start = KiAudioStartHardware(endpoint); + if (!KSUCCESS(start)) { + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointFailed); + endpoint->FailureStatus = start; + KiAudioSelectDefaultEndpointLocked(); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KeSignalEvent(endpoint->WorkEvent); + return 1; + } + } + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + active = endpoint->ActiveStreamCount; + running = endpoint->HardwareRunning; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (active || running) { + while (KiAudioMixOnePeriod(endpoint)) { + } + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + active = endpoint->ActiveStreamCount; + running = endpoint->HardwareRunning; + idleDeadline = endpoint->IdleDeadlineTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + if (!active && running) { + uint64_t now = KiAudioQueryTicks(); + if (!idleDeadline) { + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->IdleDeadlineTicks = now + + (uint64_t)KE_AUDIO_IDLE_STOP_MILLISECONDS * KE_AUDIO_TICKS_PER_MILLISECOND; + idleDeadline = endpoint->IdleDeadlineTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + } + if (now >= idleDeadline) { + KiAudioStopHardware(endpoint, 1); + } else { + uint64_t remainingTicks = idleDeadline - now; + *waitMillisecondsOut = (remainingTicks + KE_AUDIO_TICKS_PER_MILLISECOND - 1u) / + KE_AUDIO_TICKS_PER_MILLISECOND; + if (!*waitMillisecondsOut) + *waitMillisecondsOut = 1; + } + } + return 1; +} + +// Runs one endpoint's real-time mixer and provider lifecycle loop. +static void KiAudioMixerThread(void* argument) { + IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)argument; + while (true) { + KeClearEvent(endpoint->WorkEvent); + uint64_t waitMilliseconds = (uint64_t)-1; + if (!KiAudioProcessEndpoint(endpoint, &waitMilliseconds)) + break; + + KSTATUS waitStatus = NTLSTATUS_SUCCESS; + if (waitMilliseconds == (uint64_t)-1) { + waitStatus = KeWaitForEvent(endpoint->WorkEvent, 0); + } else { + EVENT* events[1] = { endpoint->WorkEvent }; + uint32_t selected = 0; + waitStatus = KeWaitForMultipleEvents(events, + 1, + 0, + waitMilliseconds, + &selected); + } + if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT) + KiAudioBug("KiAudioMixerThread", "work wait failed", waitStatus); + } + + KeSignalEvent(endpoint->MixerExitEvent); + ObDereferenceObject(&endpoint->Header); +} + +// Destroys one fully unregistered endpoint after its mixer and streams leave. +static void KiAudioEndpointDestroy(OBJECT* object) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(object, IO_AUDIO_ENDPOINT, Header); + if (endpoint->Registered || endpoint->MixerThread || + !LIST_IS_EMPTY(&endpoint->StreamList) || endpoint->HardwareRunning || + endpoint->ActiveStreamCount) + KiAudioBug("KiAudioEndpointDestroy", "live endpoint reached destruction", endpoint->EndpointId); + if (endpoint->Device) + IoDereferenceDeviceObject(endpoint->Device); + if (endpoint->ControlLock) + ObDereferenceObject(&endpoint->ControlLock->Header); + if (endpoint->WorkEvent) + ObDereferenceObject(&endpoint->WorkEvent->Header); + if (endpoint->MixerExitEvent) + ObDereferenceObject(&endpoint->MixerExitEvent->Header); +} + +// Records one process handle that owns the stream. +void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream) { + if (!stream) + return; + + long handleCount = _InterlockedIncrement(&stream->ProcessHandleCount); + if (handleCount <= 0) + KiAudioBug("KeAudioStreamHandleOpened", "invalid process handle count", handleCount); +} + +// Requests passive stream closure when the final process handle disappears. +void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream) { + if (!stream) + return; + + long handleCount = _InterlockedDecrement(&stream->ProcessHandleCount); + if (handleCount < 0) + KiAudioBug("KeAudioStreamHandleClosed", "process handle underflow", handleCount); + if (handleCount != 0) + return; + + _InterlockedExchange(&stream->CloseRequested, 1); + KeSignalEvent(stream->WaitEvent); + if (stream->Endpoint) + KeSignalEvent(stream->Endpoint->WorkEvent); +} + +// Destroys one exclusively owned stream without entering blocking primitives. +static void KiAudioStreamDestroy(OBJECT* object) { + KE_AUDIO_STREAM* stream = CONTAINING_RECORD(object, KE_AUDIO_STREAM, Header); + IO_AUDIO_ENDPOINT* endpoint = stream->Endpoint; + if (endpoint) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->StreamListLock); + if (stream->EndpointLink.Fwd != &stream->EndpointLink || + stream->EndpointLink.Bwd != &stream->EndpointLink) + KeListRemoveChecked(&stream->EndpointLink); + KeReleaseSpinlock(&endpoint->StreamListLock, flags); + } + + if (stream->Lock && + (stream->Lock->Owner || !LIST_IS_EMPTY(&stream->Lock->WaitList))) + KiAudioBug("KiAudioStreamDestroy", "live stream mutex reached destruction", + (uint64_t)(uintptr_t)stream->Lock->Owner); + if (!KiAudioStreamTransitionAllowed(stream->State, KeAudioStreamClosed)) + KiAudioBug("KiAudioStreamDestroy", "invalid terminal stream state", stream->State); + int wasActive = KiAudioStreamStateIsActive(stream->State); + stream->State = KeAudioStreamClosed; + if (wasActive) + KiAudioAdjustActiveStreams(endpoint, -1); + + if (stream->Ring) + MmFreePool(stream->Ring); + if (stream->WaitEvent) + ObDereferenceObject(&stream->WaitEvent->Header); + if (stream->Lock) + ObDereferenceObject(&stream->Lock->Header); + if (endpoint) + ObDereferenceObject(&endpoint->Header); + _InterlockedIncrement64(&gAudioCoreStats.StreamsDestroyed); +} + +// Registers one hardware-provider endpoint. +static KSTATUS KiAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, + IO_AUDIO_ENDPOINT** outEndpoint) { + if (!outEndpoint) + return NTLSTATUS_INVALID_PARAMETER; + *outEndpoint = (IO_AUDIO_ENDPOINT*)0; + KSTATUS status = KiAudioValidateRegistration(registration); + if (!KSUCCESS(status)) + return status; + + IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)ObAllocateObject( + ObjectTypeAudioEndpoint, + sizeof(IO_AUDIO_ENDPOINT)); + if (!endpoint) + return NTLSTATUS_NO_MEMORY; + endpoint->Header.Destroy = KiAudioEndpointDestroy; + LIST_INIT(&endpoint->RegistryLink); + LIST_INIT(&endpoint->StreamList); + endpoint->StreamListLock = (SPINLOCK)SPINLOCK_INIT; + endpoint->NotifyLock = (SPINLOCK)SPINLOCK_INIT; + endpoint->Kind = registration->Kind; + endpoint->State = KeAudioEndpointStarting; + endpoint->Capabilities = registration->Capabilities; + endpoint->Available = registration->InitiallyAvailable ? 1u : 0u; + endpoint->Device = registration->Device; + endpoint->ProviderContext = registration->ProviderContext; + endpoint->Operations = registration->Operations; + endpoint->PeriodBuffer = (uint8_t*)registration->PeriodBuffer; + endpoint->PeriodBufferBytes = registration->PeriodBufferBytes; + endpoint->PeriodCount = registration->PeriodCount; + endpoint->FramesPerPeriod = registration->FramesPerPeriod; + endpoint->PeriodStrideBytes = registration->PeriodStrideBytes; + endpoint->HardwareFormat = registration->HardwareFormat; + endpoint->HardwareGainMinimumMilliDb = registration->HardwareGainMinimumMilliDb; + endpoint->HardwareGainMaximumMilliDb = registration->HardwareGainMaximumMilliDb; + endpoint->HardwareGainStepMilliDb = registration->HardwareGainStepMilliDb; + endpoint->MasterRequestedVolume = 1.0f; + endpoint->MasterHardwareGainMilliDb = KiAudioHighestSafeHardwareGain(endpoint); + endpoint->MasterCurrentGain = KiAudioSoftwareGainForVolume(endpoint, 1.0f); + endpoint->MasterTargetGain = endpoint->MasterCurrentGain; + endpoint->RemovalMode = IoAudioOrderlyRemoval; + memset(endpoint->PeriodBuffer, 0, endpoint->PeriodBufferBytes); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) + endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FREE; + + if (endpoint->Device) + IoReferenceDeviceObject(endpoint->Device); + + status = KeMutexCreate(&endpoint->ControlLock); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + return status; + } + status = KeEventCreate(EventManualReset, 0, &endpoint->WorkEvent); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + return status; + } + status = KeEventCreate(EventManualReset, 0, &endpoint->MixerExitEvent); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + return status; + } + + ObReferenceObject(&endpoint->Header); + const KE_THREAD_CREATE_PARAMETERS mixerParameters = + KE_THREAD_CREATE_REALTIME(KE_AUDIO_MIXER_REALTIME_PRIORITY); + status = KeCreateKernelThread((uintptr_t)KiAudioMixerThread, + (uintptr_t)endpoint, + &mixerParameters, + &endpoint->MixerThread); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + ObDereferenceObject(&endpoint->Header); + return status; + } + + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + if (gAudioEndpointCount >= KE_AUDIO_MAX_ENDPOINTS) { + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping); + endpoint->FailureStatus = NTLSTATUS_NO_MEMORY; + KeReleaseSpinlock(&gAudioRegistryLock, flags); + KeSignalEvent(endpoint->WorkEvent); + (void)KeWaitForEvent(endpoint->MixerExitEvent, 0); + ObDereferenceObject(&endpoint->MixerThread->Header); + endpoint->MixerThread = (THREAD*)0; + ObDereferenceObject(&endpoint->Header); + return NTLSTATUS_NO_MEMORY; + } + endpoint->EndpointId = gAudioNextEndpointId++; + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointReady); + endpoint->Registered = 1; + ObReferenceObject(&endpoint->Header); + KeListInsertTailChecked(&gAudioEndpointList, &endpoint->RegistryLink); + gAudioEndpointCount++; + KiAudioSelectDefaultEndpointLocked(); + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + _InterlockedIncrement64(&gAudioCoreStats.EndpointsRegistered); + *outEndpoint = endpoint; + return NTLSTATUS_SUCCESS; +} + +// Implements the provider-facing endpoint registration operation. +static KSTATUS KiAudioDdkRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, + IO_AUDIO_ENDPOINT** outEndpoint) { + return KiAudioRegisterEndpoint(registration, outEndpoint); +} + +// Removes one endpoint and waits until no provider operation can run again. +static void KiAudioDdkUnregisterEndpoint(IO_AUDIO_ENDPOINT* endpoint, + IO_AUDIO_REMOVAL_MODE mode) { + if (!endpoint || (mode != IoAudioOrderlyRemoval && mode != IoAudioSurpriseRemoval)) + KiAudioBug("KiAudioDdkUnregisterEndpoint", "invalid unregister request", mode); + + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + if (!endpoint->Registered) { + KeReleaseSpinlock(&gAudioRegistryLock, flags); + KiAudioBug("KiAudioDdkUnregisterEndpoint", "duplicate unregister", endpoint->EndpointId); + } + KeListRemoveChecked(&endpoint->RegistryLink); + endpoint->Registered = 0; + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping); + if (KSUCCESS(endpoint->FailureStatus)) + endpoint->FailureStatus = NTLSTATUS_DEVICE_REMOVED; + endpoint->RemovalMode = mode; + endpoint->SkipProviderStop = mode == IoAudioSurpriseRemoval ? 1u : 0u; + gAudioEndpointCount--; + KiAudioSelectDefaultEndpointLocked(); + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + KeSignalEvent(endpoint->WorkEvent); + (void)KeWaitForEvent(endpoint->MixerExitEvent, 0); + + flags = KeAcquireSpinlock(&gAudioRegistryLock); + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointRemoved); + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + if (!endpoint->MixerThread) + KiAudioBug("KiAudioDdkUnregisterEndpoint", "missing mixer thread", endpoint->EndpointId); + ObDereferenceObject(&endpoint->MixerThread->Header); + endpoint->MixerThread = (THREAD*)0; + + _InterlockedIncrement64(&gAudioCoreStats.EndpointsUnregistered); + ObDereferenceObject(&endpoint->Header); + ObDereferenceObject(&endpoint->Header); +} + +// Advances one provider ring period and wakes the real-time mixer. +static void KiAudioDdkNotifyPeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t hardwarePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks) { + if (!endpoint || completedPeriodIndex >= endpoint->PeriodCount) + KiAudioBug("KiAudioDdkNotifyPeriod", "invalid period notification", completedPeriodIndex); + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + KE_AUDIO_PERIOD_STATE* period = &endpoint->Periods[completedPeriodIndex]; + if (!endpoint->HardwareRunning) { + endpoint->SpuriousNotifications++; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return; + } + + int missedDeadline = period->State != KE_AUDIO_PERIOD_STATE_READY; + if (missedDeadline) { + endpoint->Underruns++; + endpoint->DiscontinuityGeneration++; + endpoint->SilenceFramesInserted += endpoint->FramesPerPeriod; + endpoint->PendingDiscontinuities++; + } + if (period->State == KE_AUDIO_PERIOD_STATE_COMPLETED || + period->State == KE_AUDIO_PERIOD_STATE_REAPING) { + endpoint->SpuriousNotifications++; + } else { + period->Generation++; + period->State = KE_AUDIO_PERIOD_STATE_COMPLETED; + } + endpoint->PeriodsCompleted++; + if (presentedFrames < endpoint->FramesPresented) { + endpoint->DiscontinuityGeneration++; + endpoint->PendingDiscontinuities++; + } else { + endpoint->FramesPresented = presentedFrames; + } + if (hardwarePositionFrames < endpoint->HardwarePositionFrames || + interruptCount < endpoint->InterruptCount) { + endpoint->DiscontinuityGeneration++; + endpoint->PendingDiscontinuities++; + } else { + endpoint->HardwarePositionFrames = hardwarePositionFrames; + endpoint->InterruptCount = interruptCount; + } + endpoint->LastPresentationTimestampTicks = timestampTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + KeSignalEvent(endpoint->WorkEvent); +} + +// Updates one endpoint's jack-derived availability and default selection. +static void KiAudioDdkSetEndpointAvailable(IO_AUDIO_ENDPOINT* endpoint, uint8_t available) { + if (!endpoint || !(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE)) + KiAudioBug("KiAudioDdkSetEndpointAvailable", "invalid availability update", 0); + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + if (endpoint->State == KeAudioEndpointReady) { + endpoint->Available = available ? 1u : 0u; + KiAudioSelectDefaultEndpointLocked(); + } + KeReleaseSpinlock(&gAudioRegistryLock, flags); +} + +// Fails one endpoint from provider completion or recovery code. +static void KiAudioDdkReportEndpointFailure(IO_AUDIO_ENDPOINT* endpoint, + KSTATUS status, + uint32_t flags) { + if (!endpoint || !KFAILED(status) || (flags & ~IO_AUDIO_FAILURE_VALID_MASK) != 0) + KiAudioBug("KiAudioDdkReportEndpointFailure", "invalid failure report", status); + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + if (flags & IO_AUDIO_FAILURE_HARDWARE_INACCESSIBLE) + endpoint->SkipProviderStop = 1; + if (endpoint->State == KeAudioEndpointReady) { + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointFailed); + endpoint->FailureStatus = status; + endpoint->Available = 0; + KiAudioSelectDefaultEndpointLocked(); + } + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + DbgPrint("audio: endpoint %llu failed status=0x%x flags=0x%x\n", + (unsigned long long)endpoint->EndpointId, + status, + flags); + KeSignalEvent(endpoint->WorkEvent); +} + +static const NTL_DDK_AUDIO gAudioDdk = { + .RegisterEndpoint = KiAudioDdkRegisterEndpoint, + .UnregisterEndpoint = KiAudioDdkUnregisterEndpoint, + .NotifyPeriod = KiAudioDdkNotifyPeriod, + .SetEndpointAvailable = KiAudioDdkSetEndpointAvailable, + .ReportEndpointFailure = KiAudioDdkReportEndpointFailure, +}; + +// Initializes the one kernel audio registry. +void KeAudioInitialize(void) { + if (gAudioInitialized) + KiAudioBug("KeAudioInitialize", "duplicate initialization", 0); + LIST_INIT(&gAudioEndpointList); + gAudioInitialized = 1; +} + +// Returns the immutable provider-facing audio table. +const NTL_DDK_AUDIO* KeAudioGetDdk(void) { + return &gAudioDdk; +} + +// Returns the current default endpoint identifier. +uint64_t KeAudioGetDefaultEndpointId(void) { + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + uint64_t endpointId = gAudioDefaultEndpointId; + KeReleaseSpinlock(&gAudioRegistryLock, flags); + return endpointId; +} + +// Copies endpoint snapshots without retaining registry locks across passive operations. +KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t capacity, + uint32_t* countOut) { + if (!countOut || (capacity && !endpoints)) + return NTLSTATUS_INVALID_PARAMETER; + + IO_AUDIO_ENDPOINT* references[KE_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + uint64_t defaultId = 0; + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + defaultId = gAudioDefaultEndpointId; + for (LIST_ENTRY* link = gAudioEndpointList.Fwd; + link != &gAudioEndpointList && count < KE_AUDIO_MAX_ENDPOINTS; + link = link->Fwd) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); + if (ObTryReferenceObject(&endpoint->Header)) + references[count++] = endpoint; + } + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + *countOut = count; + uint32_t copyCount = capacity < count ? capacity : count; + for (uint32_t i = 0; i < count; i++) { + IO_AUDIO_ENDPOINT* endpoint = references[i]; + if (i < copyCount) { + KE_AUDIO_ENDPOINT_INFO* info = &endpoints[i]; + memset(info, 0, sizeof(*info)); + flags = KeAcquireSpinlock(&gAudioRegistryLock); + info->EndpointId = endpoint->EndpointId; + info->Kind = endpoint->Kind; + info->State = endpoint->State; + info->Capabilities = endpoint->Capabilities; + info->HardwareFormat = endpoint->HardwareFormat; + info->PeriodCount = endpoint->PeriodCount; + info->FramesPerPeriod = endpoint->FramesPerPeriod; + info->Available = endpoint->Available; + info->IsDefault = endpoint->EndpointId == defaultId; + info->FailureStatus = endpoint->FailureStatus; + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + uintptr_t streamFlags = KeAcquireSpinlock(&endpoint->StreamListLock); + for (LIST_ENTRY* link = endpoint->StreamList.Fwd; + link != &endpoint->StreamList; + link = link->Fwd) + info->StreamCount++; + KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + info->ActiveStreamCount = endpoint->ActiveStreamCount; + info->HardwareRunning = endpoint->HardwareRunning; + info->MasterMuted = endpoint->MasterMuted; + info->MasterHardwareGainMilliDb = endpoint->MasterHardwareGainMilliDb; + info->MasterRequestedVolume = endpoint->MasterRequestedVolume; + info->PeriodsMixed = endpoint->PeriodsMixed; + info->PeriodsCompleted = endpoint->PeriodsCompleted; + info->FramesPresented = endpoint->FramesPresented; + info->HardwarePositionFrames = endpoint->HardwarePositionFrames; + info->InterruptCount = endpoint->InterruptCount; + info->LastPresentationTimestampTicks = endpoint->LastPresentationTimestampTicks; + info->SilenceFramesInserted = endpoint->SilenceFramesInserted; + info->Underruns = endpoint->Underruns; + info->DiscontinuityGeneration = endpoint->DiscontinuityGeneration; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + } + ObDereferenceObject(&endpoint->Header); + } + return capacity < count ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS; +} + +// Creates one logical render stream with a fixed-capacity source ring. +KSTATUS KeAudioCreateStream(uint64_t endpointId, + const IO_AUDIO_FORMAT* format, + KE_AUDIO_STREAM** outStream) { + if (!outStream || !KiAudioValidateSourceFormat(format)) + return NTLSTATUS_INVALID_PARAMETER; + *outStream = (KE_AUDIO_STREAM*)0; + IO_AUDIO_ENDPOINT* endpoint = KiAudioReferenceEndpointById(endpointId); + if (!endpoint) + return NTLSTATUS_NOT_FOUND; + + KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)ObAllocateObject( + ObjectTypeAudioStream, + sizeof(KE_AUDIO_STREAM)); + if (!stream) { + ObDereferenceObject(&endpoint->Header); + return NTLSTATUS_NO_MEMORY; + } + stream->Header.Destroy = KiAudioStreamDestroy; + LIST_INIT(&stream->EndpointLink); + stream->Endpoint = endpoint; + stream->State = KeAudioStreamCreated; + stream->Format = *format; + stream->FrameBytes = KiAudioFrameBytes(format); + stream->ResampleStepQ32 = ((uint64_t)format->SampleRate << 32) / + KE_AUDIO_OUTPUT_SAMPLE_RATE; + stream->WaitMask = KE_AUDIO_WAIT_WRITABLE; + stream->WaitWritableFrames = 1; + stream->Volume = 1.0f; + stream->CurrentLeftGain = 1.0f; + stream->CurrentRightGain = 1.0f; + stream->TargetLeftGain = 1.0f; + stream->TargetRightGain = 1.0f; + + KSTATUS status = KeMutexCreate(&stream->Lock); + if (!KSUCCESS(status)) { + ObDereferenceObject(&stream->Header); + return status; + } + status = KeEventCreate(EventManualReset, 1, &stream->WaitEvent); + if (!KSUCCESS(status)) { + ObDereferenceObject(&stream->Header); + return status; + } + stream->Ring = (uint8_t*)MmAllocatePoolTag( + (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes, + KE_AUDIO_RING_TAG); + if (!stream->Ring) { + ObDereferenceObject(&stream->Header); + return NTLSTATUS_NO_MEMORY; + } + memset(stream->Ring, + 0, + (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes); + + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + int endpointReady = endpoint->Registered && endpoint->State == KeAudioEndpointReady; + uint32_t streamCount = 0; + uintptr_t streamFlags = KeAcquireSpinlock(&endpoint->StreamListLock); + if (endpointReady) { + for (LIST_ENTRY* link = endpoint->StreamList.Fwd; + link != &endpoint->StreamList; + link = link->Fwd) + streamCount++; + } + if (!endpointReady || streamCount >= KE_AUDIO_MAX_STREAMS_PER_ENDPOINT) { + KSTATUS createStatus = !endpointReady + ? (KFAILED(endpoint->FailureStatus) + ? endpoint->FailureStatus + : NTLSTATUS_DEVICE_REMOVED) + : NTLSTATUS_NO_MEMORY; + KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + ObDereferenceObject(&stream->Header); + return createStatus; + } + KeListInsertTailChecked(&endpoint->StreamList, &stream->EndpointLink); + KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + + _InterlockedIncrement64(&gAudioCoreStats.StreamsCreated); + *outStream = stream; + return NTLSTATUS_SUCCESS; +} + +// Returns the stream event used by future waitable handles. +EVENT* KeAudioStreamWaitEvent(KE_AUDIO_STREAM* stream) { + return stream ? stream->WaitEvent : (EVENT*)0; +} + +// Arms one wait condition while holding the stream state that drives its event. +KSTATUS KeAudioArmStreamWait(KE_AUDIO_STREAM* stream, + const KE_AUDIO_STREAM_WAIT_CONDITION* condition) { + if (!stream || !condition || !condition->Mask || + (condition->Mask & ~KE_AUDIO_WAIT_VALID_MASK) != 0 || + ((condition->Mask & KE_AUDIO_WAIT_WRITABLE) && + (!condition->WritableFrames || + condition->WritableFrames > KE_AUDIO_STREAM_RING_FRAMES))) + return NTLSTATUS_INVALID_PARAMETER; + + KiAudioAcquireMutex(stream->Lock, "KeAudioArmStreamWait"); + if (stream->State == KeAudioStreamClosed) { + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_OBJECT_DEAD; + } + stream->WaitMask = condition->Mask; + stream->WaitWritableFrames = condition->WritableFrames; + stream->WaitConsumedFrames = condition->ConsumedFrames; + stream->WaitPresentedFrames = condition->PresentedFrames; + stream->WaitDrainedGeneration = condition->DrainedGeneration; + stream->WaitDiscontinuityGeneration = condition->DiscontinuityGeneration; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_SUCCESS; +} + +// Copies a bounded source-frame run into one stream ring. +KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream, + const void* frames, + uint32_t requestedFrames, + uint32_t* writtenFramesOut) { + if (!stream || !writtenFramesOut || (requestedFrames && !frames)) + return NTLSTATUS_INVALID_PARAMETER; + *writtenFramesOut = 0; + if (!requestedFrames) + return NTLSTATUS_SUCCESS; + + KiAudioAcquireMutex(stream->Lock, "KeAudioWriteStream"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + if (stream->State == KeAudioStreamDraining) { + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_INVALID_PARAMETER; + } + + uint32_t writable = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; + if (!writable) { + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_WOULD_BLOCK; + } + uint32_t toWrite = requestedFrames < writable ? requestedFrames : writable; + uint32_t firstFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->WriteFrame; + if (firstFrames > toWrite) firstFrames = toWrite; + size_t firstBytes = (size_t)firstFrames * stream->FrameBytes; + memcpy(stream->Ring + (size_t)stream->WriteFrame * stream->FrameBytes, + frames, + firstBytes); + uint32_t secondFrames = toWrite - firstFrames; + if (secondFrames) { + memcpy(stream->Ring, + (const uint8_t*)frames + firstBytes, + (size_t)secondFrames * stream->FrameBytes); + } + stream->WriteFrame = (stream->WriteFrame + toWrite) % KE_AUDIO_STREAM_RING_FRAMES; + stream->BufferedFrames += toWrite; + stream->FramesSubmitted += toWrite; + *writtenFramesOut = toWrite; + KiAudioRefreshStreamEventLocked(stream); + int active = KiAudioStreamStateIsActive(stream->State); + KiAudioReleaseMutex(stream->Lock); + + if (active) + KeSignalEvent(stream->Endpoint->WorkEvent); + return NTLSTATUS_SUCCESS; +} + +// Starts one created or stopped stream. +KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioStartStream"); + if (stream->State != KeAudioStreamCreated && stream->State != KeAudioStreamStopped) { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioSetStreamStateLocked(stream, KeAudioStreamRunning); + stream->FailureStatus = NTLSTATUS_SUCCESS; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + _InterlockedIncrement64(&gAudioCoreStats.Starts); + return NTLSTATUS_SUCCESS; +} + +// Pauses one running stream without discarding source frames. +KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioPauseStream"); + if (stream->State != KeAudioStreamRunning) { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioSetStreamStateLocked(stream, KeAudioStreamPaused); + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + _InterlockedIncrement64(&gAudioCoreStats.Pauses); + return NTLSTATUS_SUCCESS; +} + +// Resumes one paused stream. +KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioResumeStream"); + if (stream->State != KeAudioStreamPaused) { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioSetStreamStateLocked(stream, KeAudioStreamRunning); + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + _InterlockedIncrement64(&gAudioCoreStats.Resumes); + return NTLSTATUS_SUCCESS; +} + +// Stops accepting writes and plays all currently buffered frames. +KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioDrainStream"); + if (stream->State == KeAudioStreamRunning || + stream->State == KeAudioStreamPaused) { + KiAudioSetStreamStateLocked(stream, KeAudioStreamDraining); + if (KiAudioDrainIsCompleteLocked(stream)) + KiAudioCompleteDrainLocked(stream); + } else { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + _InterlockedIncrement64(&gAudioCoreStats.Drains); + return NTLSTATUS_SUCCESS; +} + +// Stops future mixing and discards source frames not yet committed to endpoint periods. +KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioStopStream"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + stream->FramesDiscarded += stream->BufferedFrames; + KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->BufferedFrames = 0; + stream->ResamplePositionQ32 = 0; + stream->ConsecutiveUnderrunPeriods = 0; + stream->DiscontinuityGeneration++; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + _InterlockedIncrement64(&gAudioCoreStats.Stops); + return NTLSTATUS_SUCCESS; +} + +// Resets one live stream and invalidates presentation accounting from earlier periods. +KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioResetStream"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + stream->BufferGeneration++; + KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->BufferedFrames = 0; + stream->ResamplePositionQ32 = 0; + stream->FramesSubmitted = 0; + stream->FramesConsumed = 0; + stream->FramesPresented = 0; + stream->FramesDiscarded = 0; + stream->OutputFramesMixed = 0; + stream->SilenceFramesInserted = 0; + stream->Underruns = 0; + stream->ConsecutiveUnderrunPeriods = 0; + stream->DiscontinuityGeneration++; + memset(stream->Ring, + 0, + (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes); + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + _InterlockedIncrement64(&gAudioCoreStats.Resets); + return NTLSTATUS_SUCCESS; +} + +// Updates per-stream volume, linear balance pan, and mute targets. +KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream, + float volume, + float pan, + bool muted) { + if (!stream || !KiAudioFloatIsFinite(volume) || !KiAudioFloatIsFinite(pan) || + volume < 0.0f || volume > 1.0f || pan < -1.0f || pan > 1.0f) + return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioSetStreamVolume"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + stream->Volume = volume; + stream->Pan = pan; + stream->Muted = muted ? 1u : 0u; + if (stream->State == KeAudioStreamCreated || stream->State == KeAudioStreamStopped) { + KiAudioComputeStreamTargets(stream->Volume, + stream->Pan, + stream->Muted, + &stream->CurrentLeftGain, + &stream->CurrentRightGain); + stream->TargetLeftGain = stream->CurrentLeftGain; + stream->TargetRightGain = stream->CurrentRightGain; + stream->GainRampRemaining = 0; + stream->LeftGainStep = 0.0f; + stream->RightGainStep = 0.0f; + } else { + KiAudioBeginStreamGainRampLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + return NTLSTATUS_SUCCESS; +} + +// Updates endpoint master software gain and separate logical mute state. +KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted) { + if (!KiAudioFloatIsFinite(volume) || volume < 0.0f || volume > 1.0f) + return NTLSTATUS_INVALID_PARAMETER; + IO_AUDIO_ENDPOINT* endpoint = KiAudioReferenceEndpointById(endpointId); + if (!endpoint) + return NTLSTATUS_NOT_FOUND; + + KiAudioAcquireMutex(endpoint->ControlLock, "KeAudioSetMasterVolume"); + float softwareGain = muted ? 0.0f + : KiAudioSoftwareGainForVolume(endpoint, volume); + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + if (endpoint->State != KeAudioEndpointReady) { + KSTATUS failure = endpoint->FailureStatus; + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KiAudioReleaseMutex(endpoint->ControlLock); + ObDereferenceObject(&endpoint->Header); + return KFAILED(failure) ? failure : NTLSTATUS_OBJECT_DEAD; + } + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->MasterRequestedVolume = volume; + endpoint->MasterTargetGain = softwareGain; + if (endpoint->HardwareRunning) { + endpoint->MasterGainStep = + (endpoint->MasterTargetGain - endpoint->MasterCurrentGain) / + (float)KE_AUDIO_GAIN_RAMP_FRAMES; + endpoint->MasterRampRemaining = KE_AUDIO_GAIN_RAMP_FRAMES; + } else { + endpoint->MasterCurrentGain = endpoint->MasterTargetGain; + endpoint->MasterGainStep = 0.0f; + endpoint->MasterRampRemaining = 0; + } + endpoint->MasterGainGeneration++; + endpoint->MasterMuted = muted ? 1u : 0u; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KiAudioReleaseMutex(endpoint->ControlLock); + + KeSignalEvent(endpoint->WorkEvent); + ObDereferenceObject(&endpoint->Header); + return NTLSTATUS_SUCCESS; +} + +// Copies one stream snapshot under its priority-inheriting mutex. +KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut) { + if (!stream || !infoOut) + return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioQueryStream"); + memset(infoOut, 0, sizeof(*infoOut)); + infoOut->State = stream->State; + infoOut->Format = stream->Format; + infoOut->CapacityFrames = KE_AUDIO_STREAM_RING_FRAMES; + infoOut->BufferedFrames = stream->BufferedFrames; + infoOut->WritableFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; + infoOut->FramesSubmitted = stream->FramesSubmitted; + infoOut->FramesConsumed = stream->FramesConsumed; + infoOut->FramesPresented = stream->FramesPresented; + infoOut->FramesDiscarded = stream->FramesDiscarded; + infoOut->OutputFramesMixed = stream->OutputFramesMixed; + infoOut->SilenceFramesInserted = stream->SilenceFramesInserted; + infoOut->Underruns = stream->Underruns; + infoOut->DrainedGeneration = stream->DrainedGeneration; + infoOut->DiscontinuityGeneration = stream->DiscontinuityGeneration; + infoOut->FailureStatus = stream->FailureStatus; + infoOut->Volume = stream->Volume; + infoOut->Pan = stream->Pan; + infoOut->Muted = stream->Muted; + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_SUCCESS; +} + +// Copies one endpoint snapshot by stable identifier. +static KSTATUS KiAudioSelfTestQueryEndpoint(uint64_t endpointId, + KE_AUDIO_ENDPOINT_INFO* infoOut) { + KE_AUDIO_ENDPOINT_INFO infos[KE_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + KSTATUS status = KeAudioQueryEndpoints(infos, KE_AUDIO_MAX_ENDPOINTS, &count); + if (KFAILED(status)) + return status; + for (uint32_t index = 0; index < count; index++) { + if (infos[index].EndpointId == endpointId) { + *infoOut = infos[index]; + return NTLSTATUS_SUCCESS; + } + } + return NTLSTATUS_NOT_FOUND; +} + +// Generates one bounded run of a ramped 440 Hz stereo sine wave. +static void KiAudioSelfTestGenerateTone(int16_t* frames, + uint32_t frameCount, + uint64_t firstFrame, + double* sineInOut, + double* cosineInOut) { + const uint64_t totalFrames = 3ull * KE_AUDIO_OUTPUT_SAMPLE_RATE; + const uint64_t rampFrames = 384ull; + const double amplitude = 8231.0; + const double sineStep = 0.057564026959567284; + const double cosineStep = 0.9983418166140283; + double sine = *sineInOut; + double cosine = *cosineInOut; + + for (uint32_t index = 0; index < frameCount; index++) { + uint64_t frame = firstFrame + index; + uint64_t framesFromEnd = totalFrames - 1ull - frame; + double envelope = 1.0; + if (frame < rampFrames) + envelope = (double)frame / (double)(rampFrames - 1ull); + if (framesFromEnd < rampFrames) { + double fadeOut = (double)framesFromEnd / (double)(rampFrames - 1ull); + if (fadeOut < envelope) + envelope = fadeOut; + } + + double scaled = sine * amplitude * envelope; + int32_t sample = scaled >= 0.0 ? (int32_t)(scaled + 0.5) + : (int32_t)(scaled - 0.5); + frames[index * 2u] = (int16_t)sample; + frames[index * 2u + 1u] = (int16_t)sample; + + double nextSine = sine * cosineStep + cosine * sineStep; + double nextCosine = cosine * cosineStep - sine * sineStep; + sine = nextSine; + cosine = nextCosine; + } + + *sineInOut = sine; + *cosineInOut = cosine; +} + +// Waits for one selftest stream condition with a finite boot-time deadline. +static KSTATUS KiAudioSelfTestWait(KE_AUDIO_STREAM* stream) { + EVENT* event = KeAudioStreamWaitEvent(stream); + if (!event) + return NTLSTATUS_OBJECT_DEAD; + uint32_t selected = 0; + return KeWaitForMultipleEvents(&event, + 1, + 0, + KE_AUDIO_SELFTEST_WAIT_MILLISECONDS, + &selected); +} + +// Writes one generated run while using the stream event for bounded backpressure. +static KSTATUS KiAudioSelfTestWriteFrames(KE_AUDIO_STREAM* stream, + const int16_t* frames, + uint32_t frameCount, + uint64_t expectedDiscontinuity) { + uint32_t offset = 0; + while (offset < frameCount) { + uint32_t written = 0; + KSTATUS status = KeAudioWriteStream(stream, + frames + offset * 2u, + frameCount - offset, + &written); + if (KSUCCESS(status) && written) { + offset += written; + continue; + } + if (status != NTLSTATUS_WOULD_BLOCK) + return KFAILED(status) ? status : NTLSTATUS_FAILURE; + + KE_AUDIO_STREAM_INFO info; + status = KeAudioQueryStream(stream, &info); + if (KFAILED(status)) + return status; + if (info.State == KeAudioStreamFailed) + return KFAILED(info.FailureStatus) ? info.FailureStatus : NTLSTATUS_DEVICE_ERROR; + if (info.DiscontinuityGeneration != expectedDiscontinuity) + return NTLSTATUS_DEVICE_ERROR; + + KE_AUDIO_STREAM_WAIT_CONDITION condition = { + .Mask = KE_AUDIO_WAIT_WRITABLE | KE_AUDIO_WAIT_FAILED | KE_AUDIO_WAIT_DISCONTINUITY, + .WritableFrames = 1, + .DiscontinuityGeneration = expectedDiscontinuity, + }; + status = KeAudioArmStreamWait(stream, &condition); + if (KFAILED(status)) + return status; + status = KiAudioSelfTestWait(stream); + if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Prints the common endpoint and stream counters for one selftest outcome. +static void KiAudioSelfTestPrintCounters(uint64_t endpointId, + const KE_AUDIO_ENDPOINT_INFO* endpointBefore, + const KE_AUDIO_ENDPOINT_INFO* endpointAfter, + const KE_AUDIO_STREAM_INFO* streamInfo, + uint64_t expectedDiscontinuity) { + uint64_t mixedPeriods = endpointAfter->PeriodsMixed - endpointBefore->PeriodsMixed; + uint64_t completedPeriods = endpointAfter->PeriodsCompleted - endpointBefore->PeriodsCompleted; + uint64_t interruptCount = endpointAfter->InterruptCount - endpointBefore->InterruptCount; + uint64_t endpointUnderruns = endpointAfter->Underruns - endpointBefore->Underruns; + uint64_t endpointDiscontinuities = endpointAfter->DiscontinuityGeneration - + endpointBefore->DiscontinuityGeneration; + uint64_t positionFrames = endpointAfter->HardwarePositionFrames - + endpointBefore->HardwarePositionFrames; + uint64_t streamDiscontinuities = streamInfo->DiscontinuityGeneration >= expectedDiscontinuity + ? streamInfo->DiscontinuityGeneration - expectedDiscontinuity + : 0; + + DbgPrint("AUDIO final selftest: endpoint=%llu submitted=%llu consumed=%llu presented=%llu\n", + (unsigned long long)endpointId, + (unsigned long long)streamInfo->FramesSubmitted, + (unsigned long long)streamInfo->FramesConsumed, + (unsigned long long)streamInfo->FramesPresented); + DbgPrint("AUDIO final selftest: periods mixed=%llu completed=%llu interrupts=%llu\n", + (unsigned long long)mixedPeriods, + (unsigned long long)completedPeriods, + (unsigned long long)interruptCount); + DbgPrint("AUDIO final selftest: underruns endpoint=%llu stream=%llu discontinuities endpoint=%llu stream=%llu\n", + (unsigned long long)endpointUnderruns, + (unsigned long long)streamInfo->Underruns, + (unsigned long long)endpointDiscontinuities, + (unsigned long long)streamDiscontinuities); + DbgPrint("AUDIO final selftest: position start=%llu end=%llu delta=%llu\n", + (unsigned long long)endpointBefore->HardwarePositionFrames, + (unsigned long long)endpointAfter->HardwarePositionFrames, + (unsigned long long)positionFrames); +} + +// Plays the final-path sine test through the selected hardware endpoint and mixer. +void KiRunAudioFinalPathSelfTest(void) { + const uint64_t totalFrames = 3ull * KE_AUDIO_OUTPUT_SAMPLE_RATE; + const uint32_t chunkFrames = 1024u; + const char* failurePhase = (const char*)0; + KSTATUS failureStatus = NTLSTATUS_SUCCESS; + uint64_t failureValue = 0; + uint64_t endpointId = KeAudioGetDefaultEndpointId(); + KE_AUDIO_ENDPOINT_INFO endpointBefore; + KE_AUDIO_ENDPOINT_INFO endpointAfter; + KE_AUDIO_STREAM_INFO startInfo; + KE_AUDIO_STREAM_INFO streamAfter; + KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)0; + int16_t* chunk = (int16_t*)0; + uint64_t expectedDiscontinuity = 0; + int haveEndpointBefore = 0; + int haveEndpointAfter = 0; + int haveStreamAfter = 0; + + memset(&endpointBefore, 0, sizeof(endpointBefore)); + memset(&endpointAfter, 0, sizeof(endpointAfter)); + memset(&startInfo, 0, sizeof(startInfo)); + memset(&streamAfter, 0, sizeof(streamAfter)); + + if (!endpointId) { + DbgPrint("AUDIO final selftest: SKIP no default endpoint\n"); + return; + } + + KSTATUS status = KiAudioSelfTestQueryEndpoint(endpointId, &endpointBefore); + if (KFAILED(status) || !endpointBefore.Available || + endpointBefore.State != KeAudioEndpointReady) { + failurePhase = "endpoint-ready"; + failureStatus = KFAILED(status) ? status : NTLSTATUS_DEVICE_ERROR; + goto cleanup; + } + haveEndpointBefore = 1; + if (endpointBefore.HardwareFormat.SampleRate != KE_AUDIO_OUTPUT_SAMPLE_RATE || + endpointBefore.HardwareFormat.ChannelCount != KE_AUDIO_OUTPUT_CHANNELS || + endpointBefore.HardwareFormat.SampleFormat != IoAudioSampleS16) { + failurePhase = "endpoint-format"; + failureStatus = NTLSTATUS_NOT_SUPPORTED; + failureValue = endpointBefore.HardwareFormat.SampleFormat; + goto cleanup; + } + + DbgPrint("AUDIO final selftest: begin endpoint=%llu kind=%u format=48000/S16/stereo\n", + (unsigned long long)endpointId, + (unsigned)endpointBefore.Kind); + + IO_AUDIO_FORMAT format = { + .SampleRate = KE_AUDIO_OUTPUT_SAMPLE_RATE, + .ChannelCount = KE_AUDIO_OUTPUT_CHANNELS, + .SampleFormat = IoAudioSampleS16, + }; + status = KeAudioCreateStream(endpointId, &format, &stream); + if (KFAILED(status)) { + failurePhase = "stream-create"; + failureStatus = status; + goto cleanup; + } + + chunk = (int16_t*)MmAllocatePoolTag( + (size_t)chunkFrames * KE_AUDIO_OUTPUT_BYTES_PER_FRAME, + KE_AUDIO_SELFTEST_TAG); + if (!chunk) { + failurePhase = "tone-allocation"; + failureStatus = NTLSTATUS_NO_MEMORY; + goto cleanup; + } + + double sine = 0.0; + double cosine = 1.0; + uint64_t submitted = 0; + while (submitted < KE_AUDIO_STREAM_RING_FRAMES) { + uint32_t count = (uint32_t)(KE_AUDIO_STREAM_RING_FRAMES - submitted); + if (count > chunkFrames) + count = chunkFrames; + KiAudioSelfTestGenerateTone(chunk, count, submitted, &sine, &cosine); + uint32_t written = 0; + status = KeAudioWriteStream(stream, chunk, count, &written); + if (KFAILED(status) || written != count) { + failurePhase = "initial-prefill"; + failureStatus = KFAILED(status) ? status : NTLSTATUS_FAILURE; + failureValue = written; + goto cleanup; + } + submitted += written; + } + + status = KeAudioStartStream(stream); + if (KFAILED(status)) { + failurePhase = "stream-start"; + failureStatus = status; + goto cleanup; + } + + status = KeAudioQueryStream(stream, &startInfo); + if (KFAILED(status)) { + failurePhase = "start-query"; + failureStatus = status; + goto cleanup; + } + expectedDiscontinuity = startInfo.DiscontinuityGeneration; + + while (submitted < totalFrames) { + uint64_t remaining = totalFrames - submitted; + uint32_t count = remaining < chunkFrames ? (uint32_t)remaining : chunkFrames; + KiAudioSelfTestGenerateTone(chunk, count, submitted, &sine, &cosine); + status = KiAudioSelfTestWriteFrames(stream, + chunk, + count, + expectedDiscontinuity); + if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) { + failurePhase = "stream-write"; + failureStatus = status; + failureValue = submitted; + goto cleanup; + } + submitted += count; + } + + status = KeAudioDrainStream(stream); + if (KFAILED(status)) { + failurePhase = "stream-drain"; + failureStatus = status; + goto cleanup; + } + KE_AUDIO_STREAM_WAIT_CONDITION drainCondition = { + .Mask = KE_AUDIO_WAIT_DRAINED | KE_AUDIO_WAIT_FAILED | KE_AUDIO_WAIT_DISCONTINUITY, + .DrainedGeneration = startInfo.DrainedGeneration, + .DiscontinuityGeneration = expectedDiscontinuity, + }; + status = KeAudioArmStreamWait(stream, &drainCondition); + if (KFAILED(status)) { + failurePhase = "drain-arm"; + failureStatus = status; + goto cleanup; + } + status = KiAudioSelfTestWait(stream); + if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) { + failurePhase = "drain-wait"; + failureStatus = status; + goto cleanup; + } + + status = KeAudioQueryStream(stream, &streamAfter); + if (KFAILED(status)) { + failurePhase = "final-stream-query"; + failureStatus = status; + goto cleanup; + } + haveStreamAfter = 1; + if (streamAfter.State != KeAudioStreamStopped || + streamAfter.DrainedGeneration != startInfo.DrainedGeneration + 1u || + streamAfter.BufferedFrames != 0 || streamAfter.FramesSubmitted != totalFrames || + streamAfter.FramesConsumed != totalFrames || streamAfter.FramesPresented != totalFrames || + streamAfter.FramesDiscarded != 0 || streamAfter.Underruns != 0 || + streamAfter.DiscontinuityGeneration != expectedDiscontinuity) { + failurePhase = "final-stream-counters"; + failureStatus = NTLSTATUS_DEVICE_ERROR; + failureValue = streamAfter.FramesPresented; + goto cleanup; + } + + status = KiAudioSelfTestQueryEndpoint(endpointId, &endpointAfter); + if (KFAILED(status)) { + failurePhase = "final-endpoint-query"; + failureStatus = status; + goto cleanup; + } + haveEndpointAfter = 1; + uint64_t mixedPeriods = endpointAfter.PeriodsMixed - endpointBefore.PeriodsMixed; + uint64_t completedPeriods = endpointAfter.PeriodsCompleted - endpointBefore.PeriodsCompleted; + uint64_t interruptCount = endpointAfter.InterruptCount - endpointBefore.InterruptCount; + uint64_t endpointUnderruns = endpointAfter.Underruns - endpointBefore.Underruns; + uint64_t endpointDiscontinuities = endpointAfter.DiscontinuityGeneration - + endpointBefore.DiscontinuityGeneration; + uint64_t positionFrames = endpointAfter.HardwarePositionFrames - + endpointBefore.HardwarePositionFrames; + if (!mixedPeriods || !completedPeriods || !interruptCount || + endpointUnderruns || endpointDiscontinuities || + positionFrames + endpointAfter.FramesPerPeriod < totalFrames) { + failurePhase = "final-endpoint-counters"; + failureStatus = NTLSTATUS_DEVICE_ERROR; + failureValue = positionFrames; + goto cleanup; + } + +cleanup: + if (stream && !haveStreamAfter && KSUCCESS(KeAudioQueryStream(stream, &streamAfter))) + haveStreamAfter = 1; + if (haveEndpointBefore && !haveEndpointAfter && + KSUCCESS(KiAudioSelfTestQueryEndpoint(endpointId, &endpointAfter))) + haveEndpointAfter = 1; + if (haveEndpointBefore && haveEndpointAfter && haveStreamAfter) + KiAudioSelfTestPrintCounters(endpointId, + &endpointBefore, + &endpointAfter, + &streamAfter, + expectedDiscontinuity); + + if (stream) { + KE_AUDIO_STREAM_INFO cleanupInfo; + if (KSUCCESS(KeAudioQueryStream(stream, &cleanupInfo)) && + cleanupInfo.State != KeAudioStreamStopped && + cleanupInfo.State != KeAudioStreamFailed && + cleanupInfo.State != KeAudioStreamClosed) + (void)KeAudioStopStream(stream); + } + if (chunk) + MmFreePool(chunk); + if (stream) + ObDereferenceObject(&stream->Header); + + if (failurePhase) { + DbgPrint("AUDIO final selftest: FAIL phase=%s status=0x%x value=%llu\n", + failurePhase, + failureStatus, + (unsigned long long)failureValue); + return; + } + DbgPrint("AUDIO final selftest: PASS\n"); +} diff --git a/code/kernel/ke_cpu.c b/code/kernel/ke_cpu.c index 36e58e9..441d2b1 100644 --- a/code/kernel/ke_cpu.c +++ b/code/kernel/ke_cpu.c @@ -4,19 +4,148 @@ #include #include #include +#include +#include #define BOOTSTRAP_CPU_STACK_SIZE 0x4000u #define IA32_GS_BASE 0xC0000101u #define IA32_KERNEL_GS_BASE 0xC0000102u +#define IA32_MTRRCAP 0x000000FEu +#define IA32_MTRR_PHYSBASE0 0x00000200u +#define IA32_MTRR_PHYSMASK0 0x00000201u +#define IA32_PAT 0x00000277u +#define IA32_MTRR_DEF_TYPE 0x000002FFu +#define CPUID_FEATURE_MTRR (1u << 12) +#define CPUID_FEATURE_PAT (1u << 16) +#define MTRR_CAP_VARIABLE_COUNT 0x000000FFull +#define MTRR_MASK_VALID (1ull << 11) +#define MTRR_DEF_TYPE_MASK 0x000000FFull +#define MTRR_DEF_FIXED_ENABLE (1ull << 10) +#define MTRR_DEF_ENABLE (1ull << 11) +#define MEMORY_TYPE_UC 0x00ull +#define MEMORY_TYPE_WC 0x01ull +#define MEMORY_TYPE_WB 0x06ull +#define CR0_CACHE_DISABLE (1ull << 30) +#define CR0_NOT_WRITE_THROUGH (1ull << 29) +#define CR4_PAGE_GLOBAL_ENABLE (1ull << 7) +#define CR4_PCID_ENABLE (1ull << 17) +#define PAT_ENTRY(type, index) ((uint64_t)(type) << ((index) * 8u)) +#define CANONICAL_PAT (PAT_ENTRY(MEMORY_TYPE_WB, 0) | \ + PAT_ENTRY(MEMORY_TYPE_WC, 1) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 2) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 3) | \ + PAT_ENTRY(MEMORY_TYPE_WB, 4) | \ + PAT_ENTRY(MEMORY_TYPE_WC, 5) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 6) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 7)) KCPU gCpus[KE_MAX_CPUS]; uint32_t gCpuCount = 0; static uint8_t gBootstrapCpuStack[BOOTSTRAP_CPU_STACK_SIZE]; static volatile long gKernelGsReady = 0; +static volatile long gWriteCombiningAvailable = 0; extern KCPU* KeReadCurrentCpuFromGs(void); +// Returns whether this CPU exposes the memory-type mechanisms required by the kernel policy. +static bool KiMemoryTypeFeaturesAvailable(void) { + int regs[4]; + __cpuid(regs, 1); + uint32_t features = (uint32_t)regs[3]; + return (features & (CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT)) == + (CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT); +} + +// Flushes all translations before memory-type state changes and leaves global pages disabled. +static uintptr_t KiDisableGlobalTranslations(void) { + uintptr_t cr4 = __readcr4(); + + // nt-lite does not enable PCID. Refuse inherited PCID state rather than + // pretending that a CR3 reload flushed every tagged translation. + if (cr4 & CR4_PCID_ENABLE) + HalStopProcessor(); + + if (cr4 & CR4_PAGE_GLOBAL_ENABLE) + __writecr4(cr4 & ~CR4_PAGE_GLOBAL_ENABLE); + else + __writecr3(__readcr3()); + + return cr4; +} + +// Returns whether this CPU accepted the complete canonical PAT and MTRR policy. +static bool KiMemoryTypePolicyInstalled(uint32_t variableCount) { + if (__readmsr(IA32_PAT) != CANONICAL_PAT) + return false; + + uint64_t expectedDefault = MTRR_DEF_ENABLE | MEMORY_TYPE_WB; + uint64_t defaultType = __readmsr(IA32_MTRR_DEF_TYPE); + uint64_t relevantDefaultBits = MTRR_DEF_TYPE_MASK | MTRR_DEF_FIXED_ENABLE | MTRR_DEF_ENABLE; + if ((defaultType & relevantDefaultBits) != expectedDefault) + return false; + + for (uint32_t i = 0; i < variableCount; i++) { + uint64_t base = __readmsr(IA32_MTRR_PHYSBASE0 + i * 2u); + uint64_t mask = __readmsr(IA32_MTRR_PHYSMASK0 + i * 2u); + if (base != 0 || (mask & MTRR_MASK_VALID) != 0) + return false; + } + + return true; +} + +// Programs the kernel-owned PAT policy over a uniform write-back MTRR baseline. +void KeInitializeMemoryTypes(void) { + if (!KiMemoryTypeFeaturesAvailable()) + HalStopProcessor(); + + uint32_t variableCount = (uint32_t)(__readmsr(IA32_MTRRCAP) & MTRR_CAP_VARIABLE_COUNT); + if (KiMemoryTypePolicyInstalled(variableCount)) { + _InterlockedExchange(&gWriteCombiningAvailable, 1); + return; + } + + uintptr_t interruptFlags = HalDisableInterrupts(); + uintptr_t cr0 = __readcr0(); + + // Every CPU runs this before entering normal kernel execution. The BSP + // installs the policy while it is alone, and each AP installs it before + // publishing its started state. + __writecr0((cr0 | CR0_CACHE_DISABLE) & ~CR0_NOT_WRITE_THROUGH); + __wbinvd(); + uintptr_t cr4 = KiDisableGlobalTranslations(); + + // MTRRs provide only a uniform WB baseline. PAT is the sole owner of the + // page-granular WB, WC, and UC policy selected by kernel mappings. + __writemsr(IA32_MTRR_DEF_TYPE, 0); + for (uint32_t i = 0; i < variableCount; i++) { + __writemsr(IA32_MTRR_PHYSBASE0 + i * 2u, 0); + __writemsr(IA32_MTRR_PHYSMASK0 + i * 2u, 0); + } + + __writemsr(IA32_PAT, CANONICAL_PAT); + __writemsr(IA32_MTRR_DEF_TYPE, MTRR_DEF_ENABLE | MEMORY_TYPE_WB); + + __wbinvd(); + __writecr3(__readcr3()); + __writecr0(cr0); + if (cr4 & CR4_PAGE_GLOBAL_ENABLE) + __writecr4(cr4); + + bool installed = KiMemoryTypePolicyInstalled(variableCount); + HalRestoreInterrupts(interruptFlags); + if (!installed) + HalStopProcessor(); + + _InterlockedExchange(&gWriteCombiningAvailable, 1); +} + +// Reports whether PAT write-combining is available on the current platform. +bool KeWriteCombiningAvailable(void) { + return _InterlockedCompareExchange(&gWriteCombiningAvailable, 0, 0) != 0; +} + // Loads kernel Gs Base. void KeLoadKernelGsBase(KCPU* cpu) { if (!cpu) @@ -69,6 +198,7 @@ void KeInitializeBootstrapCpu(void) { cpu->Tss.RSP0 = cpu->KernelStackTop; gCpuCount = 1; + KeInitializeMemoryTypes(); KeLoadKernelGsBase(cpu); KeSetUserGsBase(0); } diff --git a/code/kernel/ke_debug.c b/code/kernel/ke_debug.c index 239ebcc..8359cc7 100644 --- a/code/kernel/ke_debug.c +++ b/code/kernel/ke_debug.c @@ -220,7 +220,29 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) { line, length); } -// Flushes debug Log To File. +// Writes one debug-log segment and rejects successful zero-progress writes. +static KSTATUS KiDebugLogWriteAll(FILE_OBJECT* file, + const char* data, + size_t length, + size_t* totalWritten) { + if (!file || (!data && length) || !totalWritten) + return NTLSTATUS_INVALID_PARAMETER; + + while (length != 0) { + size_t written = 0; + KSTATUS st = IoWriteFile(file, data, length, &written); + if (!KSUCCESS(st)) + return st; + if (written == 0) + return NTLSTATUS_DEVICE_ERROR; + data += written; + length -= written; + *totalWritten += written; + } + return NTLSTATUS_SUCCESS; +} + +// Persists the in-memory debug ring and flushes it through the storage stack. __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0) return NTLSTATUS_INVALID_PARAMETER; @@ -237,11 +259,6 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0; KeReleaseSpinlock(&gDebugPrintLock, flags); - if (used == 0) { - _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0); - return NTLSTATUS_SUCCESS; - } - (void)IoDeleteFile(path); FILE_OBJECT* file = (FILE_OBJECT*)0; @@ -251,23 +268,32 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { return st; } - size_t written = 0; - size_t totalWritten = 0; - size_t first = gDebugLogSize - start; - if (first > used) first = used; - st = IoWriteFile(file, gDebugLogBuffer + start, first, &written); - totalWritten += written; + // Shutdown logging must not depend on deferred file-cache writeback. FAT + // still updates the directory entry and cluster chain through its normal + // write path, while IoFlushFile pushes metadata and the device cache. + file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND; - if (KSUCCESS(st) && used > first) { - written = 0; - st = IoWriteFile(file, gDebugLogBuffer, used - first, &written); - totalWritten += written; + size_t totalWritten = 0; + if (used != 0) { + size_t first = gDebugLogSize - start; + if (first > used) + first = used; + + st = KiDebugLogWriteAll(file, gDebugLogBuffer + start, first, &totalWritten); + if (KSUCCESS(st) && used > first) { + st = KiDebugLogWriteAll(file, gDebugLogBuffer, + used - first, &totalWritten); + } } - IoCloseFile(file); + if (KSUCCESS(st) && totalWritten != used) + st = NTLSTATUS_DEVICE_ERROR; + if (KSUCCESS(st)) + st = IoFlushFile(file); + IoCloseFile(file); _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0); - return (KSUCCESS(st) && totalWritten == used) ? NTLSTATUS_SUCCESS : st; + return st; } // Attempts to persist the debug log during a panic when the filesystem is usable. @@ -863,6 +889,18 @@ __declspec(dllexport) KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) { if (!KeFaultFrameFromUser(frame)) KiPanicKernelFault(proc, thread, frame); + DbgPrint("USERFAULT: pid=%u tid=%u vector=%llu rip=0x%016llX rsp=0x%016llX cr2=0x%016llX error=0x%016llX rcx=0x%016llX rbx=0x%016llX rax=0x%016llX\n", + proc ? proc->ProcessId : 0, + thread ? thread->ThreadId : 0, + frame->Vector, + (uint64_t)frame->Rip, + (uint64_t)frame->Rsp, + (uint64_t)frame->Cr2, + (uint64_t)frame->ErrorCode, + (uint64_t)frame->Rcx, + (uint64_t)frame->Rbx, + (uint64_t)frame->Rax); + KSTATUS delivered = KeDeliverUserException(frame); if (KSUCCESS(delivered)) return NTLSTATUS_SUCCESS; diff --git a/code/kernel/ke_display.c b/code/kernel/ke_display.c index d8ddf0d..800b87e 100644 --- a/code/kernel/ke_display.c +++ b/code/kernel/ke_display.c @@ -19,6 +19,7 @@ #include #include #include +#include #define DISPLAY_DEFAULT_FG 0x00CCCCCCu #define DISPLAY_DEFAULT_BG 0x00012456u @@ -44,6 +45,15 @@ typedef struct DISPLAY_OUTPUT_GLYPH { uint32_t Background; } DISPLAY_OUTPUT_GLYPH; +typedef struct DISPLAY_OUTPUT_DRAW { + char Character; + uint32_t Foreground; + uint32_t Background; + int Column; + int Row; + size_t ScrollCount; +} DISPLAY_OUTPUT_DRAW; + typedef struct BOOTFB_CONTEXT { const HAL_API* Hal; uint32_t* Framebuffer; @@ -59,6 +69,7 @@ static DISPLAY_DEVICE* gDisplayList; static DISPLAY_DEVICE* gPrimaryDisplay; static SPINLOCK gDisplayLock = SPINLOCK_INIT; static MUTEX* gDisplayProviderMutex; +static MUTEX* gDisplayOutputMutex; static KWORK_ITEM gDisplayCursorWorkItem; static volatile long gDisplayCursorWorkInitialized; static bool gDisplayTransitionInProgress; @@ -104,8 +115,10 @@ static KWORK_ITEM gDisplayOutputWorkItem; static volatile long gDisplayOutputWorkInitialized; static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg); +static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count); static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode); -static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels); +static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); +static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color); static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h); static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride); @@ -119,6 +132,7 @@ static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) static const DISPLAY_DEVICE_OPS gBootFbOps = { .GetMode = BootFbGetMode, .Blit = BootFbBlit, + .PresentPitched = BootFbPresentPitched, .FillRect = BootFbFillRect, .CopyRect = BootFbCopyRect, .ReadRect = BootFbReadRect, @@ -192,10 +206,15 @@ static KSTATUS KiDisplayAcquireProviderMutex(bool* acquired) { if (KeGetDebugFlags() & KE_DEBUG_DISPLAY_ASSERTS) KeAssertIrql(PASSIVE_LEVEL); - KSTATUS s = KeAcquireMutex(gDisplayProviderMutex); - if (KSUCCESS(s) && acquired) + KSTATUS s = KeAcquireMutex(gDisplayProviderMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s)) + return s; + + if (acquired) *acquired = true; - return s; + + // Provider operations still own the mutex after an abandoned acquisition. + return NTLSTATUS_SUCCESS; } static void KiDisplayReleaseProviderMutex(bool acquired) { @@ -449,6 +468,34 @@ static bool KiDisplayConsoleGridFromMode(const DISPLAY_MODE* mode, int* outCols, return true; } +// Returns whether two dirty rectangles overlap or share an edge. +static bool KiDisplayDirtyRectsMergeable(const DISPLAY_RECT* a, const DISPLAY_RECT* b) { + if (!a || !b) + return false; + + int64_t aRight = (int64_t)a->X + a->W; + int64_t aBottom = (int64_t)a->Y + a->H; + int64_t bRight = (int64_t)b->X + b->W; + int64_t bBottom = (int64_t)b->Y + b->H; + return (int64_t)a->X <= bRight && (int64_t)b->X <= aRight && + (int64_t)a->Y <= bBottom && (int64_t)b->Y <= aBottom; +} + +// Unions two clipped dirty rectangles. +static DISPLAY_RECT KiDisplayUnionDirtyRects(const DISPLAY_RECT* a, const DISPLAY_RECT* b) { + int64_t left = a->X < b->X ? a->X : b->X; + int64_t top = a->Y < b->Y ? a->Y : b->Y; + int64_t aRight = (int64_t)a->X + a->W; + int64_t bRight = (int64_t)b->X + b->W; + int64_t aBottom = (int64_t)a->Y + a->H; + int64_t bBottom = (int64_t)b->Y + b->H; + int64_t right = aRight > bRight ? aRight : bRight; + int64_t bottom = aBottom > bBottom ? aBottom : bBottom; + DISPLAY_RECT result = { (int32_t)left, (int32_t)top, + (int32_t)(right - left), (int32_t)(bottom - top) }; + return result; +} + static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { if (!mode || gDirtyAll || w <= 0 || h <= 0) return; @@ -457,6 +504,19 @@ static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int if (!KiDisplayClipRect(mode, &rect)) return; + for (size_t i = 0; i < gDirtyRectCount;) { + if (!KiDisplayDirtyRectsMergeable(&rect, &gDirtyRects[i])) { + i++; + continue; + } + + rect = KiDisplayUnionDirtyRects(&rect, &gDirtyRects[i]); + gDirtyRectCount--; + if (i != gDirtyRectCount) + gDirtyRects[i] = gDirtyRects[gDirtyRectCount]; + i = 0; + } + if (rect.X == 0 && rect.Y == 0 && rect.W == (int32_t)mode->Width && rect.H == (int32_t)mode->Height) { KiDisplayDirtyAll(); @@ -478,6 +538,24 @@ static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, in KiDisplayAddDirtyForModeLocked(mode, x, y, w, h); } +static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { + if (!mode || w <= 0 || h <= 0) + return; + + DISPLAY_RECT rect = { x, y, w, h }; + if (!KiDisplayClipRect(mode, &rect)) + return; + + uint64_t bytes = (uint64_t)(uint32_t)rect.W * + (uint64_t)(uint32_t)rect.H * sizeof(uint32_t); + gDisplayStats.BlitCount++; + gDisplayStats.PresentCount++; + gDisplayStats.PresentRectCount++; + gDisplayStats.DirectPresentCount++; + gDisplayStats.ScanoutBytes += bytes; + gDisplayStats.BytesCopied += bytes; +} + static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { gDisplayStats.FillRectCount++; gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t); @@ -660,7 +738,7 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device, uint32_t glyphPixels[FONT_W * FONT_H]; KiDisplayBuildGlyphPixels(c, fg, bg, glyphPixels); - KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels); + KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels, FONT_W * sizeof(uint32_t)); if (KSUCCESS(s)) { uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); if (device == gPrimaryDisplay) @@ -670,20 +748,26 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device, return s; } -static KSTATUS KiDisplayDrawBlitSnapshot(DISPLAY_DEVICE* device, - const DISPLAY_DEVICE_OPS* ops, - const DISPLAY_MODE* mode, - int x, - int y, - int w, - int h, - const uint32_t* pixels) { - if (!pixels || w <= 0 || h <= 0) +// Draws a pitched source rectangle and records it as one display operation. +static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + const DISPLAY_MODE* mode, + int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; - if (!device || !ops || !ops->Blit || !mode) + if (!device || !ops || !mode) return NTLSTATUS_NOT_SUPPORTED; - KSTATUS s = ops->Blit(device, x, y, w, h, pixels); + if (!ops->Blit) + return NTLSTATUS_NOT_SUPPORTED; + + KSTATUS s = ops->Blit(device, x, y, w, h, pixels, pitchBytes); + if (KSUCCESS(s)) { uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); if (device == gPrimaryDisplay) @@ -867,7 +951,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3 if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Blit) return NTLSTATUS_NOT_SUPPORTED; - KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels); + KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels, (uint32_t)w * sizeof(uint32_t)); if (KSUCCESS(s)) { gDisplayStats.BlitCount++; gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t); @@ -1043,11 +1127,18 @@ static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uin } // Drains buffered output through the active display provider. -static void KiDisplayOutputWorker(void* context) { - (void)context; +static KSTATUS KiDisplayDrainOutput(void) { DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT]; + bool mutexHeld = false; - for (;;) { + if (gDisplayOutputMutex) { + KSTATUS acquireStatus = KeAcquireMutex(gDisplayOutputMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(acquireStatus)) + return acquireStatus; + mutexHeld = true; + } + + while (true) { size_t count = 0; uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock); while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) { @@ -1059,12 +1150,28 @@ static void KiDisplayOutputWorker(void* context) { if (count == 0) break; - for (size_t i = 0; i < count; i++) { - (void)KiDisplayPrintCharInternal(glyphs[i].Character, - glyphs[i].Foreground, - glyphs[i].Background); + KSTATUS drawStatus = KiDisplayPrintGlyphsInternal(glyphs, count); + if (!KSUCCESS(drawStatus)) { + if (mutexHeld) + KeReleaseMutex(gDisplayOutputMutex); + return drawStatus; } } + + if (mutexHeld) + KeReleaseMutex(gDisplayOutputMutex); + return NTLSTATUS_SUCCESS; +} + +// Drains buffered output from the passive work queue. +static void KiDisplayOutputWorker(void* context) { + (void)context; + (void)KiDisplayDrainOutput(); +} + +// Completes buffered console output before a caller overlays console pixels. +KSTATUS KeDisplayFlushConsoleOutput(void) { + return KiDisplayDrainOutput(); } // Queues the runtime display output worker when passive work is available. @@ -1157,6 +1264,8 @@ void KeDisplayInitialize(const HAL_API* hal) { _InterlockedExchange(&gDisplayClockInitialized, 1); if (!gDisplayProviderMutex) (void)KeMutexCreate(&gDisplayProviderMutex); + if (!gDisplayOutputMutex) + (void)KeMutexCreate(&gDisplayOutputMutex); KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0); _InterlockedExchange(&gDisplayCursorWorkInitialized, 1); KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0); @@ -1212,8 +1321,9 @@ void KeDisplayInitialize(const HAL_API* hal) { (void)KeDisplaySetPrimary(&gBootFbDevice); DISPLAY_MODE mode; if (KSUCCESS(KeDisplayGetMode(&mode))) { - DbgPrint("Display: bootfb primary %ux%u stride=%u\n", - mode.Width, mode.Height, mode.Stride); + DbgPrint("Display: bootfb primary %ux%u stride=%u cache=%s\n", + mode.Width, mode.Height, mode.Stride, + KeWriteCombiningAvailable() ? "WC" : "WT"); } // The registration messages above were routed through the HAL boot @@ -1277,7 +1387,7 @@ DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) { // Signals every display clock once for a provider-owned frame generation. static void KiDisplaySignalFrameGeneration(uint64_t generation) { - for (;;) { + while (true) { EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH]; uint32_t count = 0; @@ -1340,7 +1450,7 @@ DISPLAY_API void KeDisplayTick(void) { return; uint64_t now = KiDisplayClockReadSchedulerTick(); - for (;;) { + while (true) { EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH]; uint32_t count = 0; @@ -1424,7 +1534,8 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE s = oldDevice->Ops->ReadRect(oldDevice, 0, (int)y, (int)newMode.Width, 1, row, newMode.Width); if (!KSUCCESS(s)) break; - s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row); + s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row, + newMode.Width * sizeof(uint32_t)); if (!KSUCCESS(s)) break; } @@ -1610,6 +1721,8 @@ DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) { for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) count++; info.DeviceCount = count; + if (KeWriteCombiningAvailable()) + info.Flags |= DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE; info.ManagerStats = gDisplayStats; DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0; @@ -1660,7 +1773,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); if (KSUCCESS(s)) { (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); - s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y, w, h, pixels); + s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, + (uint32_t)w * sizeof(uint32_t)); s = KiDisplayFinishProviderDraw(device, ops, caps, s); } @@ -1668,8 +1782,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi return s; } -DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch) { - if (!source || !scratch || w <= 0 || h <= 0) +DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) { + if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; bool providerHeld = false; @@ -1684,14 +1798,7 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); if (KSUCCESS(s)) { (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); - for (int row = 0; row < h; ++row) { - s = source(context, row, scratch, (size_t)w); - if (!KSUCCESS(s)) - break; - s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y + row, w, 1, scratch); - if (!KSUCCESS(s)) - break; - } + s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, pitchBytes); s = KiDisplayFinishProviderDraw(device, ops, caps, s); } @@ -1699,6 +1806,47 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW return s; } +DISPLAY_API KSTATUS KeDisplayPresentPitched(int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t)) + return NTLSTATUS_INVALID_PARAMETER; + + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (KSUCCESS(s)) { + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + if (ops->PresentPitched) { + s = ops->PresentPitched(device, x, y, w, h, pixels, pitchBytes); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) + KiDisplayRecordDirectPresentLocked(&mode, x, y, w, h); + KeReleaseSpinlock(&gDisplayLock, flags); + } + KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps); + } else { + s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, + x, y, w, h, pixels, pitchBytes); + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + } + } + + KiDisplayReleaseProviderMutex(providerHeld); + return s; +} + DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) { bool providerHeld = false; KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); @@ -1835,18 +1983,28 @@ DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t return s; } -static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { +// Renders one or more console glyphs as one provider transaction and one present. +static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count) { + if (!glyphs && count != 0) + return NTLSTATUS_INVALID_PARAMETER; + if (count == 0) + return NTLSTATUS_SUCCESS; + if (count > DISPLAY_OUTPUT_DRAIN_COUNT) + return NTLSTATUS_INVALID_PARAMETER; + if (KiDisplayOutputSuspended()) { - if (!KeCanWait()) - return KiDisplayFallbackPrintChar(c, fg, bg); + if (!KeCanWait()) { + for (size_t i = 0; i < count; i++) + (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background); + } return NTLSTATUS_SUCCESS; } - // Once the scheduler/work-queue era begins, display provider callbacks are - // PASSIVE_LEVEL-only. The HAL output endpoint buffers normal text so this - // fallback is reserved for explicit display calls made at elevated IRQL. - if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) - return KiDisplayFallbackPrintChar(c, fg, bg); + if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) { + for (size_t i = 0; i < count; i++) + (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background); + return NTLSTATUS_SUCCESS; + } bool providerHeld = false; KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); @@ -1861,8 +2019,10 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { if (!KSUCCESS(s)) { KiDisplayReleaseProviderMutex(providerHeld); const HAL_API* hal = gDisplayHal; - if (hal && hal->Screen.PrintChar) - hal->Screen.PrintChar(c, fg, bg); + if (hal && hal->Screen.PrintChar) { + for (size_t i = 0; i < count; i++) + hal->Screen.PrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background); + } return s; } @@ -1875,71 +2035,94 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); - bool drawChar = false; - bool needScroll = false; - int drawCol = 0; - int drawRow = 0; + DISPLAY_OUTPUT_DRAW draws[DISPLAY_OUTPUT_DRAIN_COUNT]; + size_t drawCount = 0; + size_t scrollCount = 0; - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - if (c == '\n') { - gConsoleX = 0; - gConsoleY++; - } else if (c == '\r') { - gConsoleX = 0; - } else if (c == '\b') { - if (gConsoleX > 0) { - gConsoleX--; - } else if (gConsoleY > 0) { - gConsoleY--; - gConsoleX = cols - 1; - } - } else { - drawChar = true; - drawCol = gConsoleX; - drawRow = gConsoleY; - if (++gConsoleX >= cols) { + for (size_t glyphIndex = 0; glyphIndex < count; glyphIndex++) { + char c = glyphs[glyphIndex].Character; + int drawCol = 0; + int drawRow = 0; + bool drawChar = false; + bool needScroll = false; + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (c == '\n') { gConsoleX = 0; gConsoleY++; + } else if (c == '\r') { + gConsoleX = 0; + } else if (c == '\b') { + if (gConsoleX > 0) { + gConsoleX--; + } else if (gConsoleY > 0) { + gConsoleY--; + gConsoleX = cols - 1; + } + } else { + drawChar = true; + drawCol = gConsoleX; + drawRow = gConsoleY; + if (++gConsoleX >= cols) { + gConsoleX = 0; + gConsoleY++; + } + + if (drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) { + size_t cell = KiDisplayConsoleIndex(drawCol, drawRow); + gConsoleCells[cell] = c; + gConsoleAttrs[cell] = (uint8_t)gConsoleAttribute; + } + } + + if (gConsoleY >= rows) { + KiDisplayConsoleScrollCellsLocked(cols, rows, 1); + gConsoleY = rows - 1; + if (gConsoleX < 0) gConsoleX = 0; + if (gConsoleX >= cols) gConsoleX = cols - 1; + needScroll = true; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + if (drawChar) { + DISPLAY_OUTPUT_DRAW* draw = &draws[drawCount++]; + draw->Character = c; + draw->Foreground = glyphs[glyphIndex].Foreground; + draw->Background = glyphs[glyphIndex].Background; + draw->Column = drawCol; + draw->Row = drawRow; + draw->ScrollCount = scrollCount; + } + if (needScroll) + scrollCount++; + } + + if (scrollCount != 0) { + uint64_t scrollPixels64 = (uint64_t)scrollCount * FONT_H; + int scrollPixels = scrollPixels64 >= mode.Height ? (int)mode.Height : (int)scrollPixels64; + if ((uint32_t)scrollPixels >= mode.Height) { + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0, + (int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG); + } else { + s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, scrollPixels, + (int)mode.Width, (int)(mode.Height - (uint32_t)scrollPixels)); + if (KSUCCESS(s)) { + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, + (int)(mode.Height - (uint32_t)scrollPixels), + (int)mode.Width, scrollPixels, DISPLAY_DEFAULT_BG); + } } } - if (gConsoleY >= rows) - needScroll = true; - KeReleaseSpinlock(&gDisplayLock, flags); - - if (drawChar) { - s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, drawCol * FONT_W, drawRow * FONT_H, fg, bg); - if (KSUCCESS(s)) { - flags = KeAcquireSpinlock(&gDisplayLock); - if (device == gPrimaryDisplay && drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) { - size_t i = KiDisplayConsoleIndex(drawCol, drawRow); - gConsoleCells[i] = c; - gConsoleAttrs[i] = (uint8_t)gConsoleAttribute; - } - KeReleaseSpinlock(&gDisplayLock, flags); - } - } else { - s = NTLSTATUS_SUCCESS; - } - - if (KSUCCESS(s) && needScroll) { - s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, FONT_H, - (int)mode.Width, (int)(mode.Height - FONT_H)); - if (KSUCCESS(s)) { - s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, - (int)(mode.Height - FONT_H), - (int)mode.Width, FONT_H, DISPLAY_DEFAULT_BG); - } - if (KSUCCESS(s)) { - flags = KeAcquireSpinlock(&gDisplayLock); - if (device == gPrimaryDisplay) { - KiDisplayConsoleScrollCellsLocked(cols, rows, 1); - gConsoleY = rows - 1; - if (gConsoleX < 0) gConsoleX = 0; - if (gConsoleX >= cols) gConsoleX = cols - 1; - } - KeReleaseSpinlock(&gDisplayLock, flags); - } + for (size_t i = 0; i < drawCount && KSUCCESS(s); i++) { + DISPLAY_OUTPUT_DRAW* draw = &draws[i]; + size_t laterScrolls = scrollCount - draw->ScrollCount; + int finalRow = draw->Row - (int)laterScrolls; + if (finalRow < 0 || finalRow >= rows) + continue; + s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, draw->Character, + draw->Column * FONT_W, finalRow * FONT_H, + draw->Foreground, draw->Background); } s = KiDisplayFinishProviderDraw(device, ops, caps, s); @@ -1947,6 +2130,12 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { return s; } +// Renders one console character through the shared batched console path. +static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { + DISPLAY_OUTPUT_GLYPH glyph = { c, fg, bg }; + return KiDisplayPrintGlyphsInternal(&glyph, 1); +} + DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) { return KiDisplayPrintCharInternal(c, fg, bg); } @@ -2285,10 +2474,114 @@ static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) { return NTLSTATUS_SUCCESS; } -// Implements the boot Fb Blit operation. -static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) { +// Copies one pixel row into the write-combined scanout aperture. +static void BootFbCopyToScanout(uint32_t* destination, + const uint32_t* source, + size_t pixelCount) { + if (!destination || !source || pixelCount == 0) + return; + + if (!KeWriteCombiningAvailable()) { + memcpy(destination, source, pixelCount * sizeof(uint32_t)); + return; + } + + while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) { + *destination++ = *source++; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_stream_si128((__m128i*)(destination + 0), v0); + _mm_stream_si128((__m128i*)(destination + 4), v1); + _mm_stream_si128((__m128i*)(destination + 8), v2); + _mm_stream_si128((__m128i*)(destination + 12), v3); + source += 16; + destination += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_stream_si128((__m128i*)destination, value); + source += 4; + destination += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + *destination++ = *source++; + pixelCount--; + } +} + +// Copies one source row into the retained shadow and scanout in one read pass. +static void BootFbCopyToShadowAndScanout(uint32_t* shadow, + uint32_t* scanout, + const uint32_t* source, + size_t pixelCount) { + if (!shadow || !scanout || !source || pixelCount == 0) + return; + + if (!KeWriteCombiningAvailable()) { + memcpy(shadow, source, pixelCount * sizeof(uint32_t)); + memcpy(scanout, source, pixelCount * sizeof(uint32_t)); + return; + } + + while (pixelCount != 0 && ((uintptr_t)scanout & 15u) != 0) { + uint32_t value = *source++; + *shadow++ = value; + *scanout++ = value; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_storeu_si128((__m128i*)(shadow + 0), v0); + _mm_storeu_si128((__m128i*)(shadow + 4), v1); + _mm_storeu_si128((__m128i*)(shadow + 8), v2); + _mm_storeu_si128((__m128i*)(shadow + 12), v3); + _mm_stream_si128((__m128i*)(scanout + 0), v0); + _mm_stream_si128((__m128i*)(scanout + 4), v1); + _mm_stream_si128((__m128i*)(scanout + 8), v2); + _mm_stream_si128((__m128i*)(scanout + 12), v3); + source += 16; + shadow += 16; + scanout += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_storeu_si128((__m128i*)shadow, value); + _mm_stream_si128((__m128i*)scanout, value); + source += 4; + shadow += 4; + scanout += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + uint32_t value = *source++; + *shadow++ = value; + *scanout++ = value; + pixelCount--; + } +} + +// Implements the boot framebuffer pitched blit operation. +static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) { BOOTFB_CONTEXT* fb = BootFbContext(device); - if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0) + if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; uint32_t* dstBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer; @@ -2309,12 +2602,61 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co for (int row = 0; row < endY - startY; row++) { uint32_t* dst = dstBase + ((size_t)(startY + row) * fb->Stride) + startX; - const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX; + const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX; memcpy(dst, src, bytesPerRow); } return NTLSTATUS_SUCCESS; } +// Presents one pitched source rectangle without a shadow-to-scanout second pass. +static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, + int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + BOOTFB_CONTEXT* fb = BootFbContext(device); + if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) + return NTLSTATUS_INVALID_PARAMETER; + + int startX = x < 0 ? 0 : x; + int startY = y < 0 ? 0 : y; + int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w; + int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h; + if (startX >= endX || startY >= endY) + return NTLSTATUS_SUCCESS; + + int srcOffsetX = startX - x; + int srcOffsetY = startY - y; + size_t pixelCount = (size_t)(endX - startX); + uint64_t bytes = (uint64_t)pixelCount * (uint64_t)(endY - startY) * sizeof(uint32_t); + + fb->Stats.BlitCount++; + fb->Stats.PresentCount++; + fb->Stats.PresentRectCount++; + fb->Stats.DirectPresentCount++; + fb->Stats.ScanoutBytes += bytes; + fb->Stats.BytesCopied += fb->Backbuffer ? bytes * 2u : bytes; + + for (int row = 0; row < endY - startY; row++) { + const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX; + uint32_t* scanout = fb->Framebuffer + ((size_t)(startY + row) * fb->Stride) + startX; + if (fb->Backbuffer) { + uint32_t* shadow = fb->Backbuffer + ((size_t)(startY + row) * fb->Stride) + startX; + BootFbCopyToShadowAndScanout(shadow, scanout, source, pixelCount); + } else { + BootFbCopyToScanout(scanout, source, pixelCount); + } + } + + _mm_sfence(); + return NTLSTATUS_SUCCESS; +} + // Implements the boot Fb Fill Rect operation. static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) { BOOTFB_CONTEXT* fb = BootFbContext(device); @@ -2425,14 +2767,18 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, DISPLAY_RECT r = rects[i]; if (!KiDisplayClipRect(&mode, &r)) continue; - fb->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t); - size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t); + uint64_t bytes = (uint64_t)(uint32_t)r.W * + (uint64_t)(uint32_t)r.H * sizeof(uint32_t); + fb->Stats.BytesCopied += bytes; + fb->Stats.ScanoutBytes += bytes; + size_t pixelCount = (size_t)r.W; for (int row = 0; row < r.H; row++) { uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X; uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X; - memcpy(dst, src, bytesPerRow); + BootFbCopyToScanout(dst, src, pixelCount); } } + _mm_sfence(); return NTLSTATUS_SUCCESS; } @@ -2540,6 +2886,7 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) { fb->Stats.CursorHideCount++; BootFbRestoreCursor(fb); } + _mm_sfence(); return NTLSTATUS_SUCCESS; } diff --git a/code/kernel/ke_dpc.c b/code/kernel/ke_dpc.c index d0b1a41..6e81e43 100644 --- a/code/kernel/ke_dpc.c +++ b/code/kernel/ke_dpc.c @@ -81,7 +81,7 @@ void KiDrainDpcQueue(void) { if (oldIrql < DISPATCH_LEVEL) KeRaiseIrql(DISPATCH_LEVEL); - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&gDpcLock); LIST_ENTRY* node = KeListPopHeadChecked(&gDpcQueue); if (!node) { diff --git a/code/kernel/ke_exception.c b/code/kernel/ke_exception.c index 9eb59f1..c8dafb6 100644 --- a/code/kernel/ke_exception.c +++ b/code/kernel/ke_exception.c @@ -60,7 +60,7 @@ static _Noreturn void KiTerminateCurrentExceptionProcess(KSTATUS exitStatus) { if (thread) KeExitThread(thread, exitCode); - for (;;) { + while (true) { KeYield(); } } diff --git a/code/kernel/ke_input.c b/code/kernel/ke_input.c index 5220ffc..ca54e56 100644 --- a/code/kernel/ke_input.c +++ b/code/kernel/ke_input.c @@ -390,7 +390,7 @@ static KSTATUS KiInputWaitKey(KE_INPUT_CONTEXT* context, uint8_t alertable, uint if (!context->WaitEvent) return NTLSTATUS_FAILURE; - for (;;) { + while (true) { EVENT* event = context->WaitEvent; uint32_t selected = 0; KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected); diff --git a/code/kernel/ke_irp.c b/code/kernel/ke_irp.c index 05ad272..d269f33 100644 --- a/code/kernel/ke_irp.c +++ b/code/kernel/ke_irp.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -38,12 +39,13 @@ __declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) { return CONTAINING_RECORD(node, IRP, Link); } -__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, - KSTATUS status, - IRP_QUEUE_CANCEL_CALLBACK callback, - void* context) { +// Cancels every queued IRP and invokes an optional ownership callback. +__declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q, + KSTATUS status, + IRP_QUEUE_CANCEL_CALLBACK callback, + void* context) { if (!q) return; - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&q->Lock); LIST_ENTRY* node = ListPopHead(&q->Head); KeReleaseSpinlock(&q->Lock, flags); @@ -60,7 +62,7 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, // Implements the irp Queue Cancel All operation. __declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) { - IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0); + IrpQueueCancelAllWithCallback(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0); } // Remove locks. @@ -284,6 +286,14 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t irp->Status = status; irp->Information = information; + // Completion callbacks may recycle an embedded IRP, while a synchronous + // waiter may release stack storage as soon as its event is signaled. + // Snapshot every completion target before either ownership handoff. + KAPC* completionApc = (KAPC*)irp->CompletionApc; + EVENT* completionEvent = (EVENT*)irp->CompletionEvent; + IRP_COMPLETION_FN completionFn = irp->CompletionFn; + void* completionCtx = irp->CompletionCtx; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0; if (dev) { @@ -292,12 +302,18 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t ios->DeviceObject = (DEVICE_OBJECT*)0; } - if (irp->CompletionApc) - KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status); - if (irp->CompletionEvent) - KeSignalEvent((EVENT*)irp->CompletionEvent); - if (irp->CompletionFn) - irp->CompletionFn(irp, irp->CompletionCtx); + if (completionApc) + KeInsertQueueApc(completionApc, irp, (void*)(uintptr_t)status); + + // The callback owns the final callback-based handoff. Do not dereference + // the IRP after it returns because embedded requests may be freed there. + if (completionFn) + completionFn(irp, completionCtx); + + // The event is the final synchronous ownership handoff. A waiter may + // invalidate a stack IRP immediately after this signal becomes visible. + if (completionEvent) + KeSignalEvent(completionEvent); return NTLSTATUS_SUCCESS; } @@ -314,6 +330,7 @@ __declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) { // Submits irp Sync. __declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) { if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER; + if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; EVENT* ev = (EVENT*)0; KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev); @@ -321,12 +338,22 @@ __declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) { irp->CompletionEvent = ev; + // Drivers may retain this IRP, its transfer buffer, and the caller's + // kernel stack until completion. Defer process termination across that + // ownership interval rather than letting teardown free live storage. + KeEnterTerminationDeferral(); st = IoSubmitIrp(device, irp); if (st == NTLSTATUS_PENDING) { - KeWaitForEvent(ev, 0); + do { + st = KeWaitForEvent(ev, 0); + } while (st != NTLSTATUS_SUCCESS); st = irp->Status; } + // Completion events are signaled only after IoCompleteRequest has finished + // every operation that can dereference a synchronous caller's stack IRP. + irp->CompletionEvent = (EVENT*)0; ObDereferenceObject(&ev->Header); + KeLeaveTerminationDeferral(); return st; } diff --git a/code/kernel/ke_irq.c b/code/kernel/ke_irq.c index e15526f..e5722ab 100644 --- a/code/kernel/ke_irq.c +++ b/code/kernel/ke_irq.c @@ -126,7 +126,9 @@ void KiExitInterrupt(void) { // high-IRQL region. if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) { THREAD* current = cpu->CurrentThread; - if (current && !current->IsIdleThread && current->Owner && KeIsProcessTerminating(current->Owner)) { + if (current && !current->IsIdleThread && current->Owner && + KeIsProcessTerminating(current->Owner) && + !KeThreadTerminationDeferred(current)) { KeExitThread(current, current->Owner->ExitCode); while (true) ; } diff --git a/code/kernel/ke_mutex.c b/code/kernel/ke_mutex.c index fee35fd..91fdb3a 100644 --- a/code/kernel/ke_mutex.c +++ b/code/kernel/ke_mutex.c @@ -1,4 +1,4 @@ -// Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups. +// Kernel mutex primitive. This file owns exclusive ownership, abandonment, wait queues, and priority inheritance. #include #include @@ -6,76 +6,285 @@ #include #include #include +#include #include #include -// Implements the mutex Destroy operation. -static void KiMutexDestroy(OBJECT* obj) { (void)obj; } +// Reports an invalid mutex ownership transition. +static void KiMutexBug(const char* where, MUTEX* mutex, THREAD* thread, const char* why) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "mutex invariant failed: %s: mutex=0x%016llX thread=0x%016llX tid=%u owner=0x%016llX refs=%ld waitEmpty=%u ownerLink=0x%016llX/0x%016llX threadWaiting=0x%016llX threadOwnedEmpty=%u: %s\n", + where, + (uint64_t)mutex, + (uint64_t)thread, + thread ? thread->ThreadId : 0, + mutex ? (uint64_t)mutex->Owner : 0, + mutex ? mutex->Header.RefCount : 0, + mutex ? (LIST_IS_EMPTY(&mutex->WaitList) ? 1u : 0u) : 1u, + mutex ? (uint64_t)mutex->OwnerLink.Fwd : 0, + mutex ? (uint64_t)mutex->OwnerLink.Bwd : 0, + thread ? (uint64_t)thread->WaitingMutex : 0, + thread ? (LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u) : 1u, + why ? why : "?"); +} -// Implements the mutex Create operation. -__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) { +// Stabilizes an owner pointer before it crosses the mutex lock boundary. +static void KiMutexReferenceOwnerLocked(MUTEX* mutex, THREAD* owner) { + if (!owner || !ObTryReferenceObject(&owner->Header)) + KiMutexBug("KiMutexReferenceOwnerLocked", mutex, owner, + "live mutex owner could not be referenced"); +} + +// Tests whether one scheduler link is currently attached to a list. +static int KiMutexWaiterIsLinked(const THREAD* thread) { + if (!thread) return 0; + return thread->SchedulerLink.Fwd != &thread->SchedulerLink || + thread->SchedulerLink.Bwd != &thread->SchedulerLink; +} + +// Propagates one effective-rank change without carrying raw owners across locks. +static void KiMutexPropagatePriority(THREAD* thread) { + int releaseThread = 0; + + for (uint32_t depth = 1; thread && depth <= KE_PRIORITY_INHERITANCE_MAX_DEPTH; depth++) { + MUTEX* waitingMutex = (MUTEX*)0; + int changed = KiSchedulerRefreshEffectivePriority(thread, &waitingMutex); + if (!changed) { + if (releaseThread) ObDereferenceObject(&thread->Header); + return; + } + + KiSchedulerNotePriorityPropagation(depth); + if (!waitingMutex) { + if (releaseThread) ObDereferenceObject(&thread->Header); + return; + } + + uintptr_t flags = KeAcquireSpinlock(&waitingMutex->Lock); + if (thread->WaitingMutex != waitingMutex || + thread->WaitObject != &waitingMutex->Header || + thread->WaitListHead != &waitingMutex->WaitList || + !KiMutexWaiterIsLinked(thread)) { + KeReleaseSpinlock(&waitingMutex->Lock, flags); + ObDereferenceObject(&waitingMutex->Header); + if (releaseThread) ObDereferenceObject(&thread->Header); + return; + } + + THREAD* owner = waitingMutex->Owner; + if (owner) KiMutexReferenceOwnerLocked(waitingMutex, owner); + int donationChanged = KiSchedulerRefreshMutexDonation(waitingMutex); + KeReleaseSpinlock(&waitingMutex->Lock, flags); + ObDereferenceObject(&waitingMutex->Header); + + if (releaseThread) ObDereferenceObject(&thread->Header); + if (!donationChanged || !owner) { + if (owner) ObDereferenceObject(&owner->Header); + return; + } + + thread = owner; + releaseThread = 1; + } + + if (thread) + KiMutexBug("KiMutexPropagatePriority", thread->WaitingMutex, thread, + "priority inheritance chain exceeded the supported depth"); +} + +// Initializes one mutex's ownership bookkeeping. +static void KiMutexInitialize(MUTEX* mutex) { + mutex->Lock = (SPINLOCK)SPINLOCK_INIT; + mutex->Owner = (THREAD*)0; + LIST_INIT(&mutex->WaitList); + LIST_INIT(&mutex->OwnerLink); + mutex->DonationClass = KeSchedulingNormal; + mutex->DonationPriority = PriorityLow; + mutex->HasDonation = 0; + mutex->Abandoned = 0; +} + +// Destroys one unowned mutex object. +static void KiMutexDestroy(OBJECT* object) { + MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header); + if (mutex->Owner || !LIST_IS_EMPTY(&mutex->WaitList) || + mutex->OwnerLink.Fwd != &mutex->OwnerLink || + mutex->OwnerLink.Bwd != &mutex->OwnerLink) { + KiMutexBug("KiMutexDestroy", mutex, mutex->Owner, + "owned or waited mutex reached object destruction"); + } +} + +// Assigns an unowned mutex to one thread and consumes abandonment state. +static KSTATUS KiMutexAcquireUnownedLocked(MUTEX* mutex, THREAD* thread) { + KSTATUS status = mutex->Abandoned ? NTLSTATUS_ABANDONED : NTLSTATUS_SUCCESS; + mutex->Abandoned = 0; + ObReferenceObject(&mutex->Header); + KiSchedulerAcquireMutexOwnership(thread, mutex); + return status; +} + +// Creates a mutex object. +__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out) { if (!out) return NTLSTATUS_INVALID_PARAMETER; - MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX)); - if (!m) return NTLSTATUS_NO_MEMORY; - m->Header.Destroy = KiMutexDestroy; - m->Lock = (SPINLOCK)SPINLOCK_INIT; - m->Owner = (struct THREAD *)0; - LIST_INIT(&m->WaitList); - *out = m; + MUTEX* mutex = (MUTEX*)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX)); + if (!mutex) return NTLSTATUS_NO_MEMORY; + mutex->Header.Destroy = KiMutexDestroy; + KiMutexInitialize(mutex); + *out = mutex; return NTLSTATUS_SUCCESS; } -// Implements the acquire Mutex operation. -__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) { +// Acquires a mutex within the requested timeout and donates priority while blocked. +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds) { if (!mutex) return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; + if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL; + THREAD* self = KeGetCurrentThread(); + if (!self) return NTLSTATUS_FAILURE; uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); if (!mutex->Owner) { - mutex->Owner = self; + KSTATUS status = KiMutexAcquireUnownedLocked(mutex, self); KeReleaseSpinlock(&mutex->Lock, flags); - return NTLSTATUS_SUCCESS; + return status; + } + + if (mutex->Owner == self) { + KeReleaseSpinlock(&mutex->Lock, flags); + return NTLSTATUS_DEADLOCK; + } + + if (timeoutMilliseconds == 0) { + KeReleaseSpinlock(&mutex->Lock, flags); + return NTLSTATUS_TIMEOUT; + } + + THREAD* owner = mutex->Owner; + KiMutexReferenceOwnerLocked(mutex, owner); + KSTATUS waitStatus = KiSchedulerPrepareMutexWait(self, mutex, owner); + if (!KSUCCESS(waitStatus)) { + KeReleaseSpinlock(&mutex->Lock, flags); + ObDereferenceObject(&owner->Header); + return waitStatus; } - KePrepareBlockThread(self, &mutex->Header, 0); KeListInsertTailChecked(&mutex->WaitList, &self->SchedulerLink); - KeSetThreadWaitList(self, &mutex->WaitList); + if (timeoutMilliseconds != KE_WAIT_INFINITE) { + KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT); + if (!KSUCCESS(timeoutStatus)) { + KeListRemoveChecked(&self->SchedulerLink); + KiSchedulerCancelMutexWait(self, mutex); + (void)KiSchedulerRefreshMutexDonation(mutex); + KeUnblockThreadWithStatus(self, timeoutStatus); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + ObDereferenceObject(&owner->Header); + return timeoutStatus; + } + } + (void)KiSchedulerRefreshMutexDonation(mutex); KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + ObDereferenceObject(&owner->Header); return KeCommitBlockThread(self, 0); } -// Implements the try Acquire Mutex operation. -__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) { - if (!mutex) return NTLSTATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - if (!mutex->Owner) { - mutex->Owner = self; - KeReleaseSpinlock(&mutex->Lock, flags); - return NTLSTATUS_SUCCESS; - } - KeReleaseSpinlock(&mutex->Lock, flags); - return NTLSTATUS_FAILURE; -} - -// Releases mutex. +// Releases a mutex and hands ownership to its highest-ranked waiter. __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) { if (!mutex) return; - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - if (!LIST_IS_EMPTY(&mutex->WaitList)) { - LIST_ENTRY* node = KeListPopHeadChecked(&mutex->WaitList); - THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeSetThreadWaitList(waiter, (LIST_ENTRY*)0); - // Direct handoff: the selected waiter owns the mutex when its wait - // returns. Do not drop Owner to NULL, or another thread can steal it. - mutex->Owner = waiter; - KeReleaseSpinlock(&mutex->Lock, flags); + THREAD* self = KeGetCurrentThread(); + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (!self || mutex->Owner != self) + KiMutexBug("KeReleaseMutex", mutex, self, "release by non-owner"); + + THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex); + if (waiter) { + KeListRemoveChecked(&waiter->SchedulerLink); + KiSchedulerTransferMutexOwnership(self, waiter, mutex); + (void)KiSchedulerRefreshMutexDonation(mutex); + (void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0); KeUnblockThreadWithStatus(waiter, NTLSTATUS_SUCCESS); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(self); return; } - mutex->Owner = (struct THREAD *)0; + KiSchedulerReleaseMutexOwnership(self, mutex); + (void)KiSchedulerRefreshMutexDonation(mutex); KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(self); + ObDereferenceObject(&mutex->Header); +} + +// Removes one blocked mutex waiter during thread-wait rundown. +int KiMutexAbortWait(MUTEX* mutex, THREAD* thread) { + if (!mutex || !thread) return 0; + + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (thread->WaitingMutex != mutex || + thread->WaitObject != &mutex->Header || + thread->WaitListHead != &mutex->WaitList || + !KiMutexWaiterIsLinked(thread)) { + KeReleaseSpinlock(&mutex->Lock, flags); + return 0; + } + + KeListRemoveChecked(&thread->SchedulerLink); + KiSchedulerCancelMutexWait(thread, mutex); + THREAD* owner = mutex->Owner; + if (owner) KiMutexReferenceOwnerLocked(mutex, owner); + (void)KiSchedulerRefreshMutexDonation(mutex); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + if (owner) ObDereferenceObject(&owner->Header); + KiMutexPropagatePriority(thread); + return 1; +} + +// Abandons one mutex owned by a terminating thread. +static void KiMutexAbandonOwned(MUTEX* mutex, THREAD* owner) { + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (mutex->Owner != owner) + KiMutexBug("KiMutexAbandonOwned", mutex, owner, "ownership list mismatch"); + + THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex); + if (waiter) { + KeListRemoveChecked(&waiter->SchedulerLink); + KiSchedulerTransferMutexOwnership(owner, waiter, mutex); + mutex->Abandoned = 0; + (void)KiSchedulerRefreshMutexDonation(mutex); + (void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0); + KeUnblockThreadWithStatus(waiter, NTLSTATUS_ABANDONED); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + return; + } + + KiSchedulerReleaseMutexOwnership(owner, mutex); + mutex->Abandoned = 1; + (void)KiSchedulerRefreshMutexDonation(mutex); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + ObDereferenceObject(&mutex->Header); +} + +// Releases every mutex still owned by a terminating thread. +void KiMutexRundownThread(THREAD* thread) { + if (!thread) return; + + while (true) { + MUTEX* mutex = KiSchedulerFirstOwnedMutex(thread); + if (!mutex) + return; + KiMutexAbandonOwned(mutex, thread); + } } diff --git a/code/kernel/ke_process.c b/code/kernel/ke_process.c index 127c180..5670a48 100644 --- a/code/kernel/ke_process.c +++ b/code/kernel/ke_process.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include @@ -217,6 +219,7 @@ static uint32_t KiThreadStateToSys(THREAD_STATE state) { case ThreadRunning: return SYS_THREAD_STATE_RUNNING; case ThreadWaiting: return SYS_THREAD_STATE_WAITING; case ThreadDead: return SYS_THREAD_STATE_DEAD; + case ThreadSuspended: return SYS_THREAD_STATE_SUSPENDED; default: return SYS_THREAD_STATE_DEAD; } } @@ -265,25 +268,33 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s SYS_THREAD_INFO* info = &infoOut[count]; memset(info, 0, sizeof(*info)); info->Size = sizeof(*info); + KE_THREAD_SCHEDULER_SNAPSHOT snapshot; + KiSchedulerSnapshotThread(thread, &snapshot); + info->ProcessId = process->ProcessId; info->ThreadId = thread->ThreadId; - info->State = KiThreadStateToSys(thread->State); - info->Priority = (uint32_t)thread->Priority; - info->RunningCpu = thread->RunningCpu; - info->LastCpu = thread->LastCpu; - info->QuantumRemaining = thread->QuantumRemaining; - info->QuantumTicks = thread->QuantumTicks; - info->CpuBurstTicks = thread->CpuBurstTicks; - info->LastRunTicks = thread->LastRunTicks; - info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE; - info->WaitStatus = (uint32_t)thread->WaitStatus; + info->State = KiThreadStateToSys(snapshot.State); + info->BaseSchedulingClass = (uint32_t)snapshot.BaseSchedulingClass; + info->BasePriority = snapshot.BasePriority; + info->EffectiveSchedulingClass = (uint32_t)snapshot.EffectiveSchedulingClass; + info->EffectivePriority = snapshot.EffectivePriority; + info->RunningCpu = snapshot.RunningCpu; + info->LastCpu = snapshot.LastCpu; + info->QuantumRemaining = snapshot.QuantumRemaining; + info->QuantumTicks = snapshot.QuantumTicks; + info->CpuBurstTicks = snapshot.CpuBurstTicks; + info->LastRunTicks = snapshot.LastRunTicks; + info->WaitObjectType = snapshot.WaitObjectType != 0 + ? KiWaitObjectTypeToSys((OBJECT_TYPE)snapshot.WaitObjectType) + : SYS_THREAD_WAIT_NONE; + info->WaitStatus = (uint32_t)snapshot.WaitStatus; info->KernelApcDisable = thread->KernelApcDisable; if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT; if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE; if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL; - if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED; - if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT; + if (snapshot.Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED; + if (snapshot.AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT; if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING; if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING; } @@ -296,10 +307,26 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s return NTLSTATUS_SUCCESS; } -// Releases all kernel objects still reachable from a process handle table. -// This is part of process exit, not merely object destruction: a parent may keep -// a handle to an exited process for wait/query, but the dead process must not -// keep its own files, mappings, queues, sockets, or synchronization objects alive. +// Records type-specific ownership after a handle slot becomes visible. +static void KiProcessHandleOpened(OBJECT* object) { + if (!object) + return; + + if (object->Type == ObjectTypeAudioStream) + KeAudioStreamHandleOpened((KE_AUDIO_STREAM*)object); +} + +// Releases type-specific ownership before the handle-table reference drops. +static void KiProcessReleaseHandleObject(OBJECT* object) { + if (!object) + return; + + if (object->Type == ObjectTypeAudioStream) + KeAudioStreamHandleClosed((KE_AUDIO_STREAM*)object); + ObDereferenceObject(object); +} + +// Releases every handle-table reference owned by one process. static void KiProcessReleaseAllHandles(PROCESS* proc) { if (!proc) return; @@ -317,7 +344,7 @@ static void KiProcessReleaseAllHandles(PROCESS* proc) { if (obj) { KeNamedQueueUnpublishHandle(proc, (uint32_t)i, obj); - ObDereferenceObject(obj); + KiProcessReleaseHandleObject(obj); } } } @@ -671,14 +698,16 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) { if (t == self) continue; - // A terminating process must not be held open by worker threads that - // are blocked in non-alertable waits. User-mode DLLs such as Bink can - // leave IO/submit workers parked on semaphores; after the foreground - // thread crashes, those waiters would otherwise keep ThreadCount > 0 - // and prevent the process ExitEvent from waking the shell. - if (t->State == ThreadWaiting) + // Most user waits must be interrupted immediately. A thread can defer + // termination only while synchronous kernel work still owns its stack + // or process memory; that thread is allowed to finish the ownership + // interval before the normal termination gates reap it. + int deferred = KeThreadTerminationDeferred(t); + if (t->State == ThreadSuspended && !deferred) + (void)KeResumeThread(t, (uint32_t*)0); + else if (t->State == ThreadWaiting && !deferred) KeAbortThreadWait(t, NTLSTATUS_ALERTED); - else + else if (!deferred) (void)KeAlertThread(t); if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) { @@ -727,6 +756,7 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri process->Handles[i].Object = object; process->Handles[i].Rights = rights; process->Handles[i].Flags = 0; + KiProcessHandleOpened(object); *handleOut = (uint32_t)i; KeReleaseSpinlock(&process->HandleLock, flags); return NTLSTATUS_SUCCESS; @@ -756,7 +786,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) { if (obj->Type == ObjectTypeFile) (void)IoFlushFile((FILE_OBJECT*)obj); KeNamedQueueUnpublishHandle(process, handle, obj); - ObDereferenceObject(obj); + KiProcessReleaseHandleObject(obj); return; } KeReleaseSpinlock(&process->HandleLock, flags); diff --git a/code/kernel/ke_queue.c b/code/kernel/ke_queue.c index 17a8193..b11601d 100644 --- a/code/kernel/ke_queue.c +++ b/code/kernel/ke_queue.c @@ -133,7 +133,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMi if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE) deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds); - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&queue->Lock); if (queue->Count < QUEUE_DEPTH) { uint32_t previousCount = queue->Count; @@ -165,7 +165,7 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilli if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE) deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds); - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&queue->Lock); if (queue->Count > 0) { uint32_t previousCount = queue->Count; diff --git a/code/kernel/ke_sched.c b/code/kernel/ke_sched.c index 632cba6..67afa95 100644 --- a/code/kernel/ke_sched.c +++ b/code/kernel/ke_sched.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -15,17 +16,117 @@ #include #include #include +#include #include #include -static LIST_ENTRY gRunQueue[PriorityCount]; -static LIST_ENTRY gSleepList; +static LIST_ENTRY gNormalRunQueue[PriorityCount]; +static LIST_ENTRY gRealtimeRunQueue[KE_REALTIME_PRIORITY_COUNT]; +static LIST_ENTRY gTimeoutList; +static KWORK_ITEM gTimeoutWorkItem; static SPINLOCK gSchedLock; +static void KiTimeoutWorkRoutine(void* context); + static uint32_t gAgingCounter = 0; static KE_SCHED_STATS gSchedStats; static PROCESS* gIdleProcess; +// Tests whether one scheduling rank outranks another. +static int KiRankOutranks(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority, + KE_SCHEDULING_CLASS rightClass, uint32_t rightPriority) { + if (leftClass != rightClass) + return leftClass == KeSchedulingRealtime; + return leftPriority > rightPriority; +} + +// Tests whether two scheduling ranks are equal. +static int KiRanksEqual(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority, + KE_SCHEDULING_CLASS rightClass, uint32_t rightPriority) { + return leftClass == rightClass && leftPriority == rightPriority; +} + +// Compares two threads by effective scheduling rank. +static int KiSchedulerThreadOutranks(const THREAD* left, const THREAD* right) { + if (!left) return 0; + if (!right) return 1; + return KiRankOutranks(left->EffectiveSchedulingClass, left->EffectivePriority, + right->EffectiveSchedulingClass, right->EffectivePriority); +} + +// Copies scheduler-owned thread state into one coherent snapshot. +void KiSchedulerSnapshotThread(THREAD* thread, KE_THREAD_SCHEDULER_SNAPSHOT* snapshot) { + if (!thread || !snapshot) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + snapshot->State = thread->State; + snapshot->BaseSchedulingClass = thread->BaseSchedulingClass; + snapshot->BasePriority = thread->BasePriority; + snapshot->EffectiveSchedulingClass = thread->EffectiveSchedulingClass; + snapshot->EffectivePriority = thread->EffectivePriority; + snapshot->RunningCpu = thread->RunningCpu; + snapshot->LastCpu = thread->LastCpu; + snapshot->QuantumRemaining = thread->QuantumRemaining; + snapshot->QuantumTicks = thread->QuantumTicks; + snapshot->CpuBurstTicks = thread->CpuBurstTicks; + snapshot->LastRunTicks = thread->LastRunTicks; + snapshot->WaitObjectType = thread->WaitObject ? (uint32_t)thread->WaitObject->Type : 0; + snapshot->WaitStatus = thread->WaitStatus; + snapshot->WaitingMutex = thread->WaitingMutex; + snapshot->Queued = thread->Queued; + snapshot->AlertableWait = thread->AlertableWait; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Selects the highest-ranked waiter while the caller holds the mutex lock. +THREAD* KiSchedulerSelectMutexWaiter(MUTEX* mutex) { + if (!mutex) return (THREAD*)0; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + THREAD* best = (THREAD*)0; + LIST_ENTRY* head = &mutex->WaitList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!best || KiSchedulerThreadOutranks(waiter, best)) + best = waiter; + } + KeReleaseSpinlock(&gSchedLock, flags); + return best; +} + +// Refreshes one mutex donation while the caller holds the mutex lock. +int KiSchedulerRefreshMutexDonation(MUTEX* mutex) { + if (!mutex) return 0; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + THREAD* highest = (THREAD*)0; + LIST_ENTRY* head = &mutex->WaitList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!highest || KiSchedulerThreadOutranks(waiter, highest)) + highest = waiter; + } + + KE_SCHEDULING_CLASS oldClass = mutex->DonationClass; + uint32_t oldPriority = mutex->DonationPriority; + uint8_t oldHasDonation = mutex->HasDonation; + if (highest) { + mutex->DonationClass = highest->EffectiveSchedulingClass; + mutex->DonationPriority = highest->EffectivePriority; + mutex->HasDonation = 1; + } else { + mutex->DonationClass = KeSchedulingNormal; + mutex->DonationPriority = PriorityLow; + mutex->HasDonation = 0; + } + + int changed = oldHasDonation != mutex->HasDonation || + oldClass != mutex->DonationClass || + oldPriority != mutex->DonationPriority; + KeReleaseSpinlock(&gSchedLock, flags); + return changed; +} + // Implements the default Thread Affinity Mask operation. static uint64_t KiDefaultThreadAffinityMask(void) { // Normal kernel, user, and driver-created threads are BSP-affine by @@ -36,7 +137,9 @@ static uint64_t KiDefaultThreadAffinityMask(void) { // Implements the thread Is Killed Locked operation. static int KiThreadIsKilledLocked(const THREAD* thread) { - return thread && !thread->IsIdleThread && thread->Owner && KeIsProcessTerminating(thread->Owner); + return thread && !thread->IsIdleThread && thread->Owner && + KeIsProcessTerminating(thread->Owner) && + !KeThreadTerminationDeferred(thread); } // Implements the thread Can Run On Cpu Locked operation. @@ -124,8 +227,10 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) { gSchedStats.SmtPrimaryPokes++; KiRequestRescheduleCpuLocked(cpu); requestedIdle = 1; - } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) { + } else if (cur->State == ThreadRunning && KiSchedulerThreadOutranks(thread, cur)) { gSchedStats.PriorityPreemptions++; + if (thread->EffectiveSchedulingClass == KeSchedulingRealtime) + gSchedStats.RealtimePreemptions++; KiRequestRescheduleCpuLocked(cpu); } } @@ -150,8 +255,8 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) { } // Implements the base Quantum For Priority operation. -static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) { - switch (priority) { +static uint32_t KiBaseQuantumForPriority(uint32_t priority) { + switch ((THREAD_PRIORITY)priority) { case PriorityHigh: return SCHED_HIGH_QUANTUM_TICKS; case PriorityNormal: return SCHED_NORMAL_QUANTUM_TICKS; case PriorityLow: return SCHED_LOW_QUANTUM_TICKS; @@ -164,13 +269,16 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) { if (!thread || thread == KeGetCurrentCpu()->IdleThread) return 1; - uint32_t quantum = KiBaseQuantumForPriority(thread->Priority); + if (thread->EffectiveSchedulingClass == KeSchedulingRealtime) + return SCHED_REALTIME_QUANTUM_TICKS; + + uint32_t quantum = KiBaseQuantumForPriority(thread->EffectivePriority); // Low-priority CPU-bound threads get longer batches once they have proven // they mostly burn CPU. Higher-priority work still preempts them as soon // as it becomes ready, so this reduces switch overhead without hurting // interactive wake latency. - if (thread->Priority == PriorityLow && thread->CpuBurstTicks >= + if (thread->EffectivePriority == PriorityLow && thread->CpuBurstTicks >= (SCHED_DEMOTE_AFTER_BURST_TICKS * 2u)) { quantum += 4; } @@ -256,17 +364,330 @@ static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) { KiSchedulerBug(where, thread, "picked thread was not ready"); } -// Inserts ready Locked. +// Inserts one thread into the queue selected by its effective rank. static void KiInsertReadyLocked(THREAD* thread) { KiAssertReadyQueueCandidateLocked("KiInsertReadyLocked", thread); - int p = (int)thread->Priority; - if (p < 0) p = 0; - if (p >= PriorityCount) p = PriorityCount - 1; + LIST_ENTRY* queue; + if (thread->EffectiveSchedulingClass == KeSchedulingRealtime) { + uint32_t priority = thread->EffectivePriority; + if (priority > KE_REALTIME_PRIORITY_MAX) + priority = KE_REALTIME_PRIORITY_MAX; + queue = &gRealtimeRunQueue[priority]; + } else { + uint32_t priority = thread->EffectivePriority; + if (priority >= PriorityCount) + priority = PriorityCount - 1; + queue = &gNormalRunQueue[priority]; + } thread->State = ThreadReady; thread->Queued = 1; - KeListInsertTailChecked(&gRunQueue[p], &thread->SchedulerLink); + KeListInsertTailChecked(queue, &thread->SchedulerLink); +} + +// Tests whether any ready thread outranks one running thread on a CPU. +static int KiAnyRunnableOutrankingThreadForCpuLocked(KCPU* cpu, const THREAD* current) { + if (!cpu || !current) return 0; + + for (int p = (int)KE_REALTIME_PRIORITY_MAX; p >= 0; p--) { + LIST_ENTRY* head = &gRealtimeRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiSchedulerThreadOutranks(thread, current)) + return 1; + } + } + + for (int p = PriorityCount - 1; p >= 0; p--) { + LIST_ENTRY* head = &gNormalRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiSchedulerThreadOutranks(thread, current)) + return 1; + } + } + + return 0; +} + +// Tests whether a real-time peer at the current or higher priority is ready. +static int KiAnyRealtimePeerForCpuLocked(KCPU* cpu, uint32_t minimumPriority) { + if (!cpu) return 0; + if (minimumPriority > KE_REALTIME_PRIORITY_MAX) + minimumPriority = KE_REALTIME_PRIORITY_MAX; + + for (int p = (int)KE_REALTIME_PRIORITY_MAX; p >= (int)minimumPriority; p--) { + LIST_ENTRY* head = &gRealtimeRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (KiThreadCanRunOnCpuLocked(thread, cpu)) + return 1; + } + } + return 0; +} + +// Computes one thread's effective rank from its base rank and mutex donations. +static void KiComputeEffectiveRankLocked(THREAD* thread, + KE_SCHEDULING_CLASS* schedulingClassOut, + uint32_t* priorityOut) { + KE_SCHEDULING_CLASS schedulingClass = thread->BaseSchedulingClass; + uint32_t priority = thread->BasePriority; + + LIST_ENTRY* head = &thread->OwnedMutexList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + MUTEX* mutex = CONTAINING_RECORD(node, MUTEX, OwnerLink); + if (!mutex->HasDonation) + continue; + if (KiRankOutranks(mutex->DonationClass, mutex->DonationPriority, + schedulingClass, priority)) { + schedulingClass = mutex->DonationClass; + priority = mutex->DonationPriority; + } + } + + *schedulingClassOut = schedulingClass; + *priorityOut = priority; +} + +// Applies one effective-rank change and preserves ready-queue ordering. +static int KiApplyEffectiveRankLocked(THREAD* thread, + KE_SCHEDULING_CLASS schedulingClass, + uint32_t priority) { + KE_SCHEDULING_CLASS oldClass = thread->EffectiveSchedulingClass; + uint32_t oldPriority = thread->EffectivePriority; + if (KiRanksEqual(oldClass, oldPriority, schedulingClass, priority)) + return 0; + + int wasQueued = thread->Queued != 0; + if (wasQueued) { + KeListRemoveChecked(&thread->SchedulerLink); + thread->Queued = 0; + } + + thread->EffectiveSchedulingClass = schedulingClass; + thread->EffectivePriority = priority; + thread->QuantumRemaining = 0; + + if (wasQueued) { + KiInsertReadyLocked(thread); + KiPokeRunnableCpusLocked(thread); + } else if (thread->State == ThreadRunning && thread->RunningCpu < gCpuCount) { + KCPU* cpu = &gCpus[thread->RunningCpu]; + if (KiAnyRunnableOutrankingThreadForCpuLocked(cpu, thread)) + KiRequestRescheduleCpuLocked(cpu); + } + + return 1; +} + +// Recomputes effective rank from current base rank and owned mutexes. +int KiSchedulerRefreshEffectivePriority(THREAD* thread, MUTEX** waitingMutexOut) { + if (!thread) return 0; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + KE_SCHEDULING_CLASS oldClass = thread->EffectiveSchedulingClass; + uint32_t oldPriority = thread->EffectivePriority; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &priority); + int changed = KiApplyEffectiveRankLocked(thread, schedulingClass, priority); + if (changed) { + if (KiRankOutranks(schedulingClass, priority, oldClass, oldPriority)) + gSchedStats.PriorityInheritanceBoosts++; + else + gSchedStats.PriorityInheritanceDrops++; + } + if (waitingMutexOut) { + *waitingMutexOut = (MUTEX*)0; + if (changed && thread->WaitingMutex) { + if (!ObTryReferenceObject(&thread->WaitingMutex->Header)) + KiSchedulerBug("KiSchedulerRefreshEffectivePriority", thread, + "waiting mutex could not be referenced"); + *waitingMutexOut = thread->WaitingMutex; + } + } + KeReleaseSpinlock(&gSchedLock, flags); + return changed; +} + +// Publishes one wait edge atomically so concurrent acquisitions cannot create a cycle. +KSTATUS KiSchedulerPrepareMutexWait(THREAD* thread, MUTEX* mutex, THREAD* owner) { + if (!thread || !mutex || !owner) return NTLSTATUS_INVALID_PARAMETER; + + KCPU* cpu = KeGetCurrentCpu(); + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + + THREAD* current = owner; + uint32_t depth = 0; + while (current && depth < KE_PRIORITY_INHERITANCE_MAX_DEPTH) { + if (current == thread) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_DEADLOCK; + } + + MUTEX* waitingMutex = current->WaitingMutex; + current = waitingMutex ? waitingMutex->Owner : (THREAD*)0; + depth++; + } + + if (current) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_DEADLOCK; + } + + if (thread->IsIdleThread) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "idle thread cannot block"); + if (thread->Queued) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "queued thread cannot block"); + if (!cpu || thread != cpu->CurrentThread) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "only the current thread can prepare a mutex wait"); + if (thread->RunningCpu != cpu->CpuIndex) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "thread running CPU does not match current CPU"); + if (thread->State != ThreadRunning || thread->WaitingMutex || thread->WaitObject || + thread->WaitListHead || thread->SchedulerLink.Fwd != &thread->SchedulerLink || + thread->SchedulerLink.Bwd != &thread->SchedulerLink || thread->TimeoutArmed || + thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "thread already has wait state"); + } + + thread->State = ThreadWaiting; + thread->WaitObject = &mutex->Header; + thread->WaitStatus = NTLSTATUS_PENDING; + thread->WaitListHead = &mutex->WaitList; + thread->AlertableWait = 0; + thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0; + thread->WaitRundownContext = (void*)0; + thread->WaitingMutex = mutex; + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_SUCCESS; +} + +// Removes one published wait edge after its wait-list node is detached. +void KiSchedulerCancelMutexWait(THREAD* thread, MUTEX* mutex) { + if (!thread || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->WaitingMutex != mutex || thread->WaitObject != &mutex->Header || + thread->WaitListHead != &mutex->WaitList || + thread->SchedulerLink.Fwd != &thread->SchedulerLink || + thread->SchedulerLink.Bwd != &thread->SchedulerLink) { + KiSchedulerBug("KiSchedulerCancelMutexWait", thread, "mutex wait state is inconsistent"); + } + + thread->WaitingMutex = (MUTEX*)0; + thread->WaitListHead = (LIST_ENTRY*)0; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Assigns one unowned mutex while the caller holds its lock. +void KiSchedulerAcquireMutexOwnership(THREAD* thread, MUTEX* mutex) { + if (!thread || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (mutex->Owner || + mutex->OwnerLink.Fwd != &mutex->OwnerLink || + mutex->OwnerLink.Bwd != &mutex->OwnerLink) { + KiSchedulerBug("KiSchedulerAcquireMutexOwnership", thread, + "mutex already belongs to an owner"); + } + + mutex->Owner = thread; + KeListInsertTailChecked(&thread->OwnedMutexList, &mutex->OwnerLink); + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Transfers one mutex directly to a blocked waiter. +void KiSchedulerTransferMutexOwnership(THREAD* oldOwner, THREAD* newOwner, + MUTEX* mutex) { + if (!oldOwner || !newOwner || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (mutex->Owner != oldOwner || + mutex->OwnerLink.Fwd == &mutex->OwnerLink || + mutex->OwnerLink.Bwd == &mutex->OwnerLink) { + KiSchedulerBug("KiSchedulerTransferMutexOwnership", oldOwner, + "mutex ownership graph is inconsistent"); + } + if (newOwner->WaitingMutex != mutex || + newOwner->State != ThreadWaiting || + newOwner->WaitObject != &mutex->Header || + newOwner->WaitListHead != &mutex->WaitList || + newOwner->SchedulerLink.Fwd != &newOwner->SchedulerLink || + newOwner->SchedulerLink.Bwd != &newOwner->SchedulerLink) { + KiSchedulerBug("KiSchedulerTransferMutexOwnership", newOwner, + "new owner was not in the mutex handoff state"); + } + + KeListRemoveChecked(&mutex->OwnerLink); + newOwner->WaitingMutex = (MUTEX*)0; + newOwner->WaitListHead = (LIST_ENTRY*)0; + mutex->Owner = newOwner; + KeListInsertTailChecked(&newOwner->OwnedMutexList, &mutex->OwnerLink); + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Removes one mutex from its current owner. +void KiSchedulerReleaseMutexOwnership(THREAD* thread, MUTEX* mutex) { + if (!thread || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (mutex->Owner != thread || + mutex->OwnerLink.Fwd == &mutex->OwnerLink || + mutex->OwnerLink.Bwd == &mutex->OwnerLink) { + KiSchedulerBug("KiSchedulerReleaseMutexOwnership", thread, + "mutex was not linked to the stated owner"); + } + + KeListRemoveChecked(&mutex->OwnerLink); + mutex->Owner = (THREAD*)0; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Returns one owned mutex while the ownership list is stable. +MUTEX* KiSchedulerFirstOwnedMutex(THREAD* thread) { + if (!thread) return (MUTEX*)0; + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + MUTEX* mutex = LIST_IS_EMPTY(&thread->OwnedMutexList) + ? (MUTEX*)0 + : CONTAINING_RECORD(thread->OwnedMutexList.Fwd, MUTEX, OwnerLink); + KeReleaseSpinlock(&gSchedLock, flags); + return mutex; +} + +// Records one transitive priority-inheritance propagation step. +void KiSchedulerNotePriorityPropagation(uint32_t depth) { + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + gSchedStats.PriorityInheritancePropagations++; + if (depth > gSchedStats.PriorityInheritanceMaxDepth) + gSchedStats.PriorityInheritanceMaxDepth = depth; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Changes one normal thread's dynamic base priority. +KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority) { + if (!thread || priority >= PriorityCount) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->BaseSchedulingClass != KeSchedulingNormal) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_ACCESS_DENIED; + } + + thread->BasePriority = (uint32_t)priority; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t effectivePriority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &effectivePriority); + (void)KiApplyEffectiveRankLocked(thread, schedulingClass, effectivePriority); + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_SUCCESS; } static int KiAnyRunnableForCpuLocked(KCPU* cpu); @@ -282,7 +703,7 @@ static int KiAnyRunnableForCurrentCpu(void) { // Implements the idle Loop operation. static void KiIdleLoop(void) { - for (;;) { + while (true) { KCPU* cpu = KeGetCurrentCpu(); cpu->SchedulerTicks++; @@ -304,8 +725,10 @@ static void KiIdleLoop(void) { // Initializes scheduler Global. void KiInitializeSchedulerGlobal(void) { - for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]); - LIST_INIT(&gSleepList); + for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gNormalRunQueue[i]); + for (uint32_t i = 0; i < KE_REALTIME_PRIORITY_COUNT; i++) LIST_INIT(&gRealtimeRunQueue[i]); + LIST_INIT(&gTimeoutList); + KeInitializeWorkItem(&gTimeoutWorkItem, KiTimeoutWorkRoutine, (void*)0); gSchedLock = (SPINLOCK)SPINLOCK_INIT; KSTATUS s = KeCreateProcess("idle", &gIdleProcess); @@ -333,14 +756,20 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { idle->Header.Destroy = (void *)0; idle->ThreadId = cpu->CpuIndex ? (0x80000000u | cpu->CpuIndex) : 0; - idle->State = ThreadRunning; - idle->Priority = PriorityLow; - idle->QuantaConsumed = 0; + idle->State = ThreadRunning; + idle->BaseSchedulingClass = KeSchedulingNormal; + idle->BasePriority = PriorityLow; + idle->EffectiveSchedulingClass = KeSchedulingNormal; + idle->EffectivePriority = PriorityLow; + idle->QuantaConsumed = 0; idle->QuantumTicks = 1; idle->QuantumRemaining = 1; idle->CpuBurstTicks = 0; idle->LastRunTicks = 0; idle->RunningCpu = cpu->CpuIndex; + idle->SuspendCount = 0; + LIST_INIT(&idle->OwnedMutexList); + idle->WaitingMutex = (MUTEX*)0; idle->Queued = 0; idle->IsIdleThread = 1; idle->SchedulerFlags = 0; @@ -351,7 +780,9 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { idle->WaitStatus = NTLSTATUS_SUCCESS; idle->WaitListHead = (LIST_ENTRY*)0; idle->AlertableWait = 0; - idle->SleepUntilTick = 0; + LIST_INIT(&idle->TimeoutLink); + idle->TimeoutArmed = 0; + idle->TimeoutTick = 0; idle->TimeoutStatus = NTLSTATUS_SUCCESS; idle->UserStackBase = (void*)0; idle->Teb = (void*)0; @@ -425,14 +856,20 @@ void KeSchedulerPostSwitch(void) { THREAD* cur = cpu->CurrentThread; KeReleaseSpinlock(&gSchedLock, KiPostSwitchUnlockCookie(cur)); - KiDeliverKernelApcs(); - if (cpu->OutgoingThread && cpu->OutgoingThread->State == ThreadDead) { THREAD* dead = cpu->OutgoingThread; cpu->OutgoingThread = (THREAD*)0; dead->RunningCpu = KE_INVALID_CPU; + + // Forced process termination can bypass KeExitThread. Finalize thread + // ownership after the dead thread switches out, before releasing the + // scheduler lifetime reference. + KeFinalizeThreadExit(dead); + KiMutexRundownThread(dead); ObDereferenceObject(&dead->Header); } + + KiDeliverKernelApcs(); } // Updates thread Affinity Mask. @@ -465,6 +902,63 @@ void KeReadyThread(THREAD* thread) { KeReleaseSpinlock(&gSchedLock, flags); } +// Decrements one thread's suspend count and queues it when the count reaches zero. +__declspec(dllexport) KSTATUS KeResumeThread(THREAD* thread, + uint32_t* previousSuspendCountOut) { + if (!thread || thread->IsIdleThread) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->State == ThreadDead) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_OBJECT_DEAD; + } + + uint32_t previous = thread->SuspendCount; + if (previous != 0) { + if (thread->State != ThreadSuspended || thread->Queued || + thread->RunningCpu != KE_INVALID_CPU) { + KiSchedulerBug("KeResumeThread", thread, + "suspend count did not match scheduler state"); + } + + thread->SuspendCount = previous - 1; + if (thread->SuspendCount == 0) { + KiInsertReadyLocked(thread); + KiPokeRunnableCpusLocked(thread); + } + } + KeReleaseSpinlock(&gSchedLock, flags); + + if (previousSuspendCountOut) + *previousSuspendCountOut = previous; + return NTLSTATUS_SUCCESS; +} + +// Cancels a thread that has never left its initial suspended state. +KSTATUS KeCancelSuspendedThread(THREAD* thread) { + if (!thread || thread->IsIdleThread) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->State != ThreadSuspended || thread->SuspendCount == 0 || + thread->Queued || thread->RunningCpu != KE_INVALID_CPU) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_INVALID_PARAMETER; + } + + thread->SuspendCount = 0; + thread->State = ThreadDead; + KeReleaseSpinlock(&gSchedLock, flags); + + KeFinalizeThreadExit(thread); + + // Drop the scheduler-owned lifetime reference. The creator still owns the + // reference returned by KeCreateThread(). + ObDereferenceObject(&thread->Header); + return NTLSTATUS_SUCCESS; +} + // Implements the unready Thread operation. void KeUnreadyThread(THREAD* thread) { if (!thread) return; @@ -480,8 +974,20 @@ void KeUnreadyThread(THREAD* thread) { static int KiAnyRunnableForCpuLocked(KCPU* cpu) { if (!cpu) cpu = KeGetCurrentCpu(); + for (uint32_t p = 0; p < KE_REALTIME_PRIORITY_COUNT; p++) { + LIST_ENTRY* head = &gRealtimeRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) + continue; + return 1; + } + } + for (int p = 0; p < PriorityCount; p++) { - LIST_ENTRY* head = &gRunQueue[p]; + LIST_ENTRY* head = &gNormalRunQueue[p]; for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); if (!KiThreadCanRunOnCpuLocked(thread, cpu)) @@ -494,28 +1000,43 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) { return 0; } +// Removes the first eligible thread from one ready queue. +static THREAD* KiPickFromQueueLocked(KCPU* cpu, LIST_ENTRY* head) { + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) { + gSchedStats.SmtSiblingDeferrals++; + continue; + } + + KeListRemoveChecked(node); + thread->Queued = 0; + KiAssertPickedThreadLocked("KiPickFromQueueLocked", thread); + return thread; + } + return (THREAD*)0; +} + // Implements the pick Next For Cpu operation. static THREAD* KiPickNextForCpu(KCPU* cpu) { if (!cpu) cpu = KeGetCurrentCpu(); - for (int p = PriorityCount - 1; p >= 0; p--) { - LIST_ENTRY* head = &gRunQueue[p]; - for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { - THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); - if (!KiThreadCanRunOnCpuLocked(thread, cpu)) - continue; - if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) { - gSchedStats.SmtSiblingDeferrals++; - continue; - } - - KeListRemoveChecked(node); - thread->Queued = 0; - KiAssertPickedThreadLocked("KiPickNextForCpu", thread); + for (int p = (int)KE_REALTIME_PRIORITY_MAX; p >= 0; p--) { + THREAD* thread = KiPickFromQueueLocked(cpu, &gRealtimeRunQueue[p]); + if (thread) { + gSchedStats.RealtimeDispatches++; return thread; } } + for (int p = PriorityCount - 1; p >= 0; p--) { + THREAD* thread = KiPickFromQueueLocked(cpu, &gNormalRunQueue[p]); + if (thread) + return thread; + } + return cpu->IdleThread; } @@ -561,6 +1082,25 @@ static void KiSchedule(void) { } THREAD* next = KiPickNextForCpu(cpu); + + // A voluntary reschedule must not let lower-ranked work displace a running + // real-time thread. Equal-ranked peers may still rotate at the quantum + // boundary, and higher-ranked real-time work may preempt immediately. + if (cur != idle && cur->State == ThreadRunning && + cur->EffectiveSchedulingClass == KeSchedulingRealtime && + next != cur && + !KiSchedulerThreadOutranks(next, cur) && + !KiRanksEqual(next->EffectiveSchedulingClass, next->EffectivePriority, + cur->EffectiveSchedulingClass, cur->EffectivePriority)) { + if (next != idle) + KiInsertReadyLocked(next); + if (cur->QuantumRemaining == 0) + KiRefillQuantumLocked(cur); + gSchedStats.KeepCurrentDecisions++; + KeReleaseSpinlock(&gSchedLock, flags); + return; + } + if (next == idle) gSchedStats.IdleSelections++; if (next == cur) { if (cur != idle) @@ -620,11 +1160,36 @@ void KiSchedulerMarkThreadWaiting(THREAD* thread) { KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread running CPU does not match current CPU"); if (thread->State != ThreadRunning) KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread was not running before wait"); + if (thread->TimeoutArmed || + thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread already has a timeout"); + } thread->State = ThreadWaiting; KeReleaseSpinlock(&gSchedLock, flags); } +// Cancels one armed deadline while the scheduler lock is held. +static void KiCancelThreadTimeoutLocked(THREAD* thread) { + if (!thread) return; + + if (thread->TimeoutArmed) { + if (thread->TimeoutLink.Fwd == &thread->TimeoutLink || + thread->TimeoutLink.Bwd == &thread->TimeoutLink) { + KiSchedulerBug("KiCancelThreadTimeoutLocked", thread, "armed timeout was not linked"); + } + KeListRemoveChecked(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + } else if (thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KiSchedulerBug("KiCancelThreadTimeoutLocked", thread, "unarmed timeout remained linked"); + } + + thread->TimeoutTick = 0; + thread->TimeoutStatus = NTLSTATUS_SUCCESS; +} + // Implements the scheduler Note Voluntary Block operation. void KiSchedulerNoteVoluntaryBlock(THREAD* thread) { if (!thread) return; @@ -643,17 +1208,17 @@ static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) { // already on the object's wait list and has State=ThreadWaiting, but it is // still executing with RunningCpu set until it actually calls KeYield(). // That case must complete in-place, not enqueue the still-running thread. - if (thread->State != ThreadWaiting) { - thread->WaitStatus = status; + // Once another path has completed the wait, a stale abort or duplicate + // signal must not replace the result that made the thread runnable. + if (thread->State != ThreadWaiting) return; - } + + KiCancelThreadTimeoutLocked(thread); thread->WaitObject = (OBJECT*)0; thread->WaitStatus = status; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; thread->CpuBurstTicks = 0; thread->QuantaConsumed = 0; @@ -667,10 +1232,15 @@ static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) { return; } - if (thread->Priority < PriorityHigh) { - thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1); + if (thread->BaseSchedulingClass == KeSchedulingNormal && + thread->BasePriority < PriorityHigh) { + thread->BasePriority++; } + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &priority); + (void)KiApplyEffectiveRankLocked(thread, schedulingClass, priority); KiInsertReadyLocked(thread); KiPokeRunnableCpusLocked(thread); } @@ -682,8 +1252,6 @@ void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) { if (thread->State == ThreadWaiting && thread->WaitListHead) { KeListRemoveChecked(&thread->SchedulerLink); thread->WaitListHead = (LIST_ENTRY*)0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; } KiWakeThreadLocked(thread, status); KeReleaseSpinlock(&gSchedLock, flags); @@ -696,22 +1264,58 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { KeReleaseSpinlock(&gSchedLock, flags); } -// Wakes threads whose scheduler-tick sleep deadline has expired. -static void KiWakeExpiredSleepersLocked(uint64_t nowTick) { - LIST_ENTRY* node = gSleepList.Fwd; - while (node != &gSleepList) { - LIST_ENTRY* next = node->Fwd; - THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); +// Detaches one expired deadline while the scheduler lock is held. +static THREAD* KiTakeExpiredTimeoutLocked(uint64_t nowTick, KSTATUS* statusOut) { + LIST_ENTRY* head = &gTimeoutList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, TimeoutLink); + if (!thread->TimeoutArmed) + KiSchedulerBug("KiTakeExpiredTimeoutLocked", thread, "timeout list contains an unarmed thread"); + if (thread->State != ThreadWaiting) + KiSchedulerBug("KiTakeExpiredTimeoutLocked", thread, "timeout list contains a non-waiting thread"); + if (thread->TimeoutTick > nowTick) + continue; - if (thread->State == ThreadWaiting && - thread->WaitListHead == &gSleepList && - thread->SleepUntilTick <= nowTick) { - KeListRemoveChecked(node); - thread->WaitListHead = (LIST_ENTRY*)0; - KiWakeThreadLocked(thread, thread->TimeoutStatus); - } + KSTATUS timeoutStatus = thread->TimeoutStatus; + KeListRemoveChecked(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + thread->TimeoutTick = 0; + thread->TimeoutStatus = NTLSTATUS_SUCCESS; + ObReferenceObject(&thread->Header); + if (statusOut) *statusOut = timeoutStatus; + return thread; + } - node = next; + return (THREAD*)0; +} + +// Reports whether one deadline is ready while the scheduler lock is held. +static int KiHasExpiredTimeoutLocked(uint64_t nowTick) { + LIST_ENTRY* head = &gTimeoutList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, TimeoutLink); + if (thread->TimeoutTick <= nowTick) + return 1; + } + return 0; +} + +// Resolves expired deadlines at passive level through each object's abort path. +static void KiTimeoutWorkRoutine(void* context) { + (void)context; + KeAssertIrql(PASSIVE_LEVEL); + + while (true) { + KSTATUS timeoutStatus = NTLSTATUS_TIMEOUT; + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + uint64_t nowTick = gSchedStats.Ticks; + THREAD* thread = KiTakeExpiredTimeoutLocked(nowTick, &timeoutStatus); + KeReleaseSpinlock(&gSchedLock, flags); + if (!thread) + return; + + (void)KeAbortThreadWait(thread, timeoutStatus); + ObDereferenceObject(&thread->Header); } } @@ -724,36 +1328,42 @@ static int KiCpuOwnsGlobalSchedulerTick(const KCPU* cpu) { return cpu && cpu->IsBsp; } -// Ages ready threads after enough global scheduler ticks have elapsed. +// Ages ready normal-class threads after enough global ticks have elapsed. static void KiAgeReadyThreadsLocked(void) { - // Periodic aging prevents indefinite starvation of PriorityLow and - // PriorityNormal threads. Ready threads promoted by aging get a fresh CPU - // burst history; they earned the boost by waiting, not by burning CPU. + // Real-time queues are static and never participate in aging. Normal + // threads promoted by aging get a fresh CPU-burst history because they + // earned the boost by waiting rather than by burning CPU. gAgingCounter++; if (gAgingCounter < SCHED_AGING_INTERVAL_TICKS) return; gAgingCounter = 0; for (int p = 0; p < PriorityHigh; p++) { - while (!LIST_IS_EMPTY(&gRunQueue[p])) { - LIST_ENTRY* node = KeListPopHeadChecked(&gRunQueue[p]); - THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); - t->Priority = PriorityHigh; - t->CpuBurstTicks = 0; - t->QuantaConsumed = 0; - t->QuantumRemaining = 0; + while (!LIST_IS_EMPTY(&gNormalRunQueue[p])) { + LIST_ENTRY* node = KeListPopHeadChecked(&gNormalRunQueue[p]); + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + thread->Queued = 0; + thread->BasePriority = PriorityHigh; + thread->CpuBurstTicks = 0; + thread->QuantaConsumed = 0; + thread->QuantumRemaining = 0; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &priority); + thread->EffectiveSchedulingClass = schedulingClass; + thread->EffectivePriority = priority; gSchedStats.AgingPromotions++; - KeListInsertTailChecked(&gRunQueue[PriorityHigh], &t->SchedulerLink); + KiInsertReadyLocked(thread); } } } // Runs scheduler work that must be single-sourced for the whole system. -static void KiRunGlobalSchedulerTickLocked(void) { +static int KiRunGlobalSchedulerTickLocked(void) { gSchedStats.Ticks++; - KiWakeExpiredSleepersLocked(gSchedStats.Ticks); KeInputTimerTick(); KiAgeReadyThreadsLocked(); + return KiHasExpiredTimeoutLocked(gSchedStats.Ticks); } // Services the current CPU's local scheduler timer interrupt. @@ -766,16 +1376,22 @@ static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) { } else if (KiThreadIsKilledLocked(cur)) { requestReschedule = 1; } else { - cur->CpuBurstTicks++; - cur->QuantaConsumed = cur->CpuBurstTicks; + if (cur->EffectiveSchedulingClass == KeSchedulingNormal) { + cur->CpuBurstTicks++; + cur->QuantaConsumed = cur->CpuBurstTicks; - if (cur->CpuBurstTicks >= SCHED_DEMOTE_AFTER_BURST_TICKS) { - cur->CpuBurstTicks = 0; - cur->QuantaConsumed = 0; - if (cur->Priority > PriorityLow) { - cur->Priority = (THREAD_PRIORITY)(cur->Priority - 1); - gSchedStats.Demotions++; - gSchedStats.CpuBurstDemotions++; + if (cur->CpuBurstTicks >= SCHED_DEMOTE_AFTER_BURST_TICKS) { + cur->CpuBurstTicks = 0; + cur->QuantaConsumed = 0; + if (cur->BasePriority > PriorityLow) { + cur->BasePriority--; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(cur, &schedulingClass, &priority); + (void)KiApplyEffectiveRankLocked(cur, schedulingClass, priority); + gSchedStats.Demotions++; + gSchedStats.CpuBurstDemotions++; + } } } @@ -786,13 +1402,15 @@ static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) { cur->QuantumRemaining--; if (cur->QuantumRemaining == 0) { - if (KiAnyRunnableForCpuLocked(cpu)) { + int havePeer = cur->EffectiveSchedulingClass == KeSchedulingRealtime + ? KiAnyRealtimePeerForCpuLocked(cpu, cur->EffectivePriority) + : KiAnyRunnableForCpuLocked(cpu); + if (havePeer) { gSchedStats.QuantumExpirations++; requestReschedule = 1; } else { - // No peer is ready, so keep the CPU hot and refill the budget. - // This avoids reintroducing the old every-other-tick idle dance - // as a pointless reschedule-pending chatter loop. + // No eligible peer is ready, so keep the CPU hot and refill the + // budget instead of turning a periodic tick into idle chatter. KiRefillQuantumLocked(cur); gSchedStats.KeepCurrentDecisions++; } @@ -810,18 +1428,21 @@ void KeSchedulerTick(void) { if (!cur) return; int ownsGlobalTick = KiCpuOwnsGlobalSchedulerTick(cpu); + int queueTimeoutWork = 0; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); KiArmPostSwitchUnlock(cur, flags); if (ownsGlobalTick) - KiRunGlobalSchedulerTickLocked(); + queueTimeoutWork = KiRunGlobalSchedulerTickLocked(); int requestReschedule = KiRunLocalSchedulerTickLocked(cpu, cur, idle); KeReleaseSpinlock(&gSchedLock, flags); if (ownsGlobalTick) { + if (queueTimeoutWork) + (void)KeQueueWorkItem(&gTimeoutWorkItem); KeTimerTick(); KeDisplayTick(); } @@ -834,7 +1455,7 @@ void KeSchedulerTick(void) { KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) { if (milliseconds == 0) return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) + if (KeGetCurrentIrql() > DISPATCH_LEVEL) return NTLSTATUS_INVALID_IRQL; THREAD* self = KeGetCurrentThread(); @@ -848,8 +1469,11 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) ticks = 1; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - if (self->State != ThreadWaiting || self->WaitListHead != (LIST_ENTRY*)0 || - self->RunningCpu != KeGetCurrentCpu()->CpuIndex) { + if (self->State != ThreadWaiting || + self->RunningCpu != KeGetCurrentCpu()->CpuIndex || + self->TimeoutArmed || + self->TimeoutLink.Fwd != &self->TimeoutLink || + self->TimeoutLink.Bwd != &self->TimeoutLink) { KeReleaseSpinlock(&gSchedLock, flags); return NTLSTATUS_INVALID_PARAMETER; } @@ -859,10 +1483,10 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) if (wakeTick < now) wakeTick = UINT64_MAX; - self->WaitListHead = &gSleepList; - self->SleepUntilTick = wakeTick; + self->TimeoutTick = wakeTick; self->TimeoutStatus = timeoutStatus; - KeListInsertTailChecked(&gSleepList, &self->SchedulerLink); + self->TimeoutArmed = 1; + KeListInsertTailChecked(&gTimeoutList, &self->TimeoutLink); KeReleaseSpinlock(&gSchedLock, flags); return NTLSTATUS_SUCCESS; } @@ -890,7 +1514,10 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) { uintptr_t flags = KeAcquireSpinlock(&gSchedLock); if (self->Queued || self->State != ThreadRunning || - self->RunningCpu != KeGetCurrentCpu()->CpuIndex) { + self->RunningCpu != KeGetCurrentCpu()->CpuIndex || + self->TimeoutArmed || + self->TimeoutLink.Fwd != &self->TimeoutLink || + self->TimeoutLink.Bwd != &self->TimeoutLink) { KeReleaseSpinlock(&gSchedLock, flags); return NTLSTATUS_INVALID_PARAMETER; } @@ -903,11 +1530,12 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) { self->State = ThreadWaiting; self->WaitObject = (OBJECT*)0; self->WaitStatus = NTLSTATUS_PENDING; - self->WaitListHead = &gSleepList; + self->WaitListHead = (LIST_ENTRY*)0; self->AlertableWait = 0; - self->SleepUntilTick = wakeTick; + self->TimeoutTick = wakeTick; self->TimeoutStatus = NTLSTATUS_SUCCESS; - KeListInsertTailChecked(&gSleepList, &self->SchedulerLink); + self->TimeoutArmed = 1; + KeListInsertTailChecked(&gTimeoutList, &self->TimeoutLink); KeReleaseSpinlock(&gSchedLock, flags); return KeCommitBlockThread(self, 0); diff --git a/code/kernel/ke_selftest.c b/code/kernel/ke_selftest.c index 7b3cab9..f745cdf 100644 --- a/code/kernel/ke_selftest.c +++ b/code/kernel/ke_selftest.c @@ -11,11 +11,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -240,9 +243,10 @@ static void KiRunKernelApcSelfTest(void) { s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: FinishedEvent create failed %d\n", s); + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker, (uintptr_t)&st, - PriorityNormal, + &workerParameters, &st.WorkerThread); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker create failed %d\n", s); @@ -310,9 +314,10 @@ static void KiRunKernelApcSelfTest(void) { s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert FinishedEvent create failed %d\n", s); + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker, (uintptr_t)&st, - PriorityNormal, + &workerParameters, &st.WorkerThread); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable worker create failed %d\n", s); @@ -584,6 +589,213 @@ static void KiRunPfnDatabaseSelfTest(void) { DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n"); } +// Verifies common-buffer, direct mapping, bounce mapping, and teardown accounting. +static void KiRunDmaSelfTest(void) { + const NTL_DDK_DMA* dma = IoGetDmaDdk(); + if (!dma) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: DDK table missing\n"); + + IO_DMA_STATS before; + IO_DMA_STATS after; + dma->QueryStats(&before); + + DRIVER_OBJECT driver; + memset(&driver, 0, sizeof(driver)); + IoInitializeDriverObject(&driver, SYS_SV_LITERAL("dma-selftest")); + + DEVICE_OBJECT* device = (DEVICE_OBJECT*)0; + KSTATUS status = IoCreateDevice(&driver, 0, &device); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: device creation failed status=%u\n", status); + + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = UINT64_MAX; + requirements.Alignment = PAGE_SIZE; + requirements.MaximumSegmentLength = PAGE_SIZE; + requirements.MaximumSegments = 4; + + IO_DMA_ADAPTER* adapter = (IO_DMA_ADAPTER*)0; + IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0; + MM_PINNED_MEMORY* pinned = (MM_PINNED_MEMORY*)0; + IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)0; + + status = dma->GetAdapter(device, &requirements, &adapter); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: adapter creation failed status=%u\n", status); + + status = dma->AllocateCommonBuffer(adapter, PAGE_SIZE, PAGE_SIZE, &buffer); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: common buffer allocation failed status=%u\n", status); + + uint8_t* bytes = (uint8_t*)dma->GetCommonBufferAddress(buffer); + uint64_t deviceAddress = dma->GetCommonBufferDeviceAddress(buffer); + if (!bytes || !deviceAddress || (deviceAddress & (PAGE_SIZE - 1)) != 0 || + dma->GetCommonBufferSize(buffer) != PAGE_SIZE) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: common buffer metadata invalid address=0x%016llX\n", + (unsigned long long)deviceAddress); + + status = dma->PinKernelMemory(bytes, PAGE_SIZE, IoDmaBidirectional, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: kernel pin failed status=%u\n", status); + status = dma->MapPinnedMemory(adapter, + pinned, + 0, + PAGE_SIZE, + IoDmaBidirectional, + IO_DMA_MAP_NO_BOUNCE, + &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 || + dma->GetSegments(mapping)[0].DeviceAddress != deviceAddress) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: direct mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + + for (size_t i = 0; i < 128; i++) + bytes[i + 1] = (uint8_t)(i ^ 0x5Au); + status = dma->PinKernelMemory(bytes + 1, 128, IoDmaToDevice, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: unaligned kernel pin failed status=%u\n", status); + status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaToDevice, 0, &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 || + (dma->GetSegments(mapping)[0].DeviceAddress & (PAGE_SIZE - 1)) != 0 || + dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: bounce mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + status = dma->SynchronizeForDevice(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: bounce synchronization failed status=%u\n", status); + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + + memset(bytes + 1, 0, 128); + status = dma->PinKernelMemory(bytes + 1, 128, IoDmaFromDevice, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive pin failed status=%u\n", status); + status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaFromDevice, 0, &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 || + dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: receive bounce mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + status = dma->SynchronizeForDevice(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive handoff failed status=%u\n", status); + + uint64_t receiveDeviceAddress = dma->GetSegments(mapping)[0].DeviceAddress; + uint8_t* receiveDeviceAlias = (uint8_t*)MmPfnToVirtual(receiveDeviceAddress & PAGE_MASK); + if (!receiveDeviceAlias) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive device alias missing\n"); + receiveDeviceAlias += receiveDeviceAddress & (PAGE_SIZE - 1); + for (size_t i = 0; i < 128; i++) + receiveDeviceAlias[i] = (uint8_t)(i ^ 0xA5u); + + status = dma->SynchronizeForCpu(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive synchronization failed status=%u\n", status); + for (size_t i = 0; i < 128; i++) { + if (bytes[i + 1] != (uint8_t)(i ^ 0xA5u)) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: receive bounce copy mismatch offset=%llu\n", + (unsigned long long)i); + } + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + + PROCESS* process = KeGetCurrentProcess(); + if (!process) + process = KeGetSystemProcess(); + if (!process) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: no process for user pinning\n"); + + void* userBase = (void*)0; + status = MmVadCreateCommitted(process, + PAGE_SIZE, + PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, + MmVadGeneric, + &userBase); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD creation failed status=%u\n", status); + status = MmVadEnsureResident(process, userBase, PAGE_SIZE, PAGE_WRITE | PAGE_USER); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD materialization failed status=%u\n", status); + ((uint8_t*)userBase)[0] = 0xD5u; + + status = dma->PinUserMemory(process, userBase, PAGE_SIZE, IoDmaToDevice, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user pin failed status=%u\n", status); + status = MmVadDestroy(process, userBase); + if (status != NTLSTATUS_ACCESS_DENIED) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: pinned user VAD was destroyable status=%u\n", + status); + status = dma->MapPinnedMemory(adapter, + pinned, + 0, + PAGE_SIZE, + IoDmaToDevice, + IO_DMA_MAP_NO_BOUNCE, + &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: user mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + status = dma->SynchronizeForDevice(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user synchronization failed status=%u\n", status); + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + status = MmVadDestroy(process, userBase); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD destroy failed status=%u\n", status); + + dma->FreeCommonBuffer(buffer); + dma->PutAdapter(adapter); + IoDeleteDevice(device); + + dma->QueryStats(&after); + if (after.AdaptersCreated != before.AdaptersCreated + 1 || + after.AdaptersDestroyed != before.AdaptersDestroyed + 1 || + after.AdaptersInUse != before.AdaptersInUse || + after.CommonBuffersAllocated < before.CommonBuffersAllocated + 3 || + after.CommonBuffersFreed < before.CommonBuffersFreed + 3 || + after.CommonBuffersInUse != before.CommonBuffersInUse || + after.CommonBufferBytesInUse != before.CommonBufferBytesInUse || + after.PinnedRangesInUse != before.PinnedRangesInUse || + after.PinnedPagesInUse != before.PinnedPagesInUse || + after.PinnedKernelRanges != before.PinnedKernelRanges + 3 || + after.PinnedUserRanges != before.PinnedUserRanges + 1 || + after.PinnedRangesReleased != before.PinnedRangesReleased + 4 || + after.StreamingMappings != before.StreamingMappings + 4 || + after.StreamingMappingsUnmapped != before.StreamingMappingsUnmapped + 4 || + after.StreamingMappingsInUse != before.StreamingMappingsInUse || + after.DirectMappings != before.DirectMappings + 2 || + after.BounceMappings != before.BounceMappings + 2 || + after.BounceBytesCopiedToDevice < before.BounceBytesCopiedToDevice + 128 || + after.BounceBytesCopiedToCpu < before.BounceBytesCopiedToCpu + 128) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: accounting mismatch\n"); + + DbgPrint("DMA selftest: common/kernel-pin/user-pin/direct/send-bounce/receive-bounce/teardown PASS\n"); +} + // Verifies the page-table manager invariants that the higher VMM will depend on. static void KiRunMemoryMappingSelfTest(void) { PHYSICAL_ADDRESS phys = 0; @@ -1155,6 +1367,380 @@ static void KiRunAsyncFileIoSelfTest(void) { DbgPrint("FILE/ASYNC selftest: PASS\n"); } +typedef struct KI_SCHEDULER_SELFTEST_STATE { + MUTEX* GateMutex; + MUTEX* LinkMutex; + EVENT* ChainReadyEvent; + EVENT* RealtimeReadyEvent; + EVENT* ChainFinishedEvent; + EVENT* RealtimeFinishedEvent; + volatile KSTATUS ChainAcquireStatus; + volatile KSTATUS RealtimeAcquireStatus; +} KI_SCHEDULER_SELFTEST_STATE; + +// Waits until one worker has entered the expected mutex wait state. +static KE_THREAD_SCHEDULER_SNAPSHOT KiSchedulerAwaitMutexWait(THREAD* thread, MUTEX* mutex) { + KE_THREAD_SCHEDULER_SNAPSHOT snapshot; + memset(&snapshot, 0, sizeof(snapshot)); + + for (uint32_t attempt = 0; attempt < 100; attempt++) { + KiSchedulerSnapshotThread(thread, &snapshot); + if (snapshot.State == ThreadWaiting && snapshot.WaitingMutex == mutex) + return snapshot; + + KSTATUS status = KeDelayThread(SCHED_TICK_MS); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: state wait failed status=%u\n", status); + } + + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: thread %u did not block on mutex=0x%016llX state=%u waiting=0x%016llX\n", + thread ? thread->ThreadId : 0, + (uint64_t)mutex, + (unsigned)snapshot.State, + (uint64_t)snapshot.WaitingMutex); +} + +typedef struct KI_SYNC_TIMEOUT_SELFTEST_STATE { + MUTEX* Mutex; + EVENT* ReadyEvent; + EVENT* FinishedEvent; + volatile KSTATUS MutexStatus; +} KI_SYNC_TIMEOUT_SELFTEST_STATE; + +// Waits on the held mutex long enough for the timeout engine to remove donation state. +static void KiSynchronizationTimeoutWorker(void* argument) { + KI_SYNC_TIMEOUT_SELFTEST_STATE* state = (KI_SYNC_TIMEOUT_SELFTEST_STATE*)argument; + KeSignalEvent(state->ReadyEvent); + + state->MutexStatus = KeAcquireMutex(state->Mutex, SCHED_TICK_MS * 20ULL); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(state->MutexStatus)) + KeReleaseMutex(state->Mutex); + + KeSignalEvent(state->FinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +// Verifies finite mutex and semaphore waits consume no object state after timeout. +static void KiRunSynchronizationTimeoutSelfTest(void) { + DbgPrint("SYNC/TIMEOUT selftest: begin\n"); + + KI_SYNC_TIMEOUT_SELFTEST_STATE state; + memset(&state, 0, sizeof(state)); + state.MutexStatus = NTLSTATUS_PENDING; + + KSTATUS status = KeMutexCreate(&state.Mutex); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: mutex create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.ReadyEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: ready event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.FinishedEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finished event create failed status=%u\n", status); + + status = KeAcquireMutex(state.Mutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: owner acquire failed status=%u\n", status); + + THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_REALTIME(12); + status = KeCreateKernelThread((uintptr_t)KiSynchronizationTimeoutWorker, + (uintptr_t)&state, + ¶meters, + &worker); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: worker create failed status=%u\n", status); + + status = KeWaitForEvent(state.ReadyEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: ready wait failed status=%u\n", status); + + (void)KiSchedulerAwaitMutexWait(worker, state.Mutex); + + THREAD* self = KeGetCurrentThread(); + KE_THREAD_SCHEDULER_SNAPSHOT snapshot; + KiSchedulerSnapshotThread(self, &snapshot); + if (snapshot.EffectiveSchedulingClass != KeSchedulingRealtime || + snapshot.EffectivePriority != 12) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finite waiter donation missing class=%u priority=%u\n", + snapshot.EffectiveSchedulingClass, + snapshot.EffectivePriority); + } + + status = KeWaitForEvent(state.FinishedEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finished wait failed status=%u\n", status); + if (state.MutexStatus != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: mutex wait returned status=%u\n", + state.MutexStatus); + + KiSchedulerSnapshotThread(self, &snapshot); + if (snapshot.EffectiveSchedulingClass != snapshot.BaseSchedulingClass || + snapshot.EffectivePriority != snapshot.BasePriority) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: timed-out donation remained base=%u/%u effective=%u/%u\n", + snapshot.BaseSchedulingClass, + snapshot.BasePriority, + snapshot.EffectiveSchedulingClass, + snapshot.EffectivePriority); + } + + KeReleaseMutex(state.Mutex); + status = KeAcquireMutex(state.Mutex, 0); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: zero-timeout mutex acquire failed status=%u\n", status); + KeReleaseMutex(state.Mutex); + + SEMAPHORE* semaphore = (SEMAPHORE*)0; + status = KeSemaphoreCreate(0, 1, &semaphore); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: semaphore create failed status=%u\n", status); + + status = KeWaitSemaphore(semaphore, 0); + if (status != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: zero-timeout semaphore wait returned status=%u\n", status); + + status = KeWaitSemaphore(semaphore, SCHED_TICK_MS * 3ULL); + if (status != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finite semaphore wait returned status=%u\n", status); + + KeSignalSemaphore(semaphore); + status = KeWaitSemaphore(semaphore, 0); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: signaled semaphore probe failed status=%u\n", status); + status = KeWaitSemaphore(semaphore, 0); + if (status != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: semaphore count was consumed twice status=%u\n", status); + + ObDereferenceObject(&worker->Header); + ObDereferenceObject(&semaphore->Header); + ObDereferenceObject(&state.Mutex->Header); + ObDereferenceObject(&state.ReadyEvent->Header); + ObDereferenceObject(&state.FinishedEvent->Header); + + DbgPrint("SYNC/TIMEOUT selftest: PASS\n"); +} + +// Owns the link mutex and then blocks behind the self-test thread. +static void KiSchedulerChainWorker(void* argument) { + KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument; + + KSTATUS status = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain link acquire failed status=%u\n", status); + + KeSignalEvent(state->ChainReadyEvent); + state->ChainAcquireStatus = KeAcquireMutex(state->GateMutex, KE_WAIT_INFINITE); + if (state->ChainAcquireStatus != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain gate acquire failed status=%u\n", + state->ChainAcquireStatus); + + KeReleaseMutex(state->GateMutex); + KeReleaseMutex(state->LinkMutex); + KeSignalEvent(state->ChainFinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +// Blocks a real-time thread behind the normal chain worker. +static void KiSchedulerRealtimeWorker(void* argument) { + KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument; + KeSignalEvent(state->RealtimeReadyEvent); + + state->RealtimeAcquireStatus = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE); + if (state->RealtimeAcquireStatus != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time acquire failed status=%u\n", + state->RealtimeAcquireStatus); + + KeReleaseMutex(state->LinkMutex); + KeSignalEvent(state->RealtimeFinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +// Verifies real-time dispatch and transitive mutex priority inheritance. +static void KiRunRealtimeSchedulerSelfTest(void) { + DbgPrint("SCHED selftest: begin\n"); + + THREAD* self = KeGetCurrentThread(); + if (!self || self->BaseSchedulingClass != KeSchedulingNormal) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: caller is not a normal thread\n"); + + KE_SCHED_STATS before; + KE_SCHED_STATS after; + KeGetSchedulerStats(&before); + + KI_SCHEDULER_SELFTEST_STATE state; + memset(&state, 0, sizeof(state)); + state.ChainAcquireStatus = NTLSTATUS_PENDING; + state.RealtimeAcquireStatus = NTLSTATUS_PENDING; + + KSTATUS status = KeMutexCreate(&state.GateMutex); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: gate mutex create failed status=%u\n", status); + + status = KeMutexCreate(&state.LinkMutex); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: link mutex create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.ChainReadyEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain ready event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.RealtimeReadyEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time ready event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.ChainFinishedEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain finished event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.RealtimeFinishedEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time finished event create failed status=%u\n", status); + + status = KeAcquireMutex(state.GateMutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: owner acquire failed status=%u\n", status); + + THREAD* chainWorker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS chainParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + status = KeCreateKernelThread((uintptr_t)KiSchedulerChainWorker, + (uintptr_t)&state, + &chainParameters, + &chainWorker); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain worker create failed status=%u\n", status); + + status = KeWaitForEvent(state.ChainReadyEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain ready wait failed status=%u\n", status); + + (void)KiSchedulerAwaitMutexWait(chainWorker, state.GateMutex); + + status = KeAcquireMutex(state.LinkMutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_DEADLOCK) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: mutex cycle was not rejected status=%u\n", status); + + THREAD* realtimeWorker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS realtimeParameters = KE_THREAD_CREATE_REALTIME(16); + status = KeCreateKernelThread((uintptr_t)KiSchedulerRealtimeWorker, + (uintptr_t)&state, + &realtimeParameters, + &realtimeWorker); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time worker create failed status=%u\n", status); + + status = KeWaitForEvent(state.RealtimeReadyEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time ready wait failed status=%u\n", status); + + KE_THREAD_SCHEDULER_SNAPSHOT realtimeSnapshot = + KiSchedulerAwaitMutexWait(realtimeWorker, state.LinkMutex); + (void)realtimeSnapshot; + + KE_THREAD_SCHEDULER_SNAPSHOT chainSnapshot; + KE_THREAD_SCHEDULER_SNAPSHOT selfSnapshot; + KiSchedulerSnapshotThread(chainWorker, &chainSnapshot); + KiSchedulerSnapshotThread(self, &selfSnapshot); + + if (chainSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime || + chainSnapshot.EffectivePriority != 16) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: intermediate donation missing class=%u priority=%u\n", + chainSnapshot.EffectiveSchedulingClass, + chainSnapshot.EffectivePriority); + + if (selfSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime || + selfSnapshot.EffectivePriority != 16) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: transitive donation missing class=%u priority=%u\n", + selfSnapshot.EffectiveSchedulingClass, + selfSnapshot.EffectivePriority); + + KeReleaseMutex(state.GateMutex); + + KiSchedulerSnapshotThread(self, &selfSnapshot); + if (selfSnapshot.EffectiveSchedulingClass != selfSnapshot.BaseSchedulingClass || + selfSnapshot.EffectivePriority != selfSnapshot.BasePriority) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: inherited priority was not restored base=%u/%u effective=%u/%u\n", + selfSnapshot.BaseSchedulingClass, + selfSnapshot.BasePriority, + selfSnapshot.EffectiveSchedulingClass, + selfSnapshot.EffectivePriority); + + status = KeWaitForEvent(state.ChainFinishedEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain finished wait failed status=%u\n", status); + + status = KeWaitForEvent(state.RealtimeFinishedEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time finished wait failed status=%u\n", status); + + if (state.ChainAcquireStatus != NTLSTATUS_SUCCESS || + state.RealtimeAcquireStatus != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: handoff statuses chain=%u realtime=%u\n", + state.ChainAcquireStatus, + state.RealtimeAcquireStatus); + + KeGetSchedulerStats(&after); + if (after.RealtimeDispatches <= before.RealtimeDispatches || + after.PriorityInheritanceBoosts < before.PriorityInheritanceBoosts + 2 || + after.PriorityInheritanceDrops < before.PriorityInheritanceDrops + 2 || + after.PriorityInheritancePropagations < before.PriorityInheritancePropagations + 2 || + after.PriorityInheritanceMaxDepth < 2) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: scheduler counters did not prove transitive inheritance\n"); + + ObDereferenceObject(&chainWorker->Header); + ObDereferenceObject(&realtimeWorker->Header); + ObDereferenceObject(&state.GateMutex->Header); + ObDereferenceObject(&state.LinkMutex->Header); + ObDereferenceObject(&state.ChainReadyEvent->Header); + ObDereferenceObject(&state.RealtimeReadyEvent->Header); + ObDereferenceObject(&state.ChainFinishedEvent->Header); + ObDereferenceObject(&state.RealtimeFinishedEvent->Header); + + DbgPrint("SCHED selftest: PASS\n"); +} + // Runs trap Frame Self Test. static void KiRunTrapFrameSelfTest(void) { DbgPrint("TRAP selftest: begin\n"); @@ -1251,8 +1837,11 @@ void KiRunEarlyKernelSelfTests(void) { // Runs kernel Self Tests. void KiRunKernelSelfTests(const HAL_API* hal) { DbgPrint("Kernel selftests: begin\n"); + KiRunSynchronizationTimeoutSelfTest(); + KiRunRealtimeSchedulerSelfTest(); KiRunKernelApcSelfTest(); KiRunPfnDatabaseSelfTest(); + KiRunDmaSelfTest(); KiRunMemoryMappingSelfTest(); KiRunVadSparseSelfTest(); KiRunVadCopyOnWriteSelfTest(); diff --git a/code/kernel/ke_semaphore.c b/code/kernel/ke_semaphore.c index 7daa3fc..73ceb70 100644 --- a/code/kernel/ke_semaphore.c +++ b/code/kernel/ke_semaphore.c @@ -26,10 +26,10 @@ __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SE return NTLSTATUS_SUCCESS; } -// Waits for semaphore. -__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) { +// Waits for one semaphore count within the requested timeout. +__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem, uint64_t timeoutMilliseconds) { if (!sem) return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; + if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL; THREAD* self = KeGetCurrentThread(); uintptr_t flags = KeAcquireSpinlock(&sem->Lock); @@ -39,24 +39,26 @@ __declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) { return NTLSTATUS_SUCCESS; } + if (timeoutMilliseconds == 0) { + KeReleaseSpinlock(&sem->Lock, flags); + return NTLSTATUS_TIMEOUT; + } + KePrepareBlockThread(self, &sem->Header, 0); KeListInsertTailChecked(&sem->WaitList, &self->SchedulerLink); KeSetThreadWaitList(self, &sem->WaitList); - KeReleaseSpinlock(&sem->Lock, flags); - return KeCommitBlockThread(self, 0); -} - -// Implements the try Semaphore operation. -__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) { - if (!sem) return NTLSTATUS_INVALID_PARAMETER; - uintptr_t flags = KeAcquireSpinlock(&sem->Lock); - if (sem->Count > 0) { - sem->Count--; - KeReleaseSpinlock(&sem->Lock, flags); - return NTLSTATUS_SUCCESS; + if (timeoutMilliseconds != KE_WAIT_INFINITE) { + KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT); + if (!KSUCCESS(timeoutStatus)) { + KeListRemoveChecked(&self->SchedulerLink); + KeSetThreadWaitList(self, (LIST_ENTRY*)0); + KeUnblockThreadWithStatus(self, timeoutStatus); + KeReleaseSpinlock(&sem->Lock, flags); + return timeoutStatus; + } } KeReleaseSpinlock(&sem->Lock, flags); - return NTLSTATUS_FAILURE; + return KeCommitBlockThread(self, 0); } // Signals semaphore. diff --git a/code/kernel/ke_smp.c b/code/kernel/ke_smp.c index 1607242..44f0321 100644 --- a/code/kernel/ke_smp.c +++ b/code/kernel/ke_smp.c @@ -289,6 +289,8 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) { if ((cr4 & AP_CR4_REQUIRED) != AP_CR4_REQUIRED) __writecr4(cr4 | AP_CR4_REQUIRED); + KeInitializeMemoryTypes(); + if (gApBootBlock) { gApBootBlock->Stage = AP_STAGE_ENTRY; gApBootBlock->State = AP_STATE_KERNEL_ENTRY; @@ -320,7 +322,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) { gApBootBlock->State = AP_STATE_KERNEL_PARKED; } - for (;;) { + while (true) { cpu->SchedulerTicks++; if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) { @@ -980,11 +982,12 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) { cpuIndex, st); } - st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker, - (uintptr_t)arg, - PriorityNormal, - (1ULL << cpuIndex), - (THREAD**)0); + KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + workerParameters.AffinityMask = 1ULL << cpuIndex; + st = KeCreateKernelThread((uintptr_t)KiSmpApControlledWorker, + (uintptr_t)arg, + &workerParameters, + (THREAD**)0); if (!KSUCCESS(st)) { KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: AP%u worker create failed status=0x%x\n", diff --git a/code/kernel/ke_switch.asm b/code/kernel/ke_switch.asm index f7b7874..09709fb 100644 --- a/code/kernel/ke_switch.asm +++ b/code/kernel/ke_switch.asm @@ -2,16 +2,16 @@ ; ; Struct offsets: ; sizeof(OBJECT) = 24 (type:4, refcount:4, state:4, flags:4, destroy:8) -; THREAD.SavedContext offset = 96 -; THREAD.TssRsp0 offset = 112 -; THREAD.Teb offset = 120 -; THREAD.FpuContext offset = 176 +; THREAD.SavedContext offset = 112 +; THREAD.TssRsp0 offset = 128 +; THREAD.Teb offset = 136 +; THREAD.FpuContext offset = 192 ; TSS64.RSP0 offset = 4 ; -THREAD_SavedContext EQU 96 -THREAD_TssRsp0 EQU 112 -THREAD_Teb EQU 120 -THREAD_FpuContext EQU 176 +THREAD_SavedContext EQU 112 +THREAD_TssRsp0 EQU 128 +THREAD_Teb EQU 136 +THREAD_FpuContext EQU 192 TSS_RSP0_OFFSET EQU 4 EXTERN KeSetCurrentTssRsp0 : PROC @@ -131,12 +131,14 @@ KeStartInitialThread ENDP ; We load RCX from the stack (first arg per MS x64), then IRETQ. ; ------------------------------------------------------------------------- EXTERN KeSchedulerPostSwitch : PROC +EXTERN KePrepareUserThreadStart : PROC EXTERN KeGetCurrentThread : PROC EXTERN KeExitThread : PROC KeThreadEntry PROC sub rsp, 40 ; Reserve shadow space and restore MS x64 call alignment. call KeSchedulerPostSwitch + call KePrepareUserThreadStart add rsp, 40 pop rcx ; load user arg swapgs ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU diff --git a/code/kernel/ke_syscall.c b/code/kernel/ke_syscall.c index ff87faa..969376f 100644 --- a/code/kernel/ke_syscall.c +++ b/code/kernel/ke_syscall.c @@ -71,7 +71,9 @@ #include #include #include +#include #include +#include #include // HAL pointer - set once at boot by KeSyscallSetHal(). @@ -182,7 +184,8 @@ typedef struct SURFACE_OBJECT { uint64_t PresentBytesCopied; PHYSICAL_ADDRESS* Frames; void* UserBase; - uint32_t* RowScratch; + MM_KERNEL_VIEW* KernelView; + void* KernelBase; MUTEX* Lock; } SURFACE_OBJECT; @@ -274,6 +277,12 @@ static void KiSurfaceDestroy(OBJECT* obj) { surface->UserBase = (void*)0; } + if (surface->KernelView) { + MmDestroyKernelView(surface->KernelView); + surface->KernelView = (MM_KERNEL_VIEW*)0; + surface->KernelBase = (void*)0; + } + if (surface->Lock) { ObDereferenceObject(&surface->Lock->Header); surface->Lock = (MUTEX*)0; @@ -288,52 +297,19 @@ static void KiSurfaceDestroy(OBJECT* obj) { surface->Frames = (PHYSICAL_ADDRESS*)0; } - if (surface->RowScratch) { - MmFreePool(surface->RowScratch); - surface->RowScratch = (uint32_t*)0; - } } -typedef struct SURFACE_ROW_CONTEXT { - SURFACE_OBJECT* Surface; - uint32_t SourceX; - uint32_t SourceY; -} SURFACE_ROW_CONTEXT; - -static KSTATUS KiSurfaceCopyRow(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount) { - SURFACE_ROW_CONTEXT* ctx = (SURFACE_ROW_CONTEXT*)context; - if (!ctx || !ctx->Surface || !scratch || rowIndex < 0) +// Creates the process mapping used for user drawing. +static KSTATUS KiSurfaceEnsureViewLocked(SURFACE_OBJECT* surface) { + if (!surface || !surface->Owner || !surface->Frames || !surface->PageCount) return NTLSTATUS_INVALID_PARAMETER; + if (surface->UserBase) + return NTLSTATUS_SUCCESS; - SURFACE_OBJECT* surface = ctx->Surface; - uint64_t row = (uint64_t)ctx->SourceY + (uint64_t)(uint32_t)rowIndex; - uint64_t offset = row * (uint64_t)surface->PitchBytes + (uint64_t)ctx->SourceX * sizeof(uint32_t); - uint64_t bytes = (uint64_t)pixelCount * sizeof(uint32_t); - if (offset > (uint64_t)surface->PayloadSize || bytes > (uint64_t)surface->PayloadSize - offset) - return NTLSTATUS_INVALID_PARAMETER; - - uint8_t* dst = (uint8_t*)scratch; - while (bytes) { - size_t frameIndex = (size_t)(offset / PAGE_SIZE); - size_t pageOffset = (size_t)(offset & (PAGE_SIZE - 1)); - if (frameIndex >= surface->PageCount || !surface->Frames[frameIndex]) - return NTLSTATUS_INVALID_PARAMETER; - - size_t chunk = PAGE_SIZE - pageOffset; - if (chunk > bytes) - chunk = (size_t)bytes; - - const uint8_t* src = (const uint8_t*)MmPfnToVirtual(surface->Frames[frameIndex]); - if (!src) - return NTLSTATUS_INVALID_PARAMETER; - memcpy(dst, src + pageOffset, chunk); - - dst += chunk; - offset += chunk; - bytes -= chunk; - } - - return NTLSTATUS_SUCCESS; + return MmVadCreateSharedViewFromFrames(surface->Owner, surface->Frames, surface->PageCount, + surface->PayloadSize, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, MmVadShared, &surface->UserBase); } // Implements the syscall Init operation. @@ -513,8 +489,9 @@ static uint64_t SyscallExitThread(uint64_t a0, uint64_t a1, uint64_t a2, } // SYS_CREATE_THREAD a0=*SYS_CREATE_THREAD_REQUEST. -// Creates a runnable user thread and optionally returns both its waitable -// handle and diagnostic TID. The handle is the synchronization identity. +// Creates a user thread and optionally returns both its waitable handle and +// diagnostic TID. Suspended creation leaves the thread off every ready queue +// until SYS_RESUME_THREAD decrements its initial suspend count. static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; @@ -527,8 +504,11 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, if (!proc) return NTLSTATUS_FAILURE; if (!req.Entry) return NTLSTATUS_INVALID_PARAMETER; if (req.Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; - if (req.Flags) return NTLSTATUS_INVALID_PARAMETER; + if (req.Flags & ~SYS_CREATE_THREAD_FLAG_SUSPENDED) + return NTLSTATUS_INVALID_PARAMETER; if (!req.ThreadHandleOut && !req.ThreadIdOut) return NTLSTATUS_INVALID_PARAMETER; + if ((req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) && !req.ThreadHandleOut) + return NTLSTATUS_INVALID_PARAMETER; s = MmProbeUserRead((const void*)req.Entry, 1); if (!KSUCCESS(s)) return s; @@ -542,16 +522,22 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, } THREAD* newt = NULL; + KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(req.Priority); + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + threadParameters.Flags |= KE_THREAD_CREATE_FLAG_SUSPENDED; s = KeCreateThread(proc, req.Entry, req.Argument, - (THREAD_PRIORITY)req.Priority, &newt); + &threadParameters, &newt); if (!KSUCCESS(s)) return s; uint32_t threadHandle = 0; if (req.ThreadHandleOut) { s = KeAllocateHandleWithRights(proc, &newt->Header, - HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY, + HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY | + HANDLE_RIGHT_RESUME, &threadHandle); if (!KSUCCESS(s)) { + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + (void)KeCancelSuspendedThread(newt); ObDereferenceObject(&newt->Header); return s; } @@ -561,6 +547,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadHandleOut, threadHandle); if (!KSUCCESS(s)) { KeCloseHandle(proc, threadHandle); + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + (void)KeCancelSuspendedThread(newt); ObDereferenceObject(&newt->Header); return s; } @@ -570,6 +558,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadIdOut, newt->ThreadId); if (!KSUCCESS(s)) { if (threadHandle) KeCloseHandle(proc, threadHandle); + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + (void)KeCancelSuspendedThread(newt); ObDereferenceObject(&newt->Header); return s; } @@ -579,6 +569,35 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, return NTLSTATUS_SUCCESS; } +// SYS_RESUME_THREAD a0=threadHandle, a1=previousSuspendCountOut. +static uint64_t SyscallResumeThread(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a2; (void)a3; (void)a4; (void)a5; + if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(uint32_t)); + if (!KSUCCESS(status)) return status; + + PROCESS* process = KeGetCurrentProcess(); + if (!process) return NTLSTATUS_FAILURE; + + OBJECT* object = KeLookupHandleWithRights(process, (uint32_t)a0, + HANDLE_RIGHT_RESUME); + if (!object) return NTLSTATUS_INVALID_HANDLE; + if (object->Type != ObjectTypeThread) { + ObDereferenceObject(object); + return NTLSTATUS_INVALID_HANDLE; + } + + THREAD* thread = CONTAINING_RECORD(object, THREAD, Header); + uint32_t previousSuspendCount = 0; + status = KeResumeThread(thread, &previousSuspendCount); + ObDereferenceObject(object); + if (!KSUCCESS(status)) return status; + + return KiCopyU32ToUser(a1, previousSuspendCount); +} + static void KiFinishAlertableUserWait(uint8_t alertable) { if (!alertable) return; @@ -619,6 +638,9 @@ static KSTATUS KiHandleToWaitEvent(PROCESS* proc, uint32_t handle, EVENT** event case ObjectTypeDisplayClock: event = KeDisplayClockWaitEvent((DISPLAY_CLOCK_OBJECT*)obj); break; + case ObjectTypeAudioStream: + event = KeAudioStreamWaitEvent((KE_AUDIO_STREAM*)obj); + break; default: ObDereferenceObject(obj); return NTLSTATUS_NOT_SUPPORTED; @@ -656,28 +678,14 @@ static uint64_t SyscallWaitSingle(uint64_t a0, uint64_t a1, uint64_t a2, if (obj->Type == ObjectTypeMutex) { if (alertable) s = NTLSTATUS_NOT_SUPPORTED; - else if (timeoutMs == 0) { - s = KeTryAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header)); - if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT; - } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) { - s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header)); - } else { - s = NTLSTATUS_NOT_SUPPORTED; - } + else s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header), timeoutMs); ObDereferenceObject(obj); return s; } if (obj->Type == ObjectTypeSemaphore) { if (alertable) s = NTLSTATUS_NOT_SUPPORTED; - else if (timeoutMs == 0) { - s = KeTrySemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header)); - if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT; - } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) { - s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header)); - } else { - s = NTLSTATUS_NOT_SUPPORTED; - } + else s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header), timeoutMs); ObDereferenceObject(obj); return s; } @@ -992,8 +1000,7 @@ static uint64_t SyscallSetThreadPriority(uint64_t a0, uint64_t a1, uint64_t a2, if (a0 >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; THREAD* t = KeGetCurrentThread(); if (!t) return NTLSTATUS_FAILURE; - t->Priority = (THREAD_PRIORITY)a0; - return NTLSTATUS_SUCCESS; + return KiSchedulerSetNormalPriority(t, (THREAD_PRIORITY)a0); } // SYS_VM_* page-granular user virtual memory. @@ -1163,7 +1170,8 @@ static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex, HANDLE_RIGHT_WAIT); if (!mx) return NTLSTATUS_NOT_FOUND; - KSTATUS s = KeTryAcquireMutex(mx); + KSTATUS s = KeAcquireMutex(mx, 0); + if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE; ObDereferenceObject(&mx->Header); return s; } @@ -1204,7 +1212,8 @@ static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore, HANDLE_RIGHT_WAIT); if (!sem) return NTLSTATUS_NOT_FOUND; - KSTATUS s = KeTrySemaphore(sem); + KSTATUS s = KeWaitSemaphore(sem, 0); + if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE; ObDereferenceObject(&sem->Header); return s; } @@ -1672,7 +1681,7 @@ static void* KiAllocateSyscallIoBuffer(size_t requested, size_t* actualOut) { if (!requested) return NULL; size_t chunk = requested < SYSCALL_FILE_IO_CHUNK_BYTES ? requested : SYSCALL_FILE_IO_CHUNK_BYTES; - for (;;) { + while (true) { void* buf = MmAllocatePoolTag(chunk, MM_POOL_TAG('s', 'i', 'o', ' ')); if (buf) { if (actualOut) *actualOut = chunk; @@ -2275,9 +2284,8 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2, surface->PageCount = pageCount; surface->Frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS), MM_POOL_TAG('s', 'f', 'r', 'm')); - surface->RowScratch = (uint32_t*)MmAllocatePoolTag(rowBytes, MM_POOL_TAG('s', 'r', 'o', 'w')); s = KeMutexCreate(&surface->Lock); - if (!surface->Frames || !surface->RowScratch || !KSUCCESS(s)) { + if (!surface->Frames || !KSUCCESS(s)) { ObDereferenceObject(&surface->Header); return NTLSTATUS_NO_MEMORY; } @@ -2295,6 +2303,14 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2, memset(kva, 0, PAGE_SIZE); } + s = MmCreateKernelViewFromFrames(surface->Frames, surface->PageCount, + &surface->KernelView); + if (!KSUCCESS(s)) { + ObDereferenceObject(&surface->Header); + return s; + } + surface->KernelBase = MmKernelViewBase(surface->KernelView); + uint32_t handle = 0; s = KiAllocHandleForCurrentWithRights(&surface->Header, HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE | HANDLE_RIGHT_QUERY, &handle); if (!KSUCCESS(s)) @@ -2327,18 +2343,13 @@ static uint64_t SyscallMapSurface(uint64_t a0, uint64_t a1, uint64_t a2, return NTLSTATUS_ACCESS_DENIED; } - KSTATUS s = KeAcquireMutex(surface->Lock); - if (!KSUCCESS(s)) { + KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s)) { ObDereferenceObject(&surface->Header); return s; } - if (!surface->UserBase) { - s = MmVadCreateSharedViewFromFrames(current, surface->Frames, surface->PageCount, - surface->PayloadSize, - PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, - MM_VAD_ZERO, MmVadShared, &surface->UserBase); - } + s = KiSurfaceEnsureViewLocked(surface); void* userBase = surface->UserBase; uint32_t pitchBytes = surface->PitchBytes; @@ -2374,8 +2385,8 @@ static uint64_t SyscallQuerySurface(uint64_t a0, uint64_t a1, uint64_t a2, SYS_SURFACE_DESC desc; memset(&desc, 0, sizeof(desc)); - KSTATUS s = KeAcquireMutex(surface->Lock); - if (KSUCCESS(s)) { + KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(s)) { desc.Width = surface->Width; desc.Height = surface->Height; desc.PitchBytes = surface->PitchBytes; @@ -2475,20 +2486,21 @@ static uint64_t SyscallPresentSurface(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t rowsCopied = 0; uint64_t bytesCopied = 0; - KSTATUS s = NTLSTATUS_SUCCESS; - for (uint32_t i = 0; i < rectCount; ++i) { + KSTATUS s = surface->KernelBase ? NTLSTATUS_SUCCESS : NTLSTATUS_DEVICE_ERROR; + const uint8_t* sourceBase = (const uint8_t*)surface->KernelBase; + + for (uint32_t i = 0; i < rectCount && KSUCCESS(s); ++i) { DISPLAY_RECT clipped; if (!KiClipSurfaceRect(surface, &rects[i], &clipped)) { emptyClips++; continue; } - SURFACE_ROW_CONTEXT ctx; - ctx.Surface = surface; - ctx.SourceX = (uint32_t)clipped.X; - ctx.SourceY = (uint32_t)clipped.Y; - s = KeDisplayBlitRows(clipped.X, clipped.Y, clipped.W, clipped.H, - KiSurfaceCopyRow, &ctx, surface->RowScratch); + const uint32_t* source = (const uint32_t*)(sourceBase + + (size_t)(uint32_t)clipped.Y * surface->PitchBytes + + (size_t)(uint32_t)clipped.X * sizeof(uint32_t)); + s = KeDisplayPresentPitched(clipped.X, clipped.Y, clipped.W, clipped.H, + source, surface->PitchBytes); if (!KSUCCESS(s)) break; @@ -2657,7 +2669,7 @@ static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; if (!a0) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { char ch = KeInputReadChar(); if (ch) return CopyToUser((void*)(uintptr_t)a0, &ch, sizeof(ch)); @@ -2674,7 +2686,7 @@ static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; if (!a0) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { INPUT_KEY key = KeInputReadKey(); if (key != INPUT_KEY_NONE) return CopyToUser((void*)(uintptr_t)a0, &key, sizeof(key)); @@ -2722,7 +2734,7 @@ static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2, if (!KSUCCESS(s)) return s; INPUT_KEY_RECORD record; - for (;;) { + while (true) { bool available = queue ? KeInputQueueReadEvent(queue, &record) : KeInputReadEvent(&record); if (available) { @@ -2759,7 +2771,7 @@ static uint64_t SyscallKbPeekInputEvent(uint64_t a0, uint64_t a1, uint64_t a2, if (!KSUCCESS(s)) return s; INPUT_KEY_RECORD record; - for (;;) { + while (true) { bool available = queue ? KeInputQueuePeekEvent(queue, &record) : KeInputPeekEvent(&record); if (available) { @@ -2995,7 +3007,8 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, THREAD* thread; uintptr_t entryPoint = LdrGetEntryPoint(imageBase); - s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread); + const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + s = KeCreateThread(proc, entryPoint, 0, &threadParameters, &thread); if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint); MmVadDebugDumpProcess(proc); @@ -3070,7 +3083,7 @@ static uint64_t SyscallGetProcessExitCode(uint64_t a0, uint64_t a1, uint64_t a2, } // SYS_WRITE_CONSOLE (60) a0=*buf a1=len -// Writes user console text in white; distinct from green SYS_PRINT_DEBUG. +// Writes user console text and completes display delivery before returning. static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; @@ -3092,7 +3105,7 @@ static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2, KePrintConsole(chunk, todo); offset += todo; } - return NTLSTATUS_SUCCESS; + return KeDisplayFlushConsoleOutput(); } // SYS_OPEN_DIR (54) a0=*dirPath a1=*handleOut. @@ -3309,7 +3322,7 @@ void KeNamedQueueUnpublishHandle(PROCESS* process, uint32_t handle, OBJECT* obje if (!process || !object || object->Type != ObjectTypeQueue) return; - for (;;) { + while (true) { QUEUE* queue = (QUEUE*)0; uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock); for (int i = 0; i < NAMED_QUEUE_MAX; i++) { @@ -4473,7 +4486,7 @@ static uint64_t SyscallPoll(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t start = gHal ? gHal->Timer.QueryTicks() : 0; size_t ready = 0; - for (;;) { + while (true) { ready = 0; for (size_t i = 0; i < count; i++) { fds[i].ReturnedEvents = 0; @@ -4982,6 +4995,7 @@ static uint64_t SyscallQueryDisplayInfo(uint64_t a0, uint64_t a1, uint64_t a2, out.Mode = ki.Mode; out.Caps = ki.Caps; out.DeviceCount = ki.DeviceCount; + out.Flags = ki.Flags; out.ManagerStats = ki.ManagerStats; out.DeviceStats = ki.DeviceStats; @@ -5012,6 +5026,12 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, us.CpuBurstDemotions = ks.CpuBurstDemotions; us.SmtSiblingDeferrals = ks.SmtSiblingDeferrals; us.SmtPrimaryPokes = ks.SmtPrimaryPokes; + us.RealtimeDispatches = ks.RealtimeDispatches; + us.RealtimePreemptions = ks.RealtimePreemptions; + us.PriorityInheritanceBoosts = ks.PriorityInheritanceBoosts; + us.PriorityInheritanceDrops = ks.PriorityInheritanceDrops; + us.PriorityInheritancePropagations = ks.PriorityInheritancePropagations; + us.PriorityInheritanceMaxDepth = ks.PriorityInheritanceMaxDepth; return KiReturnKernelStats(a1, (size_t)a2, a3, &us, sizeof(us)); } @@ -5199,19 +5219,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, um.DropFailures = km.DropFailures; um.StandbyTransitions = km.StandbyTransitions; um.StandbyHits = km.StandbyHits; - um.ReadAheadRequests = km.ReadAheadRequests; - um.ReadAheadPagesRequested = km.ReadAheadPagesRequested; - um.ReadAheadPagesQueued = km.ReadAheadPagesQueued; - um.ReadAheadPagesLoaded = km.ReadAheadPagesLoaded; - um.ReadAheadPagesSkipped = km.ReadAheadPagesSkipped; - um.ReadAheadFailures = km.ReadAheadFailures; - um.ReadAheadPagesThrottled = km.ReadAheadPagesThrottled; - um.MaxReadAheadPagesPerRequest = km.MaxReadAheadPagesPerRequest; - um.BatchReads = km.BatchReads; - um.BatchPagesRequested = km.BatchPagesRequested; - um.BatchPagesLoaded = km.BatchPagesLoaded; - um.MaxBatchPagesRequested = km.MaxBatchPagesRequested; - um.MaxBatchPagesLoaded = km.MaxBatchPagesLoaded; um.ResidentPages = km.ResidentPages; um.ActivePages = km.ActivePages; um.StandbyPages = km.StandbyPages; @@ -5285,17 +5292,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, um.WriteSectors = km.WriteSectors; um.MaxReadSectors = km.MaxReadSectors; um.MaxWriteSectors = km.MaxWriteSectors; - um.ReadaheadHits = km.ReadaheadHits; - um.ReadaheadMisses = km.ReadaheadMisses; - um.ReadaheadStores = km.ReadaheadStores; - um.ReadaheadInvalidations = km.ReadaheadInvalidations; - um.ReadaheadBytesServed = km.ReadaheadBytesServed; - um.ReadaheadSectorsLoaded = km.ReadaheadSectorsLoaded; - um.ReadaheadSequentialDetections = km.ReadaheadSequentialDetections; - um.ReadaheadSuppressed = km.ReadaheadSuppressed; - um.ReadaheadAllocFailures = km.ReadaheadAllocFailures; - um.ReadaheadReadFailures = km.ReadaheadReadFailures; - um.MaxReadaheadSectors = km.MaxReadaheadSectors; return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); } @@ -5374,6 +5370,466 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, } } +// Converts one public audio format into the kernel mixer format. +static KSTATUS KiImportAudioFormat(const SYS_AUDIO_FORMAT* source, IO_AUDIO_FORMAT* destination) { + if (!source || !destination) + return NTLSTATUS_INVALID_PARAMETER; + if (source->SampleFormat != SysAudioSampleS16 || + (source->ChannelCount != 1 && source->ChannelCount != 2) || + (source->SampleRate != 11025u && source->SampleRate != 22050u && + source->SampleRate != 44100u && source->SampleRate != 48000u)) + return NTLSTATUS_NOT_SUPPORTED; + + destination->SampleRate = source->SampleRate; + destination->ChannelCount = source->ChannelCount; + destination->SampleFormat = IoAudioSampleS16; + return NTLSTATUS_SUCCESS; +} + +// Converts one kernel audio format into the public syscall representation. +static SYS_AUDIO_FORMAT KiExportAudioFormat(const IO_AUDIO_FORMAT* source) { + SYS_AUDIO_FORMAT destination; + destination.SampleRate = source->SampleRate; + destination.ChannelCount = source->ChannelCount; + destination.SampleFormat = source->SampleFormat == IoAudioSampleS16 + ? SysAudioSampleS16 + : 0; + return destination; +} + +// Converts one normalized kernel gain into unsigned Q16. +static uint32_t KiExportAudioGainQ16(float gain) { + if (gain <= 0.0f) return 0; + if (gain >= 1.0f) return SYS_AUDIO_GAIN_Q16_UNITY; + return (uint32_t)(gain * (float)SYS_AUDIO_GAIN_Q16_UNITY + 0.5f); +} + +// Converts one normalized kernel pan into signed Q16. +static int32_t KiExportAudioPanQ16(float pan) { + if (pan <= -1.0f) return SYS_AUDIO_PAN_Q16_LEFT; + if (pan >= 1.0f) return SYS_AUDIO_PAN_Q16_RIGHT; + float scaled = pan * (float)SYS_AUDIO_GAIN_Q16_UNITY; + return (int32_t)(scaled + (scaled >= 0.0f ? 0.5f : -0.5f)); +} + +// Converts one public unsigned Q16 gain into the kernel representation. +static float KiImportAudioGainQ16(uint32_t gainQ16) { + return (float)gainQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY; +} + +// Converts one public signed Q16 pan into the kernel representation. +static float KiImportAudioPanQ16(int32_t panQ16) { + return (float)panQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY; +} + +// Converts one kernel endpoint kind into its public ABI value. +static uint32_t KiExportAudioEndpointKind(IO_AUDIO_ENDPOINT_KIND kind) { + switch (kind) { + case IoAudioEndpointLineOut: return SysAudioEndpointLineOut; + case IoAudioEndpointSpeakerHeadphone: return SysAudioEndpointSpeakerHeadphone; + case IoAudioEndpointOtherAnalog: return SysAudioEndpointOtherAnalog; + default: return 0; + } +} + +// Converts provider endpoint capabilities into public capability bits. +static uint32_t KiExportAudioEndpointCapabilities(uint32_t capabilities) { + uint32_t result = 0; + if (capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE) + result |= SYS_AUDIO_ENDPOINT_CAP_JACK_SENSE; + if (capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) + result |= SYS_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN; + return result; +} + +// Converts one endpoint lifecycle value into its public ABI value. +static uint32_t KiExportAudioEndpointState(KE_AUDIO_ENDPOINT_STATE state) { + switch (state) { + case KeAudioEndpointStarting: return SysAudioEndpointStarting; + case KeAudioEndpointReady: return SysAudioEndpointReady; + case KeAudioEndpointStopping: return SysAudioEndpointStopping; + case KeAudioEndpointFailed: return SysAudioEndpointFailed; + case KeAudioEndpointRemoved: return SysAudioEndpointRemoved; + default: return 0; + } +} + +// Converts one stream lifecycle value into its public ABI value. +static uint32_t KiExportAudioStreamState(KE_AUDIO_STREAM_STATE state) { + switch (state) { + case KeAudioStreamCreated: return SysAudioStreamCreated; + case KeAudioStreamRunning: return SysAudioStreamRunning; + case KeAudioStreamPaused: return SysAudioStreamPaused; + case KeAudioStreamDraining: return SysAudioStreamDraining; + case KeAudioStreamStopped: return SysAudioStreamStopped; + case KeAudioStreamFailed: return SysAudioStreamFailed; + case KeAudioStreamClosed: + default: + return 0; + } +} + +// Converts public wait flags into the kernel stream condition mask. +static uint32_t KiImportAudioWaitMask(uint32_t mask) { + uint32_t result = 0; + if (mask & SYS_AUDIO_WAIT_WRITABLE) result |= KE_AUDIO_WAIT_WRITABLE; + if (mask & SYS_AUDIO_WAIT_CONSUMED) result |= KE_AUDIO_WAIT_CONSUMED; + if (mask & SYS_AUDIO_WAIT_PRESENTED) result |= KE_AUDIO_WAIT_PRESENTED; + if (mask & SYS_AUDIO_WAIT_STOPPED) result |= KE_AUDIO_WAIT_STOPPED; + if (mask & SYS_AUDIO_WAIT_DRAINED) result |= KE_AUDIO_WAIT_DRAINED; + if (mask & SYS_AUDIO_WAIT_FAILED) result |= KE_AUDIO_WAIT_FAILED; + if (mask & SYS_AUDIO_WAIT_DISCONTINUITY) result |= KE_AUDIO_WAIT_DISCONTINUITY; + return result; +} + +// Looks up one audio stream handle with the requested operation rights. +static KE_AUDIO_STREAM* KiLookupAudioStream(uint32_t handle, uint32_t rights) { + OBJECT* object = KiHandleToObject(handle, ObjectTypeAudioStream, rights); + return object ? (KE_AUDIO_STREAM*)object : (KE_AUDIO_STREAM*)0; +} + +// SYS_AUDIO_ENUMERATE_ENDPOINTS a0=*SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST. +static uint64_t SyscallAudioEnumerateEndpoints(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 || + !request.CountOut || (request.Capacity && !request.Endpoints)) + return NTLSTATUS_INVALID_PARAMETER; + + status = MmProbeUserWrite(request.CountOut, sizeof(*request.CountOut)); + if (!KSUCCESS(status)) return status; + uint32_t copyCapacity = request.Capacity < SYS_AUDIO_MAX_ENDPOINTS + ? request.Capacity + : SYS_AUDIO_MAX_ENDPOINTS; + if (copyCapacity) { + status = MmProbeUserWrite(request.Endpoints, + (size_t)copyCapacity * sizeof(*request.Endpoints)); + if (!KSUCCESS(status)) return status; + } + + KE_AUDIO_ENDPOINT_INFO kernelInfo[KE_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + status = KeAudioQueryEndpoints(kernelInfo, KE_AUDIO_MAX_ENDPOINTS, &count); + if (KFAILED(status)) return status; + + SYS_AUDIO_ENDPOINT_INFO publicInfo[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t copyCount = copyCapacity < count ? copyCapacity : count; + for (uint32_t index = 0; index < copyCount; index++) { + const KE_AUDIO_ENDPOINT_INFO* source = &kernelInfo[index]; + SYS_AUDIO_ENDPOINT_INFO* destination = &publicInfo[index]; + memset(destination, 0, sizeof(*destination)); + destination->Size = sizeof(*destination); + destination->Version = SYS_AUDIO_ABI_VERSION; + destination->EndpointId = source->EndpointId; + destination->Kind = KiExportAudioEndpointKind(source->Kind); + if (source->Available) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE; + if (source->IsDefault) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_DEFAULT; + destination->Capabilities = KiExportAudioEndpointCapabilities(source->Capabilities); + destination->State = KiExportAudioEndpointState(source->State); + destination->HardwareFormat = KiExportAudioFormat(&source->HardwareFormat); + destination->PeriodCount = source->PeriodCount; + destination->FramesPerPeriod = source->FramesPerPeriod; + destination->MasterVolumeQ16 = KiExportAudioGainQ16(source->MasterRequestedVolume); + destination->MasterMuted = source->MasterMuted ? 1u : 0u; + destination->FailureStatus = source->FailureStatus; + } + + if (copyCount) { + status = CopyToUser(request.Endpoints, + publicInfo, + (size_t)copyCount * sizeof(publicInfo[0])); + if (!KSUCCESS(status)) return status; + } + status = CopyToUser(request.CountOut, &count, sizeof(count)); + if (!KSUCCESS(status)) return status; + return request.Capacity < count ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS; +} + +// SYS_AUDIO_OPEN_STREAM a0=*SYS_AUDIO_OPEN_STREAM_REQUEST. +static uint64_t SyscallAudioOpenStream(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_OPEN_STREAM_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 || + !request.EndpointId || !request.StreamHandleOut) + return NTLSTATUS_INVALID_PARAMETER; + + status = MmProbeUserWrite(request.StreamHandleOut, sizeof(*request.StreamHandleOut)); + if (!KSUCCESS(status)) return status; + + IO_AUDIO_FORMAT format; + status = KiImportAudioFormat(&request.Format, &format); + if (!KSUCCESS(status)) return status; + + KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)0; + status = KeAudioCreateStream(request.EndpointId, &format, &stream); + if (!KSUCCESS(status)) return status; + + uint32_t handle = 0; + status = KiAllocHandleForCurrentWithRights((OBJECT*)stream, + HANDLE_RIGHT_WRITE | + HANDLE_RIGHT_WAIT | + HANDLE_RIGHT_QUERY, + &handle); + if (!KSUCCESS(status)) return status; + + status = CopyToUser(request.StreamHandleOut, &handle, sizeof(handle)); + if (!KSUCCESS(status)) { + PROCESS* process = KeGetCurrentProcess(); + if (process) KeCloseHandle(process, handle); + } + return status; +} + +// SYS_AUDIO_WRITE a0=*SYS_AUDIO_WRITE_REQUEST. +static uint64_t SyscallAudioWrite(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_WRITE_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle || + !request.WrittenFramesOut || (request.FrameCount && !request.Frames)) + return NTLSTATUS_INVALID_PARAMETER; + + status = MmProbeUserWrite(request.WrittenFramesOut, sizeof(*request.WrittenFramesOut)); + if (!KSUCCESS(status)) return status; + uint32_t written = 0; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + if (!request.FrameCount) { + ObDereferenceObject((OBJECT*)stream); + return CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + } + + KE_AUDIO_STREAM_INFO info; + status = KeAudioQueryStream(stream, &info); + if (!KSUCCESS(status)) { + ObDereferenceObject((OBJECT*)stream); + return status; + } + if (info.State == KeAudioStreamFailed) { + status = KFAILED(info.FailureStatus) ? info.FailureStatus : NTLSTATUS_DEVICE_ERROR; + ObDereferenceObject((OBJECT*)stream); + (void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + return status; + } + if (info.State == KeAudioStreamDraining) { + ObDereferenceObject((OBJECT*)stream); + (void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + return NTLSTATUS_INVALID_PARAMETER; + } + if (!info.WritableFrames) { + ObDereferenceObject((OBJECT*)stream); + status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + return KSUCCESS(status) ? NTLSTATUS_WOULD_BLOCK : status; + } + + uint32_t frameCount = request.FrameCount < info.WritableFrames + ? request.FrameCount + : info.WritableFrames; + uint32_t frameBytes = (uint32_t)info.Format.ChannelCount * sizeof(int16_t); + size_t byteCount = (size_t)frameCount * frameBytes; + status = MmProbeUserRead(request.Frames, byteCount); + if (!KSUCCESS(status)) { + ObDereferenceObject((OBJECT*)stream); + return status; + } + + void* buffer = MmAllocatePoolTag(byteCount, MM_POOL_TAG('a', 'u', 'w', 'r')); + if (!buffer) { + ObDereferenceObject((OBJECT*)stream); + return NTLSTATUS_NO_MEMORY; + } + status = CopyFromUser(buffer, request.Frames, byteCount); + if (KSUCCESS(status)) + status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + if (KSUCCESS(status)) + status = KeAudioWriteStream(stream, buffer, frameCount, &written); + MmFreePool(buffer); + ObDereferenceObject((OBJECT*)stream); + if (!KSUCCESS(status)) return status; + return CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); +} + +// SYS_AUDIO_ARM_WAIT a0=*SYS_AUDIO_ARM_WAIT_REQUEST. +static uint64_t SyscallAudioArmWait(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_ARM_WAIT_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 || + !request.StreamHandle || !request.Mask || + (request.Mask & ~SYS_AUDIO_WAIT_VALID_MASK) != 0) + return NTLSTATUS_INVALID_PARAMETER; + + KE_AUDIO_STREAM_WAIT_CONDITION condition; + condition.Mask = KiImportAudioWaitMask(request.Mask); + condition.WritableFrames = request.WritableFrames; + condition.ConsumedFrames = request.ConsumedFrames; + condition.PresentedFrames = request.PresentedFrames; + condition.DrainedGeneration = request.DrainedGeneration; + condition.DiscontinuityGeneration = request.DiscontinuityGeneration; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WAIT); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + status = KeAudioArmStreamWait(stream, &condition); + ObDereferenceObject((OBJECT*)stream); + return status; +} + +// SYS_AUDIO_QUERY_STREAM a0=streamHandle a1=*SYS_AUDIO_STREAM_INFO. +static uint64_t SyscallAudioQueryStream(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a2; (void)a3; (void)a4; (void)a5; + if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER; + + uint32_t userSize = 0; + KSTATUS status = CopyFromUser(&userSize, (const void*)(uintptr_t)a1, sizeof(userSize)); + if (!KSUCCESS(status)) return status; + if (userSize != sizeof(SYS_AUDIO_STREAM_INFO)) + return NTLSTATUS_INVALID_PARAMETER; + status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(SYS_AUDIO_STREAM_INFO)); + if (!KSUCCESS(status)) return status; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream((uint32_t)a0, HANDLE_RIGHT_QUERY); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + KE_AUDIO_STREAM_INFO kernelInfo; + status = KeAudioQueryStream(stream, &kernelInfo); + ObDereferenceObject((OBJECT*)stream); + if (!KSUCCESS(status)) return status; + + SYS_AUDIO_STREAM_INFO publicInfo; + memset(&publicInfo, 0, sizeof(publicInfo)); + publicInfo.Size = sizeof(publicInfo); + publicInfo.Version = SYS_AUDIO_ABI_VERSION; + publicInfo.State = KiExportAudioStreamState(kernelInfo.State); + publicInfo.Format = KiExportAudioFormat(&kernelInfo.Format); + publicInfo.CapacityFrames = kernelInfo.CapacityFrames; + publicInfo.BufferedFrames = kernelInfo.BufferedFrames; + publicInfo.WritableFrames = kernelInfo.WritableFrames; + publicInfo.FailureStatus = kernelInfo.FailureStatus; + publicInfo.FramesSubmitted = kernelInfo.FramesSubmitted; + publicInfo.FramesConsumed = kernelInfo.FramesConsumed; + publicInfo.FramesPresented = kernelInfo.FramesPresented; + publicInfo.FramesDiscarded = kernelInfo.FramesDiscarded; + publicInfo.OutputFramesMixed = kernelInfo.OutputFramesMixed; + publicInfo.SilenceFramesInserted = kernelInfo.SilenceFramesInserted; + publicInfo.Underruns = kernelInfo.Underruns; + publicInfo.DrainedGeneration = kernelInfo.DrainedGeneration; + publicInfo.DiscontinuityGeneration = kernelInfo.DiscontinuityGeneration; + publicInfo.VolumeQ16 = KiExportAudioGainQ16(kernelInfo.Volume); + publicInfo.PanQ16 = KiExportAudioPanQ16(kernelInfo.Pan); + publicInfo.Muted = kernelInfo.Muted; + return CopyToUser((void*)(uintptr_t)a1, &publicInfo, sizeof(publicInfo)); +} + +typedef KSTATUS (*KI_AUDIO_STREAM_OPERATION)(KE_AUDIO_STREAM* stream); + +// Invokes one existing kernel stream operation through a write-capable handle. +static KSTATUS KiCallAudioStreamOperation(uint32_t handle, KI_AUDIO_STREAM_OPERATION operation) { + if (!handle || !operation) return NTLSTATUS_INVALID_PARAMETER; + KE_AUDIO_STREAM* stream = KiLookupAudioStream(handle, HANDLE_RIGHT_WRITE); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + KSTATUS status = operation(stream); + ObDereferenceObject((OBJECT*)stream); + return status; +} + +// SYS_AUDIO_START a0=streamHandle. +static uint64_t SyscallAudioStart(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStartStream); +} + +// SYS_AUDIO_PAUSE a0=streamHandle. +static uint64_t SyscallAudioPause(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioPauseStream); +} + +// SYS_AUDIO_RESUME a0=streamHandle. +static uint64_t SyscallAudioResume(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResumeStream); +} + +// SYS_AUDIO_DRAIN a0=streamHandle. +static uint64_t SyscallAudioDrain(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioDrainStream); +} + +// SYS_AUDIO_STOP a0=streamHandle. +static uint64_t SyscallAudioStop(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStopStream); +} + +// SYS_AUDIO_RESET a0=streamHandle. +static uint64_t SyscallAudioReset(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResetStream); +} + +// SYS_AUDIO_SET_VOLUME a0=*SYS_AUDIO_SET_VOLUME_REQUEST. +static uint64_t SyscallAudioSetVolume(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_SET_VOLUME_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle || + request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY || + request.PanQ16 < SYS_AUDIO_PAN_Q16_LEFT || + request.PanQ16 > SYS_AUDIO_PAN_Q16_RIGHT) + return NTLSTATUS_INVALID_PARAMETER; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + status = KeAudioSetStreamVolume(stream, + KiImportAudioGainQ16(request.VolumeQ16), + KiImportAudioPanQ16(request.PanQ16), + request.Muted != 0); + ObDereferenceObject((OBJECT*)stream); + return status; +} + +// SYS_AUDIO_SET_MASTER_VOLUME a0=*SYS_AUDIO_SET_MASTER_VOLUME_REQUEST. +static uint64_t SyscallAudioSetMasterVolume(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || !request.EndpointId || + request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY) + return NTLSTATUS_INVALID_PARAMETER; + + return KeAudioSetMasterVolume(request.EndpointId, + KiImportAudioGainQ16(request.VolumeQ16), + request.Muted != 0); +} + // Dispatch table keyed by SYS_* constants. typedef uint64_t (*SYSCALL_HANDLER)(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); @@ -5389,6 +5845,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { [SYS_GET_PROCESS_LIST] = SyscallGetProcessList, [SYS_GET_PROCESS_PID] = SyscallGetProcessPid, [SYS_CREATE_THREAD] = SyscallCreateThread, + [SYS_RESUME_THREAD] = SyscallResumeThread, [SYS_GET_THREAD_LIST] = SyscallGetThreadList, [SYS_LOAD_MODULE] = SyscallLoadModule, [SYS_UNLOAD_MODULE] = SyscallUnloadModule, @@ -5480,6 +5937,21 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { [SYS_QUERY_SURFACE] = SyscallQuerySurface, [SYS_PRESENT_SURFACE] = SyscallPresentSurface, + // Native render audio. + [SYS_AUDIO_ENUMERATE_ENDPOINTS] = SyscallAudioEnumerateEndpoints, + [SYS_AUDIO_OPEN_STREAM] = SyscallAudioOpenStream, + [SYS_AUDIO_WRITE] = SyscallAudioWrite, + [SYS_AUDIO_ARM_WAIT] = SyscallAudioArmWait, + [SYS_AUDIO_QUERY_STREAM] = SyscallAudioQueryStream, + [SYS_AUDIO_START] = SyscallAudioStart, + [SYS_AUDIO_PAUSE] = SyscallAudioPause, + [SYS_AUDIO_RESUME] = SyscallAudioResume, + [SYS_AUDIO_DRAIN] = SyscallAudioDrain, + [SYS_AUDIO_STOP] = SyscallAudioStop, + [SYS_AUDIO_RESET] = SyscallAudioReset, + [SYS_AUDIO_SET_VOLUME] = SyscallAudioSetVolume, + [SYS_AUDIO_SET_MASTER_VOLUME] = SyscallAudioSetMasterVolume, + // Input. [SYS_POLL_KEYBOARD] = SyscallKbPoll, [SYS_IS_KEY_DOWN] = SyscallKbIsKeyDown, @@ -5532,6 +6004,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1, THREAD* self = KeGetCurrentThread(); PROCESS* owner = self ? self->Owner : (PROCESS*)0; if (owner && KeIsProcessTerminating(owner) && + !KeThreadTerminationDeferred(self) && nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) { int32_t exitCode = owner->ExitCode; KeExitThread(self, exitCode); @@ -5547,6 +6020,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1, // has been marked terminating. owner = self ? self->Owner : (PROCESS*)0; if (owner && KeIsProcessTerminating(owner) && + !KeThreadTerminationDeferred(self) && nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) { int32_t exitCode = owner->ExitCode; KeExitThread(self, exitCode); diff --git a/code/kernel/ke_system_root.c b/code/kernel/ke_system_root.c index eb7bd14..ab9c2ad 100644 --- a/code/kernel/ke_system_root.c +++ b/code/kernel/ke_system_root.c @@ -7,10 +7,15 @@ #include #include #include +#include static uint8_t gSystemRootDrive; static bool gSystemRootReady; +#define KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS 30000ull +#define KE_SYSTEM_ROOT_WAIT_SLICE_MS 1000ull +#define KE_SYSTEM_ROOT_LOG_INTERVAL_MS 5000ull + // Builds a drive-qualified path without depending on any process CWD. static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) { if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES) @@ -50,23 +55,76 @@ static bool KiDriveLooksLikeSystemRoot(uint8_t drive) { return true; } -// Selects the mounted volume that contains the kernel/system tree. -KSTATUS KeInitializeSystemRootVolume(void) { +// Returns the current scheduler time in milliseconds. +static uint64_t KiSystemRootNowMilliseconds(void) { + KE_SCHED_STATS stats; + KeGetSchedulerStats(&stats); + return stats.Ticks * SCHED_TICK_MS; +} + +// Scans the currently mounted volumes for the system tree. +static KSTATUS KiTrySelectSystemRootVolume(uint32_t* mountedOut) { + uint32_t mounted = 0; for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) { if (!IoFilesystemPresent(drive)) continue; + mounted++; if (!KiDriveLooksLikeSystemRoot(drive)) continue; gSystemRootDrive = drive; gSystemRootReady = true; + if (mountedOut) + *mountedOut = mounted; DbgPrint("System root: drive=%u\n", drive); return NTLSTATUS_SUCCESS; } + if (mountedOut) + *mountedOut = mounted; return NTLSTATUS_NOT_FOUND; } +// Waits for the mounted boot volume instead of racing asynchronous bus discovery. +KSTATUS KeInitializeSystemRootVolume(void) { + uint64_t start = KiSystemRootNowMilliseconds(); + uint64_t deadline = start + KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS; + uint64_t nextLog = start; + + for (;;) { + uint64_t generation = IoGetVolumeNamespaceGeneration(); + uint32_t mounted = 0; + KSTATUS status = KiTrySelectSystemRootVolume(&mounted); + if (KSUCCESS(status)) + return status; + + uint64_t now = KiSystemRootNowMilliseconds(); + if (now >= deadline) { + DbgPrint("System root: discovery timed out after %llu ms; mounted=%u generation=%llu\n", + (unsigned long long)KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS, + mounted, + (unsigned long long)generation); + return NTLSTATUS_TIMEOUT; + } + + if (now >= nextLog) { + DbgPrint("System root: waiting for boot volume; mounted=%u generation=%llu remaining=%llu ms\n", + mounted, + (unsigned long long)generation, + (unsigned long long)(deadline - now)); + nextLog = now + KE_SYSTEM_ROOT_LOG_INTERVAL_MS; + } + + uint64_t remaining = deadline - now; + uint64_t wait = remaining < KE_SYSTEM_ROOT_WAIT_SLICE_MS + ? remaining + : KE_SYSTEM_ROOT_WAIT_SLICE_MS; + KSTATUS waitStatus = IoWaitForVolumeNamespaceChange(generation, wait); + if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT) + return waitStatus; + } +} + // Returns whether the system root volume has been selected. bool KeSystemRootReady(void) { return gSystemRootReady; diff --git a/code/kernel/ke_thread.c b/code/kernel/ke_thread.c index 47e5d39..a5f7a2a 100644 --- a/code/kernel/ke_thread.c +++ b/code/kernel/ke_thread.c @@ -27,14 +27,105 @@ _Static_assert(offsetof(type, field) == (expected), \ #type "." #field " offset must match assembly") -STATIC_ASSERT_OFFSET(THREAD, SavedContext, 96); -STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 112); -STATIC_ASSERT_OFFSET(THREAD, Teb, 120); -STATIC_ASSERT_OFFSET(THREAD, FpuContext, 176); +STATIC_ASSERT_OFFSET(THREAD, SavedContext, 112); +STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 128); +STATIC_ASSERT_OFFSET(THREAD, Teb, 136); +STATIC_ASSERT_OFFSET(THREAD, FpuContext, 192); static uint32_t gNextThreadId = 1; extern void KeThreadEntry(void); +// Prevents a newly scheduled user thread from entering a terminating process. +void KePrepareUserThreadStart(void) { + THREAD* thread = KeGetCurrentThread(); + PROCESS* owner = thread ? thread->Owner : (PROCESS*)0; + if (!thread || !owner || !KeIsProcessTerminating(owner)) + return; + + KeExitThread(thread, owner->ExitCode); + while (true) { + KeYield(); + } +} + +// Finalizes one dead thread's completion, user memory, and process membership. +void KeFinalizeThreadExit(THREAD* thread) { + if (!thread) + return; + if (thread->State != ThreadDead) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread finalized before death: thread=0x%016llX tid=%u state=%u\n", + (uint64_t)thread, + thread->ThreadId, + (unsigned)thread->State); + } + + // Thread completion is manual-reset and idempotent. Publish it before + // process-owned handles can be released by the final live-thread exit. + if (thread->ExitEvent) + KeSignalEvent(thread->ExitEvent); + + PROCESS* owner = thread->Owner; + if (!owner) + return; + + // Every exiting thread releases its own user allocations before leaving + // the live-thread set. Therefore the final detach can destroy the remaining + // process VADs without racing a later THREAD destructor. + if (!owner->ResourcesReleased) { + if (thread->Teb && thread->TebSize) + MmFreeUserMemoryForProcess(owner, thread->Teb); + if (thread->UserStackBase) + MmFreeUserMemoryForProcess(owner, thread->UserStackBase); + } + thread->Teb = (void*)0; + thread->TebSize = 0; + thread->UserStackBase = (void*)0; + + uint32_t remaining = 0; + int detached = 0; + + uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock); + int fwdDetached = thread->ProcessLink.Fwd == &thread->ProcessLink; + int bwdDetached = thread->ProcessLink.Bwd == &thread->ProcessLink; + if (fwdDetached != bwdDetached) { + KeReleaseSpinlock(&owner->HandleLock, flags); + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread process linkage corrupt: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->ProcessLink.Fwd, + (uint64_t)thread->ProcessLink.Bwd); + } + + if (!fwdDetached) { + if (owner->ThreadCount == 0) { + KeReleaseSpinlock(&owner->HandleLock, flags); + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread process count underflow: thread=0x%016llX tid=%u pid=%u\n", + (uint64_t)thread, + thread->ThreadId, + owner->ProcessId); + } + + KeListRemoveChecked(&thread->ProcessLink); + owner->ThreadCount--; + detached = 1; + } + remaining = owner->ThreadCount; + KeReleaseSpinlock(&owner->HandleLock, flags); + + if (!detached) + return; + + if (remaining == 0) { + // ThreadCount represents live process membership, not retained thread + // handles. Dead THREAD objects may remain waitable after process exit. + _InterlockedExchange(&owner->Terminating, 1); + KeCompleteProcessExitIfLastThread(owner); + } +} + // Implements the thread Destroy operation. static void KiThreadDestroy(OBJECT* obj) { THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header); @@ -44,6 +135,26 @@ static void KiThreadDestroy(OBJECT* obj) { // wait syscall to run normal cleanup. KeRundownThreadWait(thread, NTLSTATUS_ALERTED); + if (thread->WaitingMutex || !LIST_IS_EMPTY(&thread->OwnedMutexList)) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed with mutex state: thread=0x%016llX tid=%u waiting=0x%016llX ownedEmpty=%u\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->WaitingMutex, + LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u); + } + if (thread->TimeoutArmed || + thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed with timeout state: thread=0x%016llX tid=%u armed=%u link=0x%016llX/0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (unsigned)thread->TimeoutArmed, + (uint64_t)thread->TimeoutLink.Fwd, + (uint64_t)thread->TimeoutLink.Bwd); + } + KiRundownApcState(thread); if (thread->ExitEvent) { @@ -51,39 +162,28 @@ static void KiThreadDestroy(OBJECT* obj) { thread->ExitEvent = (EVENT*)0; } - PROCESS* ownerForVadList = thread->Owner; - if (ownerForVadList) { - if (thread->Teb && thread->TebSize) { - MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb); - thread->Teb = (void*)0; - thread->TebSize = 0; - } + if (thread->Teb || thread->TebSize || thread->UserStackBase) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed with user memory: thread=0x%016llX tid=%u teb=0x%016llX tebSize=%llu stack=0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->Teb, + (uint64_t)thread->TebSize, + (uint64_t)thread->UserStackBase); + } - if (thread->UserStackBase) { - MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase); - thread->UserStackBase = (void *)0; - } + if (thread->ProcessLink.Fwd != &thread->ProcessLink || + thread->ProcessLink.Bwd != &thread->ProcessLink) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed while still process-linked: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->ProcessLink.Fwd, + (uint64_t)thread->ProcessLink.Bwd); } if (thread->Owner) { - PROCESS* owner = thread->Owner; - uint32_t remaining = 0; - uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock); - KeListRemoveChecked(&thread->ProcessLink); - if (owner->ThreadCount) owner->ThreadCount--; - remaining = owner->ThreadCount; - KeReleaseSpinlock(&owner->HandleLock, flags); - - if (remaining == 0) { - // The last thread leaving a process is process exit even if user - // code called ExitThread instead of ExitProcess. KeExitProcess / - // KeTerminateProcess set Terminating earlier; this covers the last - // thread voluntarily exiting without an explicit process-exit call. - _InterlockedExchange(&owner->Terminating, 1); - KeCompleteProcessExitIfLastThread(owner); - } - - ObDereferenceObject(&owner->Header); + ObDereferenceObject(&thread->Owner->Header); thread->Owner = (PROCESS *)0; } @@ -94,35 +194,51 @@ static void KiThreadDestroy(OBJECT* obj) { } __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out) { - if (!owner || !out) return NTLSTATUS_INVALID_PARAMETER; - if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) { + if (!owner || !parameters || !out) return NTLSTATUS_INVALID_PARAMETER; + if (parameters->SchedulingClass != KeSchedulingNormal || + parameters->Priority >= PriorityCount || + (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED)) + return NTLSTATUS_INVALID_PARAMETER; + + uint8_t initiallySuspended = + (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u; THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); if (!thread) return NTLSTATUS_NO_MEMORY; thread->Header.Destroy = KiThreadDestroy; thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1); - thread->State = ThreadReady; - thread->Priority = priority; - thread->QuantaConsumed = 0; + thread->State = initiallySuspended ? ThreadSuspended : ThreadReady; + thread->BaseSchedulingClass = parameters->SchedulingClass; + thread->BasePriority = parameters->Priority; + thread->EffectiveSchedulingClass = parameters->SchedulingClass; + thread->EffectivePriority = parameters->Priority; + thread->QuantaConsumed = 0; thread->QuantumTicks = 0; thread->QuantumRemaining = 0; thread->CpuBurstTicks = 0; thread->LastRunTicks = 0; thread->RunningCpu = KE_INVALID_CPU; + thread->SuspendCount = initiallySuspended ? 1u : 0u; + LIST_INIT(&thread->OwnedMutexList); + thread->WaitingMutex = (MUTEX*)0; thread->Queued = 0; thread->IsIdleThread = 0; thread->SchedulerFlags = 0; thread->LastCpu = KE_INVALID_CPU; - thread->AffinityMask = KeGetDefaultThreadAffinityMask(); + thread->AffinityMask = parameters->AffinityMask + ? parameters->AffinityMask + : KeGetDefaultThreadAffinityMask(); LIST_INIT(&thread->ProcessLink); LIST_INIT(&thread->SchedulerLink); thread->WaitObject = (OBJECT *)0; thread->WaitStatus = NTLSTATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; + LIST_INIT(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + thread->TimeoutTick = 0; thread->TimeoutStatus = NTLSTATUS_SUCCESS; thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); @@ -131,6 +247,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui thread->UserExceptionNoncontinuableMask = 0; thread->UserExceptionFrameUserAddress = 0; memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus)); + thread->TerminationDeferralCount = 0; void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k')); if (!kstack) { @@ -245,24 +362,58 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui ObReferenceObject(&thread->Header); *out = thread; - KeReadyThread(thread); + if (!initiallySuspended) + KeReadyThread(thread); return NTLSTATUS_SUCCESS; } + +// Enters a region that must retain the current thread during process termination. +void KeEnterTerminationDeferral(void) { + KeAssertIrql(PASSIVE_LEVEL); + THREAD* thread = KeGetCurrentThread(); + if (!thread || thread->IsIdleThread) + return; + _InterlockedIncrement(&thread->TerminationDeferralCount); +} + +// Leaves a process-termination deferral region. +void KeLeaveTerminationDeferral(void) { + KeAssertIrql(PASSIVE_LEVEL); + THREAD* thread = KeGetCurrentThread(); + if (!thread || thread->IsIdleThread) + return; + + long remaining = _InterlockedDecrement(&thread->TerminationDeferralCount); + if (remaining < 0) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread termination deferral underflow: thread=0x%016llX tid=%u\n", + (uint64_t)thread, + thread->ThreadId); + } + + if (remaining == 0 && thread->Owner && KeIsProcessTerminating(thread->Owner)) + KiRequestReschedule(); +} + +// Returns whether one thread still owns termination-sensitive kernel work. +int KeThreadTerminationDeferred(const THREAD* thread) { + return thread && _InterlockedCompareExchange((volatile long*)&thread->TerminationDeferralCount, + 0, + 0) > 0; +} + // Implements the exit Thread operation. __declspec(dllexport) void KeExitThread(THREAD* thread, int code) { (void)code; if (!thread) return; KiRundownApcState(thread); - - // Wake any pthread_join waiters before the thread is marked dead and - // potentially freed. Manual-reset means all concurrent waiters unblock. - if (thread->ExitEvent) - KeSignalEvent(thread->ExitEvent); + KiMutexRundownThread(thread); thread->State = ThreadDead; + KeFinalizeThreadExit(thread); KeYield(); } @@ -277,10 +428,9 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) { // [ksp - 16 - sizeof(KCONTEXT)] KCONTEXT { .Rip = KeKernelThreadEntry }. extern void KeKernelThreadEntry(void); -static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, - uint64_t affinityMask, - THREAD** out) { +static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg, + const KE_THREAD_CREATE_PARAMETERS* parameters, + THREAD** out) { // Kernel worker threads are owned by the system process. The scheduler's // hand-built CPU idle thread stays attached to PID 0 (idle), while driver, // async I/O, and init worker threads are grouped under PID 1 (system). @@ -289,34 +439,57 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, THREAD* currentThread = KeGetCurrentThread(); owner = currentThread ? currentThread->Owner : (PROCESS*)0; } - if (!owner) return NTLSTATUS_INVALID_PARAMETER; - if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; + if (!owner || !parameters) return NTLSTATUS_INVALID_PARAMETER; + if (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED) + return NTLSTATUS_INVALID_PARAMETER; + if ((parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) && !out) + return NTLSTATUS_INVALID_PARAMETER; + if (parameters->SchedulingClass == KeSchedulingNormal) { + if (parameters->Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; + } else if (parameters->SchedulingClass == KeSchedulingRealtime) { + if (parameters->Priority > KE_REALTIME_PRIORITY_MAX) return NTLSTATUS_INVALID_PARAMETER; + } else { + return NTLSTATUS_INVALID_PARAMETER; + } + + uint8_t initiallySuspended = + (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u; THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); if (!thread) return NTLSTATUS_NO_MEMORY; thread->Header.Destroy = KiThreadDestroy; thread->ThreadId = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1); - thread->State = ThreadReady; - thread->Priority = priority; - thread->QuantaConsumed = 0; + thread->State = initiallySuspended ? ThreadSuspended : ThreadReady; + thread->BaseSchedulingClass = parameters->SchedulingClass; + thread->BasePriority = parameters->Priority; + thread->EffectiveSchedulingClass = parameters->SchedulingClass; + thread->EffectivePriority = parameters->Priority; + thread->QuantaConsumed = 0; thread->QuantumTicks = 0; thread->QuantumRemaining = 0; thread->CpuBurstTicks = 0; thread->LastRunTicks = 0; thread->RunningCpu = KE_INVALID_CPU; + thread->SuspendCount = initiallySuspended ? 1u : 0u; + LIST_INIT(&thread->OwnedMutexList); + thread->WaitingMutex = (MUTEX*)0; thread->Queued = 0; thread->IsIdleThread = 0; thread->SchedulerFlags = 0; thread->LastCpu = KE_INVALID_CPU; - thread->AffinityMask = affinityMask ? affinityMask : KeGetDefaultThreadAffinityMask(); + thread->AffinityMask = parameters->AffinityMask + ? parameters->AffinityMask + : KeGetDefaultThreadAffinityMask(); LIST_INIT(&thread->ProcessLink); LIST_INIT(&thread->SchedulerLink); thread->WaitObject = (OBJECT*)0; thread->WaitStatus = NTLSTATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; + LIST_INIT(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + thread->TimeoutTick = 0; thread->TimeoutStatus = NTLSTATUS_SUCCESS; thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); @@ -325,6 +498,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, thread->UserExceptionNoncontinuableMask = 0; thread->UserExceptionFrameUserAddress = 0; memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus)); + thread->TerminationDeferralCount = 0; thread->UserStackBase = (void*)0; // no user stack thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx thread->TebSize = 0; @@ -371,20 +545,15 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, } KeInitThreadFpuContext(thread); - KeReadyThread(thread); + if (!initiallySuspended) + KeReadyThread(thread); return NTLSTATUS_SUCCESS; } __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out) { - return KiCreateKernelThreadWithAffinity(fn, arg, priority, - KeGetDefaultThreadAffinityMask(), out); -} - -__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, uint64_t affinityMask, + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) { - return KiCreateKernelThreadWithAffinity(fn, arg, priority, affinityMask, out); + return KiCreateKernelThread(fn, arg, parameters, out); } // Prepares block Thread. @@ -438,16 +607,6 @@ void KeRundownThreadWait(THREAD* thread, KSTATUS status) { routine(thread, context, status); } -// Implements the finish Wait operation. -static void KiFinishWait(THREAD* thread, KSTATUS status) { - thread->WaitObject = (OBJECT*)0; - thread->WaitStatus = status; - thread->WaitListHead = (LIST_ENTRY*)0; - thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; -} - // Implements the commit Block Thread operation. KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { KeAssertIrql(PASSIVE_LEVEL); @@ -463,19 +622,14 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { if (thread->State != ThreadWaiting) { KSTATUS status = thread->WaitStatus; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; return status; } // If a user APC was already pending before the thread fully blocked, // consume the alertable wait immediately instead of going to sleep. if (alertable && thread->UserApcPending && thread->UserApcDispatcher) { - if (thread->WaitListHead) - KeListRemoveChecked(&thread->SchedulerLink); - KiFinishWait(thread, NTLSTATUS_USER_APC); - thread->State = ThreadRunning; - return NTLSTATUS_USER_APC; + if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC)) + return NTLSTATUS_USER_APC; } KiSchedulerNoteVoluntaryBlock(thread); @@ -486,8 +640,6 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { KiDeliverKernelApcs(); thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; return thread->WaitStatus; } @@ -525,8 +677,6 @@ static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* wa KeListRemoveChecked(&thread->SchedulerLink); thread->WaitListHead = (LIST_ENTRY*)0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; return 1; } @@ -546,10 +696,7 @@ static int KiAbortObjectWait(THREAD* thread) { } case ObjectTypeMutex: { MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header); - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList); - KeReleaseSpinlock(&mutex->Lock, flags); - return removed; + return KiMutexAbortWait(mutex, thread); } case ObjectTypeSemaphore: { SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header); @@ -569,24 +716,29 @@ static int KiAbortObjectWait(THREAD* thread) { // unsignaled after the foreground thread has faulted. Object wait-list removal // must use the object's own lock; otherwise signal/abort races can leave stale // scheduler links in an event, mutex, or semaphore wait queue. -void KeAbortThreadWait(THREAD* thread, KSTATUS status) { - if (!thread) return; - if (thread->State != ThreadWaiting) return; +int KeAbortThreadWait(THREAD* thread, KSTATUS status) { + if (!thread) return 0; + if (thread->State != ThreadWaiting) return 0; if (thread->WaitRundownRoutine) { KeRundownThreadWait(thread, status); - return; + return thread->State != ThreadWaiting; } if (KiAbortObjectWait(thread)) { KeUnblockThreadWithStatus(thread, status); - return; + return 1; } - // Sleep/timeouts and already-dequeued handoffs are owned by the scheduler. - // This path removes from the scheduler sleep list when present and otherwise - // completes the wait in place. + // A signaler that already dequeued an object waiter owns completion. Let + // that handoff finish rather than returning timeout after ownership or a + // semaphore count has already transferred to this thread. + if (thread->WaitObject) + return 0; + + // Delays and auxiliary scheduler waits have no object-owned queue. KeUnblockThreadFromWaitListWithStatus(thread, status); + return 1; } // Implements the unblock Thread operation. @@ -601,11 +753,9 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) { if (thread->State != ThreadWaiting || !thread->AlertableWait) return NTLSTATUS_PENDING; - if (thread->WaitListHead) - KeListRemoveChecked(&thread->SchedulerLink); - - KeUnblockThreadWithStatus(thread, status); - return NTLSTATUS_SUCCESS; + return KeAbortThreadWait(thread, status) + ? NTLSTATUS_SUCCESS + : NTLSTATUS_PENDING; } // Implements the alert Thread operation. diff --git a/code/kernel/ke_time.c b/code/kernel/ke_time.c index 86a6a90..fe069ed 100644 --- a/code/kernel/ke_time.c +++ b/code/kernel/ke_time.c @@ -99,7 +99,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out uint32_t sod = (uint32_t)(epochSeconds % 86400u); uint32_t year = 1970u; - for (;;) { + while (true) { uint32_t diy = KiIsLeapYear(year) ? 366u : 365u; if (days < diy) break; days -= diy; @@ -107,7 +107,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out } uint32_t month = 1u; - for (;;) { + while (true) { uint32_t dim = KiDaysInMonth(year, month); if (days < dim) break; days -= dim; @@ -519,6 +519,13 @@ __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime) { return NTLSTATUS_SUCCESS; } +// Returns the monotonic platform counter in microseconds. +uint64_t KeQueryMonotonicTicks(void) { + return (gTimeHal && gTimeHal->Timer.QueryTicks) + ? gTimeHal->Timer.QueryTicks() + : 0; +} + // Returns unix Time Seconds. __declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void) { SYS_SYSTEM_TIME t; diff --git a/code/kernel/ke_work.c b/code/kernel/ke_work.c index 609877c..78122ec 100644 --- a/code/kernel/ke_work.c +++ b/code/kernel/ke_work.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -17,17 +18,31 @@ static SPINLOCK gWorkQueueLock = SPINLOCK_INIT; static LIST_ENTRY gWorkQueueList; static EVENT* gWorkQueueEvent; +static KDPC gWorkQueueWakeDpc; static volatile long gWorkQueueListReady; static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started. +// Signals the passive worker after an enqueue from hardware interrupt context. +static void KiWorkQueueWakeDpc(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)context; + (void)systemArgument1; + (void)systemArgument2; + KeAssertIrql(DISPATCH_LEVEL); + + EVENT* event = gWorkQueueEvent; + if (event) + KeSignalEvent(event); +} + static void KiWorkQueueWorker(void* arg) { (void)arg; KeAssertIrql(PASSIVE_LEVEL); - for (;;) { + while (true) { (void)KeWaitForEvent(gWorkQueueEvent, 0); - for (;;) { + while (true) { KWORK_ITEM* item = (KWORK_ITEM*)0; uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock); if (!LIST_IS_EMPTY(&gWorkQueueList)) { @@ -79,8 +94,14 @@ bool KeQueueWorkItem(KWORK_ITEM* item) { EVENT* event = gWorkQueueEvent; KeReleaseSpinlock(&gWorkQueueLock, flags); - if (event) - KeSignalEvent(event); + if (event) { + // Event signaling can detach waiters, so callers above passive level + // defer the wake until no caller-owned spinlock remains held. + if (KeGetCurrentIrql() == PASSIVE_LEVEL) + KeSignalEvent(event); + else + (void)KeInsertQueueDpc(&gWorkQueueWakeDpc, (void*)0, (void*)0); + } return true; } @@ -95,14 +116,16 @@ KSTATUS KeStartWorkQueue(void) { } LIST_INIT(&gWorkQueueList); + KeInitializeDpc(&gWorkQueueWakeDpc, KiWorkQueueWakeDpc, (void*)0); _InterlockedExchange(&gWorkQueueListReady, 1); KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker, 0, - PriorityNormal, + ¶meters, &worker); // Persistent system worker; keep the creator reference for kernel life. (void)worker; diff --git a/code/kernel/kmain.c b/code/kernel/kmain.c index ed11472..13dbb1d 100644 --- a/code/kernel/kmain.c +++ b/code/kernel/kmain.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -422,7 +423,8 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { uintptr_t entryPoint = LdrGetEntryPoint(imageBase); // Create the main thread last. KeCreateThread queues it before returning. - s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread); + const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, &threadParameters, &thread); if (!KSUCCESS(s)) { DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s); proc->ImageBase = 0; @@ -447,6 +449,9 @@ void SystemInitThread(void* arg) { KiRunEarlyKernelSelfTests(); KiInitializeDrivers(args->bootInfo, args->hal); + + if (args->bootInfo->Debug & KE_DEBUG_AUDIO_SELFTEST) + KiRunAudioFinalPathSelfTest(); KeTimeReloadSettings(); KiInitializeSwapFromSettings(); @@ -532,6 +537,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) { gKmainHal = hal; KeInitDebug(hal, bootInfo->Debug); + DbgPrint("CPU: PAT owns WB/WC/UC policy; MTRR default=WB fixed=off variable=off.\n"); KeTimeInitialize(bootInfo, hal); KeDisplayInitialize(hal); @@ -566,6 +572,15 @@ extern void KernelMain(BOOT_INFO* bootInfo) { KeInitializeScheduler(); + KSTATUS volumeNamespaceStatus = IoInitializeVolumeNamespace(); + if (!KSUCCESS(volumeNamespaceStatus)) { + KePanic(KE_PANIC_CODE_INIT, + "I/O: failed to initialize volume namespace notifications, status=0x%x\n", + volumeNamespaceStatus); + } + + KeAudioInitialize(); + KSTATUS workStatus = KeStartWorkQueue(); if (!KSUCCESS(workStatus)) { KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus); @@ -585,9 +600,10 @@ extern void KernelMain(BOOT_INFO* bootInfo) { // Spawn the Initialization Thread. No creator reference is needed; the // scheduler owns the live thread lifetime. + const KE_THREAD_CREATE_PARAMETERS initThreadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread, (uintptr_t)initArgs, - PriorityNormal, (THREAD**)0); + &initThreadParameters, (THREAD**)0); if (!KSUCCESS(st)) { KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st); diff --git a/code/kernel/ldr_section.c b/code/kernel/ldr_section.c index a23d770..b105a68 100644 --- a/code/kernel/ldr_section.c +++ b/code/kernel/ldr_section.c @@ -327,8 +327,18 @@ static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section, if (!size) return NTLSTATUS_SUCCESS; + // Image prototype pages are the canonical cache for executable image bytes. + // Read the backing filesystem directly so the fault path has one cache owner. + if (!section->BackingFile->Fs || !section->BackingFile->Fs->Read) + return NTLSTATUS_NOT_SUPPORTED; + size_t got = 0; - KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got); + KSTATUS s = section->BackingFile->Fs->Read(section->BackingFile->Fs, + section->BackingFile->FileCtx, + fileOffset, + dst, + size, + &got); if (!KSUCCESS(s)) return s; if (got != size) diff --git a/code/kernel/mm_file_cache.c b/code/kernel/mm_file_cache.c index 37b6b82..b22e50d 100644 --- a/code/kernel/mm_file_cache.c +++ b/code/kernel/mm_file_cache.c @@ -16,19 +16,12 @@ #include #include #include -#include #include #include #include #define MM_FILE_CACHE_BUCKETS 64u #define MM_FILE_CACHE_SOFT_MAX_PAGES 512u -#define MM_FILE_CACHE_READAHEAD_MAX_REQUESTS 64u -#define MM_FILE_CACHE_READAHEAD_MAX_PAGES 32u -#define MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET 128u -#define MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES 16u -#define MM_FILE_CACHE_BATCH_MAX_PAGES 16u -#define MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES 64u #define MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES 128u #define MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES 192u #define MM_FILE_CACHE_WRITEBACK_BATCH_PAGES 16u @@ -48,13 +41,6 @@ typedef struct MM_FILE_CACHE_ENTRY { uint64_t DirtySequence; } MM_FILE_CACHE_ENTRY; -typedef struct MM_FILE_CACHE_READAHEAD_REQUEST { - LIST_ENTRY Link; - FILE_OBJECT* File; - uint64_t Offset; - uint32_t Pages; -} MM_FILE_CACHE_READAHEAD_REQUEST; - static SPINLOCK gFileCacheLock = SPINLOCK_INIT; static MM_FILE_CACHE_ENTRY* gFileCacheBuckets[MM_FILE_CACHE_BUCKETS]; static uint64_t gFileCacheResidentPages; @@ -65,13 +51,6 @@ static uint64_t gFileCacheMaxDirtyPages; static uint64_t gFileCacheMaxResidentPages; static MM_FILE_CACHE_STATS gFileCacheStats; -static SPINLOCK gReadAheadLock = SPINLOCK_INIT; -static LIST_ENTRY gReadAheadQueue; -static SEMAPHORE* gReadAheadSemaphore; -static uint32_t gReadAheadQueuedRequests; -static uint32_t gReadAheadQueuedPages; -static int gReadAheadQueueReady; -static int gReadAheadStartState; static SPINLOCK gWritebackLock = SPINLOCK_INIT; static SEMAPHORE* gWritebackSemaphore; @@ -253,7 +232,8 @@ static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOff return NTLSTATUS_NOT_FOUND; } -static void KiFileCacheAddReadAheadStat(uint64_t* field, uint64_t value) { +// Adds to one file-cache statistic under the cache lock. +static void KiFileCacheAddStat(uint64_t* field, uint64_t value) { uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); *field += value; KeReleaseSpinlock(&gFileCacheLock, flags); @@ -265,17 +245,6 @@ static void KiFileCacheSignalWriteback(void) { KeSignalSemaphore(gWritebackSemaphore); } -// Updates a maximum counter while the file-cache stats lock is held. -static void KiFileCacheMaxStatLocked(uint64_t* field, uint64_t value) { - if (*field < value) - *field = value; -} - -static int KiFileCachePageExistsLocked(FILE_OBJECT* file, uint64_t offset) { - uint32_t bucket = KiFileCacheHash(file, offset); - return KiFileCacheFindLocked(file, offset, bucket) != (MM_FILE_CACHE_ENTRY*)0; -} - // Finds the lowest dirty page for a file, or globally when file is NULL. static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) { MM_FILE_CACHE_ENTRY* best = (MM_FILE_CACHE_ENTRY*)0; @@ -292,115 +261,6 @@ static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) return best; } -static MM_FILE_CACHE_READAHEAD_REQUEST* KiFileCacheReadAheadDequeue(void) { - uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); - LIST_ENTRY* node = ListPopHead(&gReadAheadQueue); - if (node) { - MM_FILE_CACHE_READAHEAD_REQUEST* req = CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link); - if (gReadAheadQueuedRequests) - gReadAheadQueuedRequests--; - if (gReadAheadQueuedPages >= req->Pages) - gReadAheadQueuedPages -= req->Pages; - else - gReadAheadQueuedPages = 0; - } - KeReleaseSpinlock(&gReadAheadLock, flags); - - if (!node) - return (MM_FILE_CACHE_READAHEAD_REQUEST*)0; - return CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link); -} - -static void KiFileCacheReadAheadWorker(void* arg) { - (void)arg; - KeAssertIrql(PASSIVE_LEVEL); - - for (;;) { - KSTATUS ws = KeWaitSemaphore(gReadAheadSemaphore); - if (!KSUCCESS(ws)) { - KeYield(); - continue; - } - - MM_FILE_CACHE_READAHEAD_REQUEST* req = KiFileCacheReadAheadDequeue(); - if (!req) - continue; - - for (uint32_t i = 0; i < req->Pages; ++i) { - uint64_t offset = req->Offset + ((uint64_t)i * PAGE_SIZE); - - uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); - if (gFileCacheResidentPages + MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES >= - MM_FILE_CACHE_SOFT_MAX_PAGES) { - gFileCacheStats.ReadAheadPagesThrottled += (uint64_t)(req->Pages - i); - KeReleaseSpinlock(&gFileCacheLock, flags); - break; - } - int exists = KiFileCachePageExistsLocked(req->File, offset); - if (exists) - gFileCacheStats.ReadAheadPagesSkipped++; - KeReleaseSpinlock(&gFileCacheLock, flags); - if (exists) - continue; - - MM_FILE_CACHE_PAGE page; - KSTATUS s = MmFileCacheGetPage(req->File, offset, &page); - flags = KeAcquireSpinlock(&gFileCacheLock); - if (KSUCCESS(s)) - gFileCacheStats.ReadAheadPagesLoaded++; - else - gFileCacheStats.ReadAheadFailures++; - KeReleaseSpinlock(&gFileCacheLock, flags); - - if (!KSUCCESS(s)) - break; - MmFileCacheReleasePage(&page); - } - - ObDereferenceObject(&req->File->Header); - MmFreePool(req); - } -} - -static KSTATUS KiFileCacheEnsureReadAheadStarted(void) { - uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); - if (gReadAheadStartState == 2) { - KeReleaseSpinlock(&gReadAheadLock, flags); - return NTLSTATUS_SUCCESS; - } - if (gReadAheadStartState == 1) { - KeReleaseSpinlock(&gReadAheadLock, flags); - while (gReadAheadStartState == 1) - KeYield(); - return gReadAheadStartState == 2 ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE; - } - - gReadAheadStartState = 1; - KeReleaseSpinlock(&gReadAheadLock, flags); - - LIST_INIT(&gReadAheadQueue); - gReadAheadQueuedRequests = 0; - gReadAheadQueuedPages = 0; - gReadAheadQueueReady = 1; - - KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gReadAheadSemaphore); - if (KSUCCESS(s)) { - THREAD* worker = (THREAD*)0; - s = KeCreateKernelThread((uintptr_t)KiFileCacheReadAheadWorker, - 0, - PriorityNormal, - &worker); - (void)worker; - } - - flags = KeAcquireSpinlock(&gReadAheadLock); - gReadAheadStartState = KSUCCESS(s) ? 2 : 0; - if (!KSUCCESS(s)) - gReadAheadQueueReady = 0; - KeReleaseSpinlock(&gReadAheadLock, flags); - return s; -} - // Flushes the next dirty cache page for file, or globally when file is NULL. static KSTATUS KiFileCacheFlushOneDirtyPage(FILE_OBJECT* file, int* didWork) { if (didWork) *didWork = 0; @@ -462,8 +322,8 @@ static void KiFileCacheWritebackWorker(void* arg) { (void)arg; KeAssertIrql(PASSIVE_LEVEL); - for (;;) { - KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore); + while (true) { + KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(ws)) { KeYield(); continue; @@ -504,9 +364,10 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) { KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gWritebackSemaphore); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiFileCacheWritebackWorker, 0, - PriorityNormal, + ¶meters, &worker); (void)worker; } @@ -517,15 +378,13 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) { return s; } -static size_t KiMinSize(size_t a, size_t b) { return a < b ? a : b; } - static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_PAGE* outPage, int waitForLoading) { if (outPage) memset(outPage, 0, sizeof(*outPage)); uint32_t bucket = KiFileCacheHash(file, aligned); int counted = 0; - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); if (!counted) { gFileCacheStats.Lookups++; @@ -589,7 +448,7 @@ static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CA } static void KiFileCacheKeepBounded(FILE_OBJECT* skipFile, uint64_t skipOffset) { - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); int over = gFileCacheResidentPages > MM_FILE_CACHE_SOFT_MAX_PAGES; KeReleaseSpinlock(&gFileCacheLock, flags); @@ -718,42 +577,12 @@ static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry, return NTLSTATUS_SUCCESS; } -static uint32_t KiFileCacheChooseForegroundBatchPages(void) { - uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); - uint64_t resident = gFileCacheResidentPages; - KeReleaseSpinlock(&gFileCacheLock, flags); - - if (resident >= MM_FILE_CACHE_SOFT_MAX_PAGES) - return 1; - - uint64_t headroom = MM_FILE_CACHE_SOFT_MAX_PAGES - resident; - if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) - return 1; - if (headroom <= MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES) - return 2; - if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 2u)) - return 4; - if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 4u)) - return 8; - return MM_FILE_CACHE_BATCH_MAX_PAGES; -} - -static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file, +// Loads exactly one demanded file page into the coherent cache. +static KSTATUS KiFileCacheLoadPage(FILE_OBJECT* file, uint64_t aligned, - uint32_t maxPages, - MM_FILE_CACHE_PAGE* requestedPage, - uint32_t* loadedPagesOut) { - if (loadedPagesOut) - *loadedPagesOut = 0; - if (!file || maxPages == 0) + MM_FILE_CACHE_PAGE* outPage) { + if (!file || !outPage) return NTLSTATUS_INVALID_PARAMETER; - if (maxPages > MM_FILE_CACHE_BATCH_MAX_PAGES) - maxPages = MM_FILE_CACHE_BATCH_MAX_PAGES; - - uintptr_t statFlags = KeAcquireSpinlock(&gFileCacheLock); - gFileCacheStats.BatchPagesRequested += maxPages; - KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesRequested, maxPages); - KeReleaseSpinlock(&gFileCacheLock, statFlags); uintptr_t fileFlags = KeAcquireSpinlock(&file->Lock); uint64_t fileSize = file->Size; @@ -761,134 +590,55 @@ static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file, if (aligned >= fileSize) return NTLSTATUS_SUCCESS; - MM_FILE_CACHE_ENTRY* entries[MM_FILE_CACHE_BATCH_MAX_PAGES]; - PHYSICAL_ADDRESS frames[MM_FILE_CACHE_BATCH_MAX_PAGES]; - memset(entries, 0, sizeof(entries)); - memset(frames, 0, sizeof(frames)); + MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)0; + KSTATUS s = KiFileCacheReserveLoadingPage(file, aligned, &entry); + if (s == NTLSTATUS_ALREADY_EXISTS) + return KiFileCacheTryHit(file, aligned, outPage, 1); + if (!KSUCCESS(s)) + return s; - uint32_t pages = 0; - KSTATUS reserveStatus = NTLSTATUS_SUCCESS; - while (pages < maxPages) { - uint64_t pageOffset = aligned + ((uint64_t)pages * PAGE_SIZE); - if (pageOffset >= fileSize) - break; - - reserveStatus = KiFileCacheReserveLoadingPage(file, pageOffset, &entries[pages]); - if (!KSUCCESS(reserveStatus)) - break; - pages++; - } - - if (pages == 0) { - if (requestedPage) - return KiFileCacheTryHit(file, aligned, requestedPage, 1); - return reserveStatus == NTLSTATUS_ALREADY_EXISTS ? NTLSTATUS_ALREADY_EXISTS : reserveStatus; - } - - uint8_t* readBuffer = (uint8_t*)MmAllocatePoolTag((size_t)pages * PAGE_SIZE, MM_POOL_TAG('f', 'c', 'b', 'l')); - if (!readBuffer) { - for (uint32_t i = 0; i < pages; ++i) - KiFileCacheCancelLoadingPage(entries[i]); - KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseCleanFileCache); + if (!frame) { + KiFileCacheCancelLoadingPage(entry); + KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1); return NTLSTATUS_NO_MEMORY; } - memset(readBuffer, 0, (size_t)pages * PAGE_SIZE); - for (uint32_t i = 0; i < pages; ++i) { - frames[i] = MmPfnAllocate(MmPfnUseCleanFileCache); - if (!frames[i]) { - for (uint32_t j = 0; j < i; ++j) - (void)MmPfnRelease(frames[j]); - for (uint32_t j = 0; j < pages; ++j) - KiFileCacheCancelLoadingPage(entries[j]); - MmFreePool(readBuffer); - KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); - return NTLSTATUS_NO_MEMORY; - } + void* page = MmPfnToVirtual(frame); + if (!page) { + (void)MmPfnRelease(frame); + KiFileCacheCancelLoadingPage(entry); + KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1); + return NTLSTATUS_FAILURE; } + memset(page, 0, PAGE_SIZE); + + size_t toRead = PAGE_SIZE; + uint64_t remaining = fileSize - aligned; + if (remaining < PAGE_SIZE) + toRead = (size_t)remaining; - size_t toRead = KiMinSize((size_t)pages * PAGE_SIZE, (size_t)(fileSize - aligned)); size_t got = 0; - KSTATUS s = NTLSTATUS_NOT_SUPPORTED; + s = NTLSTATUS_NOT_SUPPORTED; if (file->Fs && file->Fs->Read) - s = file->Fs->Read(file->Fs, file->FileCtx, aligned, readBuffer, toRead, &got); + s = file->Fs->Read(file->Fs, file->FileCtx, aligned, page, toRead, &got); if (!KSUCCESS(s)) { - for (uint32_t i = 0; i < pages; ++i) { - if (frames[i]) - (void)MmPfnRelease(frames[i]); - KiFileCacheCancelLoadingPage(entries[i]); - } - MmFreePool(readBuffer); - KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); + (void)MmPfnRelease(frame); + KiFileCacheCancelLoadingPage(entry); + KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1); return s; } if (got > toRead) got = toRead; + if (got < PAGE_SIZE) + memset((uint8_t*)page + got, 0, PAGE_SIZE - got); - uint64_t published = 0; - KSTATUS requestedStatus = NTLSTATUS_NOT_FOUND; - for (uint32_t i = 0; i < pages; ++i) { - uint64_t pageOffset = aligned + ((uint64_t)i * PAGE_SIZE); - size_t pageBase = (size_t)i * PAGE_SIZE; - if (pageBase >= got && pageOffset >= fileSize) { - (void)MmPfnRelease(frames[i]); - frames[i] = 0; - KiFileCacheCancelLoadingPage(entries[i]); - continue; - } - - uint32_t validBytes = 0; - if (pageBase < got) { - size_t avail = got - pageBase; - if (avail > PAGE_SIZE) avail = PAGE_SIZE; - validBytes = (uint32_t)avail; - } - - void* page = MmPfnToVirtual(frames[i]); - if (!page) { - (void)MmPfnRelease(frames[i]); - frames[i] = 0; - KiFileCacheCancelLoadingPage(entries[i]); - continue; - } - memcpy(page, readBuffer + pageBase, validBytes); - if (validBytes < PAGE_SIZE) - memset((uint8_t*)page + validBytes, 0, PAGE_SIZE - validBytes); - - MM_FILE_CACHE_PAGE* out = (pageOffset == aligned) ? requestedPage : (MM_FILE_CACHE_PAGE*)0; - KSTATUS ps = KiFileCacheCompleteLoadingPage(entries[i], frames[i], validBytes, out, &published); - if (KSUCCESS(ps)) { - if (pageOffset == aligned) - requestedStatus = NTLSTATUS_SUCCESS; - frames[i] = 0; - } else { - if (pageOffset == aligned) - requestedStatus = ps; - frames[i] = 0; - } - } - - MmFreePool(readBuffer); - - uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); - if (published) { - gFileCacheStats.BatchReads++; - gFileCacheStats.BatchPagesLoaded += published; - KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesLoaded, published); - } - KeReleaseSpinlock(&gFileCacheLock, flags); - - if (loadedPagesOut) - *loadedPagesOut = (uint32_t)published; + s = KiFileCacheCompleteLoadingPage(entry, frame, (uint32_t)got, outPage, (uint64_t*)0); + if (!KSUCCESS(s)) + return s; KiFileCacheKeepBounded(file, aligned); - - if (requestedPage) { - if (KSUCCESS(requestedStatus)) - return NTLSTATUS_SUCCESS; - return KiFileCacheTryHit(file, aligned, requestedPage, 1); - } - return published ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND; + return NTLSTATUS_SUCCESS; } // Returns a retained cache page suitable for in-place buffered writes. @@ -906,7 +656,7 @@ static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file, return NTLSTATUS_SUCCESS; if (needExistingBytes && aligned < oldFileSize) { - s = KiFileCacheLoadSpan(file, aligned, 1, outPage, (uint32_t*)0); + s = KiFileCacheLoadPage(file, aligned, outPage); if (KSUCCESS(s)) return NTLSTATUS_SUCCESS; if (s != NTLSTATUS_NOT_FOUND) @@ -1063,7 +813,7 @@ KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file) { KeReleaseSpinlock(&gFileCacheLock, flags); KSTATUS firstError = NTLSTATUS_SUCCESS; - for (;;) { + while (true) { int didWork = 0; KSTATUS s = KiFileCacheFlushOneDirtyPage(file, &didWork); if (!didWork) @@ -1090,7 +840,7 @@ KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAG KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1); if (KSUCCESS(s)) return NTLSTATUS_SUCCESS; - return KiFileCacheLoadSpan(file, aligned, KiFileCacheChooseForegroundBatchPages(), outPage, (uint32_t*)0); + return KiFileCacheLoadPage(file, aligned, outPage); } void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page) { @@ -1111,7 +861,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) { } for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { - for (;;) { + while (true) { MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0; uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; @@ -1145,7 +895,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) { void MmFileCacheInvalidateAll(FILE_OBJECT* file) { if (!file) return; for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { - for (;;) { + while (true) { MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0; uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; @@ -1179,83 +929,6 @@ KSTATUS MmFileCacheDropOnePageForPressure(void) { return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0); } -KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages) { - if (!file || pages == 0) - return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) - return NTLSTATUS_INVALID_IRQL; - - uint32_t requestedPages = pages; - if (pages > MM_FILE_CACHE_READAHEAD_MAX_PAGES) - pages = MM_FILE_CACHE_READAHEAD_MAX_PAGES; - - uintptr_t cacheFlags = KeAcquireSpinlock(&gFileCacheLock); - gFileCacheStats.ReadAheadPagesRequested += requestedPages; - KiFileCacheMaxStatLocked(&gFileCacheStats.MaxReadAheadPagesPerRequest, requestedPages); - uint64_t resident = gFileCacheResidentPages; - uint64_t headroom = resident < MM_FILE_CACHE_SOFT_MAX_PAGES - ? MM_FILE_CACHE_SOFT_MAX_PAGES - resident - : 0; - if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) { - gFileCacheStats.ReadAheadPagesThrottled += pages; - KeReleaseSpinlock(&gFileCacheLock, cacheFlags); - return NTLSTATUS_QUEUE_FULL; - } - headroom -= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES; - if ((uint64_t)pages > headroom) - pages = (uint32_t)headroom; - KeReleaseSpinlock(&gFileCacheLock, cacheFlags); - if (pages == 0) - return NTLSTATUS_QUEUE_FULL; - - KSTATUS s = KiFileCacheEnsureReadAheadStarted(); - if (!KSUCCESS(s)) { - KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1); - return s; - } - - MM_FILE_CACHE_READAHEAD_REQUEST* req = - (MM_FILE_CACHE_READAHEAD_REQUEST*)MmAllocatePoolTag(sizeof(*req), MM_POOL_TAG('r', 'd', 'a', 'h')); - if (!req) { - KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1); - return NTLSTATUS_NO_MEMORY; - } - - LIST_INIT(&req->Link); - req->File = file; - req->Offset = offset & ~(uint64_t)(PAGE_SIZE - 1); - req->Pages = pages; - ObReferenceObject(&file->Header); - - uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); - if (!gReadAheadQueueReady || - gReadAheadQueuedRequests >= MM_FILE_CACHE_READAHEAD_MAX_REQUESTS || - gReadAheadQueuedPages >= MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET) { - KeReleaseSpinlock(&gReadAheadLock, flags); - ObDereferenceObject(&file->Header); - MmFreePool(req); - KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadPagesThrottled, pages); - return NTLSTATUS_QUEUE_FULL; - } - uint32_t queueHeadroom = MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET - gReadAheadQueuedPages; - if (pages > queueHeadroom) - pages = queueHeadroom; - req->Pages = pages; - - ListInsertTail(&gReadAheadQueue, &req->Link); - gReadAheadQueuedRequests++; - gReadAheadQueuedPages += pages; - KeReleaseSpinlock(&gReadAheadLock, flags); - - flags = KeAcquireSpinlock(&gFileCacheLock); - gFileCacheStats.ReadAheadRequests++; - gFileCacheStats.ReadAheadPagesQueued += pages; - KeReleaseSpinlock(&gFileCacheLock, flags); - - KeSignalSemaphore(gReadAheadSemaphore); - return NTLSTATUS_SUCCESS; -} - void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out) { if (!out) return; uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); diff --git a/code/kernel/mm_phys.c b/code/kernel/mm_phys.c index 1680ff7..0a636fc 100644 --- a/code/kernel/mm_phys.c +++ b/code/kernel/mm_phys.c @@ -472,8 +472,13 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min return (PHYSICAL_ADDRESS)0; } -// Allocates a physically contiguous run for DMA or hardware structures. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) { +// Allocates a physically contiguous run inside an address and alignment window. +PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count, + PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive, + size_t alignment, + size_t boundary, + MM_PFN_USAGE usage) { if (!count || count > gMaxPfn) { uintptr_t flags = KeAcquireSpinlock(&gPfnLock); gPfnFailedContiguousAllocations++; @@ -482,6 +487,50 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_ return (PHYSICAL_ADDRESS)0; } + if (!alignment) + alignment = PAGE_SIZE; + if (alignment < PAGE_SIZE) + alignment = PAGE_SIZE; + if ((alignment & (alignment - 1)) != 0 || + (boundary && (boundary & (boundary - 1)) != 0)) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + + if (count > ((size_t)-1) / PAGE_SIZE) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + + size_t byteCount = count * PAGE_SIZE; + if (boundary && byteCount > boundary) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + + minInclusive = KiAlignUp(minInclusive); + maxExclusive = KiAlignDown(maxExclusive); + PHYSICAL_ADDRESS managedLimit = (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE; + if (maxExclusive > managedLimit) + maxExclusive = managedLimit; + if (maxExclusive <= minInclusive || + (PHYSICAL_ADDRESS)byteCount > maxExclusive - minInclusive) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + if (!KiPfnUsageMayUseReserve(usage)) KiPfnReclaimForPressure(); @@ -493,23 +542,31 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_ return (PHYSICAL_ADDRESS)0; } - size_t runStart = 0; - size_t runLength = 0; - - for (size_t pfn = 1; pfn < gMaxPfn; pfn++) { - if (KiPfnAvailableLocked(pfn)) { - if (!runLength) - runStart = pfn; - runLength++; - if (runLength == count) { - for (size_t i = 0; i < count; i++) - KiClaimPfnLocked(runStart + i, usage); - KeReleaseSpinlock(&gPfnLock, flags); - return MmPfnAddress(runStart); - } - } else { - runLength = 0; + size_t firstPfn = (size_t)(minInclusive / PAGE_SIZE); + size_t limitPfn = (size_t)(maxExclusive / PAGE_SIZE); + for (size_t runStart = firstPfn; runStart + count <= limitPfn; ++runStart) { + PHYSICAL_ADDRESS runAddress = MmPfnAddress(runStart); + if ((runAddress & (alignment - 1)) != 0) + continue; + if (boundary) { + PHYSICAL_ADDRESS runEnd = runAddress + (PHYSICAL_ADDRESS)byteCount - 1; + if ((runAddress & ~((PHYSICAL_ADDRESS)boundary - 1)) != + (runEnd & ~((PHYSICAL_ADDRESS)boundary - 1))) + continue; } + + size_t available = 0; + while (available < count && KiPfnAvailableLocked(runStart + available)) + available++; + if (available != count) { + runStart += available; + continue; + } + + for (size_t i = 0; i < count; ++i) + KiClaimPfnLocked(runStart + i, usage); + KeReleaseSpinlock(&gPfnLock, flags); + return runAddress; } gPfnFailedContiguousAllocations++; @@ -518,6 +575,16 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_ return (PHYSICAL_ADDRESS)0; } +// Allocates a physically contiguous run for DMA or hardware structures. +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) { + return MmPfnAllocateContiguousInRange(count, + PAGE_SIZE, + (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE, + PAGE_SIZE, + 0, + usage); +} + // Increments the reference count on an allocated frame. __declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame) { size_t pfn = MmPfnIndex(frame); diff --git a/code/kernel/mm_pte.c b/code/kernel/mm_pte.c index 0383584..2a71465 100644 --- a/code/kernel/mm_pte.c +++ b/code/kernel/mm_pte.c @@ -60,6 +60,14 @@ static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose return purpose; } +// Returns whether flags select one canonical PAT cache policy. +static bool KiPteCacheFlagsValid(uint64_t flags) { + uint64_t cacheFlags = flags & PAGE_CACHE_MASK; + return cacheFlags == PAGE_CACHE_WB || + cacheFlags == PAGE_CACHE_WC || + cacheFlags == PAGE_CACHE_UC; +} + // Ensures that one parent entry points to a child page table. // New child tables are zeroed before publication so no stale mappings become visible. static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys, @@ -200,6 +208,10 @@ int MmPteHandleShootdownNmi(void) { // Maps or replaces one 4 KiB leaf PTE in the shared page tables. __declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) { KiPteStatInc(&gPteStats.MapCalls); + if (!KiPteCacheFlagsValid(flags)) { + KiPteStatInc(&gPteStats.MapFailures); + return NTLSTATUS_INVALID_PARAMETER; + } unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0; uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); @@ -286,6 +298,10 @@ __declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) { // Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them. __declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) { KiPteStatInc(&gPteStats.ProtectCalls); + if (!KiPteCacheFlagsValid(flags)) { + KiPteStatInc(&gPteStats.ProtectFailures); + return NTLSTATUS_INVALID_PARAMETER; + } if (!size) return NTLSTATUS_SUCCESS; // The protection operation is page-granular even when callers pass an interior address. diff --git a/code/kernel/mm_virt.c b/code/kernel/mm_virt.c index 422a3ff..218cced 100644 --- a/code/kernel/mm_virt.c +++ b/code/kernel/mm_virt.c @@ -7,6 +7,7 @@ // FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings. // FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map. // FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys. +// FFFFA00000000000-FFFFAFFFFFFFFFFF is the kernel shared-view arena. // FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool. // // The current VM model uses one shared PML4, so user pointers are globally @@ -112,6 +113,12 @@ typedef struct MM_VAD_PAGE { uint32_t SwapSlot; } MM_VAD_PAGE; +struct MM_KERNEL_VIEW { + LIST_ENTRY Link; + uintptr_t Base; + size_t PageCount; +}; + typedef struct MM_VAD { LIST_ENTRY ProcessLink; LIST_ENTRY GlobalLink; @@ -143,13 +150,19 @@ static LIST_ENTRY gVadList; static SPINLOCK gVadLock = SPINLOCK_INIT; static int gVadListReady = 0; static MM_VAD_STATS gVadStats; +static LIST_ENTRY gKernelViewList; +static SPINLOCK gKernelViewLock = SPINLOCK_INIT; -#define MM_VAD_ARENA_BASE 0x0000100000000000ULL +#define MM_VAD_ARENA_BASE 0x0000100000000000ULL +#define MM_KERNEL_VIEW_BASE 0xFFFFA00000000000ULL +#define MM_KERNEL_VIEW_LIMIT 0xFFFFB00000000000ULL // Initializes the VAD list after the shared page table and kernel pool are ready. static void KiVadInitialize(void) { LIST_INIT(&gVadList); + LIST_INIT(&gKernelViewList); gVadLock = (SPINLOCK)SPINLOCK_INIT; + gKernelViewLock = (SPINLOCK)SPINLOCK_INIT; memset(&gVadStats, 0, sizeof(gVadStats)); gVadListReady = 1; } @@ -170,6 +183,108 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) { (void)MmPfnPin(gVadZeroPageFrame); } +// Inserts a kernel view before one list node while preserving address order. +static void KiKernelViewInsertBefore(LIST_ENTRY* before, MM_KERNEL_VIEW* view) { + view->Link.Fwd = before; + view->Link.Bwd = before->Bwd; + before->Bwd->Fwd = &view->Link; + before->Bwd = &view->Link; +} + +// Maps existing frames into the kernel shared-view arena. +KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames, + size_t pageCount, + MM_KERNEL_VIEW** outView) { + if (!outView) + return NTLSTATUS_INVALID_PARAMETER; + *outView = (MM_KERNEL_VIEW*)0; + if (!frames || pageCount == 0 || pageCount > (SIZE_MAX / PAGE_SIZE)) + return NTLSTATUS_INVALID_PARAMETER; + + size_t viewBytes = pageCount * PAGE_SIZE; + MM_KERNEL_VIEW* view = (MM_KERNEL_VIEW*)MmAllocatePoolTag(sizeof(MM_KERNEL_VIEW), + MM_POOL_TAG('k', 'v', 'i', 'w')); + if (!view) + return NTLSTATUS_NO_MEMORY; + memset(view, 0, sizeof(*view)); + LIST_INIT(&view->Link); + + uintptr_t candidate = MM_KERNEL_VIEW_BASE; + LIST_ENTRY* before = &gKernelViewList; + uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock); + for (LIST_ENTRY* it = gKernelViewList.Fwd; it != &gKernelViewList; it = it->Fwd) { + MM_KERNEL_VIEW* existing = CONTAINING_RECORD(it, MM_KERNEL_VIEW, Link); + if (candidate <= existing->Base && + viewBytes <= (size_t)(existing->Base - candidate)) { + before = it; + break; + } + + uintptr_t existingEnd = existing->Base + existing->PageCount * PAGE_SIZE; + if (existingEnd > candidate) + candidate = existingEnd; + } + + if (candidate > MM_KERNEL_VIEW_LIMIT || viewBytes > MM_KERNEL_VIEW_LIMIT - candidate) { + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + MmFreePool(view); + return NTLSTATUS_NO_MEMORY; + } + + view->Base = candidate; + view->PageCount = pageCount; + KiKernelViewInsertBefore(before, view); + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + + const uint64_t hardwareFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX; + size_t mappedPages = 0; + KSTATUS s = NTLSTATUS_SUCCESS; + for (; mappedPages < pageCount; mappedPages++) { + if (!frames[mappedPages]) { + s = NTLSTATUS_INVALID_PARAMETER; + break; + } + s = MmPteMapPage(candidate + mappedPages * PAGE_SIZE, frames[mappedPages], + hardwareFlags, MmPteMapFailIfPresent); + if (!KSUCCESS(s)) + break; + } + + if (!KSUCCESS(s)) { + while (mappedPages != 0) { + mappedPages--; + (void)MmPteUnmapPage(candidate + mappedPages * PAGE_SIZE); + } + lockFlags = KeAcquireSpinlock(&gKernelViewLock); + ListRemove(&view->Link); + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + MmFreePool(view); + return s; + } + + *outView = view; + return NTLSTATUS_SUCCESS; +} + +// Returns the base address of one kernel shared view. +void* MmKernelViewBase(const MM_KERNEL_VIEW* view) { + return view ? (void*)view->Base : (void*)0; +} + +// Unmaps and releases one kernel shared view without releasing its frames. +void MmDestroyKernelView(MM_KERNEL_VIEW* view) { + if (!view) + return; + + for (size_t i = 0; i < view->PageCount; i++) + (void)MmPteUnmapPage(view->Base + i * PAGE_SIZE); + + uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock); + ListRemove(&view->Link); + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + MmFreePool(view); +} + // VAD utility helpers. static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) { if (align < PAGE_SIZE) align = PAGE_SIZE; @@ -199,7 +314,7 @@ static KSTATUS KiFindVadRange(size_t reservationSize, size_t alignment, uintptr_ uintptr_t candidate = KiAlignUpPtr(MM_VAD_ARENA_BASE, alignment); if (!candidate) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { if (candidate < MM_VAD_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate)) return NTLSTATUS_NO_MEMORY; @@ -438,7 +553,7 @@ static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) { static int KiVadTryBeginPaging(MM_VAD_PAGE* page) { if (!page) return 0; - for (;;) { + while (true) { uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0); if (oldFlags & MM_VAD_PAGE_PAGING) return 0; @@ -454,7 +569,7 @@ static int KiVadTryBeginPaging(MM_VAD_PAGE* page) { static void KiVadEndPaging(MM_VAD_PAGE* page) { if (!page) return; - for (;;) { + while (true) { uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0); uint32_t newFlags = oldFlags & ~MM_VAD_PAGE_PAGING; if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, @@ -1157,6 +1272,18 @@ __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, return NTLSTATUS_SUCCESS; } +// Returns the page count held by a locked user buffer. +size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer) { + return buffer ? buffer->PageCount : 0; +} + +// Returns one pinned frame held by a locked user buffer. +PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index) { + if (!buffer || index >= buffer->PageCount) + return (PHYSICAL_ADDRESS)0; + return buffer->Pages[index].Frame; +} + // Unlocks a user buffer locked for asynchronous kernel access. __declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer) { if (!buffer) @@ -2256,7 +2383,7 @@ void MmVadDestroyAllForProcess(PROCESS* process) { KeAssertIrql(PASSIVE_LEVEL); if (!process) return; - for (;;) { + while (true) { uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); LIST_ENTRY* node = (LIST_ENTRY*)0; int sawLocked = 0; diff --git a/code/kernel/net_socket.c b/code/kernel/net_socket.c index 307a23e..2638d8d 100644 --- a/code/kernel/net_socket.c +++ b/code/kernel/net_socket.c @@ -379,8 +379,8 @@ static void NetAsyncSocketExecute(NET_ASYNC_SOCKET_REQUEST_OBJECT* req, static void NetAsyncSocketWorker(void* arg) { (void)arg; - for (;;) { - KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore); + while (true) { + KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(waitStatus)) { KeYield(); continue; @@ -426,9 +426,10 @@ static KSTATUS NetEnsureAsyncSocketWorkerStarted(void) { KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncSocketSemaphore); if (KSUCCESS(status)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); status = KeCreateKernelThread((uintptr_t)NetAsyncSocketWorker, 0, - PriorityNormal, + ¶meters, &worker); (void)worker; } diff --git a/code/kernel/ob_object.c b/code/kernel/ob_object.c index bde8914..3e59ab7 100644 --- a/code/kernel/ob_object.c +++ b/code/kernel/ob_object.c @@ -31,6 +31,8 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) { case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' '); case ObjectTypeModule: return MM_POOL_TAG('m', 'o', 'd', 'u'); case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k'); + case ObjectTypeAudioEndpoint: return MM_POOL_TAG('a', 'e', 'p', 't'); + case ObjectTypeAudioStream: return MM_POOL_TAG('a', 's', 't', 'r'); default: return MM_POOL_TAG('o', 'b', 'j', ' '); } } @@ -64,7 +66,7 @@ __declspec(dllexport) bool ObTryReferenceObject(OBJECT* object) { if (!object) return false; - for (;;) { + while (true) { if (_InterlockedCompareExchange(&object->LifetimeState, 0, 0) != ObjectAlive) return false; if ((_InterlockedCompareExchange(&object->Flags, 0, 0) & OBJECT_FLAG_INSERTED) == 0) diff --git a/code/public/ntlite/audio.h b/code/public/ntlite/audio.h new file mode 100644 index 0000000..5c93c48 --- /dev/null +++ b/code/public/ntlite/audio.h @@ -0,0 +1,220 @@ +// Public native render-stream syscall ABI declarations. + +#pragma once + +#include +#include + +#define SYS_AUDIO_ABI_VERSION 3u +#define SYS_AUDIO_MAX_ENDPOINTS 32u +#define SYS_AUDIO_STREAM_CAPACITY_FRAMES 4096u + +#define SYS_AUDIO_GAIN_Q16_UNITY 65536u +#define SYS_AUDIO_PAN_Q16_LEFT (-65536) +#define SYS_AUDIO_PAN_Q16_CENTER 0 +#define SYS_AUDIO_PAN_Q16_RIGHT 65536 + +// Identifies one logical render destination. +typedef enum SYS_AUDIO_ENDPOINT_KIND { + SysAudioEndpointLineOut = 1, + SysAudioEndpointSpeakerHeadphone, + SysAudioEndpointOtherAnalog +} SYS_AUDIO_ENDPOINT_KIND; + +// Identifies the interleaved PCM sample representation accepted by native streams. +typedef enum SYS_AUDIO_SAMPLE_FORMAT { + SysAudioSampleS16 = 1 +} SYS_AUDIO_SAMPLE_FORMAT; + +// Describes one interleaved PCM stream format. +typedef struct SYS_AUDIO_FORMAT { + uint32_t SampleRate; + uint16_t ChannelCount; + uint16_t SampleFormat; +} SYS_AUDIO_FORMAT; + +#define SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE 0x00000001u +#define SYS_AUDIO_ENDPOINT_FLAG_DEFAULT 0x00000002u +#define SYS_AUDIO_ENDPOINT_FLAG_VALID_MASK 0x00000003u + +#define SYS_AUDIO_ENDPOINT_CAP_JACK_SENSE 0x00000001u +#define SYS_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN 0x00000002u +#define SYS_AUDIO_ENDPOINT_CAP_VALID_MASK 0x00000003u + +// Identifies the lifecycle state reported for one render endpoint. +typedef enum SYS_AUDIO_ENDPOINT_STATE { + SysAudioEndpointStarting = 1, + SysAudioEndpointReady, + SysAudioEndpointStopping, + SysAudioEndpointFailed, + SysAudioEndpointRemoved +} SYS_AUDIO_ENDPOINT_STATE; + +// Reports one endpoint snapshot keyed by its stable kernel identifier. +typedef struct SYS_AUDIO_ENDPOINT_INFO { + uint32_t Size; + uint32_t Version; + uint64_t EndpointId; + uint32_t Kind; + uint32_t Flags; + uint32_t Capabilities; + uint32_t State; + SYS_AUDIO_FORMAT HardwareFormat; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + uint32_t MasterVolumeQ16; + uint32_t MasterMuted; + KSTATUS FailureStatus; + uint32_t Reserved; +} SYS_AUDIO_ENDPOINT_INFO; + +// Selects the endpoint array used by one enumeration call. +typedef struct SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST { + uint32_t Size; + uint32_t Flags; + SYS_AUDIO_ENDPOINT_INFO* Endpoints; + uint32_t Capacity; + uint32_t Reserved; + uint32_t* CountOut; +} SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST; + +// Selects one endpoint and source format for a new render stream handle. +typedef struct SYS_AUDIO_OPEN_STREAM_REQUEST { + uint32_t Size; + uint32_t Flags; + uint64_t EndpointId; + SYS_AUDIO_FORMAT Format; + uint32_t Reserved; + uint32_t* StreamHandleOut; +} SYS_AUDIO_OPEN_STREAM_REQUEST; + +// Describes one bounded nonblocking stream write. +typedef struct SYS_AUDIO_WRITE_REQUEST { + uint32_t Size; + uint32_t Flags; + uint32_t StreamHandle; + uint32_t FrameCount; + const void* Frames; + uint32_t* WrittenFramesOut; +} SYS_AUDIO_WRITE_REQUEST; + +#define SYS_AUDIO_WAIT_WRITABLE 0x00000001u +#define SYS_AUDIO_WAIT_CONSUMED 0x00000002u +#define SYS_AUDIO_WAIT_PRESENTED 0x00000004u +#define SYS_AUDIO_WAIT_STOPPED 0x00000008u +#define SYS_AUDIO_WAIT_DRAINED 0x00000010u +#define SYS_AUDIO_WAIT_FAILED 0x00000020u +#define SYS_AUDIO_WAIT_DISCONTINUITY 0x00000040u +#define SYS_AUDIO_WAIT_VALID_MASK 0x0000007Fu + +// Arms the stream handle event for one or more progress conditions. +typedef struct SYS_AUDIO_ARM_WAIT_REQUEST { + uint32_t Size; + uint32_t Flags; + uint32_t StreamHandle; + uint32_t Mask; + // Frame counters are inclusive absolute targets and generations are change baselines. + uint32_t WritableFrames; + uint32_t Reserved; + uint64_t ConsumedFrames; + uint64_t PresentedFrames; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; +} SYS_AUDIO_ARM_WAIT_REQUEST; + +// Identifies the public lifecycle state of one native render stream. +typedef enum SYS_AUDIO_STREAM_STATE { + SysAudioStreamCreated = 1, + SysAudioStreamRunning, + SysAudioStreamPaused, + SysAudioStreamDraining, + SysAudioStreamStopped, + SysAudioStreamFailed +} SYS_AUDIO_STREAM_STATE; + +// Reports one coherent stream snapshot. +typedef struct SYS_AUDIO_STREAM_INFO { + uint32_t Size; + uint32_t Version; + uint32_t State; + uint32_t Reserved; + SYS_AUDIO_FORMAT Format; + uint32_t CapacityFrames; + uint32_t BufferedFrames; + uint32_t WritableFrames; + KSTATUS FailureStatus; + uint64_t FramesSubmitted; + uint64_t FramesConsumed; + uint64_t FramesPresented; + uint64_t FramesDiscarded; + uint64_t OutputFramesMixed; + uint64_t SilenceFramesInserted; + // Underruns count recoverable source starvation periods. + uint64_t Underruns; + uint64_t DrainedGeneration; + // Discontinuities report explicit control or hardware timeline changes. + uint64_t DiscontinuityGeneration; + uint32_t VolumeQ16; + int32_t PanQ16; + uint8_t Muted; + uint8_t ReservedBytes[7]; +} SYS_AUDIO_STREAM_INFO; + +// Updates one stream's software gain, balance, and mute targets. +// Volume uses unsigned Q16 in [0, 1] and pan uses signed Q16 in [-1, 1]. +typedef struct SYS_AUDIO_SET_VOLUME_REQUEST { + uint32_t Size; + uint32_t Flags; + uint32_t StreamHandle; + uint32_t Muted; + uint32_t VolumeQ16; + int32_t PanQ16; +} SYS_AUDIO_SET_VOLUME_REQUEST; + +// Updates one endpoint's master software gain and mute state. +typedef struct SYS_AUDIO_SET_MASTER_VOLUME_REQUEST { + uint32_t Size; + uint32_t Flags; + uint64_t EndpointId; + uint32_t Muted; + uint32_t VolumeQ16; +} SYS_AUDIO_SET_MASTER_VOLUME_REQUEST; + +// Enumerates render endpoints and returns the complete count even when capacity is insufficient. +KSTATUS SysAudioEnumerateEndpoints(const SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST* request); + +// Opens one waitable, process-owned native render stream handle. +KSTATUS SysAudioOpenStream(const SYS_AUDIO_OPEN_STREAM_REQUEST* request); + +// Copies as many complete frames as currently fit and returns WOULD_BLOCK when none fit. +KSTATUS SysAudioWrite(const SYS_AUDIO_WRITE_REQUEST* request); + +// Replaces the any-condition progress targets that drive one stream handle's wait state. +KSTATUS SysAudioArmWait(const SYS_AUDIO_ARM_WAIT_REQUEST* request); + +// Copies one coherent stream snapshot after the caller initializes infoOut->Size. +KSTATUS SysAudioQueryStream(uint32_t streamHandle, SYS_AUDIO_STREAM_INFO* infoOut); + +// Starts one created or stopped stream. +KSTATUS SysAudioStart(uint32_t streamHandle); + +// Pauses future mixing while preserving unconsumed source frames. +KSTATUS SysAudioPause(uint32_t streamHandle); + +// Resumes one paused stream. +KSTATUS SysAudioResume(uint32_t streamHandle); + +// Rejects new writes and stops after all submitted frames reach a terminal disposition. +KSTATUS SysAudioDrain(uint32_t streamHandle); + +// Stops future mixing and discards source frames not yet committed to hardware periods. +KSTATUS SysAudioStop(uint32_t streamHandle); + +// Returns one stopped stream to an empty position-zero state. +KSTATUS SysAudioReset(uint32_t streamHandle); + +// Updates one stream's software volume, pan, and mute state. +KSTATUS SysAudioSetVolume(const SYS_AUDIO_SET_VOLUME_REQUEST* request); + +// Updates one endpoint's master software volume and mute state. +KSTATUS SysAudioSetMasterVolume(const SYS_AUDIO_SET_MASTER_VOLUME_REQUEST* request); diff --git a/code/public/ntlite/ddi/display.h b/code/public/ntlite/ddi/display.h index bdb1392..4eb5508 100644 --- a/code/public/ntlite/ddi/display.h +++ b/code/public/ntlite/ddi/display.h @@ -25,7 +25,13 @@ typedef struct DISPLAY_DEVICE_OPS { KSTATUS (*Blit)(DISPLAY_DEVICE* device, int x, int y, int w, int h, - const uint32_t* pixels); + const uint32_t* pixels, + uint32_t pitchBytes); + + KSTATUS (*PresentPitched)(DISPLAY_DEVICE* device, + int x, int y, int w, int h, + const uint32_t* pixels, + uint32_t pitchBytes); KSTATUS (*FillRect)(DISPLAY_DEVICE* device, int x, int y, int w, int h, diff --git a/code/public/ntlite/ddi/io_driver.h b/code/public/ntlite/ddi/io_driver.h index ec5a7b5..6d7ccf4 100644 --- a/code/public/ntlite/ddi/io_driver.h +++ b/code/public/ntlite/ddi/io_driver.h @@ -20,6 +20,7 @@ typedef void (*DRIVER_UNLOAD_FN)(void); typedef struct DRIVER_OBJECT DRIVER_OBJECT; typedef struct DEVICE_OBJECT DEVICE_OBJECT; typedef struct IRP IRP; +struct IO_DMA_ADAPTER; typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, IRP* irp); typedef KSTATUS (*PDRIVER_ADD_DEVICE)(DRIVER_OBJECT* driver, @@ -89,6 +90,7 @@ struct DEVICE_OBJECT { DEVICE_OBJECT* LowerDevice; // device directly below this FDO/filter void* DeviceExtension; + struct IO_DMA_ADAPTER* DmaAdapter; uint32_t Flags; uint32_t Characteristics; @@ -183,17 +185,6 @@ typedef struct IO_DISK_STATS { uint64_t WriteSectors; uint64_t MaxReadSectors; uint64_t MaxWriteSectors; - uint64_t ReadaheadHits; - uint64_t ReadaheadMisses; - uint64_t ReadaheadStores; - uint64_t ReadaheadInvalidations; - uint64_t ReadaheadBytesServed; - uint64_t ReadaheadSectorsLoaded; - uint64_t ReadaheadSequentialDetections; - uint64_t ReadaheadSuppressed; - uint64_t ReadaheadAllocFailures; - uint64_t ReadaheadReadFailures; - uint64_t MaxReadaheadSectors; } IO_DISK_STATS; __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, @@ -220,4 +211,5 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, uint64_t lba, uint32_t count, const void* buffer); +__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex); __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out); diff --git a/code/public/ntlite/ddk/audio.h b/code/public/ntlite/ddk/audio.h new file mode 100644 index 0000000..ecabab2 --- /dev/null +++ b/code/public/ntlite/ddk/audio.h @@ -0,0 +1,105 @@ +// Public nt-lite kernel-audio provider declarations. + +#pragma once + +#include +#include +#include + +struct DEVICE_OBJECT; +typedef struct IO_AUDIO_ENDPOINT IO_AUDIO_ENDPOINT; + +// Identifies one logical render destination rather than one codec pin. +typedef enum IO_AUDIO_ENDPOINT_KIND { + IoAudioEndpointLineOut = 1, + IoAudioEndpointSpeakerHeadphone, + IoAudioEndpointOtherAnalog +} IO_AUDIO_ENDPOINT_KIND; + +// Identifies the canonical interleaved PCM sample representation. +typedef enum IO_AUDIO_SAMPLE_FORMAT { + IoAudioSampleS16 = 1 +} IO_AUDIO_SAMPLE_FORMAT; + +// Describes interleaved PCM frames. +typedef struct IO_AUDIO_FORMAT { + uint32_t SampleRate; + uint16_t ChannelCount; + uint16_t SampleFormat; +} IO_AUDIO_FORMAT; + +#define IO_AUDIO_ENDPOINT_CAP_JACK_SENSE 0x00000001u +#define IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN 0x00000002u +#define IO_AUDIO_ENDPOINT_CAP_VALID_MASK 0x00000003u + +// Bounds provider-advertised hardware gain ranges in millidecibels. +#define IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB 200000 + +// Distinguishes orderly teardown from surprise removal where registers may no longer be accessible. +typedef enum IO_AUDIO_REMOVAL_MODE { + IoAudioOrderlyRemoval = 1, + IoAudioSurpriseRemoval +} IO_AUDIO_REMOVAL_MODE; + +#define IO_AUDIO_FAILURE_HARDWARE_INACCESSIBLE 0x00000001u +#define IO_AUDIO_FAILURE_VALID_MASK 0x00000001u + +// Provider operations are invoked only at PASSIVE_LEVEL by the endpoint mixer thread. +typedef struct IO_AUDIO_PROVIDER_OPERATIONS { + // Starts DMA only after the ring contains at least two prepared periods. + // Failure must leave the provider stopped; Stop may still be called for partial-start cleanup. + KSTATUS (*Start)(void* providerContext); + // Stops DMA before unregister permits provider storage to be released. + void (*Stop)(void* providerContext); + // Applies endpoint gain and mute only with IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN. + KSTATUS (*SetGain)(void* providerContext, int32_t gainMilliDb, uint8_t muted); +} IO_AUDIO_PROVIDER_OPERATIONS; + +// Registers one fixed-format render ring owned by a hardware provider. +// The provider must keep DMA stopped and the physical output muted until Start is called. +typedef struct IO_AUDIO_ENDPOINT_REGISTRATION { + struct DEVICE_OBJECT* Device; + IO_AUDIO_ENDPOINT_KIND Kind; + uint32_t Capabilities; + uint8_t InitiallyAvailable; + void* ProviderContext; + IO_AUDIO_PROVIDER_OPERATIONS Operations; + void* PeriodBuffer; + size_t PeriodBufferBytes; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + size_t PeriodStrideBytes; + IO_AUDIO_FORMAT HardwareFormat; + // Gain fields must be zero unless IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN is present. + int32_t HardwareGainMinimumMilliDb; + int32_t HardwareGainMaximumMilliDb; + uint32_t HardwareGainStepMilliDb; +} IO_AUDIO_ENDPOINT_REGISTRATION; + +// Driver-visible services for registering and advancing kernel audio endpoints. +typedef struct NTL_DDK_AUDIO { + // Registers one endpoint and returns a provider-owned reference valid until unregister returns. + KSTATUS (*RegisterEndpoint)(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, + IO_AUDIO_ENDPOINT** outEndpoint); + + // Stops the endpoint, removes it from enumeration, and consumes the provider-owned reference. + void (*UnregisterEndpoint)(IO_AUDIO_ENDPOINT* endpoint, IO_AUDIO_REMOVAL_MODE mode); + + // Reports one completed hardware period from ISR/DPC completion processing. + // presentedFrames, hardwarePositionFrames, and interruptCount must be monotonic for the + // endpoint lifetime. timestampTicks uses the DDK monotonic microsecond source. + void (*NotifyPeriod)(IO_AUDIO_ENDPOINT* endpoint, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t hardwarePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks); + + // Updates jack-derived endpoint availability without changing endpoint identity. + void (*SetEndpointAvailable)(IO_AUDIO_ENDPOINT* endpoint, uint8_t available); + + // Fails an endpoint and wakes every stream owner. + void (*ReportEndpointFailure)(IO_AUDIO_ENDPOINT* endpoint, + KSTATUS status, + uint32_t flags); +} NTL_DDK_AUDIO; diff --git a/code/public/ntlite/ddk/ddk.h b/code/public/ntlite/ddk/ddk.h index 9ad13de..b24ee96 100644 --- a/code/public/ntlite/ddk/ddk.h +++ b/code/public/ntlite/ddk/ddk.h @@ -4,7 +4,9 @@ #include #include +#include #include +#include #include #include #include @@ -25,12 +27,14 @@ #include #include -#define NTL_DDK_VERSION 2u +#define NTL_DDK_VERSION 1u // Root DDK table passed to kernel drivers by the driver loader. typedef struct NTL_DDK { uint32_t Version; const NTL_DDK_DEBUG* Debug; + const NTL_DDK_AUDIO* Audio; + const NTL_DDK_DMA* Dma; const NTL_DDK_INTERRUPT* Interrupt; const NTL_DDK_MEMORY* Memory; const NTL_DDK_PLATFORM* Platform; diff --git a/code/public/ntlite/ddk/debug.h b/code/public/ntlite/ddk/debug.h index 18ccd3b..b6863ee 100644 --- a/code/public/ntlite/ddk/debug.h +++ b/code/public/ntlite/ddk/debug.h @@ -2,11 +2,29 @@ #pragma once +#include #include +#define KE_DEBUG_CONSOLE 0x00000001u +#define KE_DEBUG_SELFTEST 0x00000002u +#define KE_DEBUG_FILE_LOG 0x00000004u +#define KE_DEBUG_POOL_CANARY 0x00000008u +#define KE_DEBUG_POOL_POISON 0x00000010u +#define KE_DEBUG_POOL_QUARANTINE 0x00000020u +#define KE_DEBUG_POOL_CALLERS 0x00000040u +#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u +#define KE_DEBUG_DISPLAY_TRACE 0x00000100u +#define KE_DEBUG_INPUT_TRACE 0x00000200u +#define KE_DEBUG_VM_ASSERTS 0x00000400u +#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u +#define KE_DEBUG_LIST_CHECKS 0x00001000u +#define KE_DEBUG_AUDIO 0x00002000u +#define KE_DEBUG_AUDIO_SELFTEST 0x00004000u + // Driver-visible debug output service table passed through NTL_DDK. typedef struct NTL_DDK_DEBUG { void (*Print)(const char* fmt, ...); + uint32_t (*GetFlags)(void); } NTL_DDK_DEBUG; // Current fatal categories used by kernel-mode driver diagnostics. diff --git a/code/public/ntlite/ddk/display.h b/code/public/ntlite/ddk/display.h index 0f571b6..0c7c96a 100644 --- a/code/public/ntlite/ddk/display.h +++ b/code/public/ntlite/ddk/display.h @@ -21,6 +21,8 @@ typedef struct KE_DISPLAY_INFO { DISPLAY_MODE Mode; uint32_t Caps; uint32_t DeviceCount; + uint32_t Flags; + uint32_t Reserved; DISPLAY_STATS ManagerStats; DISPLAY_STATS DeviceStats; } KE_DISPLAY_INFO; @@ -40,10 +42,9 @@ DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode); // Queries the current display size. DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight); -typedef KSTATUS (*KE_DISPLAY_ROW_SOURCE)(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount); - DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels); -DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch); +DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); +DISPLAY_API KSTATUS KeDisplayPresentPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color); DISPLAY_API KSTATUS KeDisplayClear(uint32_t color); DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h); diff --git a/code/public/ntlite/ddk/dma.h b/code/public/ntlite/ddk/dma.h new file mode 100644 index 0000000..eb651ec --- /dev/null +++ b/code/public/ntlite/ddk/dma.h @@ -0,0 +1,146 @@ +// Public nt-lite DMA declarations. + +#pragma once + +#include +#include +#include + +struct DEVICE_OBJECT; +struct PROCESS; +typedef struct IO_DMA_ADAPTER IO_DMA_ADAPTER; +typedef struct IO_DMA_BUFFER IO_DMA_BUFFER; +typedef struct IO_DMA_MAPPING IO_DMA_MAPPING; +typedef struct MM_PINNED_MEMORY MM_PINNED_MEMORY; + +typedef enum IO_DMA_DIRECTION { + IoDmaToDevice = 1, + IoDmaFromDevice, + IoDmaBidirectional +} IO_DMA_DIRECTION; + +// Fails a streaming mapping instead of allocating a bounce buffer. +#define IO_DMA_MAP_NO_BOUNCE 0x00000001u + +// Describes immutable constraints shared by one device adapter. Address, +// alignment, and boundary limits apply to common buffers. Segment limits apply +// to streaming mappings. Zero selects the unconstrained default for each field. +typedef struct IO_DMA_REQUIREMENTS { + uint64_t MaximumDeviceAddress; + size_t Alignment; + size_t Boundary; + size_t MaximumSegmentLength; + uint32_t MaximumSegments; +} IO_DMA_REQUIREMENTS; + +// Describes one device-visible span in a streaming mapping. +typedef struct IO_DMA_SEGMENT { + uint64_t DeviceAddress; + size_t Length; +} IO_DMA_SEGMENT; + +// Reports cumulative activity and live ownership for validation. +typedef struct IO_DMA_STATS { + uint64_t AdaptersCreated; + uint64_t AdaptersDestroyed; + uint64_t AdaptersInUse; + uint64_t CommonBuffersAllocated; + uint64_t CommonBuffersFreed; + uint64_t CommonBuffersInUse; + uint64_t CommonBufferBytesInUse; + uint64_t PinnedKernelRanges; + uint64_t PinnedUserRanges; + uint64_t PinnedRangesReleased; + uint64_t PinnedRangesInUse; + uint64_t PinnedPagesInUse; + uint64_t StreamingMappings; + uint64_t StreamingMappingsUnmapped; + uint64_t StreamingMappingsInUse; + uint64_t DirectMappings; + uint64_t BounceMappings; + uint64_t BounceBytesCopiedToDevice; + uint64_t BounceBytesCopiedToCpu; + uint64_t FailedAllocations; + uint64_t FailedMappings; +} IO_DMA_STATS; + +// Driver-visible DMA services. Device addresses are intentionally distinct +// from physical addresses even while the first implementation maps them one to +// one on x86-64. +typedef struct NTL_DDK_DMA { + // Returns the one adapter and requirement set owned by a device object. + KSTATUS (*GetAdapter)(struct DEVICE_OBJECT* device, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_ADAPTER** outAdapter); + + // Releases one caller-owned adapter reference. + void (*PutAdapter)(IO_DMA_ADAPTER* adapter); + + // Allocates coherent memory with stable CPU and base device addresses. The + // caller must quiesce every device reference before releasing the buffer. + KSTATUS (*AllocateCommonBuffer)(IO_DMA_ADAPTER* adapter, + size_t size, + size_t alignment, + IO_DMA_BUFFER** outBuffer); + + // Releases a coherent common buffer. + void (*FreeCommonBuffer)(IO_DMA_BUFFER* buffer); + + // Returns the stable CPU address of a common buffer. + void* (*GetCommonBufferAddress)(const IO_DMA_BUFFER* buffer); + + // Returns the stable device address of a common buffer. + uint64_t (*GetCommonBufferDeviceAddress)(const IO_DMA_BUFFER* buffer); + + // Returns the caller-requested size of a common buffer. + size_t (*GetCommonBufferSize)(const IO_DMA_BUFFER* buffer); + + // Stabilizes a kernel range for one or more streaming mappings. Pinning + // preserves the pages, but does not transfer ownership to a device. + KSTATUS (*PinKernelMemory)(void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory); + + // Stabilizes a process-owned user range for one or more mappings. The + // descriptor retains the process and range until the final unpin. + KSTATUS (*PinUserMemory)(struct PROCESS* process, + void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory); + + // Releases one pinned-memory reference after all mappings leave. + void (*UnpinMemory)(MM_PINNED_MEMORY* memory); + + // Maps a pinned subrange and creates a constrained segment list. A new + // mapping begins CPU-owned and requires external operation serialization. + KSTATUS (*MapPinnedMemory)(IO_DMA_ADAPTER* adapter, + MM_PINNED_MEMORY* memory, + size_t offset, + size_t length, + IO_DMA_DIRECTION direction, + uint32_t flags, + IO_DMA_MAPPING** outMapping); + + // Releases a mapping and performs a final receive-side bounce copy only + // when ownership was transferred to the device and not returned to the CPU. + void (*UnmapMemory)(IO_DMA_MAPPING* mapping); + + // Transfers mapping ownership from the CPU to the device. Streaming DMA + // must not begin before this operation succeeds. + KSTATUS (*SynchronizeForDevice)(IO_DMA_MAPPING* mapping); + + // Transfers mapping ownership from the device to the CPU. CPU readers must + // not consume receive data before this operation succeeds. + KSTATUS (*SynchronizeForCpu)(IO_DMA_MAPPING* mapping); + + // Returns the immutable segment count owned by a mapping. + uint32_t (*GetSegmentCount)(const IO_DMA_MAPPING* mapping); + + // Returns the immutable segment array owned by a mapping. + const IO_DMA_SEGMENT* (*GetSegments)(const IO_DMA_MAPPING* mapping); + + // Copies current DMA counters into caller-owned storage. + void (*QueryStats)(IO_DMA_STATS* outStats); +} NTL_DDK_DMA; diff --git a/code/public/ntlite/ddk/file.h b/code/public/ntlite/ddk/file.h index ec0b760..ad27a7c 100644 --- a/code/public/ntlite/ddk/file.h +++ b/code/public/ntlite/ddk/file.h @@ -59,6 +59,7 @@ typedef struct IO_FILESYSTEM { KSTATUS (*QueryInfo)(struct IO_FILESYSTEM* fs, void* fileCtx, SYS_FILE_STAT* out); KSTATUS (*Stat)(struct IO_FILESYSTEM* fs, SYS_STRING_VIEW path, SYS_FILE_STAT* out); KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx, IO_FILE_IDENTITY* out); + KSTATUS (*Flush)(struct IO_FILESYSTEM* fs, void* fileCtx); void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx); void (*QueryStats)(struct IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out); void* DriverCtx; @@ -84,10 +85,6 @@ typedef struct FILE_OBJECT { uint64_t Size; uint64_t Offset; uint32_t Flags; - uint32_t ReadaheadScore; - uint64_t ReadaheadNextOffset; - uint64_t ReadaheadIssuedUntil; - uint64_t ReadaheadSuppressedUntil; } FILE_OBJECT; typedef struct DIR_OBJECT { diff --git a/code/public/ntlite/ddk/irp.h b/code/public/ntlite/ddk/irp.h index c6a5bbe..ece9216 100644 --- a/code/public/ntlite/ddk/irp.h +++ b/code/public/ntlite/ddk/irp.h @@ -23,11 +23,12 @@ typedef enum IO_MAJOR_FUNCTION { IRP_MJ_CLOSE = 1, IRP_MJ_READ = 2, IRP_MJ_WRITE = 3, - IRP_MJ_DEVICE_CONTROL = 4, - IRP_MJ_INTERNAL_DEVICE_CONTROL = 5, - IRP_MJ_PNP = 6, - IRP_MJ_POWER = 7, - IRP_MJ_MAXIMUM_FUNCTION = 8, + IRP_MJ_FLUSH_BUFFERS = 4, + IRP_MJ_DEVICE_CONTROL = 5, + IRP_MJ_INTERNAL_DEVICE_CONTROL = 6, + IRP_MJ_PNP = 7, + IRP_MJ_POWER = 8, + IRP_MJ_MAXIMUM_FUNCTION = 9, } IO_MAJOR_FUNCTION; // PnP minor functions. @@ -159,10 +160,10 @@ __declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp); __declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q); // Cancels every IRP in a queue and invokes a callback for each one. -__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, - KSTATUS status, - IRP_QUEUE_CANCEL_CALLBACK callback, - void* context); +__declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q, + KSTATUS status, + IRP_QUEUE_CANCEL_CALLBACK callback, + void* context); // Cancels every IRP in a queue. __declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status); diff --git a/code/public/ntlite/ddk/memory.h b/code/public/ntlite/ddk/memory.h index c9450ec..f1fe3d2 100644 --- a/code/public/ntlite/ddk/memory.h +++ b/code/public/ntlite/ddk/memory.h @@ -14,12 +14,14 @@ #define PAGE_MASK (~(PAGE_SIZE - 1ULL)) #endif -#define NTL_DDK_PAGE_SIZE PAGE_SIZE -#define NTL_DDK_PAGE_PRESENT (1ULL << 0) -#define NTL_DDK_PAGE_WRITE (1ULL << 1) -#define NTL_DDK_PAGE_WRITETHROUGH (1ULL << 3) -#define NTL_DDK_PAGE_NOCACHE (1ULL << 4) -#define NTL_DDK_PAGE_NX (1ULL << 63) +#define NTL_DDK_PAGE_SIZE PAGE_SIZE +#define NTL_DDK_PAGE_PRESENT (1ULL << 0) +#define NTL_DDK_PAGE_WRITE (1ULL << 1) +#define NTL_DDK_PAGE_CACHE_WB 0ULL +#define NTL_DDK_PAGE_CACHE_WC (1ULL << 3) +#define NTL_DDK_PAGE_CACHE_UC (1ULL << 4) +#define NTL_DDK_PAGE_CACHE_MASK (NTL_DDK_PAGE_CACHE_WC | NTL_DDK_PAGE_CACHE_UC) +#define NTL_DDK_PAGE_NX (1ULL << 63) #ifndef PAGE_PRESENT #define PAGE_PRESENT (1ULL << 0) @@ -30,11 +32,17 @@ #ifndef PAGE_USER #define PAGE_USER (1ULL << 2) #endif -#ifndef PAGE_WRITETHROUGH -#define PAGE_WRITETHROUGH (1ULL << 3) +#ifndef PAGE_CACHE_WB +#define PAGE_CACHE_WB 0ULL #endif -#ifndef PAGE_NOCACHE -#define PAGE_NOCACHE (1ULL << 4) +#ifndef PAGE_CACHE_WC +#define PAGE_CACHE_WC (1ULL << 3) +#endif +#ifndef PAGE_CACHE_UC +#define PAGE_CACHE_UC (1ULL << 4) +#endif +#ifndef PAGE_CACHE_MASK +#define PAGE_CACHE_MASK (PAGE_CACHE_WC | PAGE_CACHE_UC) #endif #ifndef PAGE_ACCESSED #define PAGE_ACCESSED (1ULL << 5) @@ -56,55 +64,6 @@ typedef enum NTL_DDK_MAP_MODE { NtlDdkMapReplaceExisting = 1, } NTL_DDK_MAP_MODE; -#ifndef NTL_MM_PTE_MAP_MODE_DEFINED -typedef enum MM_PTE_MAP_MODE { - MmPteMapFailIfPresent = 0, - MmPteMapReplaceExisting = 1, -} MM_PTE_MAP_MODE; -#define NTL_MM_PTE_MAP_MODE_DEFINED 1 -#endif - -#ifndef NTL_MM_PFN_TYPES_DEFINED -typedef enum MM_PFN_STATE { - MmPfnStateInvalid = 0, - MmPfnStateReserved, - MmPfnStateFree, - MmPfnStateZeroed, - MmPfnStateActive, - MmPfnStateStandby, - MmPfnStateModified, -} MM_PFN_STATE; - -typedef enum MM_PFN_USAGE { - MmPfnUseNone = 0, - MmPfnUseKernel, - MmPfnUsePageTable, - MmPfnUseUser, - MmPfnUseCleanFileCache, - MmPfnUseImage, - MmPfnUseMappedFile, - MmPfnUseAnonymous, - MmPfnUseCow, - MmPfnUseSurface, - MmPfnUseDriver, - MmPfnUseDma, - MmPfnUseBootloader, - MmPfnUsePfnDatabase, - MmPfnUseMmio, -} MM_PFN_USAGE; - -typedef struct MM_PFN { - uint16_t State; - uint16_t Usage; - uint16_t RefCount; - uint16_t PinCount; - uint32_t Flags; - uint32_t MappedCount; - void* Owner; - uint64_t BackingPage; -} MM_PFN; -#define NTL_MM_PFN_TYPES_DEFINED 1 -#endif #define MM_POOL_TAG(a, b, c, d) \ ((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \ @@ -131,56 +90,3 @@ __declspec(dllexport) void MmFreePool(void* ptr); // Reallocates kernel pool memory. __declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize); - -// Allocates one physical frame for a specific driver-visible usage. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage); - -// Allocates one physical frame below maxExclusive. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage); - -// Allocates one physical frame in a range. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive, - PHYSICAL_ADDRESS maxExclusive, - MM_PFN_USAGE usage); - -// Allocates contiguous physical frames. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage); - -// Releases a frame reference. -__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame); - -// Retains a physical frame. -__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame); - -// Releases a retained physical frame. -__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame); - -// Pins a physical frame. -__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame); - -// Unpins a physical frame. -__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame); - -// Returns the PFN database entry for a frame. -__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame); - -// Returns the direct-map alias for a physical frame. -__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame); - -// Maps one page into the shared kernel address space. -__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, - PHYSICAL_ADDRESS phys, - uint64_t flags, - MM_PTE_MAP_MODE mode); - -// Removes one page mapping from the shared kernel address space. -__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt); - -// Replaces flags on a mapped range. -__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags); - -// Translates a virtual address through the shared kernel page tables. -__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt); - -// Returns the physical address and flags for a mapped page. -__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut); diff --git a/code/public/ntlite/ddk/mutex.h b/code/public/ntlite/ddk/mutex.h index 1dc405b..b00628f 100644 --- a/code/public/ntlite/ddk/mutex.h +++ b/code/public/ntlite/ddk/mutex.h @@ -13,16 +13,23 @@ typedef struct MUTEX { SPINLOCK Lock; THREAD* Owner; LIST_ENTRY WaitList; + LIST_ENTRY OwnerLink; + KE_SCHEDULING_CLASS DonationClass; + uint32_t DonationPriority; + uint8_t HasDonation; + uint8_t Abandoned; } MUTEX; +// Tests whether a mutex acquisition transferred ownership to the caller. +#define KE_MUTEX_ACQUIRE_SUCCEEDED(status) \ + ((status) == NTLSTATUS_SUCCESS || (status) == NTLSTATUS_ABANDONED) + // Creates a mutex object. __declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out); -// Acquires a mutex, blocking when necessary. -__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex); - -// Attempts to acquire a mutex without blocking. -__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex); +// Acquires a mutex within the requested timeout and reports abandonment on transfer. +// Zero performs a nonblocking probe; KE_WAIT_INFINITE waits without a timeout. +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds); // Releases a mutex. __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex); diff --git a/code/public/ntlite/ddk/object.h b/code/public/ntlite/ddk/object.h index 98bf575..17a61b9 100644 --- a/code/public/ntlite/ddk/object.h +++ b/code/public/ntlite/ddk/object.h @@ -29,6 +29,8 @@ typedef enum OBJECT_TYPE { ObjectTypeTimer = 19, ObjectTypeModule = 20, ObjectTypeDisplayClock = 21, + ObjectTypeAudioEndpoint = 22, + ObjectTypeAudioStream = 23, } OBJECT_TYPE; typedef enum OBJECT_LIFETIME_STATE { diff --git a/code/public/ntlite/ddk/semaphore.h b/code/public/ntlite/ddk/semaphore.h index 74db887..35eac0b 100644 --- a/code/public/ntlite/ddk/semaphore.h +++ b/code/public/ntlite/ddk/semaphore.h @@ -19,11 +19,9 @@ typedef struct SEMAPHORE { // Creates a counting semaphore. __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE** out); -// Waits for a semaphore count to become available. -__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore); - -// Attempts to take a semaphore without blocking. -__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* semaphore); +// Waits for a semaphore count within the requested timeout. +// Zero performs a nonblocking probe; KE_WAIT_INFINITE waits without a timeout. +__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore, uint64_t timeoutMilliseconds); // Releases one semaphore count. __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* semaphore); diff --git a/code/public/ntlite/ddk/sync.h b/code/public/ntlite/ddk/sync.h index d4cb9f0..56c548d 100644 --- a/code/public/ntlite/ddk/sync.h +++ b/code/public/ntlite/ddk/sync.h @@ -14,6 +14,9 @@ typedef struct SPINLOCK { #define SPINLOCK_INIT { 0 } +// Requests a synchronization wait without a timeout. +#define KE_WAIT_INFINITE UINT64_MAX + #define KI_SPINLOCK_IRQL_SHIFT ((unsigned)(sizeof(uintptr_t) * 8u - 8u)) #define KI_SPINLOCK_FLAGS_MASK (~(((uintptr_t)0xFFu) << KI_SPINLOCK_IRQL_SHIFT)) diff --git a/code/public/ntlite/ddk/thread.h b/code/public/ntlite/ddk/thread.h index 28187e3..d0c0228 100644 --- a/code/public/ntlite/ddk/thread.h +++ b/code/public/ntlite/ddk/thread.h @@ -17,18 +17,49 @@ typedef enum THREAD_PRIORITY { #define NTL_THREAD_PRIORITY_DEFINED 1 #endif -// Creates a system-process kernel thread for a kernel-mode driver. +// Selects dynamic normal scheduling or static real-time scheduling. +typedef enum KE_SCHEDULING_CLASS { + KeSchedulingNormal = 0, + KeSchedulingRealtime = 1, + KeSchedulingClassCount = 2, +} KE_SCHEDULING_CLASS; + +// Higher real-time values outrank lower values. Real-time threads always +// outrank normal threads and round-robin with equal-priority peers. +#define KE_REALTIME_PRIORITY_MIN 0u +#define KE_REALTIME_PRIORITY_MAX 31u +#define KE_REALTIME_PRIORITY_COUNT 32u + +// Creates a thread without placing it on a scheduler ready queue. The caller +// must retain the returned THREAD reference and call KeResumeThread(). +#define KE_THREAD_CREATE_FLAG_SUSPENDED 0x00000001u + +// Describes initial scheduling, execution state, and affinity for one new +// thread. Zero affinity selects the kernel default rather than an empty CPU +// set. +typedef struct KE_THREAD_CREATE_PARAMETERS { + KE_SCHEDULING_CLASS SchedulingClass; + uint32_t Priority; + uint32_t Flags; + uint64_t AffinityMask; +} KE_THREAD_CREATE_PARAMETERS; + +#define KE_THREAD_CREATE_NORMAL(priorityValue) \ + { KeSchedulingNormal, (uint32_t)(priorityValue), 0u, 0ULL } + +#define KE_THREAD_CREATE_REALTIME(priorityValue) \ + { KeSchedulingRealtime, (uint32_t)(priorityValue), 0u, 0ULL } + +// Creates a system-process kernel thread with explicit scheduling parameters. __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out); -// Creates a system-process kernel thread with an explicit affinity mask. -__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, - uintptr_t arg, - THREAD_PRIORITY priority, - uint64_t affinityMask, - THREAD** out); +// Decrements one thread's suspend count and queues it when the count reaches +// zero. previousSuspendCountOut may be NULL. +__declspec(dllexport) KSTATUS KeResumeThread(THREAD* thread, + uint32_t* previousSuspendCountOut); // Yields the current processor to another runnable thread. __declspec(dllexport) void KeYield(void); diff --git a/code/public/ntlite/display.h b/code/public/ntlite/display.h index c978269..9bc9f66 100644 --- a/code/public/ntlite/display.h +++ b/code/public/ntlite/display.h @@ -77,6 +77,8 @@ typedef struct DISPLAY_STATS { uint64_t ReadRectCount; uint64_t PresentCount; uint64_t PresentRectCount; + uint64_t DirectPresentCount; + uint64_t ScanoutBytes; uint64_t DirtyRectCount; uint64_t DirtyAllCount; uint64_t BytesCopied; @@ -98,11 +100,15 @@ typedef struct DISPLAY_STATS { #define SYS_DISPLAY_NAME_MAX 32u +#define DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE 0x00000001u + typedef struct SYS_DISPLAY_INFO { char PrimaryName[SYS_DISPLAY_NAME_MAX]; DISPLAY_MODE Mode; uint32_t Caps; uint32_t DeviceCount; + uint32_t Flags; + uint32_t Reserved; DISPLAY_STATS ManagerStats; DISPLAY_STATS DeviceStats; } SYS_DISPLAY_INFO; diff --git a/code/public/ntlite/status.h b/code/public/ntlite/status.h index fba063f..5362e37 100644 --- a/code/public/ntlite/status.h +++ b/code/public/ntlite/status.h @@ -36,6 +36,7 @@ typedef uint32_t KSTATUS; #define NTLSTATUS_ALERTED KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000003u) #define NTLSTATUS_KERNEL_APC KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000004u) #define NTLSTATUS_USER_APC KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000005u) +#define NTLSTATUS_ABANDONED KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000006u) // System-facility failures. // NTLSTATUS_WOULD_BLOCK is the canonical result for immediate nonblocking operations that cannot make progress. @@ -59,6 +60,8 @@ typedef uint32_t KSTATUS; #define NTLSTATUS_NO_MORE_FILES KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000011u) #define NTLSTATUS_INVALID_IRQL KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000012u) #define NTLSTATUS_WOULD_BLOCK KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000013u) +#define NTLSTATUS_DEADLOCK KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000014u) +#define NTLSTATUS_DEVICE_REMOVED KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000015u) // Process termination and fault statuses use the process facility. #define NTLSTATUS_PROCESS_BASE KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_PROCESS, 0x000000u) diff --git a/code/public/ntlite/syscall.h b/code/public/ntlite/syscall.h index 2b8961c..f181621 100644 --- a/code/public/ntlite/syscall.h +++ b/code/public/ntlite/syscall.h @@ -12,6 +12,7 @@ #include #include #include +#include // Syscall ABI convention: status-returning calls use KSTATUS. New or revised // operations use a request struct when they need five or more logical arguments @@ -494,6 +495,9 @@ typedef struct SYS_CREATE_THREAD_REQUEST { uint32_t* ThreadIdOut; } SYS_CREATE_THREAD_REQUEST; +#define SYS_CREATE_THREAD_FLAG_NONE 0x00000000u +#define SYS_CREATE_THREAD_FLAG_SUSPENDED 0x00000001u + void SysExitProcess(int32_t exitCode); void SysExitThread(int32_t exitCode); KSTATUS SysCreateProcess(const SYS_CREATE_PROCESS_REQUEST* request); @@ -533,6 +537,10 @@ KSTATUS SysKillProcess(uint32_t processHandle, int32_t exitCode); #define SYS_THREAD_STATE_RUNNING 1u #define SYS_THREAD_STATE_WAITING 2u #define SYS_THREAD_STATE_DEAD 3u +#define SYS_THREAD_STATE_SUSPENDED 4u + +#define SYS_SCHEDULING_NORMAL 0u +#define SYS_SCHEDULING_REALTIME 1u #define SYS_THREAD_FLAG_CURRENT 0x00000001u #define SYS_THREAD_FLAG_IDLE 0x00000002u @@ -569,7 +577,11 @@ typedef struct SYS_THREAD_INFO { uint32_t ThreadId; uint32_t State; - uint32_t Priority; + uint32_t BaseSchedulingClass; + uint32_t BasePriority; + uint32_t EffectiveSchedulingClass; + uint32_t EffectivePriority; + uint32_t Flags; uint32_t RunningCpu; uint32_t LastCpu; @@ -674,6 +686,7 @@ KSTATUS SysGetModuleExport(const SYS_GET_MODULE_EXPORT_REQUEST* request); KSTATUS SysGetModuleList(SYS_MODULE_INFO* modulesOut, size_t maxCount, size_t* countOut); KSTATUS SysCreateThread(const SYS_CREATE_THREAD_REQUEST* request); +KSTATUS SysResumeThread(uint32_t threadHandle, uint32_t* previousSuspendCountOut); uint32_t SysGetCurrentTid(void); uint32_t SysGetCurrentPid(void); KSTATUS SysSetThreadPriority(uint32_t priority); @@ -771,7 +784,8 @@ KSTATUS SysCloseHandle(uint32_t handle); #define SYS_HANDLE_RIGHT_QUERY 0x00000020u #define SYS_HANDLE_RIGHT_TERMINATE 0x00000040u #define SYS_HANDLE_RIGHT_LOCK 0x00000080u -#define SYS_HANDLE_RIGHT_ALL 0x000000FFu +#define SYS_HANDLE_RIGHT_RESUME 0x00000100u +#define SYS_HANDLE_RIGHT_ALL 0x000001FFu #define SYS_DUPLICATE_HANDLE_FLAG_NONE 0x00000000u #define SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS 0x00000001u @@ -1080,7 +1094,7 @@ KSTATUS SysRecvSocket(uint32_t socket, void* buffer, size_t length, uint32_t fla KSTATUS SysShutdownSocket(uint32_t socket, uint32_t how); KSTATUS SysSendToSocket(uint32_t socket, const SYS_SOCKET_SEND_TO_REQUEST* request); KSTATUS SysRecvFromSocket(uint32_t socket, const SYS_SOCKET_RECV_FROM_REQUEST* request); -KSTATUS SysResolveHost(const SYS_RESOLVE_HOST_REQUEST* request); // Cry some more, microkernel purists! +KSTATUS SysResolveHost(const SYS_RESOLVE_HOST_REQUEST* request); // Cry about it. KSTATUS SysSetSocketOption(uint32_t socket, const SYS_SOCKET_OPTION_REQUEST* request); KSTATUS SysGetSocketOption(uint32_t socket, const SYS_SOCKET_OPTION_REQUEST* request); KSTATUS SysPoll(const SYS_POLL_REQUEST* request); @@ -1147,6 +1161,12 @@ typedef struct SYS_SCHEDULER_STATS { uint64_t CpuBurstDemotions; uint64_t SmtSiblingDeferrals; uint64_t SmtPrimaryPokes; + uint64_t RealtimeDispatches; + uint64_t RealtimePreemptions; + uint64_t PriorityInheritanceBoosts; + uint64_t PriorityInheritanceDrops; + uint64_t PriorityInheritancePropagations; + uint64_t PriorityInheritanceMaxDepth; } SYS_SCHEDULER_STATS; typedef struct SYS_PMM_STATS { @@ -1310,19 +1330,6 @@ typedef struct SYS_FILE_CACHE_STATS { uint64_t DropFailures; uint64_t StandbyTransitions; uint64_t StandbyHits; - uint64_t ReadAheadRequests; - uint64_t ReadAheadPagesRequested; - uint64_t ReadAheadPagesQueued; - uint64_t ReadAheadPagesLoaded; - uint64_t ReadAheadPagesSkipped; - uint64_t ReadAheadFailures; - uint64_t ReadAheadPagesThrottled; - uint64_t MaxReadAheadPagesPerRequest; - uint64_t BatchReads; - uint64_t BatchPagesRequested; - uint64_t BatchPagesLoaded; - uint64_t MaxBatchPagesRequested; - uint64_t MaxBatchPagesLoaded; uint64_t ResidentPages; uint64_t ActivePages; uint64_t StandbyPages; @@ -1360,17 +1367,6 @@ typedef struct SYS_DISK_STATS { uint64_t WriteSectors; uint64_t MaxReadSectors; uint64_t MaxWriteSectors; - uint64_t ReadaheadHits; - uint64_t ReadaheadMisses; - uint64_t ReadaheadStores; - uint64_t ReadaheadInvalidations; - uint64_t ReadaheadBytesServed; - uint64_t ReadaheadSectorsLoaded; - uint64_t ReadaheadSequentialDetections; - uint64_t ReadaheadSuppressed; - uint64_t ReadaheadAllocFailures; - uint64_t ReadaheadReadFailures; - uint64_t MaxReadaheadSectors; } SYS_DISK_STATS; #define SYS_NET_IRP_OPERATION_COUNT 16u diff --git a/code/user/apps/doom/PureDOOM.h b/code/user/apps/doom/PureDOOM.h index 7275eb5..262750d 100644 --- a/code/user/apps/doom/PureDOOM.h +++ b/code/user/apps/doom/PureDOOM.h @@ -7549,15 +7549,25 @@ void doom_init(int argc, const char** argv, int flags) if (!doom_exit) doom_exit = doom_exit_impl; if (!doom_getenv) doom_getenv = doom_getenv_impl; + doom_print("doom_init: allocating screen buffer\n"); screen_buffer = (unsigned char*)doom_malloc(SCREENWIDTH * SCREENHEIGHT); + doom_print("doom_init: screen buffer ready\n"); + + doom_print("doom_init: allocating final screen buffer\n"); final_screen_buffer = (unsigned char*)doom_malloc(SCREENWIDTH * SCREENHEIGHT * 4); + doom_print("doom_init: final screen buffer ready\n"); + + doom_print("doom_init: reading initial time\n"); last_update_time = I_GetTime(); + doom_print("doom_init: initial time ready\n"); myargc = argc; myargv = (char**)argv; // unsafe my ass doom_flags = flags; + doom_print("doom_init: entering D_DoomMain\n"); D_DoomMain(); + doom_print("doom_init: D_DoomMain returned\n"); } @@ -9790,7 +9800,27 @@ void FindResponseFile(void) { int i; + doom_print("FindResponseFile: entered\n"); + if (myargc <= 0) + { + doom_print("FindResponseFile: argc <= 0\n"); + return; + } + if (myargc == 1) + { + doom_print("FindResponseFile: argc == 1, returning\n"); + return; + } + doom_print("FindResponseFile: argc > 1\n"); + for (i = 1; i < myargc; i++) + { + doom_print("FindResponseFile: checking argument\n"); + if (!myargv || !myargv[i]) + { + doom_print("FindResponseFile: invalid argv entry\n"); + return; + } if (myargv[i][0] == '@') { void* handle = 0; @@ -9860,6 +9890,9 @@ void FindResponseFile(void) break; } + } + + doom_print("FindResponseFile: scan complete\n"); } @@ -9871,9 +9904,13 @@ void D_DoomMain(void) int p; char file[256]; + doom_print("D_DoomMain: FindResponseFile\n"); FindResponseFile(); + doom_print("D_DoomMain: FindResponseFile done\n"); + doom_print("D_DoomMain: IdentifyVersion\n"); IdentifyVersion(); + doom_print("D_DoomMain: IdentifyVersion done\n"); modifiedgame = doom_false; @@ -23741,19 +23778,67 @@ void M_SaveDefaults(void) // // M_LoadDefaults // +// Reads one complete defaults line without allowing malformed files to escape the caller's buffer. +static int M_ReadDefaultsLine(void* handle, char* line, int capacity) +{ + int length = 0; + doom_boolean overflow = doom_false; + + if (!handle || !line || capacity < 2) + return 0; + + for (;;) + { + char c = 0; + int read = doom_read(handle, &c, 1); + if (read != 1) + { + if (!length && !overflow) + return 0; + break; + } + + if (c == '\n') + break; + if (c == '\r') + continue; + + if (length + 1 < capacity) + line[length++] = c; + else + overflow = doom_true; + } + + line[length] = 0; + return overflow ? -1 : 1; +} + + +// Returns the first non-whitespace byte in a mutable defaults token. +static char* M_SkipDefaultWhitespace(char* text) +{ + while (*text == ' ' || *text == '\t') + text++; + return text; +} + + +// Removes horizontal whitespace from the end of a mutable defaults token. +static void M_TrimDefaultWhitespace(char* text) +{ + int length = (int)doom_strlen(text); + while (length > 0 && (text[length - 1] == ' ' || text[length - 1] == '\t')) + text[--length] = 0; +} + + +// Loads bounded name/value records from the optional defaults file. void M_LoadDefaults(void) { + char line[384]; int i; - int len; - void* f; - char def[80]; - char strparm[100]; - char* newstring; - int parm; - doom_boolean isstring; - // set everything to base values - // numdefaults = sizeof(defaults)/sizeof(defaults[0]); + // Set everything to base values. for (i = 0; i < numdefaults; i++) { if (defaults[i].defaultvalue == 0xFFFF) @@ -23762,104 +23847,80 @@ void M_LoadDefaults(void) *defaults[i].location = (int)defaults[i].defaultvalue; } - // check for a custom default file + // Check for a custom default file. i = M_CheckParm((char*)"-config"); if (i && i < myargc - 1) { defaultfile = myargv[i + 1]; - //doom_print(" default file: %s\n", defaultfile); doom_print(" default file: "); doom_print(defaultfile); doom_print("\n"); } else - defaultfile = basedefault; - - // read the file in, overriding any set defaults - f = doom_open(defaultfile, "r"); - if (f) { - while (!doom_eof(f)) + defaultfile = basedefault; + } + + void* f = doom_open(defaultfile, "r"); + if (!f) + return; + + for (;;) + { + int lineStatus = M_ReadDefaultsLine(f, line, (int)sizeof(line)); + if (!lineStatus) + break; + if (lineStatus < 0) { - // def - int arg_read = 0; - char c; - for (i = 0; i < 79; ++i) - { - doom_read(f, &c, 1); - if (c == ' ' || c == '\n' || c == '\t') - { - if (i > 0) arg_read++; - break; - } - def[i] = c; - } - def[i] = '\0'; - - // Ignore spaces - if (c != '\n') - { - while (1) - { - doom_read(f, &c, 1); - if (c != ' ' && c != '\t') break; - } - - // strparam - i = 0; - if (c != '\n') - { - for (; i < 260;) - { - strparm[i++] = c; - doom_read(f, &c, 1); - if (c == '\n') - { - if (i > 0) arg_read++; - break; - } - } - } - strparm[i] = '\0'; - } - - isstring = doom_false; - //if (fscanf(f, "%79s %[^\n]\n", def, strparm) == 2) - if (arg_read == 2) - { - if (strparm[0] == '"') - { - // get a string default - isstring = doom_true; - len = (int)doom_strlen(strparm); - newstring = (char*)doom_malloc(len); - strparm[len - 1] = 0; - doom_strcpy(newstring, strparm + 1); - } - else if (strparm[0] == '0' && strparm[1] == 'x') - { - //sscanf(strparm + 2, "%x", &parm); - parm = doom_atox(strparm + 2); - } - else - { - //sscanf(strparm, "%i", &parm); - parm = doom_atoi(strparm); - } - for (i = 0; i < numdefaults; i++) - if (!doom_strcmp(def, defaults[i].name)) - { - if (!isstring) - *defaults[i].location = parm; - else - *defaults[i].text_location = newstring; - break; - } - } + doom_print("M_LoadDefaults: ignored an overlong defaults line.\n"); + continue; } - doom_close(f); + char* name = M_SkipDefaultWhitespace(line); + if (!*name || *name == '#') + continue; + + char* value = name; + while (*value && *value != ' ' && *value != '\t') + value++; + if (!*value) + continue; + *value++ = 0; + value = M_SkipDefaultWhitespace(value); + M_TrimDefaultWhitespace(value); + if (!*value) + continue; + + default_t* setting = get_default(name); + if (!setting) + continue; + + if (value[0] == '"') + { + int length = (int)doom_strlen(value); + if (length < 2 || value[length - 1] != '"' || !setting->text_location) + continue; + + int textLength = length - 2; + char* newstring = (char*)doom_malloc(textLength + 1); + if (!newstring) + continue; + doom_memcpy(newstring, value + 1, textLength); + newstring[textLength] = 0; + *setting->text_location = newstring; + } + else if (setting->location) + { + int parsed; + if (value[0] == '0' && value[1] == 'x') + parsed = doom_atox(value + 2); + else + parsed = doom_atoi(value); + *setting->location = parsed; + } } + + doom_close(f); } @@ -37068,10 +37129,6 @@ void R_InitTextures(void) int* directory; - int temp1; - int temp2; - int temp3; - // Load the patch names from pnames.lmp. name[8] = 0; names = (char*)W_CacheLumpName((char*)"PNAMES", PU_STATIC); @@ -37118,23 +37175,10 @@ void R_InitTextures(void) totalwidth = 0; - // Really complex printing shit... - temp1 = W_GetNumForName((char*)"S_START"); // P_??????? - temp2 = W_GetNumForName((char*)"S_END") - 1; - temp3 = ((temp2 - temp1 + 63) / 64) + ((numtextures + 63) / 64); - doom_print("["); - for (i = 0; i < temp3; i++) - doom_print(" "); - doom_print(" ]"); - for (i = 0; i < temp3; i++) - doom_print("\x8"); - doom_print("\x8\x8\x8\x8\x8\x8\x8\x8\x8\x8"); + // Cursor-control progress output can block simple platform consoles. for (i = 0; i < numtextures; i++, directory++) { - if (!(i & 63)) - doom_print("."); - if (i == numtextures1) { // Start looking in second texture file. @@ -37285,14 +37329,14 @@ void R_InitColormaps(void) // void R_InitData(void) { - R_InitTextures(); doom_print("\nInitTextures"); - R_InitFlats(); + R_InitTextures(); doom_print("\nInitFlats"); - R_InitSpriteLumps(); + R_InitFlats(); doom_print("\nInitSprites"); - R_InitColormaps(); + R_InitSpriteLumps(); doom_print("\nInitColormaps"); + R_InitColormaps(); } diff --git a/code/user/apps/doom/doom.c b/code/user/apps/doom/doom.c index c71bc39..07924ea 100644 --- a/code/user/apps/doom/doom.c +++ b/code/user/apps/doom/doom.c @@ -1,6 +1,6 @@ // doom.c — PureDOOM port // Native personality carrying all the weight. -// Drop doom1.wad (or doom.wad) next to this binary on your FAT32 volume. +// Place the Doom WAD and optional doom.sf2 in the process working directory. #include #include @@ -10,6 +10,8 @@ #include #include +#include "doom_audio.h" + #define DOOM_IMPLEMENTATION #include "PureDOOM.h" @@ -189,6 +191,7 @@ static void doom_gettime_wrapper(int* sec, int* usec) // --------------------------------------------------------------------------- static void doom_exit_wrapper(int code) { + DoomAudioAbort(); SysExitProcess(code); } @@ -445,7 +448,7 @@ static int doom_now_tic(void) // Waits until Doom has a new simulation tic to render. static int wait_for_next_doom_tic(int lastTic) { - for (;;) { + while (true) { int now = doom_now_tic(); if (now != lastTic) return now; @@ -495,9 +498,18 @@ int main(void) char argvStorage[512]; const char* argv[32]; int argc = build_argv_from_command_line(cmdline, argvStorage, sizeof(argvStorage), argv, 32); + if (argc == 1) + doom_print_wrapper("doom: parsed argc == 1\n"); + else if (argc > 1) + doom_print_wrapper("doom: parsed argc > 1\n"); + else + doom_print_wrapper("doom: parsed argc <= 0\n"); init_surface(); init_display_clock(); + do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: starting doom\n") }; SysWriteConsole(&req); } while (0); doom_init(argc, argv, 0); + do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: starting audio\n") }; SysWriteConsole(&req); } while (0); + DoomAudioStart(); do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: entering game loop\n") }; SysWriteConsole(&req); } while (0); @@ -508,9 +520,12 @@ int main(void) int frameTic = wait_for_next_doom_tic(lastFrameTic); lastFrameTic = frameTic; + DoomAudioLockCore(); poll_input(); doom_update(); - render_frame((const uint8_t*)doom_get_framebuffer(4 /* RGBA */)); + const uint8_t* framebuffer = (const uint8_t*)doom_get_framebuffer(4 /* RGBA */); + DoomAudioUnlockCore(); + render_frame(framebuffer); } SysExitProcess(0); diff --git a/code/user/apps/doom/doom_audio.c b/code/user/apps/doom/doom_audio.c new file mode 100644 index 0000000..b9e8932 --- /dev/null +++ b/code/user/apps/doom/doom_audio.c @@ -0,0 +1,802 @@ +// Doom audio mixes PureDOOM SFX with optional SoundFont MIDI into one native render stream. + +#include +#include +#include +#include +#include +#include + +#include "doom_audio.h" +#include "PureDOOM.h" + +#define DOOM_AUDIO_SAMPLE_RATE DOOM_SAMPLERATE +#define DOOM_AUDIO_CHANNELS 2u +#define DOOM_AUDIO_BLOCK_FRAMES 512u +#define DOOM_AUDIO_WRITE_QUANTUM_FRAMES 256u +#define DOOM_AUDIO_PREFILL_FRAMES DOOM_AUDIO_BLOCK_FRAMES +#define DOOM_AUDIO_REFILL_WATERMARK_FRAMES DOOM_AUDIO_WRITE_QUANTUM_FRAMES +#define DOOM_AUDIO_MIDI_BATCH_MESSAGES 256u +#define DOOM_AUDIO_SOUNDFONT_PATH "doom.sf2" +#define DOOM_AUDIO_MAX_SOUNDFONT_BYTES (64u * 1024u * 1024u) +#define DOOM_AUDIO_WAIT_MILLISECONDS 100u +#define DOOM_AUDIO_WORKER_PRIORITY 2u +#define DOOM_AUDIO_SYNTH_LOADER_PRIORITY 0u +#define DOOM_AUDIO_MAX_VOICES 128 +#define DOOM_AUDIO_MUSIC_GAIN 1.0f + +// Computes a natural logarithm for the positive finite values used by TinySoundFont. +static double DoomAudioLog(double value) { + union { + double Value; + uint64_t Bits; + } bits; + const double ln2 = 0.69314718055994530942; + int exponent; + double mantissa; + double ratio; + double ratio2; + double term; + double sum; + + if (value <= 0.0) + return -1.0e300; + + bits.Value = value; + exponent = (int)((bits.Bits >> 52) & 0x7FFu) - 1023; + bits.Bits = (bits.Bits & UINT64_C(0x000FFFFFFFFFFFFF)) | + UINT64_C(0x3FF0000000000000); + mantissa = bits.Value; + ratio = (mantissa - 1.0) / (mantissa + 1.0); + ratio2 = ratio * ratio; + term = ratio; + sum = term; + term *= ratio2; + sum += term / 3.0; + term *= ratio2; + sum += term / 5.0; + term *= ratio2; + sum += term / 7.0; + term *= ratio2; + sum += term / 9.0; + term *= ratio2; + sum += term / 11.0; + term *= ratio2; + sum += term / 13.0; + return (double)exponent * ln2 + 2.0 * sum; +} + +// Scales a normal double by an integral power of two without a runtime math library. +static double DoomAudioScalePowerOfTwo(double value, int exponent) { + union { + double Value; + uint64_t Bits; + } scale; + + if (exponent > 1023) + return 1.0e300; + if (exponent < -1022) + return 0.0; + + scale.Bits = (uint64_t)(exponent + 1023) << 52; + return value * scale.Value; +} + +// Computes an exponential over the bounded ranges used by TinySoundFont envelopes. +static double DoomAudioExp(double value) { + const double ln2 = 0.69314718055994530942; + const double inverseLn2 = 1.4426950408889634074; + int exponent; + double reduced; + double result; + + if (value >= 700.0) + return 1.0e300; + if (value <= -700.0) + return 0.0; + + exponent = (int)(value * inverseLn2 + (value >= 0.0 ? 0.5 : -0.5)); + reduced = value - (double)exponent * ln2; + result = 1.0 + reduced * + (1.0 + reduced * + (0.5 + reduced * + (0.16666666666666666667 + reduced * + (0.04166666666666666667 + reduced * + (0.00833333333333333333 + reduced * + (0.00138888888888888889 + reduced * + 0.00019841269841269841)))))); + return DoomAudioScalePowerOfTwo(result, exponent); +} + +// Computes a positive real power for TinySoundFont frequency and gain conversion. +static double DoomAudioPow(double base, double exponent) { + if (base <= 0.0) + return 0.0; + return DoomAudioExp(exponent * DoomAudioLog(base)); +} + +// Computes the float exponential required by TinySoundFont envelopes. +static float DoomAudioExpf(float value) { + return (float)DoomAudioExp((double)value); +} + +// Computes the float power required by TinySoundFont frequency and gain conversion. +static float DoomAudioPowf(float base, float exponent) { + return (float)DoomAudioPow((double)base, (double)exponent); +} + +// Computes base-ten logarithms from the shared natural logarithm approximation. +static double DoomAudioLog10(double value) { + return DoomAudioLog(value) * 0.43429448190325182765; +} + +// Computes square roots with a bit-derived seed and stable Newton refinement. +static double DoomAudioSqrt(double value) { + union { + double Value; + uint64_t Bits; + } estimate; + + if (value <= 0.0) + return 0.0; + + estimate.Value = value; + estimate.Bits = (estimate.Bits >> 1) + UINT64_C(0x1FF8000000000000); + for (int iteration = 0; iteration < 6; iteration++) + estimate.Value = 0.5 * (estimate.Value + value / estimate.Value); + return estimate.Value; +} + +// Computes the float square root required by TinySoundFont panning. +static float DoomAudioSqrtf(float value) { + return (float)DoomAudioSqrt((double)value); +} + +// Computes tangent accurately over TinySoundFont's normalized low-pass range. +static double DoomAudioTan(double value) { + double squared = value * value; + double sine = value * + (1.0 + squared * + (-0.16666666666666666667 + squared * + (0.00833333333333333333 + squared * + (-0.00019841269841269841 + squared * + (0.00000275573192239859 + squared * + (-0.00000002505210838544 + squared * + 0.00000000016059043837)))))); + double cosine = 1.0 + squared * + (-0.5 + squared * + (0.04166666666666666667 + squared * + (-0.00138888888888888889 + squared * + (0.00002480158730158730 + squared * + (-0.00000027557319223986 + squared * + (0.00000000208767569879 + squared * + -0.00000000001147074560)))))); + + if (cosine > -1.0e-12 && cosine < 1.0e-12) + cosine = cosine < 0.0 ? -1.0e-12 : 1.0e-12; + return sine / cosine; +} + +#pragma warning(push) +#pragma warning(disable: 4244 4267 4706) +#define TSF_NO_STDIO +#define TSF_MALLOC malloc +#define TSF_REALLOC realloc +#define TSF_FREE free +#define TSF_MEMCPY memcpy +#define TSF_MEMSET memset +#define TSF_POW DoomAudioPow +#define TSF_POWF DoomAudioPowf +#define TSF_EXPF DoomAudioExpf +#define TSF_LOG DoomAudioLog +#define TSF_TAN DoomAudioTan +#define TSF_LOG10 DoomAudioLog10 +#define TSF_SQRT DoomAudioSqrt +#define TSF_SQRTF DoomAudioSqrtf +#define TSF_IMPLEMENTATION +#include "third_party/tsf.h" +#pragma warning(pop) + +typedef struct DOOM_AUDIO_STATE { + uint32_t CoreMutex; + uint32_t StreamHandle; + uint32_t WorkerHandle; + uint32_t WorkerId; + uint32_t SynthLoaderHandle; + uint32_t SynthLoaderId; + volatile long StopRequested; + tsf* volatile Synth; +} DOOM_AUDIO_STATE; + +static DOOM_AUDIO_STATE g_doomAudio; + +// Writes one complete diagnostic line to the process console. +static void DoomAudioPrint(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Opens the default available native render endpoint. +static KSTATUS DoomAudioFindEndpoint(uint64_t* endpointIdOut) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + uint32_t count = 0; + const SYS_AUDIO_ENDPOINT_INFO* fallback = NULL; + KSTATUS status; + + if (!endpointIdOut) + return NTLSTATUS_INVALID_PARAMETER; + *endpointIdOut = 0; + + memset(endpoints, 0, sizeof(endpoints)); + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + request.CountOut = &count; + status = SysAudioEnumerateEndpoints(&request); + if (status != NTLSTATUS_SUCCESS) + return status; + if (count > SYS_AUDIO_MAX_ENDPOINTS) + count = SYS_AUDIO_MAX_ENDPOINTS; + + for (uint32_t index = 0; index < count; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) + fallback = endpoint; + if (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT) { + *endpointIdOut = endpoint->EndpointId; + return NTLSTATUS_SUCCESS; + } + } + + if (!fallback) + return NTLSTATUS_NOT_FOUND; + *endpointIdOut = fallback->EndpointId; + return NTLSTATUS_SUCCESS; +} + +// Returns the currently published synthesizer without taking the PureDOOM lock. +static tsf* DoomAudioGetSynth(void) { + return (tsf*)_InterlockedCompareExchangePointer( + (void* volatile*)&g_doomAudio.Synth, NULL, NULL); +} + +// Maps the optional SoundFont so TinySoundFont can parse it without a separate input copy. +static KSTATUS DoomAudioMapSoundFont(void** dataOut, uint32_t* sizeOut) { + uint32_t fileHandle = 0; + uint32_t mappingHandle = 0; + uint64_t fileSize = 0; + void* base = NULL; + KSTATUS status; + + if (!dataOut || !sizeOut) + return NTLSTATUS_INVALID_PARAMETER; + *dataOut = NULL; + *sizeOut = 0; + + SYS_OPEN_FILE_REQUEST open = SysMakeOpenFileRequest( + SysSvFromCstr(DOOM_AUDIO_SOUNDFONT_PATH), + SYS_FILE_ACCESS_QUERY | SYS_FILE_ACCESS_READ, + SYS_FILE_OPEN_EXISTING, + SYS_FILE_FLAG_NONE, + &fileHandle); + status = SysOpenFile(&open); + if (status != NTLSTATUS_SUCCESS) + return status; + + status = SysGetFileSize(fileHandle, &fileSize); + if (status != NTLSTATUS_SUCCESS || fileSize == 0 || + fileSize > DOOM_AUDIO_MAX_SOUNDFONT_BYTES || + fileSize > INT32_MAX) { + (void)SysCloseHandle(fileHandle); + return status == NTLSTATUS_SUCCESS ? NTLSTATUS_INVALID_PARAMETER : status; + } + + status = SysCreateFileMapping(fileHandle, SYS_VM_READ, SYS_MAP_FLAG_NONE, + fileSize, &mappingHandle); + if (status != NTLSTATUS_SUCCESS || !mappingHandle) { + (void)SysCloseHandle(fileHandle); + return status == NTLSTATUS_SUCCESS ? NTLSTATUS_FAILURE : status; + } + + status = SysMapFile(mappingHandle, 0, (size_t)fileSize, + SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &base); + (void)SysCloseHandle(mappingHandle); + (void)SysCloseHandle(fileHandle); + if (status != NTLSTATUS_SUCCESS || !base) + return status == NTLSTATUS_SUCCESS ? NTLSTATUS_FAILURE : status; + + *dataOut = base; + *sizeOut = (uint32_t)fileSize; + return NTLSTATUS_SUCCESS; +} + +// Configures one parsed General MIDI synthesizer for PureDOOM's native sample rate. +static KSTATUS DoomAudioConfigureSynth(tsf* synth) { + if (!synth) + return NTLSTATUS_INVALID_PARAMETER; + + tsf_set_output(synth, TSF_STEREO_INTERLEAVED, + (int)DOOM_AUDIO_SAMPLE_RATE, 0.0f); + if (!tsf_set_max_voices(synth, DOOM_AUDIO_MAX_VOICES)) + return NTLSTATUS_NO_MEMORY; + + for (int channel = 0; channel < 16; channel++) { + if (!tsf_channel_set_presetnumber(synth, channel, 0, channel == 9)) + return NTLSTATUS_FAILURE; + } + return NTLSTATUS_SUCCESS; +} + +// Loads optional MIDI state at low priority without blocking Doom or SFX startup. +static void DoomAudioSynthLoader(uintptr_t argument) { + uint32_t soundFontSize = 0; + void* soundFontBase = NULL; + tsf* synth = NULL; + KSTATUS status; + (void)argument; + + status = DoomAudioMapSoundFont(&soundFontBase, &soundFontSize); + if (status == NTLSTATUS_NOT_FOUND) { + DoomAudioPrint("doom: doom.sf2 unavailable; continuing with sound effects only\n"); + SysExitThread(0); + } + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: doom.sf2 could not be mapped; continuing with sound effects only\n"); + SysExitThread(0); + } + + if (_InterlockedCompareExchange(&g_doomAudio.StopRequested, 0, 0)) { + (void)SysFreeVm(soundFontBase); + SysExitThread(0); + } + + synth = tsf_load_memory(soundFontBase, (int)soundFontSize); + (void)SysFreeVm(soundFontBase); + if (!synth) { + DoomAudioPrint("doom: doom.sf2 is invalid or synth allocation failed; continuing with sound effects only\n"); + SysExitThread(0); + } + + status = DoomAudioConfigureSynth(synth); + if (status != NTLSTATUS_SUCCESS || + _InterlockedCompareExchange(&g_doomAudio.StopRequested, 0, 0)) { + if (status != NTLSTATUS_SUCCESS) + DoomAudioPrint("doom: MIDI synthesizer initialization failed; continuing with sound effects only\n"); + tsf_close(synth); + SysExitThread(0); + } + + (void)_InterlockedExchangePointer( + (void* volatile*)&g_doomAudio.Synth, synth); + DoomAudioPrint("doom: MIDI synthesizer ready\n"); + SysExitThread(0); +} + +// Applies one packed WinMM-style short MIDI message to TinySoundFont. +static void DoomAudioDispatchMidi(tsf* synth, uint32_t message) { + uint8_t status = (uint8_t)(message & 0xFFu); + uint8_t data1 = (uint8_t)((message >> 8) & 0x7Fu); + uint8_t data2 = (uint8_t)((message >> 16) & 0x7Fu); + int channel = status & 0x0F; + + if (!synth) + return; + + switch (status & 0xF0) { + case 0x80: + tsf_channel_note_off(synth, channel, data1); + break; + case 0x90: + if (data2) + (void)tsf_channel_note_on(synth, channel, data1, + (float)data2 / 127.0f); + else + tsf_channel_note_off(synth, channel, data1); + break; + case 0xB0: + (void)tsf_channel_midi_control(synth, channel, data1, data2); + break; + case 0xC0: + (void)tsf_channel_set_presetnumber(synth, channel, data1, + channel == 9); + break; + case 0xE0: + (void)tsf_channel_set_pitchwheel(synth, channel, + (int)data1 | ((int)data2 << 7)); + break; + default: + break; + } +} + +// Advances one PureDOOM MIDI tick and dispatches every event outside the core lock. +static void DoomAudioTickMidi(tsf* synth) { + uint32_t messages[DOOM_AUDIO_MIDI_BATCH_MESSAGES]; + int tickComplete = 0; + + if (!synth) + return; + + while (!tickComplete) { + uint32_t count = 0; + + DoomAudioLockCore(); + while (count < DOOM_AUDIO_MIDI_BATCH_MESSAGES) { + uint32_t message = (uint32_t)doom_tick_midi(); + if (!message) { + tickComplete = 1; + break; + } + messages[count++] = message; + } + DoomAudioUnlockCore(); + + for (uint32_t index = 0; index < count; index++) + DoomAudioDispatchMidi(synth, messages[index]); + } +} + +// Copies one complete PureDOOM SFX block while holding the shared core lock. +static void DoomAudioReadSfxBlock(int16_t* frames) { + short* source; + + DoomAudioLockCore(); + source = doom_get_sound_buffer(); + memcpy(frames, source, + DOOM_AUDIO_BLOCK_FRAMES * DOOM_AUDIO_CHANNELS * sizeof(int16_t)); + DoomAudioUnlockCore(); +} + +// Mixes rendered floating-point music into signed 16-bit PureDOOM SFX. +static void DoomAudioMixMusic(int16_t* destination, + const float* music, + uint32_t frameOffset, + uint32_t frameCount) { + uint32_t firstSample = frameOffset * DOOM_AUDIO_CHANNELS; + uint32_t sampleCount = frameCount * DOOM_AUDIO_CHANNELS; + + for (uint32_t index = 0; index < sampleCount; index++) { + float scaled = music[index] * (32767.0f * DOOM_AUDIO_MUSIC_GAIN); + int32_t synthSample; + int32_t mixed; + + if (scaled > 32767.0f) + synthSample = 32767; + else if (scaled < -32768.0f) + synthSample = -32768; + else + synthSample = (int32_t)scaled; + + mixed = (int32_t)destination[firstSample + index] + synthSample; + if (mixed > 32767) + mixed = 32767; + else if (mixed < -32768) + mixed = -32768; + destination[firstSample + index] = (int16_t)mixed; + } +} + +// Renders MIDI at exactly 140 ticks per second into one native PureDOOM block. +static void DoomAudioRenderMusic(tsf* synth, + int16_t* frames, + uint32_t* phaseInOut) { + float music[DOOM_AUDIO_BLOCK_FRAMES * DOOM_AUDIO_CHANNELS]; + uint32_t frameOffset = 0; + + memset(music, 0, sizeof(music)); + while (frameOffset < DOOM_AUDIO_BLOCK_FRAMES) { + uint32_t remaining; + uint32_t framesUntilTick; + + while (*phaseInOut >= DOOM_AUDIO_SAMPLE_RATE) { + *phaseInOut -= DOOM_AUDIO_SAMPLE_RATE; + DoomAudioTickMidi(synth); + } + + remaining = DOOM_AUDIO_BLOCK_FRAMES - frameOffset; + framesUntilTick = + (DOOM_AUDIO_SAMPLE_RATE - *phaseInOut + DOOM_MIDI_RATE - 1u) / + DOOM_MIDI_RATE; + if (framesUntilTick > remaining) + framesUntilTick = remaining; + + tsf_render_float(synth, + music + frameOffset * DOOM_AUDIO_CHANNELS, + (int)framesUntilTick, + 0); + DoomAudioMixMusic(frames, music + frameOffset * DOOM_AUDIO_CHANNELS, + frameOffset, framesUntilTick); + frameOffset += framesUntilTick; + *phaseInOut += framesUntilTick * DOOM_MIDI_RATE; + } +} + +// Renders one complete native-rate block containing synchronized music and SFX. +static void DoomAudioRenderBlock(int16_t* frames, + tsf** activeSynthInOut, + uint32_t* midiPhaseInOut) { + tsf* synth; + + DoomAudioReadSfxBlock(frames); + synth = DoomAudioGetSynth(); + if (synth != *activeSynthInOut) { + *activeSynthInOut = synth; + *midiPhaseInOut = DOOM_AUDIO_SAMPLE_RATE; + } + if (synth) + DoomAudioRenderMusic(synth, frames, midiPhaseInOut); +} + +// Queries one coherent native stream snapshot. +static KSTATUS DoomAudioQueryStream(SYS_AUDIO_STREAM_INFO* infoOut) { + if (!infoOut) + return NTLSTATUS_INVALID_PARAMETER; + memset(infoOut, 0, sizeof(*infoOut)); + infoOut->Size = sizeof(*infoOut); + return SysAudioQueryStream(g_doomAudio.StreamHandle, infoOut); +} + +// Arms a wait for writable space or a terminal stream condition. +static KSTATUS DoomAudioWaitWritable(const SYS_AUDIO_STREAM_INFO* info, + uint32_t writableFrames) { + SYS_AUDIO_ARM_WAIT_REQUEST request; + KSTATUS status; + + if (!info || !writableFrames || writableFrames > info->CapacityFrames) + return NTLSTATUS_INVALID_PARAMETER; + + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = g_doomAudio.StreamHandle; + request.Mask = SYS_AUDIO_WAIT_WRITABLE | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY; + request.WritableFrames = writableFrames; + request.DiscontinuityGeneration = info->DiscontinuityGeneration; + status = SysAudioArmWait(&request); + if (status != NTLSTATUS_SUCCESS) + return status; + return SysWaitSingle(g_doomAudio.StreamHandle, + SYS_WAIT_FLAG_NONE, + DOOM_AUDIO_WAIT_MILLISECONDS); +} + +// Waits until the buffered source has fallen to the low-latency refill watermark. +static KSTATUS DoomAudioWaitForRefill(const SYS_AUDIO_STREAM_INFO* info) { + if (!info || info->CapacityFrames <= DOOM_AUDIO_REFILL_WATERMARK_FRAMES) + return NTLSTATUS_INVALID_PARAMETER; + return DoomAudioWaitWritable( + info, info->CapacityFrames - DOOM_AUDIO_REFILL_WATERMARK_FRAMES); +} + +// Writes as much of one pending block as the nonblocking stream accepts. +static KSTATUS DoomAudioWritePending(const int16_t* frames, + uint32_t* frameOffsetInOut) { + SYS_AUDIO_WRITE_REQUEST request; + uint32_t written = 0; + KSTATUS status; + + if (!frames || !frameOffsetInOut || + *frameOffsetInOut >= DOOM_AUDIO_BLOCK_FRAMES) + return NTLSTATUS_INVALID_PARAMETER; + + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = g_doomAudio.StreamHandle; + request.FrameCount = DOOM_AUDIO_BLOCK_FRAMES - *frameOffsetInOut; + if (request.FrameCount > DOOM_AUDIO_WRITE_QUANTUM_FRAMES) + request.FrameCount = DOOM_AUDIO_WRITE_QUANTUM_FRAMES; + request.Frames = frames + *frameOffsetInOut * DOOM_AUDIO_CHANNELS; + request.WrittenFramesOut = &written; + status = SysAudioWrite(&request); + if (status != NTLSTATUS_SUCCESS) + return status; + if (!written) + return NTLSTATUS_WOULD_BLOCK; + + *frameOffsetInOut += written; + return NTLSTATUS_SUCCESS; +} + +// Keeps a bounded low-latency stream queue at PureDOOM's native cadence. +static void DoomAudioWorker(uintptr_t argument) { + int16_t block[DOOM_AUDIO_BLOCK_FRAMES * DOOM_AUDIO_CHANNELS]; + uint32_t pendingOffset = DOOM_AUDIO_BLOCK_FRAMES; + uint32_t midiPhase = 0; + tsf* activeSynth = NULL; + uint8_t started = 0; + (void)argument; + + while (true) { + SYS_AUDIO_STREAM_INFO info; + uint32_t requiredFrames; + KSTATUS status; + + if (_InterlockedCompareExchange(&g_doomAudio.StopRequested, 0, 0)) + break; + + status = DoomAudioQueryStream(&info); + if (status != NTLSTATUS_SUCCESS || info.State == SysAudioStreamFailed) { + DoomAudioPrint("doom: native audio stream failed\n"); + break; + } + + if (!started && info.BufferedFrames >= DOOM_AUDIO_PREFILL_FRAMES) { + status = SysAudioStart(g_doomAudio.StreamHandle); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: native audio start failed\n"); + break; + } + started = 1; + continue; + } + + if (pendingOffset == DOOM_AUDIO_BLOCK_FRAMES) { + if (started && + info.BufferedFrames > DOOM_AUDIO_REFILL_WATERMARK_FRAMES) { + status = DoomAudioWaitForRefill(&info); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_TIMEOUT) { + DoomAudioPrint("doom: native audio wait failed\n"); + break; + } + continue; + } + + DoomAudioRenderBlock(block, &activeSynth, &midiPhase); + pendingOffset = 0; + } + + requiredFrames = DOOM_AUDIO_BLOCK_FRAMES - pendingOffset; + if (requiredFrames > DOOM_AUDIO_WRITE_QUANTUM_FRAMES) + requiredFrames = DOOM_AUDIO_WRITE_QUANTUM_FRAMES; + if (info.WritableFrames < requiredFrames) { + status = DoomAudioWaitWritable(&info, requiredFrames); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_TIMEOUT) { + DoomAudioPrint("doom: native audio wait failed\n"); + break; + } + continue; + } + + status = DoomAudioWritePending(block, &pendingOffset); + if (status == NTLSTATUS_WOULD_BLOCK) + continue; + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: native audio write failed\n"); + break; + } + } + + if (started) + (void)SysAudioStop(g_doomAudio.StreamHandle); + SysExitThread(0); +} + +// Releases resources owned before or after the producer thread lifetime. +static void DoomAudioReleaseResources(void) { + if (g_doomAudio.StreamHandle) { + (void)SysCloseHandle(g_doomAudio.StreamHandle); + g_doomAudio.StreamHandle = 0; + } + tsf* synth = (tsf*)_InterlockedExchangePointer( + (void* volatile*)&g_doomAudio.Synth, NULL); + if (synth) + tsf_close(synth); + if (g_doomAudio.CoreMutex) { + (void)SysCloseHandle(g_doomAudio.CoreMutex); + g_doomAudio.CoreMutex = 0; + } +} + +// Starts one process-owned audio stream and its exclusive producer thread. +void DoomAudioStart(void) { + uint64_t endpointId = 0; + uint32_t streamHandle = 0; + KSTATUS status; + + if (DOOM_AUDIO_PREFILL_FRAMES >= SYS_AUDIO_STREAM_CAPACITY_FRAMES || + DOOM_AUDIO_REFILL_WATERMARK_FRAMES >= DOOM_AUDIO_PREFILL_FRAMES || + DOOM_AUDIO_WRITE_QUANTUM_FRAMES > DOOM_AUDIO_BLOCK_FRAMES) { + DoomAudioPrint("doom: invalid native audio buffering configuration\n"); + return; + } + + memset(&g_doomAudio, 0, sizeof(g_doomAudio)); + status = SysCreateMutex(&g_doomAudio.CoreMutex); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: audio synchronization unavailable; continuing without sound\n"); + return; + } + + status = DoomAudioFindEndpoint(&endpointId); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: no audio endpoint; continuing without sound\n"); + DoomAudioReleaseResources(); + return; + } + + SYS_AUDIO_OPEN_STREAM_REQUEST open; + memset(&open, 0, sizeof(open)); + open.Size = sizeof(open); + open.EndpointId = endpointId; + open.Format.SampleRate = DOOM_AUDIO_SAMPLE_RATE; + open.Format.ChannelCount = DOOM_AUDIO_CHANNELS; + open.Format.SampleFormat = SysAudioSampleS16; + open.StreamHandleOut = &streamHandle; + status = SysAudioOpenStream(&open); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: native audio open failed; continuing without sound\n"); + DoomAudioReleaseResources(); + return; + } + + g_doomAudio.StreamHandle = streamHandle; + + SYS_CREATE_THREAD_REQUEST thread; + memset(&thread, 0, sizeof(thread)); + thread.Entry = (uintptr_t)DoomAudioWorker; + thread.Priority = DOOM_AUDIO_WORKER_PRIORITY; + thread.ThreadHandleOut = &g_doomAudio.WorkerHandle; + thread.ThreadIdOut = &g_doomAudio.WorkerId; + status = SysCreateThread(&thread); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: audio worker creation failed; continuing without sound\n"); + DoomAudioReleaseResources(); + return; + } + + memset(&thread, 0, sizeof(thread)); + thread.Entry = (uintptr_t)DoomAudioSynthLoader; + thread.Priority = DOOM_AUDIO_SYNTH_LOADER_PRIORITY; + thread.ThreadHandleOut = &g_doomAudio.SynthLoaderHandle; + thread.ThreadIdOut = &g_doomAudio.SynthLoaderId; + status = SysCreateThread(&thread); + if (status != NTLSTATUS_SUCCESS) + DoomAudioPrint("doom: MIDI loader creation failed; continuing with sound effects only\n"); +} + +// Stops playback immediately without waiting for the producer thread. +void DoomAudioAbort(void) { + _InterlockedExchange(&g_doomAudio.StopRequested, 1); + if (g_doomAudio.StreamHandle) + (void)SysAudioStop(g_doomAudio.StreamHandle); +} + +// Stops the producer before releasing its stream, synthesizer, and core lock. +void DoomAudioShutdown(void) { + DoomAudioAbort(); + if (g_doomAudio.WorkerHandle) { + (void)SysWaitSingle(g_doomAudio.WorkerHandle, + SYS_WAIT_FLAG_NONE, + SYS_WAIT_TIMEOUT_INFINITE); + (void)SysCloseHandle(g_doomAudio.WorkerHandle); + g_doomAudio.WorkerHandle = 0; + } + if (g_doomAudio.SynthLoaderHandle) { + (void)SysWaitSingle(g_doomAudio.SynthLoaderHandle, + SYS_WAIT_FLAG_NONE, + SYS_WAIT_TIMEOUT_INFINITE); + (void)SysCloseHandle(g_doomAudio.SynthLoaderHandle); + g_doomAudio.SynthLoaderHandle = 0; + } + DoomAudioReleaseResources(); +} + +// Enters the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioLockCore(void) { + if (g_doomAudio.CoreMutex) + (void)SysWaitSingle(g_doomAudio.CoreMutex, + SYS_WAIT_FLAG_NONE, + SYS_WAIT_TIMEOUT_INFINITE); +} + +// Leaves the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioUnlockCore(void) { + if (g_doomAudio.CoreMutex) + (void)SysReleaseMutex(g_doomAudio.CoreMutex); +} diff --git a/code/user/apps/doom/doom_audio.h b/code/user/apps/doom/doom_audio.h new file mode 100644 index 0000000..9762512 --- /dev/null +++ b/code/user/apps/doom/doom_audio.h @@ -0,0 +1,18 @@ +// Doom audio owns PureDOOM sound extraction, MIDI synthesis, and native stream submission. + +#pragma once + +// Starts the optional audio worker after PureDOOM has initialized its sound state. +void DoomAudioStart(void); + +// Stops playback immediately without waiting for the worker. +void DoomAudioAbort(void); + +// Stops the audio worker and releases its process-owned resources. +void DoomAudioShutdown(void); + +// Enters the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioLockCore(void); + +// Leaves the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioUnlockCore(void); diff --git a/code/user/apps/doom/third_party/LICENSE.tsf b/code/user/apps/doom/third_party/LICENSE.tsf new file mode 100644 index 0000000..533da73 --- /dev/null +++ b/code/user/apps/doom/third_party/LICENSE.tsf @@ -0,0 +1,19 @@ +Copyright (C) 2017-2023 Bernhard Schelling (Based on SFZero, Copyright (C) 2012 Steve Folta, https://github.com/stevefolta/SFZero) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/code/user/apps/doom/third_party/README.tsf b/code/user/apps/doom/third_party/README.tsf new file mode 100644 index 0000000..f741372 --- /dev/null +++ b/code/user/apps/doom/third_party/README.tsf @@ -0,0 +1,11 @@ +TinySoundFont v0.9 + +Upstream: https://github.com/schellingb/TinySoundFont +File: tsf.h +License: MIT; see LICENSE.tsf. + +The vendored header is unchanged apart from normalized line endings. nt-lite +supplies allocator, memory, and math macros from doom_audio.c so the synthesizer +stays within the native user runtime. Doom loads the optional small `doom.sf2` +asset asynchronously, then releases the input mapping after TinySoundFont has +copied the sample bank into its owned representation. diff --git a/code/user/apps/doom/third_party/tsf.h b/code/user/apps/doom/third_party/tsf.h new file mode 100644 index 0000000..22a2f2a --- /dev/null +++ b/code/user/apps/doom/third_party/tsf.h @@ -0,0 +1,2045 @@ +/* TinySoundFont - v0.9 - SoundFont2 synthesizer - https://github.com/schellingb/TinySoundFont + no warranty implied; use at your own risk + Do this: + #define TSF_IMPLEMENTATION + before you include this file in *one* C or C++ file to create the implementation. + // i.e. it should look like this: + #include ... + #include ... + #define TSF_IMPLEMENTATION + #include "tsf.h" + + [OPTIONAL] #define TSF_NO_STDIO to remove stdio dependency + [OPTIONAL] #define TSF_MALLOC, TSF_REALLOC, and TSF_FREE to avoid stdlib.h + [OPTIONAL] #define TSF_MEMCPY, TSF_MEMSET to avoid string.h + [OPTIONAL] #define TSF_POW, TSF_POWF, TSF_EXPF, TSF_LOG, TSF_TAN, TSF_LOG10, TSF_SQRT to avoid math.h + + NOT YET IMPLEMENTED + - Support for ChorusEffectsSend and ReverbEffectsSend generators + - Better low-pass filter without lowering performance too much + - Support for modulators + + LICENSE (MIT) + + Copyright (C) 2017-2023 Bernhard Schelling + Based on SFZero, Copyright (C) 2012 Steve Folta (https://github.com/stevefolta/SFZero) + + Permission is hereby granted, free of charge, to any person obtaining a copy of this + software and associated documentation files (the "Software"), to deal in the Software + without restriction, including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + +#ifndef TSF_INCLUDE_TSF_INL +#define TSF_INCLUDE_TSF_INL + +#ifdef __cplusplus +extern "C" { +# define CPP_DEFAULT0 = 0 +#else +# define CPP_DEFAULT0 +#endif + +//define this if you want the API functions to be static +#ifdef TSF_STATIC +#define TSFDEF static +#else +#define TSFDEF extern +#endif + +// The load functions will return a pointer to a struct tsf which all functions +// thereafter take as the first parameter. +// On error the tsf_load* functions will return NULL most likely due to invalid +// data (or if the file did not exist in tsf_load_filename). +typedef struct tsf tsf; + +#ifndef TSF_NO_STDIO +// Directly load a SoundFont from a .sf2 file path +TSFDEF tsf* tsf_load_filename(const char* filename); +#endif + +// Load a SoundFont from a block of memory +TSFDEF tsf* tsf_load_memory(const void* buffer, int size); + +// Stream structure for the generic loading +struct tsf_stream +{ + // Custom data given to the functions as the first parameter + void* data; + + // Function pointer will be called to read 'size' bytes into ptr (returns number of read bytes) + int (*read)(void* data, void* ptr, unsigned int size); + + // Function pointer will be called to skip ahead over 'count' bytes (returns 1 on success, 0 on error) + int (*skip)(void* data, unsigned int count); +}; + +// Generic SoundFont loading method using the stream structure above +TSFDEF tsf* tsf_load(struct tsf_stream* stream); + +// Copy a tsf instance from an existing one, use tsf_close to close it as well. +// All copied tsf instances and their original instance are linked, and share the underlying soundfont. +// This allows loading a soundfont only once, but using it for multiple independent playbacks. +// (This function isn't thread-safe without locking.) +TSFDEF tsf* tsf_copy(tsf* f); + +// Free the memory related to this tsf instance +TSFDEF void tsf_close(tsf* f); + +// Stop all playing notes immediately and reset all channel parameters +TSFDEF void tsf_reset(tsf* f); + +// Returns the preset index from a bank and preset number, or -1 if it does not exist in the loaded SoundFont +TSFDEF int tsf_get_presetindex(const tsf* f, int bank, int preset_number); + +// Returns the number of presets in the loaded SoundFont +TSFDEF int tsf_get_presetcount(const tsf* f); + +// Returns the name of a preset index >= 0 and < tsf_get_presetcount() +TSFDEF const char* tsf_get_presetname(const tsf* f, int preset_index); + +// Returns the name of a preset by bank and preset number +TSFDEF const char* tsf_bank_get_presetname(const tsf* f, int bank, int preset_number); + +// Supported output modes by the render methods +enum TSFOutputMode +{ + // Two channels with single left/right samples one after another + TSF_STEREO_INTERLEAVED, + // Two channels with all samples for the left channel first then right + TSF_STEREO_UNWEAVED, + // A single channel (stereo instruments are mixed into center) + TSF_MONO +}; + +// Thread safety: +// +// 1. Rendering / voices: +// +// Your audio output which calls the tsf_render* functions will most likely +// run on a different thread than where the playback tsf_note* functions +// are called. In which case some sort of concurrency control like a +// mutex needs to be used so they are not called at the same time. +// Alternatively, you can pre-allocate a maximum number of voices that can +// play simultaneously by calling tsf_set_max_voices after loading. +// That way memory re-allocation will not happen during tsf_note_on and +// TSF should become mostly thread safe. +// There is a theoretical chance that ending notes would negatively influence +// a voice that is rendering at the time but it is hard to say. +// Also be aware, this has not been tested much. +// +// 2. Channels: +// +// Calls to tsf_channel_set_... functions may allocate new channels +// if no channel with that number was previously used. Make sure to +// create all channels at the beginning as required if you call tsf_render* +// from a different thread. + +// Setup the parameters for the voice render methods +// outputmode: if mono or stereo and how stereo channel data is ordered +// samplerate: the number of samples per second (output frequency) +// global_gain_db: volume gain in decibels (>0 means higher, <0 means lower) +TSFDEF void tsf_set_output(tsf* f, enum TSFOutputMode outputmode, int samplerate, float global_gain_db CPP_DEFAULT0); + +// Set the global gain as a volume factor +// global_gain: the desired volume where 1.0 is 100% +TSFDEF void tsf_set_volume(tsf* f, float global_gain); + +// Set the maximum number of voices to play simultaneously +// Depending on the soundfond, one note can cause many new voices to be started, +// so don't keep this number too low or otherwise sounds may not play. +// max_voices: maximum number to pre-allocate and set the limit to +// (tsf_set_max_voices returns 0 if allocation failed, otherwise 1) +TSFDEF int tsf_set_max_voices(tsf* f, int max_voices); + +// Start playing a note +// preset_index: preset index >= 0 and < tsf_get_presetcount() +// key: note value between 0 and 127 (60 being middle C) +// vel: velocity as a float between 0.0 (equal to note off) and 1.0 (full) +// bank: instrument bank number (alternative to preset_index) +// preset_number: preset number (alternative to preset_index) +// (tsf_note_on returns 0 if the allocation of a new voice failed, otherwise 1) +// (tsf_bank_note_on returns 0 if preset does not exist or allocation failed, otherwise 1) +TSFDEF int tsf_note_on(tsf* f, int preset_index, int key, float vel); +TSFDEF int tsf_bank_note_on(tsf* f, int bank, int preset_number, int key, float vel); + +// Stop playing a note +// (bank_note_off returns 0 if preset does not exist, otherwise 1) +TSFDEF void tsf_note_off(tsf* f, int preset_index, int key); +TSFDEF int tsf_bank_note_off(tsf* f, int bank, int preset_number, int key); + +// Stop playing all notes (end with sustain and release) +TSFDEF void tsf_note_off_all(tsf* f); + +// Returns the number of active voices +TSFDEF int tsf_active_voice_count(tsf* f); + +// Render output samples into a buffer +// You can either render as signed 16-bit values (tsf_render_short) or +// as 32-bit float values (tsf_render_float) +// buffer: target buffer of size samples * output_channels * sizeof(type) +// samples: number of samples to render +// flag_mixing: if 0 clear the buffer first, otherwise mix into existing data +TSFDEF void tsf_render_short(tsf* f, short* buffer, int samples, int flag_mixing CPP_DEFAULT0); +TSFDEF void tsf_render_float(tsf* f, float* buffer, int samples, int flag_mixing CPP_DEFAULT0); + +// Higher level channel based functions, set up channel parameters +// channel: channel number +// preset_index: preset index >= 0 and < tsf_get_presetcount() +// preset_number: preset number (alternative to preset_index) +// flag_mididrums: 0 for normal channels, otherwise apply MIDI drum channel rules +// bank: instrument bank number (alternative to preset_index) +// pan: stereo panning value from 0.0 (left) to 1.0 (right) (default 0.5 center) +// volume: linear volume scale factor (default 1.0 full) +// pitch_wheel: pitch wheel position 0 to 16383 (default 8192 unpitched) +// pitch_range: range of the pitch wheel in semitones (default 2.0, total +/- 2 semitones) +// tuning: tuning of all playing voices in semitones (default 0.0, standard (A440) tuning) +// (tsf_set_preset_number and set_bank_preset return 0 if preset does not exist, otherwise 1) +// (tsf_channel_set_... return 0 if a new channel needed allocation and that failed, otherwise 1) +TSFDEF int tsf_channel_set_presetindex(tsf* f, int channel, int preset_index); +TSFDEF int tsf_channel_set_presetnumber(tsf* f, int channel, int preset_number, int flag_mididrums CPP_DEFAULT0); +TSFDEF int tsf_channel_set_bank(tsf* f, int channel, int bank); +TSFDEF int tsf_channel_set_bank_preset(tsf* f, int channel, int bank, int preset_number); +TSFDEF int tsf_channel_set_pan(tsf* f, int channel, float pan); +TSFDEF int tsf_channel_set_volume(tsf* f, int channel, float volume); +TSFDEF int tsf_channel_set_pitchwheel(tsf* f, int channel, int pitch_wheel); +TSFDEF int tsf_channel_set_pitchrange(tsf* f, int channel, float pitch_range); +TSFDEF int tsf_channel_set_tuning(tsf* f, int channel, float tuning); + +// Start or stop playing notes on a channel (needs channel preset to be set) +// channel: channel number +// key: note value between 0 and 127 (60 being middle C) +// vel: velocity as a float between 0.0 (equal to note off) and 1.0 (full) +// (tsf_channel_note_on returns 0 on allocation failure of new voice, otherwise 1) +TSFDEF int tsf_channel_note_on(tsf* f, int channel, int key, float vel); +TSFDEF void tsf_channel_note_off(tsf* f, int channel, int key); +TSFDEF void tsf_channel_note_off_all(tsf* f, int channel); //end with sustain and release +TSFDEF void tsf_channel_sounds_off_all(tsf* f, int channel); //end immediately + +// Apply a MIDI control change to the channel (not all controllers are supported!) +// (tsf_channel_midi_control returns 0 on allocation failure of new channel, otherwise 1) +TSFDEF int tsf_channel_midi_control(tsf* f, int channel, int controller, int control_value); + +// Get current values set on the channels +TSFDEF int tsf_channel_get_preset_index(tsf* f, int channel); +TSFDEF int tsf_channel_get_preset_bank(tsf* f, int channel); +TSFDEF int tsf_channel_get_preset_number(tsf* f, int channel); +TSFDEF float tsf_channel_get_pan(tsf* f, int channel); +TSFDEF float tsf_channel_get_volume(tsf* f, int channel); +TSFDEF int tsf_channel_get_pitchwheel(tsf* f, int channel); +TSFDEF float tsf_channel_get_pitchrange(tsf* f, int channel); +TSFDEF float tsf_channel_get_tuning(tsf* f, int channel); + +#ifdef __cplusplus +# undef CPP_DEFAULT0 +} +#endif + +// end header +// --------------------------------------------------------------------------------------------------------- +#endif //TSF_INCLUDE_TSF_INL + +#ifdef TSF_IMPLEMENTATION +#undef TSF_IMPLEMENTATION + +// The lower this block size is the more accurate the effects are. +// Increasing the value significantly lowers the CPU usage of the voice rendering. +// If LFO affects the low-pass filter it can be hearable even as low as 8. +#ifndef TSF_RENDER_EFFECTSAMPLEBLOCK +#define TSF_RENDER_EFFECTSAMPLEBLOCK 64 +#endif + +// When using tsf_render_short, to do the conversion a buffer of a fixed size is +// allocated on the stack. On low memory platforms this could be made smaller. +// Increasing this above 512 should not have a significant impact on performance. +// The value should be a multiple of TSF_RENDER_EFFECTSAMPLEBLOCK. +#ifndef TSF_RENDER_SHORTBUFFERBLOCK +#define TSF_RENDER_SHORTBUFFERBLOCK 512 +#endif + +// Grace release time for quick voice off (avoid clicking noise) +#define TSF_FASTRELEASETIME 0.01f + +#if !defined(TSF_MALLOC) || !defined(TSF_FREE) || !defined(TSF_REALLOC) +# include +# define TSF_MALLOC malloc +# define TSF_FREE free +# define TSF_REALLOC realloc +#endif + +#if !defined(TSF_MEMCPY) || !defined(TSF_MEMSET) +# include +# define TSF_MEMCPY memcpy +# define TSF_MEMSET memset +#endif + +#if !defined(TSF_POW) || !defined(TSF_POWF) || !defined(TSF_EXPF) || !defined(TSF_LOG) || !defined(TSF_TAN) || !defined(TSF_LOG10) || !defined(TSF_SQRT) +# include +# if !defined(__cplusplus) && !defined(NAN) && !defined(powf) && !defined(expf) && !defined(sqrtf) +# define powf (float)pow // deal with old math.h +# define expf (float)exp // files that come without +# define sqrtf (float)sqrt // powf, expf and sqrtf +# endif +# define TSF_POW pow +# define TSF_POWF powf +# define TSF_EXPF expf +# define TSF_LOG log +# define TSF_TAN tan +# define TSF_LOG10 log10 +# define TSF_SQRTF sqrtf +#endif + +#ifndef TSF_NO_STDIO +# include +#endif + +#define TSF_TRUE 1 +#define TSF_FALSE 0 +#define TSF_BOOL char +#define TSF_PI 3.14159265358979323846264338327950288 +#define TSF_NULL 0 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef char tsf_fourcc[4]; +typedef signed char tsf_s8; +typedef unsigned char tsf_u8; +typedef unsigned short tsf_u16; +typedef signed short tsf_s16; +typedef unsigned int tsf_u32; +typedef char tsf_char20[20]; + +#define TSF_FourCCEquals(value1, value2) (value1[0] == value2[0] && value1[1] == value2[1] && value1[2] == value2[2] && value1[3] == value2[3]) + +struct tsf +{ + struct tsf_preset* presets; + float* fontSamples; + struct tsf_voice* voices; + struct tsf_channels* channels; + + int presetNum; + int voiceNum; + int maxVoiceNum; + unsigned int voicePlayIndex; + + enum TSFOutputMode outputmode; + float outSampleRate; + float globalGainDB; + int* refCount; +}; + +#ifndef TSF_NO_STDIO +static int tsf_stream_stdio_read(FILE* f, void* ptr, unsigned int size) { return (int)fread(ptr, 1, size, f); } +static int tsf_stream_stdio_skip(FILE* f, unsigned int count) { return !fseek(f, count, SEEK_CUR); } +TSFDEF tsf* tsf_load_filename(const char* filename) +{ + tsf* res; + struct tsf_stream stream = { TSF_NULL, (int(*)(void*,void*,unsigned int))&tsf_stream_stdio_read, (int(*)(void*,unsigned int))&tsf_stream_stdio_skip }; + #if __STDC_WANT_SECURE_LIB__ + FILE* f = TSF_NULL; fopen_s(&f, filename, "rb"); + #else + FILE* f = fopen(filename, "rb"); + #endif + if (!f) + { + //if (e) *e = TSF_FILENOTFOUND; + return TSF_NULL; + } + stream.data = f; + res = tsf_load(&stream); + fclose(f); + return res; +} +#endif + +struct tsf_stream_memory { const char* buffer; unsigned int total, pos; }; +static int tsf_stream_memory_read(struct tsf_stream_memory* m, void* ptr, unsigned int size) { if (size > m->total - m->pos) size = m->total - m->pos; TSF_MEMCPY(ptr, m->buffer+m->pos, size); m->pos += size; return size; } +static int tsf_stream_memory_skip(struct tsf_stream_memory* m, unsigned int count) { if (m->pos + count > m->total) return 0; m->pos += count; return 1; } +TSFDEF tsf* tsf_load_memory(const void* buffer, int size) +{ + struct tsf_stream stream = { TSF_NULL, (int(*)(void*,void*,unsigned int))&tsf_stream_memory_read, (int(*)(void*,unsigned int))&tsf_stream_memory_skip }; + struct tsf_stream_memory f = { 0, 0, 0 }; + f.buffer = (const char*)buffer; + f.total = size; + stream.data = &f; + return tsf_load(&stream); +} + +enum { TSF_LOOPMODE_NONE, TSF_LOOPMODE_CONTINUOUS, TSF_LOOPMODE_SUSTAIN }; + +enum { TSF_SEGMENT_NONE, TSF_SEGMENT_DELAY, TSF_SEGMENT_ATTACK, TSF_SEGMENT_HOLD, TSF_SEGMENT_DECAY, TSF_SEGMENT_SUSTAIN, TSF_SEGMENT_RELEASE, TSF_SEGMENT_DONE }; + +struct tsf_hydra +{ + struct tsf_hydra_phdr *phdrs; struct tsf_hydra_pbag *pbags; struct tsf_hydra_pmod *pmods; + struct tsf_hydra_pgen *pgens; struct tsf_hydra_inst *insts; struct tsf_hydra_ibag *ibags; + struct tsf_hydra_imod *imods; struct tsf_hydra_igen *igens; struct tsf_hydra_shdr *shdrs; + int phdrNum, pbagNum, pmodNum, pgenNum, instNum, ibagNum, imodNum, igenNum, shdrNum; +}; + +union tsf_hydra_genamount { struct { tsf_u8 lo, hi; } range; tsf_s16 shortAmount; tsf_u16 wordAmount; }; +struct tsf_hydra_phdr { tsf_char20 presetName; tsf_u16 preset, bank, presetBagNdx; tsf_u32 library, genre, morphology; }; +struct tsf_hydra_pbag { tsf_u16 genNdx, modNdx; }; +struct tsf_hydra_pmod { tsf_u16 modSrcOper, modDestOper; tsf_s16 modAmount; tsf_u16 modAmtSrcOper, modTransOper; }; +struct tsf_hydra_pgen { tsf_u16 genOper; union tsf_hydra_genamount genAmount; }; +struct tsf_hydra_inst { tsf_char20 instName; tsf_u16 instBagNdx; }; +struct tsf_hydra_ibag { tsf_u16 instGenNdx, instModNdx; }; +struct tsf_hydra_imod { tsf_u16 modSrcOper, modDestOper; tsf_s16 modAmount; tsf_u16 modAmtSrcOper, modTransOper; }; +struct tsf_hydra_igen { tsf_u16 genOper; union tsf_hydra_genamount genAmount; }; +struct tsf_hydra_shdr { tsf_char20 sampleName; tsf_u32 start, end, startLoop, endLoop, sampleRate; tsf_u8 originalPitch; tsf_s8 pitchCorrection; tsf_u16 sampleLink, sampleType; }; + +#define TSFR(FIELD) stream->read(stream->data, &i->FIELD, sizeof(i->FIELD)); +static void tsf_hydra_read_phdr(struct tsf_hydra_phdr* i, struct tsf_stream* stream) { TSFR(presetName) TSFR(preset) TSFR(bank) TSFR(presetBagNdx) TSFR(library) TSFR(genre) TSFR(morphology) } +static void tsf_hydra_read_pbag(struct tsf_hydra_pbag* i, struct tsf_stream* stream) { TSFR(genNdx) TSFR(modNdx) } +static void tsf_hydra_read_pmod(struct tsf_hydra_pmod* i, struct tsf_stream* stream) { TSFR(modSrcOper) TSFR(modDestOper) TSFR(modAmount) TSFR(modAmtSrcOper) TSFR(modTransOper) } +static void tsf_hydra_read_pgen(struct tsf_hydra_pgen* i, struct tsf_stream* stream) { TSFR(genOper) TSFR(genAmount) } +static void tsf_hydra_read_inst(struct tsf_hydra_inst* i, struct tsf_stream* stream) { TSFR(instName) TSFR(instBagNdx) } +static void tsf_hydra_read_ibag(struct tsf_hydra_ibag* i, struct tsf_stream* stream) { TSFR(instGenNdx) TSFR(instModNdx) } +static void tsf_hydra_read_imod(struct tsf_hydra_imod* i, struct tsf_stream* stream) { TSFR(modSrcOper) TSFR(modDestOper) TSFR(modAmount) TSFR(modAmtSrcOper) TSFR(modTransOper) } +static void tsf_hydra_read_igen(struct tsf_hydra_igen* i, struct tsf_stream* stream) { TSFR(genOper) TSFR(genAmount) } +static void tsf_hydra_read_shdr(struct tsf_hydra_shdr* i, struct tsf_stream* stream) { TSFR(sampleName) TSFR(start) TSFR(end) TSFR(startLoop) TSFR(endLoop) TSFR(sampleRate) TSFR(originalPitch) TSFR(pitchCorrection) TSFR(sampleLink) TSFR(sampleType) } +#undef TSFR + +struct tsf_riffchunk { tsf_fourcc id; tsf_u32 size; }; +struct tsf_envelope { float delay, attack, hold, decay, sustain, release, keynumToHold, keynumToDecay; }; +struct tsf_voice_envelope { float level, slope; int samplesUntilNextSegment; short segment, midiVelocity; struct tsf_envelope parameters; TSF_BOOL segmentIsExponential, isAmpEnv; }; +struct tsf_voice_lowpass { double QInv, a0, a1, b1, b2, z1, z2; TSF_BOOL active; }; +struct tsf_voice_lfo { int samplesUntil; float level, delta; }; + +struct tsf_region +{ + int loop_mode; + unsigned int sample_rate; + unsigned char lokey, hikey, lovel, hivel; + unsigned int group, offset, end, loop_start, loop_end; + int transpose, tune, pitch_keycenter, pitch_keytrack; + float attenuation, pan; + struct tsf_envelope ampenv, modenv; + int initialFilterQ, initialFilterFc; + int modEnvToPitch, modEnvToFilterFc, modLfoToFilterFc, modLfoToVolume; + float delayModLFO; + int freqModLFO, modLfoToPitch; + float delayVibLFO; + int freqVibLFO, vibLfoToPitch; +}; + +struct tsf_preset +{ + tsf_char20 presetName; + tsf_u16 preset, bank; + struct tsf_region* regions; + int regionNum; +}; + +struct tsf_voice +{ + int playingPreset, playingKey, playingChannel; + struct tsf_region* region; + double pitchInputTimecents, pitchOutputFactor; + double sourceSamplePosition; + float noteGainDB, panFactorLeft, panFactorRight; + unsigned int playIndex, loopStart, loopEnd; + struct tsf_voice_envelope ampenv, modenv; + struct tsf_voice_lowpass lowpass; + struct tsf_voice_lfo modlfo, viblfo; +}; + +struct tsf_channel +{ + unsigned short presetIndex, bank, pitchWheel, midiPan, midiVolume, midiExpression, midiRPN, midiData; + float panOffset, gainDB, pitchRange, tuning; +}; + +struct tsf_channels +{ + void (*setupVoice)(tsf* f, struct tsf_voice* voice); + int channelNum, activeChannel; + struct tsf_channel channels[1]; +}; + +static double tsf_timecents2Secsd(double timecents) { return TSF_POW(2.0, timecents / 1200.0); } +static float tsf_timecents2Secsf(float timecents) { return TSF_POWF(2.0f, timecents / 1200.0f); } +static float tsf_cents2Hertz(float cents) { return 8.176f * TSF_POWF(2.0f, cents / 1200.0f); } +static float tsf_decibelsToGain(float db) { return (db > -100.f ? TSF_POWF(10.0f, db * 0.05f) : 0); } +static float tsf_gainToDecibels(float gain) { return (gain <= .00001f ? -100.f : (float)(20.0 * TSF_LOG10(gain))); } + +static TSF_BOOL tsf_riffchunk_read(struct tsf_riffchunk* parent, struct tsf_riffchunk* chunk, struct tsf_stream* stream) +{ + TSF_BOOL IsRiff, IsList; + if (parent && sizeof(tsf_fourcc) + sizeof(tsf_u32) > parent->size) return TSF_FALSE; + if (!stream->read(stream->data, &chunk->id, sizeof(tsf_fourcc)) || *chunk->id <= ' ' || *chunk->id >= 'z') return TSF_FALSE; + if (!stream->read(stream->data, &chunk->size, sizeof(tsf_u32))) return TSF_FALSE; + if (parent && sizeof(tsf_fourcc) + sizeof(tsf_u32) + chunk->size > parent->size) return TSF_FALSE; + if (parent) parent->size -= sizeof(tsf_fourcc) + sizeof(tsf_u32) + chunk->size; + IsRiff = TSF_FourCCEquals(chunk->id, "RIFF"), IsList = TSF_FourCCEquals(chunk->id, "LIST"); + if (IsRiff && parent) return TSF_FALSE; //not allowed + if (!IsRiff && !IsList) return TSF_TRUE; //custom type without sub type + if (!stream->read(stream->data, &chunk->id, sizeof(tsf_fourcc)) || *chunk->id <= ' ' || *chunk->id >= 'z') return TSF_FALSE; + chunk->size -= sizeof(tsf_fourcc); + return TSF_TRUE; +} + +static void tsf_region_clear(struct tsf_region* i, TSF_BOOL for_relative) +{ + TSF_MEMSET(i, 0, sizeof(struct tsf_region)); + i->hikey = i->hivel = 127; + i->pitch_keycenter = 60; // C4 + if (for_relative) return; + + i->pitch_keytrack = 100; + + i->pitch_keycenter = -1; + + // SF2 defaults in timecents. + i->ampenv.delay = i->ampenv.attack = i->ampenv.hold = i->ampenv.decay = i->ampenv.release = -12000.0f; + i->modenv.delay = i->modenv.attack = i->modenv.hold = i->modenv.decay = i->modenv.release = -12000.0f; + + i->initialFilterFc = 13500; + + i->delayModLFO = -12000.0f; + i->delayVibLFO = -12000.0f; +} + +static void tsf_region_operator(struct tsf_region* region, tsf_u16 genOper, union tsf_hydra_genamount* amount, struct tsf_region* merge_region) +{ + enum + { + _GEN_TYPE_MASK = 0x0F, + GEN_FLOAT = 0x01, + GEN_INT = 0x02, + GEN_UINT_ADD = 0x03, + GEN_UINT_ADD15 = 0x04, + GEN_KEYRANGE = 0x05, + GEN_VELRANGE = 0x06, + GEN_LOOPMODE = 0x07, + GEN_GROUP = 0x08, + GEN_KEYCENTER = 0x09, + + _GEN_LIMIT_MASK = 0xF0, + GEN_INT_LIMIT12K = 0x10, //min -12000, max 12000 + GEN_INT_LIMITFC = 0x20, //min 1500, max 13500 + GEN_INT_LIMITQ = 0x30, //min 0, max 960 + GEN_INT_LIMIT960 = 0x40, //min -960, max 960 + GEN_INT_LIMIT16K4500 = 0x50, //min -16000, max 4500 + GEN_FLOAT_LIMIT12K5K = 0x60, //min -12000, max 5000 + GEN_FLOAT_LIMIT12K8K = 0x70, //min -12000, max 8000 + GEN_FLOAT_LIMIT1200 = 0x80, //min -1200, max 1200 + GEN_FLOAT_LIMITPAN = 0x90, //* .001f, min -.5f, max .5f, + GEN_FLOAT_LIMITATTN = 0xA0, //* .1f, min 0, max 144.0 + GEN_FLOAT_MAX1000 = 0xB0, //min 0, max 1000 + GEN_FLOAT_MAX1440 = 0xC0, //min 0, max 1440 + + _GEN_MAX = 59 + }; + #define _TSFREGIONOFFSET(TYPE, FIELD) (unsigned char)(((TYPE*)&((struct tsf_region*)0)->FIELD) - (TYPE*)0) + #define _TSFREGIONENVOFFSET(TYPE, ENV, FIELD) (unsigned char)(((TYPE*)&((&(((struct tsf_region*)0)->ENV))->FIELD)) - (TYPE*)0) + static const struct { unsigned char mode, offset; } genMetas[_GEN_MAX] = + { + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, offset ) }, // 0 StartAddrsOffset + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, end ) }, // 1 EndAddrsOffset + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, loop_start ) }, // 2 StartloopAddrsOffset + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, loop_end ) }, // 3 EndloopAddrsOffset + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, offset ) }, // 4 StartAddrsCoarseOffset + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modLfoToPitch ) }, // 5 ModLfoToPitch + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, vibLfoToPitch ) }, // 6 VibLfoToPitch + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modEnvToPitch ) }, // 7 ModEnvToPitch + { GEN_INT | GEN_INT_LIMITFC , _TSFREGIONOFFSET( int, initialFilterFc ) }, // 8 InitialFilterFc + { GEN_INT | GEN_INT_LIMITQ , _TSFREGIONOFFSET( int, initialFilterQ ) }, // 9 InitialFilterQ + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modLfoToFilterFc ) }, //10 ModLfoToFilterFc + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modEnvToFilterFc ) }, //11 ModEnvToFilterFc + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, end ) }, //12 EndAddrsCoarseOffset + { GEN_INT | GEN_INT_LIMIT960 , _TSFREGIONOFFSET( int, modLfoToVolume ) }, //13 ModLfoToVolume + { 0 , (0 ) }, // Unused + { 0 , (0 ) }, //15 ChorusEffectsSend (unsupported) + { 0 , (0 ) }, //16 ReverbEffectsSend (unsupported) + { GEN_FLOAT | GEN_FLOAT_LIMITPAN , _TSFREGIONOFFSET( float, pan ) }, //17 Pan + { 0 , (0 ) }, // Unused + { 0 , (0 ) }, // Unused + { 0 , (0 ) }, // Unused + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONOFFSET( float, delayModLFO ) }, //21 DelayModLFO + { GEN_INT | GEN_INT_LIMIT16K4500 , _TSFREGIONOFFSET( int, freqModLFO ) }, //22 FreqModLFO + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONOFFSET( float, delayVibLFO ) }, //23 DelayVibLFO + { GEN_INT | GEN_INT_LIMIT16K4500 , _TSFREGIONOFFSET( int, freqVibLFO ) }, //24 FreqVibLFO + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, modenv, delay ) }, //25 DelayModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, modenv, attack ) }, //26 AttackModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, modenv, hold ) }, //27 HoldModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, modenv, decay ) }, //28 DecayModEnv + { GEN_FLOAT | GEN_FLOAT_MAX1000 , _TSFREGIONENVOFFSET( float, modenv, sustain ) }, //29 SustainModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, modenv, release ) }, //30 ReleaseModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, modenv, keynumToHold ) }, //31 KeynumToModEnvHold + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, modenv, keynumToDecay) }, //32 KeynumToModEnvDecay + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, ampenv, delay ) }, //33 DelayVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, ampenv, attack ) }, //34 AttackVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, ampenv, hold ) }, //35 HoldVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, ampenv, decay ) }, //36 DecayVolEnv + { GEN_FLOAT | GEN_FLOAT_MAX1440 , _TSFREGIONENVOFFSET( float, ampenv, sustain ) }, //37 SustainVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, ampenv, release ) }, //38 ReleaseVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, ampenv, keynumToHold ) }, //39 KeynumToVolEnvHold + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, ampenv, keynumToDecay) }, //40 KeynumToVolEnvDecay + { 0 , (0 ) }, // Instrument (special) + { 0 , (0 ) }, // Reserved + { GEN_KEYRANGE , (0 ) }, //43 KeyRange + { GEN_VELRANGE , (0 ) }, //44 VelRange + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, loop_start ) }, //45 StartloopAddrsCoarseOffset + { 0 , (0 ) }, //46 Keynum (special) + { 0 , (0 ) }, //47 Velocity (special) + { GEN_FLOAT | GEN_FLOAT_LIMITATTN , _TSFREGIONOFFSET( float, attenuation ) }, //48 InitialAttenuation + { 0 , (0 ) }, // Reserved + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, loop_end ) }, //50 EndloopAddrsCoarseOffset + { GEN_INT , _TSFREGIONOFFSET( int, transpose ) }, //51 CoarseTune + { GEN_INT , _TSFREGIONOFFSET( int, tune ) }, //52 FineTune + { 0 , (0 ) }, // SampleID (special) + { GEN_LOOPMODE , _TSFREGIONOFFSET( int, loop_mode ) }, //54 SampleModes + { 0 , (0 ) }, // Reserved + { GEN_INT , _TSFREGIONOFFSET( int, pitch_keytrack ) }, //56 ScaleTuning + { GEN_GROUP , _TSFREGIONOFFSET(unsigned int, group ) }, //57 ExclusiveClass + { GEN_KEYCENTER , _TSFREGIONOFFSET( int, pitch_keycenter ) }, //58 OverridingRootKey + }; + #undef _TSFREGIONOFFSET + #undef _TSFREGIONENVOFFSET + if (amount) + { + int offset; + if (genOper >= _GEN_MAX) return; + offset = genMetas[genOper].offset; + switch (genMetas[genOper].mode & _GEN_TYPE_MASK) + { + case GEN_FLOAT: (( float*)region)[offset] = amount->shortAmount; return; + case GEN_INT: (( int*)region)[offset] = amount->shortAmount; return; + case GEN_UINT_ADD: ((unsigned int*)region)[offset] += amount->shortAmount; return; + case GEN_UINT_ADD15: ((unsigned int*)region)[offset] += amount->shortAmount<<15; return; + case GEN_KEYRANGE: region->lokey = amount->range.lo; region->hikey = amount->range.hi; return; + case GEN_VELRANGE: region->lovel = amount->range.lo; region->hivel = amount->range.hi; return; + case GEN_LOOPMODE: region->loop_mode = ((amount->wordAmount&3) == 3 ? TSF_LOOPMODE_SUSTAIN : ((amount->wordAmount&3) == 1 ? TSF_LOOPMODE_CONTINUOUS : TSF_LOOPMODE_NONE)); return; + case GEN_GROUP: region->group = amount->wordAmount; return; + case GEN_KEYCENTER: region->pitch_keycenter = amount->shortAmount; return; + } + } + else //merge regions and clamp values + { + for (genOper = 0; genOper != _GEN_MAX; genOper++) + { + int offset = genMetas[genOper].offset; + switch (genMetas[genOper].mode & _GEN_TYPE_MASK) + { + case GEN_FLOAT: + { + float *val = &((float*)region)[offset], vfactor, vmin, vmax; + *val += ((float*)merge_region)[offset]; + switch (genMetas[genOper].mode & _GEN_LIMIT_MASK) + { + case GEN_FLOAT_LIMIT12K5K: vfactor = 1.0f; vmin = -12000.0f; vmax = 5000.0f; break; + case GEN_FLOAT_LIMIT12K8K: vfactor = 1.0f; vmin = -12000.0f; vmax = 8000.0f; break; + case GEN_FLOAT_LIMIT1200: vfactor = 1.0f; vmin = -1200.0f; vmax = 1200.0f; break; + case GEN_FLOAT_LIMITPAN: vfactor = 0.001f; vmin = -0.5f; vmax = 0.5f; break; + case GEN_FLOAT_LIMITATTN: vfactor = 0.1f; vmin = 0.0f; vmax = 144.0f; break; + case GEN_FLOAT_MAX1000: vfactor = 1.0f; vmin = 0.0f; vmax = 1000.0f; break; + case GEN_FLOAT_MAX1440: vfactor = 1.0f; vmin = 0.0f; vmax = 1440.0f; break; + default: continue; + } + *val *= vfactor; + if (*val < vmin) *val = vmin; + else if (*val > vmax) *val = vmax; + continue; + } + case GEN_INT: + { + int *val = &((int*)region)[offset], vmin, vmax; + *val += ((int*)merge_region)[offset]; + switch (genMetas[genOper].mode & _GEN_LIMIT_MASK) + { + case GEN_INT_LIMIT12K: vmin = -12000; vmax = 12000; break; + case GEN_INT_LIMITFC: vmin = 1500; vmax = 13500; break; + case GEN_INT_LIMITQ: vmin = 0; vmax = 960; break; + case GEN_INT_LIMIT960: vmin = -960; vmax = 960; break; + case GEN_INT_LIMIT16K4500: vmin = -16000; vmax = 4500; break; + default: continue; + } + if (*val < vmin) *val = vmin; + else if (*val > vmax) *val = vmax; + continue; + } + case GEN_UINT_ADD: + { + ((unsigned int*)region)[offset] += ((unsigned int*)merge_region)[offset]; + continue; + } + } + } + } +} + +static void tsf_region_envtosecs(struct tsf_envelope* p, TSF_BOOL sustainIsGain) +{ + // EG times need to be converted from timecents to seconds. + // Pin very short EG segments. Timecents don't get to zero, and our EG is + // happier with zero values. + p->delay = (p->delay < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->delay)); + p->attack = (p->attack < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->attack)); + p->release = (p->release < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->release)); + + // If we have dynamic hold or decay times depending on key number we need + // to keep the values in timecents so we can calculate it during startNote + if (!p->keynumToHold) p->hold = (p->hold < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->hold)); + if (!p->keynumToDecay) p->decay = (p->decay < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->decay)); + + if (p->sustain < 0.0f) p->sustain = 0.0f; + else if (sustainIsGain) p->sustain = tsf_decibelsToGain(-p->sustain / 10.0f); + else p->sustain = 1.0f - (p->sustain / 1000.0f); +} + +static int tsf_load_presets(tsf* res, struct tsf_hydra *hydra, unsigned int fontSampleCount) +{ + enum { GenInstrument = 41, GenKeyRange = 43, GenVelRange = 44, GenSampleID = 53 }; + // Read each preset. + struct tsf_hydra_phdr *pphdr, *pphdrMax; + res->presetNum = hydra->phdrNum - 1; + res->presets = (struct tsf_preset*)TSF_MALLOC(res->presetNum * sizeof(struct tsf_preset)); + if (!res->presets) return 0; + else { int i; for (i = 0; i != res->presetNum; i++) res->presets[i].regions = TSF_NULL; } + for (pphdr = hydra->phdrs, pphdrMax = pphdr + hydra->phdrNum - 1; pphdr != pphdrMax; pphdr++) + { + int sortedIndex = 0, region_index = 0; + struct tsf_hydra_phdr *otherphdr; + struct tsf_preset* preset; + struct tsf_hydra_pbag *ppbag, *ppbagEnd; + struct tsf_region globalRegion; + for (otherphdr = hydra->phdrs; otherphdr != pphdrMax; otherphdr++) + { + if (otherphdr == pphdr || otherphdr->bank > pphdr->bank) continue; + else if (otherphdr->bank < pphdr->bank) sortedIndex++; + else if (otherphdr->preset > pphdr->preset) continue; + else if (otherphdr->preset < pphdr->preset) sortedIndex++; + else if (otherphdr < pphdr) sortedIndex++; + } + + preset = &res->presets[sortedIndex]; + TSF_MEMCPY(preset->presetName, pphdr->presetName, sizeof(preset->presetName)); + preset->presetName[sizeof(preset->presetName)-1] = '\0'; //should be zero terminated in source file but make sure + preset->bank = pphdr->bank; + preset->preset = pphdr->preset; + preset->regionNum = 0; + + //count regions covered by this preset + for (ppbag = hydra->pbags + pphdr->presetBagNdx, ppbagEnd = hydra->pbags + pphdr[1].presetBagNdx; ppbag != ppbagEnd; ppbag++) + { + unsigned char plokey = 0, phikey = 127, plovel = 0, phivel = 127; + struct tsf_hydra_pgen *ppgen, *ppgenEnd; struct tsf_hydra_inst *pinst; struct tsf_hydra_ibag *pibag, *pibagEnd; struct tsf_hydra_igen *pigen, *pigenEnd; + for (ppgen = hydra->pgens + ppbag->genNdx, ppgenEnd = hydra->pgens + ppbag[1].genNdx; ppgen != ppgenEnd; ppgen++) + { + if (ppgen->genOper == GenKeyRange) { plokey = ppgen->genAmount.range.lo; phikey = ppgen->genAmount.range.hi; continue; } + if (ppgen->genOper == GenVelRange) { plovel = ppgen->genAmount.range.lo; phivel = ppgen->genAmount.range.hi; continue; } + if (ppgen->genOper != GenInstrument) continue; + if (ppgen->genAmount.wordAmount >= hydra->instNum) continue; + pinst = hydra->insts + ppgen->genAmount.wordAmount; + for (pibag = hydra->ibags + pinst->instBagNdx, pibagEnd = hydra->ibags + pinst[1].instBagNdx; pibag != pibagEnd; pibag++) + { + unsigned char ilokey = 0, ihikey = 127, ilovel = 0, ihivel = 127; + for (pigen = hydra->igens + pibag->instGenNdx, pigenEnd = hydra->igens + pibag[1].instGenNdx; pigen != pigenEnd; pigen++) + { + if (pigen->genOper == GenKeyRange) { ilokey = pigen->genAmount.range.lo; ihikey = pigen->genAmount.range.hi; continue; } + if (pigen->genOper == GenVelRange) { ilovel = pigen->genAmount.range.lo; ihivel = pigen->genAmount.range.hi; continue; } + if (pigen->genOper == GenSampleID && ihikey >= plokey && ilokey <= phikey && ihivel >= plovel && ilovel <= phivel) preset->regionNum++; + } + } + } + } + + preset->regions = (struct tsf_region*)TSF_MALLOC(preset->regionNum * sizeof(struct tsf_region)); + if (!preset->regions) + { + int i; for (i = 0; i != res->presetNum; i++) TSF_FREE(res->presets[i].regions); + TSF_FREE(res->presets); + return 0; + } + tsf_region_clear(&globalRegion, TSF_TRUE); + + // Zones. + for (ppbag = hydra->pbags + pphdr->presetBagNdx, ppbagEnd = hydra->pbags + pphdr[1].presetBagNdx; ppbag != ppbagEnd; ppbag++) + { + struct tsf_hydra_pgen *ppgen, *ppgenEnd; struct tsf_hydra_inst *pinst; struct tsf_hydra_ibag *pibag, *pibagEnd; struct tsf_hydra_igen *pigen, *pigenEnd; + struct tsf_region presetRegion = globalRegion; + int hadGenInstrument = 0; + + // Generators. + for (ppgen = hydra->pgens + ppbag->genNdx, ppgenEnd = hydra->pgens + ppbag[1].genNdx; ppgen != ppgenEnd; ppgen++) + { + // Instrument. + if (ppgen->genOper == GenInstrument) + { + struct tsf_region instRegion; + tsf_u16 whichInst = ppgen->genAmount.wordAmount; + if (whichInst >= hydra->instNum) continue; + + tsf_region_clear(&instRegion, TSF_FALSE); + pinst = &hydra->insts[whichInst]; + for (pibag = hydra->ibags + pinst->instBagNdx, pibagEnd = hydra->ibags + pinst[1].instBagNdx; pibag != pibagEnd; pibag++) + { + // Generators. + struct tsf_region zoneRegion = instRegion; + int hadSampleID = 0; + for (pigen = hydra->igens + pibag->instGenNdx, pigenEnd = hydra->igens + pibag[1].instGenNdx; pigen != pigenEnd; pigen++) + { + if (pigen->genOper == GenSampleID) + { + struct tsf_hydra_shdr* pshdr; + + //preset region key and vel ranges are a filter for the zone regions + if (zoneRegion.hikey < presetRegion.lokey || zoneRegion.lokey > presetRegion.hikey) continue; + if (zoneRegion.hivel < presetRegion.lovel || zoneRegion.lovel > presetRegion.hivel) continue; + if (presetRegion.lokey > zoneRegion.lokey) zoneRegion.lokey = presetRegion.lokey; + if (presetRegion.hikey < zoneRegion.hikey) zoneRegion.hikey = presetRegion.hikey; + if (presetRegion.lovel > zoneRegion.lovel) zoneRegion.lovel = presetRegion.lovel; + if (presetRegion.hivel < zoneRegion.hivel) zoneRegion.hivel = presetRegion.hivel; + + //sum regions + tsf_region_operator(&zoneRegion, 0, TSF_NULL, &presetRegion); + + // EG times need to be converted from timecents to seconds. + tsf_region_envtosecs(&zoneRegion.ampenv, TSF_TRUE); + tsf_region_envtosecs(&zoneRegion.modenv, TSF_FALSE); + + // LFO times need to be converted from timecents to seconds. + zoneRegion.delayModLFO = (zoneRegion.delayModLFO < -11950.0f ? 0.0f : tsf_timecents2Secsf(zoneRegion.delayModLFO)); + zoneRegion.delayVibLFO = (zoneRegion.delayVibLFO < -11950.0f ? 0.0f : tsf_timecents2Secsf(zoneRegion.delayVibLFO)); + + // Fixup sample positions + pshdr = &hydra->shdrs[pigen->genAmount.wordAmount]; + zoneRegion.offset += pshdr->start; + zoneRegion.end += pshdr->end; + zoneRegion.loop_start += pshdr->startLoop; + zoneRegion.loop_end += pshdr->endLoop; + if (pshdr->endLoop > 0) zoneRegion.loop_end -= 1; + if (zoneRegion.loop_end > fontSampleCount) zoneRegion.loop_end = fontSampleCount; + if (zoneRegion.pitch_keycenter == -1) zoneRegion.pitch_keycenter = pshdr->originalPitch; + zoneRegion.tune += pshdr->pitchCorrection; + zoneRegion.sample_rate = pshdr->sampleRate; + if (zoneRegion.end && zoneRegion.end < fontSampleCount) zoneRegion.end++; + else zoneRegion.end = fontSampleCount; + + preset->regions[region_index] = zoneRegion; + region_index++; + hadSampleID = 1; + } + else tsf_region_operator(&zoneRegion, pigen->genOper, &pigen->genAmount, TSF_NULL); + } + + // Handle instrument's global zone. + if (pibag == hydra->ibags + pinst->instBagNdx && !hadSampleID) + instRegion = zoneRegion; + + // Modulators (TODO) + //if (ibag->instModNdx < ibag[1].instModNdx) addUnsupportedOpcode("any modulator"); + } + hadGenInstrument = 1; + } + else tsf_region_operator(&presetRegion, ppgen->genOper, &ppgen->genAmount, TSF_NULL); + } + + // Modulators (TODO) + //if (pbag->modNdx < pbag[1].modNdx) addUnsupportedOpcode("any modulator"); + + // Handle preset's global zone. + if (ppbag == hydra->pbags + pphdr->presetBagNdx && !hadGenInstrument) + globalRegion = presetRegion; + } + } + return 1; +} + +#ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H +static int tsf_decode_ogg(const tsf_u8 *pSmpl, const tsf_u8 *pSmplEnd, float** pRes, tsf_u32* pResNum, tsf_u32* pResMax, tsf_u32 resInitial) +{ + float *res = *pRes, *oldres; tsf_u32 resNum = *pResNum; tsf_u32 resMax = *pResMax; stb_vorbis *v; + + // Use whatever stb_vorbis API that is available (either pull or push) + #if !defined(STB_VORBIS_NO_PULLDATA_API) && !defined(STB_VORBIS_NO_FROMMEMORY) + v = stb_vorbis_open_memory(pSmpl, (int)(pSmplEnd - pSmpl), TSF_NULL, TSF_NULL); + #else + { int use, err; v = stb_vorbis_open_pushdata(pSmpl, (int)(pSmplEnd - pSmpl), &use, &err, TSF_NULL); pSmpl += use; } + #endif + if (v == TSF_NULL) return 0; + + while (true) + { + float** outputs; int n_samples; + + // Decode one frame of vorbis samples with whatever stb_vorbis API that is available + #if !defined(STB_VORBIS_NO_PULLDATA_API) && !defined(STB_VORBIS_NO_FROMMEMORY) + n_samples = stb_vorbis_get_frame_float(v, TSF_NULL, &outputs); + if (!n_samples) break; + #else + if (pSmpl >= pSmplEnd) break; + { int use = stb_vorbis_decode_frame_pushdata(v, pSmpl, (int)(pSmplEnd - pSmpl), TSF_NULL, &outputs, &n_samples); pSmpl += use; } + if (!n_samples) continue; + #endif + + // Expand our output buffer if necessary then copy over the decoded frame samples + resNum += n_samples; + if (resNum > resMax) + { + do { resMax += (resMax ? (resMax < 1048576 ? resMax : 1048576) : resInitial); } while (resNum > resMax); + res = (float*)TSF_REALLOC((oldres = res), resMax * sizeof(float)); + if (!res) { TSF_FREE(oldres); stb_vorbis_close(v); return 0; } + } + TSF_MEMCPY(res + resNum - n_samples, outputs[0], n_samples * sizeof(float)); + } + stb_vorbis_close(v); + *pRes = res; *pResNum = resNum; *pResMax = resMax; + return 1; +} + +static int tsf_decode_sf3_samples(const void* rawBuffer, float** pFloatBuffer, unsigned int* pSmplCount, struct tsf_hydra *hydra) +{ + const tsf_u8* smplBuffer = (const tsf_u8*)rawBuffer; + tsf_u32 smplLength = *pSmplCount, resNum = 0, resMax = 0, resInitial = (smplLength > 0x100000 ? (smplLength & ~0xFFFFF) : 65536); + float *res = TSF_NULL, *oldres; + int i, shdrLast = hydra->shdrNum - 1, is_sf3 = 0; + for (i = 0; i <= shdrLast; i++) + { + struct tsf_hydra_shdr *shdr = &hydra->shdrs[i]; + if (shdr->sampleType & 0x30) // compression flags (sometimes Vorbis flag) + { + const tsf_u8 *pSmpl = smplBuffer + shdr->start, *pSmplEnd = smplBuffer + shdr->end; + if (pSmpl + 4 > pSmplEnd || !TSF_FourCCEquals(pSmpl, "OggS")) + { + shdr->start = shdr->end = shdr->startLoop = shdr->endLoop = 0; + continue; + } + + // Fix up sample indices in shdr (end index is set after decoding) + shdr->start = resNum; + shdr->startLoop += resNum; + shdr->endLoop += resNum; + if (!tsf_decode_ogg(pSmpl, pSmplEnd, &res, &resNum, &resMax, resInitial)) { TSF_FREE(res); return 0; } + shdr->end = resNum; + is_sf3 = 1; + } + else // raw PCM sample + { + float *out; short *in = (short*)smplBuffer + resNum, *inEnd; tsf_u32 oldResNum = resNum; + if (is_sf3) // Fix up sample indices in shdr + { + tsf_u32 fix_offset = resNum - shdr->start; + in -= fix_offset; + shdr->start = resNum; + shdr->end += fix_offset; + shdr->startLoop += fix_offset; + shdr->endLoop += fix_offset; + } + inEnd = in + ((shdr->end >= shdr->endLoop ? shdr->end : shdr->endLoop) - resNum); + if (i == shdrLast || (tsf_u8*)inEnd > (smplBuffer + smplLength)) inEnd = (short*)(smplBuffer + smplLength); + if (inEnd <= in) continue; + + // expand our output buffer if necessary then convert the PCM data from short to float + resNum += (tsf_u32)(inEnd - in); + if (resNum > resMax) + { + do { resMax += (resMax ? (resMax < 1048576 ? resMax : 1048576) : resInitial); } while (resNum > resMax); + res = (float*)TSF_REALLOC((oldres = res), resMax * sizeof(float)); + if (!res) { TSF_FREE(oldres); return 0; } + } + + // Convert the samples from short to float + for (out = res + oldResNum; in < inEnd;) + *(out++) = (float)(*(in++) / 32767.0); + } + } + + // Trim the sample buffer down then return success (unless out of memory) + if (!(*pFloatBuffer = (float*)TSF_REALLOC(res, resNum * sizeof(float)))) *pFloatBuffer = res; + *pFloatBuffer = res; + *pSmplCount = resNum; + return (res ? 1 : 0); +} +#endif + +static int tsf_load_samples(void** pRawBuffer, float** pFloatBuffer, unsigned int* pSmplCount, struct tsf_riffchunk *chunkSmpl, struct tsf_stream* stream) +{ + #ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H + // With OGG Vorbis support we cannot pre-allocate the memory for tsf_decode_sf3_samples + tsf_u32 resNum, resMax; float* oldres; + *pSmplCount = chunkSmpl->size; + *pRawBuffer = (void*)TSF_MALLOC(*pSmplCount); + if (!*pRawBuffer || !stream->read(stream->data, *pRawBuffer, chunkSmpl->size)) return 0; + if (chunkSmpl->id[3] != 'o') return 1; + + // Decode custom .sfo 'smpo' format where all samples are in a single ogg stream + resNum = resMax = 0; + if (!tsf_decode_ogg((tsf_u8*)*pRawBuffer, (tsf_u8*)*pRawBuffer + chunkSmpl->size, pFloatBuffer, &resNum, &resMax, 65536)) return 0; + if (!(*pFloatBuffer = (float*)TSF_REALLOC((oldres = *pFloatBuffer), resNum * sizeof(float)))) *pFloatBuffer = oldres; + *pSmplCount = resNum; + return (*pFloatBuffer ? 1 : 0); + #else + // Inline convert the samples from short to float + float *res, *out; const short *in; + *pSmplCount = chunkSmpl->size / (unsigned int)sizeof(short); + *pFloatBuffer = (float*)TSF_MALLOC(*pSmplCount * sizeof(float)); + if (!*pFloatBuffer || !stream->read(stream->data, *pFloatBuffer, chunkSmpl->size)) return 0; + for (res = *pFloatBuffer, out = res + *pSmplCount, in = (short*)res + *pSmplCount; out != res;) + *(--out) = (float)(*(--in) / 32767.0); + return 1; + #endif +} + +static int tsf_voice_envelope_release_samples(struct tsf_voice_envelope* e, float outSampleRate) +{ + return (int)((e->parameters.release <= 0 ? TSF_FASTRELEASETIME : e->parameters.release) * outSampleRate); +} + +static void tsf_voice_envelope_nextsegment(struct tsf_voice_envelope* e, short active_segment, float outSampleRate) +{ + switch (active_segment) + { + case TSF_SEGMENT_NONE: + e->samplesUntilNextSegment = (int)(e->parameters.delay * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + e->segment = TSF_SEGMENT_DELAY; + e->segmentIsExponential = TSF_FALSE; + e->level = 0.0; + e->slope = 0.0; + return; + } + /* fall through */ + case TSF_SEGMENT_DELAY: + e->samplesUntilNextSegment = (int)(e->parameters.attack * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + if (!e->isAmpEnv) + { + //mod env attack duration scales with velocity (velocity of 1 is full duration, max velocity is 0.125 times duration) + e->samplesUntilNextSegment = (int)(e->parameters.attack * ((145 - e->midiVelocity) / 144.0f) * outSampleRate); + } + e->segment = TSF_SEGMENT_ATTACK; + e->segmentIsExponential = TSF_FALSE; + e->level = 0.0f; + e->slope = 1.0f / e->samplesUntilNextSegment; + return; + } + /* fall through */ + case TSF_SEGMENT_ATTACK: + e->samplesUntilNextSegment = (int)(e->parameters.hold * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + e->segment = TSF_SEGMENT_HOLD; + e->segmentIsExponential = TSF_FALSE; + e->level = 1.0f; + e->slope = 0.0f; + return; + } + /* fall through */ + case TSF_SEGMENT_HOLD: + e->samplesUntilNextSegment = (int)(e->parameters.decay * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + e->segment = TSF_SEGMENT_DECAY; + e->level = 1.0f; + if (e->isAmpEnv) + { + // I don't truly understand this; just following what LinuxSampler does. + float mysterySlope = -9.226f / e->samplesUntilNextSegment; + e->slope = TSF_EXPF(mysterySlope); + e->segmentIsExponential = TSF_TRUE; + if (e->parameters.sustain > 0.0f) + { + // Again, this is following LinuxSampler's example, which is similar to + // SF2-style decay, where "decay" specifies the time it would take to + // get to zero, not to the sustain level. The SFZ spec is not that + // specific about what "decay" means, so perhaps it's really supposed + // to specify the time to reach the sustain level. + e->samplesUntilNextSegment = (int)(TSF_LOG(e->parameters.sustain) / mysterySlope); + } + } + else + { + e->slope = -1.0f / e->samplesUntilNextSegment; + e->samplesUntilNextSegment = (int)(e->parameters.decay * (1.0f - e->parameters.sustain) * outSampleRate); + e->segmentIsExponential = TSF_FALSE; + } + return; + } + /* fall through */ + case TSF_SEGMENT_DECAY: + e->segment = TSF_SEGMENT_SUSTAIN; + e->level = e->parameters.sustain; + e->slope = 0.0f; + e->samplesUntilNextSegment = 0x7FFFFFFF; + e->segmentIsExponential = TSF_FALSE; + return; + case TSF_SEGMENT_SUSTAIN: + e->segment = TSF_SEGMENT_RELEASE; + e->samplesUntilNextSegment = tsf_voice_envelope_release_samples(e, outSampleRate); + if (e->isAmpEnv) + { + // I don't truly understand this; just following what LinuxSampler does. + float mysterySlope = -9.226f / e->samplesUntilNextSegment; + e->slope = TSF_EXPF(mysterySlope); + e->segmentIsExponential = TSF_TRUE; + } + else + { + e->slope = -e->level / e->samplesUntilNextSegment; + e->segmentIsExponential = TSF_FALSE; + } + return; + case TSF_SEGMENT_RELEASE: + default: + e->segment = TSF_SEGMENT_DONE; + e->segmentIsExponential = TSF_FALSE; + e->level = e->slope = 0.0f; + e->samplesUntilNextSegment = 0x7FFFFFF; + } +} + +static void tsf_voice_envelope_setup(struct tsf_voice_envelope* e, struct tsf_envelope* new_parameters, int midiNoteNumber, short midiVelocity, TSF_BOOL isAmpEnv, float outSampleRate) +{ + e->parameters = *new_parameters; + if (e->parameters.keynumToHold) + { + e->parameters.hold += e->parameters.keynumToHold * (60.0f - midiNoteNumber); + e->parameters.hold = (e->parameters.hold < -10000.0f ? 0.0f : tsf_timecents2Secsf(e->parameters.hold)); + } + if (e->parameters.keynumToDecay) + { + e->parameters.decay += e->parameters.keynumToDecay * (60.0f - midiNoteNumber); + e->parameters.decay = (e->parameters.decay < -10000.0f ? 0.0f : tsf_timecents2Secsf(e->parameters.decay)); + } + e->midiVelocity = midiVelocity; + e->isAmpEnv = isAmpEnv; + tsf_voice_envelope_nextsegment(e, TSF_SEGMENT_NONE, outSampleRate); +} + +static void tsf_voice_envelope_process(struct tsf_voice_envelope* e, int numSamples, float outSampleRate) +{ + if (e->slope) + { + if (e->segmentIsExponential) e->level *= TSF_POWF(e->slope, (float)numSamples); + else e->level += (e->slope * numSamples); + } + if ((e->samplesUntilNextSegment -= numSamples) <= 0) + tsf_voice_envelope_nextsegment(e, e->segment, outSampleRate); +} + +static void tsf_voice_lowpass_setup(struct tsf_voice_lowpass* e, float Fc) +{ + // Lowpass filter from http://www.earlevel.com/main/2012/11/26/biquad-c-source-code/ + double K = TSF_TAN(TSF_PI * Fc), KK = K * K; + double norm = 1 / (1 + K * e->QInv + KK); + e->a0 = KK * norm; + e->a1 = 2 * e->a0; + e->b1 = 2 * (KK - 1) * norm; + e->b2 = (1 - K * e->QInv + KK) * norm; +} + +static float tsf_voice_lowpass_process(struct tsf_voice_lowpass* e, double In) +{ + double Out = In * e->a0 + e->z1; e->z1 = In * e->a1 + e->z2 - e->b1 * Out; e->z2 = In * e->a0 - e->b2 * Out; return (float)Out; +} + +static void tsf_voice_lfo_setup(struct tsf_voice_lfo* e, float delay, int freqCents, float outSampleRate) +{ + e->samplesUntil = (int)(delay * outSampleRate); + e->delta = (4.0f * tsf_cents2Hertz((float)freqCents) / outSampleRate); + e->level = 0; +} + +static void tsf_voice_lfo_process(struct tsf_voice_lfo* e, int blockSamples) +{ + if (e->samplesUntil > blockSamples) { e->samplesUntil -= blockSamples; return; } + e->level += e->delta * blockSamples; + if (e->level > 1.0f) { e->delta = -e->delta; e->level = 2.0f - e->level; } + else if (e->level < -1.0f) { e->delta = -e->delta; e->level = -2.0f - e->level; } +} + +static void tsf_voice_kill(struct tsf_voice* v) +{ + v->playingPreset = -1; +} + +static void tsf_voice_end(tsf* f, struct tsf_voice* v) +{ + // if maxVoiceNum is set, assume that voice rendering and note queuing are on separate threads + // so to minimize the chance that voice rendering would advance the segment at the same time + // we just do it twice here and hope that it sticks + int repeats = (f->maxVoiceNum ? 2 : 1); + while (repeats--) + { + tsf_voice_envelope_nextsegment(&v->ampenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + tsf_voice_envelope_nextsegment(&v->modenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + if (v->region->loop_mode == TSF_LOOPMODE_SUSTAIN) + { + // Continue playing, but stop looping. + v->loopEnd = v->loopStart; + } + } +} + +static void tsf_voice_endquick(tsf* f, struct tsf_voice* v) +{ + // if maxVoiceNum is set, assume that voice rendering and note queuing are on separate threads + // so to minimize the chance that voice rendering would advance the segment at the same time + // we just do it twice here and hope that it sticks + int repeats = (f->maxVoiceNum ? 2 : 1); + while (repeats--) + { + v->ampenv.parameters.release = 0.0f; tsf_voice_envelope_nextsegment(&v->ampenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + v->modenv.parameters.release = 0.0f; tsf_voice_envelope_nextsegment(&v->modenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + } +} + +static void tsf_voice_calcpitchratio(struct tsf_voice* v, float pitchShift, float outSampleRate) +{ + double note = v->playingKey + v->region->transpose + v->region->tune / 100.0; + double adjustedPitch = v->region->pitch_keycenter + (note - v->region->pitch_keycenter) * (v->region->pitch_keytrack / 100.0); + if (pitchShift) adjustedPitch += pitchShift; + v->pitchInputTimecents = adjustedPitch * 100.0; + v->pitchOutputFactor = v->region->sample_rate / (tsf_timecents2Secsd(v->region->pitch_keycenter * 100.0) * outSampleRate); +} + +static void tsf_voice_render(tsf* f, struct tsf_voice* v, float* outputBuffer, int numSamples) +{ + struct tsf_region* region = v->region; + float* input = f->fontSamples; + float* outL = outputBuffer; + float* outR = (f->outputmode == TSF_STEREO_UNWEAVED ? outL + numSamples : TSF_NULL); + + // Cache some values, to give them at least some chance of ending up in registers. + TSF_BOOL updateModEnv = (region->modEnvToPitch || region->modEnvToFilterFc); + TSF_BOOL updateModLFO = (v->modlfo.delta && (region->modLfoToPitch || region->modLfoToFilterFc || region->modLfoToVolume)); + TSF_BOOL updateVibLFO = (v->viblfo.delta && (region->vibLfoToPitch)); + TSF_BOOL isLooping = (v->loopStart < v->loopEnd); + unsigned int tmpLoopStart = v->loopStart, tmpLoopEnd = v->loopEnd; + double tmpSampleEndDbl = (double)region->end, tmpLoopEndDbl = (double)tmpLoopEnd + 1.0; + double tmpSourceSamplePosition = v->sourceSamplePosition; + struct tsf_voice_lowpass tmpLowpass = v->lowpass; + + TSF_BOOL dynamicLowpass = (region->modLfoToFilterFc || region->modEnvToFilterFc); + float tmpSampleRate = f->outSampleRate, tmpInitialFilterFc, tmpModLfoToFilterFc, tmpModEnvToFilterFc; + + TSF_BOOL dynamicPitchRatio = (region->modLfoToPitch || region->modEnvToPitch || region->vibLfoToPitch); + double pitchRatio; + float tmpModLfoToPitch, tmpVibLfoToPitch, tmpModEnvToPitch; + + TSF_BOOL dynamicGain = (region->modLfoToVolume != 0); + float noteGain = 0, tmpModLfoToVolume; + + if (dynamicLowpass) tmpInitialFilterFc = (float)region->initialFilterFc, tmpModLfoToFilterFc = (float)region->modLfoToFilterFc, tmpModEnvToFilterFc = (float)region->modEnvToFilterFc; + else tmpInitialFilterFc = 0, tmpModLfoToFilterFc = 0, tmpModEnvToFilterFc = 0; + + if (dynamicPitchRatio) pitchRatio = 0, tmpModLfoToPitch = (float)region->modLfoToPitch, tmpVibLfoToPitch = (float)region->vibLfoToPitch, tmpModEnvToPitch = (float)region->modEnvToPitch; + else pitchRatio = tsf_timecents2Secsd(v->pitchInputTimecents) * v->pitchOutputFactor, tmpModLfoToPitch = 0, tmpVibLfoToPitch = 0, tmpModEnvToPitch = 0; + + if (dynamicGain) tmpModLfoToVolume = (float)region->modLfoToVolume * 0.1f; + else noteGain = tsf_decibelsToGain(v->noteGainDB), tmpModLfoToVolume = 0; + + while (numSamples) + { + float gainMono, gainLeft, gainRight; + int blockSamples = (numSamples > TSF_RENDER_EFFECTSAMPLEBLOCK ? TSF_RENDER_EFFECTSAMPLEBLOCK : numSamples); + numSamples -= blockSamples; + + if (dynamicLowpass) + { + float fres = tmpInitialFilterFc + v->modlfo.level * tmpModLfoToFilterFc + v->modenv.level * tmpModEnvToFilterFc; + float lowpassFc = (fres <= 13500 ? tsf_cents2Hertz(fres) / tmpSampleRate : 1.0f); + tmpLowpass.active = (lowpassFc < 0.499f); + if (tmpLowpass.active) tsf_voice_lowpass_setup(&tmpLowpass, lowpassFc); + } + + if (dynamicPitchRatio) + pitchRatio = tsf_timecents2Secsd(v->pitchInputTimecents + (v->modlfo.level * tmpModLfoToPitch + v->viblfo.level * tmpVibLfoToPitch + v->modenv.level * tmpModEnvToPitch)) * v->pitchOutputFactor; + + if (dynamicGain) + noteGain = tsf_decibelsToGain(v->noteGainDB + (v->modlfo.level * tmpModLfoToVolume)); + + gainMono = noteGain * v->ampenv.level; + + // Update EG. + tsf_voice_envelope_process(&v->ampenv, blockSamples, tmpSampleRate); + if (updateModEnv) tsf_voice_envelope_process(&v->modenv, blockSamples, tmpSampleRate); + + // Update LFOs. + if (updateModLFO) tsf_voice_lfo_process(&v->modlfo, blockSamples); + if (updateVibLFO) tsf_voice_lfo_process(&v->viblfo, blockSamples); + + switch (f->outputmode) + { + case TSF_STEREO_INTERLEAVED: + gainLeft = gainMono * v->panFactorLeft, gainRight = gainMono * v->panFactorRight; + while (blockSamples-- && tmpSourceSamplePosition < tmpSampleEndDbl) + { + unsigned int pos = (unsigned int)tmpSourceSamplePosition, nextPos = (pos >= tmpLoopEnd && isLooping ? tmpLoopStart : pos + 1); + + // Simple linear interpolation. + float alpha = (float)(tmpSourceSamplePosition - pos), val = (input[pos] * (1.0f - alpha) + input[nextPos] * alpha); + + // Low-pass filter. + if (tmpLowpass.active) val = tsf_voice_lowpass_process(&tmpLowpass, val); + + *outL++ += val * gainLeft; + *outL++ += val * gainRight; + + // Next sample. + tmpSourceSamplePosition += pitchRatio; + if (tmpSourceSamplePosition >= tmpLoopEndDbl && isLooping) tmpSourceSamplePosition -= (tmpLoopEnd - tmpLoopStart + 1.0); + } + break; + + case TSF_STEREO_UNWEAVED: + gainLeft = gainMono * v->panFactorLeft, gainRight = gainMono * v->panFactorRight; + while (blockSamples-- && tmpSourceSamplePosition < tmpSampleEndDbl) + { + unsigned int pos = (unsigned int)tmpSourceSamplePosition, nextPos = (pos >= tmpLoopEnd && isLooping ? tmpLoopStart : pos + 1); + + // Simple linear interpolation. + float alpha = (float)(tmpSourceSamplePosition - pos), val = (input[pos] * (1.0f - alpha) + input[nextPos] * alpha); + + // Low-pass filter. + if (tmpLowpass.active) val = tsf_voice_lowpass_process(&tmpLowpass, val); + + *outL++ += val * gainLeft; + *outR++ += val * gainRight; + + // Next sample. + tmpSourceSamplePosition += pitchRatio; + if (tmpSourceSamplePosition >= tmpLoopEndDbl && isLooping) tmpSourceSamplePosition -= (tmpLoopEnd - tmpLoopStart + 1.0); + } + break; + + case TSF_MONO: + while (blockSamples-- && tmpSourceSamplePosition < tmpSampleEndDbl) + { + unsigned int pos = (unsigned int)tmpSourceSamplePosition, nextPos = (pos >= tmpLoopEnd && isLooping ? tmpLoopStart : pos + 1); + + // Simple linear interpolation. + float alpha = (float)(tmpSourceSamplePosition - pos), val = (input[pos] * (1.0f - alpha) + input[nextPos] * alpha); + + // Low-pass filter. + if (tmpLowpass.active) val = tsf_voice_lowpass_process(&tmpLowpass, val); + + *outL++ += val * gainMono; + + // Next sample. + tmpSourceSamplePosition += pitchRatio; + if (tmpSourceSamplePosition >= tmpLoopEndDbl && isLooping) tmpSourceSamplePosition -= (tmpLoopEnd - tmpLoopStart + 1.0); + } + break; + } + + if (tmpSourceSamplePosition >= tmpSampleEndDbl || v->ampenv.segment == TSF_SEGMENT_DONE) + { + tsf_voice_kill(v); + return; + } + } + + v->sourceSamplePosition = tmpSourceSamplePosition; + if (tmpLowpass.active || dynamicLowpass) v->lowpass = tmpLowpass; +} + +TSFDEF tsf* tsf_load(struct tsf_stream* stream) +{ + tsf* res = TSF_NULL; + struct tsf_riffchunk chunkHead; + struct tsf_riffchunk chunkList; + struct tsf_hydra hydra; + void* rawBuffer = TSF_NULL; + float* floatBuffer = TSF_NULL; + tsf_u32 smplCount = 0; + + if (!tsf_riffchunk_read(TSF_NULL, &chunkHead, stream) || !TSF_FourCCEquals(chunkHead.id, "sfbk")) + { + //if (e) *e = TSF_INVALID_NOSF2HEADER; + return res; + } + + // Read hydra and locate sample data. + TSF_MEMSET(&hydra, 0, sizeof(hydra)); + while (tsf_riffchunk_read(&chunkHead, &chunkList, stream)) + { + struct tsf_riffchunk chunk; + if (TSF_FourCCEquals(chunkList.id, "pdta")) + { + while (tsf_riffchunk_read(&chunkList, &chunk, stream)) + { + #define HandleChunk(chunkName) (TSF_FourCCEquals(chunk.id, #chunkName) && !(chunk.size % chunkName##SizeInFile)) \ + { \ + int num = chunk.size / chunkName##SizeInFile, i; \ + hydra.chunkName##Num = num; \ + hydra.chunkName##s = (struct tsf_hydra_##chunkName*)TSF_MALLOC(num * sizeof(struct tsf_hydra_##chunkName)); \ + if (!hydra.chunkName##s) goto out_of_memory; \ + for (i = 0; i < num; ++i) tsf_hydra_read_##chunkName(&hydra.chunkName##s[i], stream); \ + } + enum + { + phdrSizeInFile = 38, pbagSizeInFile = 4, pmodSizeInFile = 10, + pgenSizeInFile = 4, instSizeInFile = 22, ibagSizeInFile = 4, + imodSizeInFile = 10, igenSizeInFile = 4, shdrSizeInFile = 46 + }; + if HandleChunk(phdr) else if HandleChunk(pbag) else if HandleChunk(pmod) + else if HandleChunk(pgen) else if HandleChunk(inst) else if HandleChunk(ibag) + else if HandleChunk(imod) else if HandleChunk(igen) else if HandleChunk(shdr) + else stream->skip(stream->data, chunk.size); + #undef HandleChunk + } + } + else if (TSF_FourCCEquals(chunkList.id, "sdta")) + { + while (tsf_riffchunk_read(&chunkList, &chunk, stream)) + { + if ((TSF_FourCCEquals(chunk.id, "smpl") + #ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H + || TSF_FourCCEquals(chunk.id, "smpo") + #endif + ) && !rawBuffer && !floatBuffer && chunk.size >= sizeof(short)) + { + if (!tsf_load_samples(&rawBuffer, &floatBuffer, &smplCount, &chunk, stream)) goto out_of_memory; + } + else stream->skip(stream->data, chunk.size); + } + } + else stream->skip(stream->data, chunkList.size); + } + if (!hydra.phdrs || !hydra.pbags || !hydra.pmods || !hydra.pgens || !hydra.insts || !hydra.ibags || !hydra.imods || !hydra.igens || !hydra.shdrs) + { + //if (e) *e = TSF_INVALID_INCOMPLETE; + } + else if (!rawBuffer && !floatBuffer) + { + //if (e) *e = TSF_INVALID_NOSAMPLEDATA; + } + else + { + #ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H + if (!floatBuffer && !tsf_decode_sf3_samples(rawBuffer, &floatBuffer, &smplCount, &hydra)) goto out_of_memory; + #endif + res = (tsf*)TSF_MALLOC(sizeof(tsf)); + if (res) TSF_MEMSET(res, 0, sizeof(tsf)); + if (!res || !tsf_load_presets(res, &hydra, smplCount)) goto out_of_memory; + res->outSampleRate = 44100.0f; + res->fontSamples = floatBuffer; + floatBuffer = TSF_NULL; // don't free below + } + if (0) + { + out_of_memory: + TSF_FREE(res); + res = TSF_NULL; + //if (e) *e = TSF_OUT_OF_MEMORY; + } + TSF_FREE(hydra.phdrs); TSF_FREE(hydra.pbags); TSF_FREE(hydra.pmods); + TSF_FREE(hydra.pgens); TSF_FREE(hydra.insts); TSF_FREE(hydra.ibags); + TSF_FREE(hydra.imods); TSF_FREE(hydra.igens); TSF_FREE(hydra.shdrs); + TSF_FREE(rawBuffer); TSF_FREE(floatBuffer); + return res; +} + +TSFDEF tsf* tsf_copy(tsf* f) +{ + tsf* res; + if (!f) return TSF_NULL; + if (!f->refCount) + { + f->refCount = (int*)TSF_MALLOC(sizeof(int)); + if (!f->refCount) return TSF_NULL; + *f->refCount = 1; + } + res = (tsf*)TSF_MALLOC(sizeof(tsf)); + if (!res) return TSF_NULL; + TSF_MEMCPY(res, f, sizeof(tsf)); + res->voices = TSF_NULL; + res->voiceNum = 0; + res->channels = TSF_NULL; + (*res->refCount)++; + return res; +} + +TSFDEF void tsf_close(tsf* f) +{ + if (!f) return; + if (!f->refCount || !--(*f->refCount)) + { + struct tsf_preset *preset = f->presets, *presetEnd = preset + f->presetNum; + for (; preset != presetEnd; preset++) TSF_FREE(preset->regions); + TSF_FREE(f->presets); + TSF_FREE(f->fontSamples); + TSF_FREE(f->refCount); + } + TSF_FREE(f->channels); + TSF_FREE(f->voices); + TSF_FREE(f); +} + +TSFDEF void tsf_reset(tsf* f) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) + if (v->playingPreset != -1 && (v->ampenv.segment < TSF_SEGMENT_RELEASE || v->ampenv.parameters.release)) + tsf_voice_endquick(f, v); + if (f->channels) { TSF_FREE(f->channels); f->channels = TSF_NULL; } +} + +TSFDEF int tsf_get_presetindex(const tsf* f, int bank, int preset_number) +{ + const struct tsf_preset *presets; + int i, iMax; + for (presets = f->presets, i = 0, iMax = f->presetNum; i < iMax; i++) + if (presets[i].preset == preset_number && presets[i].bank == bank) + return i; + return -1; +} + +TSFDEF int tsf_get_presetcount(const tsf* f) +{ + return f->presetNum; +} + +TSFDEF const char* tsf_get_presetname(const tsf* f, int preset) +{ + return (preset < 0 || preset >= f->presetNum ? TSF_NULL : f->presets[preset].presetName); +} + +TSFDEF const char* tsf_bank_get_presetname(const tsf* f, int bank, int preset_number) +{ + return tsf_get_presetname(f, tsf_get_presetindex(f, bank, preset_number)); +} + +TSFDEF void tsf_set_output(tsf* f, enum TSFOutputMode outputmode, int samplerate, float global_gain_db) +{ + f->outputmode = outputmode; + f->outSampleRate = (float)(samplerate >= 1 ? samplerate : 44100.0f); + f->globalGainDB = global_gain_db; +} + +TSFDEF void tsf_set_volume(tsf* f, float global_volume) +{ + f->globalGainDB = (global_volume == 1.0f ? 0 : -tsf_gainToDecibels(1.0f / global_volume)); +} + +TSFDEF int tsf_set_max_voices(tsf* f, int max_voices) +{ + int i = f->voiceNum; + int newVoiceNum = (f->voiceNum > max_voices ? f->voiceNum : max_voices); + struct tsf_voice *newVoices = (struct tsf_voice*)TSF_REALLOC(f->voices, newVoiceNum * sizeof(struct tsf_voice)); + if (!newVoices) return 0; + f->voices = newVoices; + f->voiceNum = f->maxVoiceNum = newVoiceNum; + for (; i < max_voices; i++) + f->voices[i].playingPreset = -1; + return 1; +} + +TSFDEF int tsf_note_on(tsf* f, int preset_index, int key, float vel) +{ + short midiVelocity = (short)(vel * 127); + int voicePlayIndex; + struct tsf_region *region, *regionEnd; + + if (preset_index < 0 || preset_index >= f->presetNum) return 1; + if (vel <= 0.0f) { tsf_note_off(f, preset_index, key); return 1; } + + // Play all matching regions. + voicePlayIndex = f->voicePlayIndex++; + for (region = f->presets[preset_index].regions, regionEnd = region + f->presets[preset_index].regionNum; region != regionEnd; region++) + { + struct tsf_voice *voice, *v, *vEnd; TSF_BOOL doLoop; float lowpassFilterQDB, lowpassFc; + if (key < region->lokey || key > region->hikey || midiVelocity < region->lovel || midiVelocity > region->hivel) continue; + + voice = TSF_NULL, v = f->voices, vEnd = v + f->voiceNum; + if (region->group) + { + for (; v != vEnd; v++) + if (v->playingPreset == preset_index && v->region->group == region->group) tsf_voice_endquick(f, v); + else if (v->playingPreset == -1 && !voice) voice = v; + } + else for (; v != vEnd; v++) if (v->playingPreset == -1) { voice = v; break; } + + if (!voice) + { + if (f->maxVoiceNum) + { + // Voices have been pre-allocated and limited to a maximum, try to kill a voice off in its release envelope + int bestKillReleaseSamplePos = -999999999; + for (v = f->voices; v != vEnd; v++) + { + if (v->ampenv.segment == TSF_SEGMENT_RELEASE) + { + // We're looking for the voice furthest into its release + int releaseSamplesDone = tsf_voice_envelope_release_samples(&v->ampenv, f->outSampleRate) - v->ampenv.samplesUntilNextSegment; + if (releaseSamplesDone > bestKillReleaseSamplePos) + { + bestKillReleaseSamplePos = releaseSamplesDone; + voice = v; + } + } + } + if (!voice) + continue; + tsf_voice_kill(voice); + } + else + { + // Allocate more voices so we don't need to kill one off. + struct tsf_voice* newVoices; + f->voiceNum += 4; + newVoices = (struct tsf_voice*)TSF_REALLOC(f->voices, f->voiceNum * sizeof(struct tsf_voice)); + if (!newVoices) return 0; + f->voices = newVoices; + voice = &f->voices[f->voiceNum - 4]; + voice[1].playingPreset = voice[2].playingPreset = voice[3].playingPreset = -1; + } + } + + voice->region = region; + voice->playingPreset = preset_index; + voice->playingKey = key; + voice->playIndex = voicePlayIndex; + voice->noteGainDB = f->globalGainDB - region->attenuation - tsf_gainToDecibels(1.0f / vel); + + if (f->channels) + { + f->channels->setupVoice(f, voice); + } + else + { + tsf_voice_calcpitchratio(voice, 0, f->outSampleRate); + // The SFZ spec is silent about the pan curve, but a 3dB pan law seems common. This sqrt() curve matches what Dimension LE does; Alchemy Free seems closer to sin(adjustedPan * pi/2). + voice->panFactorLeft = TSF_SQRTF(0.5f - region->pan); + voice->panFactorRight = TSF_SQRTF(0.5f + region->pan); + } + + // Offset/end. + voice->sourceSamplePosition = region->offset; + + // Loop. + doLoop = (region->loop_mode != TSF_LOOPMODE_NONE && region->loop_start < region->loop_end); + voice->loopStart = (doLoop ? region->loop_start : 0); + voice->loopEnd = (doLoop ? region->loop_end : 0); + + // Setup envelopes. + tsf_voice_envelope_setup(&voice->ampenv, ®ion->ampenv, key, midiVelocity, TSF_TRUE, f->outSampleRate); + tsf_voice_envelope_setup(&voice->modenv, ®ion->modenv, key, midiVelocity, TSF_FALSE, f->outSampleRate); + + // Setup lowpass filter. + lowpassFc = (region->initialFilterFc <= 13500 ? tsf_cents2Hertz((float)region->initialFilterFc) / f->outSampleRate : 1.0f); + lowpassFilterQDB = region->initialFilterQ / 10.0f; + voice->lowpass.QInv = 1.0 / TSF_POW(10.0, (lowpassFilterQDB / 20.0)); + voice->lowpass.z1 = voice->lowpass.z2 = 0; + voice->lowpass.active = (lowpassFc < 0.499f); + if (voice->lowpass.active) tsf_voice_lowpass_setup(&voice->lowpass, lowpassFc); + + // Setup LFO filters. + tsf_voice_lfo_setup(&voice->modlfo, region->delayModLFO, region->freqModLFO, f->outSampleRate); + tsf_voice_lfo_setup(&voice->viblfo, region->delayVibLFO, region->freqVibLFO, f->outSampleRate); + } + return 1; +} + +TSFDEF int tsf_bank_note_on(tsf* f, int bank, int preset_number, int key, float vel) +{ + int preset_index = tsf_get_presetindex(f, bank, preset_number); + if (preset_index == -1) return 0; + return tsf_note_on(f, preset_index, key, vel); +} + +TSFDEF void tsf_note_off(tsf* f, int preset_index, int key) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum, *vMatchFirst = TSF_NULL, *vMatchLast = TSF_NULL; + for (; v != vEnd; v++) + { + //Find the first and last entry in the voices list with matching preset, key and look up the smallest play index + if (v->playingPreset != preset_index || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE) continue; + else if (!vMatchFirst || v->playIndex < vMatchFirst->playIndex) vMatchFirst = vMatchLast = v; + else if (v->playIndex == vMatchFirst->playIndex) vMatchLast = v; + } + if (!vMatchFirst) return; + for (v = vMatchFirst; v <= vMatchLast; v++) + { + //Stop all voices with matching preset, key and the smallest play index which was enumerated above + if (v != vMatchFirst && v != vMatchLast && + (v->playIndex != vMatchFirst->playIndex || v->playingPreset != preset_index || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE)) continue; + tsf_voice_end(f, v); + } +} + +TSFDEF int tsf_bank_note_off(tsf* f, int bank, int preset_number, int key) +{ + int preset_index = tsf_get_presetindex(f, bank, preset_number); + if (preset_index == -1) return 0; + tsf_note_off(f, preset_index, key); + return 1; +} + +TSFDEF void tsf_note_off_all(tsf* f) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) if (v->playingPreset != -1 && v->ampenv.segment < TSF_SEGMENT_RELEASE) + tsf_voice_end(f, v); +} + +TSFDEF int tsf_active_voice_count(tsf* f) +{ + int count = 0; + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) if (v->playingPreset != -1) count++; + return count; +} + +TSFDEF void tsf_render_short(tsf* f, short* buffer, int samples, int flag_mixing) +{ + float outputSamples[TSF_RENDER_SHORTBUFFERBLOCK]; + int channels = (f->outputmode == TSF_MONO ? 1 : 2), maxChannelSamples = TSF_RENDER_SHORTBUFFERBLOCK / channels; + while (samples > 0) + { + int channelSamples = (samples > maxChannelSamples ? maxChannelSamples : samples); + short* bufferEnd = buffer + channelSamples * channels; + float *floatSamples = outputSamples; + tsf_render_float(f, floatSamples, channelSamples, TSF_FALSE); + samples -= channelSamples; + + if (flag_mixing) + while (buffer != bufferEnd) + { + float v = *floatSamples++; + int vi = *buffer + (v < -1.00004566f ? (int)-32768 : (v > 1.00001514f ? (int)32767 : (int)(v * 32767.5f))); + *buffer++ = (vi < -32768 ? (short)-32768 : (vi > 32767 ? (short)32767 : (short)vi)); + } + else + while (buffer != bufferEnd) + { + float v = *floatSamples++; + *buffer++ = (v < -1.00004566f ? (short)-32768 : (v > 1.00001514f ? (short)32767 : (short)(v * 32767.5f))); + } + } +} + +TSFDEF void tsf_render_float(tsf* f, float* buffer, int samples, int flag_mixing) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + if (!flag_mixing) TSF_MEMSET(buffer, 0, (f->outputmode == TSF_MONO ? 1 : 2) * sizeof(float) * samples); + for (; v != vEnd; v++) + if (v->playingPreset != -1) + tsf_voice_render(f, v, buffer, samples); +} + +static void tsf_channel_setup_voice(tsf* f, struct tsf_voice* v) +{ + struct tsf_channel* c = &f->channels->channels[f->channels->activeChannel]; + float newpan = v->region->pan + c->panOffset; + v->playingChannel = f->channels->activeChannel; + v->noteGainDB += c->gainDB; + tsf_voice_calcpitchratio(v, (c->pitchWheel == 8192 ? c->tuning : ((c->pitchWheel / 16383.0f * c->pitchRange * 2.0f) - c->pitchRange + c->tuning)), f->outSampleRate); + if (newpan <= -0.5f) { v->panFactorLeft = 1.0f; v->panFactorRight = 0.0f; } + else if (newpan >= 0.5f) { v->panFactorLeft = 0.0f; v->panFactorRight = 1.0f; } + else { v->panFactorLeft = TSF_SQRTF(0.5f - newpan); v->panFactorRight = TSF_SQRTF(0.5f + newpan); } +} + +static struct tsf_channel* tsf_channel_init(tsf* f, int channel) +{ + int i; + if (f->channels && channel < f->channels->channelNum) return &f->channels->channels[channel]; + if (!f->channels) + { + f->channels = (struct tsf_channels*)TSF_MALLOC(sizeof(struct tsf_channels) + sizeof(struct tsf_channel) * channel); + if (!f->channels) return TSF_NULL; + f->channels->setupVoice = &tsf_channel_setup_voice; + f->channels->channelNum = 0; + f->channels->activeChannel = 0; + } + else + { + struct tsf_channels *newChannels = (struct tsf_channels*)TSF_REALLOC(f->channels, sizeof(struct tsf_channels) + sizeof(struct tsf_channel) * channel); + if (!newChannels) return TSF_NULL; + f->channels = newChannels; + } + i = f->channels->channelNum; + f->channels->channelNum = channel + 1; + for (; i <= channel; i++) + { + struct tsf_channel* c = &f->channels->channels[i]; + c->presetIndex = c->bank = 0; + c->pitchWheel = c->midiPan = 8192; + c->midiVolume = c->midiExpression = 16383; + c->midiRPN = 0xFFFF; + c->midiData = 0; + c->panOffset = 0.0f; + c->gainDB = 0.0f; + c->pitchRange = 2.0f; + c->tuning = 0.0f; + } + return &f->channels->channels[channel]; +} + +static void tsf_channel_applypitch(tsf* f, int channel, struct tsf_channel* c) +{ + struct tsf_voice *v, *vEnd; + float pitchShift = (c->pitchWheel == 8192 ? c->tuning : ((c->pitchWheel / 16383.0f * c->pitchRange * 2.0f) - c->pitchRange + c->tuning)); + for (v = f->voices, vEnd = v + f->voiceNum; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel) + tsf_voice_calcpitchratio(v, pitchShift, f->outSampleRate); +} + +TSFDEF int tsf_channel_set_presetindex(tsf* f, int channel, int preset_index) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + c->presetIndex = (unsigned short)preset_index; + return 1; +} + +TSFDEF int tsf_channel_set_presetnumber(tsf* f, int channel, int preset_number, int flag_mididrums) +{ + int preset_index; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (flag_mididrums) + { + preset_index = tsf_get_presetindex(f, 128 | (c->bank & 0x7FFF), preset_number); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, 128, preset_number); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, 128, 0); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, (c->bank & 0x7FFF), preset_number); + } + else preset_index = tsf_get_presetindex(f, (c->bank & 0x7FFF), preset_number); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, 0, preset_number); + if (preset_index != -1) + { + c->presetIndex = (unsigned short)preset_index; + return 1; + } + return 0; +} + +TSFDEF int tsf_channel_set_bank(tsf* f, int channel, int bank) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + c->bank = (unsigned short)bank; + return 1; +} + +TSFDEF int tsf_channel_set_bank_preset(tsf* f, int channel, int bank, int preset_number) +{ + int preset_index; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + preset_index = tsf_get_presetindex(f, bank, preset_number); + if (preset_index == -1) return 0; + c->presetIndex = (unsigned short)preset_index; + c->bank = (unsigned short)bank; + return 1; +} + +TSFDEF int tsf_channel_set_pan(tsf* f, int channel, float pan) +{ + struct tsf_voice *v, *vEnd; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + for (v = f->voices, vEnd = v + f->voiceNum; v != vEnd; v++) + if (v->playingChannel == channel && v->playingPreset != -1) + { + float newpan = v->region->pan + pan - 0.5f; + if (newpan <= -0.5f) { v->panFactorLeft = 1.0f; v->panFactorRight = 0.0f; } + else if (newpan >= 0.5f) { v->panFactorLeft = 0.0f; v->panFactorRight = 1.0f; } + else { v->panFactorLeft = TSF_SQRTF(0.5f - newpan); v->panFactorRight = TSF_SQRTF(0.5f + newpan); } + } + c->panOffset = pan - 0.5f; + return 1; +} + +TSFDEF int tsf_channel_set_volume(tsf* f, int channel, float volume) +{ + float gainDB = tsf_gainToDecibels(volume), gainDBChange; + struct tsf_voice *v, *vEnd; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (gainDB == c->gainDB) return 1; + for (v = f->voices, vEnd = v + f->voiceNum, gainDBChange = gainDB - c->gainDB; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel) + v->noteGainDB += gainDBChange; + c->gainDB = gainDB; + return 1; +} + +TSFDEF int tsf_channel_set_pitchwheel(tsf* f, int channel, int pitch_wheel) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (c->pitchWheel == pitch_wheel) return 1; + c->pitchWheel = (unsigned short)pitch_wheel; + tsf_channel_applypitch(f, channel, c); + return 1; +} + +TSFDEF int tsf_channel_set_pitchrange(tsf* f, int channel, float pitch_range) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (c->pitchRange == pitch_range) return 1; + c->pitchRange = pitch_range; + if (c->pitchWheel != 8192) tsf_channel_applypitch(f, channel, c); + return 1; +} + +TSFDEF int tsf_channel_set_tuning(tsf* f, int channel, float tuning) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (c->tuning == tuning) return 1; + c->tuning = tuning; + tsf_channel_applypitch(f, channel, c); + return 1; +} + +TSFDEF int tsf_channel_note_on(tsf* f, int channel, int key, float vel) +{ + if (!f->channels || channel >= f->channels->channelNum) return 1; + f->channels->activeChannel = channel; + return tsf_note_on(f, f->channels->channels[channel].presetIndex, key, vel); +} + +TSFDEF void tsf_channel_note_off(tsf* f, int channel, int key) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum, *vMatchFirst = TSF_NULL, *vMatchLast = TSF_NULL; + for (; v != vEnd; v++) + { + //Find the first and last entry in the voices list with matching channel, key and look up the smallest play index + if (v->playingPreset == -1 || v->playingChannel != channel || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE) continue; + else if (!vMatchFirst || v->playIndex < vMatchFirst->playIndex) vMatchFirst = vMatchLast = v; + else if (v->playIndex == vMatchFirst->playIndex) vMatchLast = v; + } + if (!vMatchFirst) return; + for (v = vMatchFirst; v <= vMatchLast; v++) + { + //Stop all voices with matching channel, key and the smallest play index which was enumerated above + if (v != vMatchFirst && v != vMatchLast && + (v->playIndex != vMatchFirst->playIndex || v->playingPreset == -1 || v->playingChannel != channel || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE)) continue; + tsf_voice_end(f, v); + } +} + +TSFDEF void tsf_channel_note_off_all(tsf* f, int channel) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel && v->ampenv.segment < TSF_SEGMENT_RELEASE) + tsf_voice_end(f, v); +} + +TSFDEF void tsf_channel_sounds_off_all(tsf* f, int channel) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel && (v->ampenv.segment < TSF_SEGMENT_RELEASE || v->ampenv.parameters.release)) + tsf_voice_endquick(f, v); +} + +TSFDEF int tsf_channel_midi_control(tsf* f, int channel, int controller, int control_value) +{ + struct tsf_channel* c = tsf_channel_init(f, channel); + if (!c) return 0; + switch (controller) + { + case 7 /*VOLUME_MSB*/ : c->midiVolume = (unsigned short)((c->midiVolume & 0x7F ) | (control_value << 7)); goto TCMC_SET_VOLUME; + case 39 /*VOLUME_LSB*/ : c->midiVolume = (unsigned short)((c->midiVolume & 0x3F80) | control_value); goto TCMC_SET_VOLUME; + case 11 /*EXPRESSION_MSB*/ : c->midiExpression = (unsigned short)((c->midiExpression & 0x7F ) | (control_value << 7)); goto TCMC_SET_VOLUME; + case 43 /*EXPRESSION_LSB*/ : c->midiExpression = (unsigned short)((c->midiExpression & 0x3F80) | control_value); goto TCMC_SET_VOLUME; + case 10 /*PAN_MSB*/ : c->midiPan = (unsigned short)((c->midiPan & 0x7F ) | (control_value << 7)); goto TCMC_SET_PAN; + case 42 /*PAN_LSB*/ : c->midiPan = (unsigned short)((c->midiPan & 0x3F80) | control_value); goto TCMC_SET_PAN; + case 6 /*DATA_ENTRY_MSB*/ : c->midiData = (unsigned short)((c->midiData & 0x7F) | (control_value << 7)); goto TCMC_SET_DATA; + case 38 /*DATA_ENTRY_LSB*/ : c->midiData = (unsigned short)((c->midiData & 0x3F80) | control_value); goto TCMC_SET_DATA; + case 0 /*BANK_SELECT_MSB*/ : c->bank = (unsigned short)(0x8000 | control_value); return 1; //bank select MSB alone acts like LSB + case 32 /*BANK_SELECT_LSB*/ : c->bank = (unsigned short)((c->bank & 0x8000 ? ((c->bank & 0x7F) << 7) : 0) | control_value); return 1; + case 101 /*RPN_MSB*/ : c->midiRPN = (unsigned short)(((c->midiRPN == 0xFFFF ? 0 : c->midiRPN) & 0x7F ) | (control_value << 7)); return 1; + case 100 /*RPN_LSB*/ : c->midiRPN = (unsigned short)(((c->midiRPN == 0xFFFF ? 0 : c->midiRPN) & 0x3F80) | control_value); return 1; + case 98 /*NRPN_LSB*/ : c->midiRPN = 0xFFFF; return 1; + case 99 /*NRPN_MSB*/ : c->midiRPN = 0xFFFF; return 1; + case 120 /*ALL_SOUND_OFF*/ : tsf_channel_sounds_off_all(f, channel); return 1; + case 123 /*ALL_NOTES_OFF*/ : tsf_channel_note_off_all(f, channel); return 1; + case 121 /*ALL_CTRL_OFF*/ : + c->midiVolume = c->midiExpression = 16383; + c->midiPan = 8192; + c->bank = 0; + c->midiRPN = 0xFFFF; + c->midiData = 0; + tsf_channel_set_volume(f, channel, 1.0f); + tsf_channel_set_pan(f, channel, 0.5f); + tsf_channel_set_pitchrange(f, channel, 2.0f); + tsf_channel_set_tuning(f, channel, 0); + return 1; + } + return 1; +TCMC_SET_VOLUME: + //Raising to the power of 3 seems to result in a decent sounding volume curve for MIDI + tsf_channel_set_volume(f, channel, TSF_POWF((c->midiVolume / 16383.0f) * (c->midiExpression / 16383.0f), 3.0f)); + return 1; +TCMC_SET_PAN: + tsf_channel_set_pan(f, channel, c->midiPan / 16383.0f); + return 1; +TCMC_SET_DATA: + if (c->midiRPN == 0) tsf_channel_set_pitchrange(f, channel, (c->midiData >> 7) + 0.01f * (c->midiData & 0x7F)); + else if (c->midiRPN == 1) tsf_channel_set_tuning(f, channel, (int)c->tuning + ((float)c->midiData - 8192.0f) / 8192.0f); //fine tune + else if (c->midiRPN == 2 && controller == 6) tsf_channel_set_tuning(f, channel, ((float)control_value - 64.0f) + (c->tuning - (int)c->tuning)); //coarse tune + return 1; +} + +TSFDEF int tsf_channel_get_preset_index(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].presetIndex : 0); +} + +TSFDEF int tsf_channel_get_preset_bank(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? (f->channels->channels[channel].bank & 0x7FFF) : 0); +} + +TSFDEF int tsf_channel_get_preset_number(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->presets[f->channels->channels[channel].presetIndex].preset : 0); +} + +TSFDEF float tsf_channel_get_pan(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].panOffset - 0.5f : 0.5f); +} + +TSFDEF float tsf_channel_get_volume(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? tsf_decibelsToGain(f->channels->channels[channel].gainDB) : 1.0f); +} + +TSFDEF int tsf_channel_get_pitchwheel(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].pitchWheel : 8192); +} + +TSFDEF float tsf_channel_get_pitchrange(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].pitchRange : 2.0f); +} + +TSFDEF float tsf_channel_get_tuning(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].tuning : 0.0f); +} + +#ifdef __cplusplus +} +#endif + +#endif //TSF_IMPLEMENTATION diff --git a/code/user/apps/edit/edit.c b/code/user/apps/edit/edit.c index 9a4a204..7ba0191 100644 --- a/code/user/apps/edit/edit.c +++ b/code/user/apps/edit/edit.c @@ -426,7 +426,7 @@ static void load_file(void) { char buf[512]; size_t got = 0; bool truncated = false; - for (;;) { + while (true) { SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(h, buf, sizeof(buf), &got); if (SysReadFile(&readReq) != 0 || got == 0) break; for (size_t i = 0; i < got; i++) { diff --git a/code/user/apps/httpget/httpget.c b/code/user/apps/httpget/httpget.c index 663ca52..3eeed2f 100644 --- a/code/user/apps/httpget/httpget.c +++ b/code/user/apps/httpget/httpget.c @@ -279,7 +279,7 @@ int main(void) { char buf[513]; size_t total = 0; - for (;;) { + while (true) { size_t got = 0; st = SysRecvSocket(sock, buf, sizeof(buf) - 1, 0, &got); if (st != NTLSTATUS_SUCCESS) { diff --git a/code/user/apps/ping/ping.c b/code/user/apps/ping/ping.c index 5ac7f6b..9c71a46 100644 --- a/code/user/apps/ping/ping.c +++ b/code/user/apps/ping/ping.c @@ -332,7 +332,7 @@ int main(void) { bool matched = false; uint8_t lastType = 0, lastCode = 0; - for (;;) { + while (true) { uint8_t recvBuf[512]; SYS_SOCKET_ADDRESS from; size_t fromSize = sizeof(from); diff --git a/code/user/apps/shell_native/shell_native.c b/code/user/apps/shell_native/shell_native.c index 67450f0..caefe96 100644 --- a/code/user/apps/shell_native/shell_native.c +++ b/code/user/apps/shell_native/shell_native.c @@ -12,6 +12,12 @@ #define MAX_PROCS 64 // Maximum number of processes we will fetch at once #define MAX_THREADS 128 #define SHELL_PROMPT "> " +#define SHELL_GLYPH_WIDTH 8 +#define SHELL_GLYPH_HEIGHT 16 +#define SHELL_TEXT_FOREGROUND 0x00CCCCCCu +#define SHELL_TEXT_BACKGROUND 0x00012456u +#define SHELL_CURSOR_FOREGROUND 0x00000000u +#define SHELL_CURSOR_BACKGROUND 0x00E0E0E0u // Output helpers. static void print(const char* s) { SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; SysWriteConsole(&req); } @@ -192,6 +198,20 @@ static void printStringWidth(const char* s, size_t width) { static char gHistory[SHELL_HISTORY_DEPTH][MAX_CMD]; static size_t gHistoryCount; +typedef struct SHELL_EDITOR { + char* Buffer; + int Capacity; + int Length; + int Cursor; +} SHELL_EDITOR; + +typedef struct SHELL_VISIBLE_CURSOR { + int Column; + int Row; + char Character; + bool Visible; +} SHELL_VISIBLE_CURSOR; + static const char* const gShellBuiltinCommands[] = { "cd", "clear", @@ -201,6 +221,7 @@ static const char* const gShellBuiltinCommands[] = { "dir", "display", "exit", + "fbbench", "gpu", "help", "kill", @@ -211,6 +232,7 @@ static const char* const gShellBuiltinCommands[] = { "threads", "time", "type", + "volume", "write", }; @@ -280,23 +302,112 @@ static void shellHistoryAdd(const char* cmd) { gHistoryCount++; } -// Replaces the visible line while keeping the cursor at the end. -static void shellEditorReplaceLine(char* buf, int len, int* pos, const char* text) { - int oldPos = *pos; - for (int i = 0; i < oldPos; i++) print("\b"); - for (int i = 0; i < oldPos; i++) print(" "); - for (int i = 0; i < oldPos; i++) print("\b"); +// Restores the character underneath the visible edit cursor. +static void shellEditorHideVisibleCursor(SHELL_VISIBLE_CURSOR* cursor) { + if (!cursor || !cursor->Visible) return; - int newPos = 0; - if (text) { - while (text[newPos] && newPos < len - 1) { - buf[newPos] = text[newPos]; - newPos++; + (void)SysPutScreenChar(cursor->Character, + cursor->Column * SHELL_GLYPH_WIDTH, + cursor->Row * SHELL_GLYPH_HEIGHT, + SHELL_TEXT_FOREGROUND, + SHELL_TEXT_BACKGROUND); + cursor->Visible = false; +} + +// Draws the same steady block cursor used by edit without changing console state. +static void shellEditorShowVisibleCursor(SHELL_VISIBLE_CURSOR* cursor, const SHELL_EDITOR* editor) { + if (!cursor || !editor) return; + shellEditorHideVisibleCursor(cursor); + + SYS_CONSOLE_INFO info; + if (SysGetConsoleInfo(&info) != NTLSTATUS_SUCCESS) return; + + char character = editor->Cursor < editor->Length + ? editor->Buffer[editor->Cursor] + : ' '; + if (SysPutScreenChar(character, + (int)info.CursorX * SHELL_GLYPH_WIDTH, + (int)info.CursorY * SHELL_GLYPH_HEIGHT, + SHELL_CURSOR_FOREGROUND, + SHELL_CURSOR_BACKGROUND) != NTLSTATUS_SUCCESS) + return; + + cursor->Column = (int)info.CursorX; + cursor->Row = (int)info.CursorY; + cursor->Character = character; + cursor->Visible = true; +} + +// Moves the visible cursor left by the requested number of cells. +static void shellEditorMoveLeftCells(int count) { + while (count-- > 0) print("\b"); +} + +// Moves the editor cursor without changing the buffer. +static void shellEditorMoveCursor(SHELL_EDITOR* editor, int target) { + if (!editor) return; + if (target < 0) target = 0; + if (target > editor->Length) target = editor->Length; + + if (target < editor->Cursor) { + shellEditorMoveLeftCells(editor->Cursor - target); + } else { + while (editor->Cursor < target) { + char echo[2] = { editor->Buffer[editor->Cursor], '\0' }; + print(echo); + editor->Cursor++; } } - buf[newPos] = '\0'; - *pos = newPos; - print(buf); + editor->Cursor = target; +} + +// Redraws the changed tail and restores the requested cursor position. +static void shellEditorRedrawTail(SHELL_EDITOR* editor, int start, int oldLength, int targetCursor) { + if (!editor) return; + + print(editor->Buffer + start); + if (oldLength > editor->Length) printSpaces((size_t)(oldLength - editor->Length)); + + int displayedEnd = oldLength > editor->Length ? oldLength : editor->Length; + shellEditorMoveLeftCells(displayedEnd - targetCursor); + editor->Cursor = targetCursor; +} + +// Replaces one buffer range and keeps the visible line synchronized. +static bool shellEditorReplaceRange(SHELL_EDITOR* editor, int start, int end, const char* text) { + if (!editor || !editor->Buffer || editor->Capacity <= 0 || !text) return false; + if (start < 0 || end < start || end > editor->Length) return false; + + shellEditorMoveCursor(editor, start); + + int oldLength = editor->Length; + int removed = end - start; + int available = editor->Capacity - 1 - (oldLength - removed); + int replacementLength = 0; + while (text[replacementLength] && replacementLength < available) replacementLength++; + + memmove(editor->Buffer + start + replacementLength, + editor->Buffer + end, + (size_t)(oldLength - end + 1)); + if (replacementLength > 0) + memcpy(editor->Buffer + start, text, (size_t)replacementLength); + + editor->Length = oldLength - removed + replacementLength; + int targetCursor = start + replacementLength; + shellEditorRedrawTail(editor, start, oldLength, targetCursor); + return removed != replacementLength || replacementLength != 0; +} + +// Inserts text at the current cursor position. +static bool shellEditorInsert(SHELL_EDITOR* editor, const char* text) { + if (!editor) return false; + return shellEditorReplaceRange(editor, editor->Cursor, editor->Cursor, text); +} + +// Replaces the complete visible line and leaves the cursor at its end. +static void shellEditorReplaceLine(SHELL_EDITOR* editor, const char* text) { + if (!editor) return; + (void)shellEditorReplaceRange(editor, 0, editor->Length, text ? text : ""); } // Returns true when a character separates shell tokens. @@ -340,23 +451,6 @@ static void shellUpdateCommonPrefix(char* common, size_t commonSize, const char* common[i] = '\0'; } -// Appends text to the edited line and echoes it. -static bool shellEditorAppend(char* buf, int len, int* pos, const char* text) { - if (!buf || len <= 0 || !pos || !text) return false; - - bool changed = false; - while (*text && *pos < len - 1) { - buf[*pos] = *text; - (*pos)++; - buf[*pos] = '\0'; - char echo[2] = { *text, '\0' }; - print(echo); - text++; - changed = true; - } - return changed; -} - // Prints one completion candidate. static void shellPrintCompletionCandidate(const char* name, bool directory) { print(" "); @@ -454,18 +548,23 @@ static void shellCollectPathCompletions(const char* token, size_t tokenLength, } // Redraws the prompt and current edit buffer after a completion list. -static void shellRedrawPromptAndLine(const char* buf) { +static void shellRedrawPromptAndLine(SHELL_EDITOR* editor) { print(SHELL_PROMPT); - print(buf ? buf : ""); + if (!editor) return; + print(editor->Buffer); + shellEditorMoveLeftCells(editor->Length - editor->Cursor); } -// Completes the token at the end of the current line. -static void shellCompleteLine(char* buf, int len, int* pos) { - if (!buf || len <= 0 || !pos || *pos < 0) return; +// Completes the token around the current cursor position. +static void shellCompleteLine(SHELL_EDITOR* editor) { + if (!editor || !editor->Buffer || editor->Capacity <= 0 || editor->Cursor < 0) return; - size_t cursor = (size_t)*pos; + char* buf = editor->Buffer; + size_t cursor = (size_t)editor->Cursor; size_t tokenStart = cursor; while (tokenStart > 0 && !shellIsTokenSeparator(buf[tokenStart - 1])) tokenStart--; + size_t tokenEnd = cursor; + while (tokenEnd < (size_t)editor->Length && !shellIsTokenSeparator(buf[tokenEnd])) tokenEnd++; const char* token = buf + tokenStart; size_t tokenLength = cursor - tokenStart; @@ -490,14 +589,20 @@ static void shellCompleteLine(char* buf, int len, int* pos) { size_t commonLength = strlen(state.Common); bool expanded = commonLength > namePrefixLength; if (expanded) { - shellEditorAppend(buf, len, pos, state.Common + namePrefixLength); + (void)shellEditorReplaceRange(editor, editor->Cursor, (int)tokenEnd, + state.Common + namePrefixLength); } if (state.MatchCount == 1) { if (state.LastIsDirectory) { - if (*pos < len - 1 && *pos > 0 && !shellIsPathSeparator(buf[*pos - 1])) shellEditorAppend(buf, len, pos, "/"); + if (editor->Cursor < editor->Capacity - 1 && editor->Cursor > 0 && + !shellIsPathSeparator(buf[editor->Cursor - 1])) + (void)shellEditorInsert(editor, "/"); } else if (state.LastAddsSpace) { - if (*pos < len - 1 && (*pos == 0 || buf[*pos - 1] != ' ')) shellEditorAppend(buf, len, pos, " "); + if (editor->Cursor < editor->Capacity - 1 && + (editor->Cursor == editor->Length || + !shellIsTokenSeparator(buf[editor->Cursor]))) + (void)shellEditorInsert(editor, " "); } return; } @@ -509,17 +614,27 @@ static void shellCompleteLine(char* buf, int len, int* pos) { shellCollectBuiltinCompletions(token, tokenLength, NULL, true); } shellCollectPathCompletions(token, tokenLength, NULL, true); - shellRedrawPromptAndLine(buf); + shellRedrawPromptAndLine(editor); } // Reads one editable command line from the keyboard. static bool readLine(char* buf, int len) { - int pos = 0; + SHELL_EDITOR editor; + SHELL_VISIBLE_CURSOR visibleCursor; + char historyDraft[MAX_CMD]; size_t historyCursor = gHistoryCount; if (len <= 0) return false; + + editor.Buffer = buf; + editor.Capacity = len; + editor.Length = 0; + editor.Cursor = 0; + memset(&visibleCursor, 0, sizeof(visibleCursor)); buf[0] = '\0'; + historyDraft[0] = '\0'; while (true) { + shellEditorShowVisibleCursor(&visibleCursor, &editor); INPUT_KEY_RECORD input; SYS_INPUT_EVENT_REQUEST inputReq; inputReq.Size = sizeof(inputReq); @@ -527,7 +642,9 @@ static bool readLine(char* buf, int len) { inputReq.Flags = SYS_INPUT_EVENT_FLAG_NONE; inputReq.TimeoutMilliseconds = SYS_WAIT_TIMEOUT_INFINITE; inputReq.RecordOut = &input; - if (SysReadInputEvent(&inputReq) != NTLSTATUS_SUCCESS) { SysYield(); continue; } + KSTATUS inputStatus = SysReadInputEvent(&inputReq); + shellEditorHideVisibleCursor(&visibleCursor); + if (inputStatus != NTLSTATUS_SUCCESS) { SysYield(); continue; } INPUT_KEY key = input.Key; if (key == INPUT_KEY_NONE) { SysYield(); continue; } @@ -545,18 +662,45 @@ static bool readLine(char* buf, int len) { } if (key == INPUT_KEY_BACKSPACE) { - if (pos > 0) { - pos--; - buf[pos] = '\0'; - print("\b \b"); - } + if (editor.Cursor > 0) + (void)shellEditorReplaceRange(&editor, editor.Cursor - 1, editor.Cursor, ""); + continue; + } + + if (key == INPUT_KEY_DELETE) { + if (editor.Cursor < editor.Length) + (void)shellEditorReplaceRange(&editor, editor.Cursor, editor.Cursor + 1, ""); + continue; + } + + if (key == INPUT_KEY_LEFT) { + shellEditorMoveCursor(&editor, editor.Cursor - 1); + continue; + } + + if (key == INPUT_KEY_RIGHT) { + shellEditorMoveCursor(&editor, editor.Cursor + 1); + continue; + } + + if (key == INPUT_KEY_HOME) { + shellEditorMoveCursor(&editor, 0); + continue; + } + + if (key == INPUT_KEY_END) { + shellEditorMoveCursor(&editor, editor.Length); continue; } if (key == INPUT_KEY_UP) { if (historyCursor > 0) { + if (historyCursor == gHistoryCount) { + strncpy(historyDraft, editor.Buffer, sizeof(historyDraft) - 1); + historyDraft[sizeof(historyDraft) - 1] = '\0'; + } historyCursor--; - shellEditorReplaceLine(buf, len, &pos, gHistory[historyCursor]); + shellEditorReplaceLine(&editor, gHistory[historyCursor]); } continue; } @@ -564,25 +708,21 @@ static bool readLine(char* buf, int len) { if (key == INPUT_KEY_DOWN) { if (historyCursor < gHistoryCount) { historyCursor++; - shellEditorReplaceLine(buf, len, &pos, - historyCursor < gHistoryCount ? gHistory[historyCursor] : ""); + shellEditorReplaceLine(&editor, + historyCursor < gHistoryCount ? gHistory[historyCursor] : historyDraft); } continue; } if (key == INPUT_KEY_TAB) { - shellCompleteLine(buf, len, &pos); + shellCompleteLine(&editor); continue; } char c = input.Char ? input.Char : shellKeyToChar(key, modifiers); if (c < 0x20 || c == 127) continue; - if (pos < len - 1) { - buf[pos++] = c; - buf[pos] = '\0'; - char echo[2] = { c, '\0' }; - print(echo); - } + char text[2] = { c, '\0' }; + (void)shellEditorInsert(&editor, text); } return false; @@ -604,6 +744,7 @@ static void cmd_help() { " mem - Show memory usage\n" " time - Show system wall-clock time\n" " display - Show display provider and counters\n" + " fbbench - Benchmark Doom-size and full-screen presents\n" " stats - Show scheduler/CPU/MM telemetry\n" " ps - List processes\n" " threads - Show one process's thread statuses\n" @@ -611,6 +752,9 @@ static void cmd_help() { " dir [path] - List directory contents\n" " cd - Change directory\n" " type - Display file contents\n" + " volume [0-100] - Query or set master audio volume\n" + " volume mute - Mute the default audio endpoint\n" + " volume unmute - Unmute the default audio endpoint\n" " del - Delete a file\n" " write - Create a file with text\n" " clear - Clear the screen\n" @@ -639,6 +783,117 @@ static void printCpuFlags(uint32_t flags) { if (flags & SYS_CPU_FLAG_PRESENT) print("present "); } +// Selects the available default audio endpoint, or the first available endpoint. +static bool shellQueryDefaultAudioEndpoint(SYS_AUDIO_ENDPOINT_INFO* endpointOut) { + if (!endpointOut) return false; + + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + request.CountOut = &count; + + KSTATUS status = SysAudioEnumerateEndpoints(&request); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_BUFFER_TOO_SMALL) { + print("volume: endpoint query failed status="); + printUInt(status); + print("\n"); + return false; + } + + const SYS_AUDIO_ENDPOINT_INFO* fallback = NULL; + uint32_t availableCount = count < SYS_AUDIO_MAX_ENDPOINTS + ? count + : SYS_AUDIO_MAX_ENDPOINTS; + for (uint32_t index = 0; index < availableCount; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->Size != sizeof(*endpoint) || + endpoint->Version != SYS_AUDIO_ABI_VERSION || + endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) fallback = endpoint; + if (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT) { + *endpointOut = *endpoint; + return true; + } + } + + if (fallback) { + *endpointOut = *fallback; + return true; + } + + print("volume: no available audio endpoint\n"); + return false; +} + +// Converts one public Q16 gain into a rounded percentage. +static uint32_t shellAudioVolumePercent(uint32_t volumeQ16) { + uint64_t scaled = (uint64_t)volumeQ16 * 100u + SYS_AUDIO_GAIN_Q16_UNITY / 2u; + return (uint32_t)(scaled / SYS_AUDIO_GAIN_Q16_UNITY); +} + +// Converts one percentage into public Q16 gain. +static uint32_t shellAudioVolumeQ16(uint32_t percent) { + return (uint32_t)(((uint64_t)percent * SYS_AUDIO_GAIN_Q16_UNITY + 50u) / 100u); +} + +// Queries or updates the default endpoint master volume. +static void cmd_volume(const char* args) { + SYS_AUDIO_ENDPOINT_INFO endpoint; + if (!shellQueryDefaultAudioEndpoint(&endpoint)) return; + + while (args && (*args == ' ' || *args == '\t')) args++; + if (!args || *args == '\0') { + print("Volume: "); + printUInt(shellAudioVolumePercent(endpoint.MasterVolumeQ16)); + print("%"); + if (endpoint.MasterMuted) print(" (muted)"); + print("\n"); + return; + } + + uint32_t volumeQ16 = endpoint.MasterVolumeQ16; + uint32_t muted = endpoint.MasterMuted ? 1u : 0u; + if (strcmp(args, "mute") == 0) { + muted = 1u; + } else if (strcmp(args, "unmute") == 0) { + muted = 0u; + } else { + uint32_t percent = 0; + if (!parseUInt(args, &percent) || percent > 100u) { + print("Usage: volume [0-100|mute|unmute]\n"); + return; + } + volumeQ16 = shellAudioVolumeQ16(percent); + muted = 0u; + } + + SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.EndpointId = endpoint.EndpointId; + request.Muted = muted; + request.VolumeQ16 = volumeQ16; + KSTATUS status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_SUCCESS) { + print("volume: update failed status="); + printUInt(status); + print("\n"); + return; + } + + print("Volume: "); + printUInt(shellAudioVolumePercent(volumeQ16)); + print("%"); + if (muted) print(" (muted)"); + print("\n"); +} + // Implements the cmd stats operation. static void cmd_stats(void) { SYS_SCHEDULER_STATS sched; @@ -801,6 +1056,12 @@ static void cmd_stats(void) { print(" voluntary blocks: "); printUInt64(sched.VoluntaryBlocks); print("\n"); print(" SMT deferrals: "); printUInt64(sched.SmtSiblingDeferrals); print("\n"); print(" SMT primary pokes: "); printUInt64(sched.SmtPrimaryPokes); print("\n"); + print(" RT dispatches: "); printUInt64(sched.RealtimeDispatches); print("\n"); + print(" RT preemptions: "); printUInt64(sched.RealtimePreemptions); print("\n"); + print(" PI boosts: "); printUInt64(sched.PriorityInheritanceBoosts); print("\n"); + print(" PI drops: "); printUInt64(sched.PriorityInheritanceDrops); print("\n"); + print(" PI propagations: "); printUInt64(sched.PriorityInheritancePropagations); print("\n"); + print(" PI max depth: "); printUInt64(sched.PriorityInheritanceMaxDepth); print("\n"); print("Exceptions:\n"); print(" delivered: "); printUInt64(exceptionStats.Delivered); print("\n"); @@ -975,20 +1236,6 @@ static void cmd_stats(void) { print(" standby="); printUInt64(fileCache.StandbyTransitions); print(" standby-hits="); printUInt64(fileCache.StandbyHits); print(" drop-fail="); printUInt64(fileCache.DropFailures); print("\n"); - print(" read-ahead: requests="); printUInt64(fileCache.ReadAheadRequests); - print(" requested="); printUInt64(fileCache.ReadAheadPagesRequested); - print(" queued="); printUInt64(fileCache.ReadAheadPagesQueued); - print(" loaded="); printUInt64(fileCache.ReadAheadPagesLoaded); - print(" skipped="); printUInt64(fileCache.ReadAheadPagesSkipped); - print(" fail="); printUInt64(fileCache.ReadAheadFailures); - print(" throttled="); printUInt64(fileCache.ReadAheadPagesThrottled); print("\n"); - print(" read-ahead max: pages/request="); printUInt64(fileCache.MaxReadAheadPagesPerRequest); print("\n"); - print(" batch fills: reads="); printUInt64(fileCache.BatchReads); - print(" requested="); printUInt64(fileCache.BatchPagesRequested); - print(" loaded="); printUInt64(fileCache.BatchPagesLoaded); - print(" max-request="); printUInt64(fileCache.MaxBatchPagesRequested); - print(" max-loaded="); printUInt64(fileCache.MaxBatchPagesLoaded); print("\n"); - print("Filesystem:\n"); print(" path cache: lookups="); printUInt64(filesystem.PathLookups); print(" hits="); printUInt64(filesystem.PathCacheHits); @@ -1012,18 +1259,6 @@ static void cmd_stats(void) { print(" writes: requests="); printUInt64(disk.WriteRequests); print(" sectors="); printUInt64(disk.WriteSectors); print(" max-sectors="); printUInt64(disk.MaxWriteSectors); print("\n"); - print(" read-ahead: hits="); printUInt64(disk.ReadaheadHits); - print(" misses="); printUInt64(disk.ReadaheadMisses); - print(" stores="); printUInt64(disk.ReadaheadStores); - print(" sectors="); printUInt64(disk.ReadaheadSectorsLoaded); - print(" max-sectors="); printUInt64(disk.MaxReadaheadSectors); print("\n"); - print(" read-ahead misc: served-bytes="); printUInt64(disk.ReadaheadBytesServed); - print(" seq="); printUInt64(disk.ReadaheadSequentialDetections); - print(" invalidations="); printUInt64(disk.ReadaheadInvalidations); - print(" suppressed="); printUInt64(disk.ReadaheadSuppressed); - print(" alloc-fail="); printUInt64(disk.ReadaheadAllocFailures); - print(" read-fail="); printUInt64(disk.ReadaheadReadFailures); print("\n"); - print("Surfaces:\n"); print(" lifetime: created="); printUInt64(surface.CreatedSurfaces); print(" destroyed="); printUInt64(surface.DestroyedSurfaces); @@ -1123,9 +1358,11 @@ static void printDisplayStats(const char* title, const DISPLAY_STATS* st) { print(" fill: "); printUInt64(st->FillRectCount); print("\n"); print(" copy: "); printUInt64(st->CopyRectCount); print("\n"); print(" read: "); printUInt64(st->ReadRectCount); print("\n"); - print(" present: "); printUInt64(st->PresentCount); print(" rects="); printUInt64(st->PresentRectCount); print("\n"); + print(" present: "); printUInt64(st->PresentCount); print(" rects="); printUInt64(st->PresentRectCount); + print(" direct="); printUInt64(st->DirectPresentCount); print("\n"); print(" dirty: "); printUInt64(st->DirtyRectCount); print(" all="); printUInt64(st->DirtyAllCount); print("\n"); - print(" bytes: "); printUInt64(st->BytesCopied); print("\n"); + print(" bytes: logical="); printUInt64(st->BytesCopied); + print(" scanout="); printUInt64(st->ScanoutBytes); print("\n"); print(" sync: "); printUInt64(st->SyncCount); print("\n"); print(" cursor: move="); printUInt64(st->CursorMoveCount); print(" show="); printUInt64(st->CursorShowCount); @@ -1149,10 +1386,205 @@ static void cmd_display(void) { print(" mode: "); printUInt(info.Mode.Width); print("x"); printUInt(info.Mode.Height); print(" stride="); printUInt(info.Mode.Stride); print(" format="); printUInt((uint32_t)info.Mode.Format); print("\n"); print(" caps: 0x"); printHex32(info.Caps); print(" ["); printDisplayCaps(info.Caps); print("]\n"); + print(" scanout: "); + print((info.Flags & DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE) ? "PAT WC available" : "PAT WC unavailable"); + print("\n"); printDisplayStats("Manager counters", &info.ManagerStats); printDisplayStats("Provider counters", &info.DeviceStats); } +typedef struct SHELL_FB_BENCH_RESULT { + uint32_t Width; + uint32_t Height; + uint32_t Frames; + uint32_t Status; + uint64_t ElapsedTicks; + uint64_t Frequency; + uint64_t ExpectedBytes; + uint64_t ScanoutBytes; + uint64_t DirectPresents; +} SHELL_FB_BENCH_RESULT; + +// Computes a decimal fixed-point ratio with one digit after the decimal point. +static uint64_t shellRatioTenths(uint64_t numerator, uint64_t denominator) { + if (!denominator) + return 0; + if (numerator > UINT64_MAX / 10u) + return (numerator / denominator) * 10u; + return (numerator * 10u) / denominator; +} + +// Fills a benchmark surface with a deterministic high-contrast pattern. +static void shellFillBenchmarkSurface(uint32_t* pixels, + uint32_t pitchPixels, + uint32_t width, + uint32_t height) { + for (uint32_t y = 0; y < height; y++) { + uint32_t* row = pixels + (size_t)y * pitchPixels; + for (uint32_t x = 0; x < width; x++) { + uint32_t red = (x * 255u) / (width ? width : 1u); + uint32_t green = (y * 255u) / (height ? height : 1u); + uint32_t blue = ((x ^ y) & 0xFFu); + row[x] = (red << 16) | (green << 8) | blue; + } + } +} + +// Runs one synchronous surface-present benchmark case. +static void shellRunFramebufferBenchmark(uint32_t width, + uint32_t height, + uint32_t frames, + uint64_t frequency, + SHELL_FB_BENCH_RESULT* result) { + memset(result, 0, sizeof(*result)); + result->Width = width; + result->Height = height; + result->Frames = frames; + result->Frequency = frequency; + + uint32_t surface = 0; + SYS_CREATE_SURFACE_REQUEST create; + memset(&create, 0, sizeof(create)); + create.Size = sizeof(create); + create.Width = width; + create.Height = height; + create.Format = SYS_SURFACE_FORMAT_XRGB8888; + create.Flags = SYS_SURFACE_FLAG_NONE; + create.HandleOut = &surface; + + uint32_t status = SysCreateSurface(&create); + if (status != NTLSTATUS_SUCCESS || !surface) { + result->Status = status ? status : NTLSTATUS_FAILURE; + return; + } + + void* base = 0; + uint32_t pitchBytes = 0; + status = SysMapSurface(surface, &base, &pitchBytes); + if (status != NTLSTATUS_SUCCESS || !base || pitchBytes < width * sizeof(uint32_t)) { + result->Status = status ? status : NTLSTATUS_FAILURE; + SysCloseHandle(surface); + return; + } + + shellFillBenchmarkSurface((uint32_t*)base, pitchBytes / sizeof(uint32_t), width, height); + + DISPLAY_RECT rect = { 0, 0, (int32_t)width, (int32_t)height }; + for (uint32_t i = 0; i < 2; i++) { + status = SysPresentSurface(surface, &rect, 1); + if (status != NTLSTATUS_SUCCESS) + break; + } + + SYS_DISPLAY_INFO before; + SYS_DISPLAY_INFO after; + memset(&before, 0, sizeof(before)); + memset(&after, 0, sizeof(after)); + if (status == NTLSTATUS_SUCCESS) + status = SysQueryDisplayInfo(&before); + + uint64_t begin = 0; + uint64_t end = 0; + uint32_t completed = 0; + if (status == NTLSTATUS_SUCCESS) { + begin = SysGetTick(); + for (; completed < frames; completed++) { + status = SysPresentSurface(surface, &rect, 1); + if (status != NTLSTATUS_SUCCESS) + break; + } + end = SysGetTick(); + uint32_t queryStatus = SysQueryDisplayInfo(&after); + if (status == NTLSTATUS_SUCCESS && queryStatus != NTLSTATUS_SUCCESS) + status = queryStatus; + } + + result->Frames = completed; + result->Status = status; + result->ElapsedTicks = end - begin; + result->ExpectedBytes = (uint64_t)width * (uint64_t)height * sizeof(uint32_t) * completed; + if (after.DeviceStats.ScanoutBytes >= before.DeviceStats.ScanoutBytes) + result->ScanoutBytes = after.DeviceStats.ScanoutBytes - before.DeviceStats.ScanoutBytes; + if (after.DeviceStats.DirectPresentCount >= before.DeviceStats.DirectPresentCount) + result->DirectPresents = after.DeviceStats.DirectPresentCount - before.DeviceStats.DirectPresentCount; + + SysCloseHandle(surface); +} + +// Prints one framebuffer benchmark result. +static void shellPrintFramebufferBenchmark(const char* name, + const SHELL_FB_BENCH_RESULT* result) { + print(" "); print(name); print(" "); + printUInt(result->Width); print("x"); printUInt(result->Height); print(": "); + if (result->Status != NTLSTATUS_SUCCESS || !result->ElapsedTicks || !result->Frequency) { + print("failed status=0x"); printHex32(result->Status); print("\n"); + return; + } + + uint64_t elapsedUs = SysTicksToMicroseconds(result->ElapsedTicks, result->Frequency); + uint64_t fpsTenths = shellRatioTenths((uint64_t)result->Frames * result->Frequency, + result->ElapsedTicks); + uint64_t bytes = result->ScanoutBytes ? result->ScanoutBytes : result->ExpectedBytes; + uint64_t mibTenths = shellRatioTenths(bytes * result->Frequency, + result->ElapsedTicks * 1024u * 1024u); + + print("frames="); printUInt(result->Frames); + print(" time="); printUInt64(elapsedUs / 1000u); print(" ms"); + print(" fps="); printUInt64(fpsTenths / 10u); print("."); printUInt64(fpsTenths % 10u); + print(" scanout="); printUInt64(mibTenths / 10u); print("."); printUInt64(mibTenths % 10u); print(" MiB/s"); + print(" direct="); printUInt64(result->DirectPresents); + print(" bytes="); printUInt64(bytes); + print("\n"); +} + +// Benchmarks the synchronous surface-to-scanout path without console traffic. +static void cmd_fbbench(void) { + SYS_DISPLAY_INFO info; + uint32_t status = SysQueryDisplayInfo(&info); + if (status != NTLSTATUS_SUCCESS) { + print("fbbench: display query failed status=0x"); printHex32(status); print("\n"); + return; + } + + uint64_t frequency = 0; + status = SysGetTickFrequency(&frequency); + if (status != NTLSTATUS_SUCCESS || !frequency) { + print("fbbench: timer query failed status=0x"); printHex32(status); print("\n"); + return; + } + + uint32_t doomWidth = info.Mode.Width < 960u ? info.Mode.Width : 960u; + uint32_t doomHeight = info.Mode.Height < 600u ? info.Mode.Height : 600u; + uint32_t fullWidth = info.Mode.Width; + uint32_t fullHeight = info.Mode.Height; + + uint64_t targetBytes = 128ull * 1024ull * 1024ull; + uint64_t doomFrameBytes = (uint64_t)doomWidth * doomHeight * sizeof(uint32_t); + uint64_t fullFrameBytes = (uint64_t)fullWidth * fullHeight * sizeof(uint32_t); + uint32_t doomFrames = doomFrameBytes ? (uint32_t)((targetBytes + doomFrameBytes - 1u) / doomFrameBytes) : 1u; + uint32_t fullFrames = fullFrameBytes ? (uint32_t)((targetBytes + fullFrameBytes - 1u) / fullFrameBytes) : 1u; + if (doomFrames < 8u) doomFrames = 8u; + if (doomFrames > 128u) doomFrames = 128u; + if (fullFrames < 8u) fullFrames = 8u; + if (fullFrames > 64u) fullFrames = 64u; + + print("fbbench: running; the framebuffer will be overwritten temporarily\n"); + SHELL_FB_BENCH_RESULT doom; + SHELL_FB_BENCH_RESULT full; + shellRunFramebufferBenchmark(doomWidth, doomHeight, doomFrames, frequency, &doom); + shellRunFramebufferBenchmark(fullWidth, fullHeight, fullFrames, frequency, &full); + + SysClearScreen(0x00012456u); + print("Framebuffer benchmark:\n"); + print(" provider: "); print(info.PrimaryName[0] ? info.PrimaryName : "?"); + print(" mode="); printUInt(info.Mode.Width); print("x"); printUInt(info.Mode.Height); + print(" stride="); printUInt(info.Mode.Stride); + print(" cache="); print((info.Flags & DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE) ? "PAT-WC" : "fallback"); + print("\n"); + shellPrintFramebufferBenchmark("doom", &doom); + shellPrintFramebufferBenchmark("full", &full); +} + // Implements the cmd mem operation. static void cmd_mem(void) { SYS_MEMORY_STATUS st; @@ -1200,6 +1632,7 @@ static const char* threadStateName(uint32_t state) { case SYS_THREAD_STATE_RUNNING: return "run"; case SYS_THREAD_STATE_WAITING: return "wait"; case SYS_THREAD_STATE_DEAD: return "dead"; + case SYS_THREAD_STATE_SUSPENDED: return "suspend"; default: return "?"; } } @@ -1331,7 +1764,7 @@ static void cmd_threads(const char* args) { print(" (Count: "); printUInt((uint32_t)count); print("):\n"); - print(" TID State Pri CPU LastCPU QRem QTick Burst Wait WStat Flags\n"); + print(" TID State Base Eff CPU LastCPU QRem QTick Burst Wait WStat Flags\n"); print(" Flags: *=current I=idle K=kernel Q=queued A=alertable k=kernel-apc u=user-apc.\n"); size_t displayCount = count < MAX_THREADS ? count : MAX_THREADS; @@ -1342,7 +1775,11 @@ static void cmd_threads(const char* args) { print(" "); printStringWidth(threadStateName(t->State), 5); print(" "); - printUIntWidth(t->Priority, 3); + if (t->BaseSchedulingClass == SYS_SCHEDULING_REALTIME) print("R"); else print("N"); + printUIntWidth(t->BasePriority, 2); + print(" "); + if (t->EffectiveSchedulingClass == SYS_SCHEDULING_REALTIME) print("R"); else print("N"); + printUIntWidth(t->EffectivePriority, 2); print(" "); if (t->RunningCpu == 0xffffffffu) printStringWidth("-", 3); else printUIntWidth(t->RunningCpu, 3); @@ -1431,7 +1868,7 @@ static int shellCompareNames(const char* a, const char* b) { if (!a) a = ""; if (!b) b = ""; - for (;;) { + while (true) { char ca = shellAsciiLower(a[i]); char cb = shellAsciiLower(b[i]); if (ca != cb) @@ -1540,7 +1977,7 @@ static void cmd_dir(const char* path) { uint32_t count = 0; uint32_t capacity = 0; - for (;;) { + while (true) { SYS_DIR_ENTRY ke; st = SysReadDir(hDir, &ke); if (st != 0) @@ -1575,7 +2012,7 @@ static void cmd_type(const char* filename) { uint32_t hFile; if (open_file(filename, &hFile) != 0) { print("type: cannot open file\n"); return; } char buffer[512]; size_t bytesRead; - for (;;) { + while (true) { SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(hFile, buffer, sizeof(buffer) - 1, &bytesRead); if (SysReadFile(&readReq) != 0 || bytesRead == 0) break; buffer[bytesRead] = '\0'; print(buffer); @@ -1604,78 +2041,39 @@ static bool extractProgramPath(const char* cmdline, char* out, size_t outSize) { return pos != 0; } -// Consumes queued Ctrl-C input without taking ordinary foreground stdin. -static bool shellConsumeQueuedCtrlC(void) { - for (int i = 0; i < 4; i++) { - INPUT_KEY_RECORD input; - SYS_INPUT_EVENT_REQUEST inputReq; - inputReq.Size = sizeof(inputReq); - inputReq.QueueHandle = 0; - inputReq.Flags = SYS_INPUT_EVENT_FLAG_NONE; - inputReq.TimeoutMilliseconds = 0; - inputReq.RecordOut = &input; - if (SysPeekInputEvent(&inputReq) != NTLSTATUS_SUCCESS) return false; - - if ((input.Modifiers & INPUT_MOD_CTRL) && input.Key == INPUT_KEY_C) { - (void)SysReadInputEvent(&inputReq); - return true; - } - - if (input.Key == INPUT_KEY_LEFT_CTRL || input.Key == INPUT_KEY_RIGHT_CTRL) { - (void)SysReadInputEvent(&inputReq); - continue; - } - - return false; - } - - return false; -} - // Returns whether one input key is currently held. static bool shellIsKeyDown(INPUT_KEY key) { bool down = false; return SysIsKeyDown(key, &down) == NTLSTATUS_SUCCESS && down; } -// Waits for a foreground process while watching Ctrl-C without stealing ordinary stdin. +// Waits for a foreground process while polling Ctrl-C without consuming child input. static uint32_t waitForegroundProcess(uint32_t hProc) { bool sentBreak = false; - uint32_t inputQueue = 0; - if (SysOpenInputQueue(0, &inputQueue) != 0) - return SysWait(hProc, SYS_WAIT_FLAG_NONE); - - uint32_t handles[2] = { hProc, inputQueue }; - uint32_t index = 0; - SYS_WAIT_MANY_REQUEST waitReq; - waitReq.Size = sizeof(waitReq); - waitReq.Flags = SYS_WAIT_FLAG_NONE; - waitReq.Handles = handles; - waitReq.Count = 2; - waitReq.SignaledIndexOut = &index; - waitReq.TimeoutMilliseconds = SYS_WAIT_TIMEOUT_INFINITE; - - for (;;) { - index = 0; - uint32_t status = SysWaitMany(&waitReq); - if (status != NTLSTATUS_SUCCESS) { - SysCloseHandle(inputQueue); + while (true) { + uint32_t status = SysWaitSingle(hProc, + SYS_WAIT_FLAG_NONE, + SHELL_FOREGROUND_POLL_MS); + if (status == NTLSTATUS_SUCCESS) return status; - } - - if (index == 0) { - SysCloseHandle(inputQueue); + if (status != NTLSTATUS_TIMEOUT) return status; - } uint32_t modifiers = SysGetKeyModifiers(); - if (((modifiers & INPUT_MOD_CTRL) && shellIsKeyDown(INPUT_KEY_C)) || shellConsumeQueuedCtrlC()) { - if (!sentBreak) { - print("^C\n"); - (void)SysKillProcess(hProc, (int32_t)NTLSTATUS_PROCESS_KILLED); - sentBreak = true; + if (!sentBreak && (modifiers & INPUT_MOD_CTRL) && + shellIsKeyDown(INPUT_KEY_C)) { + print("^C\n"); + uint32_t killStatus = SysKillProcess( + hProc, + (int32_t)NTLSTATUS_PROCESS_KILLED); + if (killStatus != NTLSTATUS_SUCCESS) { + print("shell: foreground termination failed status="); + printUInt(killStatus); + print("\n"); + return killStatus; } + sentBreak = true; } } } @@ -1727,6 +2125,9 @@ static int runCommand(const char* cmd) { if (strcmp(cmd, "mem") == 0) { cmd_mem(); return 0; } if (strcmp(cmd, "time") == 0 || strcmp(cmd, "date") == 0) { cmd_time(); return 0; } if (strcmp(cmd, "display") == 0 || strcmp(cmd, "gpu") == 0) { cmd_display(); return 0; } + if (strcmp(cmd, "fbbench") == 0) { cmd_fbbench(); return 0; } + if (strcmp(cmd, "volume") == 0) { cmd_volume(""); return 0; } + if (strncmp(cmd, "volume ", 7) == 0) { cmd_volume(cmd + 7); return 0; } if (strcmp(cmd, "stats") == 0 || strcmp(cmd, "telemetry") == 0) { cmd_stats(); return 0; } if (strcmp(cmd, "ps") == 0) { cmd_ps(); return 0; } if (strncmp(cmd, "threads ", 8) == 0) { cmd_threads(cmd + 8); return 0; } diff --git a/code/user/apps/shell_posix/shell_posix.c b/code/user/apps/shell_posix/shell_posix.c index 81190af..49f8b9b 100644 --- a/code/user/apps/shell_posix/shell_posix.c +++ b/code/user/apps/shell_posix/shell_posix.c @@ -142,7 +142,7 @@ static int shellCompareNames(const char* a, const char* b) { if (!a) a = ""; if (!b) b = ""; - for (;;) { + while (true) { char ca = shellAsciiLower(a[i]); char cb = shellAsciiLower(b[i]); if (ca != cb) diff --git a/code/user/apps/shell_win32/shell_win32.c b/code/user/apps/shell_win32/shell_win32.c index 3f44767..12dee7e 100644 --- a/code/user/apps/shell_win32/shell_win32.c +++ b/code/user/apps/shell_win32/shell_win32.c @@ -112,7 +112,7 @@ static int shellCompareNames(const char* a, const char* b) { if (!a) a = ""; if (!b) b = ""; - for (;;) { + while (true) { char ca = shellAsciiLower(a[i]); char cb = shellAsciiLower(b[i]); if (ca != cb) diff --git a/code/user/apps/video/video.c b/code/user/apps/video/video.c index e40589c..70b57ad 100644 --- a/code/user/apps/video/video.c +++ b/code/user/apps/video/video.c @@ -50,11 +50,11 @@ static void fail_status(const char* what, uint32_t status) { print(" failed status="); print_u64(status); print("\n"); - for (;;) SysExitProcess(1); + while (true) SysExitProcess(1); } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static SYS_STRING_VIEW command_line_view(void) { @@ -227,6 +227,16 @@ int main(void) { print("video: input file mapped\n"); + if (!BinkSoundUseWaveOut()) { + print("video: BinkSoundUseWaveOut failed: "); + const char* err = BinkGetError(); + print(err ? err : "unknown"); + print("\n"); + SysFreeVm(buf); + exit_now(1); + } + print("video: WaveOut sound system ready\n"); + HBINK b = BinkOpen((const char*)buf, BINKFROMMEMORY); if (!b) { print("video: BinkOpen failed: "); @@ -237,6 +247,25 @@ int main(void) { exit_now(1); } + print("video: Bink open tracks="); + print_u64((uint64_t)(b->NumTracks < 0 ? 0 : b->NumTracks)); + print(" playing="); + print_u64(b->playingtracks); + print(" soundon="); + print_u64(b->soundon); + print("\n"); + for (int32_t track = 0; track < b->NumTracks; track++) { + print("video: track "); + print_u64((uint64_t)track); + print(" id="); + print_u64(BinkGetTrackID(b, (uint32_t)track)); + print(" type="); + print_u64(BinkGetTrackType(b, (uint32_t)track)); + print(" maxBytes="); + print_u64(BinkGetTrackMaxSize(b, (uint32_t)track)); + print("\n"); + } + uint32_t width = b->Width; uint32_t height = b->Height; if (!width || !height) { @@ -273,12 +302,16 @@ int main(void) { } print("video: native surface ready\n"); + print("video: decode loop entered\n"); DISPLAY_RECT fullFrame = { 0, 0, (int32_t)width, (int32_t)height }; + int firstFramePresented = 0; while (1) { if (BinkWait(b)) { - _mm_pause(); + // Bink can be waiting on its decode, I/O, or WaveOut worker. Yield + // a millisecond instead of monopolizing the CPU with a hot poll. + SysSleep(1); continue; } @@ -299,6 +332,16 @@ int main(void) { SysFreeVm(buf); fail_status("SysPresentSurface", status); } + if (!firstFramePresented) { + firstFramePresented = 1; + print("video: first frame presented sndBytes="); + print_u64(b->SndSize); + print(" soundon="); + print_u64(b->soundon); + print(" playing="); + print_u64(b->playingtracks); + print("\n"); + } if (b->FrameNum == b->Frames) break; @@ -306,8 +349,17 @@ int main(void) { BinkNextFrame(b); } + print("video: playback complete frame="); + print_u64(b->FrameNum); + print(" sndBytes="); + print_u64(b->SndSize); + print("\n"); + print("video: closing surface\n"); SysCloseHandle(surface); + print("video: closing Bink\n"); BinkClose(b); + print("video: Bink closed\n"); + print("video: freeing input mapping\n"); SysFreeVm(buf); print("video: goodbye\n"); diff --git a/code/user/personalities/win32/kernel32.c b/code/user/personalities/win32/kernel32.c index 0f2211a..c87d8d4 100644 --- a/code/user/personalities/win32/kernel32.c +++ b/code/user/personalities/win32/kernel32.c @@ -39,6 +39,54 @@ typedef long LONG; typedef struct { LONG x; LONG y; } POINT; typedef struct { SHORT X; SHORT Y; } COORD; typedef struct { SHORT Left; SHORT Top; SHORT Right; SHORT Bottom; } SMALL_RECT; +typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION; +typedef struct { void* _ptr; } CONDITION_VARIABLE; + +// Forward declarations used by the process-local GetProcAddress surface. +__declspec(dllexport) HANDLE CreateEventA(void*, BOOL, BOOL, const char*); +__declspec(dllexport) HANDLE CreateEventW(void*, BOOL, BOOL, const unsigned short*); +__declspec(dllexport) BOOL SetEvent(HANDLE); +__declspec(dllexport) BOOL ResetEvent(HANDLE); +__declspec(dllexport) _Noreturn void ExitThread(DWORD); +__declspec(dllexport) HANDLE CreateThread(void*, size_t, void*, void*, DWORD, DWORD*); +__declspec(dllexport) DWORD WaitForSingleObject(HANDLE, DWORD); +__declspec(dllexport) HANDLE CreateMutexA(void*, BOOL, const char*); +__declspec(dllexport) HANDLE CreateMutexW(void*, BOOL, const unsigned short*); +__declspec(dllexport) BOOL ReleaseMutex(HANDLE); +__declspec(dllexport) BOOL CloseHandle(HANDLE); +__declspec(dllexport) void InitializeCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) BOOL InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION*, DWORD); +__declspec(dllexport) void DeleteCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) BOOL TryEnterCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) void InitializeConditionVariable(CONDITION_VARIABLE*); +__declspec(dllexport) BOOL SleepConditionVariableCS(CONDITION_VARIABLE*, CRITICAL_SECTION*, DWORD); +__declspec(dllexport) void WakeConditionVariable(CONDITION_VARIABLE*); +__declspec(dllexport) void WakeAllConditionVariable(CONDITION_VARIABLE*); +__declspec(dllexport) BOOL IsDebuggerPresent(void); +extern void waveOutGetNumDevs(void); +extern void waveOutOpen(void); +extern void waveOutClose(void); +extern void waveOutPrepareHeader(void); +extern void waveOutUnprepareHeader(void); +extern void waveOutWrite(void); +extern void waveOutPause(void); +extern void waveOutRestart(void); +extern void waveOutReset(void); +extern void waveOutSetVolume(void); +extern void waveOutGetVolume(void); +extern void waveOutGetPosition(void); +extern void waveOutGetID(void); +extern void waveOutGetDevCapsA(void); +extern void waveOutGetDevCapsW(void); +extern void waveOutGetErrorTextA(void); +extern void waveOutGetErrorTextW(void); +extern void waveOutBreakLoop(void); +extern void waveOutMessage(void); +extern void timeGetTime(void); +extern void timeBeginPeriod(void); +extern void timeEndPeriod(void); typedef struct _CONSOLE_SCREEN_BUFFER_INFO { COORD dwSize; @@ -318,6 +366,9 @@ static DWORD* K32LastErrorSlot(void) { #define gLastError (*K32LastErrorSlot()) +// Returns a stable nonzero identifier for the calling thread. +static DWORD K32CurrentTidOrOne(void); + // Implements the file Exists A operation. static BOOL KiFileExistsA(const char* path) { if (!path || !*path) return FALSE; @@ -976,11 +1027,189 @@ __declspec(dllexport) BOOL DeleteFileA(const char* lpFileName) { return TRUE; } -// CloseHandle - unified close for files, threads, mutexes, semaphores, events, queues, and processes. -__declspec(dllexport) BOOL CloseHandle(HANDLE hObject) { - if (!hObject || hObject == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; } +#define K32_MUTEX_MAGIC 0x58544D4Bu - SysCloseHandle(HANDLE_TO_KH(hObject)); +typedef struct K32_MUTEX_HANDLE { + struct K32_MUTEX_HANDLE* Next; + uint32_t Magic; + volatile long StateLock; + volatile long ApiUsers; + uint32_t NativeHandle; + DWORD OwnerTid; + uint32_t Recursion; + uint32_t Waiters; + volatile long Closing; +} K32_MUTEX_HANDLE; + +static K32_MUTEX_HANDLE* gK32MutexRegistry; +static volatile long gK32MutexRegistryLock; + +// Acquires one short process-local mutex registry lock. +static void K32MutexRegistryLock(void) { + while (_InterlockedCompareExchange(&gK32MutexRegistryLock, 1, 0) != 0) + (void)SysYield(); +} + +// Releases the process-local mutex registry lock. +static void K32MutexRegistryUnlock(void) { + _InterlockedExchange(&gK32MutexRegistryLock, 0); +} + +// Acquires one short mutex-wrapper state lock. +static void K32MutexStateLock(K32_MUTEX_HANDLE* mutex) { + while (_InterlockedCompareExchange(&mutex->StateLock, 1, 0) != 0) + (void)SysYield(); +} + +// Releases one mutex-wrapper state lock. +static void K32MutexStateUnlock(K32_MUTEX_HANDLE* mutex) { + _InterlockedExchange(&mutex->StateLock, 0); +} + +// Registers one process-local Win32 mutex wrapper. +static void K32RegisterMutex(K32_MUTEX_HANDLE* mutex) { + K32MutexRegistryLock(); + mutex->Next = gK32MutexRegistry; + gK32MutexRegistry = mutex; + K32MutexRegistryUnlock(); +} + +// References one registered Win32 mutex without dereferencing unknown handles. +static K32_MUTEX_HANDLE* K32ReferenceMutex(HANDLE handle) { + K32_MUTEX_HANDLE* found = NULL; + K32MutexRegistryLock(); + for (K32_MUTEX_HANDLE* current = gK32MutexRegistry; + current; + current = current->Next) { + if ((HANDLE)current == handle && + current->Magic == K32_MUTEX_MAGIC && + _InterlockedCompareExchange(¤t->Closing, 0, 0) == 0) { + _InterlockedIncrement(¤t->ApiUsers); + found = current; + break; + } + } + K32MutexRegistryUnlock(); + return found; +} + +// Releases one mutex-wrapper API reference. +static void K32DereferenceMutex(K32_MUTEX_HANDLE* mutex) { + if (mutex) + _InterlockedDecrement(&mutex->ApiUsers); +} + +// Removes one mutex wrapper so no new API references can be acquired. +static BOOL K32RemoveMutex(K32_MUTEX_HANDLE* mutex) { + BOOL removed = FALSE; + K32MutexRegistryLock(); + K32_MUTEX_HANDLE** link = &gK32MutexRegistry; + while (*link) { + if (*link == mutex) { + *link = mutex->Next; + mutex->Next = NULL; + removed = TRUE; + break; + } + link = &(*link)->Next; + } + K32MutexRegistryUnlock(); + return removed; +} + +// Waits on one recursive Win32 mutex wrapper. +static DWORD K32WaitMutex(K32_MUTEX_HANDLE* mutex, DWORD milliseconds) { + DWORD tid = K32CurrentTidOrOne(); + K32MutexStateLock(mutex); + if (_InterlockedCompareExchange(&mutex->Closing, 0, 0) != 0) { + K32MutexStateUnlock(mutex); + gLastError = 6; + return 0xFFFFFFFFUL; + } + if (mutex->OwnerTid == tid) { + if (mutex->Recursion == UINT32_MAX) { + K32MutexStateUnlock(mutex); + gLastError = 8; + return 0xFFFFFFFFUL; + } + mutex->Recursion++; + K32MutexStateUnlock(mutex); + return 0; + } + mutex->Waiters++; + K32MutexStateUnlock(mutex); + + uint64_t timeout = milliseconds == 0xFFFFFFFFUL + ? SYS_WAIT_TIMEOUT_INFINITE + : (uint64_t)milliseconds; + uint32_t status = SysWaitSingle(mutex->NativeHandle, SYS_WAIT_FLAG_NONE, timeout); + + K32MutexStateLock(mutex); + if (mutex->Waiters) + mutex->Waiters--; + if (status == NTLSTATUS_SUCCESS || status == NTLSTATUS_ABANDONED) { + mutex->OwnerTid = tid; + mutex->Recursion = 1; + } + K32MutexStateUnlock(mutex); + + if (status == NTLSTATUS_SUCCESS) + return 0; + if (status == NTLSTATUS_ABANDONED) + return 0x00000080UL; + if (status == NTLSTATUS_TIMEOUT) + return 0x00000102UL; + gLastError = 6; + return 0xFFFFFFFFUL; +} + +// Closes one idle Win32 mutex wrapper. +static BOOL K32CloseMutex(K32_MUTEX_HANDLE* mutex) { + K32MutexStateLock(mutex); + BOOL busy = mutex->OwnerTid != 0 || mutex->Waiters != 0; + if (!busy && _InterlockedCompareExchange(&mutex->Closing, 1, 0) != 0) + busy = TRUE; + K32MutexStateUnlock(mutex); + if (busy) { + gLastError = 170; + K32DereferenceMutex(mutex); + return FALSE; + } + if (!K32RemoveMutex(mutex)) { + gLastError = 6; + K32DereferenceMutex(mutex); + return FALSE; + } + + while (mutex->ApiUsers > 1) + (void)SysYield(); + uint32_t status = SysCloseHandle(mutex->NativeHandle); + mutex->Magic = 0; + K32DereferenceMutex(mutex); + free(mutex); + if (status != NTLSTATUS_SUCCESS) { + gLastError = 6; + return FALSE; + } + return TRUE; +} + +// Closes one Win32 or native kernel handle. +__declspec(dllexport) BOOL CloseHandle(HANDLE hObject) { + if (!hObject || hObject == INVALID_HANDLE_VALUE) { + gLastError = 6; + return FALSE; + } + + K32_MUTEX_HANDLE* mutex = K32ReferenceMutex(hObject); + if (mutex) + return K32CloseMutex(mutex); + + uint32_t status = SysCloseHandle(HANDLE_TO_KH(hObject)); + if (status != NTLSTATUS_SUCCESS) { + gLastError = 6; + return FALSE; + } return TRUE; } @@ -1087,12 +1316,32 @@ static int KiStricmpA(const char* a, const char* b) { return ((unsigned char)*a | 0x20) - ((unsigned char)*b | 0x20); } -// Checks whether win32 Dll Name A. +// Matches one Win32 module basename with its optional DLL suffix. +static BOOL KiMatchesWin32DllNameA(const char* name, const char* baseName) { + if (!name || !baseName) + return FALSE; + + while (*name && *baseName && + (((unsigned char)*name | 0x20u) == ((unsigned char)*baseName | 0x20u))) { + name++; + baseName++; + } + + if (*baseName) + return FALSE; + if (!*name) + return TRUE; + return KiStricmpA(name, ".dll") == 0; +} + +// Checks whether one name identifies a process-local Win32 personality module. static BOOL KiIsWin32DllNameA(const char* name) { - return name && ( - !KiStricmpA(name, "kernel32.dll") || !KiStricmpA(name, "user32.dll") || - !KiStricmpA(name, "gdi32.dll") || !KiStricmpA(name, "ole32.dll") || - !KiStricmpA(name, "winmm.dll") || !KiStricmpA(name, "win32.dll")); + return KiMatchesWin32DllNameA(name, "kernel32") || + KiMatchesWin32DllNameA(name, "user32") || + KiMatchesWin32DllNameA(name, "gdi32") || + KiMatchesWin32DllNameA(name, "ole32") || + KiMatchesWin32DllNameA(name, "winmm") || + KiMatchesWin32DllNameA(name, "win32"); } // Implements the dbg Write A operation. @@ -1100,11 +1349,13 @@ static void KiDbgWriteA(const char* s) { if (s) K32WriteConsoleSpan(s, strlen(s)); } -// Returns module Handle A. +// Returns the process-local module handle for one ANSI module name. __declspec(dllexport) HANDLE GetModuleHandleA(const char* lpModuleName) { - if (!lpModuleName) return WIN32_HMODULE_EXE; - if (KiIsWin32DllNameA(lpModuleName)) return WIN32_HMODULE_WIN32; - gLastError = 126; // ERROR_MOD_NOT_FOUND + if (!lpModuleName) + return WIN32_HMODULE_EXE; + if (KiIsWin32DllNameA(lpModuleName)) + return WIN32_HMODULE_WIN32; + gLastError = 126; // ERROR_MOD_NOT_FOUND. return NULL; } @@ -1175,13 +1426,31 @@ __declspec(dllexport) void* GetProcAddress(HANDLE hModule, const char* lpProcNam return (void*)address; } -#define GP(name) if (strcmp(lpProcName, #name) == 0) return (void*)&name +#define GP(name) do { \ + if (strcmp(lpProcName, #name) == 0) return (void*)&name; \ +} while (0) GP(GetLastError); GP(SetLastError); GP(LoadLibraryA); GP(GetProcAddress); + GP(IsDebuggerPresent); GP(FreeLibrary); GP(VirtualAlloc); GP(VirtualFree); GP(VirtualProtect); GP(VirtualQuery); GP(GetTickCount); GP(GetTickCount64); GP(Sleep); + GP(CreateEventA); GP(CreateEventW); GP(SetEvent); GP(ResetEvent); + GP(CreateThread); GP(ExitThread); GP(WaitForSingleObject); + GP(CreateMutexA); GP(CreateMutexW); GP(ReleaseMutex); GP(CloseHandle); + GP(InitializeCriticalSection); GP(InitializeCriticalSectionAndSpinCount); + GP(DeleteCriticalSection); GP(EnterCriticalSection); GP(LeaveCriticalSection); + GP(TryEnterCriticalSection); GP(InitializeConditionVariable); + GP(SleepConditionVariableCS); GP(WakeConditionVariable); GP(WakeAllConditionVariable); GP(TlsAlloc); GP(TlsFree); GP(TlsGetValue); GP(TlsSetValue); GP(GetConsoleScreenBufferInfo); GP(FillConsoleOutputCharacterA); GP(FillConsoleOutputAttribute); GP(SetConsoleCursorPosition); + GP(waveOutGetNumDevs); GP(waveOutOpen); GP(waveOutClose); + GP(waveOutPrepareHeader); GP(waveOutUnprepareHeader); GP(waveOutWrite); + GP(waveOutPause); GP(waveOutRestart); GP(waveOutReset); + GP(waveOutSetVolume); GP(waveOutGetVolume); GP(waveOutGetPosition); GP(waveOutGetID); + GP(waveOutGetDevCapsA); GP(waveOutGetDevCapsW); + GP(waveOutGetErrorTextA); GP(waveOutGetErrorTextW); + GP(waveOutBreakLoop); GP(waveOutMessage); + GP(timeGetTime); GP(timeBeginPeriod); GP(timeEndPeriod); #undef GP KiDbgWriteA("GetProcAddress miss: "); KiDbgWriteA(lpProcName); @@ -1275,53 +1544,163 @@ __declspec(dllexport) char* GetCommandLineA(void) { #define WAIT_OBJECT_0 0x00000000UL #define WAIT_TIMEOUT 0x00000102UL #define INFINITE 0xFFFFFFFFUL +#define CREATE_SUSPENDED 0x00000004UL +typedef DWORD (__stdcall *K32_THREAD_START_ROUTINE)(void* parameter); + +typedef struct K32_THREAD_START_CONTEXT { + K32_THREAD_START_ROUTINE StartRoutine; + void* Parameter; +} K32_THREAD_START_CONTEXT; + +// Invokes one Win32 thread procedure and terminates the native thread on return. +static void K32ThreadStart(uintptr_t argument) { + K32_THREAD_START_CONTEXT* context = (K32_THREAD_START_CONTEXT*)argument; + K32_THREAD_START_ROUTINE startRoutine = context->StartRoutine; + void* parameter = context->Parameter; + free(context); + + DWORD exitCode = startRoutine(parameter); + SysExitThread((int32_t)exitCode); + while (true) { + (void)SysYield(); + } +} + +// Creates one Win32 thread with return-to-ExitThread semantics. __declspec(dllexport) HANDLE CreateThread(void* lpThreadAttributes, size_t dwStackSize, void* lpStartAddress, void* lpParameter, DWORD dwCreationFlags, DWORD* lpThreadId) { - (void)lpThreadAttributes; (void)dwStackSize; (void)dwCreationFlags; + (void)lpThreadAttributes; + (void)dwStackSize; + if (!lpStartAddress || (dwCreationFlags & ~CREATE_SUSPENDED)) { + gLastError = 87; + return NULL; + } + + K32_THREAD_START_CONTEXT* context = + (K32_THREAD_START_CONTEXT*)malloc(sizeof(*context)); + if (!context) { + gLastError = 8; + return NULL; + } + context->StartRoutine = (K32_THREAD_START_ROUTINE)lpStartAddress; + context->Parameter = lpParameter; + uint32_t tid = 0; uint32_t kh = 0; SYS_CREATE_THREAD_REQUEST req = { - (uintptr_t)lpStartAddress, - (uintptr_t)lpParameter, - 0, + (uintptr_t)K32ThreadStart, + (uintptr_t)context, 0, + (dwCreationFlags & CREATE_SUSPENDED) + ? SYS_CREATE_THREAD_FLAG_SUSPENDED + : SYS_CREATE_THREAD_FLAG_NONE, &kh, &tid }; - uint32_t s = SysCreateThread(&req); - if (s != 0) { gLastError = 11; return INVALID_HANDLE_VALUE; } - if (lpThreadId) *lpThreadId = (DWORD)tid; + uint32_t status = SysCreateThread(&req); + if (status != NTLSTATUS_SUCCESS) { + free(context); + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 11; + return NULL; + } + if (lpThreadId) + *lpThreadId = (DWORD)tid; return KH_TO_HANDLE(kh); } -// WaitForSingleObject - blocks on native waitable handles. A zero timeout is -// only a probe for mutexes/semaphores until native timeout waits exist. +// Terminates the calling Win32 thread with the supplied exit code. +__declspec(dllexport) _Noreturn void ExitThread(DWORD exitCode) { + SysExitThread((int32_t)exitCode); + while (true) { + (void)SysYield(); + } +} + +// Waits for one Win32 mutex wrapper or native waitable handle. __declspec(dllexport) DWORD WaitForSingleObject(HANDLE hHandle, DWORD dwMilliseconds) { - if (!IS_VALID_KERNEL_HANDLE(hHandle)) { gLastError = 6; return 0xFFFFFFFFUL; } - - uint32_t kh = HANDLE_TO_KH(hHandle); - if (dwMilliseconds == 0) { - uint32_t s = SysTryAcquireMutex(kh); - if (s == 0) return WAIT_OBJECT_0; - - s = SysTryWaitSem(kh); - if (s == 0) return WAIT_OBJECT_0; - - return WAIT_TIMEOUT; + if (!hHandle || hHandle == INVALID_HANDLE_VALUE) { + gLastError = 6; + return 0xFFFFFFFFUL; } - uint32_t s = SysWait(kh, SYS_WAIT_FLAG_NONE); - if (s == 0) return WAIT_OBJECT_0; + K32_MUTEX_HANDLE* mutex = K32ReferenceMutex(hHandle); + if (mutex) { + DWORD result = K32WaitMutex(mutex, dwMilliseconds); + K32DereferenceMutex(mutex); + return result; + } + if (!IS_VALID_KERNEL_HANDLE(hHandle)) { + gLastError = 6; + return 0xFFFFFFFFUL; + } + + uint64_t timeout = dwMilliseconds == INFINITE + ? SYS_WAIT_TIMEOUT_INFINITE + : (uint64_t)dwMilliseconds; + uint32_t s = SysWaitSingle(HANDLE_TO_KH(hHandle), SYS_WAIT_FLAG_NONE, timeout); + if (s == NTLSTATUS_SUCCESS) return WAIT_OBJECT_0; + if (s == NTLSTATUS_ABANDONED) return 0x00000080UL; + if (s == NTLSTATUS_TIMEOUT) return WAIT_TIMEOUT; gLastError = 6; return 0xFFFFFFFFUL; } +// Creates one unnamed native event with Win32 reset semantics. +__declspec(dllexport) HANDLE CreateEventA(void* lpEventAttributes, + BOOL bManualReset, + BOOL bInitialState, + const char* lpName) { + (void)lpEventAttributes; + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + + uint32_t handle = 0; + uint32_t status = SysCreateEvent(bManualReset ? 1u : 0u, + bInitialState ? 1u : 0u, + &handle); + if (status != NTLSTATUS_SUCCESS) { + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 87; + return NULL; + } + return KH_TO_HANDLE(handle); +} + +// Creates one unnamed native event from the wide Win32 entry point. +__declspec(dllexport) HANDLE CreateEventW(void* lpEventAttributes, + BOOL bManualReset, + BOOL bInitialState, + const unsigned short* lpName) { + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + return CreateEventA(lpEventAttributes, bManualReset, bInitialState, NULL); +} + +// Signals one native event. +__declspec(dllexport) BOOL SetEvent(HANDLE hEvent) { + if (!IS_VALID_KERNEL_HANDLE(hEvent)) { gLastError = 6; return FALSE; } + uint32_t status = SysSignalEvent(HANDLE_TO_KH(hEvent)); + if (status != NTLSTATUS_SUCCESS) { gLastError = 6; return FALSE; } + return TRUE; +} + +// Clears one native event. +__declspec(dllexport) BOOL ResetEvent(HANDLE hEvent) { + if (!IS_VALID_KERNEL_HANDLE(hEvent)) { gLastError = 6; return FALSE; } + uint32_t status = SysClearEvent(HANDLE_TO_KH(hEvent)); + if (status != NTLSTATUS_SUCCESS) { gLastError = 6; return FALSE; } + return TRUE; +} + // Implements the extract Application Path A operation. static BOOL ExtractApplicationPathA(const char* commandLine, char* out, size_t outSize) { if (!commandLine || !out || outSize == 0) return FALSE; @@ -1449,10 +1828,22 @@ __declspec(dllexport) BOOL GetExitCodeProcess(HANDLE hProcess, DWORD* lpExitCode return TRUE; } -// Implements the resume Thread operation. +// Decrements a thread's suspend count and makes it runnable at zero. __declspec(dllexport) DWORD ResumeThread(HANDLE hThread) { - (void)hThread; - return 0; // previous suspend count + if (!hThread || hThread == INVALID_HANDLE_VALUE) { + gLastError = 6; + return 0xFFFFFFFFu; + } + + uint32_t previousSuspendCount = 0; + uint32_t status = SysResumeThread(HANDLE_TO_KH(hThread), + &previousSuspendCount); + if (status != NTLSTATUS_SUCCESS) { + gLastError = status == NTLSTATUS_ACCESS_DENIED ? 5 : 6; + return 0xFFFFFFFFu; + } + + return previousSuspendCount; } #define THREAD_PRIORITY_NORMAL 0 @@ -1876,7 +2267,7 @@ __declspec(dllexport) HANDLE FindFirstFileA(const char* lpFileName, // Read the first entry that matches the pattern. KDIR_ENTRY ke; - for (;;) { + while (true) { uint32_t rs = SysReadDir(kh, &ke); if (rs != 0) { // Directory exhausted or error. @@ -1902,7 +2293,7 @@ __declspec(dllexport) BOOL FindNextFileA(HANDLE hFindFile, FIND_HANDLE* fh = (FIND_HANDLE*)hFindFile; KDIR_ENTRY ke; - for (;;) { + while (true) { uint32_t rs = SysReadDir(fh->kh, &ke); if (rs != 0) { gLastError = 18; // ERROR_NO_MORE_FILES @@ -1979,28 +2370,88 @@ __declspec(dllexport) DWORD GetCurrentDirectoryA(DWORD nBufferLength, char* lpBu return (DWORD)(needed - 1); // length without NUL, matching Win32 } +// Creates one unnamed recursive Win32 mutex over a native kernel mutex. __declspec(dllexport) HANDLE CreateMutexA(void* lpMutexAttributes, BOOL bInitialOwner, const char* lpName) { - (void)lpMutexAttributes; (void)lpName; - uint32_t kh = 0; - uint32_t s = SysCreateMutex(&kh); - if (s != 0 || !kh) { gLastError = 8; return NULL; } + (void)lpMutexAttributes; + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + + K32_MUTEX_HANDLE* mutex = (K32_MUTEX_HANDLE*)calloc(1, sizeof(*mutex)); + if (!mutex) { + gLastError = 8; + return NULL; + } + uint32_t status = SysCreateMutex(&mutex->NativeHandle); + if (status != NTLSTATUS_SUCCESS || !mutex->NativeHandle) { + free(mutex); + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 1; + return NULL; + } + mutex->Magic = K32_MUTEX_MAGIC; if (bInitialOwner) { - s = SysWait(kh, SYS_WAIT_FLAG_NONE); - if (s != 0) { - SysCloseHandle(kh); + status = SysWait(mutex->NativeHandle, SYS_WAIT_FLAG_NONE); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_ABANDONED) { + (void)SysCloseHandle(mutex->NativeHandle); + free(mutex); gLastError = 1; return NULL; } + mutex->OwnerTid = K32CurrentTidOrOne(); + mutex->Recursion = 1; } - return KH_TO_HANDLE(kh); + K32RegisterMutex(mutex); + return (HANDLE)mutex; } -// Releases mutex. +// Creates one unnamed recursive Win32 mutex from a wide-name request. +__declspec(dllexport) HANDLE CreateMutexW(void* lpMutexAttributes, BOOL bInitialOwner, + const unsigned short* lpName) { + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + return CreateMutexA(lpMutexAttributes, bInitialOwner, NULL); +} + +// Releases one recursion level from a Win32 mutex wrapper. __declspec(dllexport) BOOL ReleaseMutex(HANDLE hMutex) { - if (!IS_VALID_KERNEL_HANDLE(hMutex)) { gLastError = 6; return FALSE; } - uint32_t s = SysReleaseMutex(HANDLE_TO_KH(hMutex)); - if (s != 0) { gLastError = 6; return FALSE; } + K32_MUTEX_HANDLE* mutex = K32ReferenceMutex(hMutex); + if (!mutex) { + gLastError = 6; + return FALSE; + } + + DWORD tid = K32CurrentTidOrOne(); + K32MutexStateLock(mutex); + if (mutex->OwnerTid != tid || mutex->Recursion == 0) { + K32MutexStateUnlock(mutex); + K32DereferenceMutex(mutex); + gLastError = 288; + return FALSE; + } + mutex->Recursion--; + if (mutex->Recursion != 0) { + K32MutexStateUnlock(mutex); + K32DereferenceMutex(mutex); + return TRUE; + } + mutex->OwnerTid = 0; + K32MutexStateUnlock(mutex); + + uint32_t status = SysReleaseMutex(mutex->NativeHandle); + if (status != NTLSTATUS_SUCCESS) { + K32MutexStateLock(mutex); + mutex->OwnerTid = tid; + mutex->Recursion = 1; + K32MutexStateUnlock(mutex); + K32DereferenceMutex(mutex); + gLastError = 288; + return FALSE; + } + K32DereferenceMutex(mutex); return TRUE; } @@ -2030,8 +2481,19 @@ __declspec(dllexport) BOOL ReleaseSemaphore(HANDLE hSemaphore, long lReleaseCoun // using the storage as a small recursive spin lock. Bink uses these from its // worker/copy paths, so the old no-op implementation allowed internal state // races as soon as CreateThread started working. -typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION; -typedef struct { void* _ptr; } CONDITION_VARIABLE; +typedef struct K32_CONDITION_WAITER K32_CONDITION_WAITER; +typedef struct K32_CONDITION_STATE { + volatile long Lock; + K32_CONDITION_WAITER* Head; + K32_CONDITION_WAITER* Tail; +} K32_CONDITION_STATE; + +struct K32_CONDITION_WAITER { + K32_CONDITION_WAITER* Previous; + K32_CONDITION_WAITER* Next; + uint32_t Event; + BOOL Queued; +}; #define K32_CS_LOCK 0 #define K32_CS_OWNER 1 @@ -2067,8 +2529,12 @@ __declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS) { lpCS->_opaque[K32_CS_RECURSION]++; return; } - while (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0) + uint32_t spins = 0; + while (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0) { _mm_pause(); + if ((++spins & 0xFFFu) == 0) + (void)SysYield(); + } lpCS->_opaque[K32_CS_OWNER] = tid; lpCS->_opaque[K32_CS_RECURSION] = 1; } @@ -2100,19 +2566,183 @@ __declspec(dllexport) BOOL TryEnterCriticalSection(CRITICAL_SECTION* lpCS) { return TRUE; } -// Initializes condition Variable. -__declspec(dllexport) void InitializeConditionVariable(CONDITION_VARIABLE* cv) { (void)cv; } +// Acquires one condition-variable queue lock. +static void K32ConditionLock(K32_CONDITION_STATE* state) { + uint32_t spins = 0; + while (_InterlockedCompareExchange(&state->Lock, 1, 0) != 0) { + _mm_pause(); + if ((++spins & 0xFFFu) == 0) + (void)SysYield(); + } +} +// Releases one condition-variable queue lock. +static void K32ConditionUnlock(K32_CONDITION_STATE* state) { + _ReadWriteBarrier(); + _InterlockedExchange(&state->Lock, 0); +} + +// Returns the lazily allocated process-local condition-variable state. +static K32_CONDITION_STATE* K32ConditionState(CONDITION_VARIABLE* cv, BOOL create) { + if (!cv) + return NULL; + + K32_CONDITION_STATE* state = (K32_CONDITION_STATE*)cv->_ptr; + if (state || !create) + return state; + + K32_CONDITION_STATE* allocated = (K32_CONDITION_STATE*)calloc(1, sizeof(*allocated)); + if (!allocated) + return NULL; + + void* previous = _InterlockedCompareExchangePointer((void* volatile*)&cv->_ptr, + allocated, + NULL); + if (previous) { + free(allocated); + state = (K32_CONDITION_STATE*)previous; + } else { + state = allocated; + } + return state; +} + +// Removes one waiter while the condition-variable queue lock is held. +static void K32ConditionRemoveWaiter(K32_CONDITION_STATE* state, + K32_CONDITION_WAITER* waiter) { + if (!waiter->Queued) + return; + + if (waiter->Previous) + waiter->Previous->Next = waiter->Next; + else + state->Head = waiter->Next; + + if (waiter->Next) + waiter->Next->Previous = waiter->Previous; + else + state->Tail = waiter->Previous; + + waiter->Previous = NULL; + waiter->Next = NULL; + waiter->Queued = FALSE; +} + +// Initializes one process-local condition variable. +__declspec(dllexport) void InitializeConditionVariable(CONDITION_VARIABLE* cv) { + if (cv) + cv->_ptr = NULL; +} + +// Atomically releases a critical section and waits for one condition wake. __declspec(dllexport) BOOL SleepConditionVariableCS(CONDITION_VARIABLE* cv, CRITICAL_SECTION* cs, DWORD dwMilliseconds) { - (void)cv; (void)cs; (void)dwMilliseconds; - return TRUE; + if (!cv || !cs) { + gLastError = 87; + return FALSE; + } + + DWORD tid = K32CurrentTidOrOne(); + uintptr_t recursion = cs->_opaque[K32_CS_RECURSION]; + if ((DWORD)cs->_opaque[K32_CS_OWNER] != tid || recursion == 0) { + gLastError = 87; + return FALSE; + } + + K32_CONDITION_STATE* state = K32ConditionState(cv, TRUE); + if (!state) { + gLastError = 8; + return FALSE; + } + + uint32_t event = 0; + KSTATUS status = SysCreateEvent(0, 0, &event); + if (status != NTLSTATUS_SUCCESS) { + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 87; + return FALSE; + } + + // The waiter is heap-backed because process termination can stop a user + // thread at syscall return before it can remove a stack-backed queue node. + K32_CONDITION_WAITER* waiter = (K32_CONDITION_WAITER*)calloc(1, sizeof(*waiter)); + if (!waiter) { + (void)SysCloseHandle(event); + gLastError = 8; + return FALSE; + } + waiter->Event = event; + waiter->Queued = TRUE; + + K32ConditionLock(state); + waiter->Previous = state->Tail; + if (state->Tail) + state->Tail->Next = waiter; + else + state->Head = waiter; + state->Tail = waiter; + K32ConditionUnlock(state); + + // The waiter is visible before the lock is released, so a wake that races + // this transition is retained by the private auto-reset event. + cs->_opaque[K32_CS_RECURSION] = 0; + cs->_opaque[K32_CS_OWNER] = 0; + _ReadWriteBarrier(); + _InterlockedExchange(K32CsLockWord(cs), 0); + + uint64_t timeout = dwMilliseconds == INFINITE + ? SYS_WAIT_TIMEOUT_INFINITE + : (uint64_t)dwMilliseconds; + status = SysWaitSingle(event, SYS_WAIT_FLAG_NONE, timeout); + + K32ConditionLock(state); + BOOL wakeWon = waiter->Queued ? FALSE : TRUE; + K32ConditionRemoveWaiter(state, waiter); + K32ConditionUnlock(state); + + EnterCriticalSection(cs); + cs->_opaque[K32_CS_RECURSION] = recursion; + (void)SysCloseHandle(event); + free(waiter); + + if (status == NTLSTATUS_SUCCESS || wakeWon) + return TRUE; + if (status == NTLSTATUS_TIMEOUT) + gLastError = 1460; + else + gLastError = 6; + return FALSE; +} + +// Wakes the oldest waiter on one condition variable. +__declspec(dllexport) void WakeConditionVariable(CONDITION_VARIABLE* cv) { + K32_CONDITION_STATE* state = K32ConditionState(cv, FALSE); + if (!state) + return; + + K32ConditionLock(state); + K32_CONDITION_WAITER* waiter = state->Head; + if (waiter) { + K32ConditionRemoveWaiter(state, waiter); + (void)SysSignalEvent(waiter->Event); + } + K32ConditionUnlock(state); +} + +// Wakes every waiter currently queued on one condition variable. +__declspec(dllexport) void WakeAllConditionVariable(CONDITION_VARIABLE* cv) { + K32_CONDITION_STATE* state = K32ConditionState(cv, FALSE); + if (!state) + return; + + K32ConditionLock(state); + while (state->Head) { + K32_CONDITION_WAITER* waiter = state->Head; + K32ConditionRemoveWaiter(state, waiter); + (void)SysSignalEvent(waiter->Event); + } + K32ConditionUnlock(state); } -// Implements the wake Condition Variable operation. -__declspec(dllexport) void WakeConditionVariable(CONDITION_VARIABLE* cv) { (void)cv; } -// Implements the wake All Condition Variable operation. -__declspec(dllexport) void WakeAllConditionVariable(CONDITION_VARIABLE* cv) { (void)cv; } // Timing & Performance Counters. @@ -2961,7 +3591,7 @@ SetUnhandledExceptionFilter(LPTOP_LEVEL_EXCEPTION_FILTER lpFilter) { return NULL; // previous filter: none } -// Checks whether debugger Present. +// Reports that no user-mode debugger is attached. __declspec(dllexport) BOOL IsDebuggerPresent(void) { return FALSE; } diff --git a/code/user/personalities/win32/winmm.c b/code/user/personalities/win32/winmm.c index f3e4eaf..eceea58 100644 --- a/code/user/personalities/win32/winmm.c +++ b/code/user/personalities/win32/winmm.c @@ -1,126 +1,28 @@ -// Win32 multimedia compatibility stubs. This file provides minimal timing and sound API entry points. +// Win32 multimedia timing entry points. #include -#include #include -// Types. -typedef unsigned int MMRESULT; -typedef void* HWAVEOUT; +typedef unsigned int MMRESULT; -#define MMSYSERR_NOERROR 0 -#define MMSYSERR_NOTSUPPORTED 8 -#define WAVERR_STILLPLAYING 33 - -// WAVEFORMATEX - standard PCM audio format descriptor. -typedef struct { - unsigned short wFormatTag; - unsigned short nChannels; - unsigned int nSamplesPerSec; - unsigned int nAvgBytesPerSec; - unsigned short nBlockAlign; - unsigned short wBitsPerSample; - unsigned short cbSize; -} WAVEFORMATEX; - -// WAVEHDR - buffer header submitted to waveOutWrite. -typedef struct _WAVEHDR { - char* lpData; - unsigned int dwBufferLength; - unsigned int dwBytesRecorded; - uintptr_t dwUser; - unsigned int dwFlags; - unsigned int dwLoops; - struct _WAVEHDR* lpNext; - uintptr_t reserved; -} WAVEHDR; - -// MMTIME - used by waveOutGetPosition. -typedef struct { - unsigned int wType; - union { - unsigned int ms; - unsigned int sample; - unsigned int cb; - unsigned int ticks; - } u; -} MMTIME; - -// Timer. +#define MMSYSERR_NOERROR 0u +// Returns monotonic milliseconds since boot. __declspec(dllexport) unsigned long timeGetTime(void) { uint64_t frequency = 0; - if (SysGetTickFrequency(&frequency) != NTLSTATUS_SUCCESS || !frequency) return 0; + if (SysGetTickFrequency(&frequency) != NTLSTATUS_SUCCESS || !frequency) + return 0; return (unsigned long)SysTicksToMilliseconds(SysGetTick(), frequency); } -// Waveform audio (stubs - no audio hardware at this stage). - -__declspec(dllexport) MMRESULT waveOutOpen(HWAVEOUT* phwo, unsigned int deviceId, - const WAVEFORMATEX* fmt, uintptr_t callback, - uintptr_t callbackInstance, unsigned long flags) { - (void)deviceId; (void)fmt; (void)callback; (void)callbackInstance; (void)flags; - if (phwo) *phwo = (void*)0; - return MMSYSERR_NOTSUPPORTED; -} - -// Implements the wave Out Close operation. -__declspec(dllexport) MMRESULT waveOutClose(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -__declspec(dllexport) MMRESULT waveOutPrepareHeader(HWAVEOUT hwo, WAVEHDR* hdr, - unsigned int size) { - (void)hwo; (void)hdr; (void)size; return MMSYSERR_NOERROR; -} - -__declspec(dllexport) MMRESULT waveOutUnprepareHeader(HWAVEOUT hwo, WAVEHDR* hdr, - unsigned int size) { - (void)hwo; (void)hdr; (void)size; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Write operation. -__declspec(dllexport) MMRESULT waveOutWrite(HWAVEOUT hwo, WAVEHDR* hdr, unsigned int size) { - (void)hwo; (void)hdr; (void)size; return MMSYSERR_NOTSUPPORTED; -} - -// Implements the wave Out Pause operation. -__declspec(dllexport) MMRESULT waveOutPause(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Restart operation. -__declspec(dllexport) MMRESULT waveOutRestart(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Reset operation. -__declspec(dllexport) MMRESULT waveOutReset(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Set Volume operation. -__declspec(dllexport) MMRESULT waveOutSetVolume(HWAVEOUT hwo, unsigned long volume) { - (void)hwo; (void)volume; return MMSYSERR_NOTSUPPORTED; -} - -__declspec(dllexport) MMRESULT waveOutGetPosition(HWAVEOUT hwo, MMTIME* mmt, - unsigned int size) { - (void)hwo; (void)size; - if (mmt) { mmt->wType = 0; mmt->u.ms = 0; } +// Accepts timer-resolution requests without reprogramming the native timer. +__declspec(dllexport) MMRESULT timeBeginPeriod(unsigned int periodMilliseconds) { + (void)periodMilliseconds; return MMSYSERR_NOERROR; } -// Timer resolution control -// No hardware timer reprogramming at this stage; pretend success so callers -// that wrap their audio/game loop in timeBeginPeriod(1) don't bail out. -__declspec(dllexport) MMRESULT timeBeginPeriod(unsigned int uPeriod) { - (void)uPeriod; - return MMSYSERR_NOERROR; -} - -// Implements the time End Period operation. -__declspec(dllexport) MMRESULT timeEndPeriod(unsigned int uPeriod) { - (void)uPeriod; +// Releases one timer-resolution request without changing native timer policy. +__declspec(dllexport) MMRESULT timeEndPeriod(unsigned int periodMilliseconds) { + (void)periodMilliseconds; return MMSYSERR_NOERROR; } diff --git a/code/user/personalities/win32/winmm_waveout.c b/code/user/personalities/win32/winmm_waveout.c new file mode 100644 index 0000000..a709585 --- /dev/null +++ b/code/user/personalities/win32/winmm_waveout.c @@ -0,0 +1,1510 @@ +// WinMM waveform output over the native render-stream syscall ABI. + +#include +#include +#include +#include +#include +#include + +typedef unsigned int MMRESULT; +typedef void* HWAVEOUT; +typedef unsigned int UINT; +typedef uint32_t DWORD; +typedef uintptr_t DWORD_PTR; +typedef unsigned short WORD; +typedef int BOOL; + +#define FALSE 0 +#define TRUE 1 + +#define MMSYSERR_NOERROR 0u +#define MMSYSERR_ERROR 1u +#define MMSYSERR_BADDEVICEID 2u +#define MMSYSERR_ALLOCATED 4u +#define MMSYSERR_INVALHANDLE 5u +#define MMSYSERR_NOMEM 7u +#define MMSYSERR_NOTSUPPORTED 8u +#define MMSYSERR_INVALFLAG 10u +#define MMSYSERR_INVALPARAM 11u +#define MMSYSERR_HANDLEBUSY 12u + +#define WAVERR_BADFORMAT 32u +#define WAVERR_STILLPLAYING 33u +#define WAVERR_UNPREPARED 34u + +#define WAVE_FORMAT_PCM 1u +#define WAVE_MAPPER 0xFFFFFFFFu + +#define WAVE_FORMAT_QUERY 0x00000001u +#define WAVE_ALLOWSYNC 0x00000002u +#define WAVE_FORMAT_DIRECT 0x00000008u +#define CALLBACK_TYPEMASK 0x00070000u +#define CALLBACK_NULL 0x00000000u +#define CALLBACK_WINDOW 0x00010000u +#define CALLBACK_THREAD 0x00020000u +#define CALLBACK_FUNCTION 0x00030000u +#define CALLBACK_EVENT 0x00050000u +#define WINMM_WAVE_OPEN_SUPPORTED_FLAGS \ + (WAVE_FORMAT_QUERY | WAVE_ALLOWSYNC | WAVE_FORMAT_DIRECT | CALLBACK_TYPEMASK) + +#define WHDR_DONE 0x00000001u +#define WHDR_PREPARED 0x00000002u +#define WHDR_BEGINLOOP 0x00000004u +#define WHDR_ENDLOOP 0x00000008u +#define WHDR_INQUEUE 0x00000010u + +#define WOM_OPEN 0x03BBu +#define WOM_CLOSE 0x03BCu +#define WOM_DONE 0x03BDu + +#define TIME_MS 0x0001u +#define TIME_SAMPLES 0x0002u +#define TIME_BYTES 0x0004u + +#define WAVECAPS_VOLUME 0x0001u +#define WAVECAPS_LRVOLUME 0x0002u + +#define WAVE_FORMAT_1M16 0x00000004u +#define WAVE_FORMAT_1S16 0x00000008u +#define WAVE_FORMAT_2M16 0x00000040u +#define WAVE_FORMAT_2S16 0x00000080u +#define WAVE_FORMAT_4M16 0x00000400u +#define WAVE_FORMAT_4S16 0x00000800u +#define WAVE_FORMAT_48M16 0x00004000u +#define WAVE_FORMAT_48S16 0x00008000u + +#define WINMM_WAVEOUT_MAGIC 0x4F564157u +#define WINMM_WAIT_MILLISECONDS 5000ull +#define WINMM_WORKER_PRIORITY 1u +#define WINMM_WRITE_CHUNK_FRAMES 1024u + +typedef struct WAVEFORMATEX { + WORD wFormatTag; + WORD nChannels; + DWORD nSamplesPerSec; + DWORD nAvgBytesPerSec; + WORD nBlockAlign; + WORD wBitsPerSample; + WORD cbSize; +} WAVEFORMATEX; + +typedef struct WAVEHDR { + char* lpData; + DWORD dwBufferLength; + DWORD dwBytesRecorded; + DWORD_PTR dwUser; + DWORD dwFlags; + DWORD dwLoops; + struct WAVEHDR* lpNext; + DWORD_PTR reserved; +} WAVEHDR; + +typedef struct MMTIME { + UINT wType; + union { + DWORD ms; + DWORD sample; + DWORD cb; + DWORD ticks; + } u; +} MMTIME; + +typedef struct WAVEOUTCAPSA { + WORD wMid; + WORD wPid; + DWORD vDriverVersion; + char szPname[32]; + DWORD dwFormats; + WORD wChannels; + WORD wReserved1; + DWORD dwSupport; +} WAVEOUTCAPSA; + +typedef struct WAVEOUTCAPSW { + WORD wMid; + WORD wPid; + DWORD vDriverVersion; + unsigned short szPname[32]; + DWORD dwFormats; + WORD wChannels; + WORD wReserved1; + DWORD dwSupport; +} WAVEOUTCAPSW; + +typedef char WINMM_WAVEFORMATEX_SIZE[(sizeof(WAVEFORMATEX) == 20u) ? 1 : -1]; +typedef char WINMM_WAVEHDR_SIZE[(sizeof(WAVEHDR) == 48u) ? 1 : -1]; +typedef char WINMM_MMTIME_SIZE[(sizeof(MMTIME) == 8u) ? 1 : -1]; +typedef char WINMM_WAVEOUTCAPSA_SIZE[(sizeof(WAVEOUTCAPSA) == 52u) ? 1 : -1]; +typedef char WINMM_WAVEOUTCAPSW_SIZE[(sizeof(WAVEOUTCAPSW) == 84u) ? 1 : -1]; + +typedef void (__stdcall *WAVEOUTPROC)(HWAVEOUT waveOut, + UINT message, + DWORD_PTR instance, + DWORD_PTR parameter1, + DWORD_PTR parameter2); + +typedef enum WINMM_WAVE_COMMAND { + WinmmWaveCommandNone = 0, + WinmmWaveCommandPause, + WinmmWaveCommandRestart, + WinmmWaveCommandReset +} WINMM_WAVE_COMMAND; + +typedef struct WINMM_WAVE_BUFFER { + struct WINMM_WAVE_BUFFER* Next; + WAVEHDR* Header; + int16_t* Frames; + uint32_t FrameCount; + uint32_t SubmittedFrames; + uint64_t PresentedTarget; + uint8_t FullySubmitted; + uint64_t Sequence; +} WINMM_WAVE_BUFFER; + +typedef struct WINMM_WAVEOUT { + struct WINMM_WAVEOUT* RegistryNext; + uint32_t Magic; + volatile long Lock; + volatile long ApiUsers; + uint32_t NativeStream; + uint32_t WakeEvent; + uint32_t CommandEvent; + uint32_t ReadyEvent; + uint32_t WorkerHandle; + uint32_t WorkerTid; + uint64_t EndpointId; + WAVEFORMATEX SourceFormat; + uintptr_t Callback; + uintptr_t CallbackInstance; + uint32_t CallbackMode; + uint32_t DeviceId; + uint32_t Volume; + uint32_t Closing; + uint32_t UserPaused; + uint32_t AutoPaused; + uint32_t Started; + uint32_t TerminalError; + WINMM_WAVE_COMMAND Command; + MMRESULT CommandResult; + WINMM_WAVE_BUFFER* QueueHead; + WINMM_WAVE_BUFFER* QueueTail; + WINMM_WAVE_BUFFER* SubmitNode; + uint64_t NativeSubmitted; + uint64_t KnownDiscontinuity; + uint64_t HeadersQueued; + uint64_t HeadersCompleted; + uint64_t FramesQueued; +} WINMM_WAVEOUT; + +static WINMM_WAVEOUT* gWaveOutRegistry; +static volatile long gWaveOutRegistryLock; + + +typedef struct WINMM_DEBUG_LINE { + char Buffer[192]; + size_t Length; +} WINMM_DEBUG_LINE; + +// Appends text to one bounded WinMM diagnostic line. +static void WinmmDebugText(WINMM_DEBUG_LINE* line, const char* text) { + if (!line || !text) + return; + while (*text && line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = *text++; +} + +// Appends one unsigned decimal value to a WinMM diagnostic line. +static void WinmmDebugU64(WINMM_DEBUG_LINE* line, uint64_t value) { + char digits[24]; + size_t count = 0; + do { + digits[count++] = (char)('0' + value % 10u); + value /= 10u; + } while (value && count < sizeof(digits)); + while (count && line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = digits[--count]; +} + +// Appends one fixed-width hexadecimal value to a WinMM diagnostic line. +static void WinmmDebugHex(WINMM_DEBUG_LINE* line, uint64_t value) { + static const char digits[] = "0123456789ABCDEF"; + WinmmDebugText(line, "0x"); + int started = 0; + for (int shift = 60; shift >= 0; shift -= 4) { + uint8_t digit = (uint8_t)((value >> shift) & 0xFu); + if (!started && digit == 0 && shift) + continue; + started = 1; + if (line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = digits[digit]; + } +} + +// Emits one complete WinMM diagnostic line through the native debug console. +static void WinmmDebugCommit(WINMM_DEBUG_LINE* line) { + if (!line) + return; + if (line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = '\n'; + line->Buffer[line->Length] = '\0'; + SYS_PRINT_DEBUG_REQUEST request = { SysSvFromParts(line->Buffer, line->Length) }; + (void)SysPrintDebug(&request); +} + +// Logs one waveform-open request before validation changes its control flow. +static void WinmmDebugOpenRequest(UINT deviceId, + const WAVEFORMATEX* format, + DWORD flags) { + WINMM_DEBUG_LINE line = {0}; + WinmmDebugText(&line, "WINMM: open request device="); + WinmmDebugU64(&line, deviceId); + WinmmDebugText(&line, " tag="); + WinmmDebugU64(&line, format ? format->wFormatTag : 0); + WinmmDebugText(&line, " rate="); + WinmmDebugU64(&line, format ? format->nSamplesPerSec : 0); + WinmmDebugText(&line, " channels="); + WinmmDebugU64(&line, format ? format->nChannels : 0); + WinmmDebugText(&line, " bits="); + WinmmDebugU64(&line, format ? format->wBitsPerSample : 0); + WinmmDebugText(&line, " align="); + WinmmDebugU64(&line, format ? format->nBlockAlign : 0); + WinmmDebugText(&line, " avg="); + WinmmDebugU64(&line, format ? format->nAvgBytesPerSec : 0); + WinmmDebugText(&line, " flags="); + WinmmDebugHex(&line, flags); + WinmmDebugCommit(&line); +} + +// Logs one WinMM result with its handle and native stream identity. +static void WinmmDebugResult(const char* operation, + const WINMM_WAVEOUT* waveOut, + uint64_t value) { + WINMM_DEBUG_LINE line = {0}; + WinmmDebugText(&line, "WINMM: "); + WinmmDebugText(&line, operation); + if (waveOut) { + WinmmDebugText(&line, " handle="); + WinmmDebugHex(&line, (uintptr_t)waveOut); + WinmmDebugText(&line, " stream="); + WinmmDebugU64(&line, waveOut->NativeStream); + } + WinmmDebugText(&line, " value="); + WinmmDebugU64(&line, value); + WinmmDebugCommit(&line); +} + +// Acquires one short user-mode spin lock without entering the kernel. +static void WinmmLock(volatile long* lock) { + while (_InterlockedCompareExchange(lock, 1, 0) != 0) + (void)SysYield(); +} + +// Releases one short user-mode spin lock. +static void WinmmUnlock(volatile long* lock) { + _InterlockedExchange(lock, 0); +} + +// Maps one native status to the closest waveform-open result. +static MMRESULT WinmmMapNativeStatus(KSTATUS status) { + if (status == NTLSTATUS_SUCCESS) + return MMSYSERR_NOERROR; + if (status == NTLSTATUS_NO_MEMORY) + return MMSYSERR_NOMEM; + if (status == NTLSTATUS_NOT_FOUND || status == NTLSTATUS_DEVICE_REMOVED) + return MMSYSERR_BADDEVICEID; + if (status == NTLSTATUS_NOT_SUPPORTED) + return WAVERR_BADFORMAT; + if (status == NTLSTATUS_INVALID_PARAMETER) + return MMSYSERR_INVALPARAM; + return MMSYSERR_ERROR; +} + +// Returns whether one source rate is accepted by the native stream ABI. +static int WinmmRateIsSupported(uint32_t sampleRate) { + return sampleRate == 11025u || sampleRate == 22050u || + sampleRate == 44100u || sampleRate == 48000u; +} + +// Validates the narrow PCM surface implemented by this personality. +static MMRESULT WinmmValidateFormat(const WAVEFORMATEX* format) { + if (!format) + return MMSYSERR_INVALPARAM; + if (format->wFormatTag != WAVE_FORMAT_PCM || + (format->nChannels != 1u && format->nChannels != 2u) || + !WinmmRateIsSupported(format->nSamplesPerSec) || + (format->wBitsPerSample != 16u && format->wBitsPerSample != 24u)) + return WAVERR_BADFORMAT; + + uint32_t bytesPerSample = format->wBitsPerSample / 8u; + uint32_t blockAlign = (uint32_t)format->nChannels * bytesPerSample; + uint64_t averageBytes = (uint64_t)format->nSamplesPerSec * blockAlign; + if (format->nBlockAlign != blockAlign || + averageBytes > UINT32_MAX || + format->nAvgBytesPerSec != (uint32_t)averageBytes) + return WAVERR_BADFORMAT; + return MMSYSERR_NOERROR; +} + +// Enumerates available native endpoints into the caller's fixed snapshot. +static MMRESULT WinmmEnumerateEndpoints(SYS_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t capacity, + uint32_t* countOut) { + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = capacity; + request.CountOut = countOut; + return WinmmMapNativeStatus(SysAudioEnumerateEndpoints(&request)); +} + +// Counts endpoints that are currently ready and available. +static uint32_t WinmmAvailableEndpointCount(void) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + if (WinmmEnumerateEndpoints(endpoints, SYS_AUDIO_MAX_ENDPOINTS, &count) != MMSYSERR_NOERROR) + return 0; + if (count > SYS_AUDIO_MAX_ENDPOINTS) + count = SYS_AUDIO_MAX_ENDPOINTS; + + uint32_t available = 0; + for (uint32_t index = 0; index < count; index++) { + if (endpoints[index].State == SysAudioEndpointReady && + (endpoints[index].Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE)) + available++; + } + return available; +} + +// Resolves WAVE_MAPPER or one available-device index to a stable endpoint ID. +static MMRESULT WinmmResolveEndpoint(uint32_t deviceId, + uint64_t* endpointIdOut, + uint32_t* resolvedDeviceIdOut) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + MMRESULT result = WinmmEnumerateEndpoints(endpoints, SYS_AUDIO_MAX_ENDPOINTS, &count); + if (result != MMSYSERR_NOERROR) + return result; + if (count > SYS_AUDIO_MAX_ENDPOINTS) + count = SYS_AUDIO_MAX_ENDPOINTS; + + const SYS_AUDIO_ENDPOINT_INFO* fallback = NULL; + uint32_t fallbackIndex = 0; + uint32_t availableIndex = 0; + for (uint32_t index = 0; index < count; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) { + fallback = endpoint; + fallbackIndex = availableIndex; + } + if (deviceId == WAVE_MAPPER && + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT)) { + *endpointIdOut = endpoint->EndpointId; + *resolvedDeviceIdOut = availableIndex; + return MMSYSERR_NOERROR; + } + if (deviceId != WAVE_MAPPER && availableIndex == deviceId) { + *endpointIdOut = endpoint->EndpointId; + *resolvedDeviceIdOut = availableIndex; + return MMSYSERR_NOERROR; + } + availableIndex++; + } + + if (deviceId == WAVE_MAPPER && fallback) { + *endpointIdOut = fallback->EndpointId; + *resolvedDeviceIdOut = fallbackIndex; + return MMSYSERR_NOERROR; + } + return MMSYSERR_BADDEVICEID; +} + +// Adds one live waveform handle to the process-local registry. +static void WinmmRegisterWaveOut(WINMM_WAVEOUT* waveOut) { + WinmmLock(&gWaveOutRegistryLock); + waveOut->RegistryNext = gWaveOutRegistry; + gWaveOutRegistry = waveOut; + WinmmUnlock(&gWaveOutRegistryLock); +} + +// References one registered waveform handle without dereferencing unknown pointers. +static WINMM_WAVEOUT* WinmmReferenceWaveOut(HWAVEOUT handle) { + WINMM_WAVEOUT* found = NULL; + WinmmLock(&gWaveOutRegistryLock); + for (WINMM_WAVEOUT* current = gWaveOutRegistry; + current; + current = current->RegistryNext) { + if ((HWAVEOUT)current == handle && current->Magic == WINMM_WAVEOUT_MAGIC) { + _InterlockedIncrement(¤t->ApiUsers); + found = current; + break; + } + } + WinmmUnlock(&gWaveOutRegistryLock); + return found; +} + +// Releases one API-use reference acquired through the registry. +static void WinmmReleaseWaveOut(WINMM_WAVEOUT* waveOut) { + if (waveOut) + _InterlockedDecrement(&waveOut->ApiUsers); +} + +// Removes one waveform handle so no new API references can be acquired. +static int WinmmRemoveWaveOut(WINMM_WAVEOUT* waveOut) { + int removed = 0; + WinmmLock(&gWaveOutRegistryLock); + WINMM_WAVEOUT** link = &gWaveOutRegistry; + while (*link) { + if (*link == waveOut) { + *link = waveOut->RegistryNext; + waveOut->RegistryNext = NULL; + removed = 1; + break; + } + link = &(*link)->RegistryNext; + } + WinmmUnlock(&gWaveOutRegistryLock); + return removed; +} + +// Delivers one WinMM callback through a supported user-mode mechanism. +static void WinmmNotify(WINMM_WAVEOUT* waveOut, + uint32_t message, + uintptr_t parameter1, + uintptr_t parameter2) { + if (waveOut->CallbackMode == CALLBACK_FUNCTION) { + WAVEOUTPROC callback = (WAVEOUTPROC)waveOut->Callback; + callback((HWAVEOUT)waveOut, + message, + waveOut->CallbackInstance, + parameter1, + parameter2); + } else if (waveOut->CallbackMode == CALLBACK_EVENT) { + (void)SysSignalEvent((uint32_t)waveOut->Callback); + } +} + +// Converts one packed PCM buffer into personality-owned native S16 frames. +static MMRESULT WinmmCopyFrames(const WINMM_WAVEOUT* waveOut, + const WAVEHDR* header, + int16_t** framesOut, + uint32_t* frameCountOut) { + if (!header || !framesOut || !frameCountOut) + return MMSYSERR_INVALPARAM; + if (header->dwBufferLength && !header->lpData) + return MMSYSERR_INVALPARAM; + if (header->dwBufferLength % waveOut->SourceFormat.nBlockAlign) + return WAVERR_BADFORMAT; + + uint32_t frameCount = header->dwBufferLength / waveOut->SourceFormat.nBlockAlign; + size_t sampleCount = (size_t)frameCount * waveOut->SourceFormat.nChannels; + if (frameCount && sampleCount > SIZE_MAX / sizeof(int16_t)) + return MMSYSERR_NOMEM; + int16_t* output = sampleCount ? (int16_t*)malloc(sampleCount * sizeof(int16_t)) : NULL; + if (sampleCount && !output) + return MMSYSERR_NOMEM; + + if (waveOut->SourceFormat.wBitsPerSample == 16u) { + if (sampleCount) + memcpy(output, header->lpData, sampleCount * sizeof(int16_t)); + } else { + const uint8_t* input = (const uint8_t*)header->lpData; + for (size_t sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) { + uint32_t packed = (uint32_t)input[sampleIndex * 3u] | + ((uint32_t)input[sampleIndex * 3u + 1u] << 8u) | + ((uint32_t)input[sampleIndex * 3u + 2u] << 16u); + int32_t signedSample = (packed & 0x00800000u) + ? (int32_t)(packed | 0xFF000000u) + : (int32_t)packed; + output[sampleIndex] = (int16_t)(signedSample / 256); + } + } + + *framesOut = output; + *frameCountOut = frameCount; + return MMSYSERR_NOERROR; +} + +// Pops and completes every header whose source frames reached hardware presentation. +static void WinmmCompletePresentedBuffers(WINMM_WAVEOUT* waveOut, + uint64_t framesPresented) { + while (true) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* buffer = waveOut->QueueHead; + if (!buffer || !buffer->FullySubmitted || + buffer->PresentedTarget > framesPresented) { + WinmmUnlock(&waveOut->Lock); + return; + } + waveOut->QueueHead = buffer->Next; + if (!waveOut->QueueHead) + waveOut->QueueTail = NULL; + if (waveOut->SubmitNode == buffer) + waveOut->SubmitNode = buffer->Next; + buffer->Header->dwFlags &= ~WHDR_INQUEUE; + buffer->Header->dwFlags |= WHDR_DONE; + waveOut->HeadersCompleted++; + WinmmUnlock(&waveOut->Lock); + WinmmNotify(waveOut, WOM_DONE, (uintptr_t)buffer->Header, 0); + free(buffer->Frames); + free(buffer); + } +} + +// Detaches every queued buffer for reset or terminal failure completion. +static WINMM_WAVE_BUFFER* WinmmDetachAllBuffers(WINMM_WAVEOUT* waveOut) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* buffers = waveOut->QueueHead; + waveOut->QueueHead = NULL; + waveOut->QueueTail = NULL; + waveOut->SubmitNode = NULL; + WinmmUnlock(&waveOut->Lock); + return buffers; +} + +// Completes and releases one detached buffer list in queue order. +static void WinmmCompleteDetachedBuffers(WINMM_WAVEOUT* waveOut, + WINMM_WAVE_BUFFER* buffers) { + while (buffers) { + WINMM_WAVE_BUFFER* next = buffers->Next; + buffers->Header->dwFlags &= ~WHDR_INQUEUE; + buffers->Header->dwFlags |= WHDR_DONE; + waveOut->HeadersCompleted++; + WinmmNotify(waveOut, WOM_DONE, (uintptr_t)buffers->Header, 0); + free(buffers->Frames); + free(buffers); + buffers = next; + } +} + +// Queries one coherent native stream snapshot. +static KSTATUS WinmmQueryNativeStream(WINMM_WAVEOUT* waveOut, + SYS_AUDIO_STREAM_INFO* info) { + memset(info, 0, sizeof(*info)); + info->Size = sizeof(*info); + return SysAudioQueryStream(waveOut->NativeStream, info); +} + +// Updates the controlled discontinuity baseline after stop or reset. +static void WinmmRefreshDiscontinuity(WINMM_WAVEOUT* waveOut) { + SYS_AUDIO_STREAM_INFO info; + if (WinmmQueryNativeStream(waveOut, &info) == NTLSTATUS_SUCCESS) + waveOut->KnownDiscontinuity = info.DiscontinuityGeneration; +} + +// Processes one synchronous control command on the worker thread. +static void WinmmProcessCommand(WINMM_WAVEOUT* waveOut) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_COMMAND command = waveOut->Command; + WinmmUnlock(&waveOut->Lock); + if (command == WinmmWaveCommandNone) + return; + + MMRESULT result = MMSYSERR_NOERROR; + WINMM_WAVE_BUFFER* resetBuffers = NULL; + if (command == WinmmWaveCommandPause) { + if (!waveOut->UserPaused) { + waveOut->UserPaused = 1; + if (waveOut->Started && !waveOut->AutoPaused) { + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status == NTLSTATUS_SUCCESS && info.State == SysAudioStreamRunning) + status = SysAudioPause(waveOut->NativeStream); + if (status != NTLSTATUS_SUCCESS) + result = WinmmMapNativeStatus(status); + } + } + } else if (command == WinmmWaveCommandRestart) { + if (waveOut->UserPaused) { + waveOut->UserPaused = 0; + if (waveOut->Started && !waveOut->AutoPaused) { + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status == NTLSTATUS_SUCCESS && info.State == SysAudioStreamPaused) + status = SysAudioResume(waveOut->NativeStream); + if (status != NTLSTATUS_SUCCESS) + result = WinmmMapNativeStatus(status); + } + } + } else if (command == WinmmWaveCommandReset) { + (void)SysAudioStop(waveOut->NativeStream); + KSTATUS status = SysAudioReset(waveOut->NativeStream); + if (status != NTLSTATUS_SUCCESS) + result = WinmmMapNativeStatus(status); + else + waveOut->TerminalError = 0; + resetBuffers = WinmmDetachAllBuffers(waveOut); + waveOut->NativeSubmitted = 0; + waveOut->Started = 0; + waveOut->UserPaused = 0; + waveOut->AutoPaused = 0; + WinmmRefreshDiscontinuity(waveOut); + } + + if (resetBuffers) + WinmmCompleteDetachedBuffers(waveOut, resetBuffers); + + WinmmLock(&waveOut->Lock); + waveOut->CommandResult = result; + waveOut->Command = WinmmWaveCommandNone; + WinmmUnlock(&waveOut->Lock); + (void)SysSignalEvent(waveOut->CommandEvent); +} + +// Returns the next queued node that still has source frames to submit. +static WINMM_WAVE_BUFFER* WinmmFindSubmitNode(WINMM_WAVEOUT* waveOut) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* buffer = waveOut->SubmitNode; + while (buffer && buffer->FullySubmitted) + buffer = buffer->Next; + waveOut->SubmitNode = buffer; + WinmmUnlock(&waveOut->Lock); + return buffer; +} + +// Feeds queued buffers into the native ring until it becomes full or empty. +static KSTATUS WinmmFeedNativeStream(WINMM_WAVEOUT* waveOut, + uint32_t* madeProgressOut) { + *madeProgressOut = 0; + while (true) { + WINMM_WAVE_BUFFER* buffer = WinmmFindSubmitNode(waveOut); + if (!buffer) + return NTLSTATUS_SUCCESS; + + uint32_t remaining = buffer->FrameCount - buffer->SubmittedFrames; + if (!remaining) { + buffer->FullySubmitted = 1; + buffer->PresentedTarget = waveOut->NativeSubmitted; + continue; + } + uint32_t requestFrames = remaining < WINMM_WRITE_CHUNK_FRAMES + ? remaining + : WINMM_WRITE_CHUNK_FRAMES; + uint32_t writtenFrames = 0; + SYS_AUDIO_WRITE_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = waveOut->NativeStream; + request.FrameCount = requestFrames; + request.Frames = buffer->Frames + + (size_t)buffer->SubmittedFrames * waveOut->SourceFormat.nChannels; + request.WrittenFramesOut = &writtenFrames; + KSTATUS status = SysAudioWrite(&request); + if (status == NTLSTATUS_WOULD_BLOCK) + return status; + if (status != NTLSTATUS_SUCCESS) + return status; + if (!writtenFrames) + return NTLSTATUS_WOULD_BLOCK; + + buffer->SubmittedFrames += writtenFrames; + waveOut->NativeSubmitted += writtenFrames; + *madeProgressOut = 1; + if (buffer->SubmittedFrames == buffer->FrameCount) { + buffer->FullySubmitted = 1; + buffer->PresentedTarget = waveOut->NativeSubmitted; + } + } +} + +// Determines whether any queued buffer still needs native-ring submission. +static int WinmmHasUnsubmittedBuffer(WINMM_WAVEOUT* waveOut) { + return WinmmFindSubmitNode(waveOut) != NULL; +} + +// Starts or resumes playback after a useful prefill is available. +static KSTATUS WinmmEnsureStarted(WINMM_WAVEOUT* waveOut) { + if (waveOut->UserPaused) + return NTLSTATUS_SUCCESS; + if (!waveOut->Started) { + KSTATUS status = SysAudioStart(waveOut->NativeStream); + if (status == NTLSTATUS_SUCCESS) { + waveOut->Started = 1; + waveOut->AutoPaused = 0; + } + return status; + } + if (waveOut->AutoPaused) { + KSTATUS status = SysAudioResume(waveOut->NativeStream); + if (status == NTLSTATUS_SUCCESS) + waveOut->AutoPaused = 0; + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Stops a completed empty stream without resetting its public position counters. +static void WinmmStopEmptyStream(WINMM_WAVEOUT* waveOut) { + if (!waveOut->Started) + return; + (void)SysAudioStop(waveOut->NativeStream); + waveOut->Started = 0; + waveOut->AutoPaused = 0; + WinmmRefreshDiscontinuity(waveOut); +} + +// Arms native progress conditions and waits for either stream progress or new work. +static KSTATUS WinmmWaitForWork(WINMM_WAVEOUT* waveOut, + const SYS_AUDIO_STREAM_INFO* info, + int hasUnsubmitted) { + SYS_AUDIO_ARM_WAIT_REQUEST arm; + memset(&arm, 0, sizeof(arm)); + arm.Size = sizeof(arm); + arm.StreamHandle = waveOut->NativeStream; + arm.Mask = SYS_AUDIO_WAIT_FAILED | SYS_AUDIO_WAIT_DISCONTINUITY; + arm.DiscontinuityGeneration = waveOut->KnownDiscontinuity; + + if (hasUnsubmitted && !waveOut->UserPaused) { + arm.Mask |= SYS_AUDIO_WAIT_WRITABLE; + arm.WritableFrames = 1; + } + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* head = waveOut->QueueHead; + if (head && head->FullySubmitted) { + arm.Mask |= SYS_AUDIO_WAIT_PRESENTED; + arm.PresentedFrames = head->PresentedTarget; + } + WinmmUnlock(&waveOut->Lock); + if (!hasUnsubmitted && waveOut->Started && + info->FramesConsumed < waveOut->NativeSubmitted) { + arm.Mask |= SYS_AUDIO_WAIT_CONSUMED; + arm.ConsumedFrames = waveOut->NativeSubmitted; + } + + KSTATUS status = SysAudioArmWait(&arm); + if (status != NTLSTATUS_SUCCESS) + return status; + + uint32_t handles[2] = { waveOut->WakeEvent, waveOut->NativeStream }; + uint32_t selected = 0; + SYS_WAIT_MANY_REQUEST wait; + memset(&wait, 0, sizeof(wait)); + wait.Size = sizeof(wait); + wait.Handles = handles; + wait.Count = 2; + wait.SignaledIndexOut = &selected; + wait.TimeoutMilliseconds = SYS_WAIT_TIMEOUT_INFINITE; + return SysWaitMany(&wait); +} + +// Retires a failed stream and returns all queued headers to the application. +static void WinmmFailStream(WINMM_WAVEOUT* waveOut, MMRESULT error) { + waveOut->TerminalError = error ? error : MMSYSERR_ERROR; + WinmmDebugResult("stream failure", waveOut, waveOut->TerminalError); + (void)SysAudioStop(waveOut->NativeStream); + waveOut->Started = 0; + waveOut->AutoPaused = 0; + WINMM_WAVE_BUFFER* buffers = WinmmDetachAllBuffers(waveOut); + WinmmCompleteDetachedBuffers(waveOut, buffers); +} + +// Runs one waveform-output worker until close or process teardown. +static void WinmmWaveWorker(uintptr_t argument) { + WINMM_WAVEOUT* waveOut = (WINMM_WAVEOUT*)argument; + waveOut->WorkerTid = SysGetCurrentTid(); + WinmmNotify(waveOut, WOM_OPEN, 0, 0); + (void)SysSignalEvent(waveOut->ReadyEvent); + + while (true) { + WinmmProcessCommand(waveOut); + if (waveOut->Closing) { + WinmmStopEmptyStream(waveOut); + WinmmNotify(waveOut, WOM_CLOSE, 0, 0); + SysExitThread(0); + } + if (waveOut->TerminalError) { + uint32_t handles[1] = { waveOut->WakeEvent }; + uint32_t selected = 0; + SYS_WAIT_MANY_REQUEST wait = { + sizeof(wait), 0, handles, 1, &selected, SYS_WAIT_TIMEOUT_INFINITE + }; + (void)SysWaitMany(&wait); + continue; + } + + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS || info.State == SysAudioStreamFailed) { + WinmmFailStream(waveOut, WinmmMapNativeStatus( + status != NTLSTATUS_SUCCESS ? status : info.FailureStatus)); + continue; + } + if (info.DiscontinuityGeneration != waveOut->KnownDiscontinuity) { + // A native discontinuity is a timeline notification, not a stream + // failure. The failed state remains the terminal error authority. + waveOut->KnownDiscontinuity = info.DiscontinuityGeneration; + WinmmDebugResult("stream discontinuity", waveOut, + waveOut->KnownDiscontinuity); + } + + WinmmCompletePresentedBuffers(waveOut, info.FramesPresented); + WinmmLock(&waveOut->Lock); + int queueEmpty = waveOut->QueueHead == NULL; + WinmmUnlock(&waveOut->Lock); + if (queueEmpty) { + WinmmStopEmptyStream(waveOut); + status = WinmmWaitForWork(waveOut, &info, 0); + if (status != NTLSTATUS_SUCCESS) + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + + if (!waveOut->UserPaused) { + uint32_t madeProgress = 0; + status = WinmmFeedNativeStream(waveOut, &madeProgress); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_WOULD_BLOCK) { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + if (waveOut->NativeSubmitted && (!waveOut->Started || waveOut->AutoPaused)) { + status = WinmmEnsureStarted(waveOut); + if (status != NTLSTATUS_SUCCESS) { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + } + if (madeProgress) + continue; + } + + status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS) { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + int hasUnsubmitted = WinmmHasUnsubmittedBuffer(waveOut); + if (!hasUnsubmitted && waveOut->Started && !waveOut->UserPaused && + !waveOut->AutoPaused && info.FramesConsumed >= waveOut->NativeSubmitted) { + status = SysAudioPause(waveOut->NativeStream); + if (status == NTLSTATUS_SUCCESS) + waveOut->AutoPaused = 1; + else { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + } + + status = WinmmWaitForWork(waveOut, &info, hasUnsubmitted); + if (status != NTLSTATUS_SUCCESS) + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + } +} + +// Submits one synchronous worker command and waits for its completion. +static MMRESULT WinmmSubmitCommand(WINMM_WAVEOUT* waveOut, + WINMM_WAVE_COMMAND command) { + if (waveOut->WorkerTid == SysGetCurrentTid()) + return MMSYSERR_HANDLEBUSY; + + WinmmLock(&waveOut->Lock); + if (waveOut->Command != WinmmWaveCommandNone) { + WinmmUnlock(&waveOut->Lock); + return MMSYSERR_ALLOCATED; + } + waveOut->Command = command; + waveOut->CommandResult = MMSYSERR_ERROR; + WinmmUnlock(&waveOut->Lock); + (void)SysSignalEvent(waveOut->WakeEvent); + + KSTATUS status = SysWait(waveOut->CommandEvent, SYS_WAIT_FLAG_NONE); + if (status != NTLSTATUS_SUCCESS) + return WinmmMapNativeStatus(status); + return waveOut->CommandResult; +} + +// Returns the number of available logical waveform output devices. +__declspec(dllexport) UINT waveOutGetNumDevs(void) { + return WinmmAvailableEndpointCount(); +} + +// Opens one waveform output handle or validates one format query. +__declspec(dllexport) MMRESULT waveOutOpen(HWAVEOUT* handleOut, + UINT deviceId, + const WAVEFORMATEX* format, + DWORD_PTR callback, + DWORD_PTR callbackInstance, + DWORD flags) { + WinmmDebugOpenRequest(deviceId, format, flags); + MMRESULT result = WinmmValidateFormat(format); + if (result != MMSYSERR_NOERROR) { + WinmmDebugResult("open rejected format", NULL, result); + return result; + } + if (flags & ~WINMM_WAVE_OPEN_SUPPORTED_FLAGS) { + WinmmDebugResult("open rejected flags", NULL, flags & ~WINMM_WAVE_OPEN_SUPPORTED_FLAGS); + return MMSYSERR_INVALFLAG; + } + + uint32_t callbackMode = flags & CALLBACK_TYPEMASK; + if (callbackMode == CALLBACK_WINDOW || callbackMode == CALLBACK_THREAD) { + WinmmDebugResult("open rejected callback", NULL, callbackMode); + return MMSYSERR_NOTSUPPORTED; + } + if (callbackMode != CALLBACK_NULL && + callbackMode != CALLBACK_FUNCTION && + callbackMode != CALLBACK_EVENT) + return MMSYSERR_INVALFLAG; + if ((callbackMode == CALLBACK_FUNCTION || callbackMode == CALLBACK_EVENT) && !callback) + return MMSYSERR_INVALPARAM; + + uint64_t endpointId = 0; + uint32_t resolvedDeviceId = 0; + result = WinmmResolveEndpoint(deviceId, &endpointId, &resolvedDeviceId); + if (result != MMSYSERR_NOERROR) { + WinmmDebugResult("open rejected endpoint", NULL, result); + return result; + } + if (flags & WAVE_FORMAT_QUERY) { + return MMSYSERR_NOERROR; + } + if (!handleOut) + return MMSYSERR_INVALPARAM; + *handleOut = NULL; + + WINMM_WAVEOUT* waveOut = (WINMM_WAVEOUT*)calloc(1, sizeof(*waveOut)); + if (!waveOut) + return MMSYSERR_NOMEM; + waveOut->Magic = WINMM_WAVEOUT_MAGIC; + waveOut->EndpointId = endpointId; + waveOut->SourceFormat = *format; + waveOut->Callback = callback; + waveOut->CallbackInstance = callbackInstance; + waveOut->CallbackMode = callbackMode; + waveOut->DeviceId = resolvedDeviceId; + waveOut->Volume = 0xFFFFFFFFu; + + SYS_AUDIO_OPEN_STREAM_REQUEST open; + memset(&open, 0, sizeof(open)); + open.Size = sizeof(open); + open.EndpointId = endpointId; + open.Format.SampleRate = format->nSamplesPerSec; + open.Format.ChannelCount = format->nChannels; + open.Format.SampleFormat = SysAudioSampleS16; + open.StreamHandleOut = &waveOut->NativeStream; + KSTATUS status = SysAudioOpenStream(&open); + if (status != NTLSTATUS_SUCCESS) { + result = WinmmMapNativeStatus(status); + WinmmDebugResult("native stream open failed", waveOut, status); + free(waveOut); + return result; + } + + status = SysCreateEvent(0, 0, &waveOut->WakeEvent); + if (status == NTLSTATUS_SUCCESS) + status = SysCreateEvent(0, 0, &waveOut->CommandEvent); + if (status == NTLSTATUS_SUCCESS) + status = SysCreateEvent(0, 0, &waveOut->ReadyEvent); + if (status != NTLSTATUS_SUCCESS) { + result = WinmmMapNativeStatus(status); + WinmmDebugResult("event creation failed", waveOut, status); + if (waveOut->ReadyEvent) (void)SysCloseHandle(waveOut->ReadyEvent); + if (waveOut->CommandEvent) (void)SysCloseHandle(waveOut->CommandEvent); + if (waveOut->WakeEvent) (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return result; + } + + SYS_AUDIO_STREAM_INFO info; + status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS) { + result = WinmmMapNativeStatus(status); + (void)SysCloseHandle(waveOut->ReadyEvent); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return result; + } + waveOut->KnownDiscontinuity = info.DiscontinuityGeneration; + + WinmmRegisterWaveOut(waveOut); + SYS_CREATE_THREAD_REQUEST thread; + memset(&thread, 0, sizeof(thread)); + thread.Entry = (uintptr_t)WinmmWaveWorker; + thread.Argument = (uintptr_t)waveOut; + thread.Priority = WINMM_WORKER_PRIORITY; + thread.ThreadHandleOut = &waveOut->WorkerHandle; + thread.ThreadIdOut = &waveOut->WorkerTid; + status = SysCreateThread(&thread); + if (status != NTLSTATUS_SUCCESS) { + (void)WinmmRemoveWaveOut(waveOut); + result = WinmmMapNativeStatus(status); + WinmmDebugResult("worker creation failed", waveOut, status); + (void)SysCloseHandle(waveOut->ReadyEvent); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return result; + } + + status = SysWaitSingle(waveOut->ReadyEvent, + SYS_WAIT_FLAG_NONE, + WINMM_WAIT_MILLISECONDS); + (void)SysCloseHandle(waveOut->ReadyEvent); + waveOut->ReadyEvent = 0; + if (status != NTLSTATUS_SUCCESS) { + waveOut->Closing = 1; + (void)SysSignalEvent(waveOut->WakeEvent); + (void)SysWait(waveOut->WorkerHandle, SYS_WAIT_FLAG_NONE); + (void)WinmmRemoveWaveOut(waveOut); + (void)SysCloseHandle(waveOut->WorkerHandle); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return WinmmMapNativeStatus(status); + } + + *handleOut = (HWAVEOUT)waveOut; + WINMM_DEBUG_LINE line = {0}; + WinmmDebugText(&line, "WINMM: open success handle="); + WinmmDebugHex(&line, (uintptr_t)waveOut); + WinmmDebugText(&line, " endpoint="); + WinmmDebugU64(&line, endpointId); + WinmmDebugText(&line, " stream="); + WinmmDebugU64(&line, waveOut->NativeStream); + WinmmDebugText(&line, " worker="); + WinmmDebugU64(&line, waveOut->WorkerTid); + WinmmDebugCommit(&line); + return MMSYSERR_NOERROR; +} + +// Closes one idle waveform handle after its worker sends WOM_CLOSE. +__declspec(dllexport) MMRESULT waveOutClose(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (waveOut->WorkerTid == SysGetCurrentTid()) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_HANDLEBUSY; + } + + WinmmLock(&waveOut->Lock); + int busy = waveOut->QueueHead != NULL || waveOut->Command != WinmmWaveCommandNone; + WinmmUnlock(&waveOut->Lock); + if (busy) { + WinmmDebugResult("close still playing", waveOut, waveOut->HeadersQueued - waveOut->HeadersCompleted); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if (!WinmmRemoveWaveOut(waveOut)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALHANDLE; + } + + waveOut->Closing = 1; + (void)SysSignalEvent(waveOut->WakeEvent); + KSTATUS status = SysWait(waveOut->WorkerHandle, SYS_WAIT_FLAG_NONE); + while (waveOut->ApiUsers > 1) + (void)SysYield(); + + waveOut->Magic = 0; + (void)SysCloseHandle(waveOut->WorkerHandle); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + WinmmReleaseWaveOut(waveOut); + MMRESULT result = status == NTLSTATUS_SUCCESS ? MMSYSERR_NOERROR : WinmmMapNativeStatus(status); + WinmmDebugResult("close complete", waveOut, result); + free(waveOut); + return result; +} + +// Marks one caller-owned header ready for waveform submission. +__declspec(dllexport) MMRESULT waveOutPrepareHeader(HWAVEOUT handle, + WAVEHDR* header, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!header || size < sizeof(*header)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + WinmmLock(&waveOut->Lock); + if (header->dwFlags & WHDR_INQUEUE) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if ((header->dwFlags & WHDR_PREPARED) && + header->reserved != (DWORD_PTR)waveOut) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_ALLOCATED; + } + header->dwFlags &= ~WHDR_DONE; + header->dwFlags |= WHDR_PREPARED; + header->reserved = (DWORD_PTR)waveOut; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Releases preparation state after one header is no longer queued. +__declspec(dllexport) MMRESULT waveOutUnprepareHeader(HWAVEOUT handle, + WAVEHDR* header, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!header || size < sizeof(*header)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + WinmmLock(&waveOut->Lock); + if (header->dwFlags & WHDR_INQUEUE) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if ((header->dwFlags & WHDR_PREPARED) == 0 || + header->reserved != (DWORD_PTR)waveOut) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + header->dwFlags &= ~WHDR_PREPARED; + header->reserved = 0; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Copies one prepared buffer immediately and appends it to the user-mode queue. +__declspec(dllexport) MMRESULT waveOutWrite(HWAVEOUT handle, + WAVEHDR* header, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!header || size < sizeof(*header)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + WinmmLock(&waveOut->Lock); + if ((header->dwFlags & WHDR_PREPARED) == 0 || + header->reserved != (DWORD_PTR)waveOut) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_UNPREPARED; + } + if (header->dwFlags & WHDR_INQUEUE) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if ((header->dwFlags & (WHDR_BEGINLOOP | WHDR_ENDLOOP)) || header->dwLoops) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOTSUPPORTED; + } + if (waveOut->TerminalError) { + MMRESULT error = waveOut->TerminalError; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return error; + } + header->dwFlags &= ~WHDR_DONE; + header->dwFlags |= WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + + WINMM_WAVE_BUFFER* buffer = (WINMM_WAVE_BUFFER*)calloc(1, sizeof(*buffer)); + if (!buffer) { + WinmmLock(&waveOut->Lock); + header->dwFlags &= ~WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOMEM; + } + MMRESULT result = WinmmCopyFrames(waveOut, + header, + &buffer->Frames, + &buffer->FrameCount); + if (result != MMSYSERR_NOERROR) { + WinmmLock(&waveOut->Lock); + header->dwFlags &= ~WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + free(buffer); + WinmmReleaseWaveOut(waveOut); + return result; + } + buffer->Header = header; + if (!buffer->FrameCount) { + buffer->FullySubmitted = 1; + buffer->PresentedTarget = waveOut->NativeSubmitted; + } + + WinmmLock(&waveOut->Lock); + if (waveOut->Closing) { + header->dwFlags &= ~WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + free(buffer->Frames); + free(buffer); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALHANDLE; + } + waveOut->HeadersQueued++; + waveOut->FramesQueued += buffer->FrameCount; + buffer->Sequence = waveOut->HeadersQueued; + if (waveOut->QueueTail) + waveOut->QueueTail->Next = buffer; + else + waveOut->QueueHead = buffer; + waveOut->QueueTail = buffer; + if (!waveOut->SubmitNode) + waveOut->SubmitNode = buffer; + WinmmUnlock(&waveOut->Lock); + (void)SysSignalEvent(waveOut->WakeEvent); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Pauses future source consumption while preserving queued headers. +__declspec(dllexport) MMRESULT waveOutPause(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + MMRESULT result = WinmmSubmitCommand(waveOut, WinmmWaveCommandPause); + WinmmReleaseWaveOut(waveOut); + return result; +} + +// Resumes a user-paused waveform queue. +__declspec(dllexport) MMRESULT waveOutRestart(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + MMRESULT result = WinmmSubmitCommand(waveOut, WinmmWaveCommandRestart); + WinmmReleaseWaveOut(waveOut); + return result; +} + +// Stops playback and returns every queued header as completed. +__declspec(dllexport) MMRESULT waveOutReset(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + MMRESULT result = WinmmSubmitCommand(waveOut, WinmmWaveCommandReset); + WinmmReleaseWaveOut(waveOut); + return result; +} + +// Applies independent WinMM left and right volume values through native gain and pan. +__declspec(dllexport) MMRESULT waveOutSetVolume(HWAVEOUT handle, DWORD volume) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + + uint32_t left = volume & 0xFFFFu; + uint32_t right = (volume >> 16u) & 0xFFFFu; + uint32_t maximum = left > right ? left : right; + uint32_t volumeQ16 = maximum ? (uint32_t)(((uint64_t)maximum * 65536u) / 65535u) : 0u; + int32_t panQ16 = 0; + if (left > right && left) + panQ16 = -(int32_t)(((uint64_t)(left - right) * 65536u) / left); + else if (right > left && right) + panQ16 = (int32_t)(((uint64_t)(right - left) * 65536u) / right); + + SYS_AUDIO_SET_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = waveOut->NativeStream; + request.VolumeQ16 = volumeQ16; + request.PanQ16 = panQ16; + KSTATUS status = SysAudioSetVolume(&request); + if (status == NTLSTATUS_SUCCESS) + waveOut->Volume = volume; + WinmmReleaseWaveOut(waveOut); + return WinmmMapNativeStatus(status); +} + +// Returns the last WinMM left and right volume value accepted for this handle. +__declspec(dllexport) MMRESULT waveOutGetVolume(HWAVEOUT handle, DWORD* volumeOut) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!volumeOut) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + *volumeOut = waveOut->Volume; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Reports source-frame progress using the requested MMTIME unit. +__declspec(dllexport) MMRESULT waveOutGetPosition(HWAVEOUT handle, + MMTIME* timeOut, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!timeOut || size < sizeof(*timeOut)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS) { + WinmmReleaseWaveOut(waveOut); + return WinmmMapNativeStatus(status); + } + uint64_t frames = info.FramesPresented; + if (timeOut->wType == TIME_BYTES) { + uint64_t bytes = frames * waveOut->SourceFormat.nBlockAlign; + timeOut->u.cb = bytes > UINT32_MAX ? UINT32_MAX : (uint32_t)bytes; + } else if (timeOut->wType == TIME_SAMPLES) { + timeOut->u.sample = frames > UINT32_MAX ? UINT32_MAX : (uint32_t)frames; + } else { + uint64_t milliseconds = frames * 1000u / waveOut->SourceFormat.nSamplesPerSec; + timeOut->wType = TIME_MS; + timeOut->u.ms = milliseconds > UINT32_MAX ? UINT32_MAX : (uint32_t)milliseconds; + } + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Returns the device index selected when the waveform handle was opened. +__declspec(dllexport) MMRESULT waveOutGetID(HWAVEOUT handle, UINT* deviceIdOut) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!deviceIdOut) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + *deviceIdOut = waveOut->DeviceId; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Fills the minimal ANSI device capabilities supported by this personality. +__declspec(dllexport) MMRESULT waveOutGetDevCapsA(UINT deviceId, + WAVEOUTCAPSA* caps, + UINT size) { + uint64_t endpointId = 0; + uint32_t resolvedDeviceId = 0; + MMRESULT result = WinmmResolveEndpoint(deviceId, &endpointId, &resolvedDeviceId); + if (result != MMSYSERR_NOERROR) { + WinmmDebugResult("device caps rejected", NULL, result); + return result; + } + (void)endpointId; + (void)resolvedDeviceId; + if (!caps || size < sizeof(*caps)) + return MMSYSERR_INVALPARAM; + + memset(caps, 0, sizeof(*caps)); + caps->vDriverVersion = 0x00010000u; + memcpy(caps->szPname, "nt-lite analog output", sizeof("nt-lite analog output")); + caps->dwFormats = WAVE_FORMAT_1M16 | WAVE_FORMAT_1S16 | + WAVE_FORMAT_2M16 | WAVE_FORMAT_2S16 | + WAVE_FORMAT_4M16 | WAVE_FORMAT_4S16 | + WAVE_FORMAT_48M16 | WAVE_FORMAT_48S16; + caps->wChannels = 2; + caps->dwSupport = WAVECAPS_VOLUME | WAVECAPS_LRVOLUME; + return MMSYSERR_NOERROR; +} + +// Fills the minimal wide device capabilities supported by this personality. +__declspec(dllexport) MMRESULT waveOutGetDevCapsW(UINT deviceId, + WAVEOUTCAPSW* caps, + UINT size) { + if (!caps || size < sizeof(*caps)) + return MMSYSERR_INVALPARAM; + WAVEOUTCAPSA ansi; + MMRESULT result = waveOutGetDevCapsA(deviceId, &ansi, sizeof(ansi)); + if (result != MMSYSERR_NOERROR) + return result; + memset(caps, 0, sizeof(*caps)); + caps->wMid = ansi.wMid; + caps->wPid = ansi.wPid; + caps->vDriverVersion = ansi.vDriverVersion; + for (uint32_t index = 0; index < 32u && ansi.szPname[index]; index++) + caps->szPname[index] = (unsigned short)(uint8_t)ansi.szPname[index]; + caps->dwFormats = ansi.dwFormats; + caps->wChannels = ansi.wChannels; + caps->dwSupport = ansi.dwSupport; + return MMSYSERR_NOERROR; +} + +// Copies one fixed WinMM error description to an ANSI buffer. +__declspec(dllexport) MMRESULT waveOutGetErrorTextA(MMRESULT error, + char* text, + UINT characters) { + if (!text || !characters) + return MMSYSERR_INVALPARAM; + const char* message = "Unknown waveform output error"; + switch (error) { + case MMSYSERR_NOERROR: message = "No error"; break; + case MMSYSERR_BADDEVICEID: message = "Bad waveform output device ID"; break; + case MMSYSERR_INVALHANDLE: message = "Invalid waveform output handle"; break; + case MMSYSERR_NOMEM: message = "Not enough memory"; break; + case MMSYSERR_NOTSUPPORTED: message = "Operation not supported"; break; + case MMSYSERR_INVALFLAG: message = "Invalid waveform output flags"; break; + case MMSYSERR_INVALPARAM: message = "Invalid waveform output parameter"; break; + case WAVERR_BADFORMAT: message = "Unsupported waveform format"; break; + case WAVERR_STILLPLAYING: message = "Waveform buffer is still playing"; break; + case WAVERR_UNPREPARED: message = "Waveform header is not prepared"; break; + default: break; + } + uint32_t index = 0; + while (index + 1u < characters && message[index]) { + text[index] = message[index]; + index++; + } + text[index] = '\0'; + return MMSYSERR_NOERROR; +} + +// Copies one fixed WinMM error description to a wide buffer. +__declspec(dllexport) MMRESULT waveOutGetErrorTextW(MMRESULT error, + unsigned short* text, + UINT characters) { + if (!text || !characters) + return MMSYSERR_INVALPARAM; + char ansi[96]; + MMRESULT result = waveOutGetErrorTextA(error, ansi, sizeof(ansi)); + if (result != MMSYSERR_NOERROR) + return result; + uint32_t index = 0; + while (index + 1u < characters && ansi[index]) { + text[index] = (unsigned short)(uint8_t)ansi[index]; + index++; + } + text[index] = 0; + return MMSYSERR_NOERROR; +} + +// Rejects loop-breaking because looped headers are not part of this implementation. +__declspec(dllexport) MMRESULT waveOutBreakLoop(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOTSUPPORTED; +} + +// Rejects driver-specific waveform messages instead of pretending they succeeded. +__declspec(dllexport) MMRESULT waveOutMessage(HWAVEOUT handle, + UINT message, + DWORD_PTR parameter1, + DWORD_PTR parameter2) { + (void)message; + (void)parameter1; + (void)parameter2; + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOTSUPPORTED; +} diff --git a/code/user/runtime/heap.c b/code/user/runtime/heap.c index 39d5cc8..7af9dd0 100644 --- a/code/user/runtime/heap.c +++ b/code/user/runtime/heap.c @@ -164,7 +164,7 @@ void* _malloc(size_t size) { size = AlignSize(size); HeapLock(); - for (;;) { + while (true) { for (HEAP_SEGMENT* seg = gSegments; seg; seg = seg->Next) { for (HEAP_BLOCK* block = seg->First; block; block = block->Next) { if (block->Magic == USER_HEAP_MAGIC && block->Free && block->Size >= size) { diff --git a/code/user/runtime/native_exception.c b/code/user/runtime/native_exception.c index e3f9c5b..b9b3d0f 100644 --- a/code/user/runtime/native_exception.c +++ b/code/user/runtime/native_exception.c @@ -53,7 +53,7 @@ static void NtlpTerminateExceptionDispatch(void) { request.Disposition = SYS_EXCEPTION_DISPOSITION_TERMINATE_PROCESS; uint32_t status = SysContinueException(&request); - for (;;) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); + while (true) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); } // Asks the kernel to continue searching from the top-level native dispatcher. @@ -64,7 +64,7 @@ static void NtlpContinueExceptionSearch(void) { request.Disposition = SYS_EXCEPTION_DISPOSITION_CONTINUE_SEARCH; uint32_t status = SysContinueException(&request); - for (;;) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); + while (true) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); } // Asks the kernel to resume execution with a handler-edited user context. @@ -76,7 +76,7 @@ static void NtlpContinueExecution(SYS_USER_CONTEXT* context) { request.Context = context; uint32_t status = SysContinueException(&request); - for (;;) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); + while (true) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); } // Returns whether one kernel-provided exception frame has the expected ABI shape. diff --git a/code/user/runtime/native_start.c b/code/user/runtime/native_start.c index fdcdf35..765862f 100644 --- a/code/user/runtime/native_start.c +++ b/code/user/runtime/native_start.c @@ -10,9 +10,9 @@ extern int main(void); void NtlNativeStart(void) { uint32_t status = NtlInitializeExceptionRuntime(); if (status != NTLSTATUS_SUCCESS) { - for (;;) SysExitProcess((int32_t)status); + while (true) SysExitProcess((int32_t)status); } int exitCode = main(); - for (;;) SysExitProcess(exitCode); + while (true) SysExitProcess(exitCode); } diff --git a/code/user/runtime/posix_start.c b/code/user/runtime/posix_start.c index 4615d22..5f3a196 100644 --- a/code/user/runtime/posix_start.c +++ b/code/user/runtime/posix_start.c @@ -12,11 +12,11 @@ extern void PosixFdInit(void); void PosixStart(void) { uint32_t status = NtlInitializeExceptionRuntime(); if (status != NTLSTATUS_SUCCESS) { - for (;;) SysExitProcess((int32_t)status); + while (true) SysExitProcess((int32_t)status); } PosixFdInit(); int exitCode = main(); exit(exitCode); - for (;;) {} + while (true) {} } diff --git a/code/user/syscall/syscall_nr.inc b/code/user/syscall/syscall_nr.inc index 7fa798e..4d7599a 100644 --- a/code/user/syscall/syscall_nr.inc +++ b/code/user/syscall/syscall_nr.inc @@ -10,138 +10,154 @@ SYS_KILL_PROCESS EQU 5 SYS_GET_PROCESS_LIST EQU 6 SYS_GET_PROCESS_PID EQU 7 SYS_CREATE_THREAD EQU 8 -SYS_GET_THREAD_LIST EQU 9 -SYS_LOAD_MODULE EQU 10 -SYS_UNLOAD_MODULE EQU 11 -SYS_GET_MODULE_INFO EQU 12 -SYS_GET_MODULE_EXPORT EQU 13 -SYS_GET_MODULE_LIST EQU 14 -SYS_GET_CURRENT_TID EQU 15 -SYS_GET_CURRENT_PID EQU 16 -SYS_SET_THREAD_PRIORITY EQU 17 -SYS_YIELD EQU 18 -SYS_SLEEP EQU 19 -SYS_WAIT_SINGLE EQU 20 -SYS_WAIT_MANY EQU 21 -SYS_SET_USER_APC_DISPATCHER EQU 22 -SYS_APC_RETURN EQU 23 -SYS_SET_EXCEPTION_DISPATCHER EQU 24 -SYS_GET_EXCEPTION_DISPATCHER EQU 25 -SYS_CONTINUE_EXCEPTION EQU 26 -SYS_RAISE_EXCEPTION EQU 27 +SYS_RESUME_THREAD EQU 9 +SYS_GET_THREAD_LIST EQU 10 +SYS_LOAD_MODULE EQU 11 +SYS_UNLOAD_MODULE EQU 12 +SYS_GET_MODULE_INFO EQU 13 +SYS_GET_MODULE_EXPORT EQU 14 +SYS_GET_MODULE_LIST EQU 15 +SYS_GET_CURRENT_TID EQU 16 +SYS_GET_CURRENT_PID EQU 17 +SYS_SET_THREAD_PRIORITY EQU 18 +SYS_YIELD EQU 19 +SYS_SLEEP EQU 20 +SYS_WAIT_SINGLE EQU 21 +SYS_WAIT_MANY EQU 22 +SYS_SET_USER_APC_DISPATCHER EQU 23 +SYS_APC_RETURN EQU 24 +SYS_SET_EXCEPTION_DISPATCHER EQU 25 +SYS_GET_EXCEPTION_DISPATCHER EQU 26 +SYS_CONTINUE_EXCEPTION EQU 27 +SYS_RAISE_EXCEPTION EQU 28 ; Virtual memory, mappings, and handles. -SYS_ALLOC_VM EQU 28 -SYS_FREE_VM EQU 29 -SYS_PROTECT_VM EQU 30 -SYS_QUERY_VM EQU 31 -SYS_CLOSE_HANDLE EQU 32 -SYS_DUPLICATE_HANDLE EQU 33 -SYS_CREATE_FILE_MAPPING EQU 34 -SYS_MAP_FILE EQU 35 +SYS_ALLOC_VM EQU 29 +SYS_FREE_VM EQU 30 +SYS_PROTECT_VM EQU 31 +SYS_QUERY_VM EQU 32 +SYS_CLOSE_HANDLE EQU 33 +SYS_DUPLICATE_HANDLE EQU 34 +SYS_CREATE_FILE_MAPPING EQU 35 +SYS_MAP_FILE EQU 36 ; Synchronization objects. -SYS_CREATE_MUTEX EQU 36 -SYS_TRY_ACQUIRE_MUTEX EQU 37 -SYS_RELEASE_MUTEX EQU 38 -SYS_CREATE_SEM EQU 39 -SYS_TRY_WAIT_SEM EQU 40 -SYS_SIGNAL_SEM EQU 41 -SYS_CREATE_EVENT EQU 42 -SYS_SIGNAL_EVENT EQU 43 -SYS_CLEAR_EVENT EQU 44 +SYS_CREATE_MUTEX EQU 37 +SYS_TRY_ACQUIRE_MUTEX EQU 38 +SYS_RELEASE_MUTEX EQU 39 +SYS_CREATE_SEM EQU 40 +SYS_TRY_WAIT_SEM EQU 41 +SYS_SIGNAL_SEM EQU 42 +SYS_CREATE_EVENT EQU 43 +SYS_SIGNAL_EVENT EQU 44 +SYS_CLEAR_EVENT EQU 45 ; Queues and named queues. -SYS_CREATE_QUEUE EQU 45 -SYS_SEND_QUEUE EQU 46 -SYS_RECV_QUEUE EQU 47 -SYS_PUBLISH_QUEUE EQU 48 -SYS_OPEN_NAMED_QUEUE EQU 49 -SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS EQU 50 +SYS_CREATE_QUEUE EQU 46 +SYS_SEND_QUEUE EQU 47 +SYS_RECV_QUEUE EQU 48 +SYS_PUBLISH_QUEUE EQU 49 +SYS_OPEN_NAMED_QUEUE EQU 50 +SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS EQU 51 ; Files, directories, and current directory. -SYS_OPEN_FILE EQU 51 -SYS_READ_FILE EQU 52 -SYS_WRITE_FILE EQU 53 -SYS_READ_FILE_ASYNC EQU 54 -SYS_WRITE_FILE_ASYNC EQU 55 -SYS_CANCEL_IO EQU 56 -SYS_SEEK_FILE EQU 57 -SYS_GET_FILE_POS EQU 58 -SYS_GET_FILE_SIZE EQU 59 -SYS_GET_FILE_STAT EQU 60 -SYS_STAT_FILE EQU 61 -SYS_TRUNCATE_FILE EQU 62 -SYS_FLUSH_FILE EQU 63 -SYS_DELETE_FILE EQU 64 -SYS_RENAME_FILE EQU 65 -SYS_LOCK_FILE EQU 66 -SYS_UNLOCK_FILE EQU 67 -SYS_OPEN_DIR EQU 68 -SYS_READ_DIR EQU 69 -SYS_CREATE_DIR EQU 70 -SYS_REMOVE_DIR EQU 71 -SYS_SET_CWD EQU 72 -SYS_REAL_PATH EQU 73 +SYS_OPEN_FILE EQU 52 +SYS_READ_FILE EQU 53 +SYS_WRITE_FILE EQU 54 +SYS_READ_FILE_ASYNC EQU 55 +SYS_WRITE_FILE_ASYNC EQU 56 +SYS_CANCEL_IO EQU 57 +SYS_SEEK_FILE EQU 58 +SYS_GET_FILE_POS EQU 59 +SYS_GET_FILE_SIZE EQU 60 +SYS_GET_FILE_STAT EQU 61 +SYS_STAT_FILE EQU 62 +SYS_TRUNCATE_FILE EQU 63 +SYS_FLUSH_FILE EQU 64 +SYS_DELETE_FILE EQU 65 +SYS_RENAME_FILE EQU 66 +SYS_LOCK_FILE EQU 67 +SYS_UNLOCK_FILE EQU 68 +SYS_OPEN_DIR EQU 69 +SYS_READ_DIR EQU 70 +SYS_CREATE_DIR EQU 71 +SYS_REMOVE_DIR EQU 72 +SYS_SET_CWD EQU 73 +SYS_REAL_PATH EQU 74 ; Console, display, and surfaces. -SYS_WRITE_CONSOLE EQU 74 -SYS_GET_CONSOLE_INFO EQU 75 -SYS_SET_CONSOLE_CURSOR EQU 76 -SYS_FILL_CONSOLE_CHARACTER EQU 77 -SYS_FILL_CONSOLE_ATTRIBUTE EQU 78 -SYS_PUT_SCREEN_CHAR EQU 79 -SYS_CLEAR_SCREEN EQU 80 -SYS_GET_SCREEN_SIZE EQU 81 -SYS_QUERY_DISPLAY_INFO EQU 82 -SYS_OPEN_DISPLAY_CLOCK EQU 83 -SYS_CREATE_SURFACE EQU 84 -SYS_MAP_SURFACE EQU 85 -SYS_QUERY_SURFACE EQU 86 -SYS_PRESENT_SURFACE EQU 87 +SYS_WRITE_CONSOLE EQU 75 +SYS_GET_CONSOLE_INFO EQU 76 +SYS_SET_CONSOLE_CURSOR EQU 77 +SYS_FILL_CONSOLE_CHARACTER EQU 78 +SYS_FILL_CONSOLE_ATTRIBUTE EQU 79 +SYS_PUT_SCREEN_CHAR EQU 80 +SYS_CLEAR_SCREEN EQU 81 +SYS_GET_SCREEN_SIZE EQU 82 +SYS_QUERY_DISPLAY_INFO EQU 83 +SYS_OPEN_DISPLAY_CLOCK EQU 84 +SYS_CREATE_SURFACE EQU 85 +SYS_MAP_SURFACE EQU 86 +SYS_QUERY_SURFACE EQU 87 +SYS_PRESENT_SURFACE EQU 88 + +; Native render audio. +SYS_AUDIO_ENUMERATE_ENDPOINTS EQU 89 +SYS_AUDIO_OPEN_STREAM EQU 90 +SYS_AUDIO_WRITE EQU 91 +SYS_AUDIO_ARM_WAIT EQU 92 +SYS_AUDIO_QUERY_STREAM EQU 93 +SYS_AUDIO_START EQU 94 +SYS_AUDIO_PAUSE EQU 95 +SYS_AUDIO_RESUME EQU 96 +SYS_AUDIO_DRAIN EQU 97 +SYS_AUDIO_STOP EQU 98 +SYS_AUDIO_RESET EQU 99 +SYS_AUDIO_SET_VOLUME EQU 100 +SYS_AUDIO_SET_MASTER_VOLUME EQU 101 ; Input. -SYS_POLL_KEYBOARD EQU 88 -SYS_IS_KEY_DOWN EQU 89 -SYS_READ_KEYBOARD_CHAR EQU 90 -SYS_READ_KEY EQU 91 -SYS_GET_KEY_MODIFIERS EQU 92 -SYS_QUERY_GLOBAL_INPUT_STATE EQU 93 -SYS_OPEN_INPUT_QUEUE EQU 94 -SYS_READ_INPUT_EVENT EQU 95 -SYS_PEEK_INPUT_EVENT EQU 96 -SYS_PEEK_MOUSE EQU 97 -SYS_READ_MOUSE EQU 98 -SYS_GET_MOUSE_POS EQU 99 +SYS_POLL_KEYBOARD EQU 102 +SYS_IS_KEY_DOWN EQU 103 +SYS_READ_KEYBOARD_CHAR EQU 104 +SYS_READ_KEY EQU 105 +SYS_GET_KEY_MODIFIERS EQU 106 +SYS_QUERY_GLOBAL_INPUT_STATE EQU 107 +SYS_OPEN_INPUT_QUEUE EQU 108 +SYS_READ_INPUT_EVENT EQU 109 +SYS_PEEK_INPUT_EVENT EQU 110 +SYS_PEEK_MOUSE EQU 111 +SYS_READ_MOUSE EQU 112 +SYS_GET_MOUSE_POS EQU 113 ; Time, debug, and system information. -SYS_CREATE_TIMER EQU 100 -SYS_SET_TIMER EQU 101 -SYS_CANCEL_TIMER EQU 102 -SYS_GET_TICK EQU 103 -SYS_GET_TICK_FREQUENCY EQU 104 -SYS_GET_SYSTEM_TIME EQU 105 -SYS_PRINT_DEBUG EQU 106 -SYS_QUERY_TRAP_FRAME EQU 107 -SYS_GET_MEMORY_STATUS EQU 108 -SYS_QUERY_KERNEL_STATS EQU 109 +SYS_CREATE_TIMER EQU 114 +SYS_SET_TIMER EQU 115 +SYS_CANCEL_TIMER EQU 116 +SYS_GET_TICK EQU 117 +SYS_GET_TICK_FREQUENCY EQU 118 +SYS_GET_SYSTEM_TIME EQU 119 +SYS_PRINT_DEBUG EQU 120 +SYS_QUERY_TRAP_FRAME EQU 121 +SYS_GET_MEMORY_STATUS EQU 122 +SYS_QUERY_KERNEL_STATS EQU 123 ; Native sockets and polling. -SYS_CREATE_SOCKET EQU 110 -SYS_SOCKET_IO_ASYNC EQU 111 -SYS_BIND_SOCKET EQU 112 -SYS_LISTEN_SOCKET EQU 113 -SYS_ACCEPT_SOCKET EQU 114 -SYS_CONNECT_SOCKET EQU 115 -SYS_SEND_SOCKET EQU 116 -SYS_RECV_SOCKET EQU 117 -SYS_SHUTDOWN_SOCKET EQU 118 -SYS_SEND_TO_SOCKET EQU 119 -SYS_RECV_FROM_SOCKET EQU 120 -SYS_RESOLVE_HOST EQU 121 -SYS_SET_SOCKET_OPTION EQU 122 -SYS_GET_SOCKET_OPTION EQU 123 -SYS_POLL EQU 124 -SYS_UNPUBLISH_QUEUE EQU 125 +SYS_CREATE_SOCKET EQU 124 +SYS_SOCKET_IO_ASYNC EQU 125 +SYS_BIND_SOCKET EQU 126 +SYS_LISTEN_SOCKET EQU 127 +SYS_ACCEPT_SOCKET EQU 128 +SYS_CONNECT_SOCKET EQU 129 +SYS_SEND_SOCKET EQU 130 +SYS_RECV_SOCKET EQU 131 +SYS_SHUTDOWN_SOCKET EQU 132 +SYS_SEND_TO_SOCKET EQU 133 +SYS_RECV_FROM_SOCKET EQU 134 +SYS_RESOLVE_HOST EQU 135 +SYS_SET_SOCKET_OPTION EQU 136 +SYS_GET_SOCKET_OPTION EQU 137 +SYS_POLL EQU 138 +SYS_UNPUBLISH_QUEUE EQU 139 -SYS_MAX EQU 126 +SYS_MAX EQU 140 diff --git a/code/user/syscall/syscall_stubs.asm b/code/user/syscall/syscall_stubs.asm index 87dc71c..b0983ca 100644 --- a/code/user/syscall/syscall_stubs.asm +++ b/code/user/syscall/syscall_stubs.asm @@ -15,11 +15,6 @@ ; SYSCALL_STUB2/3 - 4 or fewer args: pure register passing, no stack read. ; SYSCALL_STUB5 - exactly 5 args: loads [rsp+28h] -> R10 only. ; SYSCALL_STUB6 - 5 or 6 args: loads [rsp+28h] -> R10 and [rsp+30h] -> R11. -; -; Handle lifecycle: -; SysCloseHandle is the single close stub for every object type. -; There is no SysCloseFile - files are closed through the unified path, -; exactly like mutexes, semaphores, events, and queues. INCLUDE syscall_nr.inc @@ -97,6 +92,7 @@ SYSCALL_STUB2 GetModuleExport, SYS_GET_MODULE_EXPORT SYSCALL_STUB2 GetModuleList, SYS_GET_MODULE_LIST SYSCALL_STUB2 CreateThread, SYS_CREATE_THREAD +SYSCALL_STUB2 ResumeThread, SYS_RESUME_THREAD SYSCALL_STUB2 GetCurrentTid, SYS_GET_CURRENT_TID SYSCALL_STUB2 GetCurrentPid, SYS_GET_CURRENT_PID SYSCALL_STUB2 SetThreadPriority, SYS_SET_THREAD_PRIORITY @@ -218,7 +214,24 @@ SYSCALL_STUB2 PresentSurface, SYS_PRESENT_SURFACE SYSCALL_STUB2 ClearScreen, SYS_CLEAR_SCREEN SYSCALL_STUB2 GetScreenSize, SYS_GET_SCREEN_SIZE SYSCALL_STUB2 QueryDisplayInfo, SYS_QUERY_DISPLAY_INFO -SYSCALL_STUB2 OpenDisplayClock, SYS_OPEN_DISPLAY_CLOCK +SYSCALL_STUB2 OpenDisplayClock, SYS_OPEN_DISPLAY_CLOCK + +; --------------------------------------------------------------------------- +; Native render audio +; --------------------------------------------------------------------------- +SYSCALL_STUB2 AudioEnumerateEndpoints, SYS_AUDIO_ENUMERATE_ENDPOINTS +SYSCALL_STUB2 AudioOpenStream, SYS_AUDIO_OPEN_STREAM +SYSCALL_STUB2 AudioWrite, SYS_AUDIO_WRITE +SYSCALL_STUB2 AudioArmWait, SYS_AUDIO_ARM_WAIT +SYSCALL_STUB2 AudioQueryStream, SYS_AUDIO_QUERY_STREAM +SYSCALL_STUB2 AudioStart, SYS_AUDIO_START +SYSCALL_STUB2 AudioPause, SYS_AUDIO_PAUSE +SYSCALL_STUB2 AudioResume, SYS_AUDIO_RESUME +SYSCALL_STUB2 AudioDrain, SYS_AUDIO_DRAIN +SYSCALL_STUB2 AudioStop, SYS_AUDIO_STOP +SYSCALL_STUB2 AudioReset, SYS_AUDIO_RESET +SYSCALL_STUB2 AudioSetVolume, SYS_AUDIO_SET_VOLUME +SYSCALL_STUB2 AudioSetMasterVolume, SYS_AUDIO_SET_MASTER_VOLUME ; --------------------------------------------------------------------------- ; Input diff --git a/code/user/tests/all_tests/all_tests.c b/code/user/tests/all_tests/all_tests.c index d53c95e..4c69636 100644 --- a/code/user/tests/all_tests/all_tests.c +++ b/code/user/tests/all_tests/all_tests.c @@ -57,7 +57,7 @@ static void print_hex32(uint32_t v) { // Exits the process without returning to the runtime. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Converts ASCII to lowercase for stable case-insensitive comparisons. @@ -69,7 +69,7 @@ static char ascii_lower(char c) { // Compares two names case-insensitively, with a case-sensitive tie-breaker. static int compare_names(const char* a, const char* b) { size_t i = 0; - for (;;) { + while (true) { char ca = ascii_lower(a[i]); char cb = ascii_lower(b[i]); if (ca != cb) return (unsigned char)ca < (unsigned char)cb ? -1 : 1; @@ -219,7 +219,7 @@ static uint32_t collect_tests(const char* dir, const char* selfName, TEST_ENTRY* uint32_t count = 0; uint32_t capacity = 0; - for (;;) { + while (true) { SYS_DIR_ENTRY entry; status = SysReadDir(hDir, &entry); if (status != NTLSTATUS_SUCCESS) break; diff --git a/code/user/tests/audio_stream_smoke/audio_stream_smoke.c b/code/user/tests/audio_stream_smoke/audio_stream_smoke.c new file mode 100644 index 0000000..7c77a15 --- /dev/null +++ b/code/user/tests/audio_stream_smoke/audio_stream_smoke.c @@ -0,0 +1,729 @@ +// Native render-stream syscall smoke test. + +#include +#include +#include + +#define TEST_WAIT_MILLISECONDS 3000ull +#define TEST_EXTRA_FRAMES 512u +#define TEST_REFILL_FRAMES 1024u +#define TEST_TONE_FRAMES 48000u +#define TEST_AUDIBLE_FRAMES SYS_AUDIO_STREAM_CAPACITY_FRAMES +#define TEST_AUDIBLE_RAMP_FRAMES 240u +#define TEST_CLOSE_RUNDOWN_ITERATIONS 72u +#define TEST_CLOSE_RUNDOWN_SLEEP_MILLISECONDS 5ull +#define TEST_IDLE_QUIESCE_MILLISECONDS 300ull +#define TEST_TONE_PHASE_MODULUS 1200u +#define TEST_TONE_PHASE_STEP 11u +#define TEST_TONE_QUARTER_PHASE 300u +#define TEST_TONE_PEAK 8192 + +static int16_t gFrames[TEST_TONE_FRAMES * 2u]; +static int16_t gAudibleFrames[TEST_AUDIBLE_FRAMES * 2u]; +static uint32_t gOpenStream; + +typedef char TEST_AUDIO_FORMAT_SIZE[(sizeof(SYS_AUDIO_FORMAT) == 8) ? 1 : -1]; +typedef char TEST_AUDIO_ENDPOINT_INFO_SIZE[(sizeof(SYS_AUDIO_ENDPOINT_INFO) == 64) ? 1 : -1]; +typedef char TEST_AUDIO_ENUMERATE_REQUEST_SIZE[(sizeof(SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST) == 32) ? 1 : -1]; +typedef char TEST_AUDIO_OPEN_REQUEST_SIZE[(sizeof(SYS_AUDIO_OPEN_STREAM_REQUEST) == 40) ? 1 : -1]; +typedef char TEST_AUDIO_WRITE_REQUEST_SIZE[(sizeof(SYS_AUDIO_WRITE_REQUEST) == 32) ? 1 : -1]; +typedef char TEST_AUDIO_ARM_WAIT_REQUEST_SIZE[(sizeof(SYS_AUDIO_ARM_WAIT_REQUEST) == 56) ? 1 : -1]; +typedef char TEST_AUDIO_STREAM_INFO_SIZE[(sizeof(SYS_AUDIO_STREAM_INFO) == 128) ? 1 : -1]; +typedef char TEST_AUDIO_VOLUME_REQUEST_SIZE[(sizeof(SYS_AUDIO_SET_VOLUME_REQUEST) == 24) ? 1 : -1]; +typedef char TEST_AUDIO_MASTER_VOLUME_REQUEST_SIZE[(sizeof(SYS_AUDIO_SET_MASTER_VOLUME_REQUEST) == 24) ? 1 : -1]; + +// Writes one NUL-terminated string to the console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Writes one unsigned integer to the console. +static void print_u64(uint64_t value) { + char digits[32]; + uint32_t count = 0; + if (!value) { + print("0"); + return; + } + while (value && count < sizeof(digits)) { + digits[count++] = (char)('0' + value % 10u); + value /= 10u; + } + while (count) { + char character = digits[--count]; + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromParts(&character, 1) }; + (void)SysWriteConsole(&request); + } +} + +// Stops and closes the canonical stream after success, failure, or process interruption. +static void cleanup_stream(void) { + if (!gOpenStream) + return; + (void)SysAudioStop(gOpenStream); + (void)SysCloseHandle(gOpenStream); + gOpenStream = 0; +} + +// Terminates the test process. +static void exit_now(int32_t code) { + cleanup_stream(); + while (true) (void)SysExitProcess(code); +} + +// Fails the test with one status value. +static void fail_status(const char* operation, KSTATUS status) { + print("audio_stream_smoke: "); + print(operation); + print(" status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +// Fails the test with one plain diagnostic. +static void fail(const char* message) { + print("audio_stream_smoke: "); + print(message); + print("\n"); + exit_now(1); +} + +// Returns one exact 440 Hz integer triangle-wave sample for the supplied phase. +static int16_t tone_sample(uint32_t phase) { + int32_t sample; + if (phase < TEST_TONE_QUARTER_PHASE) { + sample = (int32_t)(phase * TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } else if (phase < TEST_TONE_QUARTER_PHASE * 2u) { + sample = (int32_t)((TEST_TONE_QUARTER_PHASE * 2u - phase) * + TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } else if (phase < TEST_TONE_QUARTER_PHASE * 3u) { + sample = -(int32_t)((phase - TEST_TONE_QUARTER_PHASE * 2u) * + TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } else { + sample = -(int32_t)((TEST_TONE_PHASE_MODULUS - phase) * + TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } + return (int16_t)sample; +} + +// Fills contract and audible buffers without requiring floating-point support. +static void fill_test_frames(void) { + uint32_t phase = 0; + for (uint32_t frame = 0; frame < TEST_TONE_FRAMES; frame++) { + int16_t sample = tone_sample(phase); + gFrames[frame * 2u] = sample; + gFrames[frame * 2u + 1u] = sample; + phase += TEST_TONE_PHASE_STEP; + if (phase >= TEST_TONE_PHASE_MODULUS) + phase -= TEST_TONE_PHASE_MODULUS; + } + + phase = 0; + for (uint32_t frame = 0; frame < TEST_AUDIBLE_FRAMES; frame++) { + int32_t sample = tone_sample(phase); + uint32_t gain = TEST_AUDIBLE_RAMP_FRAMES; + if (frame < gain) + gain = frame; + uint32_t remaining = TEST_AUDIBLE_FRAMES - 1u - frame; + if (remaining < gain) + gain = remaining; + sample = sample * (int32_t)gain / (int32_t)TEST_AUDIBLE_RAMP_FRAMES; + gAudibleFrames[frame * 2u] = (int16_t)sample; + gAudibleFrames[frame * 2u + 1u] = (int16_t)sample; + phase += TEST_TONE_PHASE_STEP; + if (phase >= TEST_TONE_PHASE_MODULUS) + phase -= TEST_TONE_PHASE_MODULUS; + } +} + +// Queries one stream with the required public structure size. +static SYS_AUDIO_STREAM_INFO query_stream(uint32_t stream) { + SYS_AUDIO_STREAM_INFO info; + memset(&info, 0, sizeof(info)); + info.Size = sizeof(info); + KSTATUS status = SysAudioQueryStream(stream, &info); + if (status != NTLSTATUS_SUCCESS) + fail_status("query stream", status); + if (info.Version != SYS_AUDIO_ABI_VERSION) + fail("unexpected audio ABI version"); + return info; +} + +// Applies one valid stream volume, pan, and mute setting. +static void set_stream_mix(uint32_t stream, + uint32_t volumeQ16, + int32_t panQ16, + uint32_t muted) { + SYS_AUDIO_SET_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = stream; + request.Muted = muted; + request.VolumeQ16 = volumeQ16; + request.PanQ16 = panQ16; + KSTATUS status = SysAudioSetVolume(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("set stream mix", status); +} + +// Selects the available default endpoint, or the first available endpoint. +static const SYS_AUDIO_ENDPOINT_INFO* select_endpoint(const SYS_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t count) { + const SYS_AUDIO_ENDPOINT_INFO* fallback = 0; + for (uint32_t index = 0; index < count; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->Size != sizeof(*endpoint) || + endpoint->Version != SYS_AUDIO_ABI_VERSION || + endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) fallback = endpoint; + if (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT) + return endpoint; + } + return fallback; +} + +// Queries one endpoint by its stable identifier. +static SYS_AUDIO_ENDPOINT_INFO query_endpoint(uint64_t endpointId) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + request.CountOut = &count; + KSTATUS status = SysAudioEnumerateEndpoints(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("query endpoint", status); + + for (uint32_t index = 0; index < count; index++) { + if (endpoints[index].EndpointId == endpointId) + return endpoints[index]; + } + fail("endpoint disappeared during control test"); + return endpoints[0]; +} + +// Verifies endpoint master volume, mute, validation, and restoration. +static void test_master_volume(const SYS_AUDIO_ENDPOINT_INFO* endpoint) { + SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.EndpointId = endpoint->EndpointId; + request.Muted = 1u; + request.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY / 2u; + + KSTATUS status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("set master volume", status); + SYS_AUDIO_ENDPOINT_INFO updated = query_endpoint(endpoint->EndpointId); + if (updated.MasterVolumeQ16 != SYS_AUDIO_GAIN_Q16_UNITY / 2u || + updated.MasterMuted != 1u) + fail("master volume snapshot mismatch"); + + request.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY + 1u; + status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_INVALID_PARAMETER) + fail_status("reject master volume above unity", status); + + request.Muted = endpoint->MasterMuted; + request.VolumeQ16 = endpoint->MasterVolumeQ16; + status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("restore master volume", status); +} + +// Opens one stream format and checks the expected status. +static uint32_t open_stream_format(uint64_t endpointId, + uint32_t sampleRate, + uint16_t channelCount, + uint16_t sampleFormat, + KSTATUS expectedStatus) { + uint32_t stream = 0; + SYS_AUDIO_OPEN_STREAM_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.EndpointId = endpointId; + request.Format.SampleRate = sampleRate; + request.Format.ChannelCount = channelCount; + request.Format.SampleFormat = sampleFormat; + request.StreamHandleOut = &stream; + KSTATUS status = SysAudioOpenStream(&request); + if (status != expectedStatus) + fail_status("open stream format", status); + if (status == NTLSTATUS_SUCCESS && !stream) + fail("open returned a null handle"); + if (status != NTLSTATUS_SUCCESS && stream) + fail("rejected format returned a handle"); + return stream; +} + +// Proves the complete accepted and rejected source-format surface. +static void test_format_contract(uint64_t endpointId) { + static const uint32_t rates[] = { 11025u, 22050u, 44100u, 48000u }; + for (uint32_t rateIndex = 0; rateIndex < sizeof(rates) / sizeof(rates[0]); rateIndex++) { + for (uint16_t channels = 1; channels <= 2; channels++) { + uint32_t stream = open_stream_format(endpointId, + rates[rateIndex], + channels, + SysAudioSampleS16, + NTLSTATUS_SUCCESS); + KSTATUS status = SysCloseHandle(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("close format stream", status); + } + } + + (void)open_stream_format(endpointId, 48000u, 2, 2, NTLSTATUS_NOT_SUPPORTED); + (void)open_stream_format(endpointId, 32000u, 2, SysAudioSampleS16, NTLSTATUS_NOT_SUPPORTED); + (void)open_stream_format(endpointId, 48000u, 3, SysAudioSampleS16, NTLSTATUS_NOT_SUPPORTED); +} + +// Opens the canonical test stream. +static uint32_t open_stream(uint64_t endpointId) { + return open_stream_format(endpointId, + 48000u, + 2, + SysAudioSampleS16, + NTLSTATUS_SUCCESS); +} + +// Writes one sample run and returns the accepted frame count. +static uint32_t write_samples(uint32_t stream, + const int16_t* frames, + uint32_t frameCount, + KSTATUS expectedStatus) { + uint32_t written = 0xFFFFFFFFu; + SYS_AUDIO_WRITE_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = stream; + request.FrameCount = frameCount; + request.Frames = frameCount ? frames : 0; + request.WrittenFramesOut = &written; + KSTATUS status = SysAudioWrite(&request); + if (status != expectedStatus) fail_status("write stream", status); + if (status != NTLSTATUS_SUCCESS && written != 0) + fail("failed write reported accepted frames"); + return written; +} + +// Writes one contract-tone range and returns the accepted frame count. +static uint32_t write_frames(uint32_t stream, + uint32_t frameOffset, + uint32_t frameCount, + KSTATUS expectedStatus) { + if (frameOffset > TEST_TONE_FRAMES || + frameCount > TEST_TONE_FRAMES - frameOffset) + fail("test write range exceeded the tone buffer"); + return write_samples(stream, + frameCount ? gFrames + (size_t)frameOffset * 2u : 0, + frameCount, + expectedStatus); +} + +// Arms one exact set of stream progress targets. +static void arm_wait(uint32_t stream, + uint32_t mask, + uint32_t writableFrames, + uint64_t consumedFrames, + uint64_t presentedFrames, + uint64_t drainedGeneration, + uint64_t discontinuityGeneration) { + SYS_AUDIO_ARM_WAIT_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = stream; + request.Mask = mask; + request.WritableFrames = writableFrames; + request.ConsumedFrames = consumedFrames; + request.PresentedFrames = presentedFrames; + request.DrainedGeneration = drainedGeneration; + request.DiscontinuityGeneration = discontinuityGeneration; + KSTATUS status = SysAudioArmWait(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("arm wait", status); +} + +// Waits once and converts timeout or alert completion into a focused failure. +static void wait_stream(uint32_t stream, const char* operation) { + KSTATUS status = SysWaitSingle(stream, SYS_WAIT_FLAG_NONE, TEST_WAIT_MILLISECONDS); + if (status != NTLSTATUS_SUCCESS) + fail_status(operation, status); +} + +// Proves that closing the final handle retires an active stream with queued periods. +static void test_active_close_rundown(uint64_t endpointId) { + for (uint32_t iteration = 0; + iteration < TEST_CLOSE_RUNDOWN_ITERATIONS; + iteration++) { + uint32_t stream = open_stream(endpointId); + gOpenStream = stream; + set_stream_mix(stream, + SYS_AUDIO_GAIN_Q16_UNITY, + SYS_AUDIO_PAN_Q16_CENTER, + 1u); + uint32_t written = write_frames(stream, + 0, + SYS_AUDIO_STREAM_CAPACITY_FRAMES, + NTLSTATUS_SUCCESS); + if (written != SYS_AUDIO_STREAM_CAPACITY_FRAMES) + fail("active-close prefill was partial"); + + SYS_AUDIO_STREAM_INFO before = query_stream(stream); + arm_wait(stream, + SYS_AUDIO_WAIT_CONSUMED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + before.FramesConsumed + TEST_EXTRA_FRAMES, + before.FramesPresented, + before.DrainedGeneration, + before.DiscontinuityGeneration); + KSTATUS status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("active-close start", status); + wait_stream(stream, "active-close consumed wait"); + + SYS_AUDIO_STREAM_INFO active = query_stream(stream); + if (active.State == SysAudioStreamFailed) + fail_status("active-close stream failure", active.FailureStatus); + if (active.DiscontinuityGeneration != before.DiscontinuityGeneration || + active.FramesConsumed < before.FramesConsumed + TEST_EXTRA_FRAMES) + fail("active-close progress mismatch"); + + status = SysCloseHandle(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("active-close final handle", status); + gOpenStream = 0; + status = SysSleep(TEST_CLOSE_RUNDOWN_SLEEP_MILLISECONDS); + if (status != NTLSTATUS_SUCCESS) + fail_status("active-close rundown sleep", status); + } +} + +// Plays one bounded audible tone without depending on real-time user refills. +static void play_continuous_tone(uint32_t stream) { + KSTATUS status = SysAudioReset(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("audible reset", status); + set_stream_mix(stream, + SYS_AUDIO_GAIN_Q16_UNITY, + SYS_AUDIO_PAN_Q16_CENTER, + 0u); + + SYS_AUDIO_STREAM_INFO baseline = query_stream(stream); + uint32_t submitted = write_samples(stream, + gAudibleFrames, + TEST_AUDIBLE_FRAMES, + NTLSTATUS_SUCCESS); + if (submitted != TEST_AUDIBLE_FRAMES) + fail("audible prefill was partial"); + + status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("audible start", status); + status = SysAudioDrain(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("audible drain", status); + print("audio_stream_smoke: audible 440 Hz tone begin\n"); + + arm_wait(stream, + SYS_AUDIO_WAIT_DRAINED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + baseline.FramesConsumed, + baseline.FramesPresented, + baseline.DrainedGeneration, + baseline.DiscontinuityGeneration); + wait_stream(stream, "audible drained wait"); + + SYS_AUDIO_STREAM_INFO drained = query_stream(stream); + if (drained.State == SysAudioStreamFailed) + fail_status("audible stream failure", drained.FailureStatus); + if (drained.State != SysAudioStreamStopped || + drained.DrainedGeneration != baseline.DrainedGeneration + 1u || + drained.DiscontinuityGeneration != baseline.DiscontinuityGeneration || + drained.FramesSubmitted != TEST_AUDIBLE_FRAMES || + drained.FramesConsumed != TEST_AUDIBLE_FRAMES || + drained.FramesPresented != TEST_AUDIBLE_FRAMES || + drained.BufferedFrames != 0 || + drained.Underruns != 0 || + drained.SilenceFramesInserted != 0) + fail("audible tone completion mismatch"); + print("audio_stream_smoke: audible 440 Hz tone complete\n"); +} + +// Runs the complete native stream contract against one endpoint. +static void test_stream(const SYS_AUDIO_ENDPOINT_INFO* endpoint) { + test_format_contract(endpoint->EndpointId); + test_active_close_rundown(endpoint->EndpointId); + + uint32_t stream = open_stream(endpoint->EndpointId); + gOpenStream = stream; + KSTATUS status; + SYS_AUDIO_STREAM_INFO info = query_stream(stream); + if (info.State != SysAudioStreamCreated || + info.CapacityFrames != SYS_AUDIO_STREAM_CAPACITY_FRAMES || + info.Format.SampleFormat != SysAudioSampleS16) + fail("initial stream snapshot mismatch"); + + SYS_AUDIO_SET_VOLUME_REQUEST volume; + memset(&volume, 0, sizeof(volume)); + volume.Size = sizeof(volume); + volume.StreamHandle = stream; + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY / 2u; + volume.PanQ16 = SYS_AUDIO_PAN_Q16_CENTER; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_SUCCESS) + fail_status("set volume", status); + info = query_stream(stream); + if (info.VolumeQ16 != SYS_AUDIO_GAIN_Q16_UNITY / 2u || + info.PanQ16 != SYS_AUDIO_PAN_Q16_CENTER || info.Muted) + fail("volume snapshot mismatch"); + + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY + 1u; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_INVALID_PARAMETER) + fail_status("reject volume above unity", status); + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY / 2u; + volume.PanQ16 = SYS_AUDIO_PAN_Q16_RIGHT + 1; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_INVALID_PARAMETER) + fail_status("reject pan outside range", status); + + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY; + volume.PanQ16 = SYS_AUDIO_PAN_Q16_CENTER; + volume.Muted = 1u; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_SUCCESS) + fail_status("mute control stream", status); + + uint32_t written = write_frames(stream, 0, 0, NTLSTATUS_SUCCESS); + if (written != 0) fail("zero-length write accepted frames"); + written = write_frames(stream, + 0, + SYS_AUDIO_STREAM_CAPACITY_FRAMES + TEST_EXTRA_FRAMES, + NTLSTATUS_SUCCESS); + if (written != SYS_AUDIO_STREAM_CAPACITY_FRAMES) + fail("partial write did not fill exactly one stream ring"); + written = write_frames(stream, 0, 1, NTLSTATUS_WOULD_BLOCK); + if (written != 0) fail("full-ring write accepted a frame"); + + info = query_stream(stream); + arm_wait(stream, + SYS_AUDIO_WAIT_WRITABLE | + SYS_AUDIO_WAIT_CONSUMED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + TEST_EXTRA_FRAMES, + info.FramesConsumed + TEST_EXTRA_FRAMES, + 0, + info.DrainedGeneration, + info.DiscontinuityGeneration); + status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("start", status); + wait_stream(stream, "writable wait"); + + SYS_AUDIO_STREAM_INFO progressed = query_stream(stream); + if (progressed.State == SysAudioStreamFailed) + fail_status("stream failure", progressed.FailureStatus); + if (progressed.DiscontinuityGeneration != info.DiscontinuityGeneration) + fail("unexpected discontinuity before writable target"); + if (progressed.WritableFrames < TEST_EXTRA_FRAMES || + progressed.FramesConsumed < info.FramesConsumed + TEST_EXTRA_FRAMES) + fail("writable or consumed wait did not observe its target"); + + status = SysAudioPause(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("pause", status); + if (query_stream(stream).State != SysAudioStreamPaused) + fail("pause state mismatch"); + status = SysAudioResume(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("resume", status); + + uint32_t submitted = SYS_AUDIO_STREAM_CAPACITY_FRAMES; + written = write_frames(stream, submitted, TEST_EXTRA_FRAMES, NTLSTATUS_SUCCESS); + if (written != TEST_EXTRA_FRAMES) + fail("resume refill was partial"); + submitted += written; + + while (submitted < TEST_TONE_FRAMES) { + SYS_AUDIO_STREAM_INFO refill = query_stream(stream); + if (refill.State == SysAudioStreamFailed) + fail_status("stream failure", refill.FailureStatus); + if (refill.DiscontinuityGeneration != info.DiscontinuityGeneration) + fail("unexpected discontinuity during muted control refill"); + + uint32_t remaining = TEST_TONE_FRAMES - submitted; + uint32_t refillFrames = remaining < TEST_REFILL_FRAMES + ? remaining + : TEST_REFILL_FRAMES; + if (refill.WritableFrames < refillFrames) { + arm_wait(stream, + SYS_AUDIO_WAIT_WRITABLE | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + refillFrames, + refill.FramesConsumed, + refill.FramesPresented, + refill.DrainedGeneration, + info.DiscontinuityGeneration); + wait_stream(stream, "muted control refill wait"); + continue; + } + + written = write_frames(stream, submitted, refillFrames, NTLSTATUS_SUCCESS); + if (written != refillFrames) + fail("muted control refill was partial despite reserved capacity"); + submitted += written; + } + + SYS_AUDIO_STREAM_INFO beforeDrain = query_stream(stream); + if (beforeDrain.FramesSubmitted != TEST_TONE_FRAMES) + fail("muted control frame submission mismatch"); + uint64_t presentedTarget = beforeDrain.FramesSubmitted; + status = SysAudioDrain(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("drain", status); + arm_wait(stream, + SYS_AUDIO_WAIT_PRESENTED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + beforeDrain.FramesConsumed, + presentedTarget, + beforeDrain.DrainedGeneration, + beforeDrain.DiscontinuityGeneration); + wait_stream(stream, "presentation wait"); + + SYS_AUDIO_STREAM_INFO presented = query_stream(stream); + if (presented.State == SysAudioStreamFailed) + fail_status("stream failure", presented.FailureStatus); + if (presented.DiscontinuityGeneration != beforeDrain.DiscontinuityGeneration) + fail("unexpected discontinuity before presentation target"); + if (presented.FramesPresented < presentedTarget) + fail("presented wait did not observe its target"); + + if (presented.State != SysAudioStreamStopped) { + arm_wait(stream, + SYS_AUDIO_WAIT_DRAINED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + presented.FramesConsumed, + presented.FramesPresented, + beforeDrain.DrainedGeneration, + beforeDrain.DiscontinuityGeneration); + wait_stream(stream, "drained wait"); + } + + SYS_AUDIO_STREAM_INFO drained = query_stream(stream); + if (drained.State == SysAudioStreamFailed) + fail_status("stream failure", drained.FailureStatus); + if (drained.DiscontinuityGeneration != beforeDrain.DiscontinuityGeneration) + fail("unexpected discontinuity during drain"); + if (drained.State != SysAudioStreamStopped || + drained.DrainedGeneration != beforeDrain.DrainedGeneration + 1u || + drained.BufferedFrames != 0 || + drained.FramesPresented < presentedTarget || + drained.FramesPresented + drained.FramesDiscarded < drained.FramesSubmitted) + fail("drained stream snapshot mismatch"); + + status = SysAudioReset(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("reset", status); + SYS_AUDIO_STREAM_INFO reset = query_stream(stream); + if (reset.State != SysAudioStreamStopped || reset.FramesSubmitted != 0 || + reset.FramesConsumed != 0 || reset.FramesPresented != 0 || + reset.DrainedGeneration != drained.DrainedGeneration) + fail("reset stream snapshot mismatch"); + + written = write_frames(stream, 0, 128, NTLSTATUS_SUCCESS); + if (written != 128) fail("stop test write was partial"); + status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("restart", status); + SYS_AUDIO_STREAM_INFO beforeStop = query_stream(stream); + arm_wait(stream, + SYS_AUDIO_WAIT_STOPPED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + beforeStop.FramesConsumed, + beforeStop.FramesPresented, + beforeStop.DrainedGeneration, + beforeStop.DiscontinuityGeneration); + status = SysAudioStop(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("stop", status); + wait_stream(stream, "stopped wait"); + if (query_stream(stream).State != SysAudioStreamStopped) + fail("stop state mismatch"); + + play_continuous_tone(stream); + + status = SysCloseHandle(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("close stream", status); + gOpenStream = 0; + status = SysSleep(TEST_IDLE_QUIESCE_MILLISECONDS); + if (status != NTLSTATUS_SUCCESS) + fail_status("endpoint quiesce sleep", status); +} + +// Executes the native audio syscall smoke test. +int main(void) { + fill_test_frames(); + print("audio_stream_smoke: begin native render contract test\n"); + + uint32_t count = 0; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST countRequest; + memset(&countRequest, 0, sizeof(countRequest)); + countRequest.Size = sizeof(countRequest); + countRequest.CountOut = &count; + KSTATUS status = SysAudioEnumerateEndpoints(&countRequest); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_BUFFER_TOO_SMALL) + fail_status("count endpoints", status); + if (!count) { + print("audio_stream_smoke: SKIP no render endpoint\n"); + exit_now(0); + } + if (count > SYS_AUDIO_MAX_ENDPOINTS) + fail("endpoint count exceeded ABI maximum"); + + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + memset(endpoints, 0, sizeof(endpoints)); + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST enumerate; + memset(&enumerate, 0, sizeof(enumerate)); + enumerate.Size = sizeof(enumerate); + enumerate.Endpoints = endpoints; + enumerate.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + enumerate.CountOut = &count; + status = SysAudioEnumerateEndpoints(&enumerate); + if (status != NTLSTATUS_SUCCESS) + fail_status("enumerate endpoints", status); + + const SYS_AUDIO_ENDPOINT_INFO* endpoint = select_endpoint(endpoints, count); + if (!endpoint) { + print("audio_stream_smoke: SKIP no available render endpoint\n"); + exit_now(0); + } + + print("audio_stream_smoke: endpoint="); + print_u64(endpoint->EndpointId); + print("\n"); + test_master_volume(endpoint); + test_stream(endpoint); + print("audio_stream_smoke: PASS endpoint="); + print_u64(endpoint->EndpointId); + print("\n"); + exit_now(0); +} diff --git a/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c b/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c index 787ea18..6fccb5f 100644 --- a/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c +++ b/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/dirty_writeback/dirty_writeback.c b/code/user/tests/dirty_writeback/dirty_writeback.c index 40b8d0d..bab15a1 100644 --- a/code/user/tests/dirty_writeback/dirty_writeback.c +++ b/code/user/tests/dirty_writeback/dirty_writeback.c @@ -38,7 +38,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { @@ -62,9 +62,10 @@ static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); } -static void delete_path(const char* path) { +// Deletes one test path. +static uint32_t delete_path(const char* path) { SYS_PATH_REQUEST req = { SysSvFromCstr(path) }; - (void)SysDeleteFile(&req); + return SysDeleteFile(&req); } static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { @@ -80,7 +81,7 @@ static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { uint32_t h = 0; - delete_path(paths[i]); + (void)delete_path(paths[i]); SYS_OPEN_FILE_REQUEST req = SysMakeOpenFileRequest(SysSvFromCstr(paths[i]), SYS_FILE_ACCESS_READ | SYS_FILE_ACCESS_WRITE | SYS_FILE_ACCESS_QUERY, SYS_FILE_CREATE_NEW, @@ -222,7 +223,16 @@ int main(void) { s = SysCloseHandle(file); file = 0; if (s != NTLSTATUS_SUCCESS) fail_status("close reopened", s); - delete_path(path); + s = delete_path(path); + if (s != NTLSTATUS_SUCCESS) fail_status("delete temp", s); + + uint32_t deletedHandle = 0; + s = open_path(path, &deletedHandle); + if (s == NTLSTATUS_SUCCESS) { + (void)SysCloseHandle(deletedHandle); + fail("deleted file reopened"); + } + if (s != NTLSTATUS_NOT_FOUND) fail_status("verify delete", s); print("dirty_writeback: dirty-marks+="); print_u64(afterClose.DirtyMarks - before.DirtyMarks); diff --git a/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c b/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c index 1e7f2d0..f640acd 100644 --- a/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c +++ b/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/exception_smoke/exception_smoke.c b/code/user/tests/exception_smoke/exception_smoke.c index bdf6853..93987b3 100644 --- a/code/user/tests/exception_smoke/exception_smoke.c +++ b/code/user/tests/exception_smoke/exception_smoke.c @@ -40,7 +40,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/file_lock_smoke/file_lock_smoke.c b/code/user/tests/file_lock_smoke/file_lock_smoke.c index b9bbd73..54d5110 100644 --- a/code/user/tests/file_lock_smoke/file_lock_smoke.c +++ b/code/user/tests/file_lock_smoke/file_lock_smoke.c @@ -39,7 +39,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/file_stat_smoke/file_stat_smoke.c b/code/user/tests/file_stat_smoke/file_stat_smoke.c index cdd5085..32a6eee 100644 --- a/code/user/tests/file_stat_smoke/file_stat_smoke.c +++ b/code/user/tests/file_stat_smoke/file_stat_smoke.c @@ -36,7 +36,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/handle_dup_smoke/handle_dup_smoke.c b/code/user/tests/handle_dup_smoke/handle_dup_smoke.c index 64f4b63..1d95351 100644 --- a/code/user/tests/handle_dup_smoke/handle_dup_smoke.c +++ b/code/user/tests/handle_dup_smoke/handle_dup_smoke.c @@ -42,7 +42,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/input_api_smoke/input_api_smoke.c b/code/user/tests/input_api_smoke/input_api_smoke.c index e370778..9b9fc2e 100644 --- a/code/user/tests/input_api_smoke/input_api_smoke.c +++ b/code/user/tests/input_api_smoke/input_api_smoke.c @@ -32,7 +32,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/input_model_smoke/input_model_smoke.c b/code/user/tests/input_model_smoke/input_model_smoke.c index 98ea49d..8f19d80 100644 --- a/code/user/tests/input_model_smoke/input_model_smoke.c +++ b/code/user/tests/input_model_smoke/input_model_smoke.c @@ -31,7 +31,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/mmap_smoke/mmap_smoke.c b/code/user/tests/mmap_smoke/mmap_smoke.c index 438e407..8830bbb 100644 --- a/code/user/tests/mmap_smoke/mmap_smoke.c +++ b/code/user/tests/mmap_smoke/mmap_smoke.c @@ -35,7 +35,7 @@ static void print_u64(uint64_t v) { static void exit_now(int code) { // SysExitProcess should not return, but keep tests from falling through to // the synthetic entry frame if a broken kernel build returns anyway. - for (;;) + while (true) SysExitProcess(code); } diff --git a/code/user/tests/module_load_smoke/module_load_smoke.c b/code/user/tests/module_load_smoke/module_load_smoke.c index 4120dc7..f26d51d 100644 --- a/code/user/tests/module_load_smoke/module_load_smoke.c +++ b/code/user/tests/module_load_smoke/module_load_smoke.c @@ -32,7 +32,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c b/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c index 5b0d180..8bc79d6 100644 --- a/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c +++ b/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c @@ -45,7 +45,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c b/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c index 3dae79f..9fcda32 100644 --- a/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c +++ b/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c @@ -35,7 +35,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c b/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c index 72a0124..bbfac44 100644 --- a/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c +++ b/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c @@ -36,7 +36,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/net_irp_smoke/net_irp_smoke.c b/code/user/tests/net_irp_smoke/net_irp_smoke.c index f9b6cf7..88fb0c3 100644 --- a/code/user/tests/net_irp_smoke/net_irp_smoke.c +++ b/code/user/tests/net_irp_smoke/net_irp_smoke.c @@ -36,7 +36,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/pcache_invalidate/pcache_invalidate.c b/code/user/tests/pcache_invalidate/pcache_invalidate.c index 1e369b2..4bf2918 100644 --- a/code/user/tests/pcache_invalidate/pcache_invalidate.c +++ b/code/user/tests/pcache_invalidate/pcache_invalidate.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c b/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c index 21cdac9..d15d711 100644 --- a/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c +++ b/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c @@ -30,7 +30,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/pcache_multithread/pcache_multithread.c b/code/user/tests/pcache_multithread/pcache_multithread.c index 6ae8442..77c6ce8 100644 --- a/code/user/tests/pcache_multithread/pcache_multithread.c +++ b/code/user/tests/pcache_multithread/pcache_multithread.c @@ -41,7 +41,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/pcache_pressure/pcache_pressure.c b/code/user/tests/pcache_pressure/pcache_pressure.c index d33e45f..915aeb3 100644 --- a/code/user/tests/pcache_pressure/pcache_pressure.c +++ b/code/user/tests/pcache_pressure/pcache_pressure.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { @@ -147,11 +147,6 @@ int main(void) { uint64_t loads = after.Loads - before.Loads; uint64_t hits = after.Hits - before.Hits; uint64_t drops = after.Drops - before.Drops; - uint64_t raReq = after.ReadAheadRequests - before.ReadAheadRequests; - uint64_t raLoad = after.ReadAheadPagesLoaded - before.ReadAheadPagesLoaded; - uint64_t raThrottle = after.ReadAheadPagesThrottled - before.ReadAheadPagesThrottled; - uint64_t batchReads = after.BatchReads - before.BatchReads; - uint64_t batchPages = after.BatchPagesLoaded - before.BatchPagesLoaded; print("pcache_pressure: read bytes="); print_u64(total); print(" loads+="); print_u64(loads); @@ -161,11 +156,6 @@ int main(void) { print(" active="); print_u64(after.ActivePages); print(" standby="); print_u64(after.StandbyPages); print(" max="); print_u64(after.MaxResidentPages); - print(" readahead-req+="); print_u64(raReq); - print(" readahead-load+="); print_u64(raLoad); - print(" readahead-throttle+="); print_u64(raThrottle); - print(" batch-reads+="); print_u64(batchReads); - print(" batch-pages+="); print_u64(batchPages); print("\n"); if (size >= MIN_PRESSURE_BYTES) { diff --git a/code/user/tests/pcache_smoke/pcache_smoke.c b/code/user/tests/pcache_smoke/pcache_smoke.c index 3fcc798..c88d35d 100644 --- a/code/user/tests/pcache_smoke/pcache_smoke.c +++ b/code/user/tests/pcache_smoke/pcache_smoke.c @@ -32,7 +32,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/posix_module_smoke/posix_module_smoke.c b/code/user/tests/posix_module_smoke/posix_module_smoke.c index 67ee83d..00904bc 100644 --- a/code/user/tests/posix_module_smoke/posix_module_smoke.c +++ b/code/user/tests/posix_module_smoke/posix_module_smoke.c @@ -3,6 +3,7 @@ #include #include #include +#include #define RTLD_NOW 0x0002 #define RTLD_LOCAL 0x0004 @@ -23,7 +24,7 @@ static void print(const char* s) { // Terminates the current process. static void exit_now(int code) { - for (;;) exit(code); + while (true) exit(code); } // Fails the smoke test with one message. diff --git a/code/user/tests/process_protection_smoke/process_protection_smoke.c b/code/user/tests/process_protection_smoke/process_protection_smoke.c index a9c8234..aed3ceb 100644 --- a/code/user/tests/process_protection_smoke/process_protection_smoke.c +++ b/code/user/tests/process_protection_smoke/process_protection_smoke.c @@ -37,7 +37,7 @@ static void print_u32(uint32_t value) { // Terminates the current test process. static void exit_now(int32_t code) { - for (;;) (void)SysExitProcess(code); + while (true) (void)SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/realpath_smoke/realpath_smoke.c b/code/user/tests/realpath_smoke/realpath_smoke.c index e4c19c6..01973d4 100644 --- a/code/user/tests/realpath_smoke/realpath_smoke.c +++ b/code/user/tests/realpath_smoke/realpath_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/shared_mmap/shared_mmap.c b/code/user/tests/shared_mmap/shared_mmap.c index 650371d..eb610d1 100644 --- a/code/user/tests/shared_mmap/shared_mmap.c +++ b/code/user/tests/shared_mmap/shared_mmap.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/socket_addr_smoke/socket_addr_smoke.c b/code/user/tests/socket_addr_smoke/socket_addr_smoke.c index dfe2efc..79f136c 100644 --- a/code/user/tests/socket_addr_smoke/socket_addr_smoke.c +++ b/code/user/tests/socket_addr_smoke/socket_addr_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/stats_abi_smoke/stats_abi_smoke.c b/code/user/tests/stats_abi_smoke/stats_abi_smoke.c index 28ecb31..f0e2353 100644 --- a/code/user/tests/stats_abi_smoke/stats_abi_smoke.c +++ b/code/user/tests/stats_abi_smoke/stats_abi_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/surface_invalid_rect/surface_invalid_rect.c b/code/user/tests/surface_invalid_rect/surface_invalid_rect.c index 2173215..61f074a 100644 --- a/code/user/tests/surface_invalid_rect/surface_invalid_rect.c +++ b/code/user/tests/surface_invalid_rect/surface_invalid_rect.c @@ -30,7 +30,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/surface_multithread_present/surface_multithread_present.c b/code/user/tests/surface_multithread_present/surface_multithread_present.c index d242ccc..91f6725 100644 --- a/code/user/tests/surface_multithread_present/surface_multithread_present.c +++ b/code/user/tests/surface_multithread_present/surface_multithread_present.c @@ -43,7 +43,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/surface_smoke/surface_smoke.c b/code/user/tests/surface_smoke/surface_smoke.c index 0c9b9dd..14b257a 100644 --- a/code/user/tests/surface_smoke/surface_smoke.c +++ b/code/user/tests/surface_smoke/surface_smoke.c @@ -30,7 +30,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c b/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c index 70be498..c7e77c5 100644 --- a/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c +++ b/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c @@ -34,7 +34,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Checks whether a string view equals a NUL-terminated string. diff --git a/code/user/tests/timer_smoke/timer_smoke.c b/code/user/tests/timer_smoke/timer_smoke.c index d124a66..51dda9c 100644 --- a/code/user/tests/timer_smoke/timer_smoke.c +++ b/code/user/tests/timer_smoke/timer_smoke.c @@ -31,7 +31,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/win32_condition_variable_smoke/win32_condition_variable_smoke.c b/code/user/tests/win32_condition_variable_smoke/win32_condition_variable_smoke.c new file mode 100644 index 0000000..20aa869 --- /dev/null +++ b/code/user/tests/win32_condition_variable_smoke/win32_condition_variable_smoke.c @@ -0,0 +1,137 @@ +// Win32 condition-variable smoke test for Bink worker synchronization. + +#include +#include +#include + +typedef void* HANDLE; +typedef unsigned long DWORD; +typedef int BOOL; +typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION; +typedef struct { void* _ptr; } CONDITION_VARIABLE; +typedef DWORD (__stdcall *THREAD_START_ROUTINE)(void* parameter); + +#define FALSE 0 +#define TRUE 1 +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u +#define INVALID_HANDLE_VALUE ((HANDLE)(intptr_t)-1) +#define ERROR_TIMEOUT 1460u + +__declspec(dllimport) void InitializeCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void DeleteCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void EnterCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void LeaveCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void InitializeConditionVariable(CONDITION_VARIABLE* conditionVariable); +__declspec(dllimport) BOOL SleepConditionVariableCS(CONDITION_VARIABLE* conditionVariable, + CRITICAL_SECTION* criticalSection, + DWORD milliseconds); +__declspec(dllimport) void WakeConditionVariable(CONDITION_VARIABLE* conditionVariable); +__declspec(dllimport) void WakeAllConditionVariable(CONDITION_VARIABLE* conditionVariable); +__declspec(dllimport) HANDLE CreateThread(void* attributes, + size_t stackSize, + THREAD_START_ROUTINE startRoutine, + void* parameter, + DWORD creationFlags, + DWORD* threadId); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE handle, DWORD milliseconds); +__declspec(dllimport) BOOL CloseHandle(HANDLE handle); +__declspec(dllimport) DWORD GetLastError(void); + +static CRITICAL_SECTION gLock; +static CONDITION_VARIABLE gCondition; +static uint32_t gReady; +static uint32_t gRelease; +static uint32_t gWoke; +static uint32_t gWorkerFailure; + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the smoke test after one failure. +static void fail(const char* message) { + print("win32_condition_variable_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Waits until the main thread releases all condition-variable workers. +static DWORD __stdcall worker_main(void* parameter) { + (void)parameter; + EnterCriticalSection(&gLock); + gReady++; + while (!gRelease) { + if (!SleepConditionVariableCS(&gCondition, &gLock, INFINITE)) { + gWorkerFailure = 1; + LeaveCriticalSection(&gLock); + return 1; + } + } + gWoke++; + LeaveCriticalSection(&gLock); + return 0; +} + +// Waits for both workers to enter their condition wait. +static void wait_for_workers(void) { + for (uint32_t attempt = 0; attempt < 5000u; attempt++) { + EnterCriticalSection(&gLock); + uint32_t ready = gReady; + LeaveCriticalSection(&gLock); + if (ready == 2u) + return; + (void)SysSleep(1); + } + fail("workers did not reach condition wait"); +} + +// Runs wake-all and finite-timeout condition-variable checks. +int main(void) { + print("win32_condition_variable_smoke: begin\n"); + InitializeCriticalSection(&gLock); + InitializeConditionVariable(&gCondition); + + HANDLE workers[2]; + workers[0] = CreateThread(NULL, 0, worker_main, NULL, 0, NULL); + workers[1] = CreateThread(NULL, 0, worker_main, NULL, 0, NULL); + if (!workers[0] || workers[0] == INVALID_HANDLE_VALUE || + !workers[1] || workers[1] == INVALID_HANDLE_VALUE) + fail("CreateThread failed"); + + wait_for_workers(); + print("win32_condition_variable_smoke: workers waiting\n"); + EnterCriticalSection(&gLock); + gRelease = 1; + WakeAllConditionVariable(&gCondition); + LeaveCriticalSection(&gLock); + print("win32_condition_variable_smoke: wake-all sent\n"); + + for (uint32_t index = 0; index < 2u; index++) { + if (WaitForSingleObject(workers[index], 5000u) != WAIT_OBJECT_0) + fail("worker did not exit after WakeAllConditionVariable"); + if (!CloseHandle(workers[index])) + fail("CloseHandle failed"); + } + if (gWorkerFailure || gWoke != 2u) + fail("wake-all result mismatch"); + print("win32_condition_variable_smoke: workers joined\n"); + + EnterCriticalSection(&gLock); + BOOL signaled = SleepConditionVariableCS(&gCondition, &gLock, 25u); + DWORD error = GetLastError(); + gRelease = 2; + LeaveCriticalSection(&gLock); + if (signaled || error != ERROR_TIMEOUT || gRelease != 2u) + fail("finite timeout or critical-section reacquire failed"); + print("win32_condition_variable_smoke: timeout complete\n"); + + // A wake with no waiter is intentionally not retained. + WakeConditionVariable(&gCondition); + DeleteCriticalSection(&gLock); + print("win32_condition_variable_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); +} diff --git a/code/user/tests/win32_exception_smoke/win32_exception_smoke.c b/code/user/tests/win32_exception_smoke/win32_exception_smoke.c index 461e3e2..c2033f4 100644 --- a/code/user/tests/win32_exception_smoke/win32_exception_smoke.c +++ b/code/user/tests/win32_exception_smoke/win32_exception_smoke.c @@ -41,7 +41,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/win32_module_lookup_smoke/win32_module_lookup_smoke.c b/code/user/tests/win32_module_lookup_smoke/win32_module_lookup_smoke.c new file mode 100644 index 0000000..f2dfb6b --- /dev/null +++ b/code/user/tests/win32_module_lookup_smoke/win32_module_lookup_smoke.c @@ -0,0 +1,51 @@ +// Win32 process-local module lookup smoke test. + +#include +#include + +typedef void* HANDLE; +typedef int BOOL; + +typedef BOOL (__stdcall *IS_DEBUGGER_PRESENT_ROUTINE)(void); + +__declspec(dllimport) HANDLE GetModuleHandleA(const char* moduleName); +__declspec(dllimport) void* GetProcAddress(HANDLE module, const char* procedureName); + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test after one contract failure. +static void fail(const char* message) { + print("win32_module_lookup_smoke: "); + print(message); + print("\n"); + while (true) + (void)SysExitProcess(1); +} + +// Verifies extensionless system-module lookup and dynamic debugger probing. +int main(void) { + print("win32_module_lookup_smoke: begin\n"); + + HANDLE fullName = GetModuleHandleA("kernel32.dll"); + HANDLE baseName = GetModuleHandleA("KERNEL32"); + if (!fullName || !baseName || fullName != baseName) + fail("extensionless KERNEL32 lookup failed"); + + if (GetModuleHandleA("kernel32.invalid")) + fail("invalid module suffix was accepted"); + + IS_DEBUGGER_PRESENT_ROUTINE isDebuggerPresent = + (IS_DEBUGGER_PRESENT_ROUTINE)GetProcAddress(baseName, "IsDebuggerPresent"); + if (!isDebuggerPresent) + fail("IsDebuggerPresent was absent from GetProcAddress"); + if (isDebuggerPresent()) + fail("IsDebuggerPresent reported a debugger"); + + print("win32_module_lookup_smoke: PASS\n"); + while (true) + (void)SysExitProcess(0); +} diff --git a/code/user/tests/win32_mutex_smoke/win32_mutex_smoke.c b/code/user/tests/win32_mutex_smoke/win32_mutex_smoke.c new file mode 100644 index 0000000..eb0f313 --- /dev/null +++ b/code/user/tests/win32_mutex_smoke/win32_mutex_smoke.c @@ -0,0 +1,147 @@ +// Win32 recursive mutex smoke test for Bink worker teardown. + +#include +#include +#include + +typedef void* HANDLE; +typedef unsigned long DWORD; +typedef int BOOL; +typedef DWORD (__stdcall *THREAD_START_ROUTINE)(void* parameter); + +#define FALSE 0 +#define TRUE 1 +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u +#define WAIT_TIMEOUT 0x00000102u +#define WAIT_FAILED 0xFFFFFFFFu +#define INVALID_HANDLE_VALUE ((HANDLE)(intptr_t)-1) + +__declspec(dllimport) HANDLE CreateMutexA(void* attributes, BOOL initialOwner, const char* name); +__declspec(dllimport) BOOL ReleaseMutex(HANDLE mutex); +__declspec(dllimport) HANDLE CreateThread(void* attributes, + size_t stackSize, + THREAD_START_ROUTINE startRoutine, + void* parameter, + DWORD creationFlags, + DWORD* threadId); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE handle, DWORD milliseconds); +__declspec(dllimport) BOOL CloseHandle(HANDLE handle); +__declspec(dllimport) DWORD GetLastError(void); + +static HANDLE gMutex; +static DWORD gWorkerWait; +static DWORD gWorkerError; +static BOOL gWorkerRelease; + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test after one contract failure. +static void fail(const char* message) { + print("win32_mutex_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Writes one fixed-width hexadecimal value to the native console. +static void print_hex32(DWORD value) { + static const char digits[] = "0123456789ABCDEF"; + char text[] = "0x00000000"; + for (unsigned i = 0; i < 8; i++) { + text[9 - i] = digits[value & 0xFu]; + value >>= 4; + } + print(text); +} + +// Reports a failed worker wait with both Win32 result channels. +static void fail_worker_wait(const char* message) { + print("win32_mutex_smoke: "); + print(message); + print(" result="); + print_hex32(gWorkerWait); + print(" error="); + print_hex32(gWorkerError); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Attempts one wait and releases the mutex only after successful acquisition. +static DWORD __stdcall mutex_worker(void* parameter) { + DWORD timeout = (DWORD)(uintptr_t)parameter; + gWorkerWait = WaitForSingleObject(gMutex, timeout); + gWorkerError = gWorkerWait == WAIT_FAILED ? GetLastError() : 0; + if (gWorkerWait == WAIT_OBJECT_0) + gWorkerRelease = ReleaseMutex(gMutex); + return 0; +} + +// Runs one worker and waits for its normal return. +static void run_worker(DWORD timeout) { + gWorkerWait = 0xFFFFFFFFu; + gWorkerError = 0; + gWorkerRelease = FALSE; + HANDLE worker = CreateThread(NULL, 0, mutex_worker, + (void*)(uintptr_t)timeout, 0, NULL); + if (!worker || worker == INVALID_HANDLE_VALUE) + fail("CreateThread failed"); + if (WaitForSingleObject(worker, 5000u) != WAIT_OBJECT_0) + fail("worker did not return"); + if (!CloseHandle(worker)) + fail("worker CloseHandle failed"); +} + +// Verifies recursive ownership, non-owner safety, and final release transfer. +int main(void) { + print("win32_mutex_smoke: begin\n"); + gMutex = CreateMutexA(NULL, FALSE, NULL); + if (!gMutex || gMutex == INVALID_HANDLE_VALUE) + fail("CreateMutexA failed"); + + if (WaitForSingleObject(gMutex, INFINITE) != WAIT_OBJECT_0 || + WaitForSingleObject(gMutex, INFINITE) != WAIT_OBJECT_0) + fail("recursive acquisition failed"); + + run_worker(25u); + if (gWorkerWait == WAIT_FAILED) + fail_worker_wait("doubly-owned finite wait failed"); + if (gWorkerWait != WAIT_TIMEOUT || gWorkerRelease) + fail("worker acquired doubly-owned mutex"); + + if (!ReleaseMutex(gMutex)) + fail("first recursive release failed"); + run_worker(25u); + if (gWorkerWait == WAIT_FAILED) + fail_worker_wait("singly-owned finite wait failed"); + if (gWorkerWait != WAIT_TIMEOUT || gWorkerRelease) + fail("worker acquired singly-owned mutex"); + + if (!ReleaseMutex(gMutex)) + fail("final recursive release failed"); + run_worker(1000u); + if (gWorkerWait == WAIT_FAILED) + fail_worker_wait("released finite wait failed"); + if (gWorkerWait != WAIT_OBJECT_0 || !gWorkerRelease) + fail("worker did not acquire released mutex"); + + if (!CloseHandle(gMutex)) + fail("mutex CloseHandle failed"); + + gMutex = CreateMutexA(NULL, TRUE, NULL); + if (!gMutex || gMutex == INVALID_HANDLE_VALUE) + fail("initial-owner mutex failed"); + if (WaitForSingleObject(gMutex, INFINITE) != WAIT_OBJECT_0) + fail("initial-owner recursion failed"); + if (!ReleaseMutex(gMutex) || !ReleaseMutex(gMutex)) + fail("initial-owner release failed"); + if (!CloseHandle(gMutex)) + fail("initial-owner close failed"); + + print("win32_mutex_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); +} diff --git a/code/user/tests/win32_seh_smoke/win32_seh_smoke.c b/code/user/tests/win32_seh_smoke/win32_seh_smoke.c index 84d59a9..11c3f34 100644 --- a/code/user/tests/win32_seh_smoke/win32_seh_smoke.c +++ b/code/user/tests/win32_seh_smoke/win32_seh_smoke.c @@ -54,7 +54,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/win32_thread_suspend_smoke/win32_thread_suspend_smoke.c b/code/user/tests/win32_thread_suspend_smoke/win32_thread_suspend_smoke.c new file mode 100644 index 0000000..31d99db --- /dev/null +++ b/code/user/tests/win32_thread_suspend_smoke/win32_thread_suspend_smoke.c @@ -0,0 +1,97 @@ +// Win32 suspended-thread creation and resume smoke test. + +#include +#include +#include + +typedef void* HANDLE; +typedef unsigned long DWORD; +typedef int BOOL; +typedef DWORD (__stdcall *THREAD_START_ROUTINE)(void* parameter); + +#define CREATE_SUSPENDED 0x00000004u +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u +#define INVALID_HANDLE_VALUE ((HANDLE)(intptr_t)-1) + +__declspec(dllimport) HANDLE CreateThread(void* attributes, + size_t stackSize, + THREAD_START_ROUTINE startRoutine, + void* parameter, + DWORD creationFlags, + DWORD* threadId); +__declspec(dllimport) DWORD ResumeThread(HANDLE thread); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE handle, DWORD milliseconds); +__declspec(dllimport) void Sleep(DWORD milliseconds); +__declspec(dllimport) BOOL CloseHandle(HANDLE handle); + +static volatile long gStarted; +static volatile long gRelease; + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test after one contract failure. +static void fail(const char* message) { + print("win32_thread_suspend_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Records that the suspended worker reached its user entry point. +static DWORD __stdcall suspended_worker(void* parameter) { + (void)parameter; + gStarted = 1; + while (!gRelease) + Sleep(1u); + return 0; +} + +// Exits immediately so process teardown can retain its thread handle. +static DWORD __stdcall exited_worker(void* parameter) { + (void)parameter; + return 0; +} + +// Verifies that CREATE_SUSPENDED prevents execution until ResumeThread. +int main(void) { + print("win32_thread_suspend_smoke: begin\n"); + + DWORD threadId = 0; + HANDLE thread = CreateThread(NULL, 0, suspended_worker, NULL, + CREATE_SUSPENDED, &threadId); + if (!thread || thread == INVALID_HANDLE_VALUE || threadId == 0) + fail("CreateThread(CREATE_SUSPENDED) failed"); + + Sleep(25u); + if (gStarted) + fail("worker ran before ResumeThread"); + + if (ResumeThread(thread) != 1u) + fail("first ResumeThread did not return one"); + if (ResumeThread(thread) != 0u) + fail("second ResumeThread did not return zero"); + + gRelease = 1; + if (WaitForSingleObject(thread, INFINITE) != WAIT_OBJECT_0) + fail("worker wait failed"); + if (!gStarted) + fail("worker did not run after ResumeThread"); + if (!CloseHandle(thread)) + fail("CloseHandle failed"); + + // Keep one completed thread handle open until process exit. Dead THREAD + // objects may remain waitable, but they must not keep the process alive. + HANDLE retained = CreateThread(NULL, 0, exited_worker, NULL, 0, NULL); + if (!retained || retained == INVALID_HANDLE_VALUE) + fail("retained CreateThread failed"); + if (WaitForSingleObject(retained, INFINITE) != WAIT_OBJECT_0) + fail("retained worker wait failed"); + + print("win32_thread_suspend_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); +} diff --git a/code/user/tests/winmm_waveout_smoke/winmm_waveout_smoke.c b/code/user/tests/winmm_waveout_smoke/winmm_waveout_smoke.c new file mode 100644 index 0000000..77f8a26 --- /dev/null +++ b/code/user/tests/winmm_waveout_smoke/winmm_waveout_smoke.c @@ -0,0 +1,455 @@ +// WinMM waveform-output smoke test over the native render-stream personality path. + +#include +#include +#include +#include +#include + +typedef void* HANDLE; +typedef void* HWAVEOUT; +typedef unsigned int MMRESULT; +typedef unsigned int UINT; +typedef uint32_t DWORD; +typedef uintptr_t DWORD_PTR; +typedef unsigned short WORD; +typedef int BOOL; + +#define FALSE 0 +#define TRUE 1 +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u + +#define MMSYSERR_NOERROR 0u +#define MMSYSERR_NOTSUPPORTED 8u +#define WAVERR_BADFORMAT 32u +#define WAVERR_STILLPLAYING 33u + +#define WAVE_FORMAT_PCM 1u +#define WAVE_FORMAT_QUERY 0x00000001u +#define WAVE_ALLOWSYNC 0x00000002u +#define WAVE_FORMAT_DIRECT 0x00000008u +#define CALLBACK_WINDOW 0x00010000u +#define CALLBACK_THREAD 0x00020000u +#define CALLBACK_FUNCTION 0x00030000u +#define CALLBACK_EVENT 0x00050000u +#define WAVE_MAPPER 0xFFFFFFFFu + +#define WHDR_DONE 0x00000001u +#define WHDR_PREPARED 0x00000002u +#define WHDR_BEGINLOOP 0x00000004u +#define WHDR_ENDLOOP 0x00000008u +#define WHDR_INQUEUE 0x00000010u + +#define WOM_OPEN 0x03BBu +#define WOM_CLOSE 0x03BCu +#define WOM_DONE 0x03BDu + +#define TIME_SAMPLES 0x0002u + +#define TEST_FRAMES 4096u +#define TEST_CONTROL_FRAMES 16384u +#define TEST_PHASE_MODULUS 1200u +#define TEST_PHASE_STEP 11u +#define TEST_QUARTER_PHASE 300u +#define TEST_PEAK 8192 + +typedef struct WAVEFORMATEX { + WORD wFormatTag; + WORD nChannels; + DWORD nSamplesPerSec; + DWORD nAvgBytesPerSec; + WORD nBlockAlign; + WORD wBitsPerSample; + WORD cbSize; +} WAVEFORMATEX; + +typedef struct WAVEHDR { + char* lpData; + DWORD dwBufferLength; + DWORD dwBytesRecorded; + DWORD_PTR dwUser; + DWORD dwFlags; + DWORD dwLoops; + struct WAVEHDR* lpNext; + DWORD_PTR reserved; +} WAVEHDR; + +typedef struct MMTIME { + UINT wType; + union { + DWORD ms; + DWORD sample; + DWORD cb; + DWORD ticks; + } u; +} MMTIME; + +typedef struct WAVEOUTCAPSA { + WORD wMid; + WORD wPid; + DWORD vDriverVersion; + char szPname[32]; + DWORD dwFormats; + WORD wChannels; + WORD wReserved1; + DWORD dwSupport; +} WAVEOUTCAPSA; + +typedef char TEST_WAVEFORMATEX_SIZE[(sizeof(WAVEFORMATEX) == 20u) ? 1 : -1]; +typedef char TEST_WAVEHDR_SIZE[(sizeof(WAVEHDR) == 48u) ? 1 : -1]; +typedef char TEST_MMTIME_SIZE[(sizeof(MMTIME) == 8u) ? 1 : -1]; +typedef char TEST_WAVEOUTCAPSA_SIZE[(sizeof(WAVEOUTCAPSA) == 52u) ? 1 : -1]; + +typedef void (__stdcall *WAVEOUTPROC)(HWAVEOUT, UINT, DWORD_PTR, DWORD_PTR, DWORD_PTR); + +__declspec(dllimport) UINT waveOutGetNumDevs(void); +__declspec(dllimport) MMRESULT waveOutOpen(HWAVEOUT*, UINT, const WAVEFORMATEX*, DWORD_PTR, DWORD_PTR, DWORD); +__declspec(dllimport) MMRESULT waveOutClose(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutPrepareHeader(HWAVEOUT, WAVEHDR*, UINT); +__declspec(dllimport) MMRESULT waveOutUnprepareHeader(HWAVEOUT, WAVEHDR*, UINT); +__declspec(dllimport) MMRESULT waveOutWrite(HWAVEOUT, WAVEHDR*, UINT); +__declspec(dllimport) MMRESULT waveOutPause(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutRestart(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutReset(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutSetVolume(HWAVEOUT, DWORD); +__declspec(dllimport) MMRESULT waveOutGetVolume(HWAVEOUT, DWORD*); +__declspec(dllimport) MMRESULT waveOutGetPosition(HWAVEOUT, MMTIME*, UINT); +__declspec(dllimport) MMRESULT waveOutGetID(HWAVEOUT, UINT*); +__declspec(dllimport) MMRESULT waveOutGetDevCapsA(UINT, WAVEOUTCAPSA*, UINT); +__declspec(dllimport) MMRESULT waveOutGetErrorTextA(MMRESULT, char*, UINT); +__declspec(dllimport) HANDLE CreateEventA(void*, BOOL, BOOL, const char*); +__declspec(dllimport) BOOL SetEvent(HANDLE); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE, DWORD); +__declspec(dllimport) BOOL CloseHandle(HANDLE); + +static int16_t gS16Frames[TEST_FRAMES * 2u]; +static uint8_t gS24Frames[TEST_FRAMES * 2u * 3u]; +static int16_t gControlFrames[TEST_CONTROL_FRAMES * 2u]; +static HANDLE gFunctionEvent; +static volatile long gOpenCallbacks; +static volatile long gDoneCallbacks; +static volatile long gCloseCallbacks; + +// Writes one NUL-terminated string to the kernel console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test process after one failure. +static void fail(const char* message) { + print("winmm_waveout_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Verifies one WinMM result exactly. +static void expect_result(const char* operation, MMRESULT actual, MMRESULT expected) { + if (actual != expected) { + print("winmm_waveout_smoke: unexpected result from "); + print(operation); + print("\n"); + while (true) (void)SysExitProcess(1); + } +} + +// Returns one exact integer 440 Hz triangle-wave sample at 48 kHz. +static int16_t tone_sample(uint32_t phase) { + int32_t sample; + if (phase < TEST_QUARTER_PHASE) { + sample = (int32_t)(phase * TEST_PEAK / TEST_QUARTER_PHASE); + } else if (phase < TEST_QUARTER_PHASE * 2u) { + sample = (int32_t)((TEST_QUARTER_PHASE * 2u - phase) * + TEST_PEAK / TEST_QUARTER_PHASE); + } else if (phase < TEST_QUARTER_PHASE * 3u) { + sample = -(int32_t)((phase - TEST_QUARTER_PHASE * 2u) * + TEST_PEAK / TEST_QUARTER_PHASE); + } else { + sample = -(int32_t)((TEST_PHASE_MODULUS - phase) * + TEST_PEAK / TEST_QUARTER_PHASE); + } + return (int16_t)sample; +} + +// Fills S16, packed-S24, and muted control buffers without floating point. +static void fill_buffers(void) { + uint32_t phase = 0; + for (uint32_t frame = 0; frame < TEST_FRAMES; frame++) { + int16_t sample = tone_sample(phase); + gS16Frames[frame * 2u] = sample; + gS16Frames[frame * 2u + 1u] = sample; + int32_t sample24 = (int32_t)sample * 256; + for (uint32_t channel = 0; channel < 2u; channel++) { + size_t offset = ((size_t)frame * 2u + channel) * 3u; + gS24Frames[offset] = (uint8_t)sample24; + gS24Frames[offset + 1u] = (uint8_t)(sample24 >> 8u); + gS24Frames[offset + 2u] = (uint8_t)(sample24 >> 16u); + } + phase += TEST_PHASE_STEP; + if (phase >= TEST_PHASE_MODULUS) + phase -= TEST_PHASE_MODULUS; + } + memset(gControlFrames, 0, sizeof(gControlFrames)); +} + +// Receives function callbacks on the per-HWAVEOUT worker thread. +static void __stdcall wave_callback(HWAVEOUT waveOut, + UINT message, + DWORD_PTR instance, + DWORD_PTR parameter1, + DWORD_PTR parameter2) { + (void)waveOut; + (void)instance; + (void)parameter1; + (void)parameter2; + if (message == WOM_OPEN) + _InterlockedIncrement(&gOpenCallbacks); + else if (message == WOM_DONE) { + _InterlockedIncrement(&gDoneCallbacks); + (void)SetEvent(gFunctionEvent); + } else if (message == WOM_CLOSE) + _InterlockedIncrement(&gCloseCallbacks); +} + +// Waits for one callback event with a finite deadline. +static void wait_event(HANDLE event, const char* operation) { + if (WaitForSingleObject(event, 5000u) != WAIT_OBJECT_0) + fail(operation); +} + +// Returns one canonical PCM format descriptor. +static WAVEFORMATEX make_format_at(uint32_t sampleRate, + uint16_t channels, + uint16_t bitsPerSample) { + WAVEFORMATEX format; + memset(&format, 0, sizeof(format)); + format.wFormatTag = WAVE_FORMAT_PCM; + format.nChannels = channels; + format.nSamplesPerSec = sampleRate; + format.wBitsPerSample = bitsPerSample; + format.nBlockAlign = (uint16_t)(channels * (bitsPerSample / 8u)); + format.nAvgBytesPerSec = format.nSamplesPerSec * format.nBlockAlign; + return format; +} + +// Returns the default stereo 48 kHz PCM descriptor used for playback tests. +static WAVEFORMATEX make_format(uint16_t bitsPerSample) { + return make_format_at(48000u, 2u, bitsPerSample); +} + + +// Verifies device enumeration, capabilities, and fixed error descriptions. +static void test_device_surface(void) { + WAVEOUTCAPSA caps; + memset(&caps, 0, sizeof(caps)); + expect_result("device capabilities", + waveOutGetDevCapsA(WAVE_MAPPER, &caps, sizeof(caps)), + MMSYSERR_NOERROR); + if (caps.wChannels != 2u || !caps.szPname[0]) + fail("device capabilities mismatch"); + + char errorText[64]; + memset(errorText, 0, sizeof(errorText)); + expect_result("error text", + waveOutGetErrorTextA(WAVERR_BADFORMAT, errorText, sizeof(errorText)), + MMSYSERR_NOERROR); + if (!errorText[0]) + fail("error text was empty"); +} + +// Verifies function callbacks, queued completion, immediate copy, and position. +static void test_function_callback(void) { + WAVEFORMATEX format = make_format(16u); + gFunctionEvent = CreateEventA(NULL, FALSE, FALSE, NULL); + if (!gFunctionEvent) + fail("CreateEventA for function callback failed"); + + HWAVEOUT waveOut = NULL; + expect_result("function waveOutOpen", + waveOutOpen(&waveOut, + WAVE_MAPPER, + &format, + (DWORD_PTR)(WAVEOUTPROC)wave_callback, + 0, + CALLBACK_FUNCTION | WAVE_ALLOWSYNC | WAVE_FORMAT_DIRECT), + MMSYSERR_NOERROR); + if (!waveOut || gOpenCallbacks != 1) + fail("function WOM_OPEN callback missing"); + UINT deviceId = UINT32_MAX; + expect_result("get device ID", waveOutGetID(waveOut, &deviceId), MMSYSERR_NOERROR); + if (deviceId >= waveOutGetNumDevs()) + fail("resolved device ID is out of range"); + + WAVEHDR first; + WAVEHDR second; + memset(&first, 0, sizeof(first)); + memset(&second, 0, sizeof(second)); + first.lpData = (char*)gS16Frames; + first.dwBufferLength = sizeof(gS16Frames) / 2u; + second.lpData = (char*)gS16Frames + first.dwBufferLength; + second.dwBufferLength = sizeof(gS16Frames) - first.dwBufferLength; + expect_result("prepare first", waveOutPrepareHeader(waveOut, &first, sizeof(first)), MMSYSERR_NOERROR); + expect_result("prepare second", waveOutPrepareHeader(waveOut, &second, sizeof(second)), MMSYSERR_NOERROR); + expect_result("write first", waveOutWrite(waveOut, &first, sizeof(first)), MMSYSERR_NOERROR); + expect_result("write second", waveOutWrite(waveOut, &second, sizeof(second)), MMSYSERR_NOERROR); + + memset(gS16Frames, 0, sizeof(gS16Frames)); + while (gDoneCallbacks < 2) + wait_event(gFunctionEvent, "function WOM_DONE timed out"); + if (gDoneCallbacks != 2 || + (first.dwFlags & (WHDR_DONE | WHDR_PREPARED)) != (WHDR_DONE | WHDR_PREPARED) || + (second.dwFlags & (WHDR_DONE | WHDR_PREPARED)) != (WHDR_DONE | WHDR_PREPARED) || + (first.dwFlags & WHDR_INQUEUE) || (second.dwFlags & WHDR_INQUEUE)) + fail("function header completion mismatch"); + + MMTIME position; + memset(&position, 0, sizeof(position)); + position.wType = TIME_SAMPLES; + expect_result("get position", waveOutGetPosition(waveOut, &position, sizeof(position)), MMSYSERR_NOERROR); + if (position.u.sample < TEST_FRAMES) + fail("position did not reach submitted frames"); + + expect_result("set volume", waveOutSetVolume(waveOut, 0xC0008000u), MMSYSERR_NOERROR); + DWORD volume = 0; + expect_result("get volume", waveOutGetVolume(waveOut, &volume), MMSYSERR_NOERROR); + if (volume != 0xC0008000u) + fail("volume did not round trip"); + + expect_result("unprepare first", waveOutUnprepareHeader(waveOut, &first, sizeof(first)), MMSYSERR_NOERROR); + expect_result("unprepare second", waveOutUnprepareHeader(waveOut, &second, sizeof(second)), MMSYSERR_NOERROR); + expect_result("function close", waveOutClose(waveOut), MMSYSERR_NOERROR); + if (gCloseCallbacks != 1) + fail("function WOM_CLOSE callback missing"); + if (!CloseHandle(gFunctionEvent)) + fail("function callback event close failed"); + gFunctionEvent = NULL; +} + +// Verifies CALLBACK_EVENT and packed-S24 conversion through the native S16 stream. +static void test_event_s24(void) { + WAVEFORMATEX format = make_format(24u); + HANDLE event = CreateEventA(NULL, FALSE, FALSE, NULL); + if (!event) + fail("CreateEventA for event callback failed"); + + HWAVEOUT waveOut = NULL; + expect_result("event waveOutOpen", + waveOutOpen(&waveOut, + WAVE_MAPPER, + &format, + (DWORD_PTR)event, + 0, + CALLBACK_EVENT), + MMSYSERR_NOERROR); + wait_event(event, "event WOM_OPEN timed out"); + + WAVEHDR header; + memset(&header, 0, sizeof(header)); + header.lpData = (char*)gS24Frames; + header.dwBufferLength = sizeof(gS24Frames); + expect_result("event prepare", waveOutPrepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("event write", waveOutWrite(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + memset(gS24Frames, 0, sizeof(gS24Frames)); + wait_event(event, "event WOM_DONE timed out"); + if ((header.dwFlags & WHDR_DONE) == 0 || (header.dwFlags & WHDR_INQUEUE)) + fail("event header completion mismatch"); + expect_result("event unprepare", waveOutUnprepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("event close", waveOutClose(waveOut), MMSYSERR_NOERROR); + wait_event(event, "event WOM_CLOSE timed out"); + if (!CloseHandle(event)) + fail("event callback handle close failed"); +} + +// Verifies queued-close refusal and synchronous pause, restart, and reset. +static void test_control_operations(void) { + WAVEFORMATEX format = make_format(16u); + HWAVEOUT waveOut = NULL; + expect_result("control waveOutOpen", + waveOutOpen(&waveOut, WAVE_MAPPER, &format, 0, 0, 0), + MMSYSERR_NOERROR); + expect_result("control mute", waveOutSetVolume(waveOut, 0), MMSYSERR_NOERROR); + + WAVEHDR header; + memset(&header, 0, sizeof(header)); + header.lpData = (char*)gControlFrames; + header.dwBufferLength = sizeof(gControlFrames); + expect_result("control prepare", waveOutPrepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + header.dwFlags |= WHDR_BEGINLOOP | WHDR_ENDLOOP; + header.dwLoops = 2u; + expect_result("loop rejection", waveOutWrite(waveOut, &header, sizeof(header)), MMSYSERR_NOTSUPPORTED); + header.dwFlags &= ~(WHDR_BEGINLOOP | WHDR_ENDLOOP); + header.dwLoops = 0; + expect_result("control write", waveOutWrite(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("close while queued", waveOutClose(waveOut), WAVERR_STILLPLAYING); + expect_result("unprepare while queued", + waveOutUnprepareHeader(waveOut, &header, sizeof(header)), + WAVERR_STILLPLAYING); + expect_result("pause", waveOutPause(waveOut), MMSYSERR_NOERROR); + expect_result("second pause", waveOutPause(waveOut), MMSYSERR_NOERROR); + expect_result("restart", waveOutRestart(waveOut), MMSYSERR_NOERROR); + expect_result("second restart", waveOutRestart(waveOut), MMSYSERR_NOERROR); + expect_result("reset", waveOutReset(waveOut), MMSYSERR_NOERROR); + if ((header.dwFlags & WHDR_DONE) == 0 || (header.dwFlags & WHDR_INQUEUE)) + fail("reset did not complete queued header"); + expect_result("control unprepare", waveOutUnprepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("control close", waveOutClose(waveOut), MMSYSERR_NOERROR); +} + +// Verifies the complete accepted PCM matrix and explicit rejection paths. +static void test_formats_and_rejections(void) { + static const uint32_t rates[] = { 11025u, 22050u, 44100u, 48000u }; + static const uint16_t bits[] = { 16u, 24u }; + for (uint32_t rateIndex = 0; rateIndex < sizeof(rates) / sizeof(rates[0]); rateIndex++) { + for (uint16_t channels = 1u; channels <= 2u; channels++) { + for (uint32_t bitsIndex = 0; bitsIndex < sizeof(bits) / sizeof(bits[0]); bitsIndex++) { + WAVEFORMATEX format = make_format_at(rates[rateIndex], channels, bits[bitsIndex]); + expect_result("accepted PCM query", + waveOutOpen(NULL, WAVE_MAPPER, &format, 0, 0, WAVE_FORMAT_QUERY), + MMSYSERR_NOERROR); + } + } + } + + WAVEFORMATEX s16 = make_format(16u); + HWAVEOUT waveOut = NULL; + expect_result("window callback rejection", + waveOutOpen(&waveOut, WAVE_MAPPER, &s16, 1, 0, CALLBACK_WINDOW), + MMSYSERR_NOTSUPPORTED); + expect_result("thread callback rejection", + waveOutOpen(&waveOut, WAVE_MAPPER, &s16, 1, 0, CALLBACK_THREAD), + MMSYSERR_NOTSUPPORTED); + + WAVEFORMATEX floating = make_format_at(48000u, 2u, 32u); + floating.wFormatTag = 3u; + expect_result("Float32 rejection", + waveOutOpen(NULL, WAVE_MAPPER, &floating, 0, 0, WAVE_FORMAT_QUERY), + WAVERR_BADFORMAT); + + WAVEFORMATEX badRate = make_format_at(32000u, 2u, 16u); + expect_result("32 kHz rejection", + waveOutOpen(NULL, WAVE_MAPPER, &badRate, 0, 0, WAVE_FORMAT_QUERY), + WAVERR_BADFORMAT); + + WAVEFORMATEX badChannels = make_format_at(48000u, 3u, 16u); + expect_result("three-channel rejection", + waveOutOpen(NULL, WAVE_MAPPER, &badChannels, 0, 0, WAVE_FORMAT_QUERY), + WAVERR_BADFORMAT); +} + +// Runs the focused WinMM waveform-output contract test. +int main(void) { + print("winmm_waveout_smoke: begin\n"); + if (!waveOutGetNumDevs()) + fail("no waveform output devices"); + fill_buffers(); + test_device_surface(); + test_formats_and_rejections(); + test_control_operations(); + test_function_callback(); + test_event_s24(); + print("winmm_waveout_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); + return 0; +} diff --git a/code/user/tests/would_block_smoke/would_block_smoke.c b/code/user/tests/would_block_smoke/would_block_smoke.c index 097fc44..89b3c56 100644 --- a/code/user/tests/would_block_smoke/would_block_smoke.c +++ b/code/user/tests/would_block_smoke/would_block_smoke.c @@ -74,7 +74,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/docs.md b/docs.md index b7d440d..6cca3fe 100644 --- a/docs.md +++ b/docs.md @@ -95,7 +95,9 @@ Breaking changes are allowed. When making one: ## comments from developer -- 0x%016llX over %p -- never add Ex, it makes me mad, never +- 0x%016llX over %p, it looks prettier and is standard. +- never add Ex, it makes me mad, never, rearchitect instead - i do put a focus on overengineering if it means it will be cleaner and futureproof -- please do ask questions, preferably about a hundred per feature +- ASK QUESTIONS! never assume you know the right answer, if you do, you are probably wrong +- shift the syscall number table every time you add a syscall, do not append to the end +- do not worry if a patch breaks something, it's better to break, document and fix rather than work around it. diff --git a/readme.md b/readme.md index be176ce..9bf660e 100644 --- a/readme.md +++ b/readme.md @@ -79,7 +79,8 @@ Async is first-class here too, same machinery as the filesystem. ## how to build -- open the VS x64 build tools as administrator +- get mtools +- open the VS x64 build tools - change into this directory - run `build.bat` - use one of: @@ -144,6 +145,9 @@ A: Whatever I fixate on next. Q: Does it run Doom? A: `doom.exe` +Q: Can it play Crysis? +A: Into the bucket list. + Q: Does USB work? A: It did for everything I threw at it. diff --git a/run.bat b/run.bat index eb82548..a347a30 100644 --- a/run.bat +++ b/run.bat @@ -1 +1 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=raw,file=fat:rw:fatroot -serial stdio -netdev user,id=net0 -device rtl8139,netdev=net0 +@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=vpc,file=disk.vhd -serial stdio -netdev user,id=net0 -device rtl8139,netdev=net0 -audiodev dsound,id=snd0 -device intel-hda -device hda-output,audiodev=snd0 diff --git a/runa.bat b/runa.bat index 13e3da0..bda7825 100644 --- a/runa.bat +++ b/runa.bat @@ -1,20 +1,23 @@ -@start qemu-system-x86_64 ^ --vga vmware ^ --drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd ^ --drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd ^ --device ahci,id=ahci ^ --drive format=raw,file=fat:rw:fatroot,if=none,id=disk0 ^ --device ide-hd,bus=ahci.0,drive=disk0 ^ --serial stdio ^ --M q35 ^ --m 128M ^ --smp cpus=4,cores=4,threads=1,sockets=1 ^ --netdev user,id=net0,hostfwd=tcp::8080-:8080 ^ --device rtl8139,netdev=net0 ^ --display gtk,show-cursor=on ^ --device qemu-xhci,id=usb ^ --device usb-kbd,bus=usb.0 ^ --device usb-tablet,bus=usb.0 ^ --device usb-hub,id=myhub,bus=usb.0,port=3 ^ --drive if=none,id=usbflash,format=raw,file=fat:rw:usbfatroot ^ --device usb-storage,drive=usbflash,bus=usb.0,port=3.1,id=myusbflash +@start qemu-system-x86_64 ^ +-vga vmware ^ +-drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd ^ +-drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd ^ +-device ahci,id=ahci ^ +-drive format=vpc,file=disk.vhd,if=none,id=disk0 ^ +-device ide-hd,bus=ahci.0,drive=disk0 ^ +-serial stdio ^ +-M q35 ^ +-m 256M ^ +-smp cpus=4,cores=4,threads=1,sockets=1 ^ +-netdev user,id=net0,hostfwd=tcp::8080-:8080 ^ +-device rtl8139,netdev=net0 ^ +-display gtk,show-cursor=on ^ +-device qemu-xhci,id=usb ^ +-device usb-kbd,bus=usb.0 ^ +-device usb-tablet,bus=usb.0 ^ +-device usb-hub,id=myhub,bus=usb.0,port=3 ^ +-drive if=none,id=usbflash,format=raw,file=fat:rw:usbfatroot ^ +-device usb-storage,drive=usbflash,bus=usb.0,port=3.1,id=myusbflash ^ +-audiodev dsound,id=snd0 ^ +-device intel-hda ^ +-device hda-output,audiodev=snd0 diff --git a/runad.bat b/runad.bat index bcaecc6..f3bf91c 100644 --- a/runad.bat +++ b/runad.bat @@ -1 +1 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=raw,file=fat:rw:fatroot,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0 +@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=vpc,file=disk.vhd,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0 diff --git a/rund.bat b/rund.bat index 27e9dc5..f8d797c 100644 --- a/rund.bat +++ b/rund.bat @@ -1 +1 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=raw,file=fat:rw:fatroot -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0 +@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=vpc,file=disk.vhd -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0