From 4add6789ba474702e655f77c6327b653308cfe2c Mon Sep 17 00:00:00 2001 From: Maksymilian Wilczynski Date: Fri, 7 Aug 2026 11:36:50 +0200 Subject: [PATCH] add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2. --- .gitignore | 2 +- OVMF/OVMF_VARS_4M.fd | Bin 540672 -> 540672 bytes OVMF/OVMF_VARS_4M_d.fd | Bin 540672 -> 540672 bytes code/boot.ini | 1 + code/boot/uefi/main.c | 64 +- code/build/build.c | 376 ++- code/common/libc.c | 25 + code/drivers/audio/hda/hda.c | 403 +++ code/drivers/audio/hda/hda.h | 389 +++ code/drivers/audio/hda/hda_codec.c | 1928 +++++++++++++ code/drivers/audio/hda/hda_transport.c | 1421 ++++++++++ code/drivers/bus/pci/pci.c | 2 +- code/drivers/bus/usb/usb.c | 2 +- code/drivers/bus/usb/usb.h | 123 +- code/drivers/bus/usb/usb_bus.c | 25 +- code/drivers/bus/usb/usb_core.c | 7 +- code/drivers/bus/usb/usb_hid.c | 69 +- code/drivers/bus/usb/usb_hub.c | 185 +- code/drivers/bus/usb/usb_storage.c | 311 ++- code/drivers/bus/usb/usb_xhci.c | 473 +++- code/drivers/display/vmwaresvga/vmwaresvga.c | 180 +- code/drivers/drivers.ini | 1 + code/drivers/fs/fat/fat.c | 49 +- code/drivers/fs/fat/fat.h | 1 + code/drivers/fs/fat/fat_mount.c | 1 + code/drivers/net/lwip/lwip_stack.c | 116 +- code/drivers/net/lwip/lwip_sys_arch.c | 37 +- code/drivers/net/rtl8139/rtl8139.c | 111 +- code/drivers/net/rtl8139/rtl8139.h | 7 +- code/drivers/net/rtl816x/rtl816x.c | 207 +- code/drivers/net/rtl816x/rtl816x.h | 14 +- code/drivers/storage/ahci/ahci.c | 470 +++- code/drivers/storage/ahci/ahci.h | 15 +- code/drivers/storage/ahci/ahci_port.c | 15 +- code/drivers/storage/ata/ata.c | 94 +- code/kernel/hal/hal_processor.c | 2 +- code/kernel/include/kernel/io_dma.h | 13 + code/kernel/include/kernel/io_file.h | 10 + code/kernel/include/kernel/ke_audio.h | 183 ++ code/kernel/include/kernel/ke_cpu.h | 3 + code/kernel/include/kernel/ke_debug.h | 15 - code/kernel/include/kernel/ke_display.h | 3 + code/kernel/include/kernel/ke_mutex.h | 37 +- code/kernel/include/kernel/ke_process.h | 4 +- code/kernel/include/kernel/ke_sched.h | 59 +- code/kernel/include/kernel/ke_semaphore.h | 3 +- code/kernel/include/kernel/ke_thread.h | 64 +- code/kernel/include/kernel/ke_time.h | 3 + code/kernel/include/kernel/mm_file_cache.h | 17 - code/kernel/include/kernel/mm_pfn.h | 6 + code/kernel/include/kernel/mm_pte.h | 9 +- code/kernel/include/kernel/mm_virt.h | 11 + code/kernel/include/kernel/syscall_nr.h | 252 +- code/kernel/io_ddk.c | 9 + code/kernel/io_device.c | 291 +- code/kernel/io_dma.c | 1056 +++++++ code/kernel/io_file.c | 136 +- code/kernel/ke_apc.c | 6 +- code/kernel/ke_audio.c | 2460 +++++++++++++++++ code/kernel/ke_cpu.c | 130 + code/kernel/ke_debug.c | 74 +- code/kernel/ke_display.c | 579 +++- code/kernel/ke_dpc.c | 2 +- code/kernel/ke_exception.c | 2 +- code/kernel/ke_input.c | 2 +- code/kernel/ke_irp.c | 53 +- code/kernel/ke_irq.c | 4 +- code/kernel/ke_mutex.c | 297 +- code/kernel/ke_process.c | 80 +- code/kernel/ke_queue.c | 4 +- code/kernel/ke_sched.c | 824 +++++- code/kernel/ke_selftest.c | 593 +++- code/kernel/ke_semaphore.c | 34 +- code/kernel/ke_smp.c | 15 +- code/kernel/ke_switch.asm | 18 +- code/kernel/ke_syscall.c | 718 ++++- code/kernel/ke_system_root.c | 62 +- code/kernel/ke_thread.c | 360 ++- code/kernel/ke_time.c | 11 +- code/kernel/ke_work.c | 33 +- code/kernel/kmain.c | 20 +- code/kernel/ldr_section.c | 12 +- code/kernel/mm_file_cache.c | 427 +-- code/kernel/mm_phys.c | 103 +- code/kernel/mm_pte.c | 16 + code/kernel/mm_virt.c | 137 +- code/kernel/net_socket.c | 7 +- code/kernel/ob_object.c | 4 +- code/public/ntlite/audio.h | 220 ++ code/public/ntlite/ddi/display.h | 8 +- code/public/ntlite/ddi/io_driver.h | 14 +- code/public/ntlite/ddk/audio.h | 105 + code/public/ntlite/ddk/ddk.h | 6 +- code/public/ntlite/ddk/debug.h | 18 + code/public/ntlite/ddk/display.h | 7 +- code/public/ntlite/ddk/dma.h | 146 + code/public/ntlite/ddk/file.h | 5 +- code/public/ntlite/ddk/irp.h | 19 +- code/public/ntlite/ddk/memory.h | 130 +- code/public/ntlite/ddk/mutex.h | 17 +- code/public/ntlite/ddk/object.h | 2 + code/public/ntlite/ddk/semaphore.h | 8 +- code/public/ntlite/ddk/sync.h | 3 + code/public/ntlite/ddk/thread.h | 47 +- code/public/ntlite/display.h | 6 + code/public/ntlite/status.h | 3 + code/public/ntlite/syscall.h | 50 +- code/user/apps/doom/PureDOOM.h | 274 +- code/user/apps/doom/doom.c | 21 +- code/user/apps/doom/doom_audio.c | 802 ++++++ code/user/apps/doom/doom_audio.h | 18 + code/user/apps/doom/third_party/LICENSE.tsf | 19 + code/user/apps/doom/third_party/README.tsf | 11 + code/user/apps/doom/third_party/tsf.h | 2045 ++++++++++++++ code/user/apps/edit/edit.c | 2 +- code/user/apps/httpget/httpget.c | 2 +- code/user/apps/ping/ping.c | 2 +- code/user/apps/shell_native/shell_native.c | 699 ++++- code/user/apps/shell_posix/shell_posix.c | 2 +- code/user/apps/shell_win32/shell_win32.c | 2 +- code/user/apps/video/video.c | 58 +- code/user/personalities/win32/kernel32.c | 760 ++++- code/user/personalities/win32/winmm.c | 122 +- code/user/personalities/win32/winmm_waveout.c | 1510 ++++++++++ code/user/runtime/heap.c | 2 +- code/user/runtime/native_exception.c | 6 +- code/user/runtime/native_start.c | 4 +- code/user/runtime/posix_start.c | 4 +- code/user/syscall/syscall_nr.inc | 252 +- code/user/syscall/syscall_stubs.asm | 25 +- code/user/tests/all_tests/all_tests.c | 6 +- .../audio_stream_smoke/audio_stream_smoke.c | 729 +++++ code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c | 2 +- .../tests/dirty_writeback/dirty_writeback.c | 20 +- .../exception_dispatcher_smoke.c | 2 +- .../tests/exception_smoke/exception_smoke.c | 2 +- .../tests/file_lock_smoke/file_lock_smoke.c | 2 +- .../tests/file_stat_smoke/file_stat_smoke.c | 2 +- .../tests/handle_dup_smoke/handle_dup_smoke.c | 2 +- .../tests/input_api_smoke/input_api_smoke.c | 2 +- .../input_model_smoke/input_model_smoke.c | 2 +- code/user/tests/mmap_smoke/mmap_smoke.c | 2 +- .../module_load_smoke/module_load_smoke.c | 2 +- .../named_queue_lifetime_smoke.c | 2 +- .../native_exception_runtime_smoke.c | 2 +- .../native_raise_exception_smoke.c | 2 +- code/user/tests/net_irp_smoke/net_irp_smoke.c | 2 +- .../pcache_invalidate/pcache_invalidate.c | 2 +- .../pcache_mmap_read_reuse.c | 2 +- .../pcache_multithread/pcache_multithread.c | 2 +- .../tests/pcache_pressure/pcache_pressure.c | 12 +- code/user/tests/pcache_smoke/pcache_smoke.c | 2 +- .../posix_module_smoke/posix_module_smoke.c | 3 +- .../process_protection_smoke.c | 2 +- .../tests/realpath_smoke/realpath_smoke.c | 2 +- code/user/tests/shared_mmap/shared_mmap.c | 2 +- .../socket_addr_smoke/socket_addr_smoke.c | 2 +- .../tests/stats_abi_smoke/stats_abi_smoke.c | 2 +- .../surface_invalid_rect.c | 2 +- .../surface_multithread_present.c | 2 +- code/user/tests/surface_smoke/surface_smoke.c | 2 +- .../syscall_contract_smoke.c | 2 +- code/user/tests/timer_smoke/timer_smoke.c | 2 +- .../win32_condition_variable_smoke.c | 137 + .../win32_exception_smoke.c | 2 +- .../win32_module_lookup_smoke.c | 51 + .../win32_mutex_smoke/win32_mutex_smoke.c | 147 + .../tests/win32_seh_smoke/win32_seh_smoke.c | 2 +- .../win32_thread_suspend_smoke.c | 97 + .../winmm_waveout_smoke/winmm_waveout_smoke.c | 455 +++ .../would_block_smoke/would_block_smoke.c | 2 +- docs.md | 8 +- readme.md | 6 +- run.bat | 2 +- runa.bat | 43 +- runad.bat | 2 +- rund.bat | 2 +- 177 files changed, 22565 insertions(+), 3070 deletions(-) create mode 100644 code/drivers/audio/hda/hda.c create mode 100644 code/drivers/audio/hda/hda.h create mode 100644 code/drivers/audio/hda/hda_codec.c create mode 100644 code/drivers/audio/hda/hda_transport.c create mode 100644 code/kernel/include/kernel/io_dma.h create mode 100644 code/kernel/include/kernel/ke_audio.h create mode 100644 code/kernel/io_dma.c create mode 100644 code/kernel/ke_audio.c create mode 100644 code/public/ntlite/audio.h create mode 100644 code/public/ntlite/ddk/audio.h create mode 100644 code/public/ntlite/ddk/dma.h create mode 100644 code/user/apps/doom/doom_audio.c create mode 100644 code/user/apps/doom/doom_audio.h create mode 100644 code/user/apps/doom/third_party/LICENSE.tsf create mode 100644 code/user/apps/doom/third_party/README.tsf create mode 100644 code/user/apps/doom/third_party/tsf.h create mode 100644 code/user/personalities/win32/winmm_waveout.c create mode 100644 code/user/tests/audio_stream_smoke/audio_stream_smoke.c create mode 100644 code/user/tests/win32_condition_variable_smoke/win32_condition_variable_smoke.c create mode 100644 code/user/tests/win32_module_lookup_smoke/win32_module_lookup_smoke.c create mode 100644 code/user/tests/win32_mutex_smoke/win32_mutex_smoke.c create mode 100644 code/user/tests/win32_thread_suspend_smoke/win32_thread_suspend_smoke.c create mode 100644 code/user/tests/winmm_waveout_smoke/winmm_waveout_smoke.c diff --git a/.gitignore b/.gitignore index e86b1ac..bd8eb9a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ build.obj bin/ obj/ qemu.log -fatroot/ +disk.vhd usbfatroot/ code.zip code/build/build.exe diff --git a/OVMF/OVMF_VARS_4M.fd b/OVMF/OVMF_VARS_4M.fd index 4c58e5de933103edbf71a8976e7883dfd8a6031b..5f4d30b6c621ef57f574dcad1f4ade4551a8ac87 100644 GIT binary patch delta 184 zcmZo@P;6*W+%QRZa+<2r4(lA}j)nu88-`f)~W=g7&g6VBr&NwT%^?!rF$KNCBm> zpw`YydhSyev&&k{rJZpxt7j3z&+ zPkKE+eEY|EgY<^SOL*=ASOiO88LWU+um;w_2G|5PD1!>v0^48*?1C!T12wP@4!|Ke z0>|J4IG_$1;8ar6Y4BCyXR>^X_FfczH)!jc(sL48Cw9|}aK{&#FG9Wu`(mUmBH|nR C#jsNV diff --git a/OVMF/OVMF_VARS_4M_d.fd b/OVMF/OVMF_VARS_4M_d.fd index 143cbf2c9a28003128ebc6a89f4bae68f24201c8..3b482b36bbc793e2b4ebcd218e632a2a1c090bbc 100644 GIT binary patch delta 457 zcmXBRDNh4I7{>7#cDG9vtH4f418G2%0ZYP1Kp-$AtU;oXh*7JWASOYhiX|+t+%5N2 z?)xrRg5?{q>0kYl=QYoh*O;6!xw)}|o$hV716s)i8|p4Q-2WSwidtHo?}$r-7PY*^e#-W}HhJ-d@Ric$H@be? s^a;u2kTa2$Kgr*S(=wk7jyu(2IdJ{i(N;GbDK}DHq%x86%c@!Z0|P3giU0rr delta 434 zcmXBFyH3JT0D$3~vQav$sZUQdpl5oaA-&QYX?mZxX@!4|Q4k;FwN+#BC$`M=I9f8@OgLIMB43)mO!;!b Jm+4q$qHL diff --git a/code/boot.ini b/code/boot.ini index 19372a3..8fc189f 100644 --- a/code/boot.ini +++ b/code/boot.ini @@ -1,5 +1,6 @@ [kernel] ; bitmask: 1 = debug prints, 2 = self check, 4 = file log, 4096 = checked intrusive lists +; 0x8000 reports MTRRs. Add 0x10000 for one temporary framebuffer WC experiment. debug = 0xffff [init] commandline = apps/shell_native.exe diff --git a/code/boot/uefi/main.c b/code/boot/uefi/main.c index baa4e0f..4027824 100644 --- a/code/boot/uefi/main.c +++ b/code/boot/uefi/main.c @@ -26,6 +26,11 @@ int _fltused = 0; #define KERNEL_VIRT_BASE 0xFFFF800000000000ULL #define PHYS_MAP_BASE 0xFFFF880000000000ULL #define PAGE_SIZE 4096ULL +#define BOOT_PAGE_PRESENT 0x01ULL +#define BOOT_PAGE_WRITE 0x02ULL +#define BOOT_PAGE_CACHE_WB 0x00ULL +#define BOOT_PAGE_CACHE_WC 0x08ULL +#define BOOT_PAGE_CACHE_UC 0x10ULL // Implements the ascii Str To Unicode Str S operation. static int AsciiStrToUnicodeStrS(const char* Source, uint16_t* Dest, size_t DestMax) { @@ -168,6 +173,35 @@ static void MapRangeSmart(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags) { } } +// Returns whether a firmware descriptor describes memory that is safe to expose through the direct map. +static bool IsDirectMapRamType(EFI_MEMORY_TYPE type) { + switch (type) { + case EfiLoaderCode: + case EfiLoaderData: + case EfiBootServicesCode: + case EfiBootServicesData: + case EfiRuntimeServicesCode: + case EfiRuntimeServicesData: + case EfiConventionalMemory: + case EfiACPIReclaimMemory: + case EfiACPIMemoryNVS: + return true; + default: + return false; + } +} + +// Converts firmware cache attributes into the kernel's canonical PAT policy. +static UINT64 DirectMapCacheFlags(const EFI_MEMORY_DESCRIPTOR* desc) { + if (!desc) + return BOOT_PAGE_CACHE_UC; + if (desc->Attribute & (EFI_MEMORY_UC | EFI_MEMORY_UCE | EFI_MEMORY_WT)) + return BOOT_PAGE_CACHE_UC; + if (desc->Attribute & EFI_MEMORY_WC) + return BOOT_PAGE_CACHE_WC; + return BOOT_PAGE_CACHE_WB; +} + static void MapRangeSkipping(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags, const UINT64* holes, UINTN holeCount) { UINT64 curVirt = virt; @@ -790,8 +824,8 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable for (UINT64 i = 0; i < fbBytes; i += PAGE_SIZE) { UINT64 phys = gop->Mode->FrameBufferBase + i; - Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // P|W|PCD (write-combined MMIO) - Map4K(phys, phys, 0x13); + Map4K(PHYS_MAP_BASE + phys, phys, BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_WC); + Map4K(phys, phys, BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_WC); } } @@ -866,21 +900,16 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable for (UINTN i = 0; i < mapSize / descSz; i++) { EFI_MEMORY_DESCRIPTOR* desc = (EFI_MEMORY_DESCRIPTOR *)((UINT8 *)efiMap + i * descSz); - // Map normal firmware-described regions into the physical window. - // PCI/firmware MMIO is mapped explicitly by the owning driver. - if (desc->Type != EfiUnusableMemory && - desc->Type != EfiMemoryMappedIO && - desc->Type != EfiMemoryMappedIOPortSpace) { + // Only firmware-described RAM receives a direct-map alias. Reserved + // and MMIO descriptors stay unmapped until an owning subsystem maps + // them with an explicit PAT policy. + if (IsDirectMapRamType(desc->Type)) { UINT64 start = desc->PhysicalStart; - UINT64 end = start + (desc->NumberOfPages * PAGE_SIZE); + UINT64 size = desc->NumberOfPages * PAGE_SIZE; + UINT64 flags = BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | DirectMapCacheFlags(desc); - // Add 2 pages (8KB) of padding to the end of the region. - // This catches ACPI tables that cross page boundaries into unmapped - // memory because of buggy UEFI size reporting. - end += 2 * PAGE_SIZE; - - MapRangeSkipping(start, start, end - start, 0x03, idHoles, idHoleCount); - MapRangeSkipping(PHYS_MAP_BASE + start, start, end - start, 0x03, winHoles, winHoleCount); + MapRangeSkipping(start, start, size, flags, idHoles, idHoleCount); + MapRangeSkipping(PHYS_MAP_BASE + start, start, size, flags, winHoles, winHoleCount); } } } @@ -891,8 +920,9 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable // We explicitly map the region from 0xFEC00000 up through 0xFEE00FFF // to guarantee the HAL can access them via the physical window. for (UINT64 phys = 0xFEC00000; phys < 0xFEE01000; phys += PAGE_SIZE) { - Map4K(phys, phys, 0x13); // Identity Map, P|W|PCD - Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // Physical Window Map, P|W|PCD + UINT64 flags = BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_UC; + Map4K(phys, phys, flags); + Map4K(PHYS_MAP_BASE + phys, phys, flags); } Print(L"8\n"); diff --git a/code/build/build.c b/code/build/build.c index baaf443..91ae5f1 100644 --- a/code/build/build.c +++ b/code/build/build.c @@ -21,6 +21,10 @@ #define LWIP_CFLAGS_LITERAL DRIVER_CFLAGS_LITERAL " /Icode\\drivers\\net\\lwip\\include /Icode\\drivers\\net\\lwip\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706" #define LFLAGS_LITERAL "/nologo /NODEFAULTLIB /DEBUG" +// Partition offsets inside disk.vhd +#define VHD_PART1 "disk.vhd@@1M" // ESP / Partition 1 (Offset 1MB) +#define VHD_PART2 "disk.vhd@@65M" // Test/Data / Partition 2 (Offset 65MB) + const char *CommonCflags = COMMON_CFLAGS_LITERAL; const char *KernelCflags = KERNEL_CFLAGS_LITERAL; const char *UserCflags = USER_CFLAGS_LITERAL; @@ -46,6 +50,7 @@ static const Driver_Target PCI_BUS_DRIVER = static const Driver_Target STANDALONE_DRIVERS[] = { DRIVER_TARGET("ps2", "obj\\drivers\\ps2", "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"), DRIVER_TARGET("serial", "obj\\drivers\\serial", "code\\drivers\\input\\serial\\serial.c"), + DRIVER_TARGET_LIBS("hda", "obj\\drivers\\hda", "code\\drivers\\audio\\hda\\hda.c code\\drivers\\audio\\hda\\hda_transport.c code\\drivers\\audio\\hda\\hda_codec.c", "bin\\pci.lib"), DRIVER_TARGET_LIBS("ata", "obj\\drivers\\ata", "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"), DRIVER_TARGET_LIBS("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"), DRIVER_TARGET("fat", "obj\\drivers\\fat", "code\\drivers\\fs\\fat\\fat.c code\\drivers\\fs\\fat\\fat_mount.c"), @@ -64,7 +69,7 @@ void create_layout() { "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", "obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget", "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop", - "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", + "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread", "obj\\all_tests", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke" }; @@ -91,44 +96,190 @@ Nob_Proc run_async(const char *cmd_str) { return proc; } -// Cleans all object output by replacing obj/ as a whole. -// Only obj/ is disposable. fatroot/ and usbfatroot/ are developer-owned vvfat roots. -bool clean_build_object_directories(void) { - const char *cmd = "if exist obj rmdir /s /q obj"; - if (!run_sync(cmd)) return false; - return nob_mkdir_if_not_exists("obj"); +// Deletes stale object files from a target directory before compiling into it. +bool clean_object_directory(const char *obj_dir) { + const char *cmd = nob_temp_sprintf("if exist %s\\*.obj del /q %s\\*.obj", obj_dir, obj_dir); + return run_sync(cmd); } -// Creates the FAT root layout used by QEMU vvfat without deleting existing files. -// The vvfat root is also a convenient developer workspace, so packaging must be non-destructive. -bool create_fatroot_layout(void) { +// Cleans object directories that are linked through wildcards. +bool clean_build_object_directories(void) { const char *dirs[] = { - "fatroot\\EFI", "fatroot\\EFI\\BOOT", - "fatroot\\system", "fatroot\\drivers", "fatroot\\apps", "fatroot\\tests" + "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal", + "obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix", + "obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native", + "obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget", + "obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", + "obj\\doom", "obj\\video", "obj\\memtop", "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap", + "obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread", + "obj\\all_tests", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke" }; for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) { - if (!nob_mkdir_if_not_exists(dirs[i])) return false; + if (!clean_object_directory(dirs[i])) return false; + } + + if (!clean_object_directory(PCI_BUS_DRIVER.obj_dir)) return false; + for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) { + if (!clean_object_directory(STANDALONE_DRIVERS[i].obj_dir)) return false; } return true; } -// Creates the USB vvfat root once and leaves its contents alone between builds. +// Creates the USB fat root once and leaves its contents alone between builds. bool create_usbfatroot_if_missing(void) { return nob_mkdir_if_not_exists("usbfatroot"); } +// Utility to copy a local file into a FAT image using mcopy without admin rights. +bool mcopy_file(const char *src, const char *vhd_target, const char *dst_path) { + const char *cmd = nob_temp_sprintf("mcopy -o -i %s %s ::%s", vhd_target, src, dst_path); + return run_sync(cmd); +} + +// Utility to create a directory in a FAT image using mmd. +bool mmd_dir(const char *vhd_target, const char *dir_path) { + const char *cmd = nob_temp_sprintf("mmd -i %s ::%s 2>nul", vhd_target, dir_path); + run_sync(cmd); + return true; +} + +// Writes a 512-byte Connectix VHD Fixed Disk Footer at the end of the file. +static void write_vpc_footer(FILE *f, uint64_t disk_size_bytes) { + uint8_t footer[512] = {0}; + + memcpy(&footer[0], "conectix", 8); + + // Features: 0x00000002 (Big Endian) + footer[11] = 0x02; + + // File Format Version: 0x00010000 + footer[13] = 0x01; + + // Data Offset: 0xFFFFFFFFFFFFFFFF (Fixed disk) + memset(&footer[16], 0xFF, 8); + + // Creator Application: "qemu" + memcpy(&footer[28], "qemu", 4); + + // Creator Version: 0x00020000 + footer[33] = 0x02; + + // Creator Host OS: "Wi2k" + memcpy(&footer[36], "Wi2k", 4); + + // Original & Current Size (uint64_t Big Endian) + for (int i = 0; i < 8; i++) { + uint8_t b = (uint8_t)(disk_size_bytes >> ((7 - i) * 8)); + footer[40 + i] = b; + footer[48 + i] = b; + } + + // Geometry for 512MB: 1024 Cylinders (0x0400), 16 Heads, 63 Sectors/Track + footer[56] = 0x04; + footer[57] = 0x00; + footer[58] = 16; + footer[59] = 63; + + // Disk Type: 2 (Fixed Hard Disk) + footer[63] = 0x02; + + // Unique ID GUID + static const uint8_t guid[16] = { + 0xA1, 0xB2, 0xC3, 0xD4, 0xE5, 0xF6, 0x78, 0x90, + 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0 + }; + memcpy(&footer[68], guid, 16); + + // Checksum: 1's complement sum of all bytes in footer with checksum cleared + uint32_t sum = 0; + for (int i = 0; i < 512; i++) { + sum += footer[i]; + } + uint32_t checksum = ~sum; + + footer[64] = (uint8_t)(checksum >> 24); + footer[65] = (uint8_t)(checksum >> 16); + footer[66] = (uint8_t)(checksum >> 8); + footer[67] = (uint8_t)(checksum & 0xFF); + + fseek(f, (long)disk_size_bytes, SEEK_SET); + fwrite(footer, 1, 512, f); +} + +// Creates a 512 MB partitioned disk.vhd image with a valid VPC footer. +bool create_disk_vhd_if_missing(void) { + if (nob_file_exists("disk.vhd")) return true; + + nob_log(NOB_INFO, "disk.vhd missing, creating new 512 MB fixed VHD image..."); + + const uint64_t disk_size = (uint64_t)512 * 1024 * 1024; // 512 MB Payload + FILE *f = fopen("disk.vhd", "wb"); + if (!f) { + nob_log(NOB_ERROR, "Failed to create disk.vhd file!"); + return false; + } + + // Expand file to 512MB payload + if (fseek(f, (long)(disk_size - 1), SEEK_SET) != 0) { + fclose(f); + return false; + } + fputc(0, f); + + // Build standard MBR partition table (Sector 0) + uint8_t mbr[512] = {0}; + mbr[510] = 0x55; + mbr[511] = 0xAA; + + // Partition 1 Entry: 64MB FAT32 @ LBA 2048 (1MB offset) + uint8_t *p1 = &mbr[0x01BE]; + p1[0] = 0x80; // Active / Bootable + p1[1] = 0x00; p1[2] = 0x02; p1[3] = 0x00; // CHS Start + p1[4] = 0x0C; // FAT32 LBA + p1[5] = 0xFE; p1[6] = 0xFF; p1[7] = 0xFF; // CHS End (LBA boundary) + uint32_t p1_lba = 2048; + uint32_t p1_sectors = 131072; // 64 MB + memcpy(&p1[8], &p1_lba, 4); + memcpy(&p1[12], &p1_sectors, 4); + + // Partition 2 Entry: 256MB FAT32 @ LBA 133120 (65MB offset) + uint8_t *p2 = &mbr[0x01CE]; + p2[0] = 0x00; // Inactive + p2[1] = 0x00; p2[2] = 0x01; p2[3] = 0x08; // CHS Start + p2[4] = 0x0C; // FAT32 LBA + p2[5] = 0xFE; p2[6] = 0xFF; p2[7] = 0xFF; // CHS End (LBA boundary) + uint32_t p2_lba = 133120; + uint32_t p2_sectors = 524288; // 256 MB + memcpy(&p2[8], &p2_lba, 4); + memcpy(&p2[12], &p2_sectors, 4); + + // Write Sector 0 + fseek(f, 0, SEEK_SET); + fwrite(mbr, 1, 512, f); + + // Append 512-byte Connectix VPC Footer + write_vpc_footer(f, disk_size); + + fclose(f); + + nob_log(NOB_INFO, "Formatting disk.vhd partitions with mformat..."); + if (!run_sync("mformat -i " VHD_PART1 " -F")) return false; + if (!run_sync("mformat -i " VHD_PART2 " -F")) return false; + + return true; +} + // Runs the requested operation. int main(int argc, char **argv) { - // The build tool rebuilds itself before doing any real work so the graph below matches the source. NOB_GO_REBUILD_URSELF(argc, argv); - if (!clean_build_object_directories()) return 1; - create_layout(); + if (!create_disk_vhd_if_missing()) return 1; if (!create_usbfatroot_if_missing()) return 1; + create_layout(); + if (!clean_build_object_directories()) return 1; - // Core images build sequentially because later images link against earlier import libraries. nob_log(NOB_INFO, "--- Building Core Subsystems ---"); const char *core_tasks[] = { @@ -165,7 +316,6 @@ int main(int argc, char **argv) { if (!run_sync(core_tasks[i])) return 1; } - // PCI builds before PCI function drivers because it exports the bus-driver import library. nob_log(NOB_INFO, "--- Building Bus Drivers ---"); const char *pci_cmd = nob_temp_sprintf( "cl %s /MP /c /Fo%s\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /IMPLIB:bin\\%s.lib /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\printf.obj obj\\libc.obj %s\\*.obj", @@ -174,12 +324,9 @@ int main(int argc, char **argv) { ); if (!run_sync(pci_cmd)) return 1; - // 4. Parallel Phase (Drivers and Applications). nob_log(NOB_INFO, "--- Compiling Drivers and Apps in Parallel ---"); Nob_Procs procs = {0}; - // Queue standalone drivers. Each driver is now a target with one or more - // translation units, but still links to one .sys image with DriverEntry. for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) { const Driver_Target *driver = &STANDALONE_DRIVERS[i]; const char *cmd = nob_temp_sprintf( @@ -190,7 +337,6 @@ int main(int argc, char **argv) { nob_da_append(&procs, run_async(cmd)); } - // Parse lwIP Sources and Queue it. Nob_String_Builder sb = {0}, lwip_files = {0}; if (nob_read_entire_file("code\\drivers\\net\\lwip\\lwip_sources.txt", &sb)) { Nob_String_View sv = nob_sv_from_parts(sb.items, sb.count); @@ -206,9 +352,7 @@ int main(int argc, char **argv) { nob_da_append(&procs, run_async(nob_temp_sprintf("cl %s /MP /c /Foobj\\lwip\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\lwip.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj obj\\lwip\\*.obj", LwipCflags, lwip_files.items))); } - // Queue Subsystem Applications. const char *apps[][4] = { - // Folder, Libs, Name, Optional Extra Link Args. {"system\\init", NATIVE_INIT_LIBS_LITERAL, "init", "/ENTRY:_start"}, {"tests\\all_tests", NATIVE_EXE_LIBS_LITERAL, "all_tests", NATIVE_EXE_ENTRY_LITERAL}, {"apps\\shell_win32", WIN32_EXE_LIBS_LITERAL, "shell_win32", NATIVE_EXE_ENTRY_LITERAL}, @@ -239,6 +383,12 @@ int main(int argc, char **argv) { {"tests\\win32_seh_smoke", WIN32_EXE_LIBS_LITERAL, "win32_seh_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\syscall_contract_smoke", NATIVE_EXE_LIBS_LITERAL, "syscall_contract_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\would_block_smoke", NATIVE_EXE_LIBS_LITERAL, "would_block_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\audio_stream_smoke", NATIVE_EXE_LIBS_LITERAL, "audio_stream_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\winmm_waveout_smoke", WIN32_EXE_LIBS_LITERAL, "winmm_waveout_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_condition_variable_smoke", WIN32_EXE_LIBS_LITERAL, "win32_condition_variable_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_mutex_smoke", WIN32_EXE_LIBS_LITERAL, "win32_mutex_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_thread_suspend_smoke", WIN32_EXE_LIBS_LITERAL, "win32_thread_suspend_smoke", NATIVE_EXE_ENTRY_LITERAL}, + {"tests\\win32_module_lookup_smoke", WIN32_EXE_LIBS_LITERAL, "win32_module_lookup_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\cwd_peb_smoke", NATIVE_EXE_LIBS_LITERAL, "cwd_peb_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\mmap_smoke", NATIVE_EXE_LIBS_LITERAL, "mmap_smoke", NATIVE_EXE_ENTRY_LITERAL}, {"tests\\pcache_smoke", NATIVE_EXE_LIBS_LITERAL, "pcache_smoke", NATIVE_EXE_ENTRY_LITERAL}, @@ -270,98 +420,118 @@ int main(int argc, char **argv) { nob_da_append(&procs, run_async(cmd)); } - // Wait for everything to link simultaneously. if (!nob_procs_wait(procs)) { nob_log(NOB_ERROR, "One or more parallel build jobs failed!"); return 1; } - // 4. Packing vvfat root. - nob_log(NOB_INFO, "--- Assembling fatroot ---"); - if (!create_fatroot_layout()) return 1; + // 4. Formatting and Packing VHD via mtools (No Admin, Fast). + nob_log(NOB_INFO, "--- Formatting & Packing disk.vhd with mtools ---"); + + run_sync("mformat -i " VHD_PART1 " -F"); + run_sync("mformat -i " VHD_PART2 " -F"); - const char *copies[][2] = { - {"bin\\bootx64.efi", "fatroot\\EFI\\BOOT\\bootx64.efi"}, - {"bin\\kernel.exe", "fatroot\\system\\colonel.exe"}, - {"bin\\hal.dll", "fatroot\\system\\hal.dll"}, - {"bin\\syscall.dll", "fatroot\\system\\syscall.dll"}, - {"bin\\win32.dll", "fatroot\\system\\win32.dll"}, - {"bin\\posix.dll", "fatroot\\system\\posix.dll"}, - {"code\\settings.ini", "fatroot\\system\\settings.ini"}, - {"bin\\init.exe", "fatroot\\system\\init.exe"}, + mmd_dir(VHD_PART1, "apps"); + mmd_dir(VHD_PART1, "system"); + mmd_dir(VHD_PART1, "drivers"); + mmd_dir(VHD_PART1, "EFI"); + mmd_dir(VHD_PART1, "EFI/BOOT"); + mmd_dir(VHD_PART2, "tests"); - {"bin\\pci.sys", "fatroot\\drivers\\pci.sys"}, - {"bin\\ps2.sys", "fatroot\\drivers\\ps2.sys"}, - {"bin\\serial.sys", "fatroot\\drivers\\serial.sys"}, - {"bin\\ata.sys", "fatroot\\drivers\\ata.sys"}, - {"bin\\ahci.sys", "fatroot\\drivers\\ahci.sys"}, - {"bin\\fat.sys", "fatroot\\drivers\\fat.sys"}, - {"bin\\ntfs.sys", "fatroot\\drivers\\ntfs.sys"}, - {"bin\\vmwaresvga.sys", "fatroot\\drivers\\vmwaresvga.sys"}, - {"bin\\lwip.sys", "fatroot\\drivers\\lwip.sys"}, - {"bin\\usb.sys", "fatroot\\drivers\\usb.sys"}, - {"bin\\rtl816x.sys", "fatroot\\drivers\\rtl816x.sys"}, - {"bin\\rtl8139.sys", "fatroot\\drivers\\rtl8139.sys"}, - {"code\\drivers\\drivers.ini", "fatroot\\drivers\\drivers.ini"}, - {"code\\boot.ini", "fatroot\\boot.ini"}, + const char *copies[][3] = { + {"bin\\bootx64.efi", VHD_PART1, "EFI/BOOT/bootx64.efi"}, + {"bin\\kernel.exe", VHD_PART1, "system/colonel.exe"}, + {"bin\\hal.dll", VHD_PART1, "system/hal.dll"}, + {"bin\\syscall.dll", VHD_PART1, "system/syscall.dll"}, + {"bin\\win32.dll", VHD_PART1, "system/win32.dll"}, + {"bin\\posix.dll", VHD_PART1, "system/posix.dll"}, + {"code\\settings.ini", VHD_PART1, "system/settings.ini"}, + {"bin\\init.exe", VHD_PART1, "system/init.exe"}, + + {"bin\\pci.sys", VHD_PART1, "drivers/pci.sys"}, + {"bin\\ps2.sys", VHD_PART1, "drivers/ps2.sys"}, + {"bin\\serial.sys", VHD_PART1, "drivers/serial.sys"}, + {"bin\\hda.sys", VHD_PART1, "drivers/hda.sys"}, + {"bin\\ata.sys", VHD_PART1, "drivers/ata.sys"}, + {"bin\\ahci.sys", VHD_PART1, "drivers/ahci.sys"}, + {"bin\\fat.sys", VHD_PART1, "drivers/fat.sys"}, + {"bin\\ntfs.sys", VHD_PART1, "drivers/ntfs.sys"}, + {"bin\\vmwaresvga.sys", VHD_PART1, "drivers/vmwaresvga.sys"}, + {"bin\\lwip.sys", VHD_PART1, "drivers/lwip.sys"}, + {"bin\\usb.sys", VHD_PART1, "drivers/usb.sys"}, + {"bin\\rtl816x.sys", VHD_PART1, "drivers/rtl816x.sys"}, + {"bin\\rtl8139.sys", VHD_PART1, "drivers/rtl8139.sys"}, + {"code\\drivers\\drivers.ini", VHD_PART1, "drivers/drivers.ini"}, + {"code\\boot.ini", VHD_PART1, "boot.ini"}, - {"bin\\shell_win32.exe", "fatroot\\apps\\shell_win32.exe"}, - {"bin\\shell_posix.exe", "fatroot\\apps\\shell_posix.exe"}, - {"bin\\shell_native.exe", "fatroot\\apps\\shell_native.exe"}, - {"bin\\edit.exe", "fatroot\\apps\\edit.exe"}, - {"bin\\httpget.exe", "fatroot\\apps\\httpget.exe"}, - {"bin\\tcpserve.exe", "fatroot\\apps\\tcpserve.exe"}, - {"bin\\udp.exe", "fatroot\\apps\\udp.exe"}, - {"bin\\resolve.exe", "fatroot\\apps\\resolve.exe"}, - {"bin\\ping.exe", "fatroot\\apps\\ping.exe"}, - {"bin\\doom.exe", "fatroot\\apps\\doom.exe"}, - {"bin\\video.exe", "fatroot\\apps\\video.exe"}, - {"bin\\memtop.exe", "fatroot\\apps\\memtop.exe"}, + {"bin\\shell_win32.exe", VHD_PART1, "apps/shell_win32.exe"}, + {"bin\\shell_posix.exe", VHD_PART1, "apps/shell_posix.exe"}, + {"bin\\shell_native.exe", VHD_PART1, "apps/shell_native.exe"}, + {"bin\\edit.exe", VHD_PART1, "apps/edit.exe"}, + {"bin\\httpget.exe", VHD_PART1, "apps/httpget.exe"}, + {"bin\\tcpserve.exe", VHD_PART1, "apps/tcpserve.exe"}, + {"bin\\udp.exe", VHD_PART1, "apps/udp.exe"}, + {"bin\\resolve.exe", VHD_PART1, "apps/resolve.exe"}, + {"bin\\ping.exe", VHD_PART1, "apps/ping.exe"}, + {"bin\\doom.exe", VHD_PART1, "apps/doom.exe"}, + {"bin\\video.exe", VHD_PART1, "apps/video.exe"}, + {"bin\\memtop.exe", VHD_PART1, "apps/memtop.exe"}, - {"bin\\all_tests.exe", "fatroot\\tests\\all_tests.exe"}, - {"bin\\async_apc.exe", "fatroot\\tests\\async_apc.exe"}, - {"bin\\async_cancel.exe", "fatroot\\tests\\async_cancel.exe"}, - {"bin\\apc_return_smoke.exe", "fatroot\\tests\\apc_return_smoke.exe"}, - {"bin\\file_stat_smoke.exe", "fatroot\\tests\\file_stat_smoke.exe"}, - {"bin\\handle_dup_smoke.exe", "fatroot\\tests\\handle_dup_smoke.exe"}, - {"bin\\named_queue_lifetime_smoke.exe", "fatroot\\tests\\named_queue_lifetime_smoke.exe"}, - {"bin\\input_api_smoke.exe", "fatroot\\tests\\input_api_smoke.exe"}, - {"bin\\input_model_smoke.exe", "fatroot\\tests\\input_model_smoke.exe"}, - {"bin\\timer_smoke.exe", "fatroot\\tests\\timer_smoke.exe"}, - {"bin\\stats_abi_smoke.exe", "fatroot\\tests\\stats_abi_smoke.exe"}, - {"bin\\realpath_smoke.exe", "fatroot\\tests\\realpath_smoke.exe"}, - {"bin\\file_lock_smoke.exe", "fatroot\\tests\\file_lock_smoke.exe"}, - {"bin\\process_protection_smoke.exe", "fatroot\\tests\\process_protection_smoke.exe"}, - {"bin\\socket_addr_smoke.exe", "fatroot\\tests\\socket_addr_smoke.exe"}, - {"bin\\net_irp_smoke.exe", "fatroot\\tests\\net_irp_smoke.exe"}, - {"bin\\module_load_smoke.exe", "fatroot\\tests\\module_load_smoke.exe"}, - {"bin\\posix_module_smoke.exe", "fatroot\\tests\\posix_module_smoke.exe"}, - {"bin\\exception_dispatcher_smoke.exe", "fatroot\\tests\\exception_dispatcher_smoke.exe"}, - {"bin\\native_exception_runtime_smoke.exe", "fatroot\\tests\\native_exception_runtime_smoke.exe"}, - {"bin\\native_raise_exception_smoke.exe", "fatroot\\tests\\native_raise_exception_smoke.exe"}, - {"bin\\exception_smoke.exe", "fatroot\\tests\\exception_smoke.exe"}, - {"bin\\win32_exception_smoke.exe", "fatroot\\tests\\win32_exception_smoke.exe"}, - {"bin\\win32_seh_smoke.exe", "fatroot\\tests\\win32_seh_smoke.exe"}, - {"bin\\syscall_contract_smoke.exe", "fatroot\\tests\\syscall_contract_smoke.exe"}, - {"bin\\would_block_smoke.exe", "fatroot\\tests\\would_block_smoke.exe"}, - {"bin\\cwd_peb_smoke.exe", "fatroot\\tests\\cwd_peb_smoke.exe"}, - {"bin\\mmap_smoke.exe", "fatroot\\tests\\mmap_smoke.exe"}, - {"bin\\pcache_smoke.exe", "fatroot\\tests\\pcache_smoke.exe"}, - {"bin\\pcache_pressure.exe", "fatroot\\tests\\pcache_pressure.exe"}, - {"bin\\dirty_writeback.exe", "fatroot\\tests\\dirty_writeback.exe"}, - {"bin\\shared_mmap.exe", "fatroot\\tests\\shared_mmap.exe"}, - {"bin\\pcache_invalidate.exe", "fatroot\\tests\\pcache_invalidate.exe"}, - {"bin\\pcache_mmap_read_reuse.exe", "fatroot\\tests\\pcache_mmap_read_reuse.exe"}, - {"bin\\pcache_multithread.exe", "fatroot\\tests\\pcache_multithread.exe"}, - {"bin\\surface_smoke.exe", "fatroot\\tests\\surface_smoke.exe"}, - {"bin\\surface_invalid_rect.exe", "fatroot\\tests\\surface_invalid_rect.exe"}, - {"bin\\surface_multithread_present.exe", "fatroot\\tests\\surface_multithread_present.exe"} + // Partition 2 Apps + {"bin\\all_tests.exe", VHD_PART2, "tests/all_tests.exe"}, + {"bin\\async_apc.exe", VHD_PART2, "tests/async_apc.exe"}, + {"bin\\async_cancel.exe", VHD_PART2, "tests/async_cancel.exe"}, + {"bin\\apc_return_smoke.exe", VHD_PART2, "tests/apc_return_smoke.exe"}, + {"bin\\file_stat_smoke.exe", VHD_PART2, "tests/file_stat_smoke.exe"}, + {"bin\\handle_dup_smoke.exe", VHD_PART2, "tests/handle_dup_smoke.exe"}, + {"bin\\named_queue_lifetime_smoke.exe", VHD_PART2, "tests/named_queue_lifetime_smoke.exe"}, + {"bin\\input_api_smoke.exe", VHD_PART2, "tests/input_api_smoke.exe"}, + {"bin\\input_model_smoke.exe", VHD_PART2, "tests/input_model_smoke.exe"}, + {"bin\\timer_smoke.exe", VHD_PART2, "tests/timer_smoke.exe"}, + {"bin\\stats_abi_smoke.exe", VHD_PART2, "tests/stats_abi_smoke.exe"}, + {"bin\\realpath_smoke.exe", VHD_PART2, "tests/realpath_smoke.exe"}, + {"bin\\file_lock_smoke.exe", VHD_PART2, "tests/file_lock_smoke.exe"}, + {"bin\\process_protection_smoke.exe", VHD_PART2, "tests/process_protection_smoke.exe"}, + {"bin\\socket_addr_smoke.exe", VHD_PART2, "tests/socket_addr_smoke.exe"}, + {"bin\\net_irp_smoke.exe", VHD_PART2, "tests/net_irp_smoke.exe"}, + {"bin\\module_load_smoke.exe", VHD_PART2, "tests/module_load_smoke.exe"}, + {"bin\\posix_module_smoke.exe", VHD_PART2, "tests/posix_module_smoke.exe"}, + {"bin\\exception_dispatcher_smoke.exe", VHD_PART2, "tests/exception_dispatcher_smoke.exe"}, + {"bin\\native_exception_runtime_smoke.exe", VHD_PART2, "tests/native_exception_runtime_smoke.exe"}, + {"bin\\native_raise_exception_smoke.exe", VHD_PART2, "tests/native_raise_exception_smoke.exe"}, + {"bin\\exception_smoke.exe", VHD_PART2, "tests/exception_smoke.exe"}, + {"bin\\win32_exception_smoke.exe", VHD_PART2, "tests/win32_exception_smoke.exe"}, + {"bin\\win32_seh_smoke.exe", VHD_PART2, "tests/win32_seh_smoke.exe"}, + {"bin\\syscall_contract_smoke.exe", VHD_PART2, "tests/syscall_contract_smoke.exe"}, + {"bin\\would_block_smoke.exe", VHD_PART2, "tests/would_block_smoke.exe"}, + {"bin\\audio_stream_smoke.exe", VHD_PART2, "tests/audio_stream_smoke.exe"}, + {"bin\\winmm_waveout_smoke.exe", VHD_PART2, "tests/winmm_waveout_smoke.exe"}, + {"bin\\win32_condition_variable_smoke.exe", VHD_PART2, "tests/win32_condition_variable_smoke.exe"}, + {"bin\\win32_mutex_smoke.exe", VHD_PART2, "tests/win32_mutex_smoke.exe"}, + {"bin\\win32_thread_suspend_smoke.exe", VHD_PART2, "tests/win32_thread_suspend_smoke.exe"}, + {"bin\\win32_module_lookup_smoke.exe", VHD_PART2, "tests/win32_module_lookup_smoke.exe"}, + {"bin\\cwd_peb_smoke.exe", VHD_PART2, "tests/cwd_peb_smoke.exe"}, + {"bin\\mmap_smoke.exe", VHD_PART2, "tests/mmap_smoke.exe"}, + {"bin\\pcache_smoke.exe", VHD_PART2, "tests/pcache_smoke.exe"}, + {"bin\\pcache_pressure.exe", VHD_PART2, "tests/pcache_pressure.exe"}, + {"bin\\dirty_writeback.exe", VHD_PART2, "tests/dirty_writeback.exe"}, + {"bin\\shared_mmap.exe", VHD_PART2, "tests/shared_mmap.exe"}, + {"bin\\pcache_invalidate.exe", VHD_PART2, "tests/pcache_invalidate.exe"}, + {"bin\\pcache_mmap_read_reuse.exe", VHD_PART2, "tests/pcache_mmap_read_reuse.exe"}, + {"bin\\pcache_multithread.exe", VHD_PART2, "tests/pcache_multithread.exe"}, + {"bin\\surface_smoke.exe", VHD_PART2, "tests/surface_smoke.exe"}, + {"bin\\surface_invalid_rect.exe", VHD_PART2, "tests/surface_invalid_rect.exe"}, + {"bin\\surface_multithread_present.exe", VHD_PART2, "tests/surface_multithread_present.exe"} }; for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) { - if (!nob_copy_file(copies[i][0], copies[i][1])) return 1; + mcopy_file(copies[i][0], copies[i][1], copies[i][2]); } + if (nob_file_exists("code\\user\\apps\\video\\bink2w64.dll")) mcopy_file("code\\user\\apps\\video\\bink2w64.dll", VHD_PART1, "apps/bink2w64.dll"); + if (nob_file_exists("code\\user\\apps\\video\\video.bk2")) mcopy_file("code\\user\\apps\\video\\video.bk2", VHD_PART1, "apps/video.bk2"); + if (nob_file_exists("code\\user\\apps\\doom\\doom1.wad")) mcopy_file("code\\user\\apps\\doom\\doom1.wad", VHD_PART1, "apps/doom1.wad"); + nob_log(NOB_INFO, "Build complete!"); return 0; } diff --git a/code/common/libc.c b/code/common/libc.c index 89dc672..05e1697 100644 --- a/code/common/libc.c +++ b/code/common/libc.c @@ -20,6 +20,31 @@ char* strncpy(char* dest, const char* src, size_t n) { return dest; } +char *strstr(const char *haystack, const char *needle) { + // An empty needle matches the beginning of haystack + if (!*needle) { + return (char *)haystack; + } + + for (; *haystack != '\0'; haystack++) { + const char *h = haystack; + const char *n = needle; + + // Match as many characters as possible + while (*h != '\0' && *n != '\0' && *h == *n) { + h++; + n++; + } + + // If we reached the end of needle, we found a match + if (*n == '\0') { + return (char *)haystack; + } + } + + return NULL; +} + // Implements the strcat operation. char* strcat(char* dest, const char* src) { char* ptr = dest; diff --git a/code/drivers/audio/hda/hda.c b/code/drivers/audio/hda/hda.c new file mode 100644 index 0000000..809e758 --- /dev/null +++ b/code/drivers/audio/hda/hda.c @@ -0,0 +1,403 @@ +// Intel High Definition Audio PCI attachment and controller lifetime management. + +#include "hda.h" + +#include + +const NTL_DDK* gHdaDdk; +static DRIVER_OBJECT gHdaDriver; +static HDA_CONTROLLER* gHdaControllers[HDA_MAX_CONTROLLERS]; +static uint32_t gHdaControllerCount; +static SPINLOCK gHdaRegistryLock = SPINLOCK_INIT; + +static const IO_DEVICE_ID gHdaIds[] = { + { IoBusPci, { "PCI\\VEN_8086&DEV_2668", sizeof("PCI\\VEN_8086&DEV_2668") - 1u } }, + { IoBusPci, { "PCI\\VEN_15AD&DEV_1977", sizeof("PCI\\VEN_15AD&DEV_1977") - 1u } }, + { IoBusPci, { "PCI\\CC_040300", sizeof("PCI\\CC_040300") - 1u } }, +}; + +static const IO_DRIVER_ID_TABLE gHdaIdTable = { + gHdaIds, + sizeof(gHdaIds) / sizeof(gHdaIds[0]), +}; + +// Registers a controller in the shared interrupt scan set. +static KSTATUS HdaRegisterController(HDA_CONTROLLER* controller) { + if (!controller) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock); + for (uint32_t index = 0; index < gHdaControllerCount; index++) { + if (gHdaControllers[index] == controller) { + KeReleaseSpinlock(&gHdaRegistryLock, flags); + return NTLSTATUS_SUCCESS; + } + } + if (gHdaControllerCount == HDA_MAX_CONTROLLERS) { + KeReleaseSpinlock(&gHdaRegistryLock, flags); + return NTLSTATUS_NO_MEMORY; + } + gHdaControllers[gHdaControllerCount++] = controller; + KeReleaseSpinlock(&gHdaRegistryLock, flags); + return NTLSTATUS_SUCCESS; +} + +// Removes a controller from the shared interrupt scan set. +static void HdaUnregisterController(HDA_CONTROLLER* controller) { + if (!controller) + return; + + uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock); + for (uint32_t index = 0; index < gHdaControllerCount; index++) { + if (gHdaControllers[index] != controller) + continue; + gHdaControllers[index] = gHdaControllers[gHdaControllerCount - 1u]; + gHdaControllers[gHdaControllerCount - 1u] = (HDA_CONTROLLER*)0; + gHdaControllerCount--; + break; + } + KeReleaseSpinlock(&gHdaRegistryLock, flags); +} + +// Dispatches a hardware interrupt to every live HDA controller. +static void HdaIsr(void) { + DEVICE_OBJECT* devices[HDA_MAX_CONTROLLERS]; + uint32_t count = 0; + + uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock); + count = gHdaControllerCount; + if (count > HDA_MAX_CONTROLLERS) + count = HDA_MAX_CONTROLLERS; + for (uint32_t index = 0; index < count; index++) { + HDA_CONTROLLER* controller = gHdaControllers[index]; + devices[index] = controller ? controller->Fdo : (DEVICE_OBJECT*)0; + if (devices[index]) + IoReferenceDeviceObject(devices[index]); + } + KeReleaseSpinlock(&gHdaRegistryLock, flags); + + for (uint32_t index = 0; index < count; index++) { + DEVICE_OBJECT* device = devices[index]; + if (!device) + continue; + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension; + if (controller && controller->Magic == HDA_CONTROLLER_MAGIC) + HdaHandleInterrupt(controller); + IoDereferenceDeviceObject(device); + } +} + +// Registers the global ISR once for each kernel IRQ slot used by HDA. +static KSTATUS HdaRegisterIrq(uint8_t irq) { + if (irq >= 24u) + return NTLSTATUS_NOT_SUPPORTED; + KeIrqRegister(irq, HdaIsr); + return NTLSTATUS_SUCCESS; +} + +// Maps BAR0 and validates the stream-descriptor aperture described by GCAP. +static KSTATUS HdaMapRegisters(HDA_CONTROLLER* controller) { + PCI_BAR bar; + KSTATUS status = PciGetBar(controller->Pdo, 0, &bar); + if (!KSUCCESS(status)) + return status; + if (bar.Type != PciBarMemory32 && bar.Type != PciBarMemory64) + return NTLSTATUS_NOT_SUPPORTED; + + status = PciMapBar(controller->Pdo, + 0, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, + (void**)&controller->Registers, + &controller->RegisterBytes); + if (!KSUCCESS(status)) + return status; + if (controller->RegisterBytes < HDA_REG_STREAM_BASE) + return NTLSTATUS_BUFFER_TOO_SMALL; + + controller->Gcap = HdaRead16(controller, HDA_REG_GCAP); + controller->InputStreamCount = + (uint8_t)((controller->Gcap & HDA_GCAP_ISS_MASK) >> HDA_GCAP_ISS_SHIFT); + controller->OutputStreamCount = + (uint8_t)((controller->Gcap & HDA_GCAP_OSS_MASK) >> HDA_GCAP_OSS_SHIFT); + controller->BidirectionalStreamCount = + (uint8_t)((controller->Gcap & HDA_GCAP_BSS_MASK) >> HDA_GCAP_BSS_SHIFT); + controller->TotalStreamCount = + (uint8_t)(controller->InputStreamCount + + controller->OutputStreamCount + + controller->BidirectionalStreamCount); + if ((controller->OutputStreamCount == 0 && + controller->BidirectionalStreamCount == 0) || + controller->TotalStreamCount == 0 || + controller->TotalStreamCount > 30u) + return NTLSTATUS_NOT_SUPPORTED; + + uint64_t requiredBytes = HDA_REG_STREAM_BASE + + (uint64_t)controller->TotalStreamCount * + HDA_REG_STREAM_STRIDE; + if (controller->RegisterBytes < requiredBytes) + return NTLSTATUS_BUFFER_TOO_SMALL; + + // Dedicated output descriptors precede bidirectional descriptors. Use the + // first bidirectional descriptor when no output-only descriptor exists. + uint8_t outputDescriptorBase = controller->InputStreamCount; + uint8_t bidirectionalDescriptorBase = + (uint8_t)(controller->InputStreamCount + controller->OutputStreamCount); + controller->Output.DescriptorIndex = controller->OutputStreamCount + ? outputDescriptorBase + : bidirectionalDescriptorBase; + controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG; + return NTLSTATUS_SUCCESS; +} + +// Configures MSI first and enables INTx only after MSI setup reports failure. +static KSTATUS HdaConfigureInterrupt(HDA_CONTROLLER* controller) { + KSTATUS status = PciEnableMsiAuto(controller->Pdo, &controller->Irq); + if (KSUCCESS(status)) { + controller->UsingMsi = true; + return NTLSTATUS_SUCCESS; + } + + gHdaDdk->Debug->Print( + "HDA: MSI setup failed status=0x%x; attempting explicit INTx fallback\n", + status); + controller->UsingMsi = false; + status = PciGetInterrupt(controller->Pdo, &controller->Irq); + if (!KSUCCESS(status)) + return status; + if (controller->Irq == 0xFFu || controller->Irq >= 24u) + return NTLSTATUS_NOT_SUPPORTED; + + status = PciEnableDevice(controller->Pdo, + PCI_ENABLE_MEMORY | + PCI_ENABLE_BUSMASTER | + PCI_ENABLE_INTX); + return status; +} + +// Waits until queued or active interrupt work no longer references the controller. +static KSTATUS HdaWaitForControllerInterruptWorkIdle(HDA_CONTROLLER* controller) { + if (!controller) + return NTLSTATUS_INVALID_PARAMETER; + for (uint32_t elapsed = 0; elapsed < 100000u; elapsed += 10u) { + if (controller->IsrActive == 0 && + controller->Dpc.Inserted == 0 && controller->DpcActive == 0) + return NTLSTATUS_SUCCESS; + gHdaDdk->Timer->Stall(10); + } + return NTLSTATUS_TIMEOUT; +} + +// Starts one PCI HDA controller and leaves endpoint registration to codec discovery. +static KSTATUS HdaStartDevice(HDA_CONTROLLER* controller) { + if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || + !controller->Pdo || !controller->Fdo) + return NTLSTATUS_INVALID_PARAMETER; + if (controller->Started) + return NTLSTATUS_SUCCESS; + + KSTATUS status = PciGetDeviceInfo(controller->Pdo, &controller->PciInfo); + if (!KSUCCESS(status)) + return status; + if (controller->PciInfo.ClassCode != 0x04u || + controller->PciInfo.Subclass != 0x03u || + controller->PciInfo.ProgIf != 0x00u) + return NTLSTATUS_NOT_SUPPORTED; + + status = PciEnableDevice(controller->Pdo, + PCI_ENABLE_MEMORY | + PCI_ENABLE_BUSMASTER | + PCI_DISABLE_INTX); + if (!KSUCCESS(status)) + return status; + + status = HdaMapRegisters(controller); + if (!KSUCCESS(status)) + return status; + + status = HdaConfigureInterrupt(controller); + if (!KSUCCESS(status)) + return status; + + status = HdaAllocateDma(controller); + if (!KSUCCESS(status)) + return status; + KeInitializeDpc(&controller->Dpc, HdaDpcRoutine, controller); + + status = HdaStartRecoveryWorker(controller); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + status = HdaInitializeController(controller); + if (!KSUCCESS(status)) { + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + + status = HdaRegisterController(controller); + if (!KSUCCESS(status)) { + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + status = HdaRegisterIrq(controller->Irq); + if (!KSUCCESS(status)) { + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + + controller->Started = true; + controller->Fdo->PnpState = DevicePnpStarted; + + status = HdaCodecInitialize(controller); + if (!KSUCCESS(status) && status != NTLSTATUS_NOT_SUPPORTED && + status != NTLSTATUS_NOT_FOUND) { + controller->Started = false; + controller->Fdo->PnpState = DevicePnpNotStarted; + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + HdaQuiesceController(controller); + HdaReleaseDma(controller); + return status; + } + if (!KSUCCESS(status)) { + gHdaDdk->Debug->Print( + "HDA: no analog endpoint registered status=0x%x\n", + status); + } + + HdaArmInterrupts(controller); + HdaHandleInterrupt(controller); + + return NTLSTATUS_SUCCESS; +} + +// Handles PnP lifetime transitions for one HDA function device object. +static KSTATUS HdaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp || !device->DeviceExtension) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension; + if (controller->Magic != HDA_CONTROLLER_MAGIC) + return NTLSTATUS_INVALID_PARAMETER; + + IO_STACK_LOCATION* stack = IoGetCurrentIrpStackLocation(irp); + if (!stack || stack->MajorFunction != IRP_MJ_PNP) + return NTLSTATUS_INVALID_PARAMETER; + + switch ((IO_PNP_MINOR_FUNCTION)stack->MinorFunction) { + case IRP_MN_START_DEVICE: + return HdaStartDevice(controller); + + case IRP_MN_QUERY_REMOVE_DEVICE: + case IRP_MN_CANCEL_REMOVE_DEVICE: + return NTLSTATUS_SUCCESS; + + case IRP_MN_SURPRISE_REMOVAL: + controller->Started = false; + controller->SurpriseRemoved = true; + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + KSTATUS surpriseDpcStatus = + HdaWaitForControllerInterruptWorkIdle(controller); + if (!KSUCCESS(surpriseDpcStatus)) { + KePanic(KE_PANIC_CODE_DRIVER, + "HDA: interrupt work did not quiesce during surprise removal status=0x%x\n", + surpriseDpcStatus); + } + HdaCodecStop(controller, IoAudioSurpriseRemoval); + HdaCodecDestroy(controller); + device->PnpState = DevicePnpSurpriseRemoved; + return NTLSTATUS_SUCCESS; + + case IRP_MN_REMOVE_DEVICE: + if (!controller->SurpriseRemoved && controller->Registers) + HdaWrite32(controller, HDA_REG_INTCTL, 0); + controller->Started = false; + HdaUnregisterController(controller); + HdaStopRecoveryWorker(controller); + KSTATUS dpcStatus = HdaWaitForControllerInterruptWorkIdle(controller); + if (!KSUCCESS(dpcStatus)) { + KePanic(KE_PANIC_CODE_DRIVER, + "HDA: interrupt work did not quiesce during removal status=0x%x\n", + dpcStatus); + } + HdaCodecStop(controller, + controller->SurpriseRemoved + ? IoAudioSurpriseRemoval + : IoAudioOrderlyRemoval); + HdaCodecDestroy(controller); + if (!controller->SurpriseRemoved) + HdaQuiesceController(controller); + controller->Removed = true; + HdaReleaseDma(controller); + if (!controller->SurpriseRemoved) + (void)PciEnableDevice(controller->Pdo, PCI_DISABLE_INTX); + device->PnpState = DevicePnpDeleted; + IoDetachDevice(device); + IoDeleteDevice(device); + return NTLSTATUS_SUCCESS; + + default: + return IoForwardIrpToLower(device, irp); + } +} + +// Attaches one HDA function device object to a matching PCI PDO. +static KSTATUS HdaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) { + if (!driver || !physicalDeviceObject || + physicalDeviceObject->Type != DeviceObjectPhysical) + return NTLSTATUS_INVALID_PARAMETER; + + DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0; + KSTATUS status = IoCreateDevice(driver, sizeof(HDA_CONTROLLER), &fdo); + if (!KSUCCESS(status)) + return status; + (void)IoSetDeviceName(fdo, SYS_SV_LITERAL("hda")); + + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)fdo->DeviceExtension; + memset(controller, 0, sizeof(*controller)); + controller->Magic = HDA_CONTROLLER_MAGIC; + controller->Fdo = fdo; + controller->Pdo = physicalDeviceObject; + controller->Irq = 0xFFu; + controller->StateLock = (SPINLOCK)SPINLOCK_INIT; + controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG; + + controller->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject); + if (!controller->LowerDevice) { + IoDeleteDevice(fdo); + return NTLSTATUS_FAILURE; + } + return NTLSTATUS_SUCCESS; +} + +// Registers the Intel HDA transport driver with the I/O manager. +KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { + (void)init; + if (!ddk || !ddk->Debug || !ddk->Debug->Print || !ddk->Dma || !ddk->Timer || + !ddk->Timer->QueryTicks || !ddk->Timer->Stall || !ddk->Audio || + !ddk->Scheduler || !ddk->Scheduler->DelayThread) + return NTLSTATUS_INVALID_PARAMETER; + + gHdaDdk = ddk; + gHdaControllerCount = 0; + memset(gHdaControllers, 0, sizeof(gHdaControllers)); + + memset(&gHdaDriver, 0, sizeof(gHdaDriver)); + IoInitializeDriverObject(&gHdaDriver, SYS_SV_LITERAL("hda")); + gHdaDriver.IdTable = &gHdaIdTable; + gHdaDriver.DriverAddDevice = HdaAddDevice; + gHdaDriver.MajorFunction[IRP_MJ_PNP] = HdaDispatchPnp; + + KSTATUS status = IoRegisterDriverObject(&gHdaDriver); + if (!KSUCCESS(status)) + gHdaDdk->Debug->Print("HDA: IoRegisterDriverObject failed status=0x%x\n", status); + return status; +} diff --git a/code/drivers/audio/hda/hda.h b/code/drivers/audio/hda/hda.h new file mode 100644 index 0000000..b307394 --- /dev/null +++ b/code/drivers/audio/hda/hda.h @@ -0,0 +1,389 @@ +// Intel High Definition Audio transport declarations shared by the controller and stream implementation. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#define HDA_CONTROLLER_MAGIC 0x4144484Eu +#define HDA_MAX_CONTROLLERS 8u +#define HDA_MAX_CODECS 15u +#define HDA_MAX_NODE_ID 0xFFu +#define HDA_MAX_UNSOLICITED_RESPONSES 32u + +#define HDA_OUTPUT_SAMPLE_RATE 48000u +#define HDA_OUTPUT_CHANNELS 2u +#define HDA_OUTPUT_VALID_BITS 16u +#define HDA_OUTPUT_BYTES_PER_SAMPLE 2u +#define HDA_OUTPUT_BYTES_PER_FRAME 4u +#define HDA_OUTPUT_PERIOD_FRAMES 512u +#define HDA_OUTPUT_PERIOD_COUNT 4u +#define HDA_OUTPUT_PERIOD_BYTES (HDA_OUTPUT_PERIOD_FRAMES * HDA_OUTPUT_BYTES_PER_FRAME) +#define HDA_OUTPUT_BUFFER_BYTES (HDA_OUTPUT_PERIOD_COUNT * HDA_OUTPUT_PERIOD_BYTES) +#define HDA_OUTPUT_STREAM_FORMAT 0x0011u +#define HDA_OUTPUT_STREAM_TAG 1u +#define HDA_MAX_LOGICAL_ENDPOINTS 3u + +#define HDA_REG_GCAP 0x00u +#define HDA_REG_VMIN 0x02u +#define HDA_REG_VMAJ 0x03u +#define HDA_REG_GCTL 0x08u +#define HDA_REG_STATESTS 0x0Eu +#define HDA_REG_INTCTL 0x20u +#define HDA_REG_INTSTS 0x24u +#define HDA_REG_WALLCLK 0x30u +#define HDA_REG_CORBLBASE 0x40u +#define HDA_REG_CORBUBASE 0x44u +#define HDA_REG_CORBWP 0x48u +#define HDA_REG_CORBRP 0x4Au +#define HDA_REG_CORBCTL 0x4Cu +#define HDA_REG_CORBSTS 0x4Du +#define HDA_REG_CORBSIZE 0x4Eu +#define HDA_REG_RIRBLBASE 0x50u +#define HDA_REG_RIRBUBASE 0x54u +#define HDA_REG_RIRBWP 0x58u +#define HDA_REG_RINTCNT 0x5Au +#define HDA_REG_RIRBCTL 0x5Cu +#define HDA_REG_RIRBSTS 0x5Du +#define HDA_REG_RIRBSIZE 0x5Eu +#define HDA_REG_DPLBASE 0x70u +#define HDA_REG_DPUBASE 0x74u +#define HDA_REG_STREAM_BASE 0x80u +#define HDA_REG_STREAM_STRIDE 0x20u + +#define HDA_GCAP_64OK 0x0001u +#define HDA_GCAP_BSS_MASK 0x00F8u +#define HDA_GCAP_BSS_SHIFT 3u +#define HDA_GCAP_ISS_MASK 0x0F00u +#define HDA_GCAP_ISS_SHIFT 8u +#define HDA_GCAP_OSS_MASK 0xF000u +#define HDA_GCAP_OSS_SHIFT 12u + +#define HDA_GCTL_RESET 0x00000001u +#define HDA_GCTL_UNSOLICITED_ENABLE 0x00000100u + +#define HDA_INT_CONTROLLER_ENABLE 0x40000000u +#define HDA_INT_GLOBAL_ENABLE 0x80000000u +#define HDA_INT_CONTROLLER_STATUS 0x40000000u +#define HDA_INT_GLOBAL_STATUS 0x80000000u + +#define HDA_CORBRP_POINTER_MASK 0x00FFu +#define HDA_CORBRP_RESET 0x8000u +#define HDA_CORBCTL_MEMORY_ERROR_INTERRUPT 0x01u +#define HDA_CORBCTL_RUN 0x02u +#define HDA_CORBSTS_MEMORY_ERROR 0x01u + +#define HDA_RIRBWP_RESET 0x8000u +#define HDA_RIRBCTL_RESPONSE_INTERRUPT 0x01u +#define HDA_RIRBCTL_RUN 0x02u +#define HDA_RIRBCTL_OVERRUN_INTERRUPT 0x04u +#define HDA_RIRBSTS_RESPONSE 0x01u +#define HDA_RIRBSTS_OVERRUN 0x04u +#define HDA_RIRB_RESPONSE_CODEC_MASK 0x0000000Fu +#define HDA_RIRB_RESPONSE_UNSOLICITED 0x00000010u + +#define HDA_RING_SIZE_CAP_2 0x10u +#define HDA_RING_SIZE_CAP_16 0x20u +#define HDA_RING_SIZE_CAP_256 0x40u +#define HDA_RING_SIZE_SELECT_2 0x00u +#define HDA_RING_SIZE_SELECT_16 0x01u +#define HDA_RING_SIZE_SELECT_256 0x02u + +#define HDA_DPLBASE_ENABLE 0x00000001u + +#define HDA_STREAM_REG_CTL 0x00u +#define HDA_STREAM_REG_STS 0x03u +#define HDA_STREAM_REG_LPIB 0x04u +#define HDA_STREAM_REG_CBL 0x08u +#define HDA_STREAM_REG_LVI 0x0Cu +#define HDA_STREAM_REG_FORMAT 0x12u +#define HDA_STREAM_REG_BDLPL 0x18u +#define HDA_STREAM_REG_BDLPU 0x1Cu + +#define HDA_STREAM_CTL_RESET 0x000001u +#define HDA_STREAM_CTL_RUN 0x000002u +#define HDA_STREAM_CTL_INTERRUPT_COMPLETE 0x000004u +#define HDA_STREAM_CTL_INTERRUPT_FIFO_ERROR 0x000008u +#define HDA_STREAM_CTL_INTERRUPT_DESCRIPTOR_ERROR 0x000010u +#define HDA_STREAM_CTL_INTERRUPT_MASK 0x00001Cu +#define HDA_STREAM_CTL_TAG_SHIFT 20u +#define HDA_STREAM_CTL_TAG_MASK 0x00F00000u + +#define HDA_STREAM_STS_COMPLETE 0x04u +#define HDA_STREAM_STS_FIFO_ERROR 0x08u +#define HDA_STREAM_STS_DESCRIPTOR_ERROR 0x10u +#define HDA_STREAM_STS_ERROR_MASK 0x18u +#define HDA_STREAM_STS_INTERRUPT_MASK 0x1Cu + +#define HDA_BDL_FLAG_INTERRUPT_ON_COMPLETION 0x00000001u + +#define HDA_VERB_GET_PARAMETER(parameterId) (0xF0000u | ((uint32_t)(parameterId) & 0xFFu)) +#define HDA_PARAMETER_VENDOR_ID 0x00u + +#define HDA_CONTROLLER_RESET_TIMEOUT_US 100000u +#define HDA_STREAM_RESET_TIMEOUT_US 10000u +#define HDA_RING_RESET_TIMEOUT_US 10000u +#define HDA_CODEC_COMMAND_TIMEOUT_US 100000u +#define HDA_CODEC_COMMAND_ATTEMPTS 2u +#define HDA_CODEC_RESET_SETTLE_US 1000u +#define HDA_STREAM_ERROR_WINDOW_US 1000000ull +#define HDA_STREAM_ERRORS_BEFORE_CONTROLLER_RESET 3u + +#define HDA_RECOVERY_NONE 0u +#define HDA_RECOVERY_STREAM 1u +#define HDA_RECOVERY_CONTROLLER 2u + +#pragma pack(push, 1) +typedef struct HDA_BDL_ENTRY { + uint64_t Address; + uint32_t Length; + uint32_t Flags; +} HDA_BDL_ENTRY; + +typedef struct HDA_RIRB_ENTRY { + uint32_t Response; + uint32_t ResponseExtra; +} HDA_RIRB_ENTRY; +#pragma pack(pop) + +_Static_assert(sizeof(HDA_BDL_ENTRY) == 16u, "HDA BDL entries must be 16 bytes"); +_Static_assert(sizeof(HDA_RIRB_ENTRY) == 8u, "HDA RIRB entries must be 8 bytes"); + +typedef struct HDA_CONTROLLER HDA_CONTROLLER; +typedef struct HDA_CODEC_STATE HDA_CODEC_STATE; + +typedef struct HDA_OUTPUT_DESCRIPTION { + void* PeriodBuffer; + size_t PeriodBufferBytes; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + size_t PeriodStrideBytes; + IO_AUDIO_FORMAT HardwareFormat; + uint8_t StreamTag; +} HDA_OUTPUT_DESCRIPTION; + +typedef struct HDA_TRANSPORT_CALLBACKS { + // Reports one BDL-owned period after completion interrupt processing. + void (*PeriodComplete)(void* context, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t finePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks); + // Delivers one unsolicited codec response outside the controller lock. + void (*UnsolicitedResponse)(void* context, + uint8_t codecAddress, + uint32_t response, + uint32_t responseExtra); + // Restores codec programming in a muted state while transport start and stop are serialized. + KSTATUS (*RestoreAfterControllerReset)(void* context); + // Unmutes the restored route only after controller recovery has restarted DMA. + KSTATUS (*ResumeAfterControllerReset)(void* context); + // Reports a recovery failure that leaves the transport stopped. + void (*TransportFailure)(void* context, KSTATUS status); +} HDA_TRANSPORT_CALLBACKS; + +typedef struct HDA_UNSOLICITED_RESPONSE { + uint32_t Response; + uint32_t ResponseExtra; +} HDA_UNSOLICITED_RESPONSE; + +typedef struct HDA_OUTPUT_RING { + IO_DMA_BUFFER* Dma; + uint8_t* Buffer; + uint64_t DeviceAddress; + uint8_t NextLogicalPeriod; + uint64_t PresentedFrames; +} HDA_OUTPUT_RING; + +typedef struct HDA_OUTPUT_STREAM { + uint8_t DescriptorIndex; + uint8_t StreamTag; + uint8_t BdlPeriodMap[HDA_OUTPUT_PERIOD_COUNT]; + uint8_t NextCompletionBdl; + bool Prepared; + bool Running; + volatile long PendingPeriodInterrupts; + volatile long PendingDmaErrors; + HDA_OUTPUT_RING Rings[HDA_MAX_LOGICAL_ENDPOINTS]; + uint8_t ActiveRingIndex; + IO_DMA_BUFFER* BdlDma; + HDA_BDL_ENTRY* Bdl; + uint64_t BdlDeviceAddress; + uint64_t PositionRunBaseFrames; + uint64_t LastPositionFrames; + uint64_t PositionWrapFrames; + uint32_t LastHardwarePositionBytes; + bool PositionInitialized; + uint64_t InterruptCount; + uint64_t PeriodCompletionCount; + uint64_t DmaErrorCount; + uint64_t StreamResetCount; + uint64_t PositionBufferReadCount; + uint64_t LpibReadCount; + uint64_t PositionDiscontinuityCount; +} HDA_OUTPUT_STREAM; + +struct HDA_CONTROLLER { + uint32_t Magic; + DEVICE_OBJECT* Fdo; + DEVICE_OBJECT* Pdo; + DEVICE_OBJECT* LowerDevice; + PCI_DEVICE_INFO PciInfo; + volatile uint8_t* Registers; + uint64_t RegisterBytes; + uint16_t Gcap; + uint8_t InputStreamCount; + uint8_t OutputStreamCount; + uint8_t BidirectionalStreamCount; + uint8_t TotalStreamCount; + uint16_t CodecMask; + uint8_t Irq; + bool UsingMsi; + bool Started; + bool Removed; + bool SurpriseRemoved; + SPINLOCK StateLock; + KDPC Dpc; + volatile long IsrActive; + volatile long DpcActive; + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* CorbDma; + IO_DMA_BUFFER* RirbDma; + IO_DMA_BUFFER* PositionDma; + uint32_t* Corb; + HDA_RIRB_ENTRY* Rirb; + volatile uint64_t* PositionBuffer; + uint64_t CorbDeviceAddress; + uint64_t RirbDeviceAddress; + uint64_t PositionDeviceAddress; + uint16_t CorbEntryCount; + uint16_t RirbEntryCount; + uint16_t CorbWritePointer; + uint16_t RirbReadPointer; + MUTEX* CommandMutex; + MUTEX* StreamMutex; + bool CommandWaiting; + bool CommandResponseReady; + uint8_t CommandCodecAddress; + uint32_t CommandResponse; + uint32_t CommandResponseExtra; + uint64_t CodecCommandCount; + uint64_t CodecCommandRetryCount; + uint64_t CodecCommandFailureCount; + uint64_t RirbResponseCount; + uint64_t RirbUnsolicitedCount; + uint64_t RirbOverrunCount; + uint64_t CorbMemoryErrorCount; + HDA_UNSOLICITED_RESPONSE Unsolicited[HDA_MAX_UNSOLICITED_RESPONSES]; + uint8_t UnsolicitedRead; + uint8_t UnsolicitedWrite; + uint8_t UnsolicitedCount; + HDA_OUTPUT_STREAM Output; + HDA_TRANSPORT_CALLBACKS Callbacks; + void* CallbackContext; + SEMAPHORE* RecoverySemaphore; + EVENT* RecoveryFinished; + THREAD* RecoveryThread; + bool RecoveryWorkerRunning; + bool RecoveryInProgress; + uint8_t PendingRecovery; + uint64_t StreamErrorWindowStart; + uint32_t StreamErrorsInWindow; + uint64_t ControllerResetCount; + HDA_CODEC_STATE* Codec; + uint64_t InterruptCount; + uint64_t ControllerInterruptCount; +}; + +extern const NTL_DDK* gHdaDdk; + +// Returns one byte from the controller register aperture. +uint8_t HdaRead8(const HDA_CONTROLLER* controller, uint32_t offset); + +// Returns one word from the controller register aperture. +uint16_t HdaRead16(const HDA_CONTROLLER* controller, uint32_t offset); + +// Returns one dword from the controller register aperture. +uint32_t HdaRead32(const HDA_CONTROLLER* controller, uint32_t offset); + +// Writes one byte to the controller register aperture. +void HdaWrite8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t value); + +// Writes one word to the controller register aperture. +void HdaWrite16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t value); + +// Writes one dword to the controller register aperture. +void HdaWrite32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t value); + +// Returns the register offset of the controller's one output stream descriptor. +uint32_t HdaOutputStreamBase(const HDA_CONTROLLER* controller); + +// Allocates the transport and output DMA resources for one controller. +KSTATUS HdaAllocateDma(HDA_CONTROLLER* controller); + +// Releases all DMA and synchronization resources owned by one controller. +void HdaReleaseDma(HDA_CONTROLLER* controller); + +// Resets and initializes the controller, command rings, position buffer, and output descriptor. +KSTATUS HdaInitializeController(HDA_CONTROLLER* controller); + +// Arms controller and output-stream interrupts after ISR registration. +void HdaArmInterrupts(HDA_CONTROLLER* controller); + +// Stops hardware access without releasing software resources. +void HdaQuiesceController(HDA_CONTROLLER* controller); + +// Sends one 20-bit codec verb payload through CORB/RIRB with one retry. +KSTATUS HdaCodecVerb(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut); + +// Copies one logical endpoint ring contract for codec and endpoint setup. +KSTATUS HdaDescribeOutput(HDA_CONTROLLER* controller, + uint8_t ringIndex, + HDA_OUTPUT_DESCRIPTION* descriptionOut); + +// Selects one logical endpoint ring while the stream is stopped. +KSTATUS HdaSelectOutputRing(HDA_CONTROLLER* controller, uint8_t ringIndex); + +// Replaces the transport callbacks while the output stream is stopped. +KSTATUS HdaSetTransportCallbacks(HDA_CONTROLLER* controller, + const HDA_TRANSPORT_CALLBACKS* callbacks, + void* context); + +// Starts the prepared fixed-format output stream. +KSTATUS HdaStartOutput(HDA_CONTROLLER* controller); + +// Stops the output stream while retaining its DMA ring. +void HdaStopOutput(HDA_CONTROLLER* controller); + +// Returns the monotonic fine position derived from the position buffer or LPIB. +uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller); + +// Captures one controller interrupt and queues deferred processing. +void HdaHandleInterrupt(HDA_CONTROLLER* controller); + +// Runs period delivery, unsolicited callbacks, and recovery scheduling at DPC level. +void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2); + +// Starts the passive recovery worker for one controller. +KSTATUS HdaStartRecoveryWorker(HDA_CONTROLLER* controller); + +// Stops and joins the passive recovery worker for one controller. +void HdaStopRecoveryWorker(HDA_CONTROLLER* controller); + +// Discovers codec topology and registers usable analog render endpoints. +KSTATUS HdaCodecInitialize(HDA_CONTROLLER* controller); + +// Stops jack workers and unregisters codec endpoints. +void HdaCodecStop(HDA_CONTROLLER* controller, IO_AUDIO_REMOVAL_MODE mode); + +// Releases codec topology storage after interrupt work is quiescent. +void HdaCodecDestroy(HDA_CONTROLLER* controller); diff --git a/code/drivers/audio/hda/hda_codec.c b/code/drivers/audio/hda/hda_codec.c new file mode 100644 index 0000000..ad341de --- /dev/null +++ b/code/drivers/audio/hda/hda_codec.c @@ -0,0 +1,1928 @@ +// Generic Intel HDA codec discovery and analog render routing. + +#include "hda.h" + +#include +#include + +#define HDA_CODEC_POOL_TAG MM_POOL_TAG('h', 'd', 'a', 'c') +#define HDA_MAX_WIDGETS (HDA_MAX_NODE_ID + 1u) +#define HDA_MAX_AUDIO_FUNCTIONS 8u +#define HDA_MAX_CONNECTIONS HDA_MAX_WIDGETS +#define HDA_MAX_ROUTE_NODES 32u +#define HDA_MAX_ROUTES 32u +#define HDA_MAX_ENDPOINT_ROUTES 16u +#define HDA_INVALID_INDEX 0xFFu +#define HDA_JACK_POLL_INTERVAL_MS 500u +#define HDA_POWER_TIMEOUT_US 10000u + +#define HDA_PARAMETER_NODE_COUNT 0x04u +#define HDA_PARAMETER_FUNCTION_TYPE 0x05u +#define HDA_PARAMETER_AUDIO_WIDGET_CAPS 0x09u +#define HDA_PARAMETER_PCM 0x0Au +#define HDA_PARAMETER_STREAM_FORMATS 0x0Bu +#define HDA_PARAMETER_PIN_CAPS 0x0Cu +#define HDA_PARAMETER_INPUT_AMP_CAPS 0x0Du +#define HDA_PARAMETER_CONNECTION_LIST_LENGTH 0x0Eu +#define HDA_PARAMETER_POWER_STATES 0x0Fu +#define HDA_PARAMETER_OUTPUT_AMP_CAPS 0x12u + +#define HDA_FUNCTION_TYPE_AUDIO 0x01u + +#define HDA_WIDGET_TYPE_OUTPUT 0x00u +#define HDA_WIDGET_TYPE_INPUT 0x01u +#define HDA_WIDGET_TYPE_MIXER 0x02u +#define HDA_WIDGET_TYPE_SELECTOR 0x03u +#define HDA_WIDGET_TYPE_PIN 0x04u +#define HDA_WIDGET_TYPE_POWER 0x05u +#define HDA_WIDGET_TYPE_VOLUME_KNOB 0x06u +#define HDA_WIDGET_TYPE_BEEP 0x07u +#define HDA_WIDGET_TYPE_VENDOR 0x0Fu + +#define HDA_WIDGET_CAP_STEREO 0x00000001u +#define HDA_WIDGET_CAP_INPUT_AMP 0x00000002u +#define HDA_WIDGET_CAP_OUTPUT_AMP 0x00000004u +#define HDA_WIDGET_CAP_AMP_OVERRIDE 0x00000008u +#define HDA_WIDGET_CAP_FORMAT_OVERRIDE 0x00000010u +#define HDA_WIDGET_CAP_UNSOLICITED 0x00000080u +#define HDA_WIDGET_CAP_CONNECTION_LIST 0x00000100u +#define HDA_WIDGET_CAP_DIGITAL 0x00000200u +#define HDA_WIDGET_CAP_POWER_CONTROL 0x00000400u +#define HDA_WIDGET_CAP_TYPE_SHIFT 20u +#define HDA_WIDGET_CAP_TYPE_MASK 0x00F00000u + +#define HDA_PCM_RATE_48000 (1u << 6) +#define HDA_PCM_BITS_16 (1u << 17) +#define HDA_STREAM_FORMAT_PCM (1u << 0) + +#define HDA_PIN_CAP_TRIGGER_REQUIRED (1u << 1) +#define HDA_PIN_CAP_PRESENCE_DETECT (1u << 2) +#define HDA_PIN_CAP_HEADPHONE_DRIVE (1u << 3) +#define HDA_PIN_CAP_OUTPUT (1u << 4) +#define HDA_PIN_CAP_HDMI (1u << 7) +#define HDA_PIN_CAP_EAPD (1u << 16) + +#define HDA_DEFAULT_DEVICE_LINE_OUT 0x0u +#define HDA_DEFAULT_DEVICE_SPEAKER 0x1u +#define HDA_DEFAULT_DEVICE_HEADPHONE 0x2u +#define HDA_DEFAULT_DEVICE_SPDIF_OUT 0x4u +#define HDA_DEFAULT_DEVICE_DIGITAL_OTHER_OUT 0x5u +#define HDA_DEFAULT_DEVICE_SHIFT 20u +#define HDA_DEFAULT_DEVICE_MASK 0x00F00000u +#define HDA_DEFAULT_LOCATION_SHIFT 24u +#define HDA_DEFAULT_LOCATION_MASK 0x3F000000u +#define HDA_DEFAULT_ASSOCIATION_SHIFT 4u +#define HDA_DEFAULT_ASSOCIATION_MASK 0x000000F0u +#define HDA_DEFAULT_SEQUENCE_MASK 0x0000000Fu +#define HDA_DEFAULT_PORT_SHIFT 30u +#define HDA_DEFAULT_PORT_MASK 0xC0000000u +#define HDA_DEFAULT_PORT_NONE 0x3u +#define HDA_LOCATION_GROSS_INTERNAL 0x1u + +#define HDA_VERB_GET_CONNECTION_LIST_ENTRY(index) (0xF0200u | ((uint32_t)(index) & 0xFFu)) +#define HDA_VERB_GET_POWER_STATE 0xF0500u +#define HDA_VERB_GET_PIN_SENSE 0xF0900u +#define HDA_VERB_GET_CONFIG_DEFAULT 0xF1C00u +#define HDA_VERB_SET_CONNECTION_SELECT(index) (0x70100u | ((uint32_t)(index) & 0xFFu)) +#define HDA_VERB_SET_POWER_STATE(state) (0x70500u | ((uint32_t)(state) & 0x0Fu)) +#define HDA_VERB_SET_CONVERTER_STREAM(tag, channel) \ + (0x70600u | (((uint32_t)(tag) & 0x0Fu) << 4) | ((uint32_t)(channel) & 0x0Fu)) +#define HDA_VERB_SET_PIN_CONTROL(value) (0x70700u | ((uint32_t)(value) & 0xFFu)) +#define HDA_VERB_SET_UNSOLICITED(value) (0x70800u | ((uint32_t)(value) & 0xFFu)) +#define HDA_VERB_EXECUTE_PIN_SENSE 0x70900u +#define HDA_VERB_SET_EAPD(value) (0x70C00u | ((uint32_t)(value) & 0xFFu)) +#define HDA_VERB_SET_CONVERTER_FORMAT(format) (0x20000u | ((uint32_t)(format) & 0xFFFFu)) +#define HDA_VERB_SET_AMP(payload) (0x30000u | ((uint32_t)(payload) & 0xFFFFu)) + +#define HDA_PIN_CONTROL_OUTPUT_ENABLE 0x40u +#define HDA_PIN_CONTROL_HEADPHONE_ENABLE 0x80u +#define HDA_EAPD_ENABLE 0x02u +#define HDA_UNSOLICITED_ENABLE 0x80u +#define HDA_UNSOLICITED_TAG_MASK 0x3Fu +#define HDA_UNSOLICITED_TAG_SHIFT 26u + +#define HDA_AMP_CAP_OFFSET_MASK 0x0000007Fu +#define HDA_AMP_CAP_STEPS_SHIFT 8u +#define HDA_AMP_CAP_STEPS_MASK 0x00007F00u +#define HDA_AMP_CAP_STEP_SIZE_SHIFT 16u +#define HDA_AMP_CAP_STEP_SIZE_MASK 0x007F0000u +#define HDA_AMP_CAP_MUTE 0x80000000u +#define HDA_AMP_SET_OUTPUT (1u << 15) +#define HDA_AMP_SET_INPUT (1u << 14) +#define HDA_AMP_SET_LEFT (1u << 13) +#define HDA_AMP_SET_RIGHT (1u << 12) +#define HDA_AMP_SET_INDEX_SHIFT 8u +#define HDA_AMP_SET_MUTE (1u << 7) +#define HDA_AMP_SET_GAIN_MASK 0x7Fu + +#define HDA_PIN_SENSE_PRESENT 0x80000000u + +#define HDA_POWER_STATE_D0 0u +#define HDA_POWER_ACTUAL_SHIFT 4u +#define HDA_POWER_ACTUAL_MASK 0xF0u + +typedef enum HDA_ROUTE_CLASS { + HdaRouteLineOut = 1, + HdaRouteSpeaker, + HdaRouteHeadphone, + HdaRouteOtherAnalog +} HDA_ROUTE_CLASS; + +typedef struct HDA_WIDGET { + bool Present; + uint8_t NodeId; + uint8_t Type; + uint8_t FunctionIndex; + uint32_t Capabilities; + uint32_t PcmCapabilities; + uint32_t StreamFormats; + uint32_t PinCapabilities; + uint32_t ConfigurationDefault; + uint32_t InputAmpCapabilities; + uint32_t OutputAmpCapabilities; + uint16_t ConnectionCount; + uint8_t Connections[HDA_MAX_CONNECTIONS]; + bool PresenceKnown; + bool PresentAtJack; + bool PollPresence; + bool UnsolicitedEnabled; + uint8_t UnsolicitedTag; +} HDA_WIDGET; + +typedef struct HDA_AUDIO_FUNCTION { + uint8_t NodeId; + uint32_t PcmCapabilities; + uint32_t StreamFormats; + uint32_t InputAmpCapabilities; + uint32_t OutputAmpCapabilities; +} HDA_AUDIO_FUNCTION; + +typedef struct HDA_CODEC_TOPOLOGY { + uint8_t Address; + uint32_t VendorId; + uint8_t FunctionCount; + HDA_AUDIO_FUNCTION Functions[HDA_MAX_AUDIO_FUNCTIONS]; + HDA_WIDGET Widgets[HDA_MAX_WIDGETS]; +} HDA_CODEC_TOPOLOGY; + +typedef struct HDA_ROUTE { + HDA_CODEC_TOPOLOGY* Codec; + HDA_ROUTE_CLASS Class; + uint8_t PinNode; + uint8_t ConverterNode; + uint8_t NodeCount; + uint8_t Nodes[HDA_MAX_ROUTE_NODES]; + uint8_t ConnectionIndices[HDA_MAX_ROUTE_NODES]; + uint32_t SortKey; + uint8_t MasterAmpNode; + uint32_t MasterAmpCapabilities; +} HDA_ROUTE; + +typedef struct HDA_LOGICAL_ENDPOINT { + struct HDA_CODEC_STATE* State; + IO_AUDIO_ENDPOINT* KernelEndpoint; + IO_AUDIO_ENDPOINT_KIND Kind; + uint8_t RingIndex; + uint8_t RouteCount; + uint8_t RouteIndices[HDA_MAX_ENDPOINT_ROUTES]; + uint8_t CurrentRouteIndex; + bool Running; + bool Starting; + bool Available; + bool HasHardwareGain; + uint32_t HardwareAmpCapabilities; + int32_t RequestedGainMilliDb; + bool RequestedMute; +} HDA_LOGICAL_ENDPOINT; + +struct HDA_CODEC_STATE { + HDA_CONTROLLER* Controller; + MUTEX* Lock; + SPINLOCK CallbackLock; + HDA_CODEC_TOPOLOGY* Codecs[HDA_MAX_CODECS]; + HDA_ROUTE Routes[HDA_MAX_ROUTES]; + uint8_t RouteCount; + HDA_LOGICAL_ENDPOINT Endpoints[HDA_MAX_LOGICAL_ENDPOINTS]; + uint8_t EndpointCount; + HDA_LOGICAL_ENDPOINT* ActiveEndpoint; + uint16_t UnsolicitedNodes[64]; + SEMAPHORE* WorkSemaphore; + EVENT* WorkFinished; + EVENT* PollFinished; + THREAD* WorkThread; + THREAD* PollThread; + volatile long WorkRunning; + volatile long PollRunning; + volatile long ShuttingDown; + uint64_t PendingUnsolicitedTags; + bool PollRequested; + bool HavePollingPins; + uint64_t JackEventCount; + uint64_t JackPollCount; + uint64_t JackChangeCount; + uint64_t RouteChangeCount; + uint64_t RouteFailureCount; +}; + +// Returns a widget only when the node was enumerated in this audio function group. +static HDA_WIDGET* HdaCodecWidget(HDA_CODEC_TOPOLOGY* codec, uint8_t nodeId) { + if (!codec || !codec->Widgets[nodeId].Present) + return (HDA_WIDGET*)0; + return &codec->Widgets[nodeId]; +} + +// Sends one codec verb through the transport owned by this codec state. +static KSTATUS HdaCodecCommand(HDA_CODEC_STATE* state, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t payload, + uint32_t* responseOut) { + return HdaCodecVerb(state->Controller, + codecAddress, + nodeId, + payload, + responseOut); +} + +// Reads one architectural codec parameter. +static KSTATUS HdaCodecGetParameter(HDA_CODEC_STATE* state, + uint8_t codecAddress, + uint8_t nodeId, + uint8_t parameter, + uint32_t* responseOut) { + return HdaCodecCommand(state, + codecAddress, + nodeId, + HDA_VERB_GET_PARAMETER(parameter), + responseOut); +} + +// Powers one function group or widget to D0 and waits for the actual state. +static KSTATUS HdaCodecPowerD0(HDA_CODEC_STATE* state, + uint8_t codecAddress, + uint8_t nodeId) { + KSTATUS status = HdaCodecCommand(state, + codecAddress, + nodeId, + HDA_VERB_SET_POWER_STATE(HDA_POWER_STATE_D0), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + for (uint32_t elapsed = 0; elapsed < HDA_POWER_TIMEOUT_US; elapsed += 10u) { + uint32_t powerState = 0; + status = HdaCodecCommand(state, + codecAddress, + nodeId, + HDA_VERB_GET_POWER_STATE, + &powerState); + if (!KSUCCESS(status)) + return status; + if ((powerState & HDA_POWER_ACTUAL_MASK) == + (HDA_POWER_STATE_D0 << HDA_POWER_ACTUAL_SHIFT)) + return NTLSTATUS_SUCCESS; + gHdaDdk->Timer->Stall(10); + } + return NTLSTATUS_TIMEOUT; +} + +// Expands one short- or long-form connection list into selectable node IDs. +static KSTATUS HdaCodecReadConnections(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + HDA_WIDGET* widget) { + uint32_t lengthParameter = 0; + KSTATUS status = HdaCodecGetParameter(state, + codec->Address, + widget->NodeId, + HDA_PARAMETER_CONNECTION_LIST_LENGTH, + &lengthParameter); + if (!KSUCCESS(status)) + return status; + uint8_t encodedCount = (uint8_t)(lengthParameter & 0x7Fu); + bool longForm = (lengthParameter & 0x80u) != 0; + uint8_t entriesPerResponse = longForm ? 2u : 4u; + uint16_t nodeMask = longForm ? 0x7FFFu : 0x007Fu; + uint16_t rangeMask = longForm ? 0x8000u : 0x0080u; + uint16_t previousNode = 0; + bool havePrevious = false; + + widget->ConnectionCount = 0; + for (uint8_t offset = 0; offset < encodedCount; offset += entriesPerResponse) { + uint32_t response = 0; + status = HdaCodecCommand(state, + codec->Address, + widget->NodeId, + HDA_VERB_GET_CONNECTION_LIST_ENTRY(offset), + &response); + if (!KSUCCESS(status)) + return status; + for (uint8_t slot = 0; slot < entriesPerResponse && + (uint8_t)(offset + slot) < encodedCount; slot++) { + uint16_t raw = longForm + ? (uint16_t)(response >> (slot * 16u)) + : (uint16_t)((response >> (slot * 8u)) & 0xFFu); + uint16_t node = raw & nodeMask; + bool range = (raw & rangeMask) != 0; + uint16_t first = node; + if (range) { + if (!havePrevious || node <= previousNode) + return NTLSTATUS_DEVICE_ERROR; + first = (uint16_t)(previousNode + 1u); + } + for (uint16_t expanded = first; expanded <= node; expanded++) { + if (expanded > HDA_MAX_NODE_ID || + widget->ConnectionCount == HDA_MAX_CONNECTIONS) + return NTLSTATUS_NOT_SUPPORTED; + widget->Connections[widget->ConnectionCount++] = (uint8_t)expanded; + } + previousNode = node; + havePrevious = true; + } + } + return NTLSTATUS_SUCCESS; +} + +// Enumerates one widget and inherits function defaults when the widget has no override. +static KSTATUS HdaCodecEnumerateWidget(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t functionIndex, + uint8_t nodeId) { + if (functionIndex >= codec->FunctionCount) + return NTLSTATUS_INVALID_PARAMETER; + HDA_AUDIO_FUNCTION* function = &codec->Functions[functionIndex]; + HDA_WIDGET* widget = &codec->Widgets[nodeId]; + uint32_t capabilities = 0; + KSTATUS status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_AUDIO_WIDGET_CAPS, + &capabilities); + if (!KSUCCESS(status)) + return status; + + memset(widget, 0, sizeof(*widget)); + widget->Present = true; + widget->NodeId = nodeId; + widget->FunctionIndex = functionIndex; + widget->Capabilities = capabilities; + widget->Type = (uint8_t)((capabilities & HDA_WIDGET_CAP_TYPE_MASK) >> + HDA_WIDGET_CAP_TYPE_SHIFT); + widget->PcmCapabilities = function->PcmCapabilities; + widget->StreamFormats = function->StreamFormats; + widget->InputAmpCapabilities = function->InputAmpCapabilities; + widget->OutputAmpCapabilities = function->OutputAmpCapabilities; + + if ((capabilities & HDA_WIDGET_CAP_FORMAT_OVERRIDE) != 0 && + (widget->Type == HDA_WIDGET_TYPE_OUTPUT || + widget->Type == HDA_WIDGET_TYPE_INPUT)) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_PCM, + &widget->PcmCapabilities); + if (!KSUCCESS(status)) + return status; + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_STREAM_FORMATS, + &widget->StreamFormats); + if (!KSUCCESS(status)) + return status; + } + + if ((capabilities & HDA_WIDGET_CAP_AMP_OVERRIDE) != 0) { + if ((capabilities & HDA_WIDGET_CAP_INPUT_AMP) != 0) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_INPUT_AMP_CAPS, + &widget->InputAmpCapabilities); + if (!KSUCCESS(status)) + return status; + } + if ((capabilities & HDA_WIDGET_CAP_OUTPUT_AMP) != 0) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_OUTPUT_AMP_CAPS, + &widget->OutputAmpCapabilities); + if (!KSUCCESS(status)) + return status; + } + } + + if ((capabilities & HDA_WIDGET_CAP_CONNECTION_LIST) != 0) { + status = HdaCodecReadConnections(state, codec, widget); + if (!KSUCCESS(status)) + return status; + } + + if (widget->Type == HDA_WIDGET_TYPE_PIN) { + status = HdaCodecGetParameter(state, + codec->Address, + nodeId, + HDA_PARAMETER_PIN_CAPS, + &widget->PinCapabilities); + if (!KSUCCESS(status)) + return status; + status = HdaCodecCommand(state, + codec->Address, + nodeId, + HDA_VERB_GET_CONFIG_DEFAULT, + &widget->ConfigurationDefault); + if (!KSUCCESS(status)) + return status; + } + + return NTLSTATUS_SUCCESS; +} + +// Enumerates one audio function group and all of its child widgets. +static KSTATUS HdaCodecEnumerateAudioFunction(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t functionNode) { + if (codec->FunctionCount == HDA_MAX_AUDIO_FUNCTIONS) + return NTLSTATUS_NOT_SUPPORTED; + uint8_t functionIndex = codec->FunctionCount++; + HDA_AUDIO_FUNCTION* function = &codec->Functions[functionIndex]; + memset(function, 0, sizeof(*function)); + function->NodeId = functionNode; + + KSTATUS status = HdaCodecPowerD0(state, codec->Address, functionNode); + if (!KSUCCESS(status)) + return status; + + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_PCM, + &function->PcmCapabilities); + if (!KSUCCESS(status)) + return status; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_STREAM_FORMATS, + &function->StreamFormats); + if (!KSUCCESS(status)) + return status; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_INPUT_AMP_CAPS, + &function->InputAmpCapabilities); + if (!KSUCCESS(status)) + function->InputAmpCapabilities = 0; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_OUTPUT_AMP_CAPS, + &function->OutputAmpCapabilities); + if (!KSUCCESS(status)) + function->OutputAmpCapabilities = 0; + + uint32_t nodeCount = 0; + status = HdaCodecGetParameter(state, + codec->Address, + functionNode, + HDA_PARAMETER_NODE_COUNT, + &nodeCount); + if (!KSUCCESS(status)) + return status; + uint8_t startNode = (uint8_t)((nodeCount >> 16) & 0xFFu); + uint8_t count = (uint8_t)(nodeCount & 0xFFu); + if (count == 0 || (uint16_t)startNode + count > HDA_MAX_WIDGETS) + return NTLSTATUS_NOT_SUPPORTED; + + for (uint16_t node = startNode; node < (uint16_t)startNode + count; node++) { + if (codec->Widgets[node].Present) + return NTLSTATUS_DEVICE_ERROR; + status = HdaCodecEnumerateWidget(state, + codec, + functionIndex, + (uint8_t)node); + if (!KSUCCESS(status)) + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Enumerates all audio function groups exposed by one responding codec. +static KSTATUS HdaCodecEnumerateOne(HDA_CODEC_STATE* state, uint8_t codecAddress) { + HDA_CODEC_TOPOLOGY* codec = (HDA_CODEC_TOPOLOGY*)MmAllocatePoolTag( + sizeof(HDA_CODEC_TOPOLOGY), + HDA_CODEC_POOL_TAG); + if (!codec) + return NTLSTATUS_NO_MEMORY; + memset(codec, 0, sizeof(*codec)); + codec->Address = codecAddress; + + KSTATUS status = HdaCodecGetParameter(state, + codecAddress, + 0, + HDA_PARAMETER_VENDOR_ID, + &codec->VendorId); + if (!KSUCCESS(status)) { + MmFreePool(codec); + return status; + } + uint32_t functionCount = 0; + status = HdaCodecGetParameter(state, + codecAddress, + 0, + HDA_PARAMETER_NODE_COUNT, + &functionCount); + if (!KSUCCESS(status)) { + MmFreePool(codec); + return status; + } + uint8_t startNode = (uint8_t)((functionCount >> 16) & 0xFFu); + uint8_t count = (uint8_t)(functionCount & 0xFFu); + if (count == 0 || (uint16_t)startNode + count > HDA_MAX_WIDGETS) { + MmFreePool(codec); + return NTLSTATUS_NOT_SUPPORTED; + } + + for (uint16_t node = startNode; node < (uint16_t)startNode + count; node++) { + uint32_t functionType = 0; + status = HdaCodecGetParameter(state, + codecAddress, + (uint8_t)node, + HDA_PARAMETER_FUNCTION_TYPE, + &functionType); + if (!KSUCCESS(status)) + continue; + if ((functionType & 0xFFu) != HDA_FUNCTION_TYPE_AUDIO) + continue; + status = HdaCodecEnumerateAudioFunction(state, codec, (uint8_t)node); + if (!KSUCCESS(status)) { + MmFreePool(codec); + return status; + } + } + if (codec->FunctionCount == 0) { + MmFreePool(codec); + return NTLSTATUS_NOT_FOUND; + } + + state->Codecs[codecAddress] = codec; + return NTLSTATUS_SUCCESS; +} + +// Enumerates every responding codec without assuming any codec node IDs. +static KSTATUS HdaCodecEnumerateAll(HDA_CODEC_STATE* state) { + KSTATUS lastStatus = NTLSTATUS_NOT_FOUND; + bool found = false; + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + if ((state->Controller->CodecMask & (uint16_t)(1u << codecAddress)) == 0) + continue; + KSTATUS status = HdaCodecEnumerateOne(state, codecAddress); + if (KSUCCESS(status)) { + found = true; + continue; + } + lastStatus = status; + gHdaDdk->Debug->Print( + "HDA: codec %u topology discovery failed status=0x%x\n", + codecAddress, + status); + } + return found ? NTLSTATUS_SUCCESS : lastStatus; +} + +// Returns whether a converter supports the fixed 48 kHz, stereo, 16-bit PCM contract. +static bool HdaCodecConverterSupportsOutput(const HDA_WIDGET* widget) { + return widget && widget->Type == HDA_WIDGET_TYPE_OUTPUT && + (widget->Capabilities & HDA_WIDGET_CAP_DIGITAL) == 0 && + (widget->Capabilities & HDA_WIDGET_CAP_STEREO) != 0 && + (widget->StreamFormats & HDA_STREAM_FORMAT_PCM) != 0 && + (widget->PcmCapabilities & HDA_PCM_RATE_48000) != 0 && + (widget->PcmCapabilities & HDA_PCM_BITS_16) != 0; +} + +// Classifies one analog output pin from its configuration default. +static HDA_ROUTE_CLASS HdaCodecClassifyPin(const HDA_WIDGET* pin) { + uint8_t device = (uint8_t)((pin->ConfigurationDefault & HDA_DEFAULT_DEVICE_MASK) >> + HDA_DEFAULT_DEVICE_SHIFT); + if (device == HDA_DEFAULT_DEVICE_LINE_OUT) + return HdaRouteLineOut; + if (device == HDA_DEFAULT_DEVICE_SPEAKER) + return HdaRouteSpeaker; + if (device == HDA_DEFAULT_DEVICE_HEADPHONE) + return HdaRouteHeadphone; + return HdaRouteOtherAnalog; +} + +// Returns whether a pin is a connected analog render destination. +static bool HdaCodecPinIsAnalogOutput(const HDA_WIDGET* pin) { + if (!pin || pin->Type != HDA_WIDGET_TYPE_PIN || + (pin->Capabilities & HDA_WIDGET_CAP_DIGITAL) != 0 || + (pin->PinCapabilities & HDA_PIN_CAP_OUTPUT) == 0 || + (pin->PinCapabilities & HDA_PIN_CAP_HDMI) != 0) + return false; + uint8_t port = (uint8_t)((pin->ConfigurationDefault & HDA_DEFAULT_PORT_MASK) >> + HDA_DEFAULT_PORT_SHIFT); + uint8_t device = (uint8_t)((pin->ConfigurationDefault & HDA_DEFAULT_DEVICE_MASK) >> + HDA_DEFAULT_DEVICE_SHIFT); + if (port == HDA_DEFAULT_PORT_NONE || + device == HDA_DEFAULT_DEVICE_SPDIF_OUT || + device == HDA_DEFAULT_DEVICE_DIGITAL_OTHER_OUT) + return false; + return true; +} + +// Computes a stable route ordering from association, sequence, and node ID. +static uint32_t HdaCodecRouteSortKey(const HDA_WIDGET* pin) { + uint32_t association = (pin->ConfigurationDefault & HDA_DEFAULT_ASSOCIATION_MASK) >> + HDA_DEFAULT_ASSOCIATION_SHIFT; + uint32_t sequence = pin->ConfigurationDefault & HDA_DEFAULT_SEQUENCE_MASK; + if (association == 0) + association = 0x10u; + return (association << 16) | (sequence << 8) | pin->NodeId; +} + +// Selects the first controllable output amplifier on a route, preferring the converter. +static void HdaCodecSelectMasterAmp(HDA_ROUTE* route) { + route->MasterAmpNode = HDA_INVALID_INDEX; + route->MasterAmpCapabilities = 0; + for (uint8_t reverse = route->NodeCount; reverse > 0; reverse--) { + uint8_t nodeId = route->Nodes[reverse - 1u]; + HDA_WIDGET* widget = HdaCodecWidget(route->Codec, nodeId); + if (!widget || (widget->Capabilities & HDA_WIDGET_CAP_OUTPUT_AMP) == 0) + continue; + route->MasterAmpNode = nodeId; + route->MasterAmpCapabilities = widget->OutputAmpCapabilities; + return; + } +} + +// Records one converter-to-pin route discovered by the reverse graph walk. +static KSTATUS HdaCodecRecordRoute(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + const uint8_t* nodes, + const uint8_t* connectionIndices, + uint8_t nodeCount) { + if (state->RouteCount == HDA_MAX_ROUTES || nodeCount < 2) + return NTLSTATUS_NOT_SUPPORTED; + HDA_ROUTE* route = &state->Routes[state->RouteCount]; + memset(route, 0, sizeof(*route)); + route->Codec = codec; + route->PinNode = nodes[0]; + route->ConverterNode = nodes[nodeCount - 1u]; + route->NodeCount = nodeCount; + memcpy(route->Nodes, nodes, nodeCount); + memcpy(route->ConnectionIndices, connectionIndices, nodeCount); + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + route->Class = HdaCodecClassifyPin(pin); + route->SortKey = HdaCodecRouteSortKey(pin); + HdaCodecSelectMasterAmp(route); + state->RouteCount++; + + return NTLSTATUS_SUCCESS; +} + +// Walks upstream connection lists from a pin until a valid output converter is reached. +static KSTATUS HdaCodecWalkRoute(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t functionIndex, + uint8_t nodeId, + uint8_t* nodes, + uint8_t* connectionIndices, + uint8_t depth, + bool* visited) { + if (depth == HDA_MAX_ROUTE_NODES) + return NTLSTATUS_NOT_SUPPORTED; + HDA_WIDGET* widget = HdaCodecWidget(codec, nodeId); + if (!widget || widget->FunctionIndex != functionIndex || visited[nodeId] || + (widget->Capabilities & HDA_WIDGET_CAP_DIGITAL) != 0) + return NTLSTATUS_NOT_FOUND; + + visited[nodeId] = true; + nodes[depth] = nodeId; + connectionIndices[depth] = HDA_INVALID_INDEX; + if (HdaCodecConverterSupportsOutput(widget)) { + KSTATUS status = HdaCodecRecordRoute(state, + codec, + nodes, + connectionIndices, + (uint8_t)(depth + 1u)); + visited[nodeId] = false; + return status; + } + if (widget->Type == HDA_WIDGET_TYPE_INPUT || widget->ConnectionCount == 0) { + visited[nodeId] = false; + return NTLSTATUS_NOT_FOUND; + } + + KSTATUS result = NTLSTATUS_NOT_FOUND; + for (uint16_t index = 0; index < widget->ConnectionCount; index++) { + connectionIndices[depth] = (uint8_t)index; + KSTATUS status = HdaCodecWalkRoute(state, + codec, + functionIndex, + widget->Connections[index], + nodes, + connectionIndices, + (uint8_t)(depth + 1u), + visited); + if (KSUCCESS(status)) { + visited[nodeId] = false; + return NTLSTATUS_SUCCESS; + } + if (status != NTLSTATUS_NOT_FOUND && result == NTLSTATUS_NOT_FOUND) + result = status; + } + visited[nodeId] = false; + return result; +} + +// Finds every usable analog path from an output converter to an output pin. +static KSTATUS HdaCodecBuildRoutes(HDA_CODEC_STATE* state) { + uint8_t nodes[HDA_MAX_ROUTE_NODES]; + uint8_t connectionIndices[HDA_MAX_ROUTE_NODES]; + bool visited[HDA_MAX_WIDGETS]; + memset(visited, 0, sizeof(visited)); + + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!HdaCodecPinIsAnalogOutput(pin)) + continue; + KSTATUS status = HdaCodecWalkRoute(state, + codec, + pin->FunctionIndex, + (uint8_t)node, + nodes, + connectionIndices, + 0, + visited); + if (status != NTLSTATUS_NOT_FOUND && !KSUCCESS(status)) + return status; + } + } + return state->RouteCount ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_SUPPORTED; +} + +// Reads the current presence state of one presence-capable analog pin. +static KSTATUS HdaCodecReadPinPresence(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + HDA_WIDGET* pin, + bool* changedOut) { + if (changedOut) + *changedOut = false; + if ((pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) == 0) { + pin->PresenceKnown = true; + pin->PresentAtJack = true; + return NTLSTATUS_SUCCESS; + } + if ((pin->PinCapabilities & HDA_PIN_CAP_TRIGGER_REQUIRED) != 0) { + KSTATUS status = HdaCodecCommand(state, + codec->Address, + pin->NodeId, + HDA_VERB_EXECUTE_PIN_SENSE, + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + uint32_t sense = 0; + KSTATUS status = HdaCodecCommand(state, + codec->Address, + pin->NodeId, + HDA_VERB_GET_PIN_SENSE, + &sense); + if (!KSUCCESS(status)) + return status; + bool present = (sense & HDA_PIN_SENSE_PRESENT) != 0; + bool changed = pin->PresenceKnown && pin->PresentAtJack != present; + pin->PresenceKnown = true; + pin->PresentAtJack = present; + if (changedOut) + *changedOut = changed; + return NTLSTATUS_SUCCESS; +} + +// Returns whether the physical pin behind one route is currently usable. +static bool HdaCodecRouteAvailable(const HDA_ROUTE* route) { + HDA_WIDGET* pin = HdaCodecWidget(route->Codec, route->PinNode); + if (!pin) + return false; + if ((pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) == 0) + return true; + return pin->PresenceKnown && pin->PresentAtJack; +} + +// Adds one route to a logical endpoint without duplicating route indices. +static KSTATUS HdaCodecEndpointAddRoute(HDA_LOGICAL_ENDPOINT* endpoint, + uint8_t routeIndex) { + if (endpoint->RouteCount == HDA_MAX_ENDPOINT_ROUTES) + return NTLSTATUS_NOT_SUPPORTED; + endpoint->RouteIndices[endpoint->RouteCount++] = routeIndex; + return NTLSTATUS_SUCCESS; +} + +// Returns whether an amplifier range fits the kernel endpoint gain contract. +static bool HdaCodecAmpRangeIsRegisterable(uint32_t capabilities) { + uint32_t offset = capabilities & HDA_AMP_CAP_OFFSET_MASK; + uint32_t steps = (capabilities & HDA_AMP_CAP_STEPS_MASK) >> + HDA_AMP_CAP_STEPS_SHIFT; + uint32_t step = (((capabilities & HDA_AMP_CAP_STEP_SIZE_MASK) >> + HDA_AMP_CAP_STEP_SIZE_SHIFT) + 1u) * 250u; + int64_t minimum = -(int64_t)offset * step; + int64_t maximum = ((int64_t)steps - offset) * step; + return minimum >= -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB && + maximum <= IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB; +} + +// Returns whether every route in one endpoint exposes the same mute-capable output amplifier. +static bool HdaCodecEndpointHasCommonGain(HDA_CODEC_STATE* state, + HDA_LOGICAL_ENDPOINT* endpoint, + uint32_t* capabilitiesOut) { + uint32_t capabilities = 0; + for (uint8_t index = 0; index < endpoint->RouteCount; index++) { + HDA_ROUTE* route = &state->Routes[endpoint->RouteIndices[index]]; + if (route->MasterAmpNode == HDA_INVALID_INDEX || + (route->MasterAmpCapabilities & HDA_AMP_CAP_MUTE) == 0) + return false; + if (index == 0) + capabilities = route->MasterAmpCapabilities; + else if (route->MasterAmpCapabilities != capabilities) + return false; + } + if (endpoint->RouteCount == 0 || !HdaCodecAmpRangeIsRegisterable(capabilities)) + return false; + if (capabilitiesOut) + *capabilitiesOut = capabilities; + return true; +} + +// Builds the line-out, speaker/headphone, and other-analog logical endpoint groups. +static KSTATUS HdaCodecBuildEndpoints(HDA_CODEC_STATE* state) { + HDA_LOGICAL_ENDPOINT groups[HDA_MAX_LOGICAL_ENDPOINTS]; + memset(groups, 0, sizeof(groups)); + groups[0].Kind = IoAudioEndpointLineOut; + groups[1].Kind = IoAudioEndpointSpeakerHeadphone; + groups[2].Kind = IoAudioEndpointOtherAnalog; + + for (uint8_t routeIndex = 0; routeIndex < state->RouteCount; routeIndex++) { + HDA_ROUTE* route = &state->Routes[routeIndex]; + uint8_t groupIndex = 2; + if (route->Class == HdaRouteLineOut) + groupIndex = 0; + else if (route->Class == HdaRouteSpeaker || route->Class == HdaRouteHeadphone) + groupIndex = 1; + KSTATUS status = HdaCodecEndpointAddRoute(&groups[groupIndex], routeIndex); + if (!KSUCCESS(status)) + return status; + } + + for (uint8_t groupIndex = 0; groupIndex < HDA_MAX_LOGICAL_ENDPOINTS; groupIndex++) { + if (groups[groupIndex].RouteCount == 0) + continue; + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[state->EndpointCount]; + *endpoint = groups[groupIndex]; + endpoint->State = state; + endpoint->RingIndex = state->EndpointCount; + endpoint->CurrentRouteIndex = HDA_INVALID_INDEX; + endpoint->RequestedGainMilliDb = 0; + endpoint->HardwareAmpCapabilities = 0; + endpoint->HasHardwareGain = HdaCodecEndpointHasCommonGain( + state, + endpoint, + &endpoint->HardwareAmpCapabilities); + state->EndpointCount++; + } + return state->EndpointCount ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_SUPPORTED; +} + +// Selects the best currently available route for one logical endpoint. +static uint8_t HdaCodecChooseRoute(HDA_LOGICAL_ENDPOINT* endpoint) { + HDA_CODEC_STATE* state = endpoint->State; + uint8_t selected = HDA_INVALID_INDEX; + uint32_t selectedKey = UINT32_MAX; + bool selectedHeadphone = false; + for (uint8_t index = 0; index < endpoint->RouteCount; index++) { + uint8_t routeIndex = endpoint->RouteIndices[index]; + HDA_ROUTE* route = &state->Routes[routeIndex]; + if (!HdaCodecRouteAvailable(route)) + continue; + bool headphone = route->Class == HdaRouteHeadphone; + if (endpoint->Kind == IoAudioEndpointSpeakerHeadphone) { + if (headphone && !selectedHeadphone) { + selected = routeIndex; + selectedKey = route->SortKey; + selectedHeadphone = true; + continue; + } + if (!headphone && selectedHeadphone) + continue; + } + if (selected == HDA_INVALID_INDEX || route->SortKey < selectedKey) { + selected = routeIndex; + selectedKey = route->SortKey; + selectedHeadphone = headphone; + } + } + return selected; +} + +// Converts amplifier capabilities into a millidecibel range and step. +static void HdaCodecAmpRange(uint32_t capabilities, + int32_t* minimumOut, + int32_t* maximumOut, + uint32_t* stepOut) { + uint32_t offset = capabilities & HDA_AMP_CAP_OFFSET_MASK; + uint32_t steps = (capabilities & HDA_AMP_CAP_STEPS_MASK) >> + HDA_AMP_CAP_STEPS_SHIFT; + uint32_t step = (((capabilities & HDA_AMP_CAP_STEP_SIZE_MASK) >> + HDA_AMP_CAP_STEP_SIZE_SHIFT) + 1u) * 250u; + *minimumOut = -(int32_t)(offset * step); + *maximumOut = (int32_t)((int64_t)steps * step - (int64_t)offset * step); + *stepOut = step; +} + +// Maps one millidecibel value to the closest supported amplifier gain index. +static uint8_t HdaCodecAmpGainForMilliDb(uint32_t capabilities, int32_t gainMilliDb) { + uint32_t offset = capabilities & HDA_AMP_CAP_OFFSET_MASK; + uint32_t steps = (capabilities & HDA_AMP_CAP_STEPS_MASK) >> + HDA_AMP_CAP_STEPS_SHIFT; + uint32_t step = (((capabilities & HDA_AMP_CAP_STEP_SIZE_MASK) >> + HDA_AMP_CAP_STEP_SIZE_SHIFT) + 1u) * 250u; + int64_t numerator = (int64_t)gainMilliDb + (int64_t)offset * step; + if (numerator <= 0) + return 0; + uint64_t rounded = ((uint64_t)numerator + step / 2u) / step; + if (rounded > steps) + rounded = steps; + return (uint8_t)rounded; +} + +// Programs both channels of one input or output amplifier. +static KSTATUS HdaCodecSetAmp(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + uint8_t nodeId, + bool output, + uint8_t connectionIndex, + uint8_t gain, + bool muted) { + uint16_t payload = (uint16_t)(HDA_AMP_SET_LEFT | HDA_AMP_SET_RIGHT | + ((uint16_t)connectionIndex << HDA_AMP_SET_INDEX_SHIFT) | + (gain & HDA_AMP_SET_GAIN_MASK)); + payload |= output ? HDA_AMP_SET_OUTPUT : HDA_AMP_SET_INPUT; + if (muted) + payload |= HDA_AMP_SET_MUTE; + return HdaCodecCommand(state, + codec->Address, + nodeId, + HDA_VERB_SET_AMP(payload), + (uint32_t*)0); +} + +// Programs selectors and amplifiers along one route while its pin remains disabled. +static KSTATUS HdaCodecPrepareRouteLocked(HDA_CODEC_STATE* state, + HDA_ROUTE* route, + int32_t gainMilliDb, + bool muted) { + HDA_CODEC_TOPOLOGY* codec = route->Codec; + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + if (!pin || pin->FunctionIndex >= codec->FunctionCount) + return NTLSTATUS_DEVICE_ERROR; + HDA_AUDIO_FUNCTION* function = &codec->Functions[pin->FunctionIndex]; + KSTATUS status = HdaCodecPowerD0(state, codec->Address, function->NodeId); + if (!KSUCCESS(status)) + return status; + + status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_PIN_CONTROL(0), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + if ((pin->PinCapabilities & HDA_PIN_CAP_EAPD) != 0) { + status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_EAPD(0), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + + for (uint8_t pathIndex = 0; pathIndex < route->NodeCount; pathIndex++) { + HDA_WIDGET* widget = HdaCodecWidget(codec, route->Nodes[pathIndex]); + if (!widget || widget->FunctionIndex != pin->FunctionIndex) + return NTLSTATUS_DEVICE_ERROR; + if ((widget->Capabilities & HDA_WIDGET_CAP_POWER_CONTROL) != 0) { + status = HdaCodecPowerD0(state, codec->Address, widget->NodeId); + if (!KSUCCESS(status)) + return status; + } + if (pathIndex + 1u < route->NodeCount && + widget->ConnectionCount > 1u && + widget->Type != HDA_WIDGET_TYPE_MIXER) { + status = HdaCodecCommand( + state, + codec->Address, + widget->NodeId, + HDA_VERB_SET_CONNECTION_SELECT(route->ConnectionIndices[pathIndex]), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + + if ((widget->Capabilities & HDA_WIDGET_CAP_INPUT_AMP) != 0 && + pathIndex + 1u < route->NodeCount) { + uint32_t caps = widget->InputAmpCapabilities; + uint8_t zeroDb = (uint8_t)(caps & HDA_AMP_CAP_OFFSET_MASK); + for (uint16_t input = 0; input < widget->ConnectionCount; input++) { + bool inputMuted = input != route->ConnectionIndices[pathIndex] && + (caps & HDA_AMP_CAP_MUTE) != 0; + status = HdaCodecSetAmp(state, + codec, + widget->NodeId, + false, + (uint8_t)input, + zeroDb, + inputMuted); + if (!KSUCCESS(status)) + return status; + } + } + + if ((widget->Capabilities & HDA_WIDGET_CAP_OUTPUT_AMP) != 0) { + uint32_t caps = widget->OutputAmpCapabilities; + uint8_t ampGain = (uint8_t)(caps & HDA_AMP_CAP_OFFSET_MASK); + bool ampMuted = false; + if (widget->NodeId == route->MasterAmpNode) { + ampGain = HdaCodecAmpGainForMilliDb(caps, gainMilliDb); + ampMuted = muted && (caps & HDA_AMP_CAP_MUTE) != 0; + } + status = HdaCodecSetAmp(state, + codec, + widget->NodeId, + true, + 0, + ampGain, + ampMuted); + if (!KSUCCESS(status)) + return status; + } + } + + status = HdaCodecCommand(state, + codec->Address, + route->ConverterNode, + HDA_VERB_SET_CONVERTER_FORMAT(HDA_OUTPUT_STREAM_FORMAT), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + return HdaCodecCommand( + state, + codec->Address, + route->ConverterNode, + HDA_VERB_SET_CONVERTER_STREAM(state->Controller->Output.StreamTag, 0), + (uint32_t*)0); +} + +// Enables the selected pin and applies the endpoint's requested gain and mute. +static KSTATUS HdaCodecActivateRouteLocked(HDA_CODEC_STATE* state, + HDA_ROUTE* route, + int32_t gainMilliDb, + bool muted) { + HDA_CODEC_TOPOLOGY* codec = route->Codec; + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + if (!pin) + return NTLSTATUS_DEVICE_ERROR; + + if ((pin->PinCapabilities & HDA_PIN_CAP_EAPD) != 0) { + KSTATUS status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_EAPD(HDA_EAPD_ENABLE), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + } + uint8_t pinControl = HDA_PIN_CONTROL_OUTPUT_ENABLE; + if (route->Class == HdaRouteHeadphone || + (pin->PinCapabilities & HDA_PIN_CAP_HEADPHONE_DRIVE) != 0) + pinControl |= HDA_PIN_CONTROL_HEADPHONE_ENABLE; + KSTATUS status = HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_PIN_CONTROL(pinControl), + (uint32_t*)0); + if (!KSUCCESS(status)) + return status; + + if (route->MasterAmpNode != HDA_INVALID_INDEX) { + uint8_t gain = HdaCodecAmpGainForMilliDb(route->MasterAmpCapabilities, + gainMilliDb); + status = HdaCodecSetAmp(state, + codec, + route->MasterAmpNode, + true, + 0, + gain, + muted); + } + return status; +} + +// Mutes and disconnects one route without stopping the controller DMA stream. +static void HdaCodecDeactivateRouteLocked(HDA_CODEC_STATE* state, HDA_ROUTE* route) { + HDA_CODEC_TOPOLOGY* codec = route->Codec; + if (route->MasterAmpNode != HDA_INVALID_INDEX && + (route->MasterAmpCapabilities & HDA_AMP_CAP_MUTE) != 0) { + uint8_t gain = HdaCodecAmpGainForMilliDb(route->MasterAmpCapabilities, 0); + (void)HdaCodecSetAmp(state, + codec, + route->MasterAmpNode, + true, + 0, + gain, + true); + } + (void)HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_PIN_CONTROL(0), + (uint32_t*)0); + HDA_WIDGET* pin = HdaCodecWidget(codec, route->PinNode); + if (pin && (pin->PinCapabilities & HDA_PIN_CAP_EAPD) != 0) { + (void)HdaCodecCommand(state, + codec->Address, + route->PinNode, + HDA_VERB_SET_EAPD(0), + (uint32_t*)0); + } + (void)HdaCodecCommand(state, + codec->Address, + route->ConverterNode, + HDA_VERB_SET_CONVERTER_STREAM(0, 0), + (uint32_t*)0); +} + +// Recomputes endpoint availability and reroutes a running combined endpoint in place. +static void HdaCodecRefreshEndpointsLocked(HDA_CODEC_STATE* state) { + for (uint8_t endpointIndex = 0; endpointIndex < state->EndpointCount; endpointIndex++) { + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[endpointIndex]; + uint8_t selectedRoute = HdaCodecChooseRoute(endpoint); + bool available = selectedRoute != HDA_INVALID_INDEX; + if (endpoint->Available != available) { + endpoint->Available = available; + if (endpoint->KernelEndpoint) + gHdaDdk->Audio->SetEndpointAvailable(endpoint->KernelEndpoint, + available ? 1u : 0u); + } + if (!endpoint->Running || endpoint->Starting || + endpoint != state->ActiveEndpoint || + endpoint->CurrentRouteIndex == selectedRoute) + continue; + + if (endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + endpoint->CurrentRouteIndex = selectedRoute; + KSTATUS status = NTLSTATUS_SUCCESS; + if (selectedRoute != HDA_INVALID_INDEX) { + HDA_ROUTE* route = &state->Routes[selectedRoute]; + status = HdaCodecPrepareRouteLocked(state, + route, + endpoint->RequestedGainMilliDb, + true); + if (KSUCCESS(status)) { + status = HdaCodecActivateRouteLocked(state, + route, + endpoint->RequestedGainMilliDb, + endpoint->RequestedMute); + } + } + if (KSUCCESS(status)) { + state->RouteChangeCount++; + } else { + state->RouteFailureCount++; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = false; + KeReleaseSpinlock(&state->CallbackLock, flags); + gHdaDdk->Debug->Print( + "HDA: jack reroute failed kind=%u route=%u status=0x%x\n", + endpoint->Kind, + selectedRoute, + status); + if (endpoint->KernelEndpoint) { + gHdaDdk->Audio->ReportEndpointFailure(endpoint->KernelEndpoint, + status, + 0); + } + } + } +} + +// Updates one pin after an event or poll and applies any logical routing change. +static void HdaCodecRefreshPinLocked(HDA_CODEC_STATE* state, + HDA_CODEC_TOPOLOGY* codec, + HDA_WIDGET* pin, + bool fromPoll) { + bool changed = false; + KSTATUS status = HdaCodecReadPinPresence(state, codec, pin, &changed); + if (!KSUCCESS(status)) { + return; + } + if (fromPoll) + state->JackPollCount++; + else + state->JackEventCount++; + if (!changed) + return; + state->JackChangeCount++; + HdaCodecRefreshEndpointsLocked(state); +} + +// Processes pending unsolicited tags and one requested polling pass. +static void HdaCodecProcessJackWork(HDA_CODEC_STATE* state) { + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return; + + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + uint64_t tags = state->PendingUnsolicitedTags; + bool poll = state->PollRequested; + state->PendingUnsolicitedTags = 0; + state->PollRequested = false; + KeReleaseSpinlock(&state->CallbackLock, flags); + + while (tags) { + uint8_t tag = 0; + while (tag < 64u && (tags & (1ull << tag)) == 0) + tag++; + if (tag == 64u) + break; + tags &= ~(1ull << tag); + uint16_t encoded = state->UnsolicitedNodes[tag]; + uint8_t codecAddress = (uint8_t)(encoded >> 8); + uint8_t nodeId = (uint8_t)encoded; + HDA_CODEC_TOPOLOGY* codec = codecAddress < HDA_MAX_CODECS + ? state->Codecs[codecAddress] + : (HDA_CODEC_TOPOLOGY*)0; + HDA_WIDGET* pin = HdaCodecWidget(codec, nodeId); + if (pin) + HdaCodecRefreshPinLocked(state, codec, pin, false); + } + + if (poll) { + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (pin && pin->PollPresence) + HdaCodecRefreshPinLocked(state, codec, pin, true); + } + } + } + KeReleaseMutex(state->Lock); +} + +// Runs immediate jack-event work at passive level. +static void HdaCodecWorkThread(uintptr_t argument) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)argument; + while (state->WorkRunning) { + KSTATUS status = KeWaitSemaphore(state->WorkSemaphore, KE_WAIT_INFINITE); + if (!KSUCCESS(status) || !state->WorkRunning) + break; + HdaCodecProcessJackWork(state); + } + KeSignalEvent(state->WorkFinished); +} + +// Requests a fallback presence polling pass every 500 milliseconds. +static void HdaCodecPollThread(uintptr_t argument) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)argument; + while (state->PollRunning) { + (void)gHdaDdk->Scheduler->DelayThread(HDA_JACK_POLL_INTERVAL_MS); + if (!state->PollRunning) + break; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + state->PollRequested = true; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeSignalSemaphore(state->WorkSemaphore); + } + KeSignalEvent(state->PollFinished); +} + +// Queues an unsolicited pin event for passive-level sense and route processing. +static void HdaCodecUnsolicitedResponse(void* context, + uint8_t codecAddress, + uint32_t response, + uint32_t responseExtra) { + (void)responseExtra; + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return; + uint8_t tag = (uint8_t)((response >> HDA_UNSOLICITED_TAG_SHIFT) & + HDA_UNSOLICITED_TAG_MASK); + if (tag == 0 || state->UnsolicitedNodes[tag] == 0) + return; + uint16_t encoded = state->UnsolicitedNodes[tag]; + if ((uint8_t)(encoded >> 8) != codecAddress) + return; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + state->PendingUnsolicitedTags |= 1ull << tag; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeSignalSemaphore(state->WorkSemaphore); +} + +// Forwards one hardware period completion to the currently active logical endpoint. +static void HdaCodecPeriodComplete(void* context, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t finePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return; + IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)0; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + if (state->ActiveEndpoint && state->ActiveEndpoint->Running) + endpoint = state->ActiveEndpoint->KernelEndpoint; + KeReleaseSpinlock(&state->CallbackLock, flags); + if (endpoint) { + gHdaDdk->Audio->NotifyPeriod(endpoint, + completedPeriodIndex, + presentedFrames, + finePositionFrames, + interruptCount, + timestampTicks); + } +} + +// Restores power, jack events, and muted route programming after controller reset. +static KSTATUS HdaCodecRestoreAfterControllerReset(void* context) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint8_t functionIndex = 0; + functionIndex < codec->FunctionCount; + functionIndex++) { + status = HdaCodecPowerD0( + state, + codecAddress, + codec->Functions[functionIndex].NodeId); + if (!KSUCCESS(status)) + break; + } + if (!KSUCCESS(status)) + break; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!pin || !pin->UnsolicitedEnabled) + continue; + status = HdaCodecCommand( + state, + codecAddress, + pin->NodeId, + HDA_VERB_SET_UNSOLICITED(HDA_UNSOLICITED_ENABLE | + pin->UnsolicitedTag), + (uint32_t*)0); + if (!KSUCCESS(status)) + break; + } + if (!KSUCCESS(status)) + break; + } + + if (KSUCCESS(status) && state->ActiveEndpoint && + state->ActiveEndpoint->Running && + state->ActiveEndpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HDA_LOGICAL_ENDPOINT* endpoint = state->ActiveEndpoint; + status = HdaCodecPrepareRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex], + endpoint->RequestedGainMilliDb, + true); + } + KeReleaseMutex(state->Lock); + return status; +} + +// Re-enables the physical route after controller recovery has restarted DMA. +static KSTATUS HdaCodecResumeAfterControllerReset(void* context) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (state->ActiveEndpoint && state->ActiveEndpoint->Running && + state->ActiveEndpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HDA_LOGICAL_ENDPOINT* endpoint = state->ActiveEndpoint; + status = HdaCodecActivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex], + endpoint->RequestedGainMilliDb, + endpoint->RequestedMute); + } + KeReleaseMutex(state->Lock); + return status; +} + +// Fails every logical endpoint after unrecoverable transport failure. +static void HdaCodecTransportFailure(void* context, KSTATUS failureStatus) { + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)context; + if (!state || state->ShuttingDown) + return; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + if (state->ActiveEndpoint) + state->ActiveEndpoint->Running = false; + KeReleaseSpinlock(&state->CallbackLock, flags); + for (uint8_t index = 0; index < state->EndpointCount; index++) { + if (state->Endpoints[index].KernelEndpoint) { + gHdaDdk->Audio->ReportEndpointFailure( + state->Endpoints[index].KernelEndpoint, + failureStatus, + 0); + } + } +} + +// Starts DMA for one logical endpoint after programming its selected analog route. +static KSTATUS HdaCodecEndpointStart(void* providerContext) { + HDA_LOGICAL_ENDPOINT* endpoint = (HDA_LOGICAL_ENDPOINT*)providerContext; + if (!endpoint || !endpoint->State) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CODEC_STATE* state = endpoint->State; + if (state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + + KSTATUS status = HdaSelectOutputRing(state->Controller, endpoint->RingIndex); + if (!KSUCCESS(status)) + return status; + status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (state->ActiveEndpoint && state->ActiveEndpoint != endpoint) { + KeReleaseMutex(state->Lock); + return NTLSTATUS_WOULD_BLOCK; + } + + uint8_t routeIndex = HdaCodecChooseRoute(endpoint); + if (routeIndex == HDA_INVALID_INDEX) { + KeReleaseMutex(state->Lock); + return NTLSTATUS_NOT_FOUND; + } + endpoint->Starting = true; + endpoint->CurrentRouteIndex = routeIndex; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + state->ActiveEndpoint = endpoint; + KeReleaseSpinlock(&state->CallbackLock, flags); + status = HdaCodecPrepareRouteLocked(state, + &state->Routes[routeIndex], + endpoint->RequestedGainMilliDb, + true); + KeReleaseMutex(state->Lock); + if (!KSUCCESS(status)) + goto failed; + + status = HdaStartOutput(state->Controller); + if (!KSUCCESS(status)) + goto failed; + + status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + goto failed_after_start; + endpoint->Starting = false; + routeIndex = HdaCodecChooseRoute(endpoint); + if (routeIndex != endpoint->CurrentRouteIndex) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + endpoint->CurrentRouteIndex = routeIndex; + if (routeIndex != HDA_INVALID_INDEX) { + status = HdaCodecPrepareRouteLocked(state, + &state->Routes[routeIndex], + endpoint->RequestedGainMilliDb, + true); + } + } + if (KSUCCESS(status) && routeIndex == HDA_INVALID_INDEX) + status = NTLSTATUS_NOT_FOUND; + if (KSUCCESS(status)) { + status = HdaCodecActivateRouteLocked(state, + &state->Routes[routeIndex], + endpoint->RequestedGainMilliDb, + endpoint->RequestedMute); + } + flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = KSUCCESS(status); + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + if (KSUCCESS(status)) + return NTLSTATUS_SUCCESS; + +failed_after_start: + HdaStopOutput(state->Controller); +failed: + if (KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(state->Lock, KE_WAIT_INFINITE))) { + if (endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + } + endpoint->Starting = false; + endpoint->CurrentRouteIndex = HDA_INVALID_INDEX; + flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = false; + if (state->ActiveEndpoint == endpoint) + state->ActiveEndpoint = (HDA_LOGICAL_ENDPOINT*)0; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + } + return status; +} + +// Mutes one logical endpoint and stops the shared output stream. +static void HdaCodecEndpointStop(void* providerContext) { + HDA_LOGICAL_ENDPOINT* endpoint = (HDA_LOGICAL_ENDPOINT*)providerContext; + if (!endpoint || !endpoint->State) + return; + HDA_CODEC_STATE* state = endpoint->State; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + if (!state->ShuttingDown && + state->ActiveEndpoint == endpoint && + endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[endpoint->CurrentRouteIndex]); + } + endpoint->Starting = false; + endpoint->CurrentRouteIndex = HDA_INVALID_INDEX; + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + endpoint->Running = false; + if (state->ActiveEndpoint == endpoint) + state->ActiveEndpoint = (HDA_LOGICAL_ENDPOINT*)0; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + } + if (!state->ShuttingDown || !state->Controller->SurpriseRemoved) + HdaStopOutput(state->Controller); +} + +// Stores or applies one endpoint hardware gain and mute request. +static KSTATUS HdaCodecEndpointSetGain(void* providerContext, + int32_t gainMilliDb, + uint8_t muted) { + HDA_LOGICAL_ENDPOINT* endpoint = (HDA_LOGICAL_ENDPOINT*)providerContext; + if (!endpoint || !endpoint->State || !endpoint->HasHardwareGain) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CODEC_STATE* state = endpoint->State; + if (state->ShuttingDown) + return NTLSTATUS_DEVICE_REMOVED; + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + endpoint->RequestedGainMilliDb = gainMilliDb; + endpoint->RequestedMute = muted != 0; + if (endpoint->Running && state->ActiveEndpoint == endpoint && + endpoint->CurrentRouteIndex != HDA_INVALID_INDEX) { + HDA_ROUTE* route = &state->Routes[endpoint->CurrentRouteIndex]; + uint8_t gain = HdaCodecAmpGainForMilliDb(route->MasterAmpCapabilities, + gainMilliDb); + status = HdaCodecSetAmp(state, + route->Codec, + route->MasterAmpNode, + true, + 0, + gain, + muted != 0); + } + KeReleaseMutex(state->Lock); + return status; +} + +// Configures initial jack state and chooses event or polling delivery per pin. +static KSTATUS HdaCodecInitializeJacks(HDA_CODEC_STATE* state) { + uint8_t nextTag = 1; + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!HdaCodecPinIsAnalogOutput(pin)) + continue; + bool changed = false; + KSTATUS status = HdaCodecReadPinPresence(state, codec, pin, &changed); + if (!KSUCCESS(status)) + return status; + if ((pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) == 0) + continue; + + bool eventCapable = (pin->Capabilities & HDA_WIDGET_CAP_UNSOLICITED) != 0 && + nextTag <= HDA_UNSOLICITED_TAG_MASK; + if (eventCapable) { + status = HdaCodecCommand( + state, + codecAddress, + pin->NodeId, + HDA_VERB_SET_UNSOLICITED(HDA_UNSOLICITED_ENABLE | nextTag), + (uint32_t*)0); + if (KSUCCESS(status)) { + pin->UnsolicitedEnabled = true; + pin->UnsolicitedTag = nextTag; + state->UnsolicitedNodes[nextTag] = + (uint16_t)(((uint16_t)codecAddress << 8) | pin->NodeId); + nextTag++; + } + } + // Polling remains active as a watchdog because event-capable pins can lose notifications. + pin->PollPresence = true; + state->HavePollingPins = true; + } + } + HdaCodecRefreshEndpointsLocked(state); + return NTLSTATUS_SUCCESS; +} + +// Starts the passive jack worker and optional 500 millisecond polling ticker. +static KSTATUS HdaCodecStartWorkers(HDA_CODEC_STATE* state) { + KSTATUS status = KeSemaphoreCreate(0, 1, &state->WorkSemaphore); + if (!KSUCCESS(status)) + return status; + status = KeEventCreate(EventManualReset, 0, &state->WorkFinished); + if (!KSUCCESS(status)) + return status; + status = KeEventCreate(EventManualReset, 0, &state->PollFinished); + if (!KSUCCESS(status)) + return status; + + state->WorkRunning = 1; + KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + status = KeCreateKernelThread((uintptr_t)HdaCodecWorkThread, + (uintptr_t)state, + ¶meters, + &state->WorkThread); + if (!KSUCCESS(status)) { + state->WorkRunning = 0; + return status; + } + if (!state->HavePollingPins) + return NTLSTATUS_SUCCESS; + + state->PollRunning = 1; + status = KeCreateKernelThread((uintptr_t)HdaCodecPollThread, + (uintptr_t)state, + ¶meters, + &state->PollThread); + if (!KSUCCESS(status)) { + state->PollRunning = 0; + state->WorkRunning = 0; + KeSignalSemaphore(state->WorkSemaphore); + (void)KeWaitForEvent(state->WorkFinished, 0); + ObDereferenceObject((OBJECT*)state->WorkThread); + state->WorkThread = (THREAD*)0; + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Registers every discovered logical endpoint against its dedicated DMA ring. +static KSTATUS HdaCodecRegisterEndpoints(HDA_CODEC_STATE* state) { + for (uint8_t index = 0; index < state->EndpointCount; index++) { + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[index]; + HDA_OUTPUT_DESCRIPTION output; + KSTATUS status = HdaDescribeOutput(state->Controller, + endpoint->RingIndex, + &output); + if (!KSUCCESS(status)) + return status; + + IO_AUDIO_ENDPOINT_REGISTRATION registration; + memset(®istration, 0, sizeof(registration)); + registration.Device = state->Controller->Fdo; + registration.Kind = endpoint->Kind; + registration.InitiallyAvailable = endpoint->Available ? 1u : 0u; + registration.ProviderContext = endpoint; + registration.Operations.Start = HdaCodecEndpointStart; + registration.Operations.Stop = HdaCodecEndpointStop; + registration.PeriodBuffer = output.PeriodBuffer; + registration.PeriodBufferBytes = output.PeriodBufferBytes; + registration.PeriodCount = output.PeriodCount; + registration.FramesPerPeriod = output.FramesPerPeriod; + registration.PeriodStrideBytes = output.PeriodStrideBytes; + registration.HardwareFormat = output.HardwareFormat; + if (endpoint->HasHardwareGain) { + registration.Capabilities |= IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN; + registration.Operations.SetGain = HdaCodecEndpointSetGain; + HdaCodecAmpRange(endpoint->HardwareAmpCapabilities, + ®istration.HardwareGainMinimumMilliDb, + ®istration.HardwareGainMaximumMilliDb, + ®istration.HardwareGainStepMilliDb); + } + for (uint8_t routeOffset = 0; routeOffset < endpoint->RouteCount; routeOffset++) { + HDA_ROUTE* route = &state->Routes[endpoint->RouteIndices[routeOffset]]; + HDA_WIDGET* pin = HdaCodecWidget(route->Codec, route->PinNode); + if (pin && (pin->PinCapabilities & HDA_PIN_CAP_PRESENCE_DETECT) != 0) { + registration.Capabilities |= IO_AUDIO_ENDPOINT_CAP_JACK_SENSE; + break; + } + } + + status = gHdaDdk->Audio->RegisterEndpoint(®istration, + &endpoint->KernelEndpoint); + if (!KSUCCESS(status)) + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Stops worker threads and releases their thread references. +static void HdaCodecStopWorkers(HDA_CODEC_STATE* state) { + if (!state) + return; + state->PollRunning = 0; + state->WorkRunning = 0; + if (state->WorkSemaphore) + KeSignalSemaphore(state->WorkSemaphore); + if (state->PollThread && state->PollFinished) + (void)KeWaitForEvent(state->PollFinished, 0); + if (state->WorkThread && state->WorkFinished) + (void)KeWaitForEvent(state->WorkFinished, 0); + if (state->PollThread) { + ObDereferenceObject((OBJECT*)state->PollThread); + state->PollThread = (THREAD*)0; + } + if (state->WorkThread) { + ObDereferenceObject((OBJECT*)state->WorkThread); + state->WorkThread = (THREAD*)0; + } +} + +// Discovers codec topology and registers usable analog render endpoints. +KSTATUS HdaCodecInitialize(HDA_CONTROLLER* controller) { + if (!controller || !gHdaDdk || !gHdaDdk->Audio || !gHdaDdk->Scheduler) + return NTLSTATUS_INVALID_PARAMETER; + HDA_CODEC_STATE* state = (HDA_CODEC_STATE*)MmAllocatePoolTag( + sizeof(HDA_CODEC_STATE), + HDA_CODEC_POOL_TAG); + if (!state) + return NTLSTATUS_NO_MEMORY; + memset(state, 0, sizeof(*state)); + state->Controller = controller; + state->CallbackLock = (SPINLOCK)SPINLOCK_INIT; + controller->Codec = state; + for (uint8_t index = 0; index < HDA_MAX_LOGICAL_ENDPOINTS; index++) + state->Endpoints[index].CurrentRouteIndex = HDA_INVALID_INDEX; + + KSTATUS status = KeMutexCreate(&state->Lock); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecEnumerateAll(state); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecBuildRoutes(state); + if (!KSUCCESS(status)) { + gHdaDdk->Debug->Print( + "HDA: no usable 48000 Hz, stereo, 16-bit analog converter-to-pin route\n"); + goto failed; + } + status = HdaCodecBuildEndpoints(state); + if (!KSUCCESS(status)) + goto failed; + + status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + goto failed; + status = HdaCodecInitializeJacks(state); + KeReleaseMutex(state->Lock); + if (!KSUCCESS(status)) + goto failed; + + HDA_TRANSPORT_CALLBACKS callbacks; + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.PeriodComplete = HdaCodecPeriodComplete; + callbacks.UnsolicitedResponse = HdaCodecUnsolicitedResponse; + callbacks.RestoreAfterControllerReset = HdaCodecRestoreAfterControllerReset; + callbacks.ResumeAfterControllerReset = HdaCodecResumeAfterControllerReset; + callbacks.TransportFailure = HdaCodecTransportFailure; + status = HdaSetTransportCallbacks(controller, &callbacks, state); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecStartWorkers(state); + if (!KSUCCESS(status)) + goto failed; + status = HdaCodecRegisterEndpoints(state); + if (!KSUCCESS(status)) + goto failed; + + return NTLSTATUS_SUCCESS; + +failed: + HdaCodecStop(controller, IoAudioOrderlyRemoval); + HdaCodecDestroy(controller); + return status; +} + +// Stops jack workers and unregisters codec endpoints. +void HdaCodecStop(HDA_CONTROLLER* controller, IO_AUDIO_REMOVAL_MODE mode) { + if (!controller || !controller->Codec) + return; + HDA_CODEC_STATE* state = controller->Codec; + state->ShuttingDown = 1; + HdaCodecStopWorkers(state); + + if (mode == IoAudioOrderlyRemoval && state->Lock) { + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + HDA_LOGICAL_ENDPOINT* active = state->ActiveEndpoint; + if (active && active->CurrentRouteIndex != HDA_INVALID_INDEX) { + HdaCodecDeactivateRouteLocked( + state, + &state->Routes[active->CurrentRouteIndex]); + active->CurrentRouteIndex = HDA_INVALID_INDEX; + } + uintptr_t flags = KeAcquireSpinlock(&state->CallbackLock); + if (active) + active->Running = false; + state->ActiveEndpoint = (HDA_LOGICAL_ENDPOINT*)0; + KeReleaseSpinlock(&state->CallbackLock, flags); + KeReleaseMutex(state->Lock); + } + } + + for (uint8_t index = state->EndpointCount; index > 0; index--) { + HDA_LOGICAL_ENDPOINT* endpoint = &state->Endpoints[index - 1u]; + if (!endpoint->KernelEndpoint) + continue; + gHdaDdk->Audio->UnregisterEndpoint(endpoint->KernelEndpoint, mode); + endpoint->KernelEndpoint = (IO_AUDIO_ENDPOINT*)0; + } + + if (mode == IoAudioOrderlyRemoval && state->Lock) { + KSTATUS status = KeAcquireMutex(state->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + HDA_CODEC_TOPOLOGY* codec = state->Codecs[codecAddress]; + if (!codec) + continue; + for (uint16_t node = 0; node < HDA_MAX_WIDGETS; node++) { + HDA_WIDGET* pin = HdaCodecWidget(codec, (uint8_t)node); + if (!pin || !pin->UnsolicitedEnabled) + continue; + (void)HdaCodecCommand(state, + codecAddress, + pin->NodeId, + HDA_VERB_SET_UNSOLICITED(0), + (uint32_t*)0); + pin->UnsolicitedEnabled = false; + } + } + KeReleaseMutex(state->Lock); + } + (void)HdaSetTransportCallbacks(controller, + (const HDA_TRANSPORT_CALLBACKS*)0, + (void*)0); + } +} + +// Releases codec topology storage after interrupt work is quiescent. +void HdaCodecDestroy(HDA_CONTROLLER* controller) { + if (!controller || !controller->Codec) + return; + HDA_CODEC_STATE* state = controller->Codec; + controller->Codec = (HDA_CODEC_STATE*)0; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + if (controller->CallbackContext == state) { + memset(&controller->Callbacks, 0, sizeof(controller->Callbacks)); + controller->CallbackContext = (void*)0; + } + KeReleaseSpinlock(&controller->StateLock, flags); + for (uint8_t codecAddress = 0; codecAddress < HDA_MAX_CODECS; codecAddress++) { + MmFreePool(state->Codecs[codecAddress]); + state->Codecs[codecAddress] = (HDA_CODEC_TOPOLOGY*)0; + } + if (state->WorkSemaphore) + ObDereferenceObject(&state->WorkSemaphore->Header); + if (state->WorkFinished) + ObDereferenceObject(&state->WorkFinished->Header); + if (state->PollFinished) + ObDereferenceObject(&state->PollFinished->Header); + if (state->Lock) + ObDereferenceObject(&state->Lock->Header); + MmFreePool(state); +} diff --git a/code/drivers/audio/hda/hda_transport.c b/code/drivers/audio/hda/hda_transport.c new file mode 100644 index 0000000..d1af379 --- /dev/null +++ b/code/drivers/audio/hda/hda_transport.c @@ -0,0 +1,1421 @@ +// Intel High Definition Audio controller transport, command rings, output DMA, and recovery. + +#include "hda.h" + +#include +#include + +// Orders CPU stores before a controller ownership transition. +static __forceinline void HdaMemoryBarrier(void) { + _ReadWriteBarrier(); + _mm_mfence(); + _ReadWriteBarrier(); +} + +// Returns the controller's monotonic microsecond counter. +static uint64_t HdaQueryTicks(void) { + if (!gHdaDdk || !gHdaDdk->Timer || !gHdaDdk->Timer->QueryTicks) + return 0; + return gHdaDdk->Timer->QueryTicks(); +} + +// Delays for a bounded controller settling interval. +static void HdaStall(uint32_t microseconds) { + if (gHdaDdk && gHdaDdk->Timer && gHdaDdk->Timer->Stall) + gHdaDdk->Timer->Stall(microseconds); +} + +// Returns one byte from the controller register aperture. +uint8_t HdaRead8(const HDA_CONTROLLER* controller, uint32_t offset) { + return *(volatile const uint8_t*)(controller->Registers + offset); +} + +// Returns one word from the controller register aperture. +uint16_t HdaRead16(const HDA_CONTROLLER* controller, uint32_t offset) { + return *(volatile const uint16_t*)(controller->Registers + offset); +} + +// Returns one dword from the controller register aperture. +uint32_t HdaRead32(const HDA_CONTROLLER* controller, uint32_t offset) { + return *(volatile const uint32_t*)(controller->Registers + offset); +} + +// Writes one byte to the controller register aperture. +void HdaWrite8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t value) { + *(volatile uint8_t*)(controller->Registers + offset) = value; +} + +// Writes one word to the controller register aperture. +void HdaWrite16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t value) { + *(volatile uint16_t*)(controller->Registers + offset) = value; +} + +// Writes one dword to the controller register aperture. +void HdaWrite32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t value) { + *(volatile uint32_t*)(controller->Registers + offset) = value; +} + +// Returns the register offset of the controller's one output stream descriptor. +uint32_t HdaOutputStreamBase(const HDA_CONTROLLER* controller) { + return HDA_REG_STREAM_BASE + + (uint32_t)controller->Output.DescriptorIndex * HDA_REG_STREAM_STRIDE; +} + +// Reads the 24-bit stream control register without crossing into status. +static uint32_t HdaReadStreamControl(const HDA_CONTROLLER* controller) { + uint32_t base = HdaOutputStreamBase(controller); + return (uint32_t)HdaRead16(controller, base + HDA_STREAM_REG_CTL) | + ((uint32_t)HdaRead8(controller, base + HDA_STREAM_REG_CTL + 2u) << 16); +} + +// Writes the 24-bit stream control register without modifying status. +static void HdaWriteStreamControl(HDA_CONTROLLER* controller, uint32_t value) { + uint32_t base = HdaOutputStreamBase(controller); + HdaWrite16(controller, base + HDA_STREAM_REG_CTL, (uint16_t)value); + HdaWrite8(controller, base + HDA_STREAM_REG_CTL + 2u, (uint8_t)(value >> 16)); +} + +// Waits until selected bits in one dword register match the requested state. +static KSTATUS HdaWait32(HDA_CONTROLLER* controller, + uint32_t offset, + uint32_t mask, + uint32_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaRead32(controller, offset) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Waits until selected bits in one byte register match the requested state. +static KSTATUS HdaWait8(HDA_CONTROLLER* controller, + uint32_t offset, + uint8_t mask, + uint8_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaRead8(controller, offset) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Waits until selected bits in one word register match the requested state. +static KSTATUS HdaWait16(HDA_CONTROLLER* controller, + uint32_t offset, + uint16_t mask, + uint16_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaRead16(controller, offset) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Waits until selected bits in the output stream control register match. +static KSTATUS HdaWaitStreamControl(HDA_CONTROLLER* controller, + uint32_t mask, + uint32_t expected, + uint32_t timeoutMicroseconds) { + for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + if ((HdaReadStreamControl(controller) & mask) == expected) + return NTLSTATUS_SUCCESS; + HdaStall(1); + } + return NTLSTATUS_TIMEOUT; +} + +// Selects the largest command-ring size advertised by the controller. +static KSTATUS HdaSelectRingSize(uint8_t capability, uint16_t* countOut) { + if (!countOut) + return NTLSTATUS_INVALID_PARAMETER; + if (capability & HDA_RING_SIZE_CAP_256) { + *countOut = 256u; + return NTLSTATUS_SUCCESS; + } + if (capability & HDA_RING_SIZE_CAP_16) { + *countOut = 16u; + return NTLSTATUS_SUCCESS; + } + if (capability & HDA_RING_SIZE_CAP_2) { + *countOut = 2u; + return NTLSTATUS_SUCCESS; + } + return NTLSTATUS_NOT_SUPPORTED; +} + +// Returns the hardware selector encoding for a supported ring entry count. +static uint8_t HdaRingSizeSelector(uint16_t count) { + if (count == 256u) + return HDA_RING_SIZE_SELECT_256; + if (count == 16u) + return HDA_RING_SIZE_SELECT_16; + return HDA_RING_SIZE_SELECT_2; +} + +// Releases all DMA and synchronization resources owned by one controller. +void HdaReleaseDma(HDA_CONTROLLER* controller) { + if (!controller || !gHdaDdk || !gHdaDdk->Dma) + return; + + if (controller->StreamMutex) { + ObDereferenceObject(&controller->StreamMutex->Header); + controller->StreamMutex = (MUTEX*)0; + } + if (controller->CommandMutex) { + ObDereferenceObject(&controller->CommandMutex->Header); + controller->CommandMutex = (MUTEX*)0; + } + if (controller->RecoverySemaphore) { + ObDereferenceObject(&controller->RecoverySemaphore->Header); + controller->RecoverySemaphore = (SEMAPHORE*)0; + } + if (controller->RecoveryFinished) { + ObDereferenceObject(&controller->RecoveryFinished->Header); + controller->RecoveryFinished = (EVENT*)0; + } + + gHdaDdk->Dma->FreeCommonBuffer(controller->Output.BdlDma); + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) + gHdaDdk->Dma->FreeCommonBuffer(controller->Output.Rings[ringIndex].Dma); + gHdaDdk->Dma->FreeCommonBuffer(controller->PositionDma); + gHdaDdk->Dma->FreeCommonBuffer(controller->RirbDma); + gHdaDdk->Dma->FreeCommonBuffer(controller->CorbDma); + controller->Output.BdlDma = (IO_DMA_BUFFER*)0; + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { + controller->Output.Rings[ringIndex].Dma = (IO_DMA_BUFFER*)0; + controller->Output.Rings[ringIndex].Buffer = (uint8_t*)0; + controller->Output.Rings[ringIndex].DeviceAddress = 0; + } + controller->PositionDma = (IO_DMA_BUFFER*)0; + controller->RirbDma = (IO_DMA_BUFFER*)0; + controller->CorbDma = (IO_DMA_BUFFER*)0; + controller->Output.Bdl = (HDA_BDL_ENTRY*)0; + controller->PositionBuffer = (volatile uint64_t*)0; + controller->Rirb = (HDA_RIRB_ENTRY*)0; + controller->Corb = (uint32_t*)0; + + if (controller->DmaAdapter) { + gHdaDdk->Dma->PutAdapter(controller->DmaAdapter); + controller->DmaAdapter = (IO_DMA_ADAPTER*)0; + } +} + +// Allocates the transport and output DMA resources for one controller. +KSTATUS HdaAllocateDma(HDA_CONTROLLER* controller) { + if (!controller || !controller->Fdo || !gHdaDdk || !gHdaDdk->Dma) + return NTLSTATUS_INVALID_PARAMETER; + + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = (controller->Gcap & HDA_GCAP_64OK) + ? UINT64_MAX + : 0xFFFFFFFFull; + requirements.Alignment = 128u; + requirements.MaximumSegments = 1u; + + KSTATUS status = gHdaDdk->Dma->GetAdapter(controller->Fdo, + &requirements, + &controller->DmaAdapter); + if (!KSUCCESS(status)) + return status; + + status = HdaSelectRingSize(HdaRead8(controller, HDA_REG_CORBSIZE), + &controller->CorbEntryCount); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = HdaSelectRingSize(HdaRead8(controller, HDA_REG_RIRBSIZE), + &controller->RirbEntryCount); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + (size_t)controller->CorbEntryCount * sizeof(uint32_t), + 128u, + &controller->CorbDma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + (size_t)controller->RirbEntryCount * sizeof(HDA_RIRB_ENTRY), + 128u, + &controller->RirbDma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + // The position buffer is optional because LPIB is the supported fallback. + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + (size_t)controller->TotalStreamCount * sizeof(uint64_t), + 128u, + &controller->PositionDma); + if (!KSUCCESS(status)) + controller->PositionDma = (IO_DMA_BUFFER*)0; + + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { + status = gHdaDdk->Dma->AllocateCommonBuffer( + controller->DmaAdapter, + HDA_OUTPUT_BUFFER_BYTES, + 128u, + &controller->Output.Rings[ringIndex].Dma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + } + status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, + HDA_OUTPUT_PERIOD_COUNT * sizeof(HDA_BDL_ENTRY), + 128u, + &controller->Output.BdlDma); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + controller->Corb = (uint32_t*)gHdaDdk->Dma->GetCommonBufferAddress(controller->CorbDma); + controller->Rirb = (HDA_RIRB_ENTRY*)gHdaDdk->Dma->GetCommonBufferAddress(controller->RirbDma); + controller->CorbDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->CorbDma); + controller->RirbDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->RirbDma); + if (controller->PositionDma) { + controller->PositionBuffer = (volatile uint64_t*) + gHdaDdk->Dma->GetCommonBufferAddress(controller->PositionDma); + controller->PositionDeviceAddress = + gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->PositionDma); + } + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { + controller->Output.Rings[ringIndex].Buffer = (uint8_t*) + gHdaDdk->Dma->GetCommonBufferAddress(controller->Output.Rings[ringIndex].Dma); + controller->Output.Rings[ringIndex].DeviceAddress = + gHdaDdk->Dma->GetCommonBufferDeviceAddress( + controller->Output.Rings[ringIndex].Dma); + } + controller->Output.Bdl = (HDA_BDL_ENTRY*) + gHdaDdk->Dma->GetCommonBufferAddress(controller->Output.BdlDma); + controller->Output.BdlDeviceAddress = + gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->Output.BdlDma); + + memset(controller->Corb, 0, (size_t)controller->CorbEntryCount * sizeof(uint32_t)); + memset(controller->Rirb, 0, (size_t)controller->RirbEntryCount * sizeof(HDA_RIRB_ENTRY)); + if (controller->PositionBuffer) + memset((void*)controller->PositionBuffer, 0, + (size_t)controller->TotalStreamCount * sizeof(uint64_t)); + for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) + memset(controller->Output.Rings[ringIndex].Buffer, 0, HDA_OUTPUT_BUFFER_BYTES); + memset(controller->Output.Bdl, 0, + HDA_OUTPUT_PERIOD_COUNT * sizeof(HDA_BDL_ENTRY)); + + status = KeMutexCreate(&controller->CommandMutex); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = KeMutexCreate(&controller->StreamMutex); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = KeSemaphoreCreate(0, 1, &controller->RecoverySemaphore); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + status = KeEventCreate(EventManualReset, 0, &controller->RecoveryFinished); + if (!KSUCCESS(status)) { + HdaReleaseDma(controller); + return status; + } + + return NTLSTATUS_SUCCESS; +} + +// Stops the output stream and waits for the run bit to clear. +static KSTATUS HdaStopOutputHardware(HDA_CONTROLLER* controller) { + uint32_t control = HdaReadStreamControl(controller); + control &= ~(HDA_STREAM_CTL_RUN | HDA_STREAM_CTL_INTERRUPT_MASK); + HdaWriteStreamControl(controller, control); + return HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RUN, + 0, + HDA_STREAM_RESET_TIMEOUT_US); +} + +// Resets the one output stream descriptor to its architected idle state. +static KSTATUS HdaResetOutputDescriptor(HDA_CONTROLLER* controller) { + KSTATUS status = HdaStopOutputHardware(controller); + if (!KSUCCESS(status)) + return status; + + uint32_t control = HdaReadStreamControl(controller) | HDA_STREAM_CTL_RESET; + HdaWriteStreamControl(controller, control); + status = HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RESET, + HDA_STREAM_CTL_RESET, + HDA_STREAM_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + + control &= ~HDA_STREAM_CTL_RESET; + HdaWriteStreamControl(controller, control); + status = HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RESET, + 0, + HDA_STREAM_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + + HdaWrite8(controller, + HdaOutputStreamBase(controller) + HDA_STREAM_REG_STS, + HDA_STREAM_STS_INTERRUPT_MASK); + return NTLSTATUS_SUCCESS; +} + +// Builds a four-entry BDL beginning with the active ring period owned by software. +static void HdaBuildOutputBdl(HDA_CONTROLLER* controller) { + HDA_OUTPUT_RING* ring = + &controller->Output.Rings[controller->Output.ActiveRingIndex]; + for (uint32_t bdlIndex = 0; bdlIndex < HDA_OUTPUT_PERIOD_COUNT; bdlIndex++) { + uint8_t period = (uint8_t)((ring->NextLogicalPeriod + bdlIndex) % + HDA_OUTPUT_PERIOD_COUNT); + controller->Output.BdlPeriodMap[bdlIndex] = period; + controller->Output.Bdl[bdlIndex].Address = + ring->DeviceAddress + + (uint64_t)period * HDA_OUTPUT_PERIOD_BYTES; + controller->Output.Bdl[bdlIndex].Length = HDA_OUTPUT_PERIOD_BYTES; + controller->Output.Bdl[bdlIndex].Flags = HDA_BDL_FLAG_INTERRUPT_ON_COMPLETION; + } + controller->Output.NextCompletionBdl = 0; + HdaMemoryBarrier(); +} + +// Programs the fixed-format output descriptor without starting DMA. +static KSTATUS HdaProgramOutputDescriptor(HDA_CONTROLLER* controller) { + KSTATUS status = HdaResetOutputDescriptor(controller); + if (!KSUCCESS(status)) + return status; + + if (controller->PositionBuffer) + controller->PositionBuffer[controller->Output.DescriptorIndex] = 0; + HdaBuildOutputBdl(controller); + uint32_t base = HdaOutputStreamBase(controller); + HdaWrite32(controller, base + HDA_STREAM_REG_CBL, HDA_OUTPUT_BUFFER_BYTES); + HdaWrite16(controller, base + HDA_STREAM_REG_LVI, + (uint16_t)(HDA_OUTPUT_PERIOD_COUNT - 1u)); + HdaWrite16(controller, base + HDA_STREAM_REG_FORMAT, HDA_OUTPUT_STREAM_FORMAT); + HdaWrite32(controller, base + HDA_STREAM_REG_BDLPL, + (uint32_t)controller->Output.BdlDeviceAddress); + HdaWrite32(controller, base + HDA_STREAM_REG_BDLPU, + (uint32_t)(controller->Output.BdlDeviceAddress >> 32)); + + uint32_t control = ((uint32_t)controller->Output.StreamTag << HDA_STREAM_CTL_TAG_SHIFT) | + HDA_STREAM_CTL_INTERRUPT_COMPLETE | + HDA_STREAM_CTL_INTERRUPT_FIFO_ERROR | + HDA_STREAM_CTL_INTERRUPT_DESCRIPTOR_ERROR; + HdaWriteStreamControl(controller, control); + controller->Output.Prepared = true; + return NTLSTATUS_SUCCESS; +} + +// Resets the controller global state after all DMA engines are stopped. +static KSTATUS HdaResetControllerHardware(HDA_CONTROLLER* controller) { + HdaWrite32(controller, HDA_REG_INTCTL, 0); + (void)HdaStopOutputHardware(controller); + HdaWrite8(controller, HDA_REG_CORBCTL, 0); + HdaWrite8(controller, HDA_REG_RIRBCTL, 0); + + uint32_t gctl = HdaRead32(controller, HDA_REG_GCTL); + HdaWrite32(controller, HDA_REG_GCTL, gctl & ~HDA_GCTL_RESET); + KSTATUS status = HdaWait32(controller, + HDA_REG_GCTL, + HDA_GCTL_RESET, + 0, + HDA_CONTROLLER_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + + HdaStall(100); + HdaWrite32(controller, HDA_REG_GCTL, HDA_GCTL_RESET); + status = HdaWait32(controller, + HDA_REG_GCTL, + HDA_GCTL_RESET, + HDA_GCTL_RESET, + HDA_CONTROLLER_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + HdaStall(HDA_CODEC_RESET_SETTLE_US); + return NTLSTATUS_SUCCESS; +} + +// Resets the CORB read pointer while accepting controllers that self-clear the reset bit. +static KSTATUS HdaResetCorbReadPointer(HDA_CONTROLLER* controller) { + HdaWrite16(controller, HDA_REG_CORBRP, HDA_CORBRP_RESET); + + for (uint32_t elapsed = 0; elapsed < HDA_RING_RESET_TIMEOUT_US; elapsed++) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + + uint16_t value = HdaRead16(controller, HDA_REG_CORBRP); + if ((value & HDA_CORBRP_RESET) != 0) { + HdaWrite16(controller, HDA_REG_CORBRP, 0); + return HdaWait16(controller, + HDA_REG_CORBRP, + HDA_CORBRP_RESET, + 0, + HDA_RING_RESET_TIMEOUT_US); + } + + // Some controllers complete the reset before software can observe the + // reset bit. A zero read pointer proves that the requested state was + // reached, so requiring an intermediate asserted state is incorrect. + if ((value & HDA_CORBRP_POINTER_MASK) == 0) { + HdaWrite16(controller, HDA_REG_CORBRP, 0); + return NTLSTATUS_SUCCESS; + } + + HdaStall(1); + } + + return NTLSTATUS_TIMEOUT; +} + +// Reports the register state that owns a controller initialization failure. +static KSTATUS HdaReportInitializationFailure(HDA_CONTROLLER* controller, + const char* stage, + KSTATUS status) { + gHdaDdk->Debug->Print( + "HDA: init stage=%s failed status=0x%x GCTL=0x%08x STATESTS=0x%04x " + "CORB=%02x/%02x/%04x/%04x RIRB=%02x/%02x/%04x SDCTL=0x%06x\n", + stage, + status, + HdaRead32(controller, HDA_REG_GCTL), + HdaRead16(controller, HDA_REG_STATESTS), + HdaRead8(controller, HDA_REG_CORBCTL), + HdaRead8(controller, HDA_REG_CORBSTS), + HdaRead16(controller, HDA_REG_CORBWP), + HdaRead16(controller, HDA_REG_CORBRP), + HdaRead8(controller, HDA_REG_RIRBCTL), + HdaRead8(controller, HDA_REG_RIRBSTS), + HdaRead16(controller, HDA_REG_RIRBWP), + HdaReadStreamControl(controller)); + return status; +} + +// Initializes CORB and RIRB with software pointers synchronized to hardware. +static KSTATUS HdaInitializeCommandRings(HDA_CONTROLLER* controller) { + HdaWrite8(controller, HDA_REG_CORBCTL, 0); + HdaWrite8(controller, HDA_REG_RIRBCTL, 0); + KSTATUS status = HdaWait8(controller, + HDA_REG_CORBCTL, + HDA_CORBCTL_RUN, + 0, + HDA_RING_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + status = HdaWait8(controller, + HDA_REG_RIRBCTL, + HDA_RIRBCTL_RUN, + 0, + HDA_RING_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + HdaWrite8(controller, HDA_REG_CORBSTS, HDA_CORBSTS_MEMORY_ERROR); + HdaWrite8(controller, HDA_REG_RIRBSTS, + HDA_RIRBSTS_RESPONSE | HDA_RIRBSTS_OVERRUN); + + HdaWrite32(controller, HDA_REG_CORBLBASE, (uint32_t)controller->CorbDeviceAddress); + HdaWrite32(controller, HDA_REG_CORBUBASE, + (uint32_t)(controller->CorbDeviceAddress >> 32)); + HdaWrite8(controller, HDA_REG_CORBSIZE, + HdaRingSizeSelector(controller->CorbEntryCount)); + HdaWrite16(controller, HDA_REG_CORBWP, 0); + status = HdaResetCorbReadPointer(controller); + if (!KSUCCESS(status)) + return status; + + HdaWrite32(controller, HDA_REG_RIRBLBASE, (uint32_t)controller->RirbDeviceAddress); + HdaWrite32(controller, HDA_REG_RIRBUBASE, + (uint32_t)(controller->RirbDeviceAddress >> 32)); + HdaWrite8(controller, HDA_REG_RIRBSIZE, + HdaRingSizeSelector(controller->RirbEntryCount)); + HdaWrite16(controller, HDA_REG_RIRBWP, HDA_RIRBWP_RESET); + HdaWrite16(controller, HDA_REG_RINTCNT, 1); + + controller->CorbWritePointer = 0; + controller->RirbReadPointer = 0; + controller->CommandWaiting = false; + controller->CommandResponseReady = false; + HdaMemoryBarrier(); + HdaWrite8(controller, HDA_REG_CORBCTL, + HDA_CORBCTL_MEMORY_ERROR_INTERRUPT | HDA_CORBCTL_RUN); + HdaWrite8(controller, HDA_REG_RIRBCTL, + HDA_RIRBCTL_RESPONSE_INTERRUPT | + HDA_RIRBCTL_OVERRUN_INTERRUPT | + HDA_RIRBCTL_RUN); + status = HdaWait8(controller, + HDA_REG_CORBCTL, + HDA_CORBCTL_RUN, + HDA_CORBCTL_RUN, + HDA_RING_RESET_TIMEOUT_US); + if (!KSUCCESS(status)) + return status; + return HdaWait8(controller, + HDA_REG_RIRBCTL, + HDA_RIRBCTL_RUN, + HDA_RIRBCTL_RUN, + HDA_RING_RESET_TIMEOUT_US); +} + +// Queues one unsolicited response for DPC-level topology handling. +static void HdaQueueUnsolicitedLocked(HDA_CONTROLLER* controller, + uint32_t response, + uint32_t responseExtra) { + if (controller->UnsolicitedCount == HDA_MAX_UNSOLICITED_RESPONSES) { + controller->UnsolicitedRead = (uint8_t)((controller->UnsolicitedRead + 1u) % + HDA_MAX_UNSOLICITED_RESPONSES); + controller->UnsolicitedCount--; + } + controller->Unsolicited[controller->UnsolicitedWrite].Response = response; + controller->Unsolicited[controller->UnsolicitedWrite].ResponseExtra = responseExtra; + controller->UnsolicitedWrite = (uint8_t)((controller->UnsolicitedWrite + 1u) % + HDA_MAX_UNSOLICITED_RESPONSES); + controller->UnsolicitedCount++; + controller->RirbUnsolicitedCount++; +} + +// Drains every hardware-owned RIRB entry while the controller lock is held. +static void HdaDrainRirbLocked(HDA_CONTROLLER* controller) { + uint16_t hardwareWritePointer = (uint16_t)(HdaRead16(controller, HDA_REG_RIRBWP) & 0x00FFu); + while (controller->RirbReadPointer != hardwareWritePointer) { + controller->RirbReadPointer = + (uint16_t)((controller->RirbReadPointer + 1u) % controller->RirbEntryCount); + HDA_RIRB_ENTRY entry = controller->Rirb[controller->RirbReadPointer]; + controller->RirbResponseCount++; + + if (entry.ResponseExtra & HDA_RIRB_RESPONSE_UNSOLICITED) { + HdaQueueUnsolicitedLocked(controller, entry.Response, entry.ResponseExtra); + continue; + } + + uint8_t codecAddress = (uint8_t)(entry.ResponseExtra & HDA_RIRB_RESPONSE_CODEC_MASK); + if (controller->CommandWaiting && + codecAddress == controller->CommandCodecAddress && + !controller->CommandResponseReady) { + controller->CommandResponse = entry.Response; + controller->CommandResponseExtra = entry.ResponseExtra; + controller->CommandResponseReady = true; + } + } +} + +// Drains RIRB responses and acknowledges response or overrun status. +static uint8_t HdaServiceRirbLocked(HDA_CONTROLLER* controller) { + uint8_t status = HdaRead8(controller, HDA_REG_RIRBSTS); + if (status & HDA_RIRBSTS_RESPONSE) + HdaDrainRirbLocked(controller); + if (status & HDA_RIRBSTS_OVERRUN) { + controller->RirbOverrunCount++; + if (controller->Started) + controller->PendingRecovery = HDA_RECOVERY_CONTROLLER; + } + if (status) + HdaWrite8(controller, HDA_REG_RIRBSTS, status); + return status; +} + +// Sends one codec verb attempt while the command mutex is held. +static KSTATUS HdaCodecVerbAttemptLocked(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut) { + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + (void)HdaServiceRirbLocked(controller); + uint64_t startingOverruns = controller->RirbOverrunCount; + + uint16_t next = (uint16_t)((controller->CorbWritePointer + 1u) % + controller->CorbEntryCount); + uint16_t hardwareReadPointer = (uint16_t)(HdaRead16(controller, HDA_REG_CORBRP) & 0x00FFu); + if (next == hardwareReadPointer) { + KeReleaseSpinlock(&controller->StateLock, flags); + return NTLSTATUS_WOULD_BLOCK; + } + + controller->CommandWaiting = true; + controller->CommandResponseReady = false; + controller->CommandCodecAddress = codecAddress; + controller->Corb[next] = ((uint32_t)codecAddress << 28) | + ((uint32_t)nodeId << 20) | + (verbPayload & 0x000FFFFFu); + HdaMemoryBarrier(); + controller->CorbWritePointer = next; + HdaWrite16(controller, HDA_REG_CORBWP, next); + KeReleaseSpinlock(&controller->StateLock, flags); + + for (uint32_t elapsed = 0; elapsed < HDA_CODEC_COMMAND_TIMEOUT_US; elapsed += 10u) { + flags = KeAcquireSpinlock(&controller->StateLock); + (void)HdaServiceRirbLocked(controller); + if (controller->CommandResponseReady) { + uint32_t response = controller->CommandResponse; + controller->CommandWaiting = false; + controller->CommandResponseReady = false; + KeReleaseSpinlock(&controller->StateLock, flags); + if (responseOut) + *responseOut = response; + return NTLSTATUS_SUCCESS; + } + bool overrun = controller->RirbOverrunCount != startingOverruns; + bool removed = controller->Removed || controller->SurpriseRemoved; + KeReleaseSpinlock(&controller->StateLock, flags); + if (overrun || removed) + break; + HdaStall(10); + } + + flags = KeAcquireSpinlock(&controller->StateLock); + controller->CommandWaiting = false; + controller->CommandResponseReady = false; + KeReleaseSpinlock(&controller->StateLock, flags); + return NTLSTATUS_TIMEOUT; +} + +// Sends one codec verb with the required retry while the command mutex is held. +static KSTATUS HdaCodecVerbSerialized(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut) { + KSTATUS finalStatus = NTLSTATUS_TIMEOUT; + for (uint32_t attempt = 0; attempt < HDA_CODEC_COMMAND_ATTEMPTS; attempt++) { + controller->CodecCommandCount++; + finalStatus = HdaCodecVerbAttemptLocked(controller, + codecAddress, + nodeId, + verbPayload, + responseOut); + if (KSUCCESS(finalStatus)) + break; + if (attempt + 1u < HDA_CODEC_COMMAND_ATTEMPTS) { + controller->CodecCommandRetryCount++; + HdaStall(100); + } + } + if (!KSUCCESS(finalStatus)) + controller->CodecCommandFailureCount++; + return finalStatus; +} + +// Sends one 20-bit codec verb payload through CORB/RIRB with one retry. +KSTATUS HdaCodecVerb(HDA_CONTROLLER* controller, + uint8_t codecAddress, + uint8_t nodeId, + uint32_t verbPayload, + uint32_t* responseOut) { + if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || + codecAddress >= HDA_MAX_CODECS || + (verbPayload & 0xFFF00000u) != 0 || !controller->CommandMutex) + return NTLSTATUS_INVALID_PARAMETER; + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + + KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + status = HdaCodecVerbSerialized(controller, + codecAddress, + nodeId, + verbPayload, + responseOut); + KeReleaseMutex(controller->CommandMutex); + + bool queueDpc = false; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + queueDpc = controller->Started && !controller->RecoveryInProgress && + (controller->UnsolicitedCount != 0 || + controller->PendingRecovery != HDA_RECOVERY_NONE); + KeReleaseSpinlock(&controller->StateLock, flags); + if (queueDpc) + (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); + return status; +} + +// Verifies responding codec addresses through the architectural root node. +static KSTATUS HdaProbeCodecs(HDA_CONTROLLER* controller) { + uint16_t present = controller->CodecMask; + controller->CodecMask = 0; + for (uint8_t codec = 0; codec < HDA_MAX_CODECS; codec++) { + if ((present & (uint16_t)(1u << codec)) == 0) + continue; + uint32_t vendorId = 0; + KSTATUS status = HdaCodecVerbSerialized( + controller, + codec, + 0, + HDA_VERB_GET_PARAMETER(HDA_PARAMETER_VENDOR_ID), + &vendorId); + if (!KSUCCESS(status)) + continue; + controller->CodecMask |= (uint16_t)(1u << codec); + } + return controller->CodecMask ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND; +} + +// Initializes the controller while no other command producer is active. +static KSTATUS HdaInitializeControllerLocked(HDA_CONTROLLER* controller) { + KSTATUS status = HdaResetControllerHardware(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "controller-reset", + status); + + uint16_t state = (uint16_t)(HdaRead16(controller, HDA_REG_STATESTS) & 0x7FFFu); + controller->CodecMask = state; + if (state) + HdaWrite16(controller, HDA_REG_STATESTS, state); + + status = HdaInitializeCommandRings(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "command-rings", + status); + + if (controller->PositionBuffer) { + HdaWrite32(controller, HDA_REG_DPLBASE, + (uint32_t)controller->PositionDeviceAddress | HDA_DPLBASE_ENABLE); + HdaWrite32(controller, HDA_REG_DPUBASE, + (uint32_t)(controller->PositionDeviceAddress >> 32)); + } else { + HdaWrite32(controller, HDA_REG_DPLBASE, 0); + HdaWrite32(controller, HDA_REG_DPUBASE, 0); + } + + status = HdaProgramOutputDescriptor(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "output-descriptor", + status); + + status = HdaProbeCodecs(controller); + if (!KSUCCESS(status)) + return HdaReportInitializationFailure(controller, + "codec-probe", + status); + + HdaWrite32(controller, + HDA_REG_GCTL, + HdaRead32(controller, HDA_REG_GCTL) | + HDA_GCTL_RESET | + HDA_GCTL_UNSOLICITED_ENABLE); + HdaWrite32(controller, HDA_REG_INTCTL, 0); + return NTLSTATUS_SUCCESS; +} + +// Resets and initializes the controller, command rings, position buffer, and output descriptor. +KSTATUS HdaInitializeController(HDA_CONTROLLER* controller) { + if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || + !controller->Registers || !controller->CommandMutex) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + status = HdaInitializeControllerLocked(controller); + KeReleaseMutex(controller->CommandMutex); + return status; +} + +// Arms controller and output-stream interrupts after ISR registration. +void HdaArmInterrupts(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers || !controller->Started || + controller->Removed || controller->SurpriseRemoved) + return; + uint32_t streamInterrupt = 1u << controller->Output.DescriptorIndex; + HdaWrite32(controller, + HDA_REG_INTCTL, + HDA_INT_GLOBAL_ENABLE | + HDA_INT_CONTROLLER_ENABLE | + streamInterrupt); +} + +// Stops hardware access without releasing software resources. +void HdaQuiesceController(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers || controller->SurpriseRemoved) + return; + + bool streamLocked = false; + bool commandLocked = false; + if (controller->StreamMutex) { + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return; + streamLocked = true; + } + if (controller->CommandMutex) { + KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { + if (streamLocked) + KeReleaseMutex(controller->StreamMutex); + return; + } + commandLocked = true; + } + HdaWrite32(controller, HDA_REG_INTCTL, 0); + (void)HdaStopOutputHardware(controller); + HdaWrite8(controller, HDA_REG_CORBCTL, 0); + HdaWrite8(controller, HDA_REG_RIRBCTL, 0); + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, flags); + if (commandLocked) + KeReleaseMutex(controller->CommandMutex); + if (streamLocked) + KeReleaseMutex(controller->StreamMutex); +} + +// Copies one logical endpoint ring contract for codec and endpoint setup. +KSTATUS HdaDescribeOutput(HDA_CONTROLLER* controller, + uint8_t ringIndex, + HDA_OUTPUT_DESCRIPTION* descriptionOut) { + if (!controller || !descriptionOut || ringIndex >= HDA_MAX_LOGICAL_ENDPOINTS || + !controller->Output.Rings[ringIndex].Buffer) + return NTLSTATUS_INVALID_PARAMETER; + descriptionOut->PeriodBuffer = controller->Output.Rings[ringIndex].Buffer; + descriptionOut->PeriodBufferBytes = HDA_OUTPUT_BUFFER_BYTES; + descriptionOut->PeriodCount = HDA_OUTPUT_PERIOD_COUNT; + descriptionOut->FramesPerPeriod = HDA_OUTPUT_PERIOD_FRAMES; + descriptionOut->PeriodStrideBytes = HDA_OUTPUT_PERIOD_BYTES; + descriptionOut->HardwareFormat.SampleRate = HDA_OUTPUT_SAMPLE_RATE; + descriptionOut->HardwareFormat.ChannelCount = HDA_OUTPUT_CHANNELS; + descriptionOut->HardwareFormat.SampleFormat = + IoAudioSampleS16; + descriptionOut->StreamTag = controller->Output.StreamTag; + return NTLSTATUS_SUCCESS; +} + +// Selects one logical endpoint ring while the stream is stopped. +KSTATUS HdaSelectOutputRing(HDA_CONTROLLER* controller, uint8_t ringIndex) { + if (!controller || !controller->StreamMutex || + ringIndex >= HDA_MAX_LOGICAL_ENDPOINTS || + !controller->Output.Rings[ringIndex].Buffer) + return NTLSTATUS_INVALID_PARAMETER; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (controller->Output.Running) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_WOULD_BLOCK; + } + controller->Output.ActiveRingIndex = ringIndex; + status = HdaProgramOutputDescriptor(controller); + KeReleaseMutex(controller->StreamMutex); + return status; +} + +// Replaces the transport callbacks while the output stream is stopped. +KSTATUS HdaSetTransportCallbacks(HDA_CONTROLLER* controller, + const HDA_TRANSPORT_CALLBACKS* callbacks, + void* context) { + if (!controller || !controller->StreamMutex) + return NTLSTATUS_INVALID_PARAMETER; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (controller->Output.Running) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_INVALID_PARAMETER; + } + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + if (callbacks) + controller->Callbacks = *callbacks; + else + memset(&controller->Callbacks, 0, sizeof(controller->Callbacks)); + controller->CallbackContext = context; + KeReleaseSpinlock(&controller->StateLock, flags); + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_SUCCESS; +} + +// Starts one already programmed output descriptor without rebuilding its BDL. +static KSTATUS HdaRunPreparedOutput(HDA_CONTROLLER* controller) { + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.PositionRunBaseFrames = controller->Output.LastPositionFrames; + controller->Output.PositionWrapFrames = 0; + controller->Output.LastHardwarePositionBytes = 0; + controller->Output.PositionInitialized = false; + KeReleaseSpinlock(&controller->StateLock, flags); + HdaMemoryBarrier(); + uint32_t control = HdaReadStreamControl(controller) | HDA_STREAM_CTL_RUN; + HdaWriteStreamControl(controller, control); + KSTATUS status = HdaWaitStreamControl(controller, + HDA_STREAM_CTL_RUN, + HDA_STREAM_CTL_RUN, + HDA_STREAM_RESET_TIMEOUT_US); + if (KSUCCESS(status)) { + flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = true; + KeReleaseSpinlock(&controller->StateLock, flags); + } + return status; +} + +// Starts the prepared fixed-format output stream. +KSTATUS HdaStartOutput(HDA_CONTROLLER* controller) { + if (!controller || !controller->StreamMutex) + return NTLSTATUS_INVALID_PARAMETER; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return status; + if (!controller->Started || controller->Removed || controller->SurpriseRemoved || + !controller->Output.Prepared) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_INVALID_PARAMETER; + } + if (controller->Output.Running) { + KeReleaseMutex(controller->StreamMutex); + return NTLSTATUS_SUCCESS; + } + + status = HdaProgramOutputDescriptor(controller); + if (KSUCCESS(status)) { + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.PendingPeriodInterrupts = 0; + controller->Output.PendingDmaErrors = 0; + KeReleaseSpinlock(&controller->StateLock, flags); + status = HdaRunPreparedOutput(controller); + } + KeReleaseMutex(controller->StreamMutex); + return status; +} + +// Stops the output stream while retaining its DMA ring. +void HdaStopOutput(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers || controller->SurpriseRemoved || + !controller->StreamMutex) + return; + KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) + return; + (void)HdaQueryOutputPositionFrames(controller); + (void)HdaStopOutputHardware(controller); + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, flags); + KeReleaseMutex(controller->StreamMutex); +} + +// Returns the current byte position from the position buffer or LPIB. +static uint32_t HdaQueryOutputPositionBytesLocked(HDA_CONTROLLER* controller) { + uint32_t position; + if (controller->PositionBuffer) { + position = (uint32_t)controller->PositionBuffer[controller->Output.DescriptorIndex]; + controller->Output.PositionBufferReadCount++; + } else { + position = HdaRead32(controller, + HdaOutputStreamBase(controller) + HDA_STREAM_REG_LPIB); + controller->Output.LpibReadCount++; + } + if (position > HDA_OUTPUT_BUFFER_BYTES) { + controller->Output.PositionDiscontinuityCount++; + position %= HDA_OUTPUT_BUFFER_BYTES; + } else if (position == HDA_OUTPUT_BUFFER_BYTES) { + position = 0; + } + return position; +} + +// Advances the monotonic fine position while the state lock is held. +static uint64_t HdaQueryOutputPositionFramesLocked(HDA_CONTROLLER* controller) { + if (!controller->Output.Running) + return controller->Output.LastPositionFrames; + + uint32_t positionBytes = HdaQueryOutputPositionBytesLocked(controller); + if (controller->Output.PositionInitialized && + positionBytes < controller->Output.LastHardwarePositionBytes) { + controller->Output.PositionWrapFrames += + HDA_OUTPUT_PERIOD_COUNT * HDA_OUTPUT_PERIOD_FRAMES; + } + controller->Output.PositionInitialized = true; + controller->Output.LastHardwarePositionBytes = positionBytes; + + uint64_t candidate = controller->Output.PositionRunBaseFrames + + controller->Output.PositionWrapFrames + + positionBytes / HDA_OUTPUT_BYTES_PER_FRAME; + if (candidate < controller->Output.LastPositionFrames) { + controller->Output.PositionDiscontinuityCount++; + candidate = controller->Output.LastPositionFrames; + } + controller->Output.LastPositionFrames = candidate; + return candidate; +} + +// Returns the monotonic fine position derived from the position buffer or LPIB. +uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller) { + if (!controller || !controller->Registers) + return 0; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + uint64_t position = HdaQueryOutputPositionFramesLocked(controller); + KeReleaseSpinlock(&controller->StateLock, flags); + return position; +} + +// Captures one controller interrupt and queues deferred processing. +void HdaHandleInterrupt(HDA_CONTROLLER* controller) { + if (!controller) + return; + _InterlockedIncrement(&controller->IsrActive); + if (!controller->Started || controller->Removed || controller->SurpriseRemoved) + goto finished; + + uint32_t interruptStatus = HdaRead32(controller, HDA_REG_INTSTS); + if (interruptStatus == 0 || interruptStatus == UINT32_MAX) + goto finished; + + bool queueDpc = false; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + bool recovering = controller->RecoveryInProgress; + controller->InterruptCount++; + + uint32_t streamBit = 1u << controller->Output.DescriptorIndex; + if (interruptStatus & streamBit) { + uint32_t base = HdaOutputStreamBase(controller); + uint8_t streamStatus = HdaRead8(controller, base + HDA_STREAM_REG_STS); + streamStatus &= HDA_STREAM_STS_INTERRUPT_MASK; + if (streamStatus) { + HdaWrite8(controller, base + HDA_STREAM_REG_STS, streamStatus); + controller->Output.InterruptCount++; + if ((streamStatus & HDA_STREAM_STS_COMPLETE) && !recovering) { + controller->Output.PendingPeriodInterrupts++; + queueDpc = true; + } + if (streamStatus & HDA_STREAM_STS_ERROR_MASK) { + controller->Output.DmaErrorCount++; + if (!recovering) { + controller->Output.PendingDmaErrors++; + queueDpc = true; + } + } + } + } + + if (interruptStatus & HDA_INT_CONTROLLER_STATUS) { + controller->ControllerInterruptCount++; + (void)HdaServiceRirbLocked(controller); + if (!recovering) { + queueDpc = queueDpc || controller->UnsolicitedCount != 0 || + controller->PendingRecovery != HDA_RECOVERY_NONE; + } + + uint8_t corbStatus = HdaRead8(controller, HDA_REG_CORBSTS); + if (corbStatus & HDA_CORBSTS_MEMORY_ERROR) { + controller->CorbMemoryErrorCount++; + HdaWrite8(controller, HDA_REG_CORBSTS, corbStatus); + controller->PendingRecovery = HDA_RECOVERY_CONTROLLER; + if (!recovering) + queueDpc = true; + } + } + KeReleaseSpinlock(&controller->StateLock, flags); + + if (queueDpc) + (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); + +finished: + _InterlockedDecrement(&controller->IsrActive); +} + +// Promotes an error to stream or controller recovery using the one-second threshold. +static void HdaRecordDmaErrorsLocked(HDA_CONTROLLER* controller, + uint32_t errorCount) { + uint64_t now = HdaQueryTicks(); + if (controller->StreamErrorsInWindow == 0 || + now - controller->StreamErrorWindowStart > HDA_STREAM_ERROR_WINDOW_US) { + controller->StreamErrorWindowStart = now; + controller->StreamErrorsInWindow = 0; + } + controller->StreamErrorsInWindow += errorCount; + uint8_t requested = controller->StreamErrorsInWindow >= + HDA_STREAM_ERRORS_BEFORE_CONTROLLER_RESET + ? HDA_RECOVERY_CONTROLLER + : HDA_RECOVERY_STREAM; + if (requested > controller->PendingRecovery) + controller->PendingRecovery = requested; +} + +// Runs period delivery, unsolicited callbacks, and recovery scheduling at DPC level. +void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)systemArgument1; + (void)systemArgument2; + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)context; + if (!controller) + return; + _InterlockedIncrement(&controller->DpcActive); + if (controller->Removed || controller->SurpriseRemoved) + goto finished; + + while (true) { + uint32_t periodCount = 0; + uint32_t errorCount = 0; + HDA_UNSOLICITED_RESPONSE unsolicited; + bool haveUnsolicited = false; + HDA_TRANSPORT_CALLBACKS callbacks; + void* callbackContext; + + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + if (controller->Output.PendingPeriodInterrupts > 0) { + periodCount = (uint32_t)controller->Output.PendingPeriodInterrupts; + controller->Output.PendingPeriodInterrupts = 0; + } + if (controller->Output.PendingDmaErrors > 0) { + errorCount = (uint32_t)controller->Output.PendingDmaErrors; + controller->Output.PendingDmaErrors = 0; + } + if (controller->UnsolicitedCount) { + unsolicited = controller->Unsolicited[controller->UnsolicitedRead]; + controller->UnsolicitedRead = + (uint8_t)((controller->UnsolicitedRead + 1u) % + HDA_MAX_UNSOLICITED_RESPONSES); + controller->UnsolicitedCount--; + haveUnsolicited = true; + } + callbacks = controller->Callbacks; + callbackContext = controller->CallbackContext; + KeReleaseSpinlock(&controller->StateLock, flags); + + for (uint32_t index = 0; index < periodCount; index++) { + flags = KeAcquireSpinlock(&controller->StateLock); + uint8_t bdlIndex = controller->Output.NextCompletionBdl; + uint32_t completedPeriod = controller->Output.BdlPeriodMap[bdlIndex]; + controller->Output.NextCompletionBdl = + (uint8_t)((bdlIndex + 1u) % HDA_OUTPUT_PERIOD_COUNT); + HDA_OUTPUT_RING* ring = + &controller->Output.Rings[controller->Output.ActiveRingIndex]; + ring->NextLogicalPeriod = + (uint8_t)((completedPeriod + 1u) % HDA_OUTPUT_PERIOD_COUNT); + ring->PresentedFrames += HDA_OUTPUT_PERIOD_FRAMES; + controller->Output.PeriodCompletionCount++; + uint64_t presentedFrames = ring->PresentedFrames; + uint64_t finePosition = HdaQueryOutputPositionFramesLocked(controller); + uint64_t interruptCount = controller->Output.InterruptCount; + KeReleaseSpinlock(&controller->StateLock, flags); + + uint64_t timestamp = HdaQueryTicks(); + if (callbacks.PeriodComplete) { + callbacks.PeriodComplete(callbackContext, + completedPeriod, + presentedFrames, + finePosition, + interruptCount, + timestamp); + } + } + + if (haveUnsolicited && callbacks.UnsolicitedResponse) { + callbacks.UnsolicitedResponse( + callbackContext, + (uint8_t)(unsolicited.ResponseExtra & HDA_RIRB_RESPONSE_CODEC_MASK), + unsolicited.Response, + unsolicited.ResponseExtra); + } + + if (errorCount) { + flags = KeAcquireSpinlock(&controller->StateLock); + HdaRecordDmaErrorsLocked(controller, errorCount); + KeReleaseSpinlock(&controller->StateLock, flags); + } + + bool signalRecovery = false; + flags = KeAcquireSpinlock(&controller->StateLock); + signalRecovery = controller->PendingRecovery != HDA_RECOVERY_NONE; + KeReleaseSpinlock(&controller->StateLock, flags); + if (signalRecovery && controller->RecoverySemaphore) + KeSignalSemaphore(controller->RecoverySemaphore); + + if (periodCount == 0 && errorCount == 0 && !haveUnsolicited) + break; + } + +finished: + _InterlockedDecrement(&controller->DpcActive); +} + +// Recovers one isolated stream error without disturbing the codec topology. +static KSTATUS HdaRecoverOutputStream(HDA_CONTROLLER* controller) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + (void)HdaQueryOutputPositionFrames(controller); + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + bool restart = controller->Output.Running; + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, flags); + KSTATUS status = HdaProgramOutputDescriptor(controller); + if (!KSUCCESS(status)) + return status; + controller->Output.StreamResetCount++; + HdaArmInterrupts(controller); + if (restart) + status = HdaRunPreparedOutput(controller); + return status; +} + +// Recovers command and DMA engines, then asks the codec layer to restore routing. +static KSTATUS HdaRecoverController(HDA_CONTROLLER* controller) { + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + (void)HdaQueryOutputPositionFrames(controller); + uintptr_t stateFlags = KeAcquireSpinlock(&controller->StateLock); + bool restart = controller->Output.Running; + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, stateFlags); + KSTATUS status = HdaInitializeController(controller); + if (!KSUCCESS(status)) + return status; + controller->ControllerResetCount++; + controller->StreamErrorWindowStart = 0; + controller->StreamErrorsInWindow = 0; + + HDA_TRANSPORT_CALLBACKS callbacks; + void* callbackContext; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + callbacks = controller->Callbacks; + callbackContext = controller->CallbackContext; + KeReleaseSpinlock(&controller->StateLock, flags); + if (callbacks.RestoreAfterControllerReset) { + status = callbacks.RestoreAfterControllerReset(callbackContext); + if (!KSUCCESS(status)) + return status; + } + HdaArmInterrupts(controller); + if (restart) { + status = HdaRunPreparedOutput(controller); + if (KSUCCESS(status) && callbacks.ResumeAfterControllerReset) + status = callbacks.ResumeAfterControllerReset(callbackContext); + if (!KSUCCESS(status)) { + (void)HdaStopOutputHardware(controller); + uintptr_t stopFlags = KeAcquireSpinlock(&controller->StateLock); + controller->Output.Running = false; + KeReleaseSpinlock(&controller->StateLock, stopFlags); + } + } + return status; +} + +// Waits until every pre-recovery ISR and DPC has left the controller object. +static KSTATUS HdaWaitForInterruptWorkIdle(HDA_CONTROLLER* controller) { + for (uint32_t elapsed = 0; elapsed < 100000u; elapsed += 10u) { + if (controller->IsrActive == 0 && + controller->Dpc.Inserted == 0 && controller->DpcActive == 0) + return NTLSTATUS_SUCCESS; + if (controller->Removed || controller->SurpriseRemoved) + return NTLSTATUS_DEVICE_REMOVED; + HdaStall(10); + } + return NTLSTATUS_TIMEOUT; +} + +// Runs passive-level reset work that cannot safely execute in an ISR or DPC. +static void HdaRecoveryWorker(uintptr_t argument) { + HDA_CONTROLLER* controller = (HDA_CONTROLLER*)argument; + while (controller->RecoveryWorkerRunning) { + KSTATUS waitStatus = KeWaitSemaphore(controller->RecoverySemaphore, KE_WAIT_INFINITE); + if (!KSUCCESS(waitStatus) || !controller->RecoveryWorkerRunning) + break; + + uint8_t recovery; + uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); + recovery = controller->PendingRecovery; + controller->PendingRecovery = HDA_RECOVERY_NONE; + controller->RecoveryInProgress = recovery != HDA_RECOVERY_NONE; + KeReleaseSpinlock(&controller->StateLock, flags); + if (recovery == HDA_RECOVERY_NONE) + continue; + + if (!controller->Removed && !controller->SurpriseRemoved) + HdaWrite32(controller, HDA_REG_INTCTL, 0); + KSTATUS status = HdaWaitForInterruptWorkIdle(controller); + bool streamLocked = false; + if (KSUCCESS(status)) { + status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); + streamLocked = KE_MUTEX_ACQUIRE_SUCCEEDED(status); + } + if (streamLocked) { + status = recovery == HDA_RECOVERY_CONTROLLER + ? HdaRecoverController(controller) + : HdaRecoverOutputStream(controller); + KeReleaseMutex(controller->StreamMutex); + } + + HDA_TRANSPORT_CALLBACKS callbacks; + void* callbackContext; + bool queueDpc; + bool repeatRecovery; + flags = KeAcquireSpinlock(&controller->StateLock); + controller->RecoveryInProgress = false; + callbacks = controller->Callbacks; + callbackContext = controller->CallbackContext; + queueDpc = controller->UnsolicitedCount != 0 || + controller->Output.PendingPeriodInterrupts != 0 || + controller->Output.PendingDmaErrors != 0; + repeatRecovery = controller->PendingRecovery != HDA_RECOVERY_NONE; + KeReleaseSpinlock(&controller->StateLock, flags); + + if (!KSUCCESS(status) && status != NTLSTATUS_DEVICE_REMOVED) { + if (callbacks.TransportFailure) + callbacks.TransportFailure(callbackContext, status); + gHdaDdk->Debug->Print("HDA: recovery failed status=0x%x\n", status); + } + if (queueDpc) + (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); + if (repeatRecovery) + KeSignalSemaphore(controller->RecoverySemaphore); + } + if (controller->RecoveryFinished) + KeSignalEvent(controller->RecoveryFinished); +} + +// Starts the passive recovery worker for one controller. +KSTATUS HdaStartRecoveryWorker(HDA_CONTROLLER* controller) { + if (!controller || !controller->RecoverySemaphore || !controller->RecoveryFinished) + return NTLSTATUS_INVALID_PARAMETER; + controller->RecoveryWorkerRunning = true; + KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityHigh); + KSTATUS status = KeCreateKernelThread((uintptr_t)HdaRecoveryWorker, + (uintptr_t)controller, + ¶meters, + &controller->RecoveryThread); + if (!KSUCCESS(status)) + controller->RecoveryWorkerRunning = false; + return status; +} + +// Stops and joins the passive recovery worker for one controller. +void HdaStopRecoveryWorker(HDA_CONTROLLER* controller) { + if (!controller) + return; + controller->RecoveryWorkerRunning = false; + if (controller->RecoverySemaphore) + KeSignalSemaphore(controller->RecoverySemaphore); + if (controller->RecoveryThread && controller->RecoveryFinished) + (void)KeWaitForEvent(controller->RecoveryFinished, 0); + if (controller->RecoveryThread) { + ObDereferenceObject((OBJECT*)controller->RecoveryThread); + controller->RecoveryThread = (THREAD*)0; + } +} diff --git a/code/drivers/bus/pci/pci.c b/code/drivers/bus/pci/pci.c index 90dae80..a72a8f1 100644 --- a/code/drivers/bus/pci/pci.c +++ b/code/drivers/bus/pci/pci.c @@ -33,7 +33,7 @@ static uint64_t PciAlignUp64(uint64_t value, uint64_t alignment) { // Implements the default Mmio Flags operation. static uint64_t PciDefaultMmioFlags(void) { - return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_WRITETHROUGH | NTL_DDK_PAGE_NOCACHE | NTL_DDK_PAGE_NX; + return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_CACHE_UC | NTL_DDK_PAGE_NX; } // Creates mmio Va. diff --git a/code/drivers/bus/usb/usb.c b/code/drivers/bus/usb/usb.c index ba1419b..5439b52 100644 --- a/code/drivers/bus/usb/usb.c +++ b/code/drivers/bus/usb/usb.c @@ -37,7 +37,7 @@ static KSTATUS UsbInitializeXhciDriver(void) { // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; + if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("usb: driver loading\n"); gUsbDdk = ddk; diff --git a/code/drivers/bus/usb/usb.h b/code/drivers/bus/usb/usb.h index ba6930c..427f003 100644 --- a/code/drivers/bus/usb/usb.h +++ b/code/drivers/bus/usb/usb.h @@ -124,6 +124,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define XHCI_TRB_SETUP_TRT_IN (3u << 16) #define XHCI_TRB_STATUS_DIR_IN (1u << 16) +#define XHCI_HCC1_AC64 (1u << 0) #define XHCI_HCC1_CSZ (1u << 2) #define XHCI_CONTEXT_ENTRIES(n) ((uint32_t)(n) << 27) @@ -199,6 +200,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; } #define SCSI_INQUIRY 0x12u #define SCSI_READ_10 0x28u #define SCSI_WRITE_10 0x2Au +#define SCSI_SYNCHRONIZE_CACHE_10 0x35u #define SCSI_READ_CAPACITY_10 0x25u #define XHCI_RING_TRBS 256u @@ -393,7 +395,11 @@ typedef struct USB_HCD_OPERATIONS { KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb); KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed); KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port); - void* (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys); + void (*DestroyPipe)(USB_HCD* hcd, USB_PIPE* pipe); + void* (*AllocateDma)(USB_HCD* hcd, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer); } USB_HCD_OPERATIONS; struct USB_HCD { @@ -454,7 +460,7 @@ struct USB_URB { USB_PIPE* Pipe; USB_SETUP_PACKET Setup; void* Buffer; - PHYSICAL_ADDRESS BufferPhys; + uint64_t BufferDeviceAddress; uint32_t Length; uint32_t ActualLength; uint32_t HcdCompletionCode; @@ -521,8 +527,9 @@ typedef struct USB_HUB_EXTENSION { uint32_t PendingHubPorts[1]; SPINLOCK PendingLock; SEMAPHORE* WorkerSemaphore; + EVENT* WorkerFinished; THREAD* WorkerThread; - bool WorkerRunning; + volatile bool WorkerRunning; bool InitialScanPending; bool InitialScanComplete; bool InitialScanInProgress; @@ -533,7 +540,8 @@ typedef struct USB_HUB_EXTENSION { uint8_t InterruptInterval; USB_PIPE* InterruptPipe; uint8_t* ChangeBitmap; - PHYSICAL_ADDRESS ChangeBitmapPhys; + uint64_t ChangeBitmapDeviceAddress; + IO_DMA_BUFFER* ChangeBitmapDma; USB_URB ChangeUrb; bool ChangeUrbPending; uint8_t PowerOnToPowerGoodMs; @@ -573,7 +581,8 @@ typedef struct USB_HID_EXTENSION { USB_PIPE* InterruptPipe; uint8_t* ReportBuffer; - PHYSICAL_ADDRESS ReportBufferPhys; + uint64_t ReportBufferDeviceAddress; + IO_DMA_BUFFER* ReportBufferDma; uint16_t ReportLength; USB_URB InterruptUrb; bool InterruptPending; @@ -584,6 +593,12 @@ typedef struct USB_HID_EXTENSION { uint32_t InputDeviceId; } USB_HID_EXTENSION; +typedef enum USB_STORAGE_FLUSH_CAPABILITY { + UsbStorageFlushUnknown = 0, + UsbStorageFlushSupported, + UsbStorageFlushUnsupported, +} USB_STORAGE_FLUSH_CAPABILITY; + typedef enum USB_STORAGE_EXTENSION_KIND { UsbStorageExtensionInterface = 0x55534946u, UsbStorageExtensionDisk = 0x5553444Bu, @@ -620,6 +635,7 @@ struct USB_STORAGE_EXTENSION { uint32_t Tag; uint32_t BlockSize; uint64_t BlockCount; + USB_STORAGE_FLUSH_CAPABILITY FlushCapability; volatile long IoBusy; }; @@ -630,14 +646,15 @@ struct USB_STORAGE_DISK_EXTENSION { typedef struct XHCI_PIPE_CONTEXT { XHCI_TRB* Ring; - PHYSICAL_ADDRESS RingPhys; + uint64_t RingDeviceAddress; + IO_DMA_BUFFER* RingDma; uint32_t Enqueue; uint8_t Cycle; uint8_t EndpointId; USB_PIPE_TYPE Type; bool Configured; bool ActiveTransfer; - uint64_t ActiveTrbPhys; + uint64_t ActiveTrbDeviceAddress; USB_URB* ActiveUrb; } XHCI_PIPE_CONTEXT; @@ -683,7 +700,7 @@ typedef struct XHCI_DEVICE { uint8_t HidLastModifiers; uint8_t HidLastKeys[6]; uint8_t HidLastButtons; - uint64_t HidLastTrbPhys; + uint64_t HidLastTrbDeviceAddress; bool MassCandidate; bool MassActive; @@ -708,15 +725,25 @@ typedef struct XHCI_DEVICE { bool MassRemoved; volatile long MassIoBusy; - PHYSICAL_ADDRESS InputContextPhys; - PHYSICAL_ADDRESS DeviceContextPhys; - PHYSICAL_ADDRESS Ep0RingPhys; - PHYSICAL_ADDRESS HidRingPhys; - PHYSICAL_ADDRESS MassInRingPhys; - PHYSICAL_ADDRESS MassOutRingPhys; - PHYSICAL_ADDRESS DescriptorPhys; - PHYSICAL_ADDRESS HidBufferPhys; - PHYSICAL_ADDRESS MassBufferPhys; + uint64_t InputContextDeviceAddress; + uint64_t DeviceContextDeviceAddress; + uint64_t Ep0RingDeviceAddress; + uint64_t HidRingDeviceAddress; + uint64_t MassInRingDeviceAddress; + uint64_t MassOutRingDeviceAddress; + uint64_t DescriptorDeviceAddress; + uint64_t HidBufferDeviceAddress; + uint64_t MassBufferDeviceAddress; + + IO_DMA_BUFFER* InputContextDma; + IO_DMA_BUFFER* DeviceContextDma; + IO_DMA_BUFFER* Ep0RingDma; + IO_DMA_BUFFER* HidRingDma; + IO_DMA_BUFFER* MassInRingDma; + IO_DMA_BUFFER* MassOutRingDma; + IO_DMA_BUFFER* DescriptorDma; + IO_DMA_BUFFER* HidBufferDma; + IO_DMA_BUFFER* MassBufferDma; uint8_t* InputContext; uint8_t* DeviceContext; @@ -746,8 +773,8 @@ typedef struct XHCI_TRANSFER_RESULT { typedef struct XHCI_WAIT_TARGET { struct XHCI_WAIT_TARGET* Next; - uint64_t CommandPhys; - uint64_t TransferPhys; + uint64_t CommandDeviceAddress; + uint64_t TransferDeviceAddress; uint8_t TransferSlotId; uint8_t TransferEndpointId; volatile bool CommandMatched; @@ -793,12 +820,20 @@ typedef struct XHCI_CONTROLLER { uint8_t PortProtocolMajor[XHCI_MAX_ROOT_PORTS + 1u]; uint8_t PortProtocolMinor[XHCI_MAX_ROOT_PORTS + 1u]; - PHYSICAL_ADDRESS DcbaaPhys; - PHYSICAL_ADDRESS ScratchpadArrayPhys; - PHYSICAL_ADDRESS ScratchpadPhys[XHCI_MAX_SCRATCHPADS]; - PHYSICAL_ADDRESS CommandRingPhys; - PHYSICAL_ADDRESS EventRingPhys; - PHYSICAL_ADDRESS ErstPhys; + uint64_t DcbaaDeviceAddress; + uint64_t ScratchpadArrayDeviceAddress; + uint64_t ScratchpadDeviceAddress[XHCI_MAX_SCRATCHPADS]; + uint64_t CommandRingDeviceAddress; + uint64_t EventRingDeviceAddress; + uint64_t ErstDeviceAddress; + + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* DcbaaDma; + IO_DMA_BUFFER* ScratchpadArrayDma; + IO_DMA_BUFFER* ScratchpadDma[XHCI_MAX_SCRATCHPADS]; + IO_DMA_BUFFER* CommandRingDma; + IO_DMA_BUFFER* EventRingDma; + IO_DMA_BUFFER* ErstDma; uint64_t* Dcbaa; uint64_t* ScratchpadArray; @@ -825,6 +860,7 @@ typedef struct XHCI_CONTROLLER { XHCI_WAIT_TARGET* WaiterList; volatile bool Running; + bool SurpriseRemoved; USB_HCD Hcd; } XHCI_CONTROLLER; @@ -867,7 +903,11 @@ void UsbDereferencePipe(USB_PIPE* pipe); KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb); KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb); KSTATUS UsbCancelUrb(USB_URB* urb); -void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys); +void* UsbAllocateDmaBuffer(USB_DEVICE* dev, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer); +void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer); KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp); KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp); @@ -882,7 +922,11 @@ void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value); void UsbDelayUs(const NTL_DDK* ddk, uint32_t us); bool UsbWaitSet32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs); bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs); -void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys); +void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer); +void XhciReleaseDeviceDma(XHCI_DEVICE* dev); uint16_t XhciScratchpadCount(uint32_t hcs2); uint32_t XhciTrbType(const XHCI_TRB* trb); uint32_t XhciCompletionCode(const XHCI_TRB* trb); @@ -918,7 +962,7 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci); KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci); KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci); KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci); -uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index); +uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index); KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, XHCI_COMMAND_RESULT* outResult); KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci); KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId); @@ -931,17 +975,17 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed); KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId); KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); -uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index); -KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys); -KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs); -KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length); +uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index); +KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress); +KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length, uint32_t timeoutUs); +KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length); KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup); KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs); KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc); KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength); uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress); uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval); -KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys); +KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress); bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key); INPUT_KEY XhciHidModifierKey(uint8_t bit); void XhciSubmitHidKey(INPUT_KEY key, bool pressed); @@ -949,24 +993,23 @@ void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len); KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); void XhciArmAllHid(XHCI_CONTROLLER* xhci); -void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue); +void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue); KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci); KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev); KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject); KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp); -void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle); +void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint8_t cycle); KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe); -KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual); -KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn); +KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t* enqueue, uint8_t* cycle, uint64_t bufferDeviceAddress, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual); +KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, uint64_t dataDeviceAddress, uint32_t dataLength, bool dataIn); uint32_t UsbBe32(const uint8_t* p); void UsbStoreBe32(uint8_t* p, uint32_t v); void UsbStoreBe16(uint8_t* p, uint16_t v); KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage); -KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage); KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage); KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage); KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer); diff --git a/code/drivers/bus/usb/usb_bus.c b/code/drivers/bus/usb/usb_bus.c index 617736b..188dafd 100644 --- a/code/drivers/bus/usb/usb_bus.c +++ b/code/drivers/bus/usb/usb_bus.c @@ -24,9 +24,12 @@ static void UsbDeviceDestroy(OBJECT* object) { } } -// Implements the pipe Destroy operation. +// Destroys a pipe after the host controller releases its private context. static void UsbPipeDestroy(OBJECT* object) { USB_PIPE* pipe = (USB_PIPE*)object; + if (pipe->Device && pipe->Device->Hcd && pipe->Device->Hcd->Operations && + pipe->Device->Hcd->Operations->DestroyPipe) + pipe->Device->Hcd->Operations->DestroyPipe(pipe->Device->Hcd, pipe); if (pipe->Device) { UsbDereferenceDevice(pipe->Device); pipe->Device = (USB_DEVICE*)0; @@ -53,11 +56,21 @@ void UsbDereferencePipe(USB_PIPE* pipe) { if (pipe) ObDereferenceObject(&pipe->Header); } -// Creates dma Buffer. -void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) { - if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma) +// Allocates a DMA buffer through the device host-controller adapter. +void* UsbAllocateDmaBuffer(USB_DEVICE* dev, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer) { + if (!dev || !dev->Hcd || !dev->Hcd->Operations || + !dev->Hcd->Operations->AllocateDma || !outDeviceAddress || !outBuffer) return (void*)0; - return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys); + return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outDeviceAddress, outBuffer); +} + +// Frees a DMA buffer allocated for a USB class driver. +void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer) { + if (buffer && gUsbDdk && gUsbDdk->Dma) + gUsbDdk->Dma->FreeCommonBuffer(buffer); } // Creates device. @@ -464,7 +477,7 @@ KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) { if (!parent) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { DEVICE_OBJECT* child = UsbReferenceFirstChild(parent); if (!child) break; diff --git a/code/drivers/bus/usb/usb_core.c b/code/drivers/bus/usb/usb_core.c index fb0e562..af656c1 100644 --- a/code/drivers/bus/usb/usb_core.c +++ b/code/drivers/bus/usb/usb_core.c @@ -258,6 +258,9 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) { XHCI_DEVICE* dev = &xhci->Devices[slotId]; + if (dev->Allocated && xhci->Running) + (void)XhciDisableSlot(xhci, slotId); + if (dev->UsbPdo) { (void)UsbRemovePdoSubtree(dev->UsbPdo, true); dev->UsbPdo = (DEVICE_OBJECT*)0; @@ -269,9 +272,7 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) { dev->UsbDevice = (USB_DEVICE*)0; } - if (dev->Allocated) - (void)XhciDisableSlot(xhci, slotId); - + XhciReleaseDeviceDma(dev); memset(dev, 0, sizeof(*dev)); } diff --git a/code/drivers/bus/usb/usb_hid.c b/code/drivers/bus/usb/usb_hid.c index 37bb02f..509909b 100644 --- a/code/drivers/bus/usb/usb_hid.c +++ b/code/drivers/bus/usb/usb_hid.c @@ -2,6 +2,16 @@ #include "usb.h" +// Releases the HID class DMA buffer after all pending transfers are cancelled. +static void UsbHidReleaseDma(USB_HID_EXTENSION* ext) { + if (!ext) + return; + UsbFreeDmaBuffer(ext->ReportBufferDma); + ext->ReportBufferDma = (IO_DMA_BUFFER*)0; + ext->ReportBuffer = (uint8_t*)0; + ext->ReportBufferDeviceAddress = 0; +} + // Implements the hid Key In Report operation. bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) { if (key == 0 || key == 1) return false; @@ -91,21 +101,21 @@ KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { memset(dev->HidBuffer, 0, dev->HidReportLength); - uint64_t trbPhys = 0; + uint64_t trbDeviceAddress = 0; KSTATUS st = XhciRingAppend(dev->HidRing, - dev->HidRingPhys, + dev->HidRingDeviceAddress, XHCI_HID_RING_TRBS, &dev->HidEnqueue, &dev->HidCycle, - dev->HidBufferPhys, + dev->HidBufferDeviceAddress, dev->HidReportLength, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, - &trbPhys); + &trbDeviceAddress); if (!KSUCCESS(st)) return st; - dev->HidLastTrbPhys = trbPhys; + dev->HidLastTrbDeviceAddress = trbDeviceAddress; dev->HidArmed = true; XhciBarrier(); XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, dev->HidEndpointId); @@ -125,13 +135,13 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) { } } -static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, +static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue) { if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return; XHCI_DEVICE* dev = &xhci->Devices[slotId]; - if (!dev->HidActive || endpointId != dev->HidEndpointId || trbPhys != dev->HidLastTrbPhys) + if (!dev->HidActive || endpointId != dev->HidEndpointId || trbDeviceAddress != dev->HidLastTrbDeviceAddress) return; dev->HidArmed = false; @@ -200,17 +210,26 @@ KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { if (dev->HidRing && dev->HidBuffer) return NTLSTATUS_SUCCESS; - dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingPhys); - dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferPhys); - if (!dev->HidRing || !dev->HidBuffer) + dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingDeviceAddress, &dev->HidRingDma); + dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferDeviceAddress, &dev->HidBufferDma); + if (!dev->HidRing || !dev->HidBuffer) { + if (gUsbDdk && gUsbDdk->Dma) { + gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma); + } + dev->HidBufferDma = (IO_DMA_BUFFER*)0; + dev->HidRingDma = (IO_DMA_BUFFER*)0; + dev->HidBuffer = (uint8_t*)0; + dev->HidRing = (XHCI_TRB*)0; return NTLSTATUS_NO_MEMORY; + } dev->HidEnqueue = 0; dev->HidCycle = 1; XHCI_TRB* link = &dev->HidRing[XHCI_HID_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)dev->HidRingPhys; - link->ParameterHi = (uint32_t)(dev->HidRingPhys >> 32); + link->ParameterLo = (uint32_t)dev->HidRingDeviceAddress; + link->ParameterHi = (uint32_t)(dev->HidRingDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | @@ -253,15 +272,15 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { ep[0] = (uint32_t)interval << 16; ep[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_INT_IN | ((uint32_t)maxPacket << 16); - ep[2] = (uint32_t)(dev->HidRingPhys | XHCI_EP_DCS); - ep[3] = (uint32_t)((dev->HidRingPhys | XHCI_EP_DCS) >> 32); + ep[2] = (uint32_t)(dev->HidRingDeviceAddress | XHCI_EP_DCS); + ep[3] = (uint32_t)((dev->HidRingDeviceAddress | XHCI_EP_DCS) >> 32); ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16); XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -499,7 +518,7 @@ static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) { static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) { if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed) return NTLSTATUS_OBJECT_DEAD; - if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys) + if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferDeviceAddress) return NTLSTATUS_INVALID_PARAMETER; if (ext->InterruptPending) return NTLSTATUS_SUCCESS; @@ -511,7 +530,7 @@ static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) { ext->InterruptUrb.Device = ext->Device; ext->InterruptUrb.Pipe = ext->InterruptPipe; ext->InterruptUrb.Buffer = ext->ReportBuffer; - ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys; + ext->InterruptUrb.BufferDeviceAddress = ext->ReportBufferDeviceAddress; ext->InterruptUrb.Length = ext->ReportLength; ext->InterruptUrb.Complete = UsbHidInterruptComplete; ext->InterruptUrb.CompleteContext = ext; @@ -565,7 +584,7 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT; ext->ReportLength = reportLength; - ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys); + ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferDeviceAddress, &ext->ReportBufferDma); if (!ext->ReportBuffer) return NTLSTATUS_NO_MEMORY; @@ -575,13 +594,22 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) ext->MaxPacketSize, ext->Interval, &ext->InterruptPipe); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + UsbHidReleaseDma(ext); return st; + } if (!ext->AbsoluteMouse) { st = UsbHidSendBootProtocol(ext); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + if (ext->InterruptPipe) { + ext->InterruptPipe->Open = false; + UsbDereferencePipe(ext->InterruptPipe); + ext->InterruptPipe = (USB_PIPE*)0; + } + UsbHidReleaseDma(ext); return st; + } } memset(ext->LastKeys, 0, sizeof(ext->LastKeys)); @@ -678,6 +706,7 @@ KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { IoDestroyRemoveLock(&ext->RemoveLock); ext->RemoveLockReady = false; } + UsbHidReleaseDma(ext); } KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS; diff --git a/code/drivers/bus/usb/usb_hub.c b/code/drivers/bus/usb/usb_hub.c index 534f57d..f4d14dc 100644 --- a/code/drivers/bus/usb/usb_hub.c +++ b/code/drivers/bus/usb/usb_hub.c @@ -8,6 +8,16 @@ DRIVER_OBJECT gUsbHubDriver; +// Releases the external-hub change bitmap after transfer rundown. +static void UsbHubReleaseDma(USB_HUB_EXTENSION* hub) { + if (!hub) + return; + UsbFreeDmaBuffer(hub->ChangeBitmapDma); + hub->ChangeBitmapDma = (IO_DMA_BUFFER*)0; + hub->ChangeBitmap = (uint8_t*)0; + hub->ChangeBitmapDeviceAddress = 0; +} + static const IO_DEVICE_ID gUsbHubIds[] = { { IoBusUsb, { "USB\\ROOT_HUB", sizeof("USB\\ROOT_HUB") - 1 } }, { IoBusUsb, { "USB\\CLASS_09", sizeof("USB\\CLASS_09") - 1 } }, @@ -61,8 +71,10 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub, if (maxAttempts == 0) maxAttempts = 1; for (uint8_t attempt = 1; attempt <= maxAttempts; attempt++) { - if (!UsbHubRootPortConnected(xhci, port)) - return NTLSTATUS_NOT_FOUND; + if (!UsbHubRootPortConnected(xhci, port)) { + last = NTLSTATUS_NOT_FOUND; + break; + } uint8_t slot = 0; last = XhciEnumeratePortEx(xhci, port, &slot); @@ -100,6 +112,40 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub, return last; } +// Enumerates one connected root port through the same debounce path at boot and hotplug. +static KSTATUS UsbHubEnumerateConnectedRootPort(USB_HUB_EXTENSION* hub, + XHCI_CONTROLLER* xhci, + uint8_t port, + uint8_t* outSlot) { + if (outSlot) + *outSlot = 0; + if (!hub || !xhci || port == 0 || port > hub->PortCount || + port > USB_ROOT_HUB_MAX_PORTS) + return NTLSTATUS_INVALID_PARAMETER; + + hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; + hub->RootPortState[port] = UsbHubPortConnectedDebounce; + + UsbDelayUs(xhci->Ddk, 30000u); + if (!UsbHubRootPortConnected(xhci, port)) { + UsbHubClearRootPortFailure(hub, port); + hub->RootPortSlot[port] = 0; + hub->RootPortState[port] = UsbHubPortDisconnected; + return NTLSTATUS_NOT_FOUND; + } + + hub->RootPortState[port] = UsbHubPortEnumerating; + uint8_t slot = 0; + KSTATUS status = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + if (KSUCCESS(status)) { + hub->RootPortSlot[port] = slot; + hub->RootPortState[port] = UsbHubPortChildPresent; + if (outSlot) + *outSlot = slot; + } + return status; +} + // Implements the hub Pop Pending Root Port operation. static bool UsbHubPopPendingRootPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) { if (!hub || !outPort) return false; @@ -184,17 +230,19 @@ static void UsbHubRemoveSlotChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t DbgPrint("usb-hub: %s port %u removing slot=%u\n", hub->Kind == UsbHubKindRoot ? "root" : "hub", port, slot); + if (xdev->Allocated && xhci->Running) + (void)XhciDisableSlot(xhci, slot); + if (xdev->UsbPdo) (void)UsbRemovePdoSubtree(xdev->UsbPdo, true); - if (xdev->Allocated) - (void)XhciDisableSlot(xhci, slot); - if (xdev->UsbDevice) { xdev->UsbDevice->State = UsbDeviceRemoved; UsbDereferenceDevice(xdev->UsbDevice); + xdev->UsbDevice = (USB_DEVICE*)0; } + XhciReleaseDeviceDma(xdev); memset(xdev, 0, sizeof(*xdev)); if (hub->Kind == UsbHubKindRoot && port <= USB_ROOT_HUB_MAX_PORTS) { hub->RootPortSlot[port] = 0; @@ -216,9 +264,9 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub, uint16_t value, uint16_t index, void* buffer, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint16_t length) { - if (!hub || !hub->LowerDevice || !buffer || !bufferPhys || !length) + if (!hub || !hub->LowerDevice || !buffer || !bufferDeviceAddress || !length) return NTLSTATUS_INVALID_PARAMETER; USB_URB urb; @@ -227,7 +275,7 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub, urb.Direction = UsbTransferIn; urb.Device = hub->Device; urb.Buffer = buffer; - urb.BufferPhys = bufferPhys; + urb.BufferDeviceAddress = bufferDeviceAddress; urb.Length = length; urb.Setup.bmRequestType = requestType; urb.Setup.bRequest = request; @@ -260,7 +308,7 @@ static KSTATUS UsbHubControlNoData(USB_HUB_EXTENSION* hub, // Implements the hub Read Descriptor operation. static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* outDesc) { - if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapPhys) + if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapDeviceAddress) return NTLSTATUS_INVALID_PARAMETER; memset(hub->ChangeBitmap, 0, USB_HUB_CHANGE_BITMAP_BYTES); @@ -270,7 +318,7 @@ static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* (uint16_t)(USB_DESC_HUB << 8), 0, hub->ChangeBitmap, - hub->ChangeBitmapPhys, + hub->ChangeBitmapDeviceAddress, USB_HUB_CHANGE_BITMAP_BYTES); if (!KSUCCESS(st)) return st; @@ -293,7 +341,7 @@ static KSTATUS UsbHubGetPortStatus(USB_HUB_EXTENSION* hub, uint8_t port, USB_HUB 0, port, hub->ChangeBitmap, - hub->ChangeBitmapPhys, + hub->ChangeBitmapDeviceAddress, sizeof(USB_HUB_PORT_STATUS)); if (!KSUCCESS(st)) return st; @@ -501,19 +549,8 @@ static void UsbHubHandleRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) { return; } - hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; - hub->RootPortState[port] = UsbHubPortConnectedDebounce; - - UsbDelayUs(xhci->Ddk, 30000); - portsc = XhciRead32(xhci->Op, XhciPortReg(port)); - if (!(portsc & XHCI_PORTSC_CCS)) { - UsbHubClearRootPortFailure(hub, port); - return; - } - - hub->RootPortState[port] = UsbHubPortEnumerating; uint8_t slot = 0; - KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot); if (KSUCCESS(st)) { hub->RootPortSlot[port] = slot; hub->RootPortState[port] = UsbHubPortChildPresent; @@ -544,9 +581,11 @@ static bool UsbHubTakeInitialRootScan(USB_HUB_EXTENSION* hub) { static void UsbHubWorker(void* arg) { USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)arg; while (hub && hub->WorkerRunning) { - KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore); + KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(st)) continue; + if (!hub->WorkerRunning) + break; if (hub->Kind == UsbHubKindRoot) { if (UsbHubTakeInitialRootScan(hub)) @@ -563,6 +602,61 @@ static void UsbHubWorker(void* arg) { UsbHubArmExternalInterrupt(hub); } } + if (hub && hub->WorkerFinished) + KeSignalEvent(hub->WorkerFinished); +} + +// Stops the hub worker and releases its synchronization objects. +static void UsbHubStopWorker(USB_HUB_EXTENSION* hub) { + if (!hub) + return; + + hub->WorkerRunning = false; + if (hub->WorkerSemaphore) + KeSignalSemaphore(hub->WorkerSemaphore); + if (hub->WorkerThread && hub->WorkerFinished) + (void)KeWaitForEvent(hub->WorkerFinished, 0); + + if (hub->WorkerThread) { + ObDereferenceObject((OBJECT*)hub->WorkerThread); + hub->WorkerThread = (THREAD*)0; + } + if (hub->WorkerFinished) { + ObDereferenceObject(&hub->WorkerFinished->Header); + hub->WorkerFinished = (EVENT*)0; + } + if (hub->WorkerSemaphore) { + ObDereferenceObject(&hub->WorkerSemaphore->Header); + hub->WorkerSemaphore = (SEMAPHORE*)0; + } +} + +// Creates the synchronization objects and thread for one hub worker. +static KSTATUS UsbHubStartWorker(USB_HUB_EXTENSION* hub) { + if (!hub) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); + if (!KSUCCESS(status)) + return status; + status = KeEventCreate(EventManualReset, 0, &hub->WorkerFinished); + if (!KSUCCESS(status)) { + UsbHubStopWorker(hub); + return status; + } + + hub->WorkerRunning = true; + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + status = KeCreateKernelThread((uintptr_t)UsbHubWorker, + (uintptr_t)hub, + &workerParameters, + &hub->WorkerThread); + if (!KSUCCESS(status)) { + hub->WorkerRunning = false; + UsbHubStopWorker(hub); + return status; + } + return NTLSTATUS_SUCCESS; } // Implements the hub Do Root Port Scan operation. @@ -592,10 +686,8 @@ static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd) { } attempted++; - hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING; - hub->RootPortState[port] = UsbHubPortEnumerating; uint8_t slot = 0; - KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot); + KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot); if (KSUCCESS(st)) { hub->RootPortSlot[port] = slot; hub->RootPortState[port] = UsbHubPortChildPresent; @@ -734,7 +826,7 @@ static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub) { hub->ChangeUrb.Device = hub->Device; hub->ChangeUrb.Pipe = hub->InterruptPipe; hub->ChangeUrb.Buffer = hub->ChangeBitmap; - hub->ChangeUrb.BufferPhys = hub->ChangeBitmapPhys; + hub->ChangeUrb.BufferDeviceAddress = hub->ChangeBitmapDeviceAddress; hub->ChangeUrb.Length = hub->InterruptMaxPacket; if (hub->ChangeUrb.Length > USB_HUB_CHANGE_BITMAP_BYTES) hub->ChangeUrb.Length = USB_HUB_CHANGE_BITMAP_BYTES; @@ -756,20 +848,10 @@ static KSTATUS UsbHubStartRootDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* h if (hub->PortCount > USB_ROOT_HUB_MAX_PORTS) hub->PortCount = USB_ROOT_HUB_MAX_PORTS; - KSTATUS st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); + KSTATUS st = UsbHubStartWorker(hub); if (!KSUCCESS(st)) return st; - hub->WorkerRunning = true; - st = KeCreateKernelThread((uintptr_t)UsbHubWorker, - (uintptr_t)hub, - PriorityNormal, - &hub->WorkerThread); - if (!KSUCCESS(st)) { - hub->WorkerRunning = false; - return st; - } - hub->Started = true; device->PnpState = DevicePnpStarted; if (hub->HubPdo) @@ -784,7 +866,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO if (!hub || !hub->Device) return NTLSTATUS_INVALID_PARAMETER; - hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapPhys); + hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapDeviceAddress, &hub->ChangeBitmapDma); if (!hub->ChangeBitmap) return NTLSTATUS_NO_MEMORY; @@ -793,6 +875,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO KSTATUS st = UsbHubReadDescriptor(hub, &desc); if (!KSUCCESS(st)) { DbgPrint("usb-hub: slot=%u descriptor read failed st=0x%x\n", hub->Device->SlotId, st); + UsbHubReleaseDma(hub); return st; } @@ -809,17 +892,9 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO UsbDelayUs(((XHCI_CONTROLLER*)hub->Hcd->ControllerContext)->Ddk, (uint32_t)hub->PowerOnToPowerGoodMs * 1000u); - st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore); - if (!KSUCCESS(st)) - return st; - - hub->WorkerRunning = true; - st = KeCreateKernelThread((uintptr_t)UsbHubWorker, - (uintptr_t)hub, - PriorityNormal, - &hub->WorkerThread); + st = UsbHubStartWorker(hub); if (!KSUCCESS(st)) { - hub->WorkerRunning = false; + UsbHubReleaseDma(hub); return st; } @@ -829,8 +904,11 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO hub->InterruptMaxPacket, hub->InterruptInterval, &hub->InterruptPipe); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + UsbHubStopWorker(hub); + UsbHubReleaseDma(hub); return st; + } hub->Started = true; device->PnpState = DevicePnpStarted; @@ -897,6 +975,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { (void)UsbCancelUrb(&hub->ChangeUrb); if (hub->WorkerSemaphore) KeSignalSemaphore(hub->WorkerSemaphore); + UsbHubStopWorker(hub); // Remove downstream PDOs owned by this hub FDO. The explicit // slot arrays are still used as the hardware lookup table; the @@ -929,8 +1008,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { UsbDereferencePipe(hub->InterruptPipe); hub->InterruptPipe = (USB_PIPE*)0; } - if (hub->WorkerSemaphore) - KeSignalSemaphore(hub->WorkerSemaphore); + UsbHubStopWorker(hub); if (hub->Kind == UsbHubKindRoot && hub->Hcd && hub->Hcd->RootHubContext == hub) hub->Hcd->RootHubContext = (void*)0; if (hub->RemoveLockReady) { @@ -938,6 +1016,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { IoDestroyRemoveLock(&hub->RemoveLock); hub->RemoveLockReady = false; } + UsbHubReleaseDma(hub); } KSTATUS lower = hub && hub->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS; diff --git a/code/drivers/bus/usb/usb_storage.c b/code/drivers/bus/usb/usb_storage.c index b5677d8..cd2c890 100644 --- a/code/drivers/bus/usb/usb_storage.c +++ b/code/drivers/bus/usb/usb_storage.c @@ -56,10 +56,10 @@ static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_E static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage, bool dataIn, - PHYSICAL_ADDRESS phys, + uint64_t deviceAddress, uint32_t length, uint32_t* outActual) { - if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0) + if (!storage || !storage->LowerDevice || !storage->Device || !deviceAddress || length == 0) return NTLSTATUS_INVALID_PARAMETER; USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe; @@ -73,7 +73,7 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage, urb.Device = storage->Device; urb.Pipe = pipe; urb.Buffer = (void*)0; - urb.BufferPhys = phys; + urb.BufferDeviceAddress = deviceAddress; urb.Length = length; KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb); @@ -83,29 +83,42 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage, } KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, - PHYSICAL_ADDRESS phys, + uint64_t deviceAddress, uint32_t length, uint32_t* outActual) { - return UsbStorageSubmitBulk(storage, true, phys, length, outActual); + return UsbStorageSubmitBulk(storage, true, deviceAddress, length, outActual); } KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, - PHYSICAL_ADDRESS phys, + uint64_t deviceAddress, uint32_t length, uint32_t* outActual) { - return UsbStorageSubmitBulk(storage, false, phys, length, outActual); + return UsbStorageSubmitBulk(storage, false, deviceAddress, length, outActual); } -KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, - const uint8_t* cdb, - uint8_t cdbLen, - void* data, - PHYSICAL_ADDRESS dataPhys, - uint32_t dataLength, - bool dataIn) { +typedef struct USB_STORAGE_SENSE_DATA { + bool Valid; + uint8_t ResponseCode; + uint8_t Key; + uint8_t Asc; + uint8_t Ascq; +} USB_STORAGE_SENSE_DATA; + +// Executes one BOT command without issuing follow-up commands. +static KSTATUS UsbStorageBotCommandRaw(USB_STORAGE_EXTENSION* storage, + const uint8_t* cdb, + uint8_t cdbLen, + void* data, + uint64_t dataDeviceAddress, + uint32_t dataLength, + bool dataIn, + uint8_t* outCswStatus, + uint32_t* outResidue) { + if (outCswStatus) *outCswStatus = 0xFFu; + if (outResidue) *outResidue = 0; if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16) return NTLSTATUS_INVALID_PARAMETER; - if (dataLength && (!data || !dataPhys)) + if (dataLength && (!data || !dataDeviceAddress)) return NTLSTATUS_INVALID_PARAMETER; XHCI_DEVICE* dev = storage->XhciDevice; @@ -120,14 +133,14 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, memcpy(cbw->CommandBlock, cdb, cdbLen); uint32_t actual = 0; - KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual); + KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorDeviceAddress, sizeof(*cbw), &actual); if (!KSUCCESS(st) || actual != sizeof(*cbw)) return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; if (dataLength) { actual = 0; - st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual) - : UsbStorageBulkOut(storage, dataPhys, dataLength, &actual); + st = dataIn ? UsbStorageBulkIn(storage, dataDeviceAddress, dataLength, &actual) + : UsbStorageBulkOut(storage, dataDeviceAddress, dataLength, &actual); if (!KSUCCESS(st)) return st; } @@ -135,7 +148,7 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer; memset(csw, 0, sizeof(*csw)); actual = 0; - st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual); + st = UsbStorageBulkIn(storage, dev->DescriptorDeviceAddress, sizeof(*csw), &actual); if (!KSUCCESS(st) || actual != sizeof(*csw)) return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; @@ -144,15 +157,136 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, dev->SlotId, csw->Signature, csw->Tag, storage->Tag); return NTLSTATUS_DEVICE_ERROR; } - if (csw->Status != USB_MASS_BOT_CSW_PASSED) { - DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u\n", - cdb[0], dev->SlotId, csw->Status, csw->DataResidue); - return NTLSTATUS_DEVICE_ERROR; + + if (outCswStatus) *outCswStatus = csw->Status; + if (outResidue) *outResidue = csw->DataResidue; + return csw->Status == USB_MASS_BOT_CSW_PASSED + ? NTLSTATUS_SUCCESS + : NTLSTATUS_DEVICE_ERROR; +} + +// Returns a readable SCSI sense-key name. +static const char* UsbStorageSenseKeyName(uint8_t key) { + switch (key) { + case 0x00u: return "NO SENSE"; + case 0x01u: return "RECOVERED ERROR"; + case 0x02u: return "NOT READY"; + case 0x03u: return "MEDIUM ERROR"; + case 0x04u: return "HARDWARE ERROR"; + case 0x05u: return "ILLEGAL REQUEST"; + case 0x06u: return "UNIT ATTENTION"; + case 0x07u: return "DATA PROTECT"; + case 0x08u: return "BLANK CHECK"; + case 0x0Bu: return "ABORTED COMMAND"; + case 0x0Eu: return "MISCOMPARE"; + default: return "UNKNOWN"; + } +} + +// Decodes fixed or descriptor-format REQUEST SENSE data. +static void UsbStorageDecodeSense(const uint8_t* bytes, + uint32_t length, + USB_STORAGE_SENSE_DATA* sense) { + if (!sense) return; + memset(sense, 0, sizeof(*sense)); + if (!bytes || length == 0) return; + + sense->ResponseCode = bytes[0] & 0x7Fu; + if ((sense->ResponseCode == 0x70u || sense->ResponseCode == 0x71u) && length >= 14u) { + sense->Key = bytes[2] & 0x0Fu; + sense->Asc = bytes[12]; + sense->Ascq = bytes[13]; + sense->Valid = true; + return; + } + if ((sense->ResponseCode == 0x72u || sense->ResponseCode == 0x73u) && length >= 4u) { + sense->Key = bytes[1] & 0x0Fu; + sense->Asc = bytes[2]; + sense->Ascq = bytes[3]; + sense->Valid = true; + } +} + +// Fetches and decodes sense data after a command failure. +static KSTATUS UsbStorageFetchSense(USB_STORAGE_EXTENSION* storage, + USB_STORAGE_SENSE_DATA* sense) { + if (!storage || !storage->XhciDevice || !sense) + return NTLSTATUS_INVALID_PARAMETER; + + XHCI_DEVICE* dev = storage->XhciDevice; + uint8_t cdb[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 }; + memset(dev->MassBuffer, 0, 18); + + uint8_t cswStatus = 0xFFu; + uint32_t residue = 0; + KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, sizeof(cdb), + dev->MassBuffer, dev->MassBufferDeviceAddress, + 18, true, &cswStatus, &residue); + if (!KSUCCESS(st)) { + DbgPrint("usb-storage: REQUEST SENSE failed slot=%u st=0x%x csw_status=%u residue=%u\n", + dev->SlotId, st, cswStatus, residue); + memset(sense, 0, sizeof(*sense)); + return st; } + uint32_t returned = residue < 18u ? 18u - residue : 0u; + UsbStorageDecodeSense(dev->MassBuffer, returned, sense); return NTLSTATUS_SUCCESS; } +// Executes one BOT command and reports SCSI sense data on command failure. +static KSTATUS UsbStorageExecuteCommand(USB_STORAGE_EXTENSION* storage, + const uint8_t* cdb, + uint8_t cdbLen, + void* data, + uint64_t dataDeviceAddress, + uint32_t dataLength, + bool dataIn, + USB_STORAGE_SENSE_DATA* outSense) { + if (outSense) memset(outSense, 0, sizeof(*outSense)); + + uint8_t cswStatus = 0xFFu; + uint32_t residue = 0; + KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, cdbLen, data, + dataDeviceAddress, dataLength, dataIn, + &cswStatus, &residue); + if (KSUCCESS(st)) return st; + + XHCI_DEVICE* dev = storage ? storage->XhciDevice : (XHCI_DEVICE*)0; + uint32_t slot = dev ? dev->SlotId : 0; + if (cswStatus != USB_MASS_BOT_CSW_FAILED || !cdb || cdb[0] == SCSI_REQUEST_SENSE) { + DbgPrint("usb-storage: command %02x failed slot=%u st=0x%x csw_status=%u residue=%u\n", + cdb ? cdb[0] : 0, slot, st, cswStatus, residue); + return st; + } + + USB_STORAGE_SENSE_DATA sense; + KSTATUS senseStatus = UsbStorageFetchSense(storage, &sense); + if (KSUCCESS(senseStatus) && sense.Valid) { + DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=%s key=%02x asc=%02x ascq=%02x\n", + cdb[0], slot, cswStatus, residue, + UsbStorageSenseKeyName(sense.Key), sense.Key, sense.Asc, sense.Ascq); + if (outSense) *outSense = sense; + } else { + DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=unavailable\n", + cdb[0], slot, cswStatus, residue); + } + return st; +} + +// Executes one BOT command through the sense-reporting path. +KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, + const uint8_t* cdb, + uint8_t cdbLen, + void* data, + uint64_t dataDeviceAddress, + uint32_t dataLength, + bool dataIn) { + return UsbStorageExecuteCommand(storage, cdb, cdbLen, data, + dataDeviceAddress, dataLength, dataIn, + (USB_STORAGE_SENSE_DATA*)0); +} + // Implements the be32 operation. uint32_t UsbBe32(const uint8_t* p) { return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | @@ -182,7 +316,7 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) { uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 }; memset(dev->MassBuffer, 0, 36); KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), - dev->MassBuffer, dev->MassBufferPhys, + dev->MassBuffer, dev->MassBufferDeviceAddress, 36, true); if (!KSUCCESS(st)) return st; @@ -195,18 +329,6 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) { return NTLSTATUS_SUCCESS; } -// Implements the storage Request Sense operation. -KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) { - if (!storage || !storage->XhciDevice) - return NTLSTATUS_INVALID_PARAMETER; - - XHCI_DEVICE* dev = storage->XhciDevice; - uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 }; - memset(dev->MassBuffer, 0, 18); - return UsbStorageBotCommand(storage, rs, sizeof(rs), - dev->MassBuffer, dev->MassBufferPhys, 18, true); -} - // Implements the storage Test Unit Ready operation. KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) { if (!storage || !storage->XhciDevice) @@ -222,7 +344,6 @@ KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) { if (KSUCCESS(st)) return NTLSTATUS_SUCCESS; last = st; - (void)UsbStorageRequestSense(storage); UsbDelayUs(xhci->Ddk, 10000u); } @@ -238,7 +359,7 @@ KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) { uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; memset(dev->MassBuffer, 0, 8); KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), - dev->MassBuffer, dev->MassBufferPhys, + dev->MassBuffer, dev->MassBufferDeviceAddress, 8, true); if (!KSUCCESS(st)) return st; @@ -288,7 +409,7 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, memcpy(dev->MassBuffer, buffer, bytes); KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb), - dev->MassBuffer, dev->MassBufferPhys, + dev->MassBuffer, dev->MassBufferDeviceAddress, bytes, !write); if (!KSUCCESS(st)) return st; @@ -297,6 +418,48 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, return NTLSTATUS_SUCCESS; } +// Returns whether sense data means SYNCHRONIZE CACHE is unsupported. +static bool UsbStorageSenseRejectsSynchronizeCache(const USB_STORAGE_SENSE_DATA* sense) { + if (!sense || !sense->Valid || sense->Key != 0x05u) + return false; + return sense->Asc == 0x20u || sense->Asc == 0x24u; +} + +// Flushes device write caches through SCSI SYNCHRONIZE CACHE(10). +static KSTATUS UsbStorageSynchronizeCache(USB_STORAGE_EXTENSION* storage) { + if (!storage || !storage->XhciDevice) + return NTLSTATUS_INVALID_PARAMETER; + if (storage->FlushCapability == UsbStorageFlushUnsupported) + return NTLSTATUS_SUCCESS; + + uint8_t cdb[10]; + memset(cdb, 0, sizeof(cdb)); + cdb[0] = SCSI_SYNCHRONIZE_CACHE_10; + + for (uint32_t attempt = 0; attempt < 2; attempt++) { + USB_STORAGE_SENSE_DATA sense; + KSTATUS st = UsbStorageExecuteCommand(storage, cdb, sizeof(cdb), + (void*)0, 0, 0, false, &sense); + if (KSUCCESS(st)) { + storage->FlushCapability = UsbStorageFlushSupported; + return NTLSTATUS_SUCCESS; + } + + if (UsbStorageSenseRejectsSynchronizeCache(&sense)) { + storage->FlushCapability = UsbStorageFlushUnsupported; + DbgPrint("usb-storage: slot=%u does not support SYNCHRONIZE CACHE; flushes use BOT command-completion semantics only\n", + storage->XhciDevice->SlotId); + return NTLSTATUS_SUCCESS; + } + + if (sense.Valid && sense.Key == 0x06u && attempt == 0) + continue; + return st; + } + + return NTLSTATUS_DEVICE_ERROR; +} + // Implements the storage Acquire Io operation. void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) { if (!storage) return; @@ -375,24 +538,70 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) { return NTLSTATUS_PENDING; } +// Flushes one USB mass-storage disk synchronously. +static KSTATUS UsbStorageDispatchFlush(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) + return NTLSTATUS_INVALID_PARAMETER; + + USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device); + if (!storage || !storage->RemoveLockReady) + return NTLSTATUS_OBJECT_DEAD; + + KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp); + if (!KSUCCESS(st)) + return st; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + return NTLSTATUS_INVALID_PARAMETER; + } + + if (!storage->Started || storage->SurpriseRemoved || storage->Removed) { + IoReleaseRemoveLock(&storage->RemoveLock, irp); + IoCompleteRequest(irp, NTLSTATUS_OBJECT_DEAD, 0); + return NTLSTATUS_PENDING; + } + + UsbStorageAcquireIo(storage); + st = UsbStorageSynchronizeCache(storage); + UsbStorageReleaseIo(storage); + + IoReleaseRemoveLock(&storage->RemoveLock, irp); + IoCompleteRequest(irp, st, 0); + return NTLSTATUS_PENDING; +} + // Creates mass Storage Memory. KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer) return NTLSTATUS_SUCCESS; - dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingPhys); - dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingPhys); - dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferPhys); - if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer) + dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingDeviceAddress, &dev->MassInRingDma); + dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingDeviceAddress, &dev->MassOutRingDma); + dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferDeviceAddress, &dev->MassBufferDma); + if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer) { + if (gUsbDdk && gUsbDdk->Dma) { + gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma); + } + dev->MassBufferDma = (IO_DMA_BUFFER*)0; + dev->MassOutRingDma = (IO_DMA_BUFFER*)0; + dev->MassInRingDma = (IO_DMA_BUFFER*)0; + dev->MassBuffer = (uint8_t*)0; + dev->MassOutRing = (XHCI_TRB*)0; + dev->MassInRing = (XHCI_TRB*)0; return NTLSTATUS_NO_MEMORY; + } dev->MassInEnqueue = 0; dev->MassInCycle = 1; dev->MassOutEnqueue = 0; dev->MassOutCycle = 1; dev->MassTag = 0x1000u + dev->SlotId; - XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingPhys, XHCI_BULK_RING_TRBS, dev->MassInCycle); - XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingPhys, XHCI_BULK_RING_TRBS, dev->MassOutCycle); + XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassInCycle); + XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassOutCycle); return NTLSTATUS_SUCCESS; } @@ -431,21 +640,21 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de epIn[0] = 0; epIn[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_IN | ((uint32_t)inMaxPacket << 16); - epIn[2] = (uint32_t)(dev->MassInRingPhys | XHCI_EP_DCS); - epIn[3] = (uint32_t)((dev->MassInRingPhys | XHCI_EP_DCS) >> 32); + epIn[2] = (uint32_t)(dev->MassInRingDeviceAddress | XHCI_EP_DCS); + epIn[3] = (uint32_t)((dev->MassInRingDeviceAddress | XHCI_EP_DCS) >> 32); epIn[4] = inMaxPacket; epOut[0] = 0; epOut[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_OUT | ((uint32_t)outMaxPacket << 16); - epOut[2] = (uint32_t)(dev->MassOutRingPhys | XHCI_EP_DCS); - epOut[3] = (uint32_t)((dev->MassOutRingPhys | XHCI_EP_DCS) >> 32); + epOut[2] = (uint32_t)(dev->MassOutRingDeviceAddress | XHCI_EP_DCS); + epOut[3] = (uint32_t)((dev->MassOutRingDeviceAddress | XHCI_EP_DCS) >> 32); epOut[4] = outMaxPacket; XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -549,6 +758,7 @@ static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSIO storage->BulkInMaxPacket = bulkIn->wMaxPacketSize; storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize; storage->Tag = 0x1000u + storage->Device->SlotId; + storage->FlushCapability = UsbStorageFlushUnknown; xdev->MassCandidate = true; xdev->MassInterface = storage->InterfaceNumber; @@ -844,6 +1054,7 @@ KSTATUS UsbInitializeStorageDriver(void) { gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice; gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite; gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite; + gUsbStorageDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = UsbStorageDispatchFlush; gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp; return IoRegisterDriverObject(&gUsbStorageDriver); } diff --git a/code/drivers/bus/usb/usb_xhci.c b/code/drivers/bus/usb/usb_xhci.c index 842b319..a62ca5a 100644 --- a/code/drivers/bus/usb/usb_xhci.c +++ b/code/drivers/bus/usb/usb_xhci.c @@ -73,32 +73,105 @@ bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, return false; } -void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, - PHYSICAL_ADDRESS* outPhys) { - if (!outPhys || pages == 0) - return NULL; +// Allocates page-aligned common-buffer memory from the controller adapter. +void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer) { + if (!xhci || !xhci->DmaAdapter || !outDeviceAddress || !outBuffer || pages == 0 || + pages > ((size_t)-1) / PAGE_SIZE) + return (void*)0; - // Most xHCI runtime objects are one page. Do not route those through the - // physically-contiguous allocator: on real machines that allocator is a - // scarce DMA primitive, while a single frame is already contiguous. QEMU - // hid this because it had simple low memory; laptop firmware does not. - PHYSICAL_ADDRESS phys = 0; - if (pages == 1) { - phys = MmPfnAllocate(MmPfnUseDma); - } else { - phys = MmPfnAllocateContiguous(pages, MmPfnUseDma); + IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0; + KSTATUS status = xhci->Ddk->Dma->AllocateCommonBuffer(xhci->DmaAdapter, + pages * (size_t)PAGE_SIZE, + PAGE_SIZE, + &buffer); + if (!KSUCCESS(status)) { + DbgPrint("usb: DMA allocation failed pages=%llu st=0x%x\n", + (unsigned long long)pages, status); + return (void*)0; } - if (!phys) { - DbgPrint("usb: DMA allocation failed pages=%llu\n", - (unsigned long long)pages); - return NULL; + *outDeviceAddress = xhci->Ddk->Dma->GetCommonBufferDeviceAddress(buffer); + *outBuffer = buffer; + return xhci->Ddk->Dma->GetCommonBufferAddress(buffer); +} + +// Releases all DMA allocations owned by one xHCI slot. +void XhciReleaseDeviceDma(XHCI_DEVICE* dev) { + if (!dev || !gUsbDdk || !gUsbDdk->Dma) + return; + + for (uint32_t endpoint = 1; endpoint < 32; endpoint++) { + USB_PIPE* pipe = dev->EndpointPipes[endpoint]; + dev->EndpointPipes[endpoint] = (USB_PIPE*)0; + if (pipe) + UsbDereferencePipe(pipe); } - void* virt = xhci->Ddk->Memory->PhysToVirt(phys); - memset(virt, 0, pages * (size_t)PAGE_SIZE); - *outPhys = phys; - return virt; + gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->DescriptorDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->Ep0RingDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->DeviceContextDma); + gUsbDdk->Dma->FreeCommonBuffer(dev->InputContextDma); + + dev->MassBufferDma = (IO_DMA_BUFFER*)0; + dev->HidBufferDma = (IO_DMA_BUFFER*)0; + dev->DescriptorDma = (IO_DMA_BUFFER*)0; + dev->MassOutRingDma = (IO_DMA_BUFFER*)0; + dev->MassInRingDma = (IO_DMA_BUFFER*)0; + dev->HidRingDma = (IO_DMA_BUFFER*)0; + dev->Ep0RingDma = (IO_DMA_BUFFER*)0; + dev->DeviceContextDma = (IO_DMA_BUFFER*)0; + dev->InputContextDma = (IO_DMA_BUFFER*)0; + + dev->MassBuffer = (uint8_t*)0; + dev->HidBuffer = (uint8_t*)0; + dev->DescriptorBuffer = (uint8_t*)0; + dev->MassOutRing = (XHCI_TRB*)0; + dev->MassInRing = (XHCI_TRB*)0; + dev->HidRing = (XHCI_TRB*)0; + dev->Ep0Ring = (XHCI_TRB*)0; + dev->DeviceContext = (uint8_t*)0; + dev->InputContext = (uint8_t*)0; +} + +// Releases controller-wide xHCI DMA allocations. +static void XhciReleaseControllerDma(XHCI_CONTROLLER* xhci) { + if (!xhci || !xhci->Ddk || !xhci->Ddk->Dma) + return; + + for (uint32_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++) + XhciReleaseDeviceDma(&xhci->Devices[slot]); + for (uint16_t i = 0; i < XHCI_MAX_SCRATCHPADS; i++) { + xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadDma[i]); + xhci->ScratchpadDma[i] = (IO_DMA_BUFFER*)0; + xhci->ScratchpadDeviceAddress[i] = 0; + } + + xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadArrayDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->ErstDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->EventRingDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->CommandRingDma); + xhci->Ddk->Dma->FreeCommonBuffer(xhci->DcbaaDma); + xhci->ScratchpadArrayDma = (IO_DMA_BUFFER*)0; + xhci->ErstDma = (IO_DMA_BUFFER*)0; + xhci->EventRingDma = (IO_DMA_BUFFER*)0; + xhci->CommandRingDma = (IO_DMA_BUFFER*)0; + xhci->DcbaaDma = (IO_DMA_BUFFER*)0; + xhci->ScratchpadArray = (uint64_t*)0; + xhci->Erst = (XHCI_ERST_ENTRY*)0; + xhci->EventRing = (XHCI_TRB*)0; + xhci->CommandRing = (XHCI_TRB*)0; + xhci->Dcbaa = (uint64_t*)0; + + xhci->Ddk->Dma->PutAdapter(xhci->DmaAdapter); + xhci->DmaAdapter = (IO_DMA_ADAPTER*)0; } // Implements the scratchpad Count operation. @@ -328,26 +401,26 @@ void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) { // Updates erdp. void XhciUpdateErdp(XHCI_CONTROLLER* xhci) { volatile uint8_t* ir0 = xhci->Runtime + 0x20u; - uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB); + uint64_t erdp = xhci->EventRingDeviceAddress + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB); XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear } // Returns command Waiter Locked. -static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) { +static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandDeviceAddress) { for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { - if (w->CommandPhys && w->CommandPhys == commandPhys) + if (w->CommandDeviceAddress && w->CommandDeviceAddress == commandDeviceAddress) return w; } return (XHCI_WAIT_TARGET*)0; } static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci, - uint64_t transferPhys, + uint64_t transferDeviceAddress, uint8_t slotId, uint8_t endpointId) { for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) { - if (w->TransferPhys && - w->TransferPhys == transferPhys && + if (w->TransferDeviceAddress && + w->TransferDeviceAddress == transferDeviceAddress && w->TransferSlotId == slotId && w->TransferEndpointId == endpointId) { return w; @@ -357,11 +430,11 @@ static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci, } static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci, - uint64_t commandPhys, + uint64_t commandDeviceAddress, uint32_t completionCode, uint8_t slotId) { uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); - XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys); + XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandDeviceAddress); if (w) { w->Command.CompletionCode = completionCode; w->Command.SlotId = slotId; @@ -373,13 +446,13 @@ static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci, } static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci, - uint64_t transferPhys, + uint64_t transferDeviceAddress, uint8_t slotId, uint8_t endpointId, uint32_t completionCode, uint32_t residue) { uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock); - XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId); + XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferDeviceAddress, slotId, endpointId); if (w) { w->Transfer.CompletionCode = completionCode; w->Transfer.SlotId = slotId; @@ -397,7 +470,7 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { (void)target; bool processed = false; - for (;;) { + while (true) { XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue]; uint32_t control = ev->Control; uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE); @@ -460,8 +533,8 @@ bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) { // Waits for target Complete. bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) { if (!target) return false; - if (target->CommandPhys && !target->CommandMatched) return false; - if (target->TransferPhys && !target->TransferMatched) return false; + if (target->CommandDeviceAddress && !target->CommandMatched) return false; + if (target->TransferDeviceAddress && !target->TransferMatched) return false; return true; } @@ -568,6 +641,21 @@ static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) { return NTLSTATUS_SUCCESS; } +// Removes a controller from the interrupt-dispatch registry. +static void XhciUnregisterController(XHCI_CONTROLLER* xhci) { + if (!xhci) + return; + for (uint32_t i = 0; i < gXhciControllerCount; i++) { + if (gXhciControllers[i] != xhci) + continue; + for (uint32_t next = i + 1; next < gXhciControllerCount; next++) + gXhciControllers[next - 1] = gXhciControllers[next]; + gXhciControllerCount--; + gXhciControllers[gXhciControllerCount] = (XHCI_CONTROLLER*)0; + return; + } +} + // Implements the isr operation. void XhciIsr(void) { for (uint32_t i = 0; i < gXhciControllerCount; i++) { @@ -663,6 +751,34 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) { return NTLSTATUS_SUCCESS; } +// Halts the controller before any device-visible memory can be released. +static void XhciHaltForDmaRelease(XHCI_CONTROLLER* xhci) { + if (!xhci || !xhci->DmaAdapter) + return; + + xhci->InterruptsEnabled = false; + xhci->Running = false; + if (xhci->SurpriseRemoved || !xhci->Op) + return; + + if (xhci->Runtime) { + volatile uint8_t* interrupter = xhci->Runtime + 0x20u; + XhciWrite32(interrupter, 0x00u, XHCI_IMAN_IP); + } + + uint32_t command = XhciRead32(xhci->Op, XHCI_OP_USBCMD); + command &= ~(XHCI_USBCMD_RS | XHCI_USBCMD_INTE); + XhciWrite32(xhci->Op, XHCI_OP_USBCMD, command); + if (!UsbWaitSet32(xhci->Ddk, + xhci->Op, + XHCI_OP_USBSTS, + XHCI_USBSTS_HCH, + 1000000u)) + KePanic(KE_PANIC_CODE_DRIVER, + "usb: xHCI failed to halt before DMA release usbsts=%08x\n", + XhciRead32(xhci->Op, XHCI_OP_USBSTS)); +} + // Creates scratchpads. KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { if (xhci->ScratchpadCount == 0) return NTLSTATUS_SUCCESS; @@ -672,7 +788,7 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { return NTLSTATUS_NOT_SUPPORTED; } - xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayPhys); + xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayDeviceAddress, &xhci->ScratchpadArrayDma); if (!xhci->ScratchpadArray) { DbgPrint("usb: xHCI scratchpad pointer-array allocation failed count=%u\n", xhci->ScratchpadCount); @@ -680,29 +796,29 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) { } for (uint16_t i = 0; i < xhci->ScratchpadCount; i++) { - void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadPhys[i]); + void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadDeviceAddress[i], &xhci->ScratchpadDma[i]); if (!page) { DbgPrint("usb: xHCI scratchpad buffer allocation failed index=%u/%u\n", i, xhci->ScratchpadCount); return NTLSTATUS_NO_MEMORY; } (void)page; - xhci->ScratchpadArray[i] = xhci->ScratchpadPhys[i]; + xhci->ScratchpadArray[i] = xhci->ScratchpadDeviceAddress[i]; } - xhci->Dcbaa[0] = xhci->ScratchpadArrayPhys; + xhci->Dcbaa[0] = xhci->ScratchpadArrayDeviceAddress; UsbTrace("usb: xHCI allocated %u scratchpad buffer(s) array=0x%016llX\n", xhci->ScratchpadCount, - (unsigned long long)xhci->ScratchpadArrayPhys); + (unsigned long long)xhci->ScratchpadArrayDeviceAddress); return NTLSTATUS_SUCCESS; } // Creates runtime Memory. KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { - xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys); - xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys); - xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingPhys); - xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstPhys); + xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaDeviceAddress, &xhci->DcbaaDma); + xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingDeviceAddress, &xhci->CommandRingDma); + xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingDeviceAddress, &xhci->EventRingDma); + xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstDeviceAddress, &xhci->ErstDma); if (!xhci->Dcbaa || !xhci->CommandRing || !xhci->EventRing || !xhci->Erst) { DbgPrint("usb: xHCI runtime DMA allocation failed dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", @@ -716,14 +832,14 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { xhci->EventCycle = 1; XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)xhci->CommandRingPhys; - link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); + link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress; + link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | xhci->CommandCycle; - xhci->Erst[0].RingSegmentBase = xhci->EventRingPhys; + xhci->Erst[0].RingSegmentBase = xhci->EventRingDeviceAddress; xhci->Erst[0].RingSegmentSize = XHCI_EVENT_TRBS; xhci->Erst[0].Reserved = 0; @@ -731,10 +847,10 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) { if (!KSUCCESS(st)) return st; UsbTrace("usb: xHCI dma dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n", - (unsigned long long)xhci->DcbaaPhys, - (unsigned long long)xhci->CommandRingPhys, - (unsigned long long)xhci->EventRingPhys, - (unsigned long long)xhci->ErstPhys); + (unsigned long long)xhci->DcbaaDeviceAddress, + (unsigned long long)xhci->CommandRingDeviceAddress, + (unsigned long long)xhci->EventRingDeviceAddress, + (unsigned long long)xhci->ErstDeviceAddress); return NTLSTATUS_SUCCESS; } @@ -743,8 +859,8 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) { uint8_t slots = xhci->MaxSlots; if (slots == 0) return NTLSTATUS_DEVICE_ERROR; - XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaPhys); - XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingPhys | xhci->CommandCycle); + XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaDeviceAddress); + XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingDeviceAddress | xhci->CommandCycle); XhciWrite32(xhci->Op, XHCI_OP_DNCTRL, 0); XhciWrite32(xhci->Op, XHCI_OP_CONFIG, slots); @@ -752,30 +868,30 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) { XhciWrite32(ir0, 0x00u, XHCI_IMAN_IP); // clear pending, leave disabled until smoke test passes XhciWrite32(ir0, 0x04u, 0); // no interrupt moderation yet XhciWrite32(ir0, 0x08u, 1); // one ERST entry - XhciWrite64(ir0, 0x10u, xhci->ErstPhys); - XhciWrite64(ir0, 0x18u, xhci->EventRingPhys); + XhciWrite64(ir0, 0x10u, xhci->ErstDeviceAddress); + XhciWrite64(ir0, 0x18u, xhci->EventRingDeviceAddress); XhciBarrier(); return NTLSTATUS_SUCCESS; } -// Implements the command Trb Phys operation. -uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) { - return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB); +// Returns the device address of one command TRB. +uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index) { + return xhci->CommandRingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB); } static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, - uint64_t* outPhys) { + uint64_t* outDeviceAddress) { if (!xhci || !xhci->CommandRing) return NTLSTATUS_INVALID_PARAMETER; if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) { XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)xhci->CommandRingPhys; - link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32); + link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress; + link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | @@ -787,7 +903,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, uint32_t index = xhci->CommandEnqueue++; XHCI_TRB* trb = &xhci->CommandRing[index]; - uint64_t trbPhys = XhciCommandTrbPhys(xhci, index); + uint64_t trbDeviceAddress = XhciCommandTrbDeviceAddress(xhci, index); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); @@ -795,7 +911,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci, XhciBarrier(); trb->Control = controlWithoutCycle | xhci->CommandCycle; - if (outPhys) *outPhys = trbPhys; + if (outDeviceAddress) *outDeviceAddress = trbDeviceAddress; return NTLSTATUS_SUCCESS; } @@ -811,14 +927,14 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, memset(&target, 0, sizeof(target)); uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock); - uint64_t trbPhys = 0; - KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys); + uint64_t trbDeviceAddress = 0; + KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbDeviceAddress); if (!KSUCCESS(st)) { KeReleaseSpinlock(&xhci->CommandLock, cmdFlags); return st; } - target.CommandPhys = trbPhys; + target.CommandDeviceAddress = trbDeviceAddress; XhciRegisterWaitTarget(xhci, &target); XhciBarrier(); @@ -912,7 +1028,7 @@ KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { memset(&result, 0, sizeof(result)); KSTATUS st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_ADDRESS_DEVICE << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -1059,20 +1175,22 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) { // Creates device Memory. KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { - dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys); - dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys); - dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingPhys); - dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorPhys); + dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextDeviceAddress, &dev->InputContextDma); + dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextDeviceAddress, &dev->DeviceContextDma); + dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingDeviceAddress, &dev->Ep0RingDma); + dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorDeviceAddress, &dev->DescriptorDma); - if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) + if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) { + XhciReleaseDeviceDma(dev); return NTLSTATUS_NO_MEMORY; + } dev->Ep0Enqueue = 0; dev->Ep0Cycle = 1; XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)dev->Ep0RingPhys; - link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); + link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress; + link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | @@ -1116,31 +1234,31 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) { ep0[0] = 0; ep0[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_CONTROL | (maxPacket << 16); - ep0[2] = (uint32_t)(dev->Ep0RingPhys | XHCI_EP_DCS); - ep0[3] = (uint32_t)((dev->Ep0RingPhys | XHCI_EP_DCS) >> 32); + ep0[2] = (uint32_t)(dev->Ep0RingDeviceAddress | XHCI_EP_DCS); + ep0[3] = (uint32_t)((dev->Ep0RingDeviceAddress | XHCI_EP_DCS) >> 32); ep0[4] = 8; - xhci->Dcbaa[dev->SlotId] = dev->DeviceContextPhys; + xhci->Dcbaa[dev->SlotId] = dev->DeviceContextDeviceAddress; XhciBarrier(); } -// Implements the ep0 Trb Phys operation. -uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) { - return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB); +// Returns the device address of one endpoint-zero TRB. +uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index) { + return dev->Ep0RingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB); } KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, - uint64_t* outPhys) { + uint64_t* outDeviceAddress) { if (!dev || !dev->Ep0Ring) return NTLSTATUS_INVALID_PARAMETER; if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) { XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u]; - link->ParameterLo = (uint32_t)dev->Ep0RingPhys; - link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32); + link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress; + link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | @@ -1152,7 +1270,7 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint32_t index = dev->Ep0Enqueue++; XHCI_TRB* trb = &dev->Ep0Ring[index]; - uint64_t phys = XhciEp0TrbPhys(dev, index); + uint64_t deviceAddress = XhciEp0TrbDeviceAddress(dev, index); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); @@ -1160,14 +1278,14 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, XhciBarrier(); trb->Control = controlWithoutCycle | dev->Ep0Cycle; - if (outPhys) *outPhys = phys; + if (outDeviceAddress) *outDeviceAddress = deviceAddress; return NTLSTATUS_SUCCESS; } KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint16_t length, uint32_t timeoutUs) { if (!buffer || length == 0 || length > PAGE_SIZE) @@ -1188,26 +1306,26 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, st = XhciEp0AppendTrb( dev, - bufferPhys, + bufferDeviceAddress, length, (XHCI_TRB_TYPE_DATA << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_DATA_DIR_IN, NULL); if (!KSUCCESS(st)) return st; - uint64_t statusTrbPhys = 0; + uint64_t statusTrbDeviceAddress = 0; st = XhciEp0AppendTrb( dev, 0, 0, (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_IOC, - &statusTrbPhys); + &statusTrbDeviceAddress); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); - target.TransferPhys = statusTrbPhys; + target.TransferDeviceAddress = statusTrbDeviceAddress; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; XhciRegisterWaitTarget(xhci, &target); @@ -1235,9 +1353,9 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint16_t length) { - return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length, + return XhciControlInTimeout(xhci, dev, setup, buffer, bufferDeviceAddress, length, USB_XHCI_CONTROL_TIMEOUT_US); } @@ -1254,7 +1372,7 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, NULL); if (!KSUCCESS(st)) return st; - uint64_t statusTrbPhys = 0; + uint64_t statusTrbDeviceAddress = 0; st = XhciEp0AppendTrb( dev, 0, @@ -1262,12 +1380,12 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, (XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_STATUS_DIR_IN | XHCI_TRB_IOC, - &statusTrbPhys); + &statusTrbDeviceAddress); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); - target.TransferPhys = statusTrbPhys; + target.TransferDeviceAddress = statusTrbDeviceAddress; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = 1; XhciRegisterWaitTarget(xhci, &target); @@ -1304,7 +1422,7 @@ KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US; KSTATUS st = XhciControlInTimeout(xhci, dev, &setup, dev->DescriptorBuffer, - dev->DescriptorPhys, + dev->DescriptorDeviceAddress, sizeof(USB_DEVICE_DESCRIPTOR), timeoutUs); if (!KSUCCESS(st)) return st; @@ -1339,7 +1457,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, KSTATUS st = XhciControlIn(xhci, dev, &setup, dev->DescriptorBuffer, - dev->DescriptorPhys, + dev->DescriptorDeviceAddress, sizeof(USB_CONFIGURATION_DESCRIPTOR)); if (!KSUCCESS(st)) return st; @@ -1366,7 +1484,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, setup.wLength = total; st = XhciControlIn(xhci, dev, &setup, dev->DescriptorBuffer, - dev->DescriptorPhys, + dev->DescriptorDeviceAddress, total); if (!KSUCCESS(st)) return st; @@ -1415,17 +1533,17 @@ uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) { return interval; } -KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, +KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, - uint32_t controlWithoutCycle, uint64_t* outPhys) { + uint32_t controlWithoutCycle, uint64_t* outDeviceAddress) { if (!ring || !enqueue || !cycle || trbCount < 2) return NTLSTATUS_INVALID_PARAMETER; if (*enqueue >= trbCount - 1u) { XHCI_TRB* link = &ring[trbCount - 1u]; - link->ParameterLo = (uint32_t)ringPhys; - link->ParameterHi = (uint32_t)(ringPhys >> 32); + link->ParameterLo = (uint32_t)ringDeviceAddress; + link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32); link->Status = 0; XhciBarrier(); link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | @@ -1437,7 +1555,7 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t index = (*enqueue)++; XHCI_TRB* trb = &ring[index]; - uint64_t phys = ringPhys + (uint64_t)index * sizeof(XHCI_TRB); + uint64_t deviceAddress = ringDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB); trb->ParameterLo = (uint32_t)parameter; trb->ParameterHi = (uint32_t)(parameter >> 32); @@ -1445,16 +1563,16 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, XhciBarrier(); trb->Control = controlWithoutCycle | *cycle; - if (outPhys) *outPhys = phys; + if (outDeviceAddress) *outDeviceAddress = deviceAddress; return NTLSTATUS_SUCCESS; } -void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, +void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint8_t cycle) { memset(ring, 0, trbCount * sizeof(XHCI_TRB)); XHCI_TRB* link = &ring[trbCount - 1u]; - link->ParameterLo = (uint32_t)ringPhys; - link->ParameterHi = (uint32_t)(ringPhys >> 32); + link->ParameterLo = (uint32_t)ringDeviceAddress; + link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32); link->Status = 0; link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_LINK_TC | @@ -1464,34 +1582,34 @@ void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, - PHYSICAL_ADDRESS ringPhys, + uint64_t ringDeviceAddress, uint32_t* enqueue, uint8_t* cycle, - PHYSICAL_ADDRESS bufferPhys, + uint64_t bufferDeviceAddress, uint32_t length, uint32_t* outActual) { - if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferPhys || length == 0) + if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferDeviceAddress || length == 0) return NTLSTATUS_INVALID_PARAMETER; if (length > USB_STORAGE_MAX_BYTES) return NTLSTATUS_INVALID_PARAMETER; - uint64_t trbPhys = 0; + uint64_t trbDeviceAddress = 0; KSTATUS st = XhciRingAppend(ring, - ringPhys, + ringDeviceAddress, XHCI_BULK_RING_TRBS, enqueue, cycle, - bufferPhys, + bufferDeviceAddress, length, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, - &trbPhys); + &trbDeviceAddress); if (!KSUCCESS(st)) return st; XHCI_WAIT_TARGET target; memset(&target, 0, sizeof(target)); - target.TransferPhys = trbPhys; + target.TransferDeviceAddress = trbDeviceAddress; target.TransferSlotId = dev->SlotId; target.TransferEndpointId = endpointId; @@ -1537,7 +1655,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P if (!ctx) return NTLSTATUS_NO_MEMORY; memset(ctx, 0, sizeof(*ctx)); - ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys); + ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingDeviceAddress, &ctx->RingDma); if (!ctx->Ring) { MmFreePool(ctx); return NTLSTATUS_NO_MEMORY; @@ -1546,7 +1664,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P ctx->Cycle = 1; ctx->EndpointId = dci; ctx->Type = pipe->Type; - XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle); + XhciInitialiseTransferRing(ctx->Ring, ctx->RingDeviceAddress, XHCI_PIPE_RING_TRBS, ctx->Cycle); memset(dev->InputContext, 0, PAGE_SIZE); uint32_t* icc = XhciInputControlContext(xhci, dev); @@ -1574,21 +1692,22 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P } else if (pipe->Type == UsbPipeBulk) { endpointType = XHCI_EP_TYPE_BULK_OUT; } else { + xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma); MmFreePool(ctx); return NTLSTATUS_NOT_SUPPORTED; } ep[0] = (uint32_t)interval << 16; ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16); - ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS); - ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32); + ep[2] = (uint32_t)(ctx->RingDeviceAddress | XHCI_EP_DCS); + ep[3] = (uint32_t)((ctx->RingDeviceAddress | XHCI_EP_DCS) >> 32); ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16); XHCI_COMMAND_RESULT result; memset(&result, 0, sizeof(result)); KSTATUS st = XhciSubmitCommand( xhci, - dev->InputContextPhys, + dev->InputContextDeviceAddress, 0, (XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) | ((uint32_t)dev->SlotId << 24), @@ -1596,6 +1715,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) { DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n", dev->SlotId, dci, pipe->Type, st, result.CompletionCode); + xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma); MmFreePool(ctx); return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; } @@ -1609,7 +1729,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P // Implements the hcd Submit Interrupt Urb operation. static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { - if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) + if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0) return NTLSTATUS_INVALID_PARAMETER; if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN)) return NTLSTATUS_NOT_SUPPORTED; @@ -1623,22 +1743,22 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev if (ctx->ActiveTransfer) return NTLSTATUS_QUEUE_FULL; - uint64_t trbPhys = 0; + uint64_t trbDeviceAddress = 0; st = XhciRingAppend(ctx->Ring, - ctx->RingPhys, + ctx->RingDeviceAddress, XHCI_PIPE_RING_TRBS, &ctx->Enqueue, &ctx->Cycle, - urb->BufferPhys, + urb->BufferDeviceAddress, urb->Length, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_ISP | XHCI_TRB_IOC, - &trbPhys); + &trbDeviceAddress); if (!KSUCCESS(st)) return st; ctx->ActiveTransfer = true; - ctx->ActiveTrbPhys = trbPhys; + ctx->ActiveTrbDeviceAddress = trbDeviceAddress; ctx->ActiveUrb = urb; urb->Pending = true; urb->Cancelled = false; @@ -1651,7 +1771,7 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev return NTLSTATUS_PENDING; } -void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, +void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue) { if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32) @@ -1663,12 +1783,12 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, return; XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; - if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys) + if (!ctx->ActiveTransfer || ctx->ActiveTrbDeviceAddress != trbDeviceAddress) return; USB_URB* urb = ctx->ActiveUrb; ctx->ActiveTransfer = false; - ctx->ActiveTrbPhys = 0; + ctx->ActiveTrbDeviceAddress = 0; ctx->ActiveUrb = (USB_URB*)0; if (!urb) @@ -1704,23 +1824,23 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, // Implements the hcd Submit Bulk Urb operation. static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) { - if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0) + if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0) return NTLSTATUS_INVALID_PARAMETER; uint8_t endpointId = urb->Pipe->EndpointId; XHCI_TRB* ring = (XHCI_TRB*)0; - PHYSICAL_ADDRESS ringPhys = 0; + uint64_t ringDeviceAddress = 0; uint32_t* enqueue = (uint32_t*)0; uint8_t* cycle = (uint8_t*)0; if (endpointId == dev->MassBulkInEndpointId) { ring = dev->MassInRing; - ringPhys = dev->MassInRingPhys; + ringDeviceAddress = dev->MassInRingDeviceAddress; enqueue = &dev->MassInEnqueue; cycle = &dev->MassInCycle; } else if (endpointId == dev->MassBulkOutEndpointId) { ring = dev->MassOutRing; - ringPhys = dev->MassOutRingPhys; + ringDeviceAddress = dev->MassOutRingDeviceAddress; enqueue = &dev->MassOutEnqueue; cycle = &dev->MassOutCycle; } else { @@ -1732,10 +1852,10 @@ static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB dev, endpointId, ring, - ringPhys, + ringDeviceAddress, enqueue, cycle, - urb->BufferPhys, + urb->BufferDeviceAddress, urb->Length, &actual); urb->ActualLength = KSUCCESS(st) ? actual : 0; @@ -1759,7 +1879,7 @@ static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) { st = XhciControlNoData(xhci, dev, &urb->Setup); urb->ActualLength = 0; } else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) { - st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length); + st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferDeviceAddress, (uint16_t)urb->Length); urb->ActualLength = KSUCCESS(st) ? urb->Length : 0; } else { st = NTLSTATUS_NOT_SUPPORTED; @@ -1796,9 +1916,29 @@ static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) { } // Implements the hcd Allocate Dma operation. -static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) { +static void* XhciHcdAllocateDma(USB_HCD* hcd, + size_t pages, + uint64_t* outDeviceAddress, + IO_DMA_BUFFER** outBuffer) { if (!hcd) return (void*)0; - return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys); + return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, + pages, + outDeviceAddress, + outBuffer); +} + +// Releases the host-controller context attached to a USB pipe. +static void XhciHcdDestroyPipe(USB_HCD* hcd, USB_PIPE* pipe) { + (void)hcd; + if (!pipe || !pipe->HcdPipeContext) + return; + XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext; + if (gUsbDdk && gUsbDdk->Dma) + gUsbDdk->Dma->FreeCommonBuffer(ctx->RingDma); + ctx->RingDma = (IO_DMA_BUFFER*)0; + ctx->Ring = (XHCI_TRB*)0; + pipe->HcdPipeContext = (void*)0; + MmFreePool(ctx); } // Implements the hcd Reset Root Port operation. @@ -1819,6 +1959,7 @@ const USB_HCD_OPERATIONS gXhciHcdOperations = { XhciHcdCancelUrb, XhciHcdResetRootPort, XhciHcdClearRootPortChanges, + XhciHcdDestroyPipe, XhciHcdAllocateDma, }; @@ -1886,7 +2027,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { uint64_t mappedLength = 0; st = PciMapBar(xhci->Pdo, 0, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&xhci->Mmio, &mappedLength); if (!KSUCCESS(st)) { @@ -1960,17 +2101,37 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { return NTLSTATUS_NOT_SUPPORTED; } + IO_DMA_REQUIREMENTS dmaRequirements; + memset(&dmaRequirements, 0, sizeof(dmaRequirements)); + dmaRequirements.MaximumDeviceAddress = + (xhci->Hcc1 & XHCI_HCC1_AC64) ? UINT64_MAX : UINT32_MAX; + dmaRequirements.Alignment = 1; + dmaRequirements.Boundary = 0x10000u; + dmaRequirements.MaximumSegmentLength = 0x10000u; + dmaRequirements.MaximumSegments = XHCI_PIPE_RING_TRBS - 1u; + st = xhci->Ddk->Dma->GetAdapter(xhci->Fdo, + &dmaRequirements, + &xhci->DmaAdapter); + if (!KSUCCESS(st)) { + DbgPrint("usb: xHCI DMA adapter creation failed st=0x%x\n", st); + return st; + } + st = XhciAllocateRuntimeMemory(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; st = XhciProgramRuntime(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; st = XhciStart(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; st = XhciSubmitNoopCommand(xhci); - if (!KSUCCESS(st)) return st; + if (!KSUCCESS(st)) + goto failDma; XhciPowerRootPorts(xhci); XhciLogPortStatus(xhci); @@ -1978,13 +2139,13 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { st = UsbCreateRootHubPdo(&xhci->Hcd); if (!KSUCCESS(st) && st != NTLSTATUS_ALREADY_EXISTS) { DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st); - return st; + goto failDma; } st = XhciRegisterController(xhci); if (!KSUCCESS(st)) { DbgPrint("usb: xHCI controller registry full st=0x%x\n", st); - return st; + goto failDma; } if (xhci->Irq != 0xFFu) { @@ -2009,6 +2170,16 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) { DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n", xhci->HidDeviceCount); return NTLSTATUS_SUCCESS; + +failDma: + XhciHaltForDmaRelease(xhci); + XhciUnregisterController(xhci); + if (xhci->Hcd.RootHubPdo) { + (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); + xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0; + } + XhciReleaseControllerDma(xhci); + return st; } // Dispatches pnp. @@ -2037,22 +2208,22 @@ KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: device->PnpState = DevicePnpSurpriseRemoved; + xhci->SurpriseRemoved = true; xhci->Running = false; xhci->InterruptsEnabled = false; - if (xhci->Op) { - uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD); - XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS); - } if (xhci->Hcd.RootHubPdo) (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: device->PnpState = DevicePnpDeleted; - xhci->Running = false; - xhci->InterruptsEnabled = false; - if (xhci->Hcd.RootHubPdo) + XhciHaltForDmaRelease(xhci); + if (xhci->Hcd.RootHubPdo) { (void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true); + xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0; + } + XhciUnregisterController(xhci); + XhciReleaseControllerDma(xhci); IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; diff --git a/code/drivers/display/vmwaresvga/vmwaresvga.c b/code/drivers/display/vmwaresvga/vmwaresvga.c index 5fc0bae..1d21eb6 100644 --- a/code/drivers/display/vmwaresvga/vmwaresvga.c +++ b/code/drivers/display/vmwaresvga/vmwaresvga.c @@ -748,10 +748,103 @@ static KSTATUS VmwSvgaDisplayGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMo return NTLSTATUS_SUCCESS; } -// Implements the display Blit operation. -static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) { +// Streams one pixel row into the write-combined SVGA framebuffer. +static void VmwSvgaCopyToFramebuffer(uint32_t* destination, + const uint32_t* source, + size_t pixelCount) { + if (!destination || !source || pixelCount == 0) + return; + + while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) { + *destination++ = *source++; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_stream_si128((__m128i*)(destination + 0), v0); + _mm_stream_si128((__m128i*)(destination + 4), v1); + _mm_stream_si128((__m128i*)(destination + 8), v2); + _mm_stream_si128((__m128i*)(destination + 12), v3); + source += 16; + destination += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_stream_si128((__m128i*)destination, value); + source += 4; + destination += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + *destination++ = *source++; + pixelCount--; + } +} + +// Copies one row into the retained SVGA shadow and framebuffer in one read pass. +static void VmwSvgaCopyToShadowAndFramebuffer(uint32_t* shadow, + uint32_t* framebuffer, + const uint32_t* source, + size_t pixelCount) { + if (!shadow || !framebuffer || !source || pixelCount == 0) + return; + + while (pixelCount != 0 && ((uintptr_t)framebuffer & 15u) != 0) { + uint32_t value = *source++; + *shadow++ = value; + *framebuffer++ = value; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_storeu_si128((__m128i*)(shadow + 0), v0); + _mm_storeu_si128((__m128i*)(shadow + 4), v1); + _mm_storeu_si128((__m128i*)(shadow + 8), v2); + _mm_storeu_si128((__m128i*)(shadow + 12), v3); + _mm_stream_si128((__m128i*)(framebuffer + 0), v0); + _mm_stream_si128((__m128i*)(framebuffer + 4), v1); + _mm_stream_si128((__m128i*)(framebuffer + 8), v2); + _mm_stream_si128((__m128i*)(framebuffer + 12), v3); + source += 16; + shadow += 16; + framebuffer += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_storeu_si128((__m128i*)shadow, value); + _mm_stream_si128((__m128i*)framebuffer, value); + source += 4; + shadow += 4; + framebuffer += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + uint32_t value = *source++; + *shadow++ = value; + *framebuffer++ = value; + pixelCount--; + } +} + +// Implements the display pitched blit operation. +static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) { VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device); - if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0) + if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; DISPLAY_RECT r = { x, y, w, h }; @@ -766,12 +859,60 @@ static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, i size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t); for (int row = 0; row < r.H; row++) { uint32_t* dst = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X; - const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX; + const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX; memcpy(dst, src, bytesPerRow); } return NTLSTATUS_SUCCESS; } +// Presents one pitched rectangle without a shadow-to-framebuffer second pass. +static KSTATUS VmwSvgaDisplayPresentPitched(DISPLAY_DEVICE* device, + int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device); + if (!v || !v->Framebuffer || !v->Backbuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) + return NTLSTATUS_INVALID_PARAMETER; + + DISPLAY_RECT r = { x, y, w, h }; + if (!VmwSvgaClipRect(v, &r)) + return NTLSTATUS_SUCCESS; + + int srcOffsetX = r.X - x; + int srcOffsetY = r.Y - y; + size_t pixelCount = (size_t)r.W; + uint64_t bytes = (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t); + v->Stats.BlitCount++; + v->Stats.PresentCount++; + v->Stats.PresentRectCount++; + v->Stats.DirectPresentCount++; + v->Stats.ScanoutBytes += bytes; + v->Stats.BytesCopied += bytes * 2u; + + for (int row = 0; row < r.H; row++) { + const uint8_t* sourceRow = (const uint8_t*)pixels + + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX; + uint32_t* shadow = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X; + uint32_t* framebuffer = (uint32_t*)(uintptr_t)(v->Framebuffer + + ((size_t)(r.Y + row) * v->BytesPerLine) + + ((size_t)r.X * sizeof(uint32_t))); + VmwSvgaCopyToShadowAndFramebuffer(shadow, framebuffer, source, pixelCount); + } + + _mm_sfence(); + _ReadWriteBarrier(); + KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H); + if (!KSUCCESS(st)) + return st; + return VmwSvgaSyncDevice(v); +} + // Implements the display Fill Rect operation. static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) { VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device); @@ -869,13 +1010,19 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* DISPLAY_RECT r = rects[i]; if (!VmwSvgaClipRect(v, &r)) continue; - v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t); - size_t rowBytes = (size_t)r.W * sizeof(uint32_t); + uint64_t bytes = (uint64_t)(uint32_t)r.W * + (uint64_t)(uint32_t)r.H * sizeof(uint32_t); + v->Stats.BytesCopied += bytes; + v->Stats.ScanoutBytes += bytes; + size_t pixelCount = (size_t)r.W; for (int row = 0; row < r.H; row++) { - volatile uint8_t* dstBytes = v->Framebuffer + ((size_t)(r.Y + row) * v->BytesPerLine) + ((size_t)r.X * sizeof(uint32_t)); + uint32_t* dst = (uint32_t*)(uintptr_t)(v->Framebuffer + + ((size_t)(r.Y + row) * v->BytesPerLine) + + ((size_t)r.X * sizeof(uint32_t))); uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X; - memcpy((void*)dstBytes, src, rowBytes); + VmwSvgaCopyToFramebuffer(dst, src, pixelCount); } + _mm_sfence(); _ReadWriteBarrier(); KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H); @@ -1019,6 +1166,7 @@ static KSTATUS VmwSvgaDisplayQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* o static const DISPLAY_DEVICE_OPS gVmwSvgaDisplayOps = { .GetMode = VmwSvgaDisplayGetMode, .Blit = VmwSvgaDisplayBlit, + .PresentPitched = VmwSvgaDisplayPresentPitched, .FillRect = VmwSvgaDisplayFillRect, .CopyRect = VmwSvgaDisplayCopyRect, .ReadRect = VmwSvgaDisplayReadRect, @@ -1060,15 +1208,19 @@ static KSTATUS VmwSvgaSelectBars(VMWSVGA_DEVICE* v) { return NTLSTATUS_SUCCESS; } -// Implements the mmio Page Flags operation. -static uint64_t VmwSvgaMmioPageFlags(void) { - return PAGE_PRESENT | PAGE_WRITE | PAGE_WRITETHROUGH | PAGE_NOCACHE | PAGE_NX; +// Returns the write-combined mapping flags used for the framebuffer aperture. +static uint64_t VmwSvgaFramebufferPageFlags(void) { + return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_WC | PAGE_NX; +} + +// Returns the uncached mapping flags used for device-control apertures. +static uint64_t VmwSvgaRegisterPageFlags(void) { + return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC | PAGE_NX; } // Maps bars. static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) { - uint64_t mmioFlags = VmwSvgaMmioPageFlags(); - KSTATUS st = PciMapBar(v->Pdo, 1, mmioFlags, + KSTATUS st = PciMapBar(v->Pdo, 1, VmwSvgaFramebufferPageFlags(), (void**)&v->FramebufferMapping, &v->FramebufferBytes); if (!KSUCCESS(st) || !v->FramebufferMapping) { DbgPrint("VMWSVGA: PciMapBar framebuffer failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n", @@ -1079,7 +1231,7 @@ static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) { return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st; } - st = PciMapBar(v->Pdo, 2, mmioFlags, + st = PciMapBar(v->Pdo, 2, VmwSvgaRegisterPageFlags(), (void**)&v->Fifo, &v->FifoBytes); if (!KSUCCESS(st) || !v->Fifo) { DbgPrint("VMWSVGA: PciMapBar FIFO failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n", diff --git a/code/drivers/drivers.ini b/code/drivers/drivers.ini index 8fe39f7..6d147b5 100644 --- a/code/drivers/drivers.ini +++ b/code/drivers/drivers.ini @@ -1,5 +1,6 @@ [boot] pci.sys +hda.sys ps2.sys serial.sys ahci.sys diff --git a/code/drivers/fs/fat/fat.c b/code/drivers/fs/fat/fat.c index 948112a..da108f8 100644 --- a/code/drivers/fs/fat/fat.c +++ b/code/drivers/fs/fat/fat.c @@ -80,6 +80,11 @@ static void FatCacheEnsureMutex(void) { gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT; gFatCacheMutex.Owner = (struct THREAD*)0; LIST_INIT(&gFatCacheMutex.WaitList); + LIST_INIT(&gFatCacheMutex.OwnerLink); + gFatCacheMutex.DonationClass = KeSchedulingNormal; + gFatCacheMutex.DonationPriority = PriorityLow; + gFatCacheMutex.HasDonation = 0; + gFatCacheMutex.Abandoned = 0; gFatCacheMutexReady = 1; } KeReleaseSpinlock(&gFatCacheMutexInitLock, flags); @@ -89,7 +94,7 @@ static int FatCacheAcquire(void) { if (!KeCanWait()) return 0; FatCacheEnsureMutex(); - return KSUCCESS(KeAcquireMutex(&gFatCacheMutex)); + return KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatCacheMutex, KE_WAIT_INFINITE)); } static void FatCacheRelease(int held) { @@ -1913,6 +1918,17 @@ KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) { return NTLSTATUS_SUCCESS; } +// Flushes FAT metadata and the backing device write cache. +KSTATUS FatFlush(IO_FILESYSTEM* fs, void* fileCtx) { + (void)fs; + FAT_FILE* f = (FAT_FILE*)fileCtx; + if (!f || !f->Vol) + return NTLSTATUS_INVALID_PARAMETER; + + FlushFatCache(f->Vol); + return IoFlushDisk(f->Vol->DriveIndex); +} + // Releases the requested object. void FatClose(IO_FILESYSTEM *fs, void *fileCtx) { (void)fs; @@ -2304,11 +2320,14 @@ KSTATUS FatDelete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) { if (!KSUCCESS(s)) return s; if (!direntLba) return NTLSTATUS_ACCESS_DENIED; - uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector); - if (!sec) return NTLSTATUS_NO_MEMORY; - s = ReadSectors(v, direntLba, 1, sec); - if (KSUCCESS(s)) { sec[direntOffset] = 0xE5; s = WriteSectors(v, direntLba, 1, sec); } - MmFreePool(sec); + uint32_t parentCluster = 0; + SYS_STRING_VIEW filenameView; + s = FatResolveParentForMutation(v, path, &parentCluster, &filenameView); + if (!KSUCCESS(s)) return s; + + // Delete the complete entry run so long-name metadata cannot survive the + // short entry and keep writable vvfat mirrors visibly present. + s = FatDeleteDirentRunV(v, parentCluster, filenameView); if (!KSUCCESS(s)) return s; if (cluster >= 2) FreeClusterChain(v, cluster); @@ -2583,7 +2602,7 @@ KSTATUS FatReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) { FAT_DIR_CTX *ctx = (FAT_DIR_CTX *)dirCtx; FAT_VOL *v = ctx->Vol; - for (;;) { + while (true) { if (ctx->Cursor != ctx->LoadedCursor) { if (!KSUCCESS(ReadSectors(v, ctx->LoadedLba, ctx->LoadedSectors, ctx->ClusterBuf))) return NTLSTATUS_NO_MORE_FILES; @@ -2918,8 +2937,8 @@ static void FatAsyncWriteSubmitNext(FAT_ASYNC_RW* ctx); static void FatAsyncWorker(void* arg) { (void)arg; - for (;;) { - KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore); + while (true) { + KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(s)) { KeYield(); continue; @@ -2959,8 +2978,9 @@ static KSTATUS FatEnsureAsyncWorkerStarted(void) { KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gFatAsyncSemaphore); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)FatAsyncWorker, 0, - PriorityNormal, &worker); + ¶meters, &worker); (void)worker; } @@ -3355,6 +3375,11 @@ static void FatMountEnsureMutex(void) { gFatMountMutex.Lock = (SPINLOCK)SPINLOCK_INIT; gFatMountMutex.Owner = (struct THREAD*)0; LIST_INIT(&gFatMountMutex.WaitList); + LIST_INIT(&gFatMountMutex.OwnerLink); + gFatMountMutex.DonationClass = KeSchedulingNormal; + gFatMountMutex.DonationPriority = PriorityLow; + gFatMountMutex.HasDonation = 0; + gFatMountMutex.Abandoned = 0; gFatMountMutexReady = 1; } KeReleaseSpinlock(&gFatMountMutexInitLock, flags); @@ -3375,7 +3400,7 @@ static uint32_t FatTryMountDisk(uint8_t disk) { return 0; FatMountEnsureMutex(); - if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex))) + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE))) return 0; if (gDiskScanned[disk]) { @@ -3509,7 +3534,7 @@ static void FatUnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) { uint32_t removed = 0; FatMountEnsureMutex(); - if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex))) + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE))) return; for (uint8_t i = 0; i < FAT_MAX_VOLUMES; i++) { diff --git a/code/drivers/fs/fat/fat.h b/code/drivers/fs/fat/fat.h index 3e6b7ab..f55d6cb 100644 --- a/code/drivers/fs/fat/fat.h +++ b/code/drivers/fs/fat/fat.h @@ -22,6 +22,7 @@ typedef struct FAT_VOL_S FAT_VOL; KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx); +KSTATUS FatFlush(IO_FILESYSTEM *fs, void *fileCtx); void FatClose(IO_FILESYSTEM *fs, void *fileCtx); KSTATUS FatRead(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead); KSTATUS FatWrite(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten); diff --git a/code/drivers/fs/fat/fat_mount.c b/code/drivers/fs/fat/fat_mount.c index 88e1289..d7ba9b1 100644 --- a/code/drivers/fs/fat/fat_mount.c +++ b/code/drivers/fs/fat/fat_mount.c @@ -19,6 +19,7 @@ void FatInitVtable(IO_FILESYSTEM* fs, FAT_VOL* vol) { fs->QueryInfo = FatQueryInfo; fs->Stat = FatStat; fs->QueryIdentity = FatQueryIdentity; + fs->Flush = FatFlush; fs->Close = FatClose; fs->DriverCtx = vol; fs->OpenDir = FatOpenDir; diff --git a/code/drivers/net/lwip/lwip_stack.c b/code/drivers/net/lwip/lwip_stack.c index 921646c..f86e5c2 100644 --- a/code/drivers/net/lwip/lwip_stack.c +++ b/code/drivers/net/lwip/lwip_stack.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -74,6 +75,15 @@ typedef struct LWIP_ATTACH_CTX { err_t Result; } LWIP_ATTACH_CTX; +typedef struct LWIP_RESOLVE_CONTEXT { + SEMAPHORE* Done; + volatile long References; + err_t Result; + u8_t DnsType; + ip_addr_t Address; + char Name[DNS_MAX_NAME_LENGTH]; +} LWIP_RESOLVE_CONTEXT; + typedef struct LWIP_PROVIDER_SOCKET { SEMAPHORE* Lock; struct netconn* Conn; @@ -118,6 +128,9 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how); static KSTATUS LwipProviderSetOption(void* socket, uint32_t option, const void* value, size_t valueSize); static KSTATUS LwipProviderGetOption(void* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut); static KSTATUS LwipProviderPoll(void* socket, uint32_t events, uint32_t* returnedEventsOut); +static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context); +static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument); +static void LwipResolveStartCallback(void* argument); static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut); static KSTATUS LwipDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp); @@ -355,7 +368,7 @@ static KSTATUS LwipLoadNetifConfig(const NET_DEVICE* device, LWIP_NETIF_CONFIG* static KSTATUS LwipSocketLock(LWIP_PROVIDER_SOCKET* s) { if (!s || !s->Lock) return NTLSTATUS_INVALID_PARAMETER; - return KeWaitSemaphore(s->Lock); + return KeWaitSemaphore(s->Lock, KE_WAIT_INFINITE); } // Implements the socket Unlock operation. @@ -464,7 +477,7 @@ static KSTATUS LwipSocketWaitRecvNetbuf(LWIP_PROVIDER_SOCKET* s, uint32_t originalTimeout = s->ReceiveTimeoutMs; uint32_t elapsed = 0; - for (;;) { + while (true) { if (LwipSocketCancelRequested(cancelRequested)) { netconn_set_recvtimeout(s->Conn, originalTimeout); return NTLSTATUS_CANCELLED; @@ -510,7 +523,7 @@ static KSTATUS LwipSocketWaitAccept(LWIP_PROVIDER_SOCKET* s, uint32_t originalTimeout = s->ReceiveTimeoutMs; uint32_t elapsed = 0; - for (;;) { + while (true) { if (LwipSocketCancelRequested(cancelRequested)) { netconn_set_recvtimeout(s->Conn, originalTimeout); return NTLSTATUS_CANCELLED; @@ -778,7 +791,7 @@ static KSTATUS LwipStackInitialize(const NTL_DDK* ddk) { return st; tcpip_init(LwipInitDone, done); - st = KeWaitSemaphore(done); + st = KeWaitSemaphore(done, KE_WAIT_INFINITE); ObDereferenceObject(&done->Header); if (!KSUCCESS(st)) return st; @@ -826,7 +839,7 @@ static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device) { err_t err = tcpip_callback(LwipAttachCallback, &ctx); if (err == ERR_OK) - st = KeWaitSemaphore(done); + st = KeWaitSemaphore(done, KE_WAIT_INFINITE); else st = NTLSTATUS_FAILURE; @@ -1135,7 +1148,7 @@ static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_K* add return LwipErrToStatus(err); } - for (;;) { + while (true) { if (LwipSocketCancelRequested(cancelRequested)) { (void)netconn_close(s->Conn); s->ShutdownRead = 1; @@ -1595,6 +1608,51 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how) { return LwipErrToStatus(err); } +// Releases one resolver owner and destroys the request after the final callback or waiter leaves. +static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context) { + if (!context) + return; + + if (_InterlockedDecrement(&context->References) != 0) + return; + + if (context->Done) + ObDereferenceObject(&context->Done->Header); + MmFreePool(context); +} + +// Completes an asynchronous DNS lookup without retaining the caller's kernel stack. +static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument) { + LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument; + (void)name; + + if (address) { + context->Address = *address; + context->Result = ERR_OK; + } else { + context->Result = ERR_VAL; + } + + KeSignalSemaphore(context->Done); + LwipResolveRelease(context); +} + +// Starts one DNS lookup in the tcpip thread and transfers its resolver reference to the callback. +static void LwipResolveStartCallback(void* argument) { + LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument; + err_t err = dns_gethostbyname_addrtype(context->Name, + &context->Address, + LwipResolveDnsDoneCallback, + context, + context->DnsType); + context->Result = err; + if (err == ERR_INPROGRESS) + return; + + KeSignalSemaphore(context->Done); + LwipResolveRelease(context); +} + // Implements the provider Resolve Host operation. static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut) { if (!name || !addressOut) @@ -1610,10 +1668,48 @@ static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SO else return NTLSTATUS_NOT_SUPPORTED; - ip_addr_t resolved; - err_t err = netconn_gethostbyname_addrtype(name, &resolved, dnsType); - if (err != ERR_OK) - return LwipErrToStatus(err); + size_t nameLength = strlen(name); + if (nameLength == 0 || nameLength >= DNS_MAX_NAME_LENGTH) + return NTLSTATUS_INVALID_PARAMETER; + + LWIP_RESOLVE_CONTEXT* context = + (LWIP_RESOLVE_CONTEXT*)MmAllocatePool(sizeof(*context)); + if (!context) + return NTLSTATUS_NO_MEMORY; + memset(context, 0, sizeof(*context)); + memcpy(context->Name, name, nameLength + 1); + context->References = 2; + context->Result = ERR_INPROGRESS; + context->DnsType = dnsType; + + KSTATUS status = KeSemaphoreCreate(0, 1, &context->Done); + if (!KSUCCESS(status)) { + MmFreePool(context); + return status; + } + + err_t queueError = tcpip_try_callback(LwipResolveStartCallback, context); + if (queueError != ERR_OK) { + LwipResolveRelease(context); + LwipResolveRelease(context); + return LwipErrToStatus(queueError); + } + + status = KeWaitSemaphore(context->Done, KE_WAIT_INFINITE); + if (!KSUCCESS(status)) { + DbgPrint("lwIP: resolve waiter left name=%s status=0x%x; DNS request remains active\n", + context->Name, + status); + LwipResolveRelease(context); + return status; + } + + err_t result = context->Result; + ip_addr_t resolved = context->Address; + LwipResolveRelease(context); + + if (result != ERR_OK) + return LwipErrToStatus(result); if ((family == SYS_AF_INET && !IP_IS_V4(&resolved)) || (family == SYS_AF_INET6 && !IP_IS_V6(&resolved))) return NTLSTATUS_NOT_FOUND; diff --git a/code/drivers/net/lwip/lwip_sys_arch.c b/code/drivers/net/lwip/lwip_sys_arch.c index 1d70545..b30f635 100644 --- a/code/drivers/net/lwip/lwip_sys_arch.c +++ b/code/drivers/net/lwip/lwip_sys_arch.c @@ -117,19 +117,14 @@ u32_t sys_arch_sem_wait(sys_sem_t* sem, u32_t timeout) { u32_t start = sys_now(); if (timeout == 0) { - KSTATUS st = KeWaitSemaphore((*sem)->Sem); + KSTATUS st = KeWaitSemaphore((*sem)->Sem, KE_WAIT_INFINITE); return KSUCCESS(st) ? (u32_t)(sys_now() - start) : SYS_ARCH_TIMEOUT; } - for (;;) { - if (KSUCCESS(KeTrySemaphore((*sem)->Sem))) - return (u32_t)(sys_now() - start); - - if ((u32_t)(sys_now() - start) >= timeout) - return SYS_ARCH_TIMEOUT; - - KeYield(); - } + KSTATUS status = KeWaitSemaphore((*sem)->Sem, timeout); + return KSUCCESS(status) + ? (u32_t)(sys_now() - start) + : SYS_ARCH_TIMEOUT; } // Implements the sys sem free operation. @@ -165,7 +160,7 @@ err_t sys_mutex_new(sys_mutex_t* mutex) { // Implements the sys mutex lock operation. void sys_mutex_lock(sys_mutex_t* mutex) { if (mutex && *mutex && (*mutex)->Sem) - (void)KeWaitSemaphore((*mutex)->Sem); + (void)KeWaitSemaphore((*mutex)->Sem, KE_WAIT_INFINITE); } // Implements the sys mutex unlock operation. @@ -279,28 +274,23 @@ u32_t sys_arch_mbox_fetch(sys_mbox_t* mbox, void** msg, u32_t timeout) { u32_t start = sys_now(); if (timeout == 0) { - KSTATUS st = KeWaitSemaphore((*mbox)->Items); + KSTATUS st = KeWaitSemaphore((*mbox)->Items, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return SYS_ARCH_TIMEOUT; return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start)); } - for (;;) { - if (KSUCCESS(KeTrySemaphore((*mbox)->Items))) - return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start)); - - if ((u32_t)(sys_now() - start) >= timeout) - return SYS_ARCH_TIMEOUT; - - KeYield(); - } + KSTATUS status = KeWaitSemaphore((*mbox)->Items, timeout); + if (!KSUCCESS(status)) + return SYS_ARCH_TIMEOUT; + return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start)); } u32_t sys_arch_mbox_tryfetch(sys_mbox_t* mbox, void** msg) { if (!mbox || !*mbox || !(*mbox)->Valid) return SYS_MBOX_EMPTY; - if (!KSUCCESS(KeTrySemaphore((*mbox)->Items))) + if (!KSUCCESS(KeWaitSemaphore((*mbox)->Items, 0))) return SYS_MBOX_EMPTY; return LwipMboxPopAfterItem(mbox, msg, 0); @@ -347,9 +337,10 @@ sys_thread_t sys_thread_new(const char* name, lwip_thread_fn thread, void* arg, // function pointer that lwIP supplied, with exactly the argument it supplied. THREAD* t = 0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(kp); KSTATUS st = KeCreateKernelThread((uintptr_t)thread, (uintptr_t)arg, - kp, + ¶meters, &t); if (!KSUCCESS(st)) KePanic(KE_PANIC_CODE_DRIVER, "lwIP: failed to create kernel thread: 0x%x\n", st); diff --git a/code/drivers/net/rtl8139/rtl8139.c b/code/drivers/net/rtl8139/rtl8139.c index e3d241e..5d58811 100644 --- a/code/drivers/net/rtl8139/rtl8139.c +++ b/code/drivers/net/rtl8139/rtl8139.c @@ -82,29 +82,79 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) { r->NetDevice.HardwareAddressLength = 6; } -// Creates dma. -static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) { - r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma); - r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma); - if (!r->RxPhys || !r->TxPhys) - return NTLSTATUS_NO_MEMORY; +// Stops bus mastering before DMA memory can be released. +static void Rtl8139StopDma(RTL8139_DEVICE* r) { + if (!r || !r->Started) + return; + r->Started = false; + if (r->SurpriseRemoved) + return; + Rtl8139Write16(r, RTL8139_REG_IMR, 0); + Rtl8139Write8(r, RTL8139_REG_CMD, 0); + KSTATUS status = Rtl8139Reset(r); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_DRIVER, + "RTL8139: failed to stop DMA before release status=0x%x\n", + status); +} - // RTL8139 has 32-bit DMA address registers. This is fine for typical QEMU - // and low-memory hobby OS configs, but make failure explicit. - if ((r->RxPhys + RTL8139_RX_ALLOC_SIZE - 1) > 0xFFFFFFFFULL || - (r->TxPhys + RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE - 1) > 0xFFFFFFFFULL) { - DbgPrint("RTL8139: DMA allocation above 4GiB is not supported by this NIC\n"); - return NTLSTATUS_NOT_SUPPORTED; +// Releases the common buffers and adapter owned by one RTL8139 device. +static void Rtl8139ReleaseDma(RTL8139_DEVICE* r) { + if (!r || !gRtl8139Ddk || !gRtl8139Ddk->Dma) + return; + gRtl8139Ddk->Dma->FreeCommonBuffer(r->TxDma); + gRtl8139Ddk->Dma->FreeCommonBuffer(r->RxDma); + r->TxDma = (IO_DMA_BUFFER*)0; + r->RxDma = (IO_DMA_BUFFER*)0; + r->TxBuffers = (uint8_t*)0; + r->RxBuffer = (uint8_t*)0; + r->TxDeviceAddress = 0; + r->RxDeviceAddress = 0; + gRtl8139Ddk->Dma->PutAdapter(r->DmaAdapter); + r->DmaAdapter = (IO_DMA_ADAPTER*)0; +} + +// Allocates the receive ring and transmit slots through the device DMA adapter. +static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) { + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = 0xFFFFFFFFULL; + requirements.Alignment = 4; + requirements.MaximumSegmentLength = RTL8139_TX_BUF_SIZE; + requirements.MaximumSegments = 1; + + KSTATUS status = gRtl8139Ddk->Dma->GetAdapter(r->Fdo, + &requirements, + &r->DmaAdapter); + if (!KSUCCESS(status)) + return status; + + status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter, + RTL8139_RX_ALLOC_SIZE, + PAGE_SIZE, + &r->RxDma); + if (!KSUCCESS(status)) { + Rtl8139ReleaseDma(r); + return status; } - r->RxBuffer = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->RxPhys); - r->TxBuffers = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->TxPhys); - memset(r->RxBuffer, 0, PAGE_SIZE * RTL8139_RX_PAGES); - memset(r->TxBuffers, 0, PAGE_SIZE * RTL8139_TX_PAGES); + status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter, + RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE, + PAGE_SIZE, + &r->TxDma); + if (!KSUCCESS(status)) { + Rtl8139ReleaseDma(r); + return status; + } + + r->RxBuffer = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->RxDma); + r->TxBuffers = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->TxDma); + r->RxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->RxDma); + r->TxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->TxDma); DbgPrint("RTL8139: rx=0x%016llX tx=0x%016llX\n", - (unsigned long long)r->RxPhys, - (unsigned long long)r->TxPhys); + (unsigned long long)r->RxDeviceAddress, + (unsigned long long)r->TxDeviceAddress); return NTLSTATUS_SUCCESS; } @@ -136,13 +186,13 @@ static void Rtl8139ProgramHardware(RTL8139_DEVICE* r) { Rtl8139Write16(r, RTL8139_REG_IMR, 0); Rtl8139Write16(r, RTL8139_REG_ISR, 0xFFFFu); - Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxPhys); + Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxDeviceAddress); r->RxOffset = 0; Rtl8139Write16(r, RTL8139_REG_CAPR, 0xFFF0u); // CAPR is rxOffset - 16. for (uint32_t i = 0; i < RTL8139_TX_SLOTS; i++) { - PHYSICAL_ADDRESS p = r->TxPhys + (PHYSICAL_ADDRESS)i * RTL8139_TX_BUF_SIZE; - Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)p); + uint64_t address = r->TxDeviceAddress + (uint64_t)i * RTL8139_TX_BUF_SIZE; + Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)address); } Rtl8139Write32(r, RTL8139_REG_TCR, @@ -360,7 +410,7 @@ static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) { if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) { KSTATUS st = PciMapBar(r->Pdo, best.Index, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&r->Mmio, &r->RegisterBytes); if (!KSUCCESS(st)) @@ -460,8 +510,10 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) { r->NetDevice.SendFrame = Rtl8139SendFrame; st = Rtl8139RegisterDevice(r); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + Rtl8139ReleaseDma(r); return st; + } KeIrqRegister(r->Irq, Rtl8139Isr); r->Started = true; @@ -470,9 +522,9 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) { st = NetRegisterDevice(&r->NetDevice); if (!KSUCCESS(st)) { - Rtl8139Write16(r, RTL8139_REG_IMR, 0); - r->Started = false; + Rtl8139StopDma(r); Rtl8139UnregisterDevice(r); + Rtl8139ReleaseDma(r); return st; } @@ -503,17 +555,16 @@ KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: r->SurpriseRemoved = true; + r->Started = false; device->PnpState = DevicePnpSurpriseRemoved; - if (r->Started) - Rtl8139Write16(r, RTL8139_REG_IMR, 0); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: r->Removed = true; device->PnpState = DevicePnpDeleted; - if (r->Started) - Rtl8139Write16(r, RTL8139_REG_IMR, 0); + Rtl8139StopDma(r); Rtl8139UnregisterDevice(r); + Rtl8139ReleaseDma(r); IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; @@ -555,7 +606,7 @@ KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) + if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("RTL8139: driver loading...\n"); diff --git a/code/drivers/net/rtl8139/rtl8139.h b/code/drivers/net/rtl8139/rtl8139.h index ccd2140..a6f8d86 100644 --- a/code/drivers/net/rtl8139/rtl8139.h +++ b/code/drivers/net/rtl8139/rtl8139.h @@ -115,8 +115,11 @@ typedef struct RTL8139_DEVICE { volatile uint8_t* Mmio; uint16_t IoBase; - PHYSICAL_ADDRESS RxPhys; - PHYSICAL_ADDRESS TxPhys; + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* RxDma; + IO_DMA_BUFFER* TxDma; + uint64_t RxDeviceAddress; + uint64_t TxDeviceAddress; uint8_t* RxBuffer; uint8_t* TxBuffers; uint8_t RxScratch[RTL8139_TX_BUF_SIZE]; diff --git a/code/drivers/net/rtl816x/rtl816x.c b/code/drivers/net/rtl816x/rtl816x.c index 2732b29..c912323 100644 --- a/code/drivers/net/rtl816x/rtl816x.c +++ b/code/drivers/net/rtl816x/rtl816x.c @@ -83,48 +83,117 @@ static void RtlReadMac(RTL816X_DEVICE* r) { r->NetDevice.HardwareAddressLength = 6; } -// Implements the rtl Allocate Dma operation. +// Stops bus mastering before DMA memory can be released. +static void RtlStopDma(RTL816X_DEVICE* r) { + if (!r || !r->Started) + return; + r->Started = false; + if (r->SurpriseRemoved) + return; + RtlWrite16(r, RTL_REG_INTR_MASK, 0); + RtlWrite8(r, RTL_REG_CHIP_CMD, 0); + KSTATUS status = RtlReset(r); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_DRIVER, + "RTL816X: failed to stop DMA before release status=0x%x\n", + status); +} + +// Releases the common buffers and adapter owned by one RTL816x device. +static void RtlReleaseDma(RTL816X_DEVICE* r) { + if (!r || !gRtl816xDdk || !gRtl816xDdk->Dma) + return; + gRtl816xDdk->Dma->FreeCommonBuffer(r->TxBuffersDma); + gRtl816xDdk->Dma->FreeCommonBuffer(r->RxBuffersDma); + gRtl816xDdk->Dma->FreeCommonBuffer(r->TxRingDma); + gRtl816xDdk->Dma->FreeCommonBuffer(r->RxRingDma); + r->TxBuffersDma = (IO_DMA_BUFFER*)0; + r->RxBuffersDma = (IO_DMA_BUFFER*)0; + r->TxRingDma = (IO_DMA_BUFFER*)0; + r->RxRingDma = (IO_DMA_BUFFER*)0; + r->TxBuffers = (uint8_t*)0; + r->RxBuffers = (uint8_t*)0; + r->TxRing = (RTL_DESC*)0; + r->RxRing = (RTL_DESC*)0; + r->TxBuffersDeviceAddress = 0; + r->RxBuffersDeviceAddress = 0; + r->TxRingDeviceAddress = 0; + r->RxRingDeviceAddress = 0; + gRtl816xDdk->Dma->PutAdapter(r->DmaAdapter); + r->DmaAdapter = (IO_DMA_ADAPTER*)0; +} + +// Allocates descriptor rings and packet buffers through the device DMA adapter. static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) { - r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma); - r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma); - r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma); - r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma); + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = (uint64_t)-1; + requirements.Alignment = 1; + requirements.MaximumSegmentLength = RTL_BUF_SIZE; + requirements.MaximumSegments = RTL_TX_COUNT; - if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys) - return NTLSTATUS_NO_MEMORY; + KSTATUS status = gRtl816xDdk->Dma->GetAdapter(r->Fdo, + &requirements, + &r->DmaAdapter); + if (!KSUCCESS(status)) + return status; - r->RxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->RxRingPhys); - r->TxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->TxRingPhys); - r->RxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->RxBuffersPhys); - r->TxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->TxBuffersPhys); + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_RING_PAGES, + PAGE_SIZE, + &r->RxRingDma); + if (!KSUCCESS(status)) goto fail; + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_RING_PAGES, + PAGE_SIZE, + &r->TxRingDma); + if (!KSUCCESS(status)) goto fail; + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_RXBUF_PAGES, + PAGE_SIZE, + &r->RxBuffersDma); + if (!KSUCCESS(status)) goto fail; + status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter, + PAGE_SIZE * RTL_TXBUF_PAGES, + PAGE_SIZE, + &r->TxBuffersDma); + if (!KSUCCESS(status)) goto fail; - memset(r->RxRing, 0, PAGE_SIZE * RTL_RING_PAGES); - memset(r->TxRing, 0, PAGE_SIZE * RTL_RING_PAGES); - memset(r->RxBuffers, 0, PAGE_SIZE * RTL_RXBUF_PAGES); - memset(r->TxBuffers, 0, PAGE_SIZE * RTL_TXBUF_PAGES); + r->RxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxRingDma); + r->TxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxRingDma); + r->RxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxBuffersDma); + r->TxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxBuffersDma); + r->RxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxRingDma); + r->TxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxRingDma); + r->RxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxBuffersDma); + r->TxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxBuffersDma); DbgPrint("RTL816X: rxRing=0x%016llX txRing=0x%016llX rxBuf=0x%016llX txBuf=0x%016llX\n", - (unsigned long long)r->RxRingPhys, - (unsigned long long)r->TxRingPhys, - (unsigned long long)r->RxBuffersPhys, - (unsigned long long)r->TxBuffersPhys); + (unsigned long long)r->RxRingDeviceAddress, + (unsigned long long)r->TxRingDeviceAddress, + (unsigned long long)r->RxBuffersDeviceAddress, + (unsigned long long)r->TxBuffersDeviceAddress); return NTLSTATUS_SUCCESS; + +fail: + RtlReleaseDma(r); + return status; } // Implements the rtl Init Descriptors operation. static void RtlInitDescriptors(RTL816X_DEVICE* r) { for (uint32_t i = 0; i < RTL_RX_COUNT; i++) { - PHYSICAL_ADDRESS p = r->RxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE; - r->RxRing[i].addrLo = (uint32_t)p; - r->RxRing[i].addrHi = (uint32_t)(p >> 32); + uint64_t address = r->RxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE; + r->RxRing[i].addrLo = (uint32_t)address; + r->RxRing[i].addrHi = (uint32_t)(address >> 32); r->RxRing[i].opts2 = 0; r->RxRing[i].opts1 = RTL_DESC_OWN | RTL_BUF_SIZE | ((i == RTL_RX_COUNT - 1) ? RTL_DESC_EOR : 0); } for (uint32_t i = 0; i < RTL_TX_COUNT; i++) { - PHYSICAL_ADDRESS p = r->TxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE; - r->TxRing[i].addrLo = (uint32_t)p; - r->TxRing[i].addrHi = (uint32_t)(p >> 32); + uint64_t address = r->TxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE; + r->TxRing[i].addrLo = (uint32_t)address; + r->TxRing[i].addrHi = (uint32_t)(address >> 32); r->TxRing[i].opts2 = 0; r->TxRing[i].opts1 = (i == RTL_TX_COUNT - 1) ? RTL_DESC_EOR : 0; } @@ -141,12 +210,12 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) { RtlWrite16(r, RTL_REG_INTR_MASK, 0); RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu); - RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingPhys); - RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingPhys >> 32)); - RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingPhys); - RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingPhys >> 32)); - RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingPhys); - RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingPhys >> 32)); + RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingDeviceAddress); + RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32)); + RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingDeviceAddress); + RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32)); + RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingDeviceAddress); + RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingDeviceAddress >> 32)); RtlWrite16(r, RTL_REG_CPLUS_CMD, RtlRead16(r, RTL_REG_CPLUS_CMD) & ~(uint16_t)0x00E0u); RtlWrite16(r, RTL_REG_RX_MAXSIZE, RTL_BUF_SIZE); @@ -366,7 +435,7 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) { if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) { KSTATUS st = PciMapBar(r->Pdo, best.Index, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&r->Mmio, &r->RegisterBytes); if (!KSUCCESS(st)) @@ -400,6 +469,41 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) { return NTLSTATUS_NOT_FOUND; } +// Configures a dedicated MSI vector and falls back to a routed INTx line. +static KSTATUS RtlConfigureInterrupt(RTL816X_DEVICE* r) { + if (!r || !r->Pdo) + return NTLSTATUS_INVALID_PARAMETER; + + r->Irq = 0xFFu; + r->UsingMsi = false; + + KSTATUS msiStatus = PciEnableMsiAuto(r->Pdo, &r->Irq); + if (KSUCCESS(msiStatus)) { + r->UsingMsi = true; + DbgPrint("RTL816X: using MSI IRQ slot %u\n", r->Irq); + return NTLSTATUS_SUCCESS; + } + + DbgPrint("RTL816X: MSI setup failed status=0x%x; attempting explicit INTx fallback\n", + msiStatus); + + KSTATUS intxStatus = PciGetInterrupt(r->Pdo, &r->Irq); + if (!KSUCCESS(intxStatus)) + return intxStatus; + if (r->Irq == 0xFFu || r->Irq >= 24u) + return NTLSTATUS_NOT_SUPPORTED; + + intxStatus = PciEnableDevice(r->Pdo, + PCI_ENABLE_BUSMASTER | + PCI_ENABLE_INTX | + (r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO)); + if (!KSUCCESS(intxStatus)) + return intxStatus; + + DbgPrint("RTL816X: using legacy INTx IRQ %u\n", r->Irq); + return NTLSTATUS_SUCCESS; +} + // Implements the rtl Start Device operation. static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { if (!r || r->Magic != RTL816X_DEVICE_MAGIC || !r->Pdo || !r->Fdo) @@ -428,18 +532,15 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { st = PciEnableDevice(r->Pdo, PCI_ENABLE_BUSMASTER | - PCI_ENABLE_INTX | + PCI_DISABLE_INTX | (r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO)); if (!KSUCCESS(st)) { DbgPrint("RTL816X: PciEnableDevice failed: 0x%x\n", st); return st; } - st = PciGetInterrupt(r->Pdo, &r->Irq); - if (!KSUCCESS(st) || r->Irq == 0xFFu) { - DbgPrint("RTL816X: no legacy PCI IRQ line wired\n"); - return KSUCCESS(st) ? NTLSTATUS_NOT_SUPPORTED : st; - } + RtlWrite16(r, RTL_REG_INTR_MASK, 0); + RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu); st = RtlReset(r); if (!KSUCCESS(st)) { @@ -447,12 +548,18 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { return st; } + st = RtlConfigureInterrupt(r); + if (!KSUCCESS(st)) { + DbgPrint("RTL816X: no usable MSI or INTx interrupt status=0x%x\n", st); + return st; + } + RtlReadMac(r); - DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u\n", + DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u mode=%s\n", r->NetDevice.HardwareAddress[0], r->NetDevice.HardwareAddress[1], r->NetDevice.HardwareAddress[2], r->NetDevice.HardwareAddress[3], r->NetDevice.HardwareAddress[4], r->NetDevice.HardwareAddress[5], - r->Irq); + r->Irq, r->UsingMsi ? "MSI" : "INTx"); st = RtlAllocateDma(r); if (!KSUCCESS(st)) @@ -466,8 +573,10 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { r->NetDevice.SendFrame = RtlSendFrame; st = RtlRegisterDevice(r); - if (!KSUCCESS(st)) + if (!KSUCCESS(st)) { + RtlReleaseDma(r); return st; + } KeIrqRegister(r->Irq, RtlIsr); r->Started = true; @@ -476,13 +585,14 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) { st = NetRegisterDevice(&r->NetDevice); if (!KSUCCESS(st)) { - RtlWrite16(r, RTL_REG_INTR_MASK, 0); - r->Started = false; + RtlStopDma(r); RtlUnregisterDevice(r); + RtlReleaseDma(r); return st; } - DbgPrint("RTL816X: initialized; RX/TX enabled on IRQ %u\n", r->Irq); + DbgPrint("RTL816X: initialized; RX/TX enabled on %s IRQ %u\n", + r->UsingMsi ? "MSI" : "INTx", r->Irq); return NTLSTATUS_SUCCESS; } @@ -509,17 +619,16 @@ KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: r->SurpriseRemoved = true; + r->Started = false; device->PnpState = DevicePnpSurpriseRemoved; - if (r->Started) - RtlWrite16(r, RTL_REG_INTR_MASK, 0); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: r->Removed = true; device->PnpState = DevicePnpDeleted; - if (r->Started) - RtlWrite16(r, RTL_REG_INTR_MASK, 0); + RtlStopDma(r); RtlUnregisterDevice(r); + RtlReleaseDma(r); IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; @@ -561,7 +670,7 @@ KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) + if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("RTL816X: driver loading...\n"); diff --git a/code/drivers/net/rtl816x/rtl816x.h b/code/drivers/net/rtl816x/rtl816x.h index cfa8d00..f84c30e 100644 --- a/code/drivers/net/rtl816x/rtl816x.h +++ b/code/drivers/net/rtl816x/rtl816x.h @@ -117,6 +117,7 @@ typedef struct RTL816X_DEVICE { PCI_BAR RegisterBar; uint64_t RegisterBytes; uint8_t Irq; + bool UsingMsi; bool Started; bool Removed; @@ -126,15 +127,20 @@ typedef struct RTL816X_DEVICE { volatile uint8_t* Mmio; uint16_t IoBase; + IO_DMA_ADAPTER* DmaAdapter; + IO_DMA_BUFFER* RxRingDma; + IO_DMA_BUFFER* TxRingDma; + IO_DMA_BUFFER* RxBuffersDma; + IO_DMA_BUFFER* TxBuffersDma; RTL_DESC* RxRing; RTL_DESC* TxRing; - PHYSICAL_ADDRESS RxRingPhys; - PHYSICAL_ADDRESS TxRingPhys; + uint64_t RxRingDeviceAddress; + uint64_t TxRingDeviceAddress; uint8_t* RxBuffers; uint8_t* TxBuffers; - PHYSICAL_ADDRESS RxBuffersPhys; - PHYSICAL_ADDRESS TxBuffersPhys; + uint64_t RxBuffersDeviceAddress; + uint64_t TxBuffersDeviceAddress; uint32_t RxIndex; uint32_t TxIndex; diff --git a/code/drivers/storage/ahci/ahci.c b/code/drivers/storage/ahci/ahci.c index 7e18152..55d1b28 100644 --- a/code/drivers/storage/ahci/ahci.c +++ b/code/drivers/storage/ahci/ahci.c @@ -9,6 +9,11 @@ static AHCI_CONTROLLER* g_controllers[MAX_AHCI_CONTROLLERS]; static int g_controllerCount; static uint8_t g_nextDiskIndex; +#define AHCI_COMMAND_TIMEOUT_MS 10000ull +#define AHCI_PORT_ERROR_MASK \ + (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS) +#define AHCI_PORT_TERMINAL_MASK (PORT_IS_DHRS | AHCI_PORT_ERROR_MASK) + static const IO_DEVICE_ID g_ahciIds[] = { { IoBusPci, { "PCI\\CC_010601", sizeof("PCI\\CC_010601") - 1 } }, }; @@ -21,20 +26,21 @@ static const IO_DRIVER_ID_TABLE g_ahciIdTable = { // Helpers. static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl); +static void AhciUnregisterController(AHCI_CONTROLLER* ctl); -// Accessors for port structure memory at known offsets from structVirt/Phys. +// Accessors for port structure memory at known CPU and device-address offsets. static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) { return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST); } -// Implements the cmd list phys operation. -static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) { - return drv->structPhys + PORT_OFF_CMDLIST; +// Returns the command-list device address. +static inline uint64_t cmd_list_device_address(AHCI_DRIVE *drv) { + return drv->structDeviceAddress + PORT_OFF_CMDLIST; } -// Implements the fis phys operation. -static inline uint64_t fis_phys(AHCI_DRIVE *drv) { - return drv->structPhys + PORT_OFF_FIS; +// Returns the received-FIS device address. +static inline uint64_t fis_device_address(AHCI_DRIVE *drv) { + return drv->structDeviceAddress + PORT_OFF_FIS; } static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) { @@ -43,9 +49,9 @@ static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) { + (uint32_t)slot * PORT_CMDTBL_SIZE); } -// Implements the cmd tbl phys operation. -static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { - return drv->structPhys + PORT_OFF_CMDTBLS +// Returns one command-table device address. +static inline uint64_t cmd_tbl_device_address(AHCI_DRIVE *drv, int slot) { + return drv->structDeviceAddress + PORT_OFF_CMDTBLS + (uint64_t)slot * PORT_CMDTBL_SIZE; } @@ -53,53 +59,108 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) { // Port memory initialisation. +// Stops one drive worker before its queue and DMA storage are released. +static void AhciStopWorker(AHCI_DRIVE* drv) { + if (!drv) + return; + + drv->workerRunning = false; + if (drv->irpQueue.Semaphore) + KeSignalSemaphore(drv->irpQueue.Semaphore); + if (drv->workerThread && drv->workerFinished) + (void)KeWaitForEvent(drv->workerFinished, 0); + + if (drv->workerThread) { + ObDereferenceObject((OBJECT*)drv->workerThread); + drv->workerThread = (THREAD*)0; + } + if (drv->workerFinished) { + ObDereferenceObject(&drv->workerFinished->Header); + drv->workerFinished = (EVENT*)0; + } +} + +// Releases the non-DMA synchronization objects owned by one AHCI drive. +static void AhciReleaseDriveRuntime(AHCI_DRIVE* drv) { + if (!drv) + return; + + AhciStopWorker(drv); + if (drv->removeLockReady) { + IoDestroyRemoveLock(&drv->removeLock); + drv->removeLockReady = false; + } + if (drv->irpQueue.Semaphore) { + ObDereferenceObject(&drv->irpQueue.Semaphore->Header); + drv->irpQueue.Semaphore = (SEMAPHORE*)0; + } + if (drv->completionEvent) { + ObDereferenceObject(&drv->completionEvent->Header); + drv->completionEvent = (EVENT*)0; + } +} + +// Releases the DMA buffers owned by one AHCI drive. +static void AhciReleaseDriveDma(AHCI_DRIVE* drv) { + if (!drv || !gAhciDdk || !gAhciDdk->Dma) + return; + if (drv->port && drv->controller && + !drv->surpriseRemoved && !drv->controller->surpriseRemoved) + port_stop(drv->port); + gAhciDdk->Dma->FreeCommonBuffer(drv->bounceDma); + gAhciDdk->Dma->FreeCommonBuffer(drv->structDma); + drv->bounceDma = (IO_DMA_BUFFER*)0; + drv->structDma = (IO_DMA_BUFFER*)0; + drv->bounceDeviceAddress = 0; + drv->structDeviceAddress = 0; + drv->bounceVirt = (void*)0; + drv->structVirt = (void*)0; +} + +// Allocates and programs one port's command, FIS, and bounce buffers. static KSTATUS port_init_memory(AHCI_DRIVE *drv) { - uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; + if (!drv || !drv->controller || !drv->controller->dmaAdapter) + return NTLSTATUS_INVALID_PARAMETER; - drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma); - if (!drv->structPhys) return NTLSTATUS_NO_MEMORY; - drv->structVirt = gAhciDdk->Memory->PhysToVirt(drv->structPhys); - // Force NOCACHE on DMA descriptor memory. - MmPteProtectRange((uintptr_t)drv->structVirt, - PORT_PAGES_STRUCT * 4096, flags); - memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096); + KSTATUS status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter, + PORT_PAGES_STRUCT * PAGE_SIZE, + PAGE_SIZE, + &drv->structDma); + if (!KSUCCESS(status)) + return status; + drv->structVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->structDma); + drv->structDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->structDma); - drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma); - if (!drv->bouncePhys) return NTLSTATUS_NO_MEMORY; - drv->bounceVirt = gAhciDdk->Memory->PhysToVirt(drv->bouncePhys); - // Bounce buffer: NOCACHE to ensure CPU writes reach physical memory - // before the HBA's DMA engine reads them. - MmPteProtectRange((uintptr_t)drv->bounceVirt, - BOUNCE_PAGES * 4096, flags); - memset(drv->bounceVirt, 0, BOUNCE_BYTES); + status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter, + BOUNCE_BYTES, + PAGE_SIZE, + &drv->bounceDma); + if (!KSUCCESS(status)) { + AhciReleaseDriveDma(drv); + return status; + } + drv->bounceVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->bounceDma); + drv->bounceDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->bounceDma); HBA_PORT *port = drv->port; - port_stop(port); - // Set command list base. - port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF); - port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32); + port->clb = (uint32_t)(cmd_list_device_address(drv) & 0xFFFFFFFF); + port->clbu = (uint32_t)(cmd_list_device_address(drv) >> 32); + port->fb = (uint32_t)(fis_device_address(drv) & 0xFFFFFFFF); + port->fbu = (uint32_t)(fis_device_address(drv) >> 32); - // Set FIS base. - port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF); - port->fbu = (uint32_t)(fis_phys(drv) >> 32); - - // Initialise all 32 command headers, pointing each to its command table. HBA_CMD_HEADER *hdrs = cmd_list(drv); for (int i = 0; i < NUM_CMD_SLOTS; i++) { hdrs[i].prdtl = PORT_PRDT_COUNT; - uint64_t tbl = cmd_tbl_phys(drv, i); - hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF); - hdrs[i].ctbau = (uint32_t)(tbl >> 32); + uint64_t table = cmd_tbl_device_address(drv, i); + hdrs[i].ctba = (uint32_t)(table & 0xFFFFFFFF); + hdrs[i].ctbau = (uint32_t)(table >> 32); } - // Clear any stale interrupts then start the port engine. port->serr = (uint32_t)-1; port->is = (uint32_t)-1; - port_start(port); - return NTLSTATUS_SUCCESS; } @@ -125,16 +186,22 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) { // Issue slot 0. port->ci = (1u << 0); - // Poll until slot 0 clears. - while (true) { - if (!(port->ci & (1u << 0))) break; - if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { + // Poll until slot 0 clears, but never let early-init I/O hang forever. + spin = 0; + while ((port->ci & (1u << 0)) && spin < POLL_TIMEOUT) { + if (port->is & AHCI_PORT_ERROR_MASK) { DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", port->is, port->tfd, port->serr); port->serr = (uint32_t)-1; port->is = (uint32_t)-1; return NTLSTATUS_DEVICE_ERROR; } + spin++; + } + if (port->ci & (1u << 0)) { + DbgPrint("AHCI: command timed out while polling CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, port->is, port->serr); + return NTLSTATUS_TIMEOUT; } if (port->is & PORT_IS_TFES) { @@ -146,6 +213,30 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) { return NTLSTATUS_SUCCESS; } +// Recovers a port after a command that cannot be trusted to have retired. +static void AhciRecoverPort(AHCI_DRIVE *drv) { + if (!drv || !drv->port) + return; + + HBA_PORT *port = drv->port; + uint32_t savedIe = port->ie; + + // Keep reset-generated FIS traffic from satisfying a command waiter. + port->ie = 0; + KSTATUS status = port_comreset(drv); + port->serr = (uint32_t)-1; + port->is = (uint32_t)-1; + drv->lastIs = 0; + if (drv->completionEvent) + KeClearEvent(drv->completionEvent); + port->ie = savedIe; + + if (!KSUCCESS(status)) { + DbgPrint("AHCI: port recovery failed st=0x%x CI=0x%x TFD=0x%x SSTS=0x%x\n", + status, port->ci, port->tfd, port->ssts); + } +} + // Interrupt bottom half. The ISR owns hardware acknowledgement and status // capture; this DPC wakes the worker thread after the hard IRQ has unwound. @@ -191,7 +282,11 @@ static void AhciIsr(void) { // Clear the HBA-level bit. ctl->hba->is = portBit; - KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0); + // DMA Setup, PIO Setup, and other progress interrupts are not + // command completion. Only wake the worker for D2H completion or + // an error that terminates the command. + if (portIs & AHCI_PORT_TERMINAL_MASK) + KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0); } } } @@ -206,8 +301,19 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { HBA_PORT *port = drv->port; - // Clear stale IS and snapshot before issuing so we don't mis-read a - // leftover value from a prior command. + // A serialized slot must be idle before reuse. A stale active bit means a + // prior command escaped normal completion and must be recovered first. + if (port->ci & (1u << 0)) { + DbgPrint("AHCI: slot 0 still active before issue CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, port->is, port->serr); + AhciRecoverPort(drv); + if (port->ci & (1u << 0)) + return NTLSTATUS_DEVICE_ERROR; + } + + // Clear stale software and hardware completion state before submission. + // The event can otherwise retain a late DPC signal from an older command. + KeClearEvent(drv->completionEvent); port->is = (uint32_t)-1; drv->lastIs = 0; @@ -216,20 +322,54 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT) spin++; if (spin == POLL_TIMEOUT) { - DbgPrint("AHCI: port hung (pre-issue)\n"); + DbgPrint("AHCI: port hung before issue TFD=0x%x IS=0x%x SERR=0x%x\n", + port->tfd, port->is, port->serr); + AhciRecoverPort(drv); return NTLSTATUS_TIMEOUT; } - // Issue slot 0, then block on the event. + // Issue slot 0, then block with a finite deadline. port->ci = (1u << 0); - KeWaitForEvent(drv->completionEvent, 0); + EVENT *events[1] = { drv->completionEvent }; + KSTATUS waitStatus = KeWaitForMultipleEvents(events, 1, 0, + AHCI_COMMAND_TIMEOUT_MS, + (uint32_t *)0); - // The ISR has cleared IS - check for errors in the snapshot. - uint32_t is = drv->lastIs; - if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) { - DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n", - is, port->tfd, port->serr); - port->serr = (uint32_t)-1; + // Merge any live status that was not consumed by the ISR. This lets a + // missed IRQ still report the hardware error that accompanied it. + uint32_t liveIs = port->is; + uint32_t is = drv->lastIs | liveIs; + if (liveIs) + port->is = liveIs; + + if (is & AHCI_PORT_ERROR_MASK) { + DbgPrint("AHCI: command error IS=0x%x CI=0x%x TFD=0x%x SERR=0x%x\n", + is, port->ci, port->tfd, port->serr); + AhciRecoverPort(drv); + return NTLSTATUS_DEVICE_ERROR; + } + + if (waitStatus == NTLSTATUS_TIMEOUT) { + if (!(port->ci & (1u << 0))) { + DbgPrint("AHCI: completion IRQ missed; command retired IS=0x%x TFD=0x%x SERR=0x%x\n", + is, port->tfd, port->serr); + return NTLSTATUS_SUCCESS; + } + + DbgPrint("AHCI: command timeout CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, is, port->serr); + AhciRecoverPort(drv); + return NTLSTATUS_TIMEOUT; + } + + if (waitStatus != NTLSTATUS_SUCCESS) + return waitStatus; + + // An interrupt is only completion if hardware has also retired slot 0. + if (port->ci & (1u << 0)) { + DbgPrint("AHCI: premature completion IRQ CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n", + port->ci, port->tfd, is, port->serr); + AhciRecoverPort(drv); return NTLSTATUS_DEVICE_ERROR; } @@ -239,7 +379,7 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) { // Build a standard DMA read or write command in slot 0. static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, - uint64_t bufPhys, bool write) { + uint64_t bufferDeviceAddress, bool write) { HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0 HBA_CMD_TBL *tbl = cmd_tbl(drv, 0); @@ -256,7 +396,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, // Fill PRDT entries. uint32_t remaining = sectors; - uint64_t buf = bufPhys; + uint64_t buf = bufferDeviceAddress; for (uint32_t i = 0; i < prdtl; i++) { uint32_t sects = (remaining > 16) ? 16 : remaining; uint32_t bytes = sects * 512; @@ -289,6 +429,31 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors, fis->counth = (uint8_t)(sectors >> 8); } +// Builds a non-data ATA FLUSH CACHE EXT command in slot 0. +static void build_flush_cmd(AHCI_DRIVE* drv) { + HBA_CMD_HEADER* hdr = cmd_list(drv); + HBA_CMD_TBL* tbl = cmd_tbl(drv, 0); + + memset(tbl, 0, PORT_CMDTBL_SIZE); + hdr->cfl = sizeof(FIS_REG_H2D) / 4; + hdr->w = 0; + hdr->prdtl = 0; + hdr->prdbc = 0; + + FIS_REG_H2D* fis = (FIS_REG_H2D*)tbl->cfis; + memset(fis, 0, sizeof(*fis)); + fis->fis_type = FIS_TYPE_REG_H2D; + fis->c = 1; + fis->command = ATA_CMD_FLUSH_EXT; +} + +// Flushes the drive write cache through the serialized command worker. +static KSTATUS do_flush(AHCI_DRIVE* drv) { + if (!drv) return NTLSTATUS_INVALID_PARAMETER; + build_flush_cmd(drv); + return port_issue_cmd(drv); +} + // IDENTIFY command - fills a 512-byte buffer and returns it. static KSTATUS port_identify(AHCI_DRIVE *drv) { @@ -305,8 +470,8 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) { hdr->prdtl = 1; hdr->prdbc = 0; - tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF); - tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32); + tbl->prdt_entry[0].dba = (uint32_t)(drv->bounceDeviceAddress & 0xFFFFFFFF); + tbl->prdt_entry[0].dbau = (uint32_t)(drv->bounceDeviceAddress >> 32); tbl->prdt_entry[0].dbc = 512 - 1; tbl->prdt_entry[0].i = 1; @@ -347,7 +512,7 @@ static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba, while (count > 0) { uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; - build_rw_cmd(drv, lba, batch, drv->bouncePhys, false); + build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, false); KSTATUS st = port_issue_cmd(drv); if (!KSUCCESS(st)) return st; @@ -368,7 +533,7 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba, uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count; memcpy(drv->bounceVirt, in, (size_t)batch * 512); - build_rw_cmd(drv, lba, batch, drv->bouncePhys, true); + build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, true); KSTATUS st = port_issue_cmd(drv); if (!KSUCCESS(st)) return st; @@ -463,13 +628,16 @@ static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: drv->surpriseRemoved = true; device->PnpState = DevicePnpSurpriseRemoved; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: drv->removed = true; device->PnpState = DevicePnpDeleted; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); - IoReleaseRemoveLockAndWait(&drv->removeLock, irp); + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + if (drv->removeLockReady) + IoReleaseRemoveLockAndWait(&drv->removeLock, irp); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); IoDeleteDevice(device); drv->deviceObject = (DEVICE_OBJECT*)0; return NTLSTATUS_SUCCESS; @@ -511,7 +679,10 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) { for (int i = 0; i < ctl->driveCount; i++) { AHCI_DRIVE* drv = &ctl->drives[i]; drv->surpriseRemoved = true; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + if (drv->deviceObject) + (void)IoNotifyDiskSurpriseRemoved(drv->deviceObject->u.Disk.DiskIndex); + else + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); } return NTLSTATUS_SUCCESS; @@ -520,9 +691,18 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) { device->PnpState = DevicePnpDeleted; for (int i = 0; i < ctl->driveCount; i++) { AHCI_DRIVE* drv = &ctl->drives[i]; - drv->removed = true; - IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + if (drv->deviceObject) { + (void)IoUnregisterDiskDevice(drv->deviceObject->u.Disk.DiskIndex); + } else { + drv->removed = true; + IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); + } } + AhciUnregisterController(ctl); + gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter); + ctl->dmaAdapter = (IO_DMA_ADAPTER*)0; IoDetachDevice(device); IoDeleteDevice(device); return NTLSTATUS_SUCCESS; @@ -532,6 +712,29 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) { } } +// Queues one write-cache flush on the per-drive worker. +static KSTATUS AhciDispatchFlush(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER; + AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension; + if (!drv || drv->magic != AHCI_DRIVE_MAGIC) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp); + if (!KSUCCESS(st)) + return st; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) { + IoReleaseRemoveLock(&drv->removeLock, irp); + return NTLSTATUS_INVALID_PARAMETER; + } + + irp->Status = NTLSTATUS_FAILURE; + irp->Information = 0; + IrpQueueEnqueue(&drv->irpQueue, irp); + return NTLSTATUS_PENDING; +} + // Dispatches pnp. static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { if (!device || !device->DeviceExtension) @@ -552,7 +755,7 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { // ever having blocked in a driver call. // NCQ remains serialized until the driver grows a multi-slot dispatcher: -// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP. +// 1. Wait for a free slot and a pending IRP through the queue semaphore. // 2. Pick the oldest IRP from the queue. // 3. Grab a free bit in drv->slotMask (under the slot spinlock). // 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA), @@ -561,30 +764,45 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { static void AhciWorker(uintptr_t arg) { AHCI_DRIVE *drv = (AHCI_DRIVE *)arg; - while (true) { + while (drv && drv->workerRunning) { // Block until AHCI dispatch signals a new IRP. - KeWaitSemaphore(drv->irpQueue.Semaphore); + KeWaitSemaphore(drv->irpQueue.Semaphore, KE_WAIT_INFINITE); + if (!drv->workerRunning) + break; IRP *irp = IrpQueueDequeue(&drv->irpQueue); if (!irp) continue; - bool write = false; - uint64_t lba = 0; - uint32_t sectors = 0; - void* buffer = (void*)0; - KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); - if (KSUCCESS(st)) { - if (write) - st = do_write(drv, lba, sectors, buffer); - else - st = do_read (drv, lba, sectors, buffer); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + KSTATUS st = NTLSTATUS_INVALID_PARAMETER; + size_t information = 0; + + if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) { + st = do_flush(drv); + } else { + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + st = AhciGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + if (KSUCCESS(st)) { + if (write) + st = do_write(drv, lba, sectors, buffer); + else + st = do_read(drv, lba, sectors, buffer); + if (KSUCCESS(st)) + information = (size_t)sectors * 512u; + } } // Publish result then fire completion hooks through the common I/O path. IoReleaseRemoveLock(&drv->removeLock, irp); - IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0); + IoCompleteRequest(irp, st, information); } + + if (drv && drv->workerFinished) + KeSignalEvent(drv->workerFinished); } // AHCI controller initialisation. @@ -597,18 +815,8 @@ static void AhciWorker(uintptr_t arg) { // FRE must already be set (port_init_memory does this) so the HBA can // capture the signature FIS the device sends after the reset. -// PORT_IE bits we care about: -// DHRS (bit 0) - D2H Register FIS received - normal completion -// PSS (bit 1) - PIO Setup FIS received -// DSS (bit 2) - DMA Setup FIS received -// SDBS (bit 3) - Set Device Bits FIS received -// TFES (bit 30) - Task File Error Status - error -// HBFS (bit 29) - Host Bus Fatal error -// HBDS (bit 28) - Host Bus Data error -// IFS (bit 27) - Interface Fatal error. -#define PORT_IE_NORMAL (0x0000000Fu) -#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS) -#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS) +// Runtime commands are non-NCQ DMA commands. DHRS is their completion signal; +// setup/progress FIS interrupts must not wake the serialized command waiter. // Registers controller. static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) { @@ -623,6 +831,20 @@ static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) { return NTLSTATUS_SUCCESS; } +// Removes a controller from the interrupt registry. +static void AhciUnregisterController(AHCI_CONTROLLER* ctl) { + if (!ctl) + return; + for (int i = 0; i < g_controllerCount; i++) { + if (g_controllers[i] != ctl) + continue; + g_controllerCount--; + g_controllers[i] = g_controllers[g_controllerCount]; + g_controllers[g_controllerCount] = (AHCI_CONTROLLER*)0; + return; + } +} + // Starts controller. static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC || !ctl->pdo || !ctl->fdo) @@ -676,7 +898,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = PciMapBar(ctl->pdo, AHCI_PCI_BAR_ABAR, - PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE, + PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC, (void**)&ctl->hba, &ctl->abarLength); if (!KSUCCESS(st)) { @@ -712,9 +934,23 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n", cap, nPorts, nSlots, (int)s64a); + IO_DMA_REQUIREMENTS dmaRequirements; + memset(&dmaRequirements, 0, sizeof(dmaRequirements)); + dmaRequirements.MaximumDeviceAddress = s64a ? (uint64_t)-1 : 0xFFFFFFFFULL; + dmaRequirements.Alignment = 2; + dmaRequirements.MaximumSegmentLength = 1u << 22; + dmaRequirements.MaximumSegments = PORT_PRDT_COUNT; + st = gAhciDdk->Dma->GetAdapter(ctl->fdo, &dmaRequirements, &ctl->dmaAdapter); + if (!KSUCCESS(st)) { + DbgPrint("AHCI: DMA adapter creation failed (0x%x)\n", st); + return st; + } + st = AhciRegisterController(ctl); if (!KSUCCESS(st)) { DbgPrint("AHCI: controller registry full (0x%x)\n", st); + gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter); + ctl->dmaAdapter = (IO_DMA_ADAPTER*)0; return st; } @@ -751,6 +987,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = port_comreset(drv); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: COMRESET failed\n", portno); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } @@ -758,6 +996,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { if (!port_is_sata_disk(port)) { DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n", portno, port->sig); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } @@ -765,6 +1005,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } KeInitializeDpc(&drv->completionDpc, AhciCompletionDpc, drv); @@ -775,19 +1017,26 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } IrpQueueInit(&drv->irpQueue, irpSem); st = IoInitializeRemoveLock(&drv->removeLock); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } + drv->removeLockReady = true; // Phase 6: IDENTIFY (polled - scheduler not yet running). st = port_identify(drv); if (!KSUCCESS(st)) { DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } @@ -799,12 +1048,14 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { // Phase 7: enable per-port interrupts. // Clear any stale IS first so we don't get a spurious signal immediately. port->is = (uint32_t)-1; - port->ie = PORT_IE_ALL; + port->ie = AHCI_PORT_TERMINAL_MASK; // Phase 8: create and register a disk DEVICE_OBJECT. st = IoCreateDevice(&g_driver, 0, &drv->deviceObject); if (!KSUCCESS(st)) { DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", g_nextDiskIndex, st); + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } drv->deviceObject->DeviceExtension = drv; @@ -822,19 +1073,29 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) { DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st); IoDeleteDevice(drv->deviceObject); drv->deviceObject = (DEVICE_OBJECT*)0; + AhciReleaseDriveRuntime(drv); + AhciReleaseDriveDma(drv); continue; } // Phase 9: spawn the per-drive IRP worker thread. // The worker owns do_read / do_write exclusively once the scheduler // is live; it is the only thread that may call port_issue_cmd(). - st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv, - PriorityHigh, &drv->workerThread); + st = KeEventCreate(EventManualReset, 0, &drv->workerFinished); + if (KSUCCESS(st)) { + drv->workerRunning = true; + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh); + st = KeCreateKernelThread((uintptr_t)AhciWorker, + (uintptr_t)drv, + &workerParameters, + &drv->workerThread); + } if (!KSUCCESS(st)) { - DbgPrint("AHCI: drive %u: worker thread create failed (0x%x)\n", - diskIndex, st); - // Fatal for this device: all disk access is IRP-based now, so - // leave the device registered but clearly fail during testing. + drv->workerRunning = false; + DbgPrint("AHCI: drive %u: worker creation failed (0x%x)\n", diskIndex, st); + (void)IoUnregisterDiskDevice(diskIndex); + ctl->driveCount--; + continue; } } @@ -889,7 +1150,7 @@ static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevic // Implements the driver Entry operation. KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { (void)init; - if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt) + if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Dma) return NTLSTATUS_INVALID_PARAMETER; DbgPrint("AHCI: driver loading...\n"); @@ -903,6 +1164,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { g_driver.DriverAddDevice = AhciAddDevice; g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite; g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite; + g_driver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AhciDispatchFlush; g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp; KSTATUS st = IoRegisterDriverObject(&g_driver); diff --git a/code/drivers/storage/ahci/ahci.h b/code/drivers/storage/ahci/ahci.h index 1547f32..f80dea8 100644 --- a/code/drivers/storage/ahci/ahci.h +++ b/code/drivers/storage/ahci/ahci.h @@ -250,6 +250,7 @@ typedef volatile struct { #define PORT_CMD_ICC_ACTIVE (0x1u << 28) // Port IS register bits (interrupt status). +#define PORT_IS_DHRS (1u << 0) // D2H Register FIS Received. #define PORT_IS_TFES (1u << 30) // Task File Error Status #define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error #define PORT_IS_HBDS (1u << 28) // Host Bus Data Error @@ -321,11 +322,13 @@ typedef struct { uint8_t portNo; // Physical port index (for ISR dispatch) // Structures block (command list + FIS + cmd tables). - uint64_t structPhys; // Physical base - void* structVirt; // Virtual base + IO_DMA_BUFFER* structDma; + uint64_t structDeviceAddress; // Device base. + void* structVirt; // CPU base. // Bounce buffer for DMA data. - uint64_t bouncePhys; + IO_DMA_BUFFER* bounceDma; + uint64_t bounceDeviceAddress; void* bounceVirt; // Signaled by the completion DPC when the port raises an interrupt. @@ -345,13 +348,16 @@ typedef struct { IO_REMOVE_LOCK removeLock; bool surpriseRemoved; bool removed; + bool removeLockReady; + volatile bool workerRunning; THREAD* workerThread; + EVENT* workerFinished; DEVICE_OBJECT* deviceObject; // Multi-slot NCQ needs per-slot ownership state in addition to the serialized queue: // uint32_t slotMask; // bitmask of in-flight slots // IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP - // uint64_t slotBouncePhys[NUM_CMD_SLOTS]; + // uint64_t slotBounceDeviceAddress[NUM_CMD_SLOTS]; // void* slotBounceVirt[NUM_CMD_SLOTS]; // volatile uint32_t prevSact; // previous port->sact snapshot. } AHCI_DRIVE; @@ -361,6 +367,7 @@ struct AHCI_CONTROLLER { DEVICE_OBJECT* fdo; DEVICE_OBJECT* pdo; DEVICE_OBJECT* lowerDevice; + IO_DMA_ADAPTER* dmaAdapter; PCI_DEVICE_INFO pciInfo; PCI_BAR abar; diff --git a/code/drivers/storage/ahci/ahci_port.c b/code/drivers/storage/ahci/ahci_port.c index 6012db0..6d82b81 100644 --- a/code/drivers/storage/ahci/ahci_port.c +++ b/code/drivers/storage/ahci/ahci_port.c @@ -11,16 +11,13 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) { int pageIndex = 0; for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) { uintptr_t virt = (uintptr_t)gAhciDdk->Memory->PhysToVirt(phys); - uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE; - KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent); - // Ignore NTLSTATUS_ALREADY_EXISTS - a prior mapping with different flags - // is possible for the first page if the bootloader was overly generous. - // Use MmPteProtectRange to force NOCACHE in that case. - if (st == NTLSTATUS_ALREADY_EXISTS) { - st = MmPteProtectRange(virt, 4096, flags); - } + uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC; + KSTATUS st = gAhciDdk->Memory->MapPage(virt, + phys, + flags, + NtlDdkMapReplaceExisting); if (!KSUCCESS(st)) { - DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n", + DbgPrint("AHCI: MMIO map failed for page %d (phys=0x%016llX, st=0x%x)\n", pageIndex, (unsigned long long)phys, st); return st; } diff --git a/code/drivers/storage/ata/ata.c b/code/drivers/storage/ata/ata.c index adb76db..af04614 100644 --- a/code/drivers/storage/ata/ata.c +++ b/code/drivers/storage/ata/ata.c @@ -92,7 +92,7 @@ static KSTATUS AtaIdentify(ATA_DEVICE* dev) { uint16_t ident[256]; memset(ident, 0, sizeof(ident)); - KSTATUS st = KeWaitSemaphore(ch->Lock); + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return st; @@ -193,7 +193,7 @@ static KSTATUS AtaReadPio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, void* b ATA_CHANNEL* ch = dev->Channel; uint16_t* out = (uint16_t*)buffer; - KSTATUS st = KeWaitSemaphore(ch->Lock); + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return st; @@ -234,6 +234,22 @@ static KSTATUS AtaFlush(ATA_DEVICE* dev) { return AtaWaitNotBusy(ch); } +// Flushes one device while holding the shared channel lock. +static KSTATUS AtaFlushSerialized(ATA_DEVICE* dev) { + if (!dev || !dev->Channel) + return NTLSTATUS_INVALID_PARAMETER; + + ATA_CHANNEL* ch = dev->Channel; + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); + if (!KSUCCESS(st)) + return st; + + AtaSelectDrive(dev); + st = AtaFlush(dev); + KeSignalSemaphore(ch->Lock); + return st; +} + // Updates pio. static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const void* buffer) { if (!dev || !buffer) @@ -246,7 +262,7 @@ static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const ATA_CHANNEL* ch = dev->Channel; const uint16_t* in = (const uint16_t*)buffer; - KSTATUS st = KeWaitSemaphore(ch->Lock); + KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE); if (!KSUCCESS(st)) return st; @@ -308,27 +324,38 @@ static void AtaWorker(uintptr_t arg) { return; while (true) { - KeWaitSemaphore(dev->IrpQueue.Semaphore); + KeWaitSemaphore(dev->IrpQueue.Semaphore, KE_WAIT_INFINITE); IRP* irp = IrpQueueDequeue(&dev->IrpQueue); if (!irp) continue; - bool write = false; - uint64_t lba = 0; - uint32_t sectors = 0; - void* buffer = (void*)0; - KSTATUS st = AtaGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); - if (KSUCCESS(st)) { - if (lba + sectors > dev->SectorCount) - st = NTLSTATUS_INVALID_PARAMETER; - else - st = write ? AtaWritePio(dev, lba, sectors, buffer) - : AtaReadPio(dev, lba, sectors, buffer); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + KSTATUS st = NTLSTATUS_INVALID_PARAMETER; + size_t information = 0; + + if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) { + st = AtaFlushSerialized(dev); + } else { + bool write = false; + uint64_t lba = 0; + uint32_t sectors = 0; + void* buffer = (void*)0; + st = AtaGetIrpReadWrite(irp, &write, &lba, §ors, &buffer); + if (KSUCCESS(st)) { + if (lba + sectors > dev->SectorCount) { + st = NTLSTATUS_INVALID_PARAMETER; + } else { + st = write ? AtaWritePio(dev, lba, sectors, buffer) + : AtaReadPio(dev, lba, sectors, buffer); + if (KSUCCESS(st)) + information = (size_t)sectors * ATA_SECTOR_SIZE; + } + } } IoReleaseRemoveLock(&dev->RemoveLock, irp); - IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * ATA_SECTOR_SIZE) : 0); + IoCompleteRequest(irp, st, information); } } @@ -363,6 +390,31 @@ KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) { return NTLSTATUS_PENDING; } +// Queues one cache flush on the device worker. +static KSTATUS AtaDispatchFlush(DEVICE_OBJECT* device, IRP* irp) { + if (!device || !irp || !device->DeviceExtension) + return NTLSTATUS_INVALID_PARAMETER; + + ATA_DEVICE* dev = (ATA_DEVICE*)device->DeviceExtension; + if (dev->Magic != ATA_DEVICE_MAGIC || !dev->Present) + return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS st = IoAcquireRemoveLock(&dev->RemoveLock, irp); + if (!KSUCCESS(st)) + return st; + + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); + if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) { + IoReleaseRemoveLock(&dev->RemoveLock, irp); + return NTLSTATUS_INVALID_PARAMETER; + } + + irp->Status = NTLSTATUS_FAILURE; + irp->Information = 0; + IrpQueueEnqueue(&dev->IrpQueue, irp); + return NTLSTATUS_PENDING; +} + // Returns free Disk Index. static uint8_t AtaFindFreeDiskIndex(void) { for (uint8_t i = 0; i < 8; i++) { @@ -486,8 +538,9 @@ static KSTATUS AtaProbeDrive(ATA_CONTROLLER* ctl, ATA_CHANNEL* ch, uint8_t drive return st; } + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh); st = KeCreateKernelThread((uintptr_t)AtaWorker, (uintptr_t)dev, - PriorityHigh, &dev->WorkerThread); + &workerParameters, &dev->WorkerThread); if (!KSUCCESS(st)) DbgPrint("ATA: drive %u worker thread create failed (0x%x)\n", dev->DiskIndex, st); return NTLSTATUS_SUCCESS; @@ -580,7 +633,7 @@ static void AtaRemoveControllerDevices(ATA_CONTROLLER* ctl, IO_DEVICE_CHANGE cha if (!dev->Present) continue; - IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); + IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); if (dev->DiskIndex != 0xFFu) { if (change == IoDeviceSurpriseRemoved) IoNotifyDiskSurpriseRemoved(dev->DiskIndex); @@ -647,12 +700,12 @@ KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) { case IRP_MN_SURPRISE_REMOVAL: device->PnpState = DevicePnpSurpriseRemoved; dev->Present = false; - IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); + IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); return NTLSTATUS_SUCCESS; case IRP_MN_REMOVE_DEVICE: device->PnpState = DevicePnpDeleted; dev->Present = false; - IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); + IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev); IoReleaseRemoveLockAndWait(&dev->RemoveLock, irp); IoDeleteDevice(device); dev->DeviceObject = (DEVICE_OBJECT*)0; @@ -708,6 +761,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) { gAtaDriver.DriverAddDevice = AtaAddDevice; gAtaDriver.MajorFunction[IRP_MJ_READ] = AtaDispatchReadWrite; gAtaDriver.MajorFunction[IRP_MJ_WRITE] = AtaDispatchReadWrite; + gAtaDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AtaDispatchFlush; gAtaDriver.MajorFunction[IRP_MJ_PNP] = AtaDispatchPnp; KSTATUS st = IoRegisterDriverObject(&gAtaDriver); diff --git a/code/kernel/hal/hal_processor.c b/code/kernel/hal/hal_processor.c index 860ba78..4ac4efe 100644 --- a/code/kernel/hal/hal_processor.c +++ b/code/kernel/hal/hal_processor.c @@ -37,6 +37,6 @@ __declspec(dllexport) void HalHalt(void) { // Masks interrupts and permanently stops the current processor. __declspec(dllexport) _Noreturn void HalStopProcessor(void) { _disable(); - for (;;) + while (true) __halt(); } diff --git a/code/kernel/include/kernel/io_dma.h b/code/kernel/include/kernel/io_dma.h new file mode 100644 index 0000000..4556a5b --- /dev/null +++ b/code/kernel/include/kernel/io_dma.h @@ -0,0 +1,13 @@ +// Kernel DMA declarations shared by the I/O manager and DDK provider. + +#pragma once + +#include + +struct DEVICE_OBJECT; + +// Returns the kernel-owned DDK DMA table. +const NTL_DDK_DMA* IoGetDmaDdk(void); + +// Detaches the adapter owned by a device before the device object can die. +void IoDmaDetachDevice(struct DEVICE_OBJECT* device); diff --git a/code/kernel/include/kernel/io_file.h b/code/kernel/include/kernel/io_file.h index 928adf8..34982d6 100644 --- a/code/kernel/include/kernel/io_file.h +++ b/code/kernel/include/kernel/io_file.h @@ -12,6 +12,16 @@ struct PROCESS; +// Initializes filesystem namespace change notification before drivers register volumes. +KSTATUS IoInitializeVolumeNamespace(void); + +// Returns the generation of the mounted-volume namespace. +uint64_t IoGetVolumeNamespaceGeneration(void); + +// Waits until the mounted-volume namespace changes or the timeout expires. +KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration, + uint64_t timeoutMilliseconds); + __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf, size_t size, diff --git a/code/kernel/include/kernel/ke_audio.h b/code/kernel/include/kernel/ke_audio.h new file mode 100644 index 0000000..eea1974 --- /dev/null +++ b/code/kernel/include/kernel/ke_audio.h @@ -0,0 +1,183 @@ +// Kernel audio-core declarations shared by the mixer, syscall layer, and selftests. + +#pragma once + +#include +#include +#include +#include +#include + +#define KE_AUDIO_MAX_ENDPOINTS 32u +#define KE_AUDIO_MAX_STREAMS_PER_ENDPOINT 64u +#define KE_AUDIO_STREAM_RING_FRAMES 4096u +#define KE_AUDIO_OUTPUT_SAMPLE_RATE 48000u +#define KE_AUDIO_OUTPUT_CHANNELS 2u +#define KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD 512u +#define KE_AUDIO_OUTPUT_PERIOD_COUNT 4u +#define KE_AUDIO_OUTPUT_BYTES_PER_FRAME 4u +#define KE_AUDIO_OUTPUT_PERIOD_BYTES 2048u +#define KE_AUDIO_GAIN_RAMP_FRAMES 480u +#define KE_AUDIO_IDLE_STOP_MILLISECONDS 250u +#define KE_AUDIO_MIXER_REALTIME_PRIORITY 16u + +#define KE_AUDIO_WAIT_WRITABLE 0x00000001u +#define KE_AUDIO_WAIT_CONSUMED 0x00000002u +#define KE_AUDIO_WAIT_PRESENTED 0x00000004u +#define KE_AUDIO_WAIT_STOPPED 0x00000008u +#define KE_AUDIO_WAIT_DRAINED 0x00000010u +#define KE_AUDIO_WAIT_FAILED 0x00000020u +#define KE_AUDIO_WAIT_DISCONTINUITY 0x00000040u +#define KE_AUDIO_WAIT_VALID_MASK 0x0000007Fu + +typedef enum KE_AUDIO_ENDPOINT_STATE { + KeAudioEndpointStarting = 1, + KeAudioEndpointReady, + KeAudioEndpointStopping, + KeAudioEndpointFailed, + KeAudioEndpointRemoved +} KE_AUDIO_ENDPOINT_STATE; + +typedef enum KE_AUDIO_STREAM_STATE { + KeAudioStreamCreated = 1, + KeAudioStreamRunning, + KeAudioStreamPaused, + KeAudioStreamDraining, + KeAudioStreamStopped, + KeAudioStreamFailed, + KeAudioStreamClosed +} KE_AUDIO_STREAM_STATE; + +typedef struct KE_AUDIO_STREAM KE_AUDIO_STREAM; + +// Reports one endpoint snapshot without exposing provider-owned pointers. +typedef struct KE_AUDIO_ENDPOINT_INFO { + uint64_t EndpointId; + IO_AUDIO_ENDPOINT_KIND Kind; + KE_AUDIO_ENDPOINT_STATE State; + uint32_t Capabilities; + IO_AUDIO_FORMAT HardwareFormat; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + uint8_t Available; + uint8_t IsDefault; + uint8_t HardwareRunning; + uint8_t MasterMuted; + int32_t MasterHardwareGainMilliDb; + float MasterRequestedVolume; + uint32_t StreamCount; + uint32_t ActiveStreamCount; + uint64_t PeriodsMixed; + uint64_t PeriodsCompleted; + uint64_t FramesPresented; + uint64_t HardwarePositionFrames; + uint64_t InterruptCount; + uint64_t LastPresentationTimestampTicks; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t DiscontinuityGeneration; + KSTATUS FailureStatus; +} KE_AUDIO_ENDPOINT_INFO; + +// Selects the next condition that should signal one stream's wait event. +typedef struct KE_AUDIO_STREAM_WAIT_CONDITION { + uint32_t Mask; + uint32_t WritableFrames; + uint64_t ConsumedFrames; + uint64_t PresentedFrames; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; +} KE_AUDIO_STREAM_WAIT_CONDITION; + +// Reports one logical stream snapshot. +typedef struct KE_AUDIO_STREAM_INFO { + KE_AUDIO_STREAM_STATE State; + IO_AUDIO_FORMAT Format; + uint32_t CapacityFrames; + uint32_t BufferedFrames; + uint32_t WritableFrames; + uint64_t FramesSubmitted; + uint64_t FramesConsumed; + uint64_t FramesPresented; + uint64_t FramesDiscarded; + uint64_t OutputFramesMixed; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; + KSTATUS FailureStatus; + float Volume; + float Pan; + uint8_t Muted; +} KE_AUDIO_STREAM_INFO; + +// Initializes endpoint registry and mixer ownership before drivers load. +void KeAudioInitialize(void); + +// Returns the provider-facing audio DDK table. +const NTL_DDK_AUDIO* KeAudioGetDdk(void); + +// Copies current endpoint snapshots and reports the full available count. +KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t capacity, + uint32_t* countOut); + +// Returns the currently selected endpoint identifier, or zero when none is usable. +uint64_t KeAudioGetDefaultEndpointId(void); + +// Creates one fixed-format logical render stream and returns a caller-owned reference. +KSTATUS KeAudioCreateStream(uint64_t endpointId, + const IO_AUDIO_FORMAT* format, + KE_AUDIO_STREAM** outStream); + +// Records one process handle that owns the stream. +void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream); + +// Releases one process-handle owner without blocking the handle-table path. +void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream); + +// Returns a borrowed passive event valid while the caller retains the stream. +EVENT* KeAudioStreamWaitEvent(KE_AUDIO_STREAM* stream); + +// Arms the stream event for one or more progress conditions without a lost-wakeup window. +KSTATUS KeAudioArmStreamWait(KE_AUDIO_STREAM* stream, + const KE_AUDIO_STREAM_WAIT_CONDITION* condition); + +// Copies as many complete source frames as fit without blocking. +KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream, + const void* frames, + uint32_t requestedFrames, + uint32_t* writtenFramesOut); + +// Starts one created or stopped stream. +KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream); + +// Pauses future mixing while preserving source frames not yet consumed. +KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream); + +// Resumes one paused stream. +KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream); + +// Rejects new writes and stops after every submitted frame is presented or discarded. +KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream); + +// Stops future mixing and discards source frames not yet committed to endpoint periods. +KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream); + +// Invalidates queued presentation accounting and returns an empty stream to position zero. +KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream); + +// Updates per-stream software gain with a ten-millisecond ramp. +KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream, + float volume, + float pan, + bool muted); + +// Updates endpoint master software gain and separate logical mute state. +KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted); + +// Copies one coherent stream snapshot. +KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut); + +// Runs the final render path selftest behind KE_DEBUG_AUDIO_SELFTEST. +void KiRunAudioFinalPathSelfTest(void); diff --git a/code/kernel/include/kernel/ke_cpu.h b/code/kernel/include/kernel/ke_cpu.h index a1987b7..7e373a4 100644 --- a/code/kernel/include/kernel/ke_cpu.h +++ b/code/kernel/include/kernel/ke_cpu.h @@ -7,6 +7,7 @@ // owns CPU-local scheduler, IRQL, GDT, and TSS state. #include +#include #include #include @@ -83,6 +84,8 @@ extern KCPU gCpus[KE_MAX_CPUS]; extern uint32_t gCpuCount; void KeInitializeBootstrapCpu(void); +void KeInitializeMemoryTypes(void); +bool KeWriteCombiningAvailable(void); void KeLoadKernelGsBase(KCPU* cpu); void KeSetUserGsBase(uintptr_t userGsBase); uintptr_t KeGetUserGsBase(void); diff --git a/code/kernel/include/kernel/ke_debug.h b/code/kernel/include/kernel/ke_debug.h index 86e2d69..946a061 100644 --- a/code/kernel/include/kernel/ke_debug.h +++ b/code/kernel/include/kernel/ke_debug.h @@ -10,21 +10,6 @@ #include #include -#define KE_DEBUG_CONSOLE 0x00000001u -#define KE_DEBUG_SELFTEST 0x00000002u -#define KE_DEBUG_FILE_LOG 0x00000004u - -#define KE_DEBUG_POOL_CANARY 0x00000008u -#define KE_DEBUG_POOL_POISON 0x00000010u -#define KE_DEBUG_POOL_QUARANTINE 0x00000020u -#define KE_DEBUG_POOL_CALLERS 0x00000040u -#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u -#define KE_DEBUG_DISPLAY_TRACE 0x00000100u -#define KE_DEBUG_INPUT_TRACE 0x00000200u -#define KE_DEBUG_VM_ASSERTS 0x00000400u -#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u -#define KE_DEBUG_LIST_CHECKS 0x00001000u - void KeInitDebug(const HAL_API* hal, uint32_t flags); uint32_t KeGetDebugFlags(void); __declspec(dllexport) void DbgPrint(const char* fmt, ...); diff --git a/code/kernel/include/kernel/ke_display.h b/code/kernel/include/kernel/ke_display.h index 733e006..66204b0 100644 --- a/code/kernel/include/kernel/ke_display.h +++ b/code/kernel/include/kernel/ke_display.h @@ -28,6 +28,9 @@ typedef struct DISPLAY_CLOCK_OBJECT { } DISPLAY_CLOCK_OBJECT; void KeDisplayInitialize(const HAL_API* hal); + +// Completes every console glyph queued before the call. +KSTATUS KeDisplayFlushConsoleOutput(void); DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock); DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock); DISPLAY_API void KeDisplayTick(void); diff --git a/code/kernel/include/kernel/ke_mutex.h b/code/kernel/include/kernel/ke_mutex.h index 6a88659..52b3aad 100644 --- a/code/kernel/include/kernel/ke_mutex.h +++ b/code/kernel/include/kernel/ke_mutex.h @@ -1,20 +1,37 @@ -// Kernel ke mutex declarations shared by implementation files and kernel clients. +// Kernel mutex declarations shared by implementation files and kernel clients. #pragma once -// ke_mutex.h - binary ownership mutex. + #include #include #include +#include #include +struct THREAD; + typedef struct MUTEX { - OBJECT Header; - SPINLOCK Lock; - struct THREAD* Owner; // NULL if unowned - LIST_ENTRY WaitList; + OBJECT Header; + SPINLOCK Lock; + struct THREAD* Owner; + LIST_ENTRY WaitList; + LIST_ENTRY OwnerLink; + KE_SCHEDULING_CLASS DonationClass; + uint32_t DonationPriority; + uint8_t HasDonation; + uint8_t Abandoned; } MUTEX; -__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out); -__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex); -__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex); -__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex); +// Tests whether a mutex acquisition transferred ownership to the caller. +#define KE_MUTEX_ACQUIRE_SUCCEEDED(status) \ + ((status) == NTLSTATUS_SUCCESS || (status) == NTLSTATUS_ABANDONED) + +__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out); +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds); +__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex); + +// Removes one blocked waiter while preserving priority-inheritance state. +int KiMutexAbortWait(MUTEX* mutex, struct THREAD* thread); + +// Releases every mutex owned by a terminating thread as abandoned. +void KiMutexRundownThread(struct THREAD* thread); diff --git a/code/kernel/include/kernel/ke_process.h b/code/kernel/include/kernel/ke_process.h index 495df82..b33d805 100644 --- a/code/kernel/include/kernel/ke_process.h +++ b/code/kernel/include/kernel/ke_process.h @@ -30,6 +30,7 @@ #define HANDLE_RIGHT_QUERY 0x00000020u #define HANDLE_RIGHT_TERMINATE 0x00000040u #define HANDLE_RIGHT_LOCK 0x00000080u +#define HANDLE_RIGHT_RESUME 0x00000100u #define HANDLE_RIGHT_ALL 0xFFFFFFFFu struct SYS_EXCEPTION_STATS; @@ -62,7 +63,8 @@ typedef struct PROCESS { uint8_t CurrentDrive; SPINLOCK CwdLock; - // Thread list. + // Live thread set. Dead THREAD objects may outlive this membership while + // retained handles remain waitable and queryable. LIST_ENTRY ThreadList; // links THREAD.ProcessLink uint32_t ThreadCount; diff --git a/code/kernel/include/kernel/ke_sched.h b/code/kernel/include/kernel/ke_sched.h index 972a8bb..e9bb39a 100644 --- a/code/kernel/include/kernel/ke_sched.h +++ b/code/kernel/include/kernel/ke_sched.h @@ -1,18 +1,20 @@ // Kernel ke sched declarations shared by implementation files and kernel clients. #pragma once -// ke_sched.h - Multi-Level Feedback Queue scheduler -// Three run queues (Low / Normal / High). LAPIC timer IRQs drive KeSchedulerTick. +// The scheduler combines dynamic normal queues with static real-time queues. +// LAPIC timer IRQs drive KeSchedulerTick. #include struct KCPU; #define SCHED_TICK_MS 10u +#define SCHED_REALTIME_QUANTUM_TICKS 1u #define SCHED_HIGH_QUANTUM_TICKS 2u #define SCHED_NORMAL_QUANTUM_TICKS 4u #define SCHED_LOW_QUANTUM_TICKS 8u #define SCHED_DEMOTE_AFTER_BURST_TICKS 16u #define SCHED_AGING_INTERVAL_TICKS 100u +#define KE_PRIORITY_INHERITANCE_MAX_DEPTH 64u typedef struct KE_SCHED_STATS { uint64_t Ticks; @@ -27,8 +29,61 @@ typedef struct KE_SCHED_STATS { uint64_t CpuBurstDemotions; uint64_t SmtSiblingDeferrals; uint64_t SmtPrimaryPokes; + uint64_t RealtimeDispatches; + uint64_t RealtimePreemptions; + uint64_t PriorityInheritanceBoosts; + uint64_t PriorityInheritanceDrops; + uint64_t PriorityInheritancePropagations; + uint64_t PriorityInheritanceMaxDepth; } KE_SCHED_STATS; + +struct MUTEX; + +typedef struct KE_THREAD_SCHEDULER_SNAPSHOT { + THREAD_STATE State; + KE_SCHEDULING_CLASS BaseSchedulingClass; + uint32_t BasePriority; + KE_SCHEDULING_CLASS EffectiveSchedulingClass; + uint32_t EffectivePriority; + uint32_t RunningCpu; + uint32_t LastCpu; + uint32_t QuantumRemaining; + uint32_t QuantumTicks; + uint32_t CpuBurstTicks; + uint32_t LastRunTicks; + uint32_t WaitObjectType; + KSTATUS WaitStatus; + struct MUTEX* WaitingMutex; + uint8_t Queued; + uint8_t AlertableWait; +} KE_THREAD_SCHEDULER_SNAPSHOT; + +// Copies scheduler-owned thread state into a stable diagnostic snapshot. +void KiSchedulerSnapshotThread(THREAD* thread, KE_THREAD_SCHEDULER_SNAPSHOT* snapshot); + +// Selects waiters and refreshes donations while the mutex lock is held. +struct THREAD* KiSchedulerSelectMutexWaiter(struct MUTEX* mutex); +int KiSchedulerRefreshMutexDonation(struct MUTEX* mutex); + +// Updates one thread's normal-class base priority. +KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority); + +// Publishes one mutex dependency atomically with cycle detection. +KSTATUS KiSchedulerPrepareMutexWait(THREAD* thread, struct MUTEX* mutex, THREAD* owner); +void KiSchedulerCancelMutexWait(THREAD* thread, struct MUTEX* mutex); + +// Associates mutex ownership with priority-inheritance bookkeeping. +void KiSchedulerAcquireMutexOwnership(THREAD* thread, struct MUTEX* mutex); +void KiSchedulerTransferMutexOwnership(THREAD* oldOwner, THREAD* newOwner, + struct MUTEX* mutex); +void KiSchedulerReleaseMutexOwnership(THREAD* thread, struct MUTEX* mutex); +struct MUTEX* KiSchedulerFirstOwnedMutex(THREAD* thread); +void KiSchedulerNotePriorityPropagation(uint32_t depth); + +// Recomputes effective rank and optionally returns a referenced upstream mutex. +int KiSchedulerRefreshEffectivePriority(THREAD* thread, struct MUTEX** waitingMutexOut); + // Called once by KernelMain before any threads are created. void KeInitializeScheduler(void); diff --git a/code/kernel/include/kernel/ke_semaphore.h b/code/kernel/include/kernel/ke_semaphore.h index 36b302b..a9be982 100644 --- a/code/kernel/include/kernel/ke_semaphore.h +++ b/code/kernel/include/kernel/ke_semaphore.h @@ -17,6 +17,5 @@ typedef struct SEMAPHORE { } SEMAPHORE; __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out); -__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore); -__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* semaphore); +__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore, uint64_t timeoutMilliseconds); __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* semaphore); diff --git a/code/kernel/include/kernel/ke_thread.h b/code/kernel/include/kernel/ke_thread.h index 6f6e397..201d446 100644 --- a/code/kernel/include/kernel/ke_thread.h +++ b/code/kernel/include/kernel/ke_thread.h @@ -18,6 +18,7 @@ #define KE_USER_EXCEPTION_MAX_DEPTH 8u typedef struct THREAD THREAD; +struct MUTEX; #ifndef NTL_THREAD_PRIORITY_DEFINED typedef enum THREAD_PRIORITY { @@ -34,6 +35,7 @@ typedef enum THREAD_STATE { ThreadRunning = 1, ThreadWaiting = 2, ThreadDead = 3, + ThreadSuspended = 4, } THREAD_STATE; typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread, @@ -51,17 +53,20 @@ typedef struct KCONTEXT { struct THREAD { OBJECT Header; // must be first - uint32_t ThreadId; - THREAD_STATE State; - THREAD_PRIORITY Priority; + uint32_t ThreadId; + THREAD_STATE State; + KE_SCHEDULING_CLASS BaseSchedulingClass; + uint32_t BasePriority; + KE_SCHEDULING_CLASS EffectiveSchedulingClass; + uint32_t EffectivePriority; // CPU-burst mirror kept here for the assembly-stable THREAD layout. // Dynamic quantum state lives later in the structure. - uint32_t QuantaConsumed; + uint32_t QuantaConsumed; PROCESS* Owner; LIST_ENTRY ProcessLink; // node in Owner->ThreadList - LIST_ENTRY SchedulerLink; // node in run queue or wait list + LIST_ENTRY SchedulerLink; // node in run queue or object wait list // Kernel stack - never PAGE_USER, pointed to by TssRsp0. void* KernelStackBase; @@ -108,6 +113,12 @@ struct THREAD { uint32_t CpuBurstTicks; uint32_t LastRunTicks; uint32_t RunningCpu; + uint32_t SuspendCount; + + // Priority inheritance ownership is scheduler-owned. Mutex wait-list + // membership remains protected by the mutex spinlock. + LIST_ENTRY OwnedMutexList; + struct MUTEX* WaitingMutex; // Scheduler hardening state. Queued means membership in a scheduler ready // queue only; wait-object membership is tracked by WaitListHead. Idle @@ -125,7 +136,9 @@ struct THREAD { KSTATUS WaitStatus; LIST_ENTRY* WaitListHead; uint8_t AlertableWait; - uint64_t SleepUntilTick; + LIST_ENTRY TimeoutLink; + uint8_t TimeoutArmed; + uint64_t TimeoutTick; KSTATUS TimeoutStatus; // Optional cleanup hook for waits that use auxiliary wait blocks. @@ -167,6 +180,11 @@ struct THREAD { uint32_t UserExceptionNoncontinuableMask; uintptr_t UserExceptionFrameUserAddress; KSTATUS UserExceptionExitStatus[KE_USER_EXCEPTION_MAX_DEPTH]; + + // Synchronous kernel work can temporarily retain caller-owned stack or + // process memory. Process termination must wait until that ownership has + // returned instead of freeing the thread underneath an active driver. + volatile long TerminationDeferralCount; }; // Creates and queues a user thread. On success, *out receives a referenced @@ -174,24 +192,31 @@ struct THREAD { // after copying any needed IDs/state. The live scheduler/thread lifetime is // separate and remains valid until the thread exits and is reaped. __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out); + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out); -// Create a kernel-mode-only thread. fn is called directly in ring 0 with +// Cancels a thread that has never left its initial suspended state. This drops +// the scheduler-owned lifetime reference but leaves the caller's reference +// intact. +KSTATUS KeCancelSuspendedThread(THREAD* thread); + +// Creates a kernel-mode-only thread. fn is called directly in ring 0 with // arg; when fn returns the thread exits automatically. No user stack or TEB -// is allocated. The thread is owned by the system process and runs at the -// given priority. On success, non-NULL out receives a referenced THREAD -// pointer owned by the caller; release it with ObDereferenceObject() after -// copying any needed IDs/state or after joining/waiting on caller-owned -// completion state. The scheduler's live thread reference is separate. +// is allocated. The thread is owned by the system process. On success, +// non-NULL out receives a referenced THREAD pointer owned by the caller; +// release it with ObDereferenceObject() after copying any needed IDs/state or +// after joining caller-owned completion state. The scheduler's live thread +// reference is separate. __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out); -__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, uint64_t affinityMask, + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out); // Mark a thread dead and deschedule it permanently. __declspec(dllexport) void KeExitThread(THREAD* thread, int code); +// Publishes completion, releases per-thread user memory, and removes one dead +// thread from its process live-thread set. Retained handles remain waitable. +void KeFinalizeThreadExit(THREAD* thread); + // Block / unblock primitives used by sync objects. // Correct wait pattern: @@ -213,12 +238,17 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable); KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable); void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status); void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status); -void KeAbortThreadWait(THREAD* thread, KSTATUS status); +int KeAbortThreadWait(THREAD* thread, KSTATUS status); void KeUnblockThread(THREAD* thread); KSTATUS KeAlertThread(THREAD* thread); void KiWakeThreadForKernelApc(THREAD* thread); void KiWakeThreadForUserApc(THREAD* thread); +// Defers process termination while kernel work owns caller-backed storage. +void KeEnterTerminationDeferral(void); +void KeLeaveTerminationDeferral(void); +int KeThreadTerminationDeferred(const THREAD* thread); + // Assembly stub - implemented in ke_switch.asm. void KeSwitchContext(THREAD* from, THREAD* to); __declspec(noreturn) void KeStartInitialThread(THREAD* thread); diff --git a/code/kernel/include/kernel/ke_time.h b/code/kernel/include/kernel/ke_time.h index 813db9a..8870f7a 100644 --- a/code/kernel/include/kernel/ke_time.h +++ b/code/kernel/include/kernel/ke_time.h @@ -11,3 +11,6 @@ void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal); KSTATUS KeTimeReloadSettings(void); __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime); __declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void); + +// Returns the monotonic platform counter in microseconds. +uint64_t KeQueryMonotonicTicks(void); diff --git a/code/kernel/include/kernel/mm_file_cache.h b/code/kernel/include/kernel/mm_file_cache.h index 603bb7c..a31e13e 100644 --- a/code/kernel/include/kernel/mm_file_cache.h +++ b/code/kernel/include/kernel/mm_file_cache.h @@ -43,19 +43,6 @@ typedef struct MM_FILE_CACHE_STATS { uint64_t DropFailures; uint64_t StandbyTransitions; uint64_t StandbyHits; - uint64_t ReadAheadRequests; - uint64_t ReadAheadPagesRequested; - uint64_t ReadAheadPagesQueued; - uint64_t ReadAheadPagesLoaded; - uint64_t ReadAheadPagesSkipped; - uint64_t ReadAheadFailures; - uint64_t ReadAheadPagesThrottled; - uint64_t MaxReadAheadPagesPerRequest; - uint64_t BatchReads; - uint64_t BatchPagesRequested; - uint64_t BatchPagesLoaded; - uint64_t MaxBatchPagesRequested; - uint64_t MaxBatchPagesLoaded; uint64_t ResidentPages; uint64_t ActivePages; uint64_t StandbyPages; @@ -92,8 +79,4 @@ void MmFileCacheInvalidateAll(FILE_OBJECT* file); // it never writes back because v1 stores only clean file data. KSTATUS MmFileCacheDropOnePageForPressure(void); -// Queues best-effort read-ahead for clean file pages. The queue owns a file -// reference until the worker has consumed the request. -KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages); - void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out); diff --git a/code/kernel/include/kernel/mm_pfn.h b/code/kernel/include/kernel/mm_pfn.h index 1ee0843..2a559b8 100644 --- a/code/kernel/include/kernel/mm_pfn.h +++ b/code/kernel/include/kernel/mm_pfn.h @@ -124,6 +124,12 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage); __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage); +PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count, + PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive, + size_t alignment, + size_t boundary, + MM_PFN_USAGE usage); __declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame); __declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame); __declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame); diff --git a/code/kernel/include/kernel/mm_pte.h b/code/kernel/include/kernel/mm_pte.h index 6ef7520..83c8497 100644 --- a/code/kernel/include/kernel/mm_pte.h +++ b/code/kernel/include/kernel/mm_pte.h @@ -12,10 +12,11 @@ #define PAGE_PRESENT (1ULL << 0) #define PAGE_WRITE (1ULL << 1) #define PAGE_USER (1ULL << 2) -#ifndef PAGE_WRITETHROUGH -#define PAGE_WRITETHROUGH (1ULL << 3) -#endif -#define PAGE_NOCACHE (1ULL << 4) +// Cache policy is selected through the canonical PAT layout. +#define PAGE_CACHE_WB 0ULL +#define PAGE_CACHE_WC (1ULL << 3) +#define PAGE_CACHE_UC (1ULL << 4) +#define PAGE_CACHE_MASK (PAGE_CACHE_WC | PAGE_CACHE_UC) #define PAGE_ACCESSED (1ULL << 5) #define PAGE_DIRTY (1ULL << 6) #define PAGE_NX (1ULL << 63) diff --git a/code/kernel/include/kernel/mm_virt.h b/code/kernel/include/kernel/mm_virt.h index c3c4ef4..6691d3c 100644 --- a/code/kernel/include/kernel/mm_virt.h +++ b/code/kernel/include/kernel/mm_virt.h @@ -66,6 +66,15 @@ typedef struct MM_PINNED_USER_PAGE { } MM_PINNED_USER_PAGE; typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER; +typedef struct MM_KERNEL_VIEW MM_KERNEL_VIEW; + +// Maps existing frames into a stable kernel-only virtual range. The view owns +// virtual address space and PTEs, but it does not own the physical frames. +KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames, + size_t pageCount, + MM_KERNEL_VIEW** outView); +void* MmKernelViewBase(const MM_KERNEL_VIEW* view); +void MmDestroyKernelView(MM_KERNEL_VIEW* view); // Pins one user page and returns a kernel direct-map alias for the requested // address. Callers may copy through KernelAddress + PageOffset while the page is @@ -80,6 +89,8 @@ __declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page); // unlocked. WriteAccess describes kernel writes to the user buffer. __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out); __declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer); +size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer); +PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index); // Virtual address descriptors. // The single shared address space still has per-process VAD ownership so process diff --git a/code/kernel/include/kernel/syscall_nr.h b/code/kernel/include/kernel/syscall_nr.h index 97ecb5f..4be8df4 100644 --- a/code/kernel/include/kernel/syscall_nr.h +++ b/code/kernel/include/kernel/syscall_nr.h @@ -17,138 +17,154 @@ #define SYS_GET_PROCESS_LIST 6 #define SYS_GET_PROCESS_PID 7 #define SYS_CREATE_THREAD 8 -#define SYS_GET_THREAD_LIST 9 -#define SYS_LOAD_MODULE 10 -#define SYS_UNLOAD_MODULE 11 -#define SYS_GET_MODULE_INFO 12 -#define SYS_GET_MODULE_EXPORT 13 -#define SYS_GET_MODULE_LIST 14 -#define SYS_GET_CURRENT_TID 15 -#define SYS_GET_CURRENT_PID 16 -#define SYS_SET_THREAD_PRIORITY 17 -#define SYS_YIELD 18 -#define SYS_SLEEP 19 -#define SYS_WAIT_SINGLE 20 -#define SYS_WAIT_MANY 21 -#define SYS_SET_USER_APC_DISPATCHER 22 -#define SYS_APC_RETURN 23 -#define SYS_SET_EXCEPTION_DISPATCHER 24 -#define SYS_GET_EXCEPTION_DISPATCHER 25 -#define SYS_CONTINUE_EXCEPTION 26 -#define SYS_RAISE_EXCEPTION 27 +#define SYS_RESUME_THREAD 9 +#define SYS_GET_THREAD_LIST 10 +#define SYS_LOAD_MODULE 11 +#define SYS_UNLOAD_MODULE 12 +#define SYS_GET_MODULE_INFO 13 +#define SYS_GET_MODULE_EXPORT 14 +#define SYS_GET_MODULE_LIST 15 +#define SYS_GET_CURRENT_TID 16 +#define SYS_GET_CURRENT_PID 17 +#define SYS_SET_THREAD_PRIORITY 18 +#define SYS_YIELD 19 +#define SYS_SLEEP 20 +#define SYS_WAIT_SINGLE 21 +#define SYS_WAIT_MANY 22 +#define SYS_SET_USER_APC_DISPATCHER 23 +#define SYS_APC_RETURN 24 +#define SYS_SET_EXCEPTION_DISPATCHER 25 +#define SYS_GET_EXCEPTION_DISPATCHER 26 +#define SYS_CONTINUE_EXCEPTION 27 +#define SYS_RAISE_EXCEPTION 28 // Virtual memory, mappings, and handles. -#define SYS_ALLOC_VM 28 -#define SYS_FREE_VM 29 -#define SYS_PROTECT_VM 30 -#define SYS_QUERY_VM 31 -#define SYS_CLOSE_HANDLE 32 -#define SYS_DUPLICATE_HANDLE 33 -#define SYS_CREATE_FILE_MAPPING 34 -#define SYS_MAP_FILE 35 +#define SYS_ALLOC_VM 29 +#define SYS_FREE_VM 30 +#define SYS_PROTECT_VM 31 +#define SYS_QUERY_VM 32 +#define SYS_CLOSE_HANDLE 33 +#define SYS_DUPLICATE_HANDLE 34 +#define SYS_CREATE_FILE_MAPPING 35 +#define SYS_MAP_FILE 36 // Synchronization objects. -#define SYS_CREATE_MUTEX 36 -#define SYS_TRY_ACQUIRE_MUTEX 37 -#define SYS_RELEASE_MUTEX 38 -#define SYS_CREATE_SEM 39 -#define SYS_TRY_WAIT_SEM 40 -#define SYS_SIGNAL_SEM 41 -#define SYS_CREATE_EVENT 42 -#define SYS_SIGNAL_EVENT 43 -#define SYS_CLEAR_EVENT 44 +#define SYS_CREATE_MUTEX 37 +#define SYS_TRY_ACQUIRE_MUTEX 38 +#define SYS_RELEASE_MUTEX 39 +#define SYS_CREATE_SEM 40 +#define SYS_TRY_WAIT_SEM 41 +#define SYS_SIGNAL_SEM 42 +#define SYS_CREATE_EVENT 43 +#define SYS_SIGNAL_EVENT 44 +#define SYS_CLEAR_EVENT 45 // Queues and named queues. -#define SYS_CREATE_QUEUE 45 -#define SYS_SEND_QUEUE 46 -#define SYS_RECV_QUEUE 47 -#define SYS_PUBLISH_QUEUE 48 -#define SYS_OPEN_NAMED_QUEUE 49 -#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 50 +#define SYS_CREATE_QUEUE 46 +#define SYS_SEND_QUEUE 47 +#define SYS_RECV_QUEUE 48 +#define SYS_PUBLISH_QUEUE 49 +#define SYS_OPEN_NAMED_QUEUE 50 +#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 51 // Files, directories, and current directory. -#define SYS_OPEN_FILE 51 -#define SYS_READ_FILE 52 -#define SYS_WRITE_FILE 53 -#define SYS_READ_FILE_ASYNC 54 -#define SYS_WRITE_FILE_ASYNC 55 -#define SYS_CANCEL_IO 56 -#define SYS_SEEK_FILE 57 -#define SYS_GET_FILE_POS 58 -#define SYS_GET_FILE_SIZE 59 -#define SYS_GET_FILE_STAT 60 -#define SYS_STAT_FILE 61 -#define SYS_TRUNCATE_FILE 62 -#define SYS_FLUSH_FILE 63 -#define SYS_DELETE_FILE 64 -#define SYS_RENAME_FILE 65 -#define SYS_LOCK_FILE 66 -#define SYS_UNLOCK_FILE 67 -#define SYS_OPEN_DIR 68 -#define SYS_READ_DIR 69 -#define SYS_CREATE_DIR 70 -#define SYS_REMOVE_DIR 71 -#define SYS_SET_CWD 72 -#define SYS_REAL_PATH 73 +#define SYS_OPEN_FILE 52 +#define SYS_READ_FILE 53 +#define SYS_WRITE_FILE 54 +#define SYS_READ_FILE_ASYNC 55 +#define SYS_WRITE_FILE_ASYNC 56 +#define SYS_CANCEL_IO 57 +#define SYS_SEEK_FILE 58 +#define SYS_GET_FILE_POS 59 +#define SYS_GET_FILE_SIZE 60 +#define SYS_GET_FILE_STAT 61 +#define SYS_STAT_FILE 62 +#define SYS_TRUNCATE_FILE 63 +#define SYS_FLUSH_FILE 64 +#define SYS_DELETE_FILE 65 +#define SYS_RENAME_FILE 66 +#define SYS_LOCK_FILE 67 +#define SYS_UNLOCK_FILE 68 +#define SYS_OPEN_DIR 69 +#define SYS_READ_DIR 70 +#define SYS_CREATE_DIR 71 +#define SYS_REMOVE_DIR 72 +#define SYS_SET_CWD 73 +#define SYS_REAL_PATH 74 // Console, display, and surfaces. -#define SYS_WRITE_CONSOLE 74 -#define SYS_GET_CONSOLE_INFO 75 -#define SYS_SET_CONSOLE_CURSOR 76 -#define SYS_FILL_CONSOLE_CHARACTER 77 -#define SYS_FILL_CONSOLE_ATTRIBUTE 78 -#define SYS_PUT_SCREEN_CHAR 79 -#define SYS_CLEAR_SCREEN 80 -#define SYS_GET_SCREEN_SIZE 81 -#define SYS_QUERY_DISPLAY_INFO 82 -#define SYS_OPEN_DISPLAY_CLOCK 83 -#define SYS_CREATE_SURFACE 84 -#define SYS_MAP_SURFACE 85 -#define SYS_QUERY_SURFACE 86 -#define SYS_PRESENT_SURFACE 87 +#define SYS_WRITE_CONSOLE 75 +#define SYS_GET_CONSOLE_INFO 76 +#define SYS_SET_CONSOLE_CURSOR 77 +#define SYS_FILL_CONSOLE_CHARACTER 78 +#define SYS_FILL_CONSOLE_ATTRIBUTE 79 +#define SYS_PUT_SCREEN_CHAR 80 +#define SYS_CLEAR_SCREEN 81 +#define SYS_GET_SCREEN_SIZE 82 +#define SYS_QUERY_DISPLAY_INFO 83 +#define SYS_OPEN_DISPLAY_CLOCK 84 +#define SYS_CREATE_SURFACE 85 +#define SYS_MAP_SURFACE 86 +#define SYS_QUERY_SURFACE 87 +#define SYS_PRESENT_SURFACE 88 + +// Native render audio. +#define SYS_AUDIO_ENUMERATE_ENDPOINTS 89 +#define SYS_AUDIO_OPEN_STREAM 90 +#define SYS_AUDIO_WRITE 91 +#define SYS_AUDIO_ARM_WAIT 92 +#define SYS_AUDIO_QUERY_STREAM 93 +#define SYS_AUDIO_START 94 +#define SYS_AUDIO_PAUSE 95 +#define SYS_AUDIO_RESUME 96 +#define SYS_AUDIO_DRAIN 97 +#define SYS_AUDIO_STOP 98 +#define SYS_AUDIO_RESET 99 +#define SYS_AUDIO_SET_VOLUME 100 +#define SYS_AUDIO_SET_MASTER_VOLUME 101 // Input. -#define SYS_POLL_KEYBOARD 88 -#define SYS_IS_KEY_DOWN 89 -#define SYS_READ_KEYBOARD_CHAR 90 -#define SYS_READ_KEY 91 -#define SYS_GET_KEY_MODIFIERS 92 -#define SYS_QUERY_GLOBAL_INPUT_STATE 93 -#define SYS_OPEN_INPUT_QUEUE 94 -#define SYS_READ_INPUT_EVENT 95 -#define SYS_PEEK_INPUT_EVENT 96 -#define SYS_PEEK_MOUSE 97 -#define SYS_READ_MOUSE 98 -#define SYS_GET_MOUSE_POS 99 +#define SYS_POLL_KEYBOARD 102 +#define SYS_IS_KEY_DOWN 103 +#define SYS_READ_KEYBOARD_CHAR 104 +#define SYS_READ_KEY 105 +#define SYS_GET_KEY_MODIFIERS 106 +#define SYS_QUERY_GLOBAL_INPUT_STATE 107 +#define SYS_OPEN_INPUT_QUEUE 108 +#define SYS_READ_INPUT_EVENT 109 +#define SYS_PEEK_INPUT_EVENT 110 +#define SYS_PEEK_MOUSE 111 +#define SYS_READ_MOUSE 112 +#define SYS_GET_MOUSE_POS 113 // Time, debug, and system information. -#define SYS_CREATE_TIMER 100 -#define SYS_SET_TIMER 101 -#define SYS_CANCEL_TIMER 102 -#define SYS_GET_TICK 103 -#define SYS_GET_TICK_FREQUENCY 104 -#define SYS_GET_SYSTEM_TIME 105 -#define SYS_PRINT_DEBUG 106 -#define SYS_QUERY_TRAP_FRAME 107 -#define SYS_GET_MEMORY_STATUS 108 -#define SYS_QUERY_KERNEL_STATS 109 +#define SYS_CREATE_TIMER 114 +#define SYS_SET_TIMER 115 +#define SYS_CANCEL_TIMER 116 +#define SYS_GET_TICK 117 +#define SYS_GET_TICK_FREQUENCY 118 +#define SYS_GET_SYSTEM_TIME 119 +#define SYS_PRINT_DEBUG 120 +#define SYS_QUERY_TRAP_FRAME 121 +#define SYS_GET_MEMORY_STATUS 122 +#define SYS_QUERY_KERNEL_STATS 123 // Native sockets and polling. -#define SYS_CREATE_SOCKET 110 -#define SYS_SOCKET_IO_ASYNC 111 -#define SYS_BIND_SOCKET 112 -#define SYS_LISTEN_SOCKET 113 -#define SYS_ACCEPT_SOCKET 114 -#define SYS_CONNECT_SOCKET 115 -#define SYS_SEND_SOCKET 116 -#define SYS_RECV_SOCKET 117 -#define SYS_SHUTDOWN_SOCKET 118 -#define SYS_SEND_TO_SOCKET 119 -#define SYS_RECV_FROM_SOCKET 120 -#define SYS_RESOLVE_HOST 121 -#define SYS_SET_SOCKET_OPTION 122 -#define SYS_GET_SOCKET_OPTION 123 -#define SYS_POLL 124 -#define SYS_UNPUBLISH_QUEUE 125 +#define SYS_CREATE_SOCKET 124 +#define SYS_SOCKET_IO_ASYNC 125 +#define SYS_BIND_SOCKET 126 +#define SYS_LISTEN_SOCKET 127 +#define SYS_ACCEPT_SOCKET 128 +#define SYS_CONNECT_SOCKET 129 +#define SYS_SEND_SOCKET 130 +#define SYS_RECV_SOCKET 131 +#define SYS_SHUTDOWN_SOCKET 132 +#define SYS_SEND_TO_SOCKET 133 +#define SYS_RECV_FROM_SOCKET 134 +#define SYS_RESOLVE_HOST 135 +#define SYS_SET_SOCKET_OPTION 136 +#define SYS_GET_SOCKET_OPTION 137 +#define SYS_POLL 138 +#define SYS_UNPUBLISH_QUEUE 139 -#define SYS_MAX 126 +#define SYS_MAX 140 diff --git a/code/kernel/io_ddk.c b/code/kernel/io_ddk.c index 595abc9..d769d78 100644 --- a/code/kernel/io_ddk.c +++ b/code/kernel/io_ddk.c @@ -5,8 +5,12 @@ #include #include #include +#include +#include +#include #include #include +#include static const HAL_API* gIoDdkHal; @@ -68,6 +72,7 @@ static KSTATUS IoDdkAllocateMsiVector(uint8_t* outIrq) { static const NTL_DDK_DEBUG gIoDdkDebug = { .Print = DbgPrint, + .GetFlags = KeGetDebugFlags, }; static const NTL_DDK_INTERRUPT gIoDdkInterrupt = { @@ -99,6 +104,8 @@ static NTL_DDK_TIMER gIoDdkTimer = { static NTL_DDK gIoDdk = { .Version = NTL_DDK_VERSION, .Debug = &gIoDdkDebug, + .Audio = (const NTL_DDK_AUDIO*)0, + .Dma = (const NTL_DDK_DMA*)0, .Interrupt = &gIoDdkInterrupt, .Memory = &gIoDdkMemory, .Platform = &gIoDdkPlatform, @@ -109,6 +116,8 @@ static NTL_DDK gIoDdk = { // Initializes the kernel-owned DDK table from platform services. void IoInitializeDdk(const HAL_API* hal) { gIoDdkHal = hal; + gIoDdk.Audio = KeAudioGetDdk(); + gIoDdk.Dma = IoGetDmaDdk(); gIoDdkTimer.QueryTicks = hal ? hal->Timer.QueryTicks : 0; gIoDdkTimer.Stall = hal ? hal->Timer.Stall : 0; } diff --git a/code/kernel/io_device.c b/code/kernel/io_device.c index 4ff8247..500b9b4 100644 --- a/code/kernel/io_device.c +++ b/code/kernel/io_device.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,7 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) { static void IoDeviceDestroy(OBJECT* object) { DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object; + IoDmaDetachDevice(dev); for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) { if (gPhysicalDevices[i] == dev) { @@ -316,12 +318,19 @@ static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) { // Releases device. __declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) { - if (!device) return; - if (device->DeletePending) + if (!device) return; + uintptr_t flags = KeAcquireSpinlock(&device->Lock); + if (device->DeletePending) { + KeReleaseSpinlock(&device->Lock, flags); + return; + } device->DeletePending = true; device->PnpState = DevicePnpDeleted; + KeReleaseSpinlock(&device->Lock, flags); + + IoDmaDetachDevice(device); IoUnlinkDeviceFromParent(device); IoRemovePhysicalDeviceRegistryEntry(device); IoDereferenceDeviceObject(device); @@ -441,167 +450,50 @@ static uint32_t gDiskChangeCallbackCount; static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS]; static uint32_t gDiskArrivalCallbackCount; -#define IO_DISK_READAHEAD_MAX_SECTORS 128u -#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u -#define IO_DISK_READAHEAD_MIN_SECTORS 8u #define IO_DISK_SECTOR_BYTES 512u -typedef struct IO_DISK_READAHEAD_STATE { +typedef struct IO_DISK_STATS_STATE { SPINLOCK Lock; - uint8_t* Buffer; - uint8_t Valid; - uint8_t Loading; - uint64_t Generation; - uint64_t BaseLba; - uint32_t Count; - uint64_t NextLba; - uint32_t SequentialScore; IO_DISK_STATS Stats; -} IO_DISK_READAHEAD_STATE; +} IO_DISK_STATS_STATE; -static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES]; +static IO_DISK_STATS_STATE gDiskStats[MAX_DISK_DEVICES]; -// Updates a maximum statistic while holding the disk read-ahead lock. -static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) { - if (field && *field < value) *field = value; -} - -// Prepares the per-disk read-ahead buffer. -static void IoDiskReadAheadPrepare(uint8_t driveIndex) { +// Initializes per-disk I/O statistics. +static void IoDiskStatsPrepare(uint8_t driveIndex) { if (driveIndex >= MAX_DISK_DEVICES) return; - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex]; state->Lock = (SPINLOCK)SPINLOCK_INIT; - state->Valid = 0; - state->Loading = 0; - state->Generation = 0; - state->BaseLba = 0; - state->Count = 0; - state->NextLba = 0; - state->SequentialScore = 0; memset(&state->Stats, 0, sizeof(state->Stats)); - - if (!state->Buffer) { - state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES, - MM_POOL_TAG('d', 'r', 'a', 'h')); - if (!state->Buffer) - state->Stats.ReadaheadAllocFailures++; - } } -// Releases the per-disk read-ahead buffer. -static void IoDiskReadAheadRelease(uint8_t driveIndex) { +// Records one disk read request. +static void IoDiskStatsNoteRead(uint8_t driveIndex, uint32_t count) { if (driveIndex >= MAX_DISK_DEVICES) return; - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - uint8_t* buffer = (uint8_t*)0; - for (;;) { - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - if (!state->Loading) { - buffer = state->Buffer; - state->Buffer = (uint8_t*)0; - state->Valid = 0; - state->Generation++; - state->Count = 0; - state->SequentialScore = 0; - KeReleaseSpinlock(&state->Lock, flags); - break; - } - KeReleaseSpinlock(&state->Lock, flags); - KeYield(); - } - - if (buffer) - MmFreePool(buffer); -} - -// Invalidates cached disk read-ahead data for one disk. -static void IoDiskReadAheadInvalidate(uint8_t driveIndex) { - if (driveIndex >= MAX_DISK_DEVICES) - return; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - state->Valid = 0; - state->Generation++; - state->Count = 0; - state->Stats.ReadaheadInvalidations++; - KeReleaseSpinlock(&state->Lock, flags); -} - -// Returns true when one range fully contains another range. -static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount, - uint64_t innerBase, uint32_t innerCount) { - uint64_t outerEnd = outerBase + outerCount; - uint64_t innerEnd = innerBase + innerCount; - if (outerEnd < outerBase || innerEnd < innerBase) - return false; - return innerBase >= outerBase && innerEnd <= outerEnd; -} - -// Attempts to satisfy a disk read from the per-disk read-ahead buffer. -static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) { - if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0) - return false; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - bool hit = state->Buffer && state->Valid && !state->Loading && - IoDiskRangeContains(state->BaseLba, state->Count, lba, count); - if (hit) { - size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES; - size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES; - memcpy(buffer, state->Buffer + offset, bytes); - state->Stats.ReadaheadHits++; - state->Stats.ReadaheadBytesServed += bytes; - } else { - state->Stats.ReadaheadMisses++; - } - KeReleaseSpinlock(&state->Lock, flags); - return hit; -} - -// Records request size and sequential access state for one disk read. -static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) { - if (driveIndex >= MAX_DISK_DEVICES || count == 0) - return 0; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex]; uintptr_t flags = KeAcquireSpinlock(&state->Lock); state->Stats.ReadRequests++; state->Stats.ReadSectors += count; - IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count); - - if (lba == state->NextLba) { - if (state->SequentialScore < 32) - state->SequentialScore++; - } else { - state->SequentialScore = 1; - } - state->NextLba = lba + count; - uint32_t score = state->SequentialScore; - if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE) - state->Stats.ReadaheadSequentialDetections++; + if (state->Stats.MaxReadSectors < count) + state->Stats.MaxReadSectors = count; KeReleaseSpinlock(&state->Lock, flags); - return score; } -// Records one disk write and invalidates stale read-ahead data. -static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) { +// Records one successful disk write request. +static void IoDiskStatsNoteWrite(uint8_t driveIndex, uint32_t count) { if (driveIndex >= MAX_DISK_DEVICES) return; - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; + IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex]; uintptr_t flags = KeAcquireSpinlock(&state->Lock); state->Stats.WriteRequests++; state->Stats.WriteSectors += count; - IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count); - state->Valid = 0; - state->Generation++; - state->Count = 0; - state->Stats.ReadaheadInvalidations++; + if (state->Stats.MaxWriteSectors < count) + state->Stats.MaxWriteSectors = count; KeReleaseSpinlock(&state->Lock, flags); } @@ -653,7 +545,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, if (pnpStatus == NTLSTATUS_NOT_SUPPORTED) device->PnpState = DevicePnpStarted; - IoDiskReadAheadPrepare(driveIndex); + IoDiskStatsPrepare(driveIndex); IoReferenceDeviceObject(device); gDiskDevices[driveIndex] = device; @@ -668,8 +560,6 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) { DEVICE_OBJECT* dev = gDiskDevices[driveIndex]; if (!dev) return NTLSTATUS_NOT_FOUND; - IoDiskReadAheadInvalidate(driveIndex); - dev->PnpState = DevicePnpSurpriseRemoved; IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved); (void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL); @@ -698,7 +588,6 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) { (void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE); gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0; - IoDiskReadAheadRelease(driveIndex); IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved); IoDereferenceDeviceObject(dev); return NTLSTATUS_SUCCESS; @@ -846,83 +735,6 @@ static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba, return IoSubmitIrpSync(dev, &irp); } -// Chooses a conservative read-ahead span from the current sequential score. -static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) { - uint32_t sectors = currentCount; - if (sectors < IO_DISK_READAHEAD_MIN_SECTORS) - sectors = IO_DISK_READAHEAD_MIN_SECTORS; - if (score >= 8 && sectors < 128u) - sectors = 128u; - else if (score >= 4 && sectors < 64u) - sectors = 64u; - else if (score >= 2 && sectors < 32u) - sectors = 32u; - if (sectors > IO_DISK_READAHEAD_MAX_SECTORS) - sectors = IO_DISK_READAHEAD_MAX_SECTORS; - return sectors; -} - -// Loads the next sequential disk range into the read-ahead buffer. -static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev, - uint64_t nextLba, uint32_t currentCount, uint32_t score) { - if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE) - return; - - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex]; - if (!state->Buffer) { - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - state->Stats.ReadaheadSuppressed++; - KeReleaseSpinlock(&state->Lock, flags); - return; - } - - uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score); - uint64_t diskSectors = dev->u.Disk.SectorCount; - if (nextLba >= diskSectors) - return; - if ((uint64_t)sectors > diskSectors - nextLba) - sectors = (uint32_t)(diskSectors - nextLba); - if (sectors == 0) - return; - - uintptr_t flags = KeAcquireSpinlock(&state->Lock); - bool covered = state->Valid && !state->Loading && - IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors); - if (covered) { - KeReleaseSpinlock(&state->Lock, flags); - return; - } - if (state->Loading) { - state->Stats.ReadaheadSuppressed++; - KeReleaseSpinlock(&state->Lock, flags); - return; - } - uint64_t generation = state->Generation; - state->Loading = 1; - state->Valid = 0; - state->Count = 0; - KeReleaseSpinlock(&state->Lock, flags); - - KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer); - - flags = KeAcquireSpinlock(&state->Lock); - state->Loading = 0; - if (KSUCCESS(st) && generation == state->Generation) { - state->BaseLba = nextLba; - state->Count = sectors; - state->Valid = 1; - state->Stats.ReadaheadStores++; - state->Stats.ReadaheadSectorsLoaded += sectors; - IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors); - } else { - state->Valid = 0; - state->Count = 0; - if (!KSUCCESS(st)) - state->Stats.ReadaheadReadFailures++; - } - KeReleaseSpinlock(&state->Lock, flags); -} - __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, uint64_t lba, uint32_t count, @@ -938,17 +750,8 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex, return rangeStatus; } - if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) { - uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count); - IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score); - IoDereferenceDeviceObject(dev); - return NTLSTATUS_SUCCESS; - } - KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer); - uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count); - if (KSUCCESS(st)) - IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score); + IoDiskStatsNoteRead(driveIndex, count); IoDereferenceDeviceObject(dev); return st; @@ -971,19 +774,39 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer); if (KSUCCESS(st)) - IoDiskReadAheadNoteWrite(driveIndex, count); + IoDiskStatsNoteWrite(driveIndex, count); IoDereferenceDeviceObject(dev); return st; } +// Flushes volatile write caches for one disk device. +__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex) { + DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex); + if (!dev) return NTLSTATUS_NOT_FOUND; + + IRP irp; + IoInitializeIrp(&irp, 1); + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(&irp); + if (!ios) { + IoDereferenceDeviceObject(dev); + return NTLSTATUS_FAILURE; + } + ios->MajorFunction = IRP_MJ_FLUSH_BUFFERS; + ios->MinorFunction = 0; + + KSTATUS st = IoSubmitIrpSync(dev, &irp); + IoDereferenceDeviceObject(dev); + return st; +} + // Queries aggregate disk I/O statistics. __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) { if (!out) return; memset(out, 0, sizeof(*out)); for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) { - IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i]; + IO_DISK_STATS_STATE* state = &gDiskStats[i]; uintptr_t flags = KeAcquireSpinlock(&state->Lock); if (gDiskDevices[i]) out->RegisteredDisks++; @@ -995,18 +818,6 @@ __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) { out->MaxReadSectors = state->Stats.MaxReadSectors; if (out->MaxWriteSectors < state->Stats.MaxWriteSectors) out->MaxWriteSectors = state->Stats.MaxWriteSectors; - out->ReadaheadHits += state->Stats.ReadaheadHits; - out->ReadaheadMisses += state->Stats.ReadaheadMisses; - out->ReadaheadStores += state->Stats.ReadaheadStores; - out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations; - out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed; - out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded; - out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections; - out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed; - out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures; - out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures; - if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors) - out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors; KeReleaseSpinlock(&state->Lock, flags); } } diff --git a/code/kernel/io_dma.c b/code/kernel/io_dma.c new file mode 100644 index 0000000..32f6065 --- /dev/null +++ b/code/kernel/io_dma.c @@ -0,0 +1,1056 @@ +// Kernel DMA core. This file owns device adapters, common buffers, pinned memory, streaming mappings, and bounce copies. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IO_DMA_ADAPTER_TAG MM_POOL_TAG('d', 'm', 'a', 'a') +#define IO_DMA_BUFFER_TAG MM_POOL_TAG('d', 'm', 'a', 'b') +#define IO_DMA_PIN_TAG MM_POOL_TAG('d', 'm', 'a', 'p') +#define IO_DMA_MAP_TAG MM_POOL_TAG('d', 'm', 'a', 'm') + +struct IO_DMA_ADAPTER { + volatile long RefCount; + volatile long Removed; + SPINLOCK Lock; + DEVICE_OBJECT* Device; + IO_DMA_REQUIREMENTS Requirements; +}; + +struct IO_DMA_BUFFER { + IO_DMA_ADAPTER* Adapter; + void* CpuAddress; + PHYSICAL_ADDRESS PhysicalAddress; + uint64_t DeviceAddress; + size_t Size; + size_t AllocationSize; + size_t PageCount; +}; + +typedef enum MM_PINNED_MEMORY_ORIGIN { + MmPinnedKernel = 1, + MmPinnedUser +} MM_PINNED_MEMORY_ORIGIN; + +struct MM_PINNED_MEMORY { + volatile long RefCount; + MM_PINNED_MEMORY_ORIGIN Origin; + IO_DMA_DIRECTION Direction; + void* Address; + size_t Size; + size_t PageCount; + size_t FirstPageOffset; + MM_LOCKED_USER_BUFFER* UserBuffer; + PHYSICAL_ADDRESS Frames[1]; +}; + +struct IO_DMA_MAPPING { + IO_DMA_ADAPTER* Adapter; + MM_PINNED_MEMORY* Memory; + IO_DMA_BUFFER* BounceBuffer; + size_t MemoryOffset; + size_t Length; + IO_DMA_DIRECTION Direction; + uint32_t SegmentCount; + int DeviceOwnsMapping; + IO_DMA_SEGMENT Segments[1]; +}; + +typedef struct IO_DMA_INTERNAL_STATS { + volatile long long AdaptersCreated; + volatile long long AdaptersDestroyed; + volatile long long AdaptersInUse; + volatile long long CommonBuffersAllocated; + volatile long long CommonBuffersFreed; + volatile long long CommonBuffersInUse; + volatile long long CommonBufferBytesInUse; + volatile long long PinnedKernelRanges; + volatile long long PinnedUserRanges; + volatile long long PinnedRangesReleased; + volatile long long PinnedRangesInUse; + volatile long long PinnedPagesInUse; + volatile long long StreamingMappings; + volatile long long StreamingMappingsUnmapped; + volatile long long StreamingMappingsInUse; + volatile long long DirectMappings; + volatile long long BounceMappings; + volatile long long BounceBytesCopiedToDevice; + volatile long long BounceBytesCopiedToCpu; + volatile long long FailedAllocations; + volatile long long FailedMappings; +} IO_DMA_INTERNAL_STATS; + +static IO_DMA_INTERNAL_STATS gIoDmaStats; + +// Returns whether a size is a nonzero power of two. +static int IoDmaIsPowerOfTwo(size_t value) { + return value && ((value & (value - 1)) == 0); +} + +// Normalizes and validates one adapter requirement set. +static KSTATUS IoDmaNormalizeRequirements(const IO_DMA_REQUIREMENTS* source, + IO_DMA_REQUIREMENTS* destination) { + if (!destination) + return NTLSTATUS_INVALID_PARAMETER; + + memset(destination, 0, sizeof(*destination)); + if (source) + *destination = *source; + + if (!destination->MaximumDeviceAddress) + destination->MaximumDeviceAddress = (uint64_t)-1; + if (!destination->Alignment) + destination->Alignment = 1; + if (!destination->MaximumSegmentLength) + destination->MaximumSegmentLength = (size_t)-1; + if (!destination->MaximumSegments) + destination->MaximumSegments = UINT32_MAX; + + if (!IoDmaIsPowerOfTwo(destination->Alignment)) + return NTLSTATUS_INVALID_PARAMETER; + if (destination->Boundary && !IoDmaIsPowerOfTwo(destination->Boundary)) + return NTLSTATUS_INVALID_PARAMETER; + return NTLSTATUS_SUCCESS; +} + +// Returns whether two normalized requirement sets are identical. +static int IoDmaRequirementsEqual(const IO_DMA_REQUIREMENTS* left, + const IO_DMA_REQUIREMENTS* right) { + return left && right && + left->MaximumDeviceAddress == right->MaximumDeviceAddress && + left->Alignment == right->Alignment && + left->Boundary == right->Boundary && + left->MaximumSegmentLength == right->MaximumSegmentLength && + left->MaximumSegments == right->MaximumSegments; +} + +// Returns whether device removal has closed the adapter to new ownership. +static int IoDmaAdapterRemoved(const IO_DMA_ADAPTER* adapter) { + if (!adapter) + return 1; + return _InterlockedCompareExchange((volatile long*)&adapter->Removed, 0, 0) != 0; +} + +// Adds one reference to an adapter that is already stabilized by its owner. +static void IoDmaReferenceAdapter(IO_DMA_ADAPTER* adapter) { + if (adapter) + _InterlockedIncrement(&adapter->RefCount); +} + +// Adds a reference only while the adapter still accepts new DMA ownership. +static int IoDmaTryReferenceAdapter(IO_DMA_ADAPTER* adapter) { + if (!adapter) + return 0; + uintptr_t flags = KeAcquireSpinlock(&adapter->Lock); + if (adapter->Removed) { + KeReleaseSpinlock(&adapter->Lock, flags); + return 0; + } + _InterlockedIncrement(&adapter->RefCount); + KeReleaseSpinlock(&adapter->Lock, flags); + return 1; +} + +// Releases one adapter reference and destroys the detached adapter at zero. +static void IoDmaPutAdapterImpl(IO_DMA_ADAPTER* adapter) { + if (!adapter) + return; + long remaining = _InterlockedDecrement(&adapter->RefCount); + if (remaining > 0) + return; + if (remaining < 0) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: adapter reference underflow adapter=0x%016llX\n", + (unsigned long long)(uintptr_t)adapter); + if (!IoDmaAdapterRemoved(adapter)) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: attached adapter reached zero references adapter=0x%016llX\n", + (unsigned long long)(uintptr_t)adapter); + _InterlockedIncrement64(&gIoDmaStats.AdaptersDestroyed); + _InterlockedDecrement64(&gIoDmaStats.AdaptersInUse); + MmFreePool(adapter); +} + +// Returns the one canonical DMA adapter owned by a device. +static KSTATUS IoDmaGetAdapterImpl(DEVICE_OBJECT* device, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_ADAPTER** outAdapter) { + if (!device || !outAdapter) + return NTLSTATUS_INVALID_PARAMETER; + *outAdapter = (IO_DMA_ADAPTER*)0; + + IO_DMA_REQUIREMENTS normalized; + KSTATUS status = IoDmaNormalizeRequirements(requirements, &normalized); + if (!KSUCCESS(status)) + return status; + + uintptr_t flags = KeAcquireSpinlock(&device->Lock); + if (device->DeletePending || device->PnpState == DevicePnpDeleted) { + KeReleaseSpinlock(&device->Lock, flags); + return NTLSTATUS_OBJECT_DEAD; + } + + IO_DMA_ADAPTER* adapter = device->DmaAdapter; + if (adapter) { + if (!IoDmaRequirementsEqual(&adapter->Requirements, &normalized)) { + KeReleaseSpinlock(&device->Lock, flags); + return NTLSTATUS_ALREADY_EXISTS; + } + IoDmaReferenceAdapter(adapter); + KeReleaseSpinlock(&device->Lock, flags); + *outAdapter = adapter; + return NTLSTATUS_SUCCESS; + } + KeReleaseSpinlock(&device->Lock, flags); + + IO_DMA_ADAPTER* candidate = (IO_DMA_ADAPTER*)MmAllocatePoolTag(sizeof(*candidate), + IO_DMA_ADAPTER_TAG); + if (!candidate) { + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + memset(candidate, 0, sizeof(*candidate)); + candidate->RefCount = 2; + candidate->Device = device; + candidate->Requirements = normalized; + + flags = KeAcquireSpinlock(&device->Lock); + if (device->DeletePending || device->PnpState == DevicePnpDeleted) { + KeReleaseSpinlock(&device->Lock, flags); + MmFreePool(candidate); + return NTLSTATUS_OBJECT_DEAD; + } + + adapter = device->DmaAdapter; + if (adapter) { + if (!IoDmaRequirementsEqual(&adapter->Requirements, &normalized)) { + KeReleaseSpinlock(&device->Lock, flags); + MmFreePool(candidate); + return NTLSTATUS_ALREADY_EXISTS; + } + IoDmaReferenceAdapter(adapter); + KeReleaseSpinlock(&device->Lock, flags); + MmFreePool(candidate); + *outAdapter = adapter; + return NTLSTATUS_SUCCESS; + } + + device->DmaAdapter = candidate; + KeReleaseSpinlock(&device->Lock, flags); + + _InterlockedIncrement64(&gIoDmaStats.AdaptersCreated); + _InterlockedIncrement64(&gIoDmaStats.AdaptersInUse); + *outAdapter = candidate; + return NTLSTATUS_SUCCESS; +} + +// Detaches a device-owned adapter so no new mappings can be created. +void IoDmaDetachDevice(DEVICE_OBJECT* device) { + if (!device) + return; + + uintptr_t flags = KeAcquireSpinlock(&device->Lock); + IO_DMA_ADAPTER* adapter = device->DmaAdapter; + device->DmaAdapter = (IO_DMA_ADAPTER*)0; + if (adapter) { + uintptr_t adapterFlags = KeAcquireSpinlock(&adapter->Lock); + adapter->Removed = 1; + adapter->Device = (DEVICE_OBJECT*)0; + KeReleaseSpinlock(&adapter->Lock, adapterFlags); + } + KeReleaseSpinlock(&device->Lock, flags); + + if (adapter) + IoDmaPutAdapterImpl(adapter); +} + +// Converts an inclusive device-address limit to the allocator's exclusive limit. +static PHYSICAL_ADDRESS IoDmaMaximumExclusive(const IO_DMA_ADAPTER* adapter) { + if (!adapter || adapter->Requirements.MaximumDeviceAddress == (uint64_t)-1) + return (PHYSICAL_ADDRESS)-1; + return adapter->Requirements.MaximumDeviceAddress + 1; +} + +// Allocates one contiguous backing run while applying address and alignment requirements. +static KSTATUS IoDmaAllocateCommonBufferInternal(IO_DMA_ADAPTER* adapter, + size_t size, + size_t alignment, + int enforceAllocationBoundary, + IO_DMA_BUFFER** outBuffer) { + if (!adapter || !outBuffer || !size) + return NTLSTATUS_INVALID_PARAMETER; + *outBuffer = (IO_DMA_BUFFER*)0; + if (IoDmaAdapterRemoved(adapter)) + return NTLSTATUS_OBJECT_DEAD; + + if (!alignment) + alignment = 1; + if (!IoDmaIsPowerOfTwo(alignment)) + return NTLSTATUS_INVALID_PARAMETER; + if (alignment < adapter->Requirements.Alignment) + alignment = adapter->Requirements.Alignment; + if (alignment < PAGE_SIZE) + alignment = PAGE_SIZE; + + if (size > ((size_t)-1) - (PAGE_SIZE - 1)) + return NTLSTATUS_NO_MEMORY; + size_t allocationSize = (size + PAGE_SIZE - 1) & PAGE_MASK; + size_t pageCount = allocationSize / PAGE_SIZE; + + size_t allocationBoundary = 0; + if (enforceAllocationBoundary && adapter->Requirements.Boundary) { + if (size > adapter->Requirements.Boundary) + return NTLSTATUS_NOT_SUPPORTED; + if (adapter->Requirements.Boundary >= PAGE_SIZE) + allocationBoundary = adapter->Requirements.Boundary; + } + + PHYSICAL_ADDRESS physical = MmPfnAllocateContiguousInRange( + pageCount, + PAGE_SIZE, + IoDmaMaximumExclusive(adapter), + alignment, + allocationBoundary, + MmPfnUseDma); + if (!physical) { + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)MmAllocatePoolTag(sizeof(*buffer), IO_DMA_BUFFER_TAG); + if (!buffer) { + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + memset(buffer, 0, sizeof(*buffer)); + buffer->Adapter = adapter; + buffer->CpuAddress = MmPfnToVirtual(physical); + buffer->PhysicalAddress = physical; + buffer->DeviceAddress = physical; + buffer->Size = size; + buffer->AllocationSize = allocationSize; + buffer->PageCount = pageCount; + if (!buffer->CpuAddress) { + MmFreePool(buffer); + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_FAILURE; + } + + memset(buffer->CpuAddress, 0, allocationSize); + if (!IoDmaTryReferenceAdapter(adapter)) { + MmFreePool(buffer); + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + return NTLSTATUS_OBJECT_DEAD; + } + _InterlockedIncrement64(&gIoDmaStats.CommonBuffersAllocated); + _InterlockedIncrement64(&gIoDmaStats.CommonBuffersInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, (long long)allocationSize); + *outBuffer = buffer; + return NTLSTATUS_SUCCESS; +} + +// Allocates one coherent common buffer represented by one device segment. +static KSTATUS IoDmaAllocateCommonBufferImpl(IO_DMA_ADAPTER* adapter, + size_t size, + size_t alignment, + IO_DMA_BUFFER** outBuffer) { + return IoDmaAllocateCommonBufferInternal(adapter, size, alignment, 1, outBuffer); +} + +// Frees one common buffer and its owned physical pages. +static void IoDmaFreeCommonBufferImpl(IO_DMA_BUFFER* buffer) { + if (!buffer) + return; + + IO_DMA_ADAPTER* adapter = buffer->Adapter; + PHYSICAL_ADDRESS physical = buffer->PhysicalAddress; + size_t pageCount = buffer->PageCount; + size_t allocationSize = buffer->AllocationSize; + memset(buffer, 0, sizeof(*buffer)); + MmFreePool(buffer); + + for (size_t i = 0; i < pageCount; ++i) + (void)MmPfnFree(physical + i * PAGE_SIZE); + + _InterlockedIncrement64(&gIoDmaStats.CommonBuffersFreed); + _InterlockedDecrement64(&gIoDmaStats.CommonBuffersInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, -(long long)allocationSize); + IoDmaPutAdapterImpl(adapter); +} + +// Returns the CPU address of a common buffer. +static void* IoDmaGetCommonBufferAddressImpl(const IO_DMA_BUFFER* buffer) { + return buffer ? buffer->CpuAddress : (void*)0; +} + +// Returns the first device address of a common buffer. +static uint64_t IoDmaGetCommonBufferDeviceAddressImpl(const IO_DMA_BUFFER* buffer) { + return buffer ? buffer->DeviceAddress : 0; +} + +// Returns the caller-requested size of a common buffer. +static size_t IoDmaGetCommonBufferSizeImpl(const IO_DMA_BUFFER* buffer) { + return buffer ? buffer->Size : 0; +} + +// Returns whether a pinned access mode can satisfy a mapping direction. +static int IoDmaDirectionAllows(IO_DMA_DIRECTION pinned, IO_DMA_DIRECTION mapped) { + if (pinned == IoDmaBidirectional) + return mapped == IoDmaToDevice || mapped == IoDmaFromDevice || mapped == IoDmaBidirectional; + return pinned == mapped; +} + +// Allocates a pinned-memory descriptor with room for every backing frame. +static MM_PINNED_MEMORY* IoDmaAllocatePinnedMemory(size_t pageCount) { + if (!pageCount || pageCount > (((size_t)-1) - sizeof(MM_PINNED_MEMORY)) / sizeof(PHYSICAL_ADDRESS)) + return (MM_PINNED_MEMORY*)0; + size_t bytes = sizeof(MM_PINNED_MEMORY) + (pageCount - 1) * sizeof(PHYSICAL_ADDRESS); + MM_PINNED_MEMORY* memory = (MM_PINNED_MEMORY*)MmAllocatePoolTag(bytes, IO_DMA_PIN_TAG); + if (memory) + memset(memory, 0, bytes); + return memory; +} + +// Releases a pinned-memory descriptor after its last mapping leaves. +static void IoDmaReleasePinnedMemory(MM_PINNED_MEMORY* memory) { + if (!memory) + return; + long remaining = _InterlockedDecrement(&memory->RefCount); + if (remaining > 0) + return; + if (remaining < 0) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: pinned-memory reference underflow memory=0x%016llX\n", + (unsigned long long)(uintptr_t)memory); + + if (memory->Origin == MmPinnedUser) { + MmUnlockUserBuffer(memory->UserBuffer); + } else { + for (size_t i = 0; i < memory->PageCount; ++i) + (void)MmPfnUnpin(memory->Frames[i]); + } + _InterlockedIncrement64(&gIoDmaStats.PinnedRangesReleased); + _InterlockedDecrement64(&gIoDmaStats.PinnedRangesInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, -(long long)memory->PageCount); + MmFreePool(memory); +} + +// Adds one mapping reference to pinned memory. +static void IoDmaReferencePinnedMemory(MM_PINNED_MEMORY* memory) { + if (memory) + _InterlockedIncrement(&memory->RefCount); +} + +// Pins a kernel virtual range for later DMA mapping. +static KSTATUS IoDmaPinKernelMemoryImpl(void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory) { + if (!outMemory) + return NTLSTATUS_INVALID_PARAMETER; + *outMemory = (MM_PINNED_MEMORY*)0; + if (!address || !size) + return NTLSTATUS_INVALID_PARAMETER; + if (direction < IoDmaToDevice || direction > IoDmaBidirectional) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t start = (uintptr_t)address; + if (start < KERNEL_BASE || start + size < start) + return NTLSTATUS_INVALID_PARAMETER; + uintptr_t firstPage = start & PAGE_MASK; + uintptr_t lastByte = start + size - 1; + size_t pageCount = ((lastByte & PAGE_MASK) - firstPage) / PAGE_SIZE + 1; + + MM_PINNED_MEMORY* memory = IoDmaAllocatePinnedMemory(pageCount); + if (!memory) { + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + memory->RefCount = 1; + memory->Origin = MmPinnedKernel; + memory->Direction = direction; + memory->Address = address; + memory->Size = size; + memory->PageCount = pageCount; + memory->FirstPageOffset = start & (PAGE_SIZE - 1); + + size_t pinned = 0; + KSTATUS status = NTLSTATUS_SUCCESS; + for (size_t i = 0; i < pageCount; ++i) { + PHYSICAL_ADDRESS physical = MmPteTranslate(firstPage + i * PAGE_SIZE); + PHYSICAL_ADDRESS frame = physical & PAGE_MASK; + if (!physical || !MmPfnGet(frame)) { + status = NTLSTATUS_NOT_FOUND; + break; + } + status = MmPfnPin(frame); + if (!KSUCCESS(status)) + break; + memory->Frames[i] = frame; + pinned++; + } + + if (!KSUCCESS(status)) { + for (size_t i = 0; i < pinned; ++i) + (void)MmPfnUnpin(memory->Frames[i]); + MmFreePool(memory); + return status; + } + + _InterlockedIncrement64(&gIoDmaStats.PinnedKernelRanges); + _InterlockedIncrement64(&gIoDmaStats.PinnedRangesInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (long long)pageCount); + *outMemory = memory; + return NTLSTATUS_SUCCESS; +} + +// Pins a current-process user range through the VAD locking contract. +static KSTATUS IoDmaPinUserMemoryImpl(struct PROCESS* process, + void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory) { + if (!outMemory) + return NTLSTATUS_INVALID_PARAMETER; + *outMemory = (MM_PINNED_MEMORY*)0; + if (!address || !size) + return NTLSTATUS_INVALID_PARAMETER; + if (direction < IoDmaToDevice || direction > IoDmaBidirectional) + return NTLSTATUS_INVALID_PARAMETER; + + if (!process) + return NTLSTATUS_ACCESS_DENIED; + + MM_LOCKED_USER_BUFFER* locked = (MM_LOCKED_USER_BUFFER*)0; + int writeAccess = direction == IoDmaFromDevice || direction == IoDmaBidirectional; + KSTATUS status = MmLockUserBuffer(process, address, size, writeAccess, &locked); + if (!KSUCCESS(status)) + return status; + + size_t pageCount = MmLockedUserBufferPageCount(locked); + MM_PINNED_MEMORY* memory = IoDmaAllocatePinnedMemory(pageCount); + if (!memory) { + MmUnlockUserBuffer(locked); + _InterlockedIncrement64(&gIoDmaStats.FailedAllocations); + return NTLSTATUS_NO_MEMORY; + } + + memory->RefCount = 1; + memory->Origin = MmPinnedUser; + memory->Direction = direction; + memory->Address = address; + memory->Size = size; + memory->PageCount = pageCount; + memory->FirstPageOffset = (uintptr_t)address & (PAGE_SIZE - 1); + memory->UserBuffer = locked; + for (size_t i = 0; i < pageCount; ++i) + memory->Frames[i] = MmLockedUserBufferPageFrame(locked, i); + + _InterlockedIncrement64(&gIoDmaStats.PinnedUserRanges); + _InterlockedIncrement64(&gIoDmaStats.PinnedRangesInUse); + _InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (long long)pageCount); + *outMemory = memory; + return NTLSTATUS_SUCCESS; +} + +// Releases the caller's pinned-memory reference. +static void IoDmaUnpinMemoryImpl(MM_PINNED_MEMORY* memory) { + IoDmaReleasePinnedMemory(memory); +} + +// Returns the physical address and bytes remaining in one pinned page span. +static KSTATUS IoDmaGetPinnedSpan(const MM_PINNED_MEMORY* memory, + size_t offset, + PHYSICAL_ADDRESS* address, + size_t* length) { + if (!memory || !address || !length || offset >= memory->Size) + return NTLSTATUS_INVALID_PARAMETER; + + size_t absolute = memory->FirstPageOffset + offset; + size_t pageIndex = absolute / PAGE_SIZE; + size_t pageOffset = absolute & (PAGE_SIZE - 1); + if (pageIndex >= memory->PageCount || !memory->Frames[pageIndex]) + return NTLSTATUS_NOT_FOUND; + + *address = memory->Frames[pageIndex] + pageOffset; + *length = PAGE_SIZE - pageOffset; + size_t remaining = memory->Size - offset; + if (*length > remaining) + *length = remaining; + return NTLSTATUS_SUCCESS; +} + +// Copies a pinned range into one linear kernel buffer. +static KSTATUS IoDmaCopyFromPinned(const MM_PINNED_MEMORY* memory, + size_t offset, + void* destination, + size_t length) { + if (!memory || (!destination && length) || offset > memory->Size || length > memory->Size - offset) + return NTLSTATUS_INVALID_PARAMETER; + + uint8_t* output = (uint8_t*)destination; + size_t copied = 0; + while (copied < length) { + size_t absolute = memory->FirstPageOffset + offset + copied; + size_t pageIndex = absolute / PAGE_SIZE; + size_t pageOffset = absolute & (PAGE_SIZE - 1); + size_t chunk = PAGE_SIZE - pageOffset; + if (chunk > length - copied) + chunk = length - copied; + void* page = MmPfnToVirtual(memory->Frames[pageIndex]); + if (!page) + return NTLSTATUS_FAILURE; + memcpy(output + copied, (uint8_t*)page + pageOffset, chunk); + copied += chunk; + } + return NTLSTATUS_SUCCESS; +} + +// Copies one linear kernel buffer back into a pinned range. +static KSTATUS IoDmaCopyToPinned(MM_PINNED_MEMORY* memory, + size_t offset, + const void* source, + size_t length) { + if (!memory || (!source && length) || offset > memory->Size || length > memory->Size - offset) + return NTLSTATUS_INVALID_PARAMETER; + + const uint8_t* input = (const uint8_t*)source; + size_t copied = 0; + while (copied < length) { + size_t absolute = memory->FirstPageOffset + offset + copied; + size_t pageIndex = absolute / PAGE_SIZE; + size_t pageOffset = absolute & (PAGE_SIZE - 1); + size_t chunk = PAGE_SIZE - pageOffset; + if (chunk > length - copied) + chunk = length - copied; + void* page = MmPfnToVirtual(memory->Frames[pageIndex]); + if (!page) + return NTLSTATUS_FAILURE; + memcpy((uint8_t*)page + pageOffset, input + copied, chunk); + copied += chunk; + } + return NTLSTATUS_SUCCESS; +} + +// Returns the maximum bytes one segment may cover from a given address. +static size_t IoDmaSegmentCapacity(const IO_DMA_REQUIREMENTS* requirements, + uint64_t address, + size_t currentLength) { + size_t capacity = requirements->MaximumSegmentLength - currentLength; + if (requirements->Boundary) { + size_t boundaryRemaining = requirements->Boundary - + (size_t)(address & (requirements->Boundary - 1)); + if (boundaryRemaining > currentLength) + boundaryRemaining -= currentLength; + else + boundaryRemaining = 0; + if (capacity > boundaryRemaining) + capacity = boundaryRemaining; + } + return capacity; +} + +// Adds physical spans to a constrained scatter/gather list. +static KSTATUS IoDmaBuildSegments(const MM_PINNED_MEMORY* memory, + size_t offset, + size_t length, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_SEGMENT* segments, + uint32_t* segmentCount) { + if (!memory || !requirements || !segmentCount || !length) + return NTLSTATUS_INVALID_PARAMETER; + + uint32_t count = 0; + uint64_t currentAddress = 0; + size_t currentLength = 0; + size_t consumed = 0; + + while (consumed < length) { + PHYSICAL_ADDRESS address = 0; + size_t spanLength = 0; + KSTATUS status = IoDmaGetPinnedSpan(memory, offset + consumed, &address, &spanLength); + if (!KSUCCESS(status)) + return status; + if (spanLength > length - consumed) + spanLength = length - consumed; + + if (address > requirements->MaximumDeviceAddress || + spanLength - 1 > requirements->MaximumDeviceAddress - address) + return NTLSTATUS_NOT_SUPPORTED; + + size_t spanConsumed = 0; + while (spanConsumed < spanLength) { + uint64_t pieceAddress = address + spanConsumed; + if (currentLength && currentAddress + currentLength == pieceAddress) { + size_t capacity = IoDmaSegmentCapacity(requirements, currentAddress, currentLength); + if (capacity) { + size_t take = spanLength - spanConsumed; + if (take > capacity) + take = capacity; + currentLength += take; + if (segments) + segments[count - 1].Length = currentLength; + spanConsumed += take; + continue; + } + } + + if ((pieceAddress & (requirements->Alignment - 1)) != 0) + return NTLSTATUS_NOT_SUPPORTED; + if (count == requirements->MaximumSegments) + return NTLSTATUS_NOT_SUPPORTED; + + size_t capacity = requirements->MaximumSegmentLength; + if (requirements->Boundary) { + size_t boundaryRemaining = requirements->Boundary - + (size_t)(pieceAddress & (requirements->Boundary - 1)); + if (capacity > boundaryRemaining) + capacity = boundaryRemaining; + } + if (!capacity) + return NTLSTATUS_NOT_SUPPORTED; + + size_t take = spanLength - spanConsumed; + if (take > capacity) + take = capacity; + currentAddress = pieceAddress; + currentLength = take; + if (segments) { + segments[count].DeviceAddress = currentAddress; + segments[count].Length = currentLength; + } + count++; + spanConsumed += take; + } + consumed += spanLength; + } + + *segmentCount = count; + return NTLSTATUS_SUCCESS; +} + +// Builds a constrained scatter/gather list for one contiguous bounce allocation. +static KSTATUS IoDmaBuildContiguousSegments(uint64_t address, + size_t length, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_SEGMENT* segments, + uint32_t* segmentCount) { + if (!requirements || !segmentCount || !address || !length) + return NTLSTATUS_INVALID_PARAMETER; + if (address > requirements->MaximumDeviceAddress || + length - 1 > requirements->MaximumDeviceAddress - address) + return NTLSTATUS_NOT_SUPPORTED; + + uint32_t count = 0; + size_t consumed = 0; + while (consumed < length) { + uint64_t pieceAddress = address + consumed; + if ((pieceAddress & (requirements->Alignment - 1)) != 0) + return NTLSTATUS_NOT_SUPPORTED; + if (count == requirements->MaximumSegments) + return NTLSTATUS_NOT_SUPPORTED; + + size_t capacity = requirements->MaximumSegmentLength; + if (requirements->Boundary) { + size_t boundaryRemaining = requirements->Boundary - + (size_t)(pieceAddress & (requirements->Boundary - 1)); + if (capacity > boundaryRemaining) + capacity = boundaryRemaining; + } + if (!capacity) + return NTLSTATUS_NOT_SUPPORTED; + + size_t take = length - consumed; + if (take > capacity) + take = capacity; + if (segments) { + segments[count].DeviceAddress = pieceAddress; + segments[count].Length = take; + } + count++; + consumed += take; + } + + *segmentCount = count; + return NTLSTATUS_SUCCESS; +} + +// Allocates one mapping object with a fixed segment array. +static IO_DMA_MAPPING* IoDmaAllocateMapping(uint32_t segmentCount) { + if (!segmentCount) + return (IO_DMA_MAPPING*)0; + size_t extraBytes = ((size_t)segmentCount - 1) * sizeof(IO_DMA_SEGMENT); + if (extraBytes / sizeof(IO_DMA_SEGMENT) != (size_t)segmentCount - 1) + return (IO_DMA_MAPPING*)0; + size_t bytes = sizeof(IO_DMA_MAPPING) + extraBytes; + if (bytes < extraBytes) + return (IO_DMA_MAPPING*)0; + IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)MmAllocatePoolTag(bytes, IO_DMA_MAP_TAG); + if (mapping) + memset(mapping, 0, bytes); + return mapping; +} + +// Maps pinned memory directly or through an automatic contiguous bounce buffer. +static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter, + MM_PINNED_MEMORY* memory, + size_t offset, + size_t length, + IO_DMA_DIRECTION direction, + uint32_t flags, + IO_DMA_MAPPING** outMapping) { + if (!outMapping) + return NTLSTATUS_INVALID_PARAMETER; + *outMapping = (IO_DMA_MAPPING*)0; + if (!adapter || !memory || !length || offset > memory->Size || length > memory->Size - offset) + return NTLSTATUS_INVALID_PARAMETER; + if (IoDmaAdapterRemoved(adapter)) + return NTLSTATUS_OBJECT_DEAD; + if (direction < IoDmaToDevice || direction > IoDmaBidirectional || + !IoDmaDirectionAllows(memory->Direction, direction)) + return NTLSTATUS_ACCESS_DENIED; + if (flags & ~IO_DMA_MAP_NO_BOUNCE) + return NTLSTATUS_INVALID_PARAMETER; + + uint32_t segmentCount = 0; + KSTATUS directStatus = IoDmaBuildSegments(memory, + offset, + length, + &adapter->Requirements, + (IO_DMA_SEGMENT*)0, + &segmentCount); + IO_DMA_BUFFER* bounce = (IO_DMA_BUFFER*)0; + KSTATUS status = directStatus; + + if (!KSUCCESS(status)) { + if (status != NTLSTATUS_NOT_SUPPORTED || (flags & IO_DMA_MAP_NO_BOUNCE)) { + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + + status = IoDmaAllocateCommonBufferInternal(adapter, + length, + adapter->Requirements.Alignment, + 0, + &bounce); + if (!KSUCCESS(status)) { + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + status = IoDmaBuildContiguousSegments(bounce->DeviceAddress, + length, + &adapter->Requirements, + (IO_DMA_SEGMENT*)0, + &segmentCount); + if (!KSUCCESS(status)) { + IoDmaFreeCommonBufferImpl(bounce); + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + } + + IO_DMA_MAPPING* mapping = IoDmaAllocateMapping(segmentCount); + if (!mapping) { + IoDmaFreeCommonBufferImpl(bounce); + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return NTLSTATUS_NO_MEMORY; + } + + if (bounce) { + status = IoDmaBuildContiguousSegments(bounce->DeviceAddress, + length, + &adapter->Requirements, + mapping->Segments, + &segmentCount); + } else { + status = IoDmaBuildSegments(memory, + offset, + length, + &adapter->Requirements, + mapping->Segments, + &segmentCount); + } + if (!KSUCCESS(status)) { + MmFreePool(mapping); + IoDmaFreeCommonBufferImpl(bounce); + _InterlockedIncrement64(&gIoDmaStats.FailedMappings); + return status; + } + + mapping->Adapter = adapter; + mapping->Memory = memory; + mapping->BounceBuffer = bounce; + mapping->MemoryOffset = offset; + mapping->Length = length; + mapping->Direction = direction; + mapping->SegmentCount = segmentCount; + if (!IoDmaTryReferenceAdapter(adapter)) { + MmFreePool(mapping); + IoDmaFreeCommonBufferImpl(bounce); + return NTLSTATUS_OBJECT_DEAD; + } + IoDmaReferencePinnedMemory(memory); + + _InterlockedIncrement64(&gIoDmaStats.StreamingMappings); + _InterlockedIncrement64(&gIoDmaStats.StreamingMappingsInUse); + if (bounce) + _InterlockedIncrement64(&gIoDmaStats.BounceMappings); + else + _InterlockedIncrement64(&gIoDmaStats.DirectMappings); + *outMapping = mapping; + return NTLSTATUS_SUCCESS; +} + +// Synchronizes CPU-written data before the device consumes a mapping. +static KSTATUS IoDmaSynchronizeForDeviceImpl(IO_DMA_MAPPING* mapping) { + if (!mapping) + return NTLSTATUS_INVALID_PARAMETER; + + if (mapping->DeviceOwnsMapping) + return NTLSTATUS_FAILURE; + + if (mapping->Direction != IoDmaFromDevice && mapping->BounceBuffer) { + KSTATUS status = IoDmaCopyFromPinned(mapping->Memory, + mapping->MemoryOffset, + mapping->BounceBuffer->CpuAddress, + mapping->Length); + if (!KSUCCESS(status)) + return status; + _InterlockedExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToDevice, + (long long)mapping->Length); + } + _ReadWriteBarrier(); + _mm_mfence(); + _ReadWriteBarrier(); + mapping->DeviceOwnsMapping = 1; + return NTLSTATUS_SUCCESS; +} + +// Synchronizes device-written data before the CPU consumes a mapping. +static KSTATUS IoDmaSynchronizeForCpuImpl(IO_DMA_MAPPING* mapping) { + if (!mapping) + return NTLSTATUS_INVALID_PARAMETER; + if (!mapping->DeviceOwnsMapping) + return NTLSTATUS_SUCCESS; + + _ReadWriteBarrier(); + _mm_mfence(); + _ReadWriteBarrier(); + if (mapping->Direction != IoDmaToDevice && mapping->BounceBuffer) { + KSTATUS status = IoDmaCopyToPinned(mapping->Memory, + mapping->MemoryOffset, + mapping->BounceBuffer->CpuAddress, + mapping->Length); + if (!KSUCCESS(status)) + return status; + _InterlockedExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToCpu, + (long long)mapping->Length); + } + mapping->DeviceOwnsMapping = 0; + return NTLSTATUS_SUCCESS; +} + +// Releases a streaming mapping and performs the final receive-side bounce copy. +static void IoDmaUnmapMemoryImpl(IO_DMA_MAPPING* mapping) { + if (!mapping) + return; + + if (mapping->DeviceOwnsMapping) { + KSTATUS status = IoDmaSynchronizeForCpuImpl(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_GENERIC, + "DMA: final receive synchronization failed mapping=0x%016llX status=%u\n", + (unsigned long long)(uintptr_t)mapping, + status); + } + IO_DMA_BUFFER* bounce = mapping->BounceBuffer; + MM_PINNED_MEMORY* memory = mapping->Memory; + IO_DMA_ADAPTER* adapter = mapping->Adapter; + memset(mapping, 0, sizeof(*mapping)); + MmFreePool(mapping); + IoDmaFreeCommonBufferImpl(bounce); + IoDmaReleasePinnedMemory(memory); + IoDmaPutAdapterImpl(adapter); + _InterlockedIncrement64(&gIoDmaStats.StreamingMappingsUnmapped); + _InterlockedDecrement64(&gIoDmaStats.StreamingMappingsInUse); +} + +// Returns the number of device segments in a streaming mapping. +static uint32_t IoDmaGetSegmentCountImpl(const IO_DMA_MAPPING* mapping) { + return mapping ? mapping->SegmentCount : 0; +} + +// Returns the immutable device-segment array owned by a mapping. +static const IO_DMA_SEGMENT* IoDmaGetSegmentsImpl(const IO_DMA_MAPPING* mapping) { + return mapping ? mapping->Segments : (const IO_DMA_SEGMENT*)0; +} + +// Copies the current DMA counters into caller-owned storage. +static void IoDmaQueryStatsImpl(IO_DMA_STATS* outStats) { + if (!outStats) + return; + outStats->AdaptersCreated = (uint64_t)gIoDmaStats.AdaptersCreated; + outStats->AdaptersDestroyed = (uint64_t)gIoDmaStats.AdaptersDestroyed; + outStats->AdaptersInUse = (uint64_t)gIoDmaStats.AdaptersInUse; + outStats->CommonBuffersAllocated = (uint64_t)gIoDmaStats.CommonBuffersAllocated; + outStats->CommonBuffersFreed = (uint64_t)gIoDmaStats.CommonBuffersFreed; + outStats->CommonBuffersInUse = (uint64_t)gIoDmaStats.CommonBuffersInUse; + outStats->CommonBufferBytesInUse = (uint64_t)gIoDmaStats.CommonBufferBytesInUse; + outStats->PinnedKernelRanges = (uint64_t)gIoDmaStats.PinnedKernelRanges; + outStats->PinnedUserRanges = (uint64_t)gIoDmaStats.PinnedUserRanges; + outStats->PinnedRangesReleased = (uint64_t)gIoDmaStats.PinnedRangesReleased; + outStats->PinnedRangesInUse = (uint64_t)gIoDmaStats.PinnedRangesInUse; + outStats->PinnedPagesInUse = (uint64_t)gIoDmaStats.PinnedPagesInUse; + outStats->StreamingMappings = (uint64_t)gIoDmaStats.StreamingMappings; + outStats->StreamingMappingsUnmapped = (uint64_t)gIoDmaStats.StreamingMappingsUnmapped; + outStats->StreamingMappingsInUse = (uint64_t)gIoDmaStats.StreamingMappingsInUse; + outStats->DirectMappings = (uint64_t)gIoDmaStats.DirectMappings; + outStats->BounceMappings = (uint64_t)gIoDmaStats.BounceMappings; + outStats->BounceBytesCopiedToDevice = (uint64_t)gIoDmaStats.BounceBytesCopiedToDevice; + outStats->BounceBytesCopiedToCpu = (uint64_t)gIoDmaStats.BounceBytesCopiedToCpu; + outStats->FailedAllocations = (uint64_t)gIoDmaStats.FailedAllocations; + outStats->FailedMappings = (uint64_t)gIoDmaStats.FailedMappings; +} + +static const NTL_DDK_DMA gIoDmaDdk = { + .GetAdapter = IoDmaGetAdapterImpl, + .PutAdapter = IoDmaPutAdapterImpl, + .AllocateCommonBuffer = IoDmaAllocateCommonBufferImpl, + .FreeCommonBuffer = IoDmaFreeCommonBufferImpl, + .GetCommonBufferAddress = IoDmaGetCommonBufferAddressImpl, + .GetCommonBufferDeviceAddress = IoDmaGetCommonBufferDeviceAddressImpl, + .GetCommonBufferSize = IoDmaGetCommonBufferSizeImpl, + .PinKernelMemory = IoDmaPinKernelMemoryImpl, + .PinUserMemory = IoDmaPinUserMemoryImpl, + .UnpinMemory = IoDmaUnpinMemoryImpl, + .MapPinnedMemory = IoDmaMapPinnedMemoryImpl, + .UnmapMemory = IoDmaUnmapMemoryImpl, + .SynchronizeForDevice = IoDmaSynchronizeForDeviceImpl, + .SynchronizeForCpu = IoDmaSynchronizeForCpuImpl, + .GetSegmentCount = IoDmaGetSegmentCountImpl, + .GetSegments = IoDmaGetSegmentsImpl, + .QueryStats = IoDmaQueryStatsImpl, +}; + +// Returns the kernel-owned DDK DMA table. +const NTL_DDK_DMA* IoGetDmaDdk(void) { + return &gIoDmaDdk; +} diff --git a/code/kernel/io_file.c b/code/kernel/io_file.c index 9e22e98..ad32018 100644 --- a/code/kernel/io_file.c +++ b/code/kernel/io_file.c @@ -26,6 +26,47 @@ typedef struct { static VOLUME_ENTRY gVolumes[MAX_DRIVES]; +static EVENT* gVolumeChangeEvent; +static SPINLOCK gVolumeNamespaceLock = SPINLOCK_INIT; +static uint64_t gVolumeNamespaceGeneration; + +// Publishes a mounted-volume namespace change to boot discovery and other waiters. +static void KiIoNotifyVolumeNamespaceChange(void) { + uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock); + gVolumeNamespaceGeneration++; + KeReleaseSpinlock(&gVolumeNamespaceLock, flags); + if (gVolumeChangeEvent) + KeSignalEvent(gVolumeChangeEvent); +} + +// Initializes the waitable mounted-volume namespace notification. +KSTATUS IoInitializeVolumeNamespace(void) { + if (gVolumeChangeEvent) + return NTLSTATUS_SUCCESS; + return KeEventCreate(EventAutoReset, 0, &gVolumeChangeEvent); +} + +// Returns the current mounted-volume namespace generation. +uint64_t IoGetVolumeNamespaceGeneration(void) { + uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock); + uint64_t generation = gVolumeNamespaceGeneration; + KeReleaseSpinlock(&gVolumeNamespaceLock, flags); + return generation; +} + +// Waits for a mounted-volume namespace change without polling driver discovery. +KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration, + uint64_t timeoutMilliseconds) { + if (IoGetVolumeNamespaceGeneration() != observedGeneration) + return NTLSTATUS_SUCCESS; + if (!gVolumeChangeEvent) + return NTLSTATUS_OBJECT_DEAD; + + EVENT* events[1] = { gVolumeChangeEvent }; + uint32_t selected = 0; + return KeWaitForMultipleEvents(events, 1, 0, timeoutMilliseconds, &selected); +} + // Async file I/O worker. typedef enum IO_ASYNC_FILE_OP { IoAsyncFileRead = 0, @@ -275,8 +316,8 @@ static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) { static void KiIoAsyncFileWorker(void* arg) { (void)arg; - for (;;) { - KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore); + while (true) { + KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(s)) { KeYield(); continue; @@ -335,9 +376,10 @@ static KSTATUS KiIoEnsureAsyncFileIoStarted(void) { KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker, 0, - PriorityNormal, + ¶meters, &worker); // Keep the creator reference: this worker is intentionally persistent // for the lifetime of the kernel. @@ -751,6 +793,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex, return NTLSTATUS_ALREADY_EXISTS; gVolumes[volumeIndex].Fs = fs; gVolumes[volumeIndex].Vol = *vol; + KiIoNotifyVolumeNamespaceChange(); return NTLSTATUS_SUCCESS; } @@ -762,18 +805,23 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) { return NTLSTATUS_NOT_FOUND; memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex])); + KiIoNotifyVolumeNamespaceChange(); return NTLSTATUS_SUCCESS; } // Implements the unregister Filesystems For Disk operation. __declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) { KSTATUS last = NTLSTATUS_NOT_FOUND; + bool changed = false; for (uint8_t i = 0; i < MAX_DRIVES; i++) { if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) { memset(&gVolumes[i], 0, sizeof(gVolumes[i])); last = NTLSTATUS_SUCCESS; + changed = true; } } + if (changed) + KiIoNotifyVolumeNamespaceChange(); return last; } @@ -819,7 +867,11 @@ __declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) { static void KiFileDestroy(OBJECT* obj) { FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header); if (f->Fs && f->FileCtx) { - (void)IoFlushFile(f); + // Close drains kernel-owned dirty pages, but stable-media semantics + // remain explicit through IoFlushFile so every close does not issue a + // whole-device cache flush. + (void)MmVadSyncFileMappings(f); + (void)MmFileCacheFlushFile(f); MmFileCacheInvalidateAll(f); f->Fs->Close(f->Fs, f->FileCtx); } @@ -853,71 +905,11 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out f->Size = fs->GetSize(fs, fileCtx); f->Offset = 0; f->Flags = 0; - f->ReadaheadScore = 0; - f->ReadaheadNextOffset = 0; - f->ReadaheadIssuedUntil = 0; - f->ReadaheadSuppressedUntil = 0; *out = f; return NTLSTATUS_SUCCESS; } -static uint64_t KiAlignUpPage64(uint64_t value) { - return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u); -} - -static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) { - if (!file || bytesRead < 512) - return; - if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0) - return; - - uint64_t end = offset + bytesRead; - uint64_t start = 0; - uint32_t pages = 0; - - uintptr_t flags = KeAcquireSpinlock(&file->Lock); - if (offset == file->ReadaheadNextOffset) { - if (file->ReadaheadScore < 32) - file->ReadaheadScore++; - } else { - file->ReadaheadScore = 1; - file->ReadaheadIssuedUntil = end; - } - file->ReadaheadNextOffset = end; - - if (end < file->ReadaheadSuppressedUntil) { - pages = 0; - } else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) { - pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u); - start = KiAlignUpPage64(end); - if (start < file->ReadaheadIssuedUntil) - start = KiAlignUpPage64(file->ReadaheadIssuedUntil); - if (start >= fileSize) { - pages = 0; - } else { - uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE); - if (maxEnd > fileSize) - pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE); - file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE); - } - } - KeReleaseSpinlock(&file->Lock, flags); - - if (pages) { - KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages); - if (ras == NTLSTATUS_QUEUE_FULL) { - flags = KeAcquireSpinlock(&file->Lock); - uint64_t suppressUntil = end + (1024u * 1024u); - if (suppressUntil < end || suppressUntil > fileSize) - suppressUntil = fileSize; - if (file->ReadaheadSuppressedUntil < suppressUntil) - file->ReadaheadSuppressedUntil = suppressUntil; - KeReleaseSpinlock(&file->Lock, flags); - } - } -} - // Explicit-offset file I/O. Async I/O is built on these routines so it does // not race the sequential FILE_OBJECT.Offset used by legacy read/write calls. __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, @@ -979,7 +971,6 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset, } *bytesRead = total; - KiIoMaybeScheduleReadAhead(file, offset, total, fileSize); return NTLSTATUS_SUCCESS; } @@ -1022,8 +1013,13 @@ __declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) { if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; KSTATUS sync = MmVadSyncFileMappings(file); - KSTATUS flush = MmFileCacheFlushFile(file); - return KSUCCESS(sync) ? flush : sync; + KSTATUS cache = MmFileCacheFlushFile(file); + KSTATUS media = NTLSTATUS_SUCCESS; + if (KSUCCESS(sync) && KSUCCESS(cache) && file->Fs && file->Fs->Flush) + media = file->Fs->Flush(file->Fs, file->FileCtx); + if (!KSUCCESS(sync)) return sync; + if (!KSUCCESS(cache)) return cache; + return media; } // IoReadFile - sequential read at current offset. @@ -1091,10 +1087,6 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o f->Size = 0; f->Offset = 0; f->Flags = 0; - f->ReadaheadScore = 0; - f->ReadaheadNextOffset = 0; - f->ReadaheadIssuedUntil = 0; - f->ReadaheadSuppressedUntil = 0; *out = f; return NTLSTATUS_SUCCESS; @@ -1278,7 +1270,7 @@ __declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint IO_FILE_LOCK_WAITER waiter; memset(&waiter, 0, sizeof(waiter)); - for (;;) { + while (true) { IO_FILE_LOCK_ENTRY* entry = (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k')); if (!entry) { if (waitEvent) ObDereferenceObject(&waitEvent->Header); diff --git a/code/kernel/ke_apc.c b/code/kernel/ke_apc.c index 9c561f8..85014f2 100644 --- a/code/kernel/ke_apc.c +++ b/code/kernel/ke_apc.c @@ -200,7 +200,7 @@ void KiDeliverKernelApcs(void) { thread->ApcDeliveryActive = 1; - for (;;) { + while (true) { if (KeGetCurrentIrql() != PASSIVE_LEVEL || thread->KernelApcDisable != 0) break; @@ -375,7 +375,7 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) { void KiRundownApcState(THREAD* thread) { if (!thread) return; - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock); LIST_ENTRY* node = ListPopHead(&thread->ApcQueue); if (LIST_IS_EMPTY(&thread->ApcQueue)) @@ -389,7 +389,7 @@ void KiRundownApcState(THREAD* thread) { apc->RundownRoutine(apc); } - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock); LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue); if (LIST_IS_EMPTY(&thread->UserApcQueue)) diff --git a/code/kernel/ke_audio.c b/code/kernel/ke_audio.c new file mode 100644 index 0000000..0699cbc --- /dev/null +++ b/code/kernel/ke_audio.c @@ -0,0 +1,2460 @@ +// Kernel audio core. This file owns endpoint identity, logical streams, mixing, and provider timing handoff. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KE_AUDIO_ENDPOINT_TAG MM_POOL_TAG('a', 'e', 'p', 't') +#define KE_AUDIO_RING_TAG MM_POOL_TAG('a', 'r', 'n', 'g') +#define KE_AUDIO_SELFTEST_TAG MM_POOL_TAG('a', 't', 's', 't') + +#define KE_AUDIO_PERIOD_STATE_FREE 0u +#define KE_AUDIO_PERIOD_STATE_FILLING 1u +#define KE_AUDIO_PERIOD_STATE_READY 2u +#define KE_AUDIO_PERIOD_STATE_COMPLETED 3u +#define KE_AUDIO_PERIOD_STATE_REAPING 4u + +#define KE_AUDIO_TICKS_PER_MILLISECOND 1000ull +#define KE_AUDIO_SELFTEST_WAIT_MILLISECONDS 5000ull + +typedef struct KE_AUDIO_PERIOD_CONTRIBUTION { + KE_AUDIO_STREAM* Stream; + uint64_t StreamGeneration; + uint32_t SourceFrames; +} KE_AUDIO_PERIOD_CONTRIBUTION; + +typedef struct KE_AUDIO_PERIOD_STATE { + uint64_t Generation; + uint32_t ContributionCount; + uint8_t State; + KE_AUDIO_PERIOD_CONTRIBUTION Contributions[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; +} KE_AUDIO_PERIOD_STATE; + +struct KE_AUDIO_STREAM { + OBJECT Header; + MUTEX* Lock; + EVENT* WaitEvent; + LIST_ENTRY EndpointLink; + IO_AUDIO_ENDPOINT* Endpoint; + KE_AUDIO_STREAM_STATE State; + volatile long ProcessHandleCount; + volatile long CloseRequested; + IO_AUDIO_FORMAT Format; + uint8_t* Ring; + uint32_t FrameBytes; + uint32_t ReadFrame; + uint32_t WriteFrame; + uint32_t BufferedFrames; + uint64_t ResamplePositionQ32; + uint64_t ResampleStepQ32; + uint64_t FramesSubmitted; + uint64_t FramesConsumed; + uint64_t FramesPresented; + uint64_t FramesDiscarded; + uint64_t OutputFramesMixed; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t ConsecutiveUnderrunPeriods; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; + uint64_t BufferGeneration; + KSTATUS FailureStatus; + uint32_t WaitMask; + uint32_t WaitWritableFrames; + uint64_t WaitConsumedFrames; + uint64_t WaitPresentedFrames; + uint64_t WaitDrainedGeneration; + uint64_t WaitDiscontinuityGeneration; + float Volume; + float Pan; + uint8_t Muted; + float CurrentLeftGain; + float CurrentRightGain; + float TargetLeftGain; + float TargetRightGain; + float LeftGainStep; + float RightGainStep; + uint32_t GainRampRemaining; +}; + +struct IO_AUDIO_ENDPOINT { + OBJECT Header; + LIST_ENTRY RegistryLink; + LIST_ENTRY StreamList; + SPINLOCK StreamListLock; + MUTEX* ControlLock; + EVENT* WorkEvent; + EVENT* MixerExitEvent; + THREAD* MixerThread; + SPINLOCK NotifyLock; + uint64_t EndpointId; + IO_AUDIO_ENDPOINT_KIND Kind; + KE_AUDIO_ENDPOINT_STATE State; + uint32_t Capabilities; + uint8_t Available; + uint8_t Registered; + uint8_t HardwareRunning; + uint8_t SkipProviderStop; + IO_AUDIO_REMOVAL_MODE RemovalMode; + DEVICE_OBJECT* Device; + void* ProviderContext; + IO_AUDIO_PROVIDER_OPERATIONS Operations; + uint8_t* PeriodBuffer; + size_t PeriodBufferBytes; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + size_t PeriodStrideBytes; + IO_AUDIO_FORMAT HardwareFormat; + int32_t HardwareGainMinimumMilliDb; + int32_t HardwareGainMaximumMilliDb; + uint32_t HardwareGainStepMilliDb; + KE_AUDIO_PERIOD_STATE Periods[KE_AUDIO_OUTPUT_PERIOD_COUNT]; + uint32_t ActiveStreamCount; + uint64_t IdleDeadlineTicks; + float MasterCurrentGain; + float MasterTargetGain; + float MasterRequestedVolume; + int32_t MasterHardwareGainMilliDb; + float MasterGainStep; + uint32_t MasterRampRemaining; + uint64_t MasterGainGeneration; + uint8_t MasterMuted; + KSTATUS FailureStatus; + uint64_t PeriodsMixed; + uint64_t PeriodsCompleted; + uint64_t FramesPresented; + uint64_t HardwarePositionFrames; + uint64_t InterruptCount; + uint64_t LastPresentationTimestampTicks; + uint64_t SilenceFramesInserted; + uint64_t Underruns; + uint64_t DiscontinuityGeneration; + uint64_t HardwareStarts; + uint64_t HardwareStops; + uint64_t SpuriousNotifications; + uint64_t PendingDiscontinuities; +}; + +typedef struct KE_AUDIO_CORE_STATS { + volatile long long EndpointsRegistered; + volatile long long EndpointsUnregistered; + volatile long long StreamsCreated; + volatile long long StreamsDestroyed; + volatile long long Starts; + volatile long long Stops; + volatile long long Pauses; + volatile long long Resumes; + volatile long long Drains; + volatile long long Resets; +} KE_AUDIO_CORE_STATS; + +typedef struct KE_AUDIO_LIFECYCLE_SNAPSHOT { + KE_AUDIO_ENDPOINT_STATE State; + KSTATUS FailureStatus; + IO_AUDIO_REMOVAL_MODE RemovalMode; + uint8_t SkipProviderStop; +} KE_AUDIO_LIFECYCLE_SNAPSHOT; + +static SPINLOCK gAudioRegistryLock = SPINLOCK_INIT; +static LIST_ENTRY gAudioEndpointList; +static uint32_t gAudioEndpointCount; +static uint64_t gAudioNextEndpointId = 1; +static uint64_t gAudioDefaultEndpointId; +static uint8_t gAudioInitialized; +static KE_AUDIO_CORE_STATS gAudioCoreStats; + +// Stops on one violated internal audio invariant. +static void KiAudioBug(const char* where, const char* reason, uint64_t value) { + KePanic(KE_PANIC_CODE_GENERIC, + "audio invariant failed: %s: %s value=0x%016llX\n", + where ? where : "?", + reason ? reason : "?", + (unsigned long long)value); +} + +// Acquires an internal mutex and rejects abandoned ownership as kernel corruption. +static void KiAudioAcquireMutex(MUTEX* mutex, const char* where) { + KSTATUS status = KeAcquireMutex(mutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KiAudioBug(where, "internal mutex acquisition failed", status); +} + +// Releases one internal mutex. +static void KiAudioReleaseMutex(MUTEX* mutex) { + KeReleaseMutex(mutex); +} + +// Returns the provider timer in microseconds when available. +static uint64_t KiAudioQueryTicks(void) { + return KeQueryMonotonicTicks(); +} + +// Returns whether one float is finite without depending on a runtime math library. +static int KiAudioFloatIsFinite(float value) { + union { + float Float; + uint32_t Bits; + } bits; + bits.Float = value; + return (bits.Bits & 0x7F800000u) != 0x7F800000u; +} + +// Clamps one mixer sample to the signed unit interval. +static float KiAudioClampSample(float value) { + const float maximum = 0.99999988079071044921875f; + if (value > maximum) return maximum; + if (value < -1.0f) return -1.0f; + return value; +} + +// Converts one normalized mixer sample to signed 16-bit PCM. +static int16_t KiAudioFloatToOutputSample(float value) { + float scaled = KiAudioClampSample(value) * 32768.0f; + int32_t sample = scaled >= 0.0f ? (int32_t)(scaled + 0.5f) + : (int32_t)(scaled - 0.5f); + if (sample > 32767) sample = 32767; + if (sample < -32768) sample = -32768; + return (int16_t)sample; +} + +// Approximates one linear gain as milli-decibels without a runtime math dependency. +static int32_t KiAudioMilliDbFromLinearGain(float gain) { + if (gain <= 0.0f) + return INT32_MIN; + + union { + float Float; + uint32_t Bits; + } bits; + bits.Float = gain; + uint32_t encodedExponent = (bits.Bits >> 23) & 0xFFu; + uint32_t fraction = bits.Bits & 0x007FFFFFu; + int32_t exponent = 0; + if (encodedExponent) { + exponent = (int32_t)encodedExponent - 127; + } else { + exponent = -126; + while ((fraction & 0x00800000u) == 0) { + fraction <<= 1; + exponent--; + } + fraction &= 0x007FFFFFu; + } + bits.Bits = fraction | 0x3F800000u; + float mantissa = bits.Float; + float y = (mantissa - 1.0f) / (mantissa + 1.0f); + float y2 = y * y; + float logarithm = 2.0f * y * + (1.0f + y2 * (0.3333333333333333f + + y2 * (0.2f + y2 * (0.1428571428571429f + y2 * 0.1111111111111111f)))); + float log2Gain = (float)exponent + logarithm * 1.4426950408889634f; + float milliDb = log2Gain * 6020.599913279624f; + return milliDb >= 0.0f ? (int32_t)(milliDb + 0.5f) + : (int32_t)(milliDb - 0.5f); +} + +// Approximates one nonpositive milli-decibel residual as a linear gain. +static float KiAudioLinearGainFromMilliDb(int32_t milliDb) { + if (milliDb >= 0) + return 1.0f; + float exponent = (float)milliDb * (1.0f / 6020.599913279624f); + int32_t whole = (int32_t)exponent; + if ((float)whole > exponent) + whole--; + float fraction = exponent - (float)whole; + float z = fraction * 0.6931471805599453f; + float z2 = z * z; + float result = 1.0f + z + z2 * 0.5f + z2 * z * (1.0f / 6.0f) + + z2 * z2 * (1.0f / 24.0f) + z2 * z2 * z * (1.0f / 120.0f); + while (whole < 0) { + result *= 0.5f; + whole++; + } + while (whole > 0) { + result *= 2.0f; + whole--; + } + return result; +} + +// Chooses the highest supported nonpositive hardware gain. +static int32_t KiAudioHighestSafeHardwareGain(const IO_AUDIO_ENDPOINT* endpoint) { + if (!(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) || + !endpoint->Operations.SetGain) + return 0; + + int32_t minimum = endpoint->HardwareGainMinimumMilliDb; + int32_t maximum = endpoint->HardwareGainMaximumMilliDb; + int64_t step = endpoint->HardwareGainStepMilliDb; + if (maximum <= 0) + return maximum; + int64_t steps = ((int64_t)0 - minimum) / step; + return (int32_t)((int64_t)minimum + steps * step); +} + +// Converts one requested master volume into software gain for the stable hardware gain. +static float KiAudioSoftwareGainForVolume(const IO_AUDIO_ENDPOINT* endpoint, + float requestedVolume) { + if (requestedVolume <= 0.0f) + return 0.0f; + if (!(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) || + !endpoint->Operations.SetGain) + return requestedVolume; + int32_t requestedMilliDb = KiAudioMilliDbFromLinearGain(requestedVolume); + int64_t residualMilliDb = (int64_t)requestedMilliDb - + endpoint->MasterHardwareGainMilliDb; + if (residualMilliDb > 0) + residualMilliDb = 0; + if (residualMilliDb < -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB) + return 0.0f; + return KiAudioLinearGainFromMilliDb((int32_t)residualMilliDb); +} + +// Returns the byte width of one source frame. +static uint32_t KiAudioFrameBytes(const IO_AUDIO_FORMAT* format) { + if (!format) return 0; + if (format->SampleFormat != IoAudioSampleS16) return 0; + return 2u * format->ChannelCount; +} + +// Validates one logical source format accepted by the first mixer contract. +static int KiAudioValidateSourceFormat(const IO_AUDIO_FORMAT* format) { + if (!format) return 0; + if (format->ChannelCount != 1 && format->ChannelCount != 2) + return 0; + if (format->SampleFormat != IoAudioSampleS16) + return 0; + return format->SampleRate == 11025u || + format->SampleRate == 22050u || + format->SampleRate == 44100u || + format->SampleRate == 48000u; +} + +// Validates the fixed endpoint ring contract used by the HDA provider series. +static KSTATUS KiAudioValidateRegistration( + const IO_AUDIO_ENDPOINT_REGISTRATION* registration) { + if (!registration || !registration->PeriodBuffer) + return NTLSTATUS_INVALID_PARAMETER; + if (!registration->Device) + return NTLSTATUS_INVALID_PARAMETER; + if (registration->Kind < IoAudioEndpointLineOut || + registration->Kind > IoAudioEndpointOtherAnalog || + (registration->Capabilities & ~IO_AUDIO_ENDPOINT_CAP_VALID_MASK) != 0) + return NTLSTATUS_INVALID_PARAMETER; + if (!registration->Operations.Start || !registration->Operations.Stop) + return NTLSTATUS_INVALID_PARAMETER; + if (registration->HardwareFormat.SampleRate != KE_AUDIO_OUTPUT_SAMPLE_RATE || + registration->HardwareFormat.ChannelCount != KE_AUDIO_OUTPUT_CHANNELS || + registration->HardwareFormat.SampleFormat != IoAudioSampleS16) + return NTLSTATUS_NOT_SUPPORTED; + if (registration->PeriodCount != KE_AUDIO_OUTPUT_PERIOD_COUNT || + registration->FramesPerPeriod != KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD || + registration->PeriodStrideBytes != KE_AUDIO_OUTPUT_PERIOD_BYTES) + return NTLSTATUS_NOT_SUPPORTED; + if (registration->PeriodBufferBytes < + registration->PeriodStrideBytes * registration->PeriodCount) + return NTLSTATUS_BUFFER_TOO_SMALL; + if (((uintptr_t)registration->PeriodBuffer & (sizeof(uint32_t) - 1u)) != 0) + return NTLSTATUS_INVALID_PARAMETER; + if ((registration->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) != 0) { + if (!registration->Operations.SetGain || !registration->HardwareGainStepMilliDb || + registration->HardwareGainStepMilliDb > INT32_MAX || + registration->HardwareGainMinimumMilliDb > registration->HardwareGainMaximumMilliDb || + registration->HardwareGainMinimumMilliDb > 0 || + registration->HardwareGainMinimumMilliDb < -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB || + registration->HardwareGainMaximumMilliDb > IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB || + ((int64_t)registration->HardwareGainMaximumMilliDb - + registration->HardwareGainMinimumMilliDb) % + registration->HardwareGainStepMilliDb != 0) + return NTLSTATUS_INVALID_PARAMETER; + } else if (registration->Operations.SetGain || + registration->HardwareGainMinimumMilliDb || + registration->HardwareGainMaximumMilliDb || + registration->HardwareGainStepMilliDb) { + return NTLSTATUS_INVALID_PARAMETER; + } + return NTLSTATUS_SUCCESS; +} + +// Returns whether one endpoint lifecycle transition is valid. +static int KiAudioEndpointTransitionAllowed(KE_AUDIO_ENDPOINT_STATE oldState, + KE_AUDIO_ENDPOINT_STATE newState) { + if (oldState == newState) + return 1; + switch (oldState) { + case KeAudioEndpointStarting: + return newState == KeAudioEndpointReady || + newState == KeAudioEndpointStopping; + case KeAudioEndpointReady: + return newState == KeAudioEndpointStopping || + newState == KeAudioEndpointFailed; + case KeAudioEndpointFailed: + return newState == KeAudioEndpointStopping; + case KeAudioEndpointStopping: + return newState == KeAudioEndpointRemoved; + case KeAudioEndpointRemoved: + return 0; + default: + return 0; + } +} + +// Changes one endpoint state while the registry lock serializes lifecycle ownership. +static void KiAudioSetEndpointStateLocked(IO_AUDIO_ENDPOINT* endpoint, + KE_AUDIO_ENDPOINT_STATE newState) { + if (!KiAudioEndpointTransitionAllowed(endpoint->State, newState)) + KiAudioBug("KiAudioSetEndpointStateLocked", "invalid endpoint transition", + ((uint64_t)endpoint->State << 32) | (uint32_t)newState); + endpoint->State = newState; +} + +// Returns the fixed endpoint preference rank used until policy moves to a service. +static uint32_t KiAudioEndpointRank(IO_AUDIO_ENDPOINT_KIND kind) { + if (kind == IoAudioEndpointLineOut) return 0; + if (kind == IoAudioEndpointSpeakerHeadphone) return 1; + return 2; +} + +// Recomputes the default endpoint while the registry lock is held. +static void KiAudioSelectDefaultEndpointLocked(void) { + uint64_t selectedId = 0; + uint32_t selectedRank = UINT32_MAX; + for (LIST_ENTRY* link = gAudioEndpointList.Fwd; + link != &gAudioEndpointList; + link = link->Fwd) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); + if (endpoint->State != KeAudioEndpointReady || !endpoint->Available) + continue; + uint32_t rank = KiAudioEndpointRank(endpoint->Kind); + if (!selectedId || rank < selectedRank || + (rank == selectedRank && endpoint->EndpointId < selectedId)) { + selectedId = endpoint->EndpointId; + selectedRank = rank; + } + } + gAudioDefaultEndpointId = selectedId; +} + +// References one ready endpoint by stable identifier. +static IO_AUDIO_ENDPOINT* KiAudioReferenceEndpointById(uint64_t endpointId) { + IO_AUDIO_ENDPOINT* result = (IO_AUDIO_ENDPOINT*)0; + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + for (LIST_ENTRY* link = gAudioEndpointList.Fwd; + link != &gAudioEndpointList; + link = link->Fwd) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); + if (endpoint->EndpointId != endpointId) + continue; + if (endpoint->State == KeAudioEndpointReady && ObTryReferenceObject(&endpoint->Header)) + result = endpoint; + break; + } + KeReleaseSpinlock(&gAudioRegistryLock, flags); + return result; +} + +// Signals one stream when an armed progress condition or terminal state is present. +static void KiAudioRefreshStreamEventLocked(KE_AUDIO_STREAM* stream) { + uint32_t writableFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; + int signaled = stream->State == KeAudioStreamFailed || + stream->State == KeAudioStreamClosed; + if ((stream->WaitMask & KE_AUDIO_WAIT_WRITABLE) && + writableFrames >= stream->WaitWritableFrames) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_CONSUMED) && + stream->FramesConsumed >= stream->WaitConsumedFrames) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_PRESENTED) && + stream->FramesPresented >= stream->WaitPresentedFrames) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_STOPPED) && + stream->State == KeAudioStreamStopped) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_DRAINED) && + stream->DrainedGeneration != stream->WaitDrainedGeneration) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_FAILED) && + stream->State == KeAudioStreamFailed) + signaled = 1; + if ((stream->WaitMask & KE_AUDIO_WAIT_DISCONTINUITY) && + stream->DiscontinuityGeneration != stream->WaitDiscontinuityGeneration) + signaled = 1; + if (signaled) + KeSignalEvent(stream->WaitEvent); + else + KeClearEvent(stream->WaitEvent); +} + +// Updates active-stream timing under the endpoint notification lock. +static void KiAudioAdjustActiveStreams(IO_AUDIO_ENDPOINT* endpoint, int delta) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + uint32_t oldCount = endpoint->ActiveStreamCount; + if (delta > 0) { + endpoint->ActiveStreamCount += (uint32_t)delta; + endpoint->IdleDeadlineTicks = 0; + } else { + uint32_t amount = (uint32_t)(-delta); + if (oldCount < amount) { + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioBug("KiAudioAdjustActiveStreams", "active stream count underflow", oldCount); + } + endpoint->ActiveStreamCount -= amount; + if (endpoint->ActiveStreamCount == 0) { + uint64_t now = KiAudioQueryTicks(); + endpoint->IdleDeadlineTicks = now + + (uint64_t)KE_AUDIO_IDLE_STOP_MILLISECONDS * KE_AUDIO_TICKS_PER_MILLISECOND; + } + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KeSignalEvent(endpoint->WorkEvent); +} + +// Returns whether one stream state contributes to the active hardware clock. +static int KiAudioStreamStateIsActive(KE_AUDIO_STREAM_STATE state) { + return state == KeAudioStreamRunning || state == KeAudioStreamDraining; +} + +// Returns whether every frame targeted by one drain has reached a terminal disposition. +static int KiAudioDrainIsCompleteLocked(const KE_AUDIO_STREAM* stream) { + return stream->BufferedFrames == 0 && + stream->FramesPresented + stream->FramesDiscarded >= stream->FramesSubmitted; +} + +// Returns whether one logical stream transition is valid. +static int KiAudioStreamTransitionAllowed(KE_AUDIO_STREAM_STATE oldState, + KE_AUDIO_STREAM_STATE newState) { + if (oldState == newState) + return 1; + if (newState == KeAudioStreamFailed || newState == KeAudioStreamClosed) + return oldState != KeAudioStreamClosed; + switch (oldState) { + case KeAudioStreamCreated: + return newState == KeAudioStreamRunning || + newState == KeAudioStreamStopped; + case KeAudioStreamRunning: + return newState == KeAudioStreamPaused || + newState == KeAudioStreamDraining || + newState == KeAudioStreamStopped; + case KeAudioStreamPaused: + return newState == KeAudioStreamRunning || + newState == KeAudioStreamDraining || + newState == KeAudioStreamStopped; + case KeAudioStreamDraining: + return newState == KeAudioStreamStopped; + case KeAudioStreamStopped: + return newState == KeAudioStreamRunning; + case KeAudioStreamFailed: + case KeAudioStreamClosed: + default: + return 0; + } +} + +// Changes one stream state and updates endpoint activity in the same locked transaction. +static void KiAudioSetStreamStateLocked(KE_AUDIO_STREAM* stream, + KE_AUDIO_STREAM_STATE newState) { + if (!KiAudioStreamTransitionAllowed(stream->State, newState)) + KiAudioBug("KiAudioSetStreamStateLocked", "invalid stream transition", + ((uint64_t)stream->State << 32) | (uint32_t)newState); + int oldActive = KiAudioStreamStateIsActive(stream->State); + int newActive = KiAudioStreamStateIsActive(newState); + stream->State = newState; + if (oldActive != newActive) + KiAudioAdjustActiveStreams(stream->Endpoint, newActive ? 1 : -1); +} + +// Records one completed drain before publishing the stopped state. +static void KiAudioCompleteDrainLocked(KE_AUDIO_STREAM* stream) { + if (stream->State != KeAudioStreamDraining || !KiAudioDrainIsCompleteLocked(stream)) + KiAudioBug("KiAudioCompleteDrainLocked", "incomplete drain reached completion", stream->State); + stream->DrainedGeneration++; + KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); +} + +// Computes the target left and right gains for one volume, pan, and mute state. +static void KiAudioComputeStreamTargets(float volume, + float pan, + uint8_t muted, + float* leftOut, + float* rightOut) { + float left = volume; + float right = volume; + if (pan > 0.0f) + left *= 1.0f - pan; + else if (pan < 0.0f) + right *= 1.0f + pan; + if (muted) { + left = 0.0f; + right = 0.0f; + } + *leftOut = left; + *rightOut = right; +} + +// Starts a ten-millisecond linear gain ramp while the stream mutex is held. +static void KiAudioBeginStreamGainRampLocked(KE_AUDIO_STREAM* stream) { + KiAudioComputeStreamTargets(stream->Volume, + stream->Pan, + stream->Muted, + &stream->TargetLeftGain, + &stream->TargetRightGain); + stream->GainRampRemaining = KE_AUDIO_GAIN_RAMP_FRAMES; + stream->LeftGainStep = + (stream->TargetLeftGain - stream->CurrentLeftGain) / + (float)KE_AUDIO_GAIN_RAMP_FRAMES; + stream->RightGainStep = + (stream->TargetRightGain - stream->CurrentRightGain) / + (float)KE_AUDIO_GAIN_RAMP_FRAMES; +} + +// Advances one stream gain ramp by one output frame. +static void KiAudioAdvanceStreamGainLocked(KE_AUDIO_STREAM* stream) { + if (!stream->GainRampRemaining) + return; + stream->CurrentLeftGain += stream->LeftGainStep; + stream->CurrentRightGain += stream->RightGainStep; + stream->GainRampRemaining--; + if (!stream->GainRampRemaining) { + stream->CurrentLeftGain = stream->TargetLeftGain; + stream->CurrentRightGain = stream->TargetRightGain; + } +} + +// Decodes one interleaved source frame into normalized stereo samples. +static void KiAudioDecodeSourceFrame(const KE_AUDIO_STREAM* stream, + uint32_t offset, + float* leftOut, + float* rightOut) { + uint32_t index = (stream->ReadFrame + offset) % KE_AUDIO_STREAM_RING_FRAMES; + const uint8_t* frame = stream->Ring + (size_t)index * stream->FrameBytes; + float left = 0.0f; + float right = 0.0f; + + const int16_t* samples = (const int16_t*)frame; + left = (float)samples[0] * (1.0f / 32768.0f); + right = stream->Format.ChannelCount == 2 + ? (float)samples[1] * (1.0f / 32768.0f) + : left; + + *leftOut = left; + *rightOut = right; +} + +// Applies one final-handle close request while the stream mutex serializes state. +static int KiAudioApplyCloseRequestLocked(KE_AUDIO_STREAM* stream) { + if (!_InterlockedCompareExchange(&stream->CloseRequested, 0, 0) || + stream->State == KeAudioStreamClosed) + return stream->State == KeAudioStreamClosed; + + uint64_t terminalFrames = stream->FramesPresented + stream->FramesDiscarded; + if (terminalFrames < stream->FramesSubmitted) + stream->FramesDiscarded += stream->FramesSubmitted - terminalFrames; + stream->BufferGeneration++; + KiAudioSetStreamStateLocked(stream, KeAudioStreamClosed); + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->BufferedFrames = 0; + stream->ResamplePositionQ32 = 0; + stream->ConsecutiveUnderrunPeriods = 0; + stream->DiscontinuityGeneration++; + KiAudioRefreshStreamEventLocked(stream); + return 1; +} + +// Mixes one logical stream into one endpoint period. +static uint32_t KiAudioMixStreamPeriod(KE_AUDIO_STREAM* stream, + float* mix, + uint32_t frames, + uint64_t* streamGenerationOut) { + KiAudioAcquireMutex(stream->Lock, "KiAudioMixStreamPeriod"); + (void)KiAudioApplyCloseRequestLocked(stream); + if (!KiAudioStreamStateIsActive(stream->State)) { + KiAudioReleaseMutex(stream->Lock); + return 0; + } + + uint64_t position = stream->ResamplePositionQ32; + uint32_t silentFrames = 0; + for (uint32_t outputFrame = 0; outputFrame < frames; outputFrame++) { + uint32_t sourceOffset = (uint32_t)(position >> 32); + if (sourceOffset >= stream->BufferedFrames) { + silentFrames += frames - outputFrame; + for (; outputFrame < frames; outputFrame++) + KiAudioAdvanceStreamGainLocked(stream); + break; + } + + float left0 = 0.0f; + float right0 = 0.0f; + float left1 = 0.0f; + float right1 = 0.0f; + KiAudioDecodeSourceFrame(stream, sourceOffset, &left0, &right0); + if (sourceOffset + 1u < stream->BufferedFrames) + KiAudioDecodeSourceFrame(stream, sourceOffset + 1u, &left1, &right1); + else { + left1 = left0; + right1 = right0; + } + + float fraction = (float)(uint32_t)position * (1.0f / 4294967296.0f); + float left = left0 + (left1 - left0) * fraction; + float right = right0 + (right1 - right0) * fraction; + mix[outputFrame * 2u] += left * stream->CurrentLeftGain; + mix[outputFrame * 2u + 1u] += right * stream->CurrentRightGain; + KiAudioAdvanceStreamGainLocked(stream); + position += stream->ResampleStepQ32; + } + + uint32_t consumed = (uint32_t)(position >> 32); + if (consumed > stream->BufferedFrames) + consumed = stream->BufferedFrames; + stream->ReadFrame = (stream->ReadFrame + consumed) % KE_AUDIO_STREAM_RING_FRAMES; + stream->BufferedFrames -= consumed; + stream->FramesConsumed += consumed; + stream->ResamplePositionQ32 = position - ((uint64_t)consumed << 32); + stream->OutputFramesMixed += frames - silentFrames; + + if (silentFrames && stream->State == KeAudioStreamRunning) { + stream->SilenceFramesInserted += silentFrames; + stream->Underruns++; + stream->ConsecutiveUnderrunPeriods++; + } else if (!silentFrames && stream->ConsecutiveUnderrunPeriods) { + stream->ConsecutiveUnderrunPeriods = 0; + } + + *streamGenerationOut = stream->BufferGeneration; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + return consumed; +} + +// Snapshots referenced streams under the nonblocking membership lock. +static uint32_t KiAudioSnapshotStreams(IO_AUDIO_ENDPOINT* endpoint, + KE_AUDIO_STREAM** streams) { + uint32_t count = 0; + uintptr_t flags = KeAcquireSpinlock(&endpoint->StreamListLock); + for (LIST_ENTRY* link = endpoint->StreamList.Fwd; + link != &endpoint->StreamList && count < KE_AUDIO_MAX_STREAMS_PER_ENDPOINT; + link = link->Fwd) { + KE_AUDIO_STREAM* stream = CONTAINING_RECORD(link, KE_AUDIO_STREAM, EndpointLink); + if (ObTryReferenceObject(&stream->Header)) + streams[count++] = stream; + } + KeReleaseSpinlock(&endpoint->StreamListLock, flags); + return count; +} + +// Applies the endpoint master ramp and writes one interleaved HDA period. +static void KiAudioWriteOutputPeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t periodIndex, + float* mix) { + KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioWriteOutputPeriod"); + float current = 1.0f; + float target = 1.0f; + float step = 0.0f; + uint32_t remaining = 0; + uint64_t generation = 0; + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + current = endpoint->MasterCurrentGain; + target = endpoint->MasterTargetGain; + step = endpoint->MasterGainStep; + remaining = endpoint->MasterRampRemaining; + generation = endpoint->MasterGainGeneration; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + int16_t* output = (int16_t*)(endpoint->PeriodBuffer + + (size_t)periodIndex * endpoint->PeriodStrideBytes); + for (uint32_t frame = 0; frame < endpoint->FramesPerPeriod; frame++) { + if (remaining) { + current += step; + remaining--; + if (!remaining) + current = target; + } + output[frame * 2u] = KiAudioFloatToOutputSample(mix[frame * 2u] * current); + output[frame * 2u + 1u] = KiAudioFloatToOutputSample(mix[frame * 2u + 1u] * current); + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (endpoint->MasterGainGeneration == generation) { + endpoint->MasterCurrentGain = current; + endpoint->MasterRampRemaining = remaining; + if (!remaining) + endpoint->MasterGainStep = 0.0f; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); +} + +// Reserves one free hardware period for passive-level mixing. +static int KiAudioReservePeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t* periodIndexOut, + uint64_t* generationOut) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + if (endpoint->Periods[i].State != KE_AUDIO_PERIOD_STATE_FREE) + continue; + if (endpoint->Periods[i].ContributionCount) + KiAudioBug("KiAudioReservePeriod", "free period retains contributions", i); + endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FILLING; + *periodIndexOut = i; + *generationOut = endpoint->Periods[i].Generation; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return 1; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return 0; +} + +// Discards one contribution array after its mixed source frames cannot be presented. +static void KiAudioDiscardContributions(KE_AUDIO_PERIOD_CONTRIBUTION* contributions, + uint32_t contributionCount) { + for (uint32_t i = 0; i < contributionCount; i++) { + KE_AUDIO_PERIOD_CONTRIBUTION* contribution = &contributions[i]; + KE_AUDIO_STREAM* stream = contribution->Stream; + KiAudioAcquireMutex(stream->Lock, "KiAudioDiscardContributions"); + if (stream->BufferGeneration == contribution->StreamGeneration) { + stream->FramesDiscarded += contribution->SourceFrames; + if (stream->State == KeAudioStreamDraining && + KiAudioDrainIsCompleteLocked(stream)) + KiAudioCompleteDrainLocked(stream); + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } +} + +// Commits one mixed period only if no missed deadline invalidated the reservation. +static int KiAudioCommitPeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t periodIndex, + uint64_t generation, + KE_AUDIO_PERIOD_CONTRIBUTION* contributions, + uint32_t contributionCount) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + KE_AUDIO_PERIOD_STATE* period = &endpoint->Periods[periodIndex]; + int committed = period->State == KE_AUDIO_PERIOD_STATE_FILLING && + period->Generation == generation; + if (committed) { + if (period->ContributionCount) + KiAudioBug("KiAudioCommitPeriod", "stale period contributions", periodIndex); + for (uint32_t i = 0; i < contributionCount; i++) + period->Contributions[i] = contributions[i]; + period->ContributionCount = contributionCount; + period->State = KE_AUDIO_PERIOD_STATE_READY; + endpoint->PeriodsMixed++; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (!committed) { + memset(endpoint->PeriodBuffer + (size_t)periodIndex * endpoint->PeriodStrideBytes, + 0, + endpoint->PeriodStrideBytes); + KiAudioDiscardContributions(contributions, contributionCount); + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (period->State == KE_AUDIO_PERIOD_STATE_FILLING || + period->State == KE_AUDIO_PERIOD_STATE_COMPLETED) + period->State = KE_AUDIO_PERIOD_STATE_FREE; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + } + return committed; +} + +// Mixes one free endpoint period from all currently referenced streams. +static int KiAudioMixOnePeriod(IO_AUDIO_ENDPOINT* endpoint) { + uint32_t periodIndex = 0; + uint64_t generation = 0; + if (!KiAudioReservePeriod(endpoint, &periodIndex, &generation)) + return 0; + + float mix[KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD * KE_AUDIO_OUTPUT_CHANNELS]; + memset(mix, 0, sizeof(mix)); + + KE_AUDIO_PERIOD_CONTRIBUTION contributions[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t contributionCount = 0; + KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); + for (uint32_t i = 0; i < streamCount; i++) { + uint64_t streamGeneration = 0; + uint32_t consumed = KiAudioMixStreamPeriod(streams[i], + mix, + endpoint->FramesPerPeriod, + &streamGeneration); + if (consumed) { + contributions[contributionCount].Stream = streams[i]; + contributions[contributionCount].StreamGeneration = streamGeneration; + contributions[contributionCount].SourceFrames = consumed; + contributionCount++; + } else { + ObDereferenceObject(&streams[i]->Header); + } + } + + KiAudioWriteOutputPeriod(endpoint, periodIndex, mix); + return KiAudioCommitPeriod(endpoint, + periodIndex, + generation, + contributions, + contributionCount); +} + +// Applies one hardware-completed period to per-stream presentation counters. +static int KiAudioReapOneCompletedPeriod(IO_AUDIO_ENDPOINT* endpoint) { + uint32_t periodIndex = 0; + KE_AUDIO_PERIOD_STATE* period = (KE_AUDIO_PERIOD_STATE*)0; + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + if (endpoint->Periods[i].State == KE_AUDIO_PERIOD_STATE_COMPLETED) { + periodIndex = i; + period = &endpoint->Periods[i]; + period->State = KE_AUDIO_PERIOD_STATE_REAPING; + break; + } + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (!period) + return 0; + + for (uint32_t i = 0; i < period->ContributionCount; i++) { + KE_AUDIO_PERIOD_CONTRIBUTION* contribution = &period->Contributions[i]; + KE_AUDIO_STREAM* stream = contribution->Stream; + KiAudioAcquireMutex(stream->Lock, "KiAudioReapOneCompletedPeriod"); + if (stream->BufferGeneration == contribution->StreamGeneration) { + stream->FramesPresented += contribution->SourceFrames; + if (stream->State == KeAudioStreamDraining && + KiAudioDrainIsCompleteLocked(stream)) + KiAudioCompleteDrainLocked(stream); + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } + + memset(endpoint->PeriodBuffer + (size_t)periodIndex * endpoint->PeriodStrideBytes, + 0, + endpoint->PeriodStrideBytes); + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + period->ContributionCount = 0; + period->State = KE_AUDIO_PERIOD_STATE_FREE; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return 1; +} + +// Counts periods ready for hardware consumption. +static uint32_t KiAudioReadyPeriodCount(IO_AUDIO_ENDPOINT* endpoint) { + uint32_t ready = 0; + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + if (endpoint->Periods[i].State == KE_AUDIO_PERIOD_STATE_READY) + ready++; + } + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return ready; +} + +// Starts provider DMA after at least two periods are prepared. +static KSTATUS KiAudioStartHardware(IO_AUDIO_ENDPOINT* endpoint) { + KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioStartHardware"); + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + if (endpoint->State != KeAudioEndpointReady) { + KSTATUS status = KFAILED(endpoint->FailureStatus) + ? endpoint->FailureStatus + : NTLSTATUS_DEVICE_REMOVED; + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KiAudioReleaseMutex(endpoint->ControlLock); + return status; + } + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (endpoint->HardwareRunning || endpoint->ActiveStreamCount == 0) { + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); + return NTLSTATUS_SUCCESS; + } + endpoint->HardwareRunning = 1; + endpoint->IdleDeadlineTicks = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + KSTATUS status = NTLSTATUS_SUCCESS; + int startAttempted = 0; + if (endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) { + status = endpoint->Operations.SetGain(endpoint->ProviderContext, + endpoint->MasterHardwareGainMilliDb, + 0); + } + if (KSUCCESS(status)) { + startAttempted = 1; + status = endpoint->Operations.Start(endpoint->ProviderContext); + } + if (!KSUCCESS(status) && startAttempted) + endpoint->Operations.Stop(endpoint->ProviderContext); + if (!KSUCCESS(status)) { + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->HardwareRunning = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); + return status; + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->HardwareStarts++; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); + return NTLSTATUS_SUCCESS; +} + +// Discards one period's unpresented stream contributions during hardware teardown. +static void KiAudioDiscardPeriodContributions(KE_AUDIO_PERIOD_STATE* period) { + KiAudioDiscardContributions(period->Contributions, period->ContributionCount); + period->ContributionCount = 0; +} + +// Stops provider DMA and returns every period to a zeroed free state. +static void KiAudioStopHardware(IO_AUDIO_ENDPOINT* endpoint, int hardwareAccessible) { + KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioStopHardware"); + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + int wasRunning = endpoint->HardwareRunning; + endpoint->HardwareRunning = 0; + endpoint->IdleDeadlineTicks = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + if (wasRunning && hardwareAccessible) { + if (endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) { + KSTATUS muteStatus = endpoint->Operations.SetGain( + endpoint->ProviderContext, + endpoint->MasterHardwareGainMilliDb, + 1); + if (!KSUCCESS(muteStatus)) { + DbgPrint("audio: endpoint %llu hardware mute failed status=0x%x\n", + (unsigned long long)endpoint->EndpointId, + muteStatus); + } + } + endpoint->Operations.Stop(endpoint->ProviderContext); + } + + while (KiAudioReapOneCompletedPeriod(endpoint)) { + } + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) + KiAudioDiscardPeriodContributions(&endpoint->Periods[i]); + memset(endpoint->PeriodBuffer, 0, endpoint->PeriodBufferBytes); + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { + endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FREE; + endpoint->Periods[i].Generation++; + } + endpoint->MasterCurrentGain = endpoint->MasterTargetGain; + endpoint->MasterGainStep = 0.0f; + endpoint->MasterRampRemaining = 0; + endpoint->MasterGainGeneration++; + if (wasRunning) + endpoint->HardwareStops++; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioReleaseMutex(endpoint->ControlLock); +} + +// Fails every logical stream while preserving final counters for later queries. +static void KiAudioFailAllStreams(IO_AUDIO_ENDPOINT* endpoint, KSTATUS status) { + KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); + for (uint32_t i = 0; i < streamCount; i++) { + KE_AUDIO_STREAM* stream = streams[i]; + KiAudioAcquireMutex(stream->Lock, "KiAudioFailAllStreams"); + if (stream->State != KeAudioStreamClosed) { + stream->FramesDiscarded += stream->BufferedFrames; + stream->BufferedFrames = 0; + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->ResamplePositionQ32 = 0; + KiAudioSetStreamStateLocked(stream, KeAudioStreamFailed); + stream->FailureStatus = status; + stream->BufferGeneration++; + stream->DiscontinuityGeneration++; + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + if (endpoint->ActiveStreamCount != 0) { + uint32_t active = endpoint->ActiveStreamCount; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KiAudioBug("KiAudioFailAllStreams", "active streams remained after failure", active); + } + endpoint->IdleDeadlineTicks = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); +} + +// Applies hardware deadline discontinuities to every live logical stream at passive level. +static void KiAudioApplyPendingDiscontinuities(IO_AUDIO_ENDPOINT* endpoint) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + uint64_t pending = endpoint->PendingDiscontinuities; + endpoint->PendingDiscontinuities = 0; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (!pending) + return; + + KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; + uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); + for (uint32_t i = 0; i < streamCount; i++) { + KE_AUDIO_STREAM* stream = streams[i]; + KiAudioAcquireMutex(stream->Lock, "KiAudioApplyPendingDiscontinuities"); + if (KiAudioStreamStateIsActive(stream->State)) { + stream->Underruns += pending; + stream->SilenceFramesInserted += pending * endpoint->FramesPerPeriod; + stream->DiscontinuityGeneration += pending; + KiAudioRefreshStreamEventLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + ObDereferenceObject(&stream->Header); + } +} + +// Returns one coherent endpoint lifecycle snapshot. +static void KiAudioSnapshotLifecycle(IO_AUDIO_ENDPOINT* endpoint, + KE_AUDIO_LIFECYCLE_SNAPSHOT* snapshot) { + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + snapshot->State = endpoint->State; + snapshot->FailureStatus = endpoint->FailureStatus; + snapshot->RemovalMode = endpoint->RemovalMode; + snapshot->SkipProviderStop = endpoint->SkipProviderStop; + KeReleaseSpinlock(&gAudioRegistryLock, flags); +} + +// Processes all currently available period and lifecycle work. +static int KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint, + uint64_t* waitMillisecondsOut) { + *waitMillisecondsOut = (uint64_t)-1; + while (KiAudioReapOneCompletedPeriod(endpoint)) { + } + KiAudioApplyPendingDiscontinuities(endpoint); + KE_AUDIO_LIFECYCLE_SNAPSHOT lifecycle; + KiAudioSnapshotLifecycle(endpoint, &lifecycle); + KSTATUS failure = lifecycle.FailureStatus; + if (lifecycle.State == KeAudioEndpointStopping || + lifecycle.State == KeAudioEndpointFailed || + lifecycle.State == KeAudioEndpointRemoved) { + int accessible = !lifecycle.SkipProviderStop && + lifecycle.RemovalMode != IoAudioSurpriseRemoval; + KiAudioStopHardware(endpoint, accessible); + if (lifecycle.State == KeAudioEndpointRemoved && KSUCCESS(failure)) + failure = NTLSTATUS_DEVICE_REMOVED; + if (KSUCCESS(failure)) + failure = NTLSTATUS_DEVICE_ERROR; + KiAudioFailAllStreams(endpoint, failure); + return 0; + } + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + uint32_t active = endpoint->ActiveStreamCount; + int running = endpoint->HardwareRunning; + uint64_t idleDeadline = endpoint->IdleDeadlineTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + if (active && !running) { + while (KiAudioReadyPeriodCount(endpoint) < 2u && + KiAudioMixOnePeriod(endpoint)) { + } + if (KiAudioReadyPeriodCount(endpoint) >= 2u) { + KSTATUS start = KiAudioStartHardware(endpoint); + if (!KSUCCESS(start)) { + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointFailed); + endpoint->FailureStatus = start; + KiAudioSelectDefaultEndpointLocked(); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KeSignalEvent(endpoint->WorkEvent); + return 1; + } + } + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + active = endpoint->ActiveStreamCount; + running = endpoint->HardwareRunning; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + if (active || running) { + while (KiAudioMixOnePeriod(endpoint)) { + } + } + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + active = endpoint->ActiveStreamCount; + running = endpoint->HardwareRunning; + idleDeadline = endpoint->IdleDeadlineTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + if (!active && running) { + uint64_t now = KiAudioQueryTicks(); + if (!idleDeadline) { + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->IdleDeadlineTicks = now + + (uint64_t)KE_AUDIO_IDLE_STOP_MILLISECONDS * KE_AUDIO_TICKS_PER_MILLISECOND; + idleDeadline = endpoint->IdleDeadlineTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + } + if (now >= idleDeadline) { + KiAudioStopHardware(endpoint, 1); + } else { + uint64_t remainingTicks = idleDeadline - now; + *waitMillisecondsOut = (remainingTicks + KE_AUDIO_TICKS_PER_MILLISECOND - 1u) / + KE_AUDIO_TICKS_PER_MILLISECOND; + if (!*waitMillisecondsOut) + *waitMillisecondsOut = 1; + } + } + return 1; +} + +// Runs one endpoint's real-time mixer and provider lifecycle loop. +static void KiAudioMixerThread(void* argument) { + IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)argument; + while (true) { + KeClearEvent(endpoint->WorkEvent); + uint64_t waitMilliseconds = (uint64_t)-1; + if (!KiAudioProcessEndpoint(endpoint, &waitMilliseconds)) + break; + + KSTATUS waitStatus = NTLSTATUS_SUCCESS; + if (waitMilliseconds == (uint64_t)-1) { + waitStatus = KeWaitForEvent(endpoint->WorkEvent, 0); + } else { + EVENT* events[1] = { endpoint->WorkEvent }; + uint32_t selected = 0; + waitStatus = KeWaitForMultipleEvents(events, + 1, + 0, + waitMilliseconds, + &selected); + } + if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT) + KiAudioBug("KiAudioMixerThread", "work wait failed", waitStatus); + } + + KeSignalEvent(endpoint->MixerExitEvent); + ObDereferenceObject(&endpoint->Header); +} + +// Destroys one fully unregistered endpoint after its mixer and streams leave. +static void KiAudioEndpointDestroy(OBJECT* object) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(object, IO_AUDIO_ENDPOINT, Header); + if (endpoint->Registered || endpoint->MixerThread || + !LIST_IS_EMPTY(&endpoint->StreamList) || endpoint->HardwareRunning || + endpoint->ActiveStreamCount) + KiAudioBug("KiAudioEndpointDestroy", "live endpoint reached destruction", endpoint->EndpointId); + if (endpoint->Device) + IoDereferenceDeviceObject(endpoint->Device); + if (endpoint->ControlLock) + ObDereferenceObject(&endpoint->ControlLock->Header); + if (endpoint->WorkEvent) + ObDereferenceObject(&endpoint->WorkEvent->Header); + if (endpoint->MixerExitEvent) + ObDereferenceObject(&endpoint->MixerExitEvent->Header); +} + +// Records one process handle that owns the stream. +void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream) { + if (!stream) + return; + + long handleCount = _InterlockedIncrement(&stream->ProcessHandleCount); + if (handleCount <= 0) + KiAudioBug("KeAudioStreamHandleOpened", "invalid process handle count", handleCount); +} + +// Requests passive stream closure when the final process handle disappears. +void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream) { + if (!stream) + return; + + long handleCount = _InterlockedDecrement(&stream->ProcessHandleCount); + if (handleCount < 0) + KiAudioBug("KeAudioStreamHandleClosed", "process handle underflow", handleCount); + if (handleCount != 0) + return; + + _InterlockedExchange(&stream->CloseRequested, 1); + KeSignalEvent(stream->WaitEvent); + if (stream->Endpoint) + KeSignalEvent(stream->Endpoint->WorkEvent); +} + +// Destroys one exclusively owned stream without entering blocking primitives. +static void KiAudioStreamDestroy(OBJECT* object) { + KE_AUDIO_STREAM* stream = CONTAINING_RECORD(object, KE_AUDIO_STREAM, Header); + IO_AUDIO_ENDPOINT* endpoint = stream->Endpoint; + if (endpoint) { + uintptr_t flags = KeAcquireSpinlock(&endpoint->StreamListLock); + if (stream->EndpointLink.Fwd != &stream->EndpointLink || + stream->EndpointLink.Bwd != &stream->EndpointLink) + KeListRemoveChecked(&stream->EndpointLink); + KeReleaseSpinlock(&endpoint->StreamListLock, flags); + } + + if (stream->Lock && + (stream->Lock->Owner || !LIST_IS_EMPTY(&stream->Lock->WaitList))) + KiAudioBug("KiAudioStreamDestroy", "live stream mutex reached destruction", + (uint64_t)(uintptr_t)stream->Lock->Owner); + if (!KiAudioStreamTransitionAllowed(stream->State, KeAudioStreamClosed)) + KiAudioBug("KiAudioStreamDestroy", "invalid terminal stream state", stream->State); + int wasActive = KiAudioStreamStateIsActive(stream->State); + stream->State = KeAudioStreamClosed; + if (wasActive) + KiAudioAdjustActiveStreams(endpoint, -1); + + if (stream->Ring) + MmFreePool(stream->Ring); + if (stream->WaitEvent) + ObDereferenceObject(&stream->WaitEvent->Header); + if (stream->Lock) + ObDereferenceObject(&stream->Lock->Header); + if (endpoint) + ObDereferenceObject(&endpoint->Header); + _InterlockedIncrement64(&gAudioCoreStats.StreamsDestroyed); +} + +// Registers one hardware-provider endpoint. +static KSTATUS KiAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, + IO_AUDIO_ENDPOINT** outEndpoint) { + if (!outEndpoint) + return NTLSTATUS_INVALID_PARAMETER; + *outEndpoint = (IO_AUDIO_ENDPOINT*)0; + KSTATUS status = KiAudioValidateRegistration(registration); + if (!KSUCCESS(status)) + return status; + + IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)ObAllocateObject( + ObjectTypeAudioEndpoint, + sizeof(IO_AUDIO_ENDPOINT)); + if (!endpoint) + return NTLSTATUS_NO_MEMORY; + endpoint->Header.Destroy = KiAudioEndpointDestroy; + LIST_INIT(&endpoint->RegistryLink); + LIST_INIT(&endpoint->StreamList); + endpoint->StreamListLock = (SPINLOCK)SPINLOCK_INIT; + endpoint->NotifyLock = (SPINLOCK)SPINLOCK_INIT; + endpoint->Kind = registration->Kind; + endpoint->State = KeAudioEndpointStarting; + endpoint->Capabilities = registration->Capabilities; + endpoint->Available = registration->InitiallyAvailable ? 1u : 0u; + endpoint->Device = registration->Device; + endpoint->ProviderContext = registration->ProviderContext; + endpoint->Operations = registration->Operations; + endpoint->PeriodBuffer = (uint8_t*)registration->PeriodBuffer; + endpoint->PeriodBufferBytes = registration->PeriodBufferBytes; + endpoint->PeriodCount = registration->PeriodCount; + endpoint->FramesPerPeriod = registration->FramesPerPeriod; + endpoint->PeriodStrideBytes = registration->PeriodStrideBytes; + endpoint->HardwareFormat = registration->HardwareFormat; + endpoint->HardwareGainMinimumMilliDb = registration->HardwareGainMinimumMilliDb; + endpoint->HardwareGainMaximumMilliDb = registration->HardwareGainMaximumMilliDb; + endpoint->HardwareGainStepMilliDb = registration->HardwareGainStepMilliDb; + endpoint->MasterRequestedVolume = 1.0f; + endpoint->MasterHardwareGainMilliDb = KiAudioHighestSafeHardwareGain(endpoint); + endpoint->MasterCurrentGain = KiAudioSoftwareGainForVolume(endpoint, 1.0f); + endpoint->MasterTargetGain = endpoint->MasterCurrentGain; + endpoint->RemovalMode = IoAudioOrderlyRemoval; + memset(endpoint->PeriodBuffer, 0, endpoint->PeriodBufferBytes); + for (uint32_t i = 0; i < endpoint->PeriodCount; i++) + endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FREE; + + if (endpoint->Device) + IoReferenceDeviceObject(endpoint->Device); + + status = KeMutexCreate(&endpoint->ControlLock); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + return status; + } + status = KeEventCreate(EventManualReset, 0, &endpoint->WorkEvent); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + return status; + } + status = KeEventCreate(EventManualReset, 0, &endpoint->MixerExitEvent); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + return status; + } + + ObReferenceObject(&endpoint->Header); + const KE_THREAD_CREATE_PARAMETERS mixerParameters = + KE_THREAD_CREATE_REALTIME(KE_AUDIO_MIXER_REALTIME_PRIORITY); + status = KeCreateKernelThread((uintptr_t)KiAudioMixerThread, + (uintptr_t)endpoint, + &mixerParameters, + &endpoint->MixerThread); + if (!KSUCCESS(status)) { + ObDereferenceObject(&endpoint->Header); + ObDereferenceObject(&endpoint->Header); + return status; + } + + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + if (gAudioEndpointCount >= KE_AUDIO_MAX_ENDPOINTS) { + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping); + endpoint->FailureStatus = NTLSTATUS_NO_MEMORY; + KeReleaseSpinlock(&gAudioRegistryLock, flags); + KeSignalEvent(endpoint->WorkEvent); + (void)KeWaitForEvent(endpoint->MixerExitEvent, 0); + ObDereferenceObject(&endpoint->MixerThread->Header); + endpoint->MixerThread = (THREAD*)0; + ObDereferenceObject(&endpoint->Header); + return NTLSTATUS_NO_MEMORY; + } + endpoint->EndpointId = gAudioNextEndpointId++; + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointReady); + endpoint->Registered = 1; + ObReferenceObject(&endpoint->Header); + KeListInsertTailChecked(&gAudioEndpointList, &endpoint->RegistryLink); + gAudioEndpointCount++; + KiAudioSelectDefaultEndpointLocked(); + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + _InterlockedIncrement64(&gAudioCoreStats.EndpointsRegistered); + *outEndpoint = endpoint; + return NTLSTATUS_SUCCESS; +} + +// Implements the provider-facing endpoint registration operation. +static KSTATUS KiAudioDdkRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, + IO_AUDIO_ENDPOINT** outEndpoint) { + return KiAudioRegisterEndpoint(registration, outEndpoint); +} + +// Removes one endpoint and waits until no provider operation can run again. +static void KiAudioDdkUnregisterEndpoint(IO_AUDIO_ENDPOINT* endpoint, + IO_AUDIO_REMOVAL_MODE mode) { + if (!endpoint || (mode != IoAudioOrderlyRemoval && mode != IoAudioSurpriseRemoval)) + KiAudioBug("KiAudioDdkUnregisterEndpoint", "invalid unregister request", mode); + + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + if (!endpoint->Registered) { + KeReleaseSpinlock(&gAudioRegistryLock, flags); + KiAudioBug("KiAudioDdkUnregisterEndpoint", "duplicate unregister", endpoint->EndpointId); + } + KeListRemoveChecked(&endpoint->RegistryLink); + endpoint->Registered = 0; + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping); + if (KSUCCESS(endpoint->FailureStatus)) + endpoint->FailureStatus = NTLSTATUS_DEVICE_REMOVED; + endpoint->RemovalMode = mode; + endpoint->SkipProviderStop = mode == IoAudioSurpriseRemoval ? 1u : 0u; + gAudioEndpointCount--; + KiAudioSelectDefaultEndpointLocked(); + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + KeSignalEvent(endpoint->WorkEvent); + (void)KeWaitForEvent(endpoint->MixerExitEvent, 0); + + flags = KeAcquireSpinlock(&gAudioRegistryLock); + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointRemoved); + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + if (!endpoint->MixerThread) + KiAudioBug("KiAudioDdkUnregisterEndpoint", "missing mixer thread", endpoint->EndpointId); + ObDereferenceObject(&endpoint->MixerThread->Header); + endpoint->MixerThread = (THREAD*)0; + + _InterlockedIncrement64(&gAudioCoreStats.EndpointsUnregistered); + ObDereferenceObject(&endpoint->Header); + ObDereferenceObject(&endpoint->Header); +} + +// Advances one provider ring period and wakes the real-time mixer. +static void KiAudioDdkNotifyPeriod(IO_AUDIO_ENDPOINT* endpoint, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t hardwarePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks) { + if (!endpoint || completedPeriodIndex >= endpoint->PeriodCount) + KiAudioBug("KiAudioDdkNotifyPeriod", "invalid period notification", completedPeriodIndex); + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + KE_AUDIO_PERIOD_STATE* period = &endpoint->Periods[completedPeriodIndex]; + if (!endpoint->HardwareRunning) { + endpoint->SpuriousNotifications++; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + return; + } + + int missedDeadline = period->State != KE_AUDIO_PERIOD_STATE_READY; + if (missedDeadline) { + endpoint->Underruns++; + endpoint->DiscontinuityGeneration++; + endpoint->SilenceFramesInserted += endpoint->FramesPerPeriod; + endpoint->PendingDiscontinuities++; + } + if (period->State == KE_AUDIO_PERIOD_STATE_COMPLETED || + period->State == KE_AUDIO_PERIOD_STATE_REAPING) { + endpoint->SpuriousNotifications++; + } else { + period->Generation++; + period->State = KE_AUDIO_PERIOD_STATE_COMPLETED; + } + endpoint->PeriodsCompleted++; + if (presentedFrames < endpoint->FramesPresented) { + endpoint->DiscontinuityGeneration++; + endpoint->PendingDiscontinuities++; + } else { + endpoint->FramesPresented = presentedFrames; + } + if (hardwarePositionFrames < endpoint->HardwarePositionFrames || + interruptCount < endpoint->InterruptCount) { + endpoint->DiscontinuityGeneration++; + endpoint->PendingDiscontinuities++; + } else { + endpoint->HardwarePositionFrames = hardwarePositionFrames; + endpoint->InterruptCount = interruptCount; + } + endpoint->LastPresentationTimestampTicks = timestampTicks; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + + KeSignalEvent(endpoint->WorkEvent); +} + +// Updates one endpoint's jack-derived availability and default selection. +static void KiAudioDdkSetEndpointAvailable(IO_AUDIO_ENDPOINT* endpoint, uint8_t available) { + if (!endpoint || !(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE)) + KiAudioBug("KiAudioDdkSetEndpointAvailable", "invalid availability update", 0); + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + if (endpoint->State == KeAudioEndpointReady) { + endpoint->Available = available ? 1u : 0u; + KiAudioSelectDefaultEndpointLocked(); + } + KeReleaseSpinlock(&gAudioRegistryLock, flags); +} + +// Fails one endpoint from provider completion or recovery code. +static void KiAudioDdkReportEndpointFailure(IO_AUDIO_ENDPOINT* endpoint, + KSTATUS status, + uint32_t flags) { + if (!endpoint || !KFAILED(status) || (flags & ~IO_AUDIO_FAILURE_VALID_MASK) != 0) + KiAudioBug("KiAudioDdkReportEndpointFailure", "invalid failure report", status); + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + if (flags & IO_AUDIO_FAILURE_HARDWARE_INACCESSIBLE) + endpoint->SkipProviderStop = 1; + if (endpoint->State == KeAudioEndpointReady) { + KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointFailed); + endpoint->FailureStatus = status; + endpoint->Available = 0; + KiAudioSelectDefaultEndpointLocked(); + } + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + DbgPrint("audio: endpoint %llu failed status=0x%x flags=0x%x\n", + (unsigned long long)endpoint->EndpointId, + status, + flags); + KeSignalEvent(endpoint->WorkEvent); +} + +static const NTL_DDK_AUDIO gAudioDdk = { + .RegisterEndpoint = KiAudioDdkRegisterEndpoint, + .UnregisterEndpoint = KiAudioDdkUnregisterEndpoint, + .NotifyPeriod = KiAudioDdkNotifyPeriod, + .SetEndpointAvailable = KiAudioDdkSetEndpointAvailable, + .ReportEndpointFailure = KiAudioDdkReportEndpointFailure, +}; + +// Initializes the one kernel audio registry. +void KeAudioInitialize(void) { + if (gAudioInitialized) + KiAudioBug("KeAudioInitialize", "duplicate initialization", 0); + LIST_INIT(&gAudioEndpointList); + gAudioInitialized = 1; +} + +// Returns the immutable provider-facing audio table. +const NTL_DDK_AUDIO* KeAudioGetDdk(void) { + return &gAudioDdk; +} + +// Returns the current default endpoint identifier. +uint64_t KeAudioGetDefaultEndpointId(void) { + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + uint64_t endpointId = gAudioDefaultEndpointId; + KeReleaseSpinlock(&gAudioRegistryLock, flags); + return endpointId; +} + +// Copies endpoint snapshots without retaining registry locks across passive operations. +KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t capacity, + uint32_t* countOut) { + if (!countOut || (capacity && !endpoints)) + return NTLSTATUS_INVALID_PARAMETER; + + IO_AUDIO_ENDPOINT* references[KE_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + uint64_t defaultId = 0; + uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); + defaultId = gAudioDefaultEndpointId; + for (LIST_ENTRY* link = gAudioEndpointList.Fwd; + link != &gAudioEndpointList && count < KE_AUDIO_MAX_ENDPOINTS; + link = link->Fwd) { + IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); + if (ObTryReferenceObject(&endpoint->Header)) + references[count++] = endpoint; + } + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + *countOut = count; + uint32_t copyCount = capacity < count ? capacity : count; + for (uint32_t i = 0; i < count; i++) { + IO_AUDIO_ENDPOINT* endpoint = references[i]; + if (i < copyCount) { + KE_AUDIO_ENDPOINT_INFO* info = &endpoints[i]; + memset(info, 0, sizeof(*info)); + flags = KeAcquireSpinlock(&gAudioRegistryLock); + info->EndpointId = endpoint->EndpointId; + info->Kind = endpoint->Kind; + info->State = endpoint->State; + info->Capabilities = endpoint->Capabilities; + info->HardwareFormat = endpoint->HardwareFormat; + info->PeriodCount = endpoint->PeriodCount; + info->FramesPerPeriod = endpoint->FramesPerPeriod; + info->Available = endpoint->Available; + info->IsDefault = endpoint->EndpointId == defaultId; + info->FailureStatus = endpoint->FailureStatus; + KeReleaseSpinlock(&gAudioRegistryLock, flags); + + uintptr_t streamFlags = KeAcquireSpinlock(&endpoint->StreamListLock); + for (LIST_ENTRY* link = endpoint->StreamList.Fwd; + link != &endpoint->StreamList; + link = link->Fwd) + info->StreamCount++; + KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); + + flags = KeAcquireSpinlock(&endpoint->NotifyLock); + info->ActiveStreamCount = endpoint->ActiveStreamCount; + info->HardwareRunning = endpoint->HardwareRunning; + info->MasterMuted = endpoint->MasterMuted; + info->MasterHardwareGainMilliDb = endpoint->MasterHardwareGainMilliDb; + info->MasterRequestedVolume = endpoint->MasterRequestedVolume; + info->PeriodsMixed = endpoint->PeriodsMixed; + info->PeriodsCompleted = endpoint->PeriodsCompleted; + info->FramesPresented = endpoint->FramesPresented; + info->HardwarePositionFrames = endpoint->HardwarePositionFrames; + info->InterruptCount = endpoint->InterruptCount; + info->LastPresentationTimestampTicks = endpoint->LastPresentationTimestampTicks; + info->SilenceFramesInserted = endpoint->SilenceFramesInserted; + info->Underruns = endpoint->Underruns; + info->DiscontinuityGeneration = endpoint->DiscontinuityGeneration; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + } + ObDereferenceObject(&endpoint->Header); + } + return capacity < count ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS; +} + +// Creates one logical render stream with a fixed-capacity source ring. +KSTATUS KeAudioCreateStream(uint64_t endpointId, + const IO_AUDIO_FORMAT* format, + KE_AUDIO_STREAM** outStream) { + if (!outStream || !KiAudioValidateSourceFormat(format)) + return NTLSTATUS_INVALID_PARAMETER; + *outStream = (KE_AUDIO_STREAM*)0; + IO_AUDIO_ENDPOINT* endpoint = KiAudioReferenceEndpointById(endpointId); + if (!endpoint) + return NTLSTATUS_NOT_FOUND; + + KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)ObAllocateObject( + ObjectTypeAudioStream, + sizeof(KE_AUDIO_STREAM)); + if (!stream) { + ObDereferenceObject(&endpoint->Header); + return NTLSTATUS_NO_MEMORY; + } + stream->Header.Destroy = KiAudioStreamDestroy; + LIST_INIT(&stream->EndpointLink); + stream->Endpoint = endpoint; + stream->State = KeAudioStreamCreated; + stream->Format = *format; + stream->FrameBytes = KiAudioFrameBytes(format); + stream->ResampleStepQ32 = ((uint64_t)format->SampleRate << 32) / + KE_AUDIO_OUTPUT_SAMPLE_RATE; + stream->WaitMask = KE_AUDIO_WAIT_WRITABLE; + stream->WaitWritableFrames = 1; + stream->Volume = 1.0f; + stream->CurrentLeftGain = 1.0f; + stream->CurrentRightGain = 1.0f; + stream->TargetLeftGain = 1.0f; + stream->TargetRightGain = 1.0f; + + KSTATUS status = KeMutexCreate(&stream->Lock); + if (!KSUCCESS(status)) { + ObDereferenceObject(&stream->Header); + return status; + } + status = KeEventCreate(EventManualReset, 1, &stream->WaitEvent); + if (!KSUCCESS(status)) { + ObDereferenceObject(&stream->Header); + return status; + } + stream->Ring = (uint8_t*)MmAllocatePoolTag( + (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes, + KE_AUDIO_RING_TAG); + if (!stream->Ring) { + ObDereferenceObject(&stream->Header); + return NTLSTATUS_NO_MEMORY; + } + memset(stream->Ring, + 0, + (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes); + + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + int endpointReady = endpoint->Registered && endpoint->State == KeAudioEndpointReady; + uint32_t streamCount = 0; + uintptr_t streamFlags = KeAcquireSpinlock(&endpoint->StreamListLock); + if (endpointReady) { + for (LIST_ENTRY* link = endpoint->StreamList.Fwd; + link != &endpoint->StreamList; + link = link->Fwd) + streamCount++; + } + if (!endpointReady || streamCount >= KE_AUDIO_MAX_STREAMS_PER_ENDPOINT) { + KSTATUS createStatus = !endpointReady + ? (KFAILED(endpoint->FailureStatus) + ? endpoint->FailureStatus + : NTLSTATUS_DEVICE_REMOVED) + : NTLSTATUS_NO_MEMORY; + KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + ObDereferenceObject(&stream->Header); + return createStatus; + } + KeListInsertTailChecked(&endpoint->StreamList, &stream->EndpointLink); + KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + + _InterlockedIncrement64(&gAudioCoreStats.StreamsCreated); + *outStream = stream; + return NTLSTATUS_SUCCESS; +} + +// Returns the stream event used by future waitable handles. +EVENT* KeAudioStreamWaitEvent(KE_AUDIO_STREAM* stream) { + return stream ? stream->WaitEvent : (EVENT*)0; +} + +// Arms one wait condition while holding the stream state that drives its event. +KSTATUS KeAudioArmStreamWait(KE_AUDIO_STREAM* stream, + const KE_AUDIO_STREAM_WAIT_CONDITION* condition) { + if (!stream || !condition || !condition->Mask || + (condition->Mask & ~KE_AUDIO_WAIT_VALID_MASK) != 0 || + ((condition->Mask & KE_AUDIO_WAIT_WRITABLE) && + (!condition->WritableFrames || + condition->WritableFrames > KE_AUDIO_STREAM_RING_FRAMES))) + return NTLSTATUS_INVALID_PARAMETER; + + KiAudioAcquireMutex(stream->Lock, "KeAudioArmStreamWait"); + if (stream->State == KeAudioStreamClosed) { + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_OBJECT_DEAD; + } + stream->WaitMask = condition->Mask; + stream->WaitWritableFrames = condition->WritableFrames; + stream->WaitConsumedFrames = condition->ConsumedFrames; + stream->WaitPresentedFrames = condition->PresentedFrames; + stream->WaitDrainedGeneration = condition->DrainedGeneration; + stream->WaitDiscontinuityGeneration = condition->DiscontinuityGeneration; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_SUCCESS; +} + +// Copies a bounded source-frame run into one stream ring. +KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream, + const void* frames, + uint32_t requestedFrames, + uint32_t* writtenFramesOut) { + if (!stream || !writtenFramesOut || (requestedFrames && !frames)) + return NTLSTATUS_INVALID_PARAMETER; + *writtenFramesOut = 0; + if (!requestedFrames) + return NTLSTATUS_SUCCESS; + + KiAudioAcquireMutex(stream->Lock, "KeAudioWriteStream"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + if (stream->State == KeAudioStreamDraining) { + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_INVALID_PARAMETER; + } + + uint32_t writable = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; + if (!writable) { + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_WOULD_BLOCK; + } + uint32_t toWrite = requestedFrames < writable ? requestedFrames : writable; + uint32_t firstFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->WriteFrame; + if (firstFrames > toWrite) firstFrames = toWrite; + size_t firstBytes = (size_t)firstFrames * stream->FrameBytes; + memcpy(stream->Ring + (size_t)stream->WriteFrame * stream->FrameBytes, + frames, + firstBytes); + uint32_t secondFrames = toWrite - firstFrames; + if (secondFrames) { + memcpy(stream->Ring, + (const uint8_t*)frames + firstBytes, + (size_t)secondFrames * stream->FrameBytes); + } + stream->WriteFrame = (stream->WriteFrame + toWrite) % KE_AUDIO_STREAM_RING_FRAMES; + stream->BufferedFrames += toWrite; + stream->FramesSubmitted += toWrite; + *writtenFramesOut = toWrite; + KiAudioRefreshStreamEventLocked(stream); + int active = KiAudioStreamStateIsActive(stream->State); + KiAudioReleaseMutex(stream->Lock); + + if (active) + KeSignalEvent(stream->Endpoint->WorkEvent); + return NTLSTATUS_SUCCESS; +} + +// Starts one created or stopped stream. +KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioStartStream"); + if (stream->State != KeAudioStreamCreated && stream->State != KeAudioStreamStopped) { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioSetStreamStateLocked(stream, KeAudioStreamRunning); + stream->FailureStatus = NTLSTATUS_SUCCESS; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + _InterlockedIncrement64(&gAudioCoreStats.Starts); + return NTLSTATUS_SUCCESS; +} + +// Pauses one running stream without discarding source frames. +KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioPauseStream"); + if (stream->State != KeAudioStreamRunning) { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioSetStreamStateLocked(stream, KeAudioStreamPaused); + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + _InterlockedIncrement64(&gAudioCoreStats.Pauses); + return NTLSTATUS_SUCCESS; +} + +// Resumes one paused stream. +KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioResumeStream"); + if (stream->State != KeAudioStreamPaused) { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioSetStreamStateLocked(stream, KeAudioStreamRunning); + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + _InterlockedIncrement64(&gAudioCoreStats.Resumes); + return NTLSTATUS_SUCCESS; +} + +// Stops accepting writes and plays all currently buffered frames. +KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioDrainStream"); + if (stream->State == KeAudioStreamRunning || + stream->State == KeAudioStreamPaused) { + KiAudioSetStreamStateLocked(stream, KeAudioStreamDraining); + if (KiAudioDrainIsCompleteLocked(stream)) + KiAudioCompleteDrainLocked(stream); + } else { + KSTATUS status = stream->State == KeAudioStreamFailed + ? stream->FailureStatus + : NTLSTATUS_INVALID_PARAMETER; + KiAudioReleaseMutex(stream->Lock); + return status; + } + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + _InterlockedIncrement64(&gAudioCoreStats.Drains); + return NTLSTATUS_SUCCESS; +} + +// Stops future mixing and discards source frames not yet committed to endpoint periods. +KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioStopStream"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + stream->FramesDiscarded += stream->BufferedFrames; + KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->BufferedFrames = 0; + stream->ResamplePositionQ32 = 0; + stream->ConsecutiveUnderrunPeriods = 0; + stream->DiscontinuityGeneration++; + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + _InterlockedIncrement64(&gAudioCoreStats.Stops); + return NTLSTATUS_SUCCESS; +} + +// Resets one live stream and invalidates presentation accounting from earlier periods. +KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream) { + if (!stream) return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioResetStream"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + stream->BufferGeneration++; + KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); + stream->ReadFrame = 0; + stream->WriteFrame = 0; + stream->BufferedFrames = 0; + stream->ResamplePositionQ32 = 0; + stream->FramesSubmitted = 0; + stream->FramesConsumed = 0; + stream->FramesPresented = 0; + stream->FramesDiscarded = 0; + stream->OutputFramesMixed = 0; + stream->SilenceFramesInserted = 0; + stream->Underruns = 0; + stream->ConsecutiveUnderrunPeriods = 0; + stream->DiscontinuityGeneration++; + memset(stream->Ring, + 0, + (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes); + KiAudioRefreshStreamEventLocked(stream); + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + _InterlockedIncrement64(&gAudioCoreStats.Resets); + return NTLSTATUS_SUCCESS; +} + +// Updates per-stream volume, linear balance pan, and mute targets. +KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream, + float volume, + float pan, + bool muted) { + if (!stream || !KiAudioFloatIsFinite(volume) || !KiAudioFloatIsFinite(pan) || + volume < 0.0f || volume > 1.0f || pan < -1.0f || pan > 1.0f) + return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioSetStreamVolume"); + if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { + KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; + KiAudioReleaseMutex(stream->Lock); + return status; + } + stream->Volume = volume; + stream->Pan = pan; + stream->Muted = muted ? 1u : 0u; + if (stream->State == KeAudioStreamCreated || stream->State == KeAudioStreamStopped) { + KiAudioComputeStreamTargets(stream->Volume, + stream->Pan, + stream->Muted, + &stream->CurrentLeftGain, + &stream->CurrentRightGain); + stream->TargetLeftGain = stream->CurrentLeftGain; + stream->TargetRightGain = stream->CurrentRightGain; + stream->GainRampRemaining = 0; + stream->LeftGainStep = 0.0f; + stream->RightGainStep = 0.0f; + } else { + KiAudioBeginStreamGainRampLocked(stream); + } + KiAudioReleaseMutex(stream->Lock); + KeSignalEvent(stream->Endpoint->WorkEvent); + return NTLSTATUS_SUCCESS; +} + +// Updates endpoint master software gain and separate logical mute state. +KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted) { + if (!KiAudioFloatIsFinite(volume) || volume < 0.0f || volume > 1.0f) + return NTLSTATUS_INVALID_PARAMETER; + IO_AUDIO_ENDPOINT* endpoint = KiAudioReferenceEndpointById(endpointId); + if (!endpoint) + return NTLSTATUS_NOT_FOUND; + + KiAudioAcquireMutex(endpoint->ControlLock, "KeAudioSetMasterVolume"); + float softwareGain = muted ? 0.0f + : KiAudioSoftwareGainForVolume(endpoint, volume); + uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); + if (endpoint->State != KeAudioEndpointReady) { + KSTATUS failure = endpoint->FailureStatus; + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KiAudioReleaseMutex(endpoint->ControlLock); + ObDereferenceObject(&endpoint->Header); + return KFAILED(failure) ? failure : NTLSTATUS_OBJECT_DEAD; + } + + uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); + endpoint->MasterRequestedVolume = volume; + endpoint->MasterTargetGain = softwareGain; + if (endpoint->HardwareRunning) { + endpoint->MasterGainStep = + (endpoint->MasterTargetGain - endpoint->MasterCurrentGain) / + (float)KE_AUDIO_GAIN_RAMP_FRAMES; + endpoint->MasterRampRemaining = KE_AUDIO_GAIN_RAMP_FRAMES; + } else { + endpoint->MasterCurrentGain = endpoint->MasterTargetGain; + endpoint->MasterGainStep = 0.0f; + endpoint->MasterRampRemaining = 0; + } + endpoint->MasterGainGeneration++; + endpoint->MasterMuted = muted ? 1u : 0u; + KeReleaseSpinlock(&endpoint->NotifyLock, flags); + KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); + KiAudioReleaseMutex(endpoint->ControlLock); + + KeSignalEvent(endpoint->WorkEvent); + ObDereferenceObject(&endpoint->Header); + return NTLSTATUS_SUCCESS; +} + +// Copies one stream snapshot under its priority-inheriting mutex. +KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut) { + if (!stream || !infoOut) + return NTLSTATUS_INVALID_PARAMETER; + KiAudioAcquireMutex(stream->Lock, "KeAudioQueryStream"); + memset(infoOut, 0, sizeof(*infoOut)); + infoOut->State = stream->State; + infoOut->Format = stream->Format; + infoOut->CapacityFrames = KE_AUDIO_STREAM_RING_FRAMES; + infoOut->BufferedFrames = stream->BufferedFrames; + infoOut->WritableFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; + infoOut->FramesSubmitted = stream->FramesSubmitted; + infoOut->FramesConsumed = stream->FramesConsumed; + infoOut->FramesPresented = stream->FramesPresented; + infoOut->FramesDiscarded = stream->FramesDiscarded; + infoOut->OutputFramesMixed = stream->OutputFramesMixed; + infoOut->SilenceFramesInserted = stream->SilenceFramesInserted; + infoOut->Underruns = stream->Underruns; + infoOut->DrainedGeneration = stream->DrainedGeneration; + infoOut->DiscontinuityGeneration = stream->DiscontinuityGeneration; + infoOut->FailureStatus = stream->FailureStatus; + infoOut->Volume = stream->Volume; + infoOut->Pan = stream->Pan; + infoOut->Muted = stream->Muted; + KiAudioReleaseMutex(stream->Lock); + return NTLSTATUS_SUCCESS; +} + +// Copies one endpoint snapshot by stable identifier. +static KSTATUS KiAudioSelfTestQueryEndpoint(uint64_t endpointId, + KE_AUDIO_ENDPOINT_INFO* infoOut) { + KE_AUDIO_ENDPOINT_INFO infos[KE_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + KSTATUS status = KeAudioQueryEndpoints(infos, KE_AUDIO_MAX_ENDPOINTS, &count); + if (KFAILED(status)) + return status; + for (uint32_t index = 0; index < count; index++) { + if (infos[index].EndpointId == endpointId) { + *infoOut = infos[index]; + return NTLSTATUS_SUCCESS; + } + } + return NTLSTATUS_NOT_FOUND; +} + +// Generates one bounded run of a ramped 440 Hz stereo sine wave. +static void KiAudioSelfTestGenerateTone(int16_t* frames, + uint32_t frameCount, + uint64_t firstFrame, + double* sineInOut, + double* cosineInOut) { + const uint64_t totalFrames = 3ull * KE_AUDIO_OUTPUT_SAMPLE_RATE; + const uint64_t rampFrames = 384ull; + const double amplitude = 8231.0; + const double sineStep = 0.057564026959567284; + const double cosineStep = 0.9983418166140283; + double sine = *sineInOut; + double cosine = *cosineInOut; + + for (uint32_t index = 0; index < frameCount; index++) { + uint64_t frame = firstFrame + index; + uint64_t framesFromEnd = totalFrames - 1ull - frame; + double envelope = 1.0; + if (frame < rampFrames) + envelope = (double)frame / (double)(rampFrames - 1ull); + if (framesFromEnd < rampFrames) { + double fadeOut = (double)framesFromEnd / (double)(rampFrames - 1ull); + if (fadeOut < envelope) + envelope = fadeOut; + } + + double scaled = sine * amplitude * envelope; + int32_t sample = scaled >= 0.0 ? (int32_t)(scaled + 0.5) + : (int32_t)(scaled - 0.5); + frames[index * 2u] = (int16_t)sample; + frames[index * 2u + 1u] = (int16_t)sample; + + double nextSine = sine * cosineStep + cosine * sineStep; + double nextCosine = cosine * cosineStep - sine * sineStep; + sine = nextSine; + cosine = nextCosine; + } + + *sineInOut = sine; + *cosineInOut = cosine; +} + +// Waits for one selftest stream condition with a finite boot-time deadline. +static KSTATUS KiAudioSelfTestWait(KE_AUDIO_STREAM* stream) { + EVENT* event = KeAudioStreamWaitEvent(stream); + if (!event) + return NTLSTATUS_OBJECT_DEAD; + uint32_t selected = 0; + return KeWaitForMultipleEvents(&event, + 1, + 0, + KE_AUDIO_SELFTEST_WAIT_MILLISECONDS, + &selected); +} + +// Writes one generated run while using the stream event for bounded backpressure. +static KSTATUS KiAudioSelfTestWriteFrames(KE_AUDIO_STREAM* stream, + const int16_t* frames, + uint32_t frameCount, + uint64_t expectedDiscontinuity) { + uint32_t offset = 0; + while (offset < frameCount) { + uint32_t written = 0; + KSTATUS status = KeAudioWriteStream(stream, + frames + offset * 2u, + frameCount - offset, + &written); + if (KSUCCESS(status) && written) { + offset += written; + continue; + } + if (status != NTLSTATUS_WOULD_BLOCK) + return KFAILED(status) ? status : NTLSTATUS_FAILURE; + + KE_AUDIO_STREAM_INFO info; + status = KeAudioQueryStream(stream, &info); + if (KFAILED(status)) + return status; + if (info.State == KeAudioStreamFailed) + return KFAILED(info.FailureStatus) ? info.FailureStatus : NTLSTATUS_DEVICE_ERROR; + if (info.DiscontinuityGeneration != expectedDiscontinuity) + return NTLSTATUS_DEVICE_ERROR; + + KE_AUDIO_STREAM_WAIT_CONDITION condition = { + .Mask = KE_AUDIO_WAIT_WRITABLE | KE_AUDIO_WAIT_FAILED | KE_AUDIO_WAIT_DISCONTINUITY, + .WritableFrames = 1, + .DiscontinuityGeneration = expectedDiscontinuity, + }; + status = KeAudioArmStreamWait(stream, &condition); + if (KFAILED(status)) + return status; + status = KiAudioSelfTestWait(stream); + if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Prints the common endpoint and stream counters for one selftest outcome. +static void KiAudioSelfTestPrintCounters(uint64_t endpointId, + const KE_AUDIO_ENDPOINT_INFO* endpointBefore, + const KE_AUDIO_ENDPOINT_INFO* endpointAfter, + const KE_AUDIO_STREAM_INFO* streamInfo, + uint64_t expectedDiscontinuity) { + uint64_t mixedPeriods = endpointAfter->PeriodsMixed - endpointBefore->PeriodsMixed; + uint64_t completedPeriods = endpointAfter->PeriodsCompleted - endpointBefore->PeriodsCompleted; + uint64_t interruptCount = endpointAfter->InterruptCount - endpointBefore->InterruptCount; + uint64_t endpointUnderruns = endpointAfter->Underruns - endpointBefore->Underruns; + uint64_t endpointDiscontinuities = endpointAfter->DiscontinuityGeneration - + endpointBefore->DiscontinuityGeneration; + uint64_t positionFrames = endpointAfter->HardwarePositionFrames - + endpointBefore->HardwarePositionFrames; + uint64_t streamDiscontinuities = streamInfo->DiscontinuityGeneration >= expectedDiscontinuity + ? streamInfo->DiscontinuityGeneration - expectedDiscontinuity + : 0; + + DbgPrint("AUDIO final selftest: endpoint=%llu submitted=%llu consumed=%llu presented=%llu\n", + (unsigned long long)endpointId, + (unsigned long long)streamInfo->FramesSubmitted, + (unsigned long long)streamInfo->FramesConsumed, + (unsigned long long)streamInfo->FramesPresented); + DbgPrint("AUDIO final selftest: periods mixed=%llu completed=%llu interrupts=%llu\n", + (unsigned long long)mixedPeriods, + (unsigned long long)completedPeriods, + (unsigned long long)interruptCount); + DbgPrint("AUDIO final selftest: underruns endpoint=%llu stream=%llu discontinuities endpoint=%llu stream=%llu\n", + (unsigned long long)endpointUnderruns, + (unsigned long long)streamInfo->Underruns, + (unsigned long long)endpointDiscontinuities, + (unsigned long long)streamDiscontinuities); + DbgPrint("AUDIO final selftest: position start=%llu end=%llu delta=%llu\n", + (unsigned long long)endpointBefore->HardwarePositionFrames, + (unsigned long long)endpointAfter->HardwarePositionFrames, + (unsigned long long)positionFrames); +} + +// Plays the final-path sine test through the selected hardware endpoint and mixer. +void KiRunAudioFinalPathSelfTest(void) { + const uint64_t totalFrames = 3ull * KE_AUDIO_OUTPUT_SAMPLE_RATE; + const uint32_t chunkFrames = 1024u; + const char* failurePhase = (const char*)0; + KSTATUS failureStatus = NTLSTATUS_SUCCESS; + uint64_t failureValue = 0; + uint64_t endpointId = KeAudioGetDefaultEndpointId(); + KE_AUDIO_ENDPOINT_INFO endpointBefore; + KE_AUDIO_ENDPOINT_INFO endpointAfter; + KE_AUDIO_STREAM_INFO startInfo; + KE_AUDIO_STREAM_INFO streamAfter; + KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)0; + int16_t* chunk = (int16_t*)0; + uint64_t expectedDiscontinuity = 0; + int haveEndpointBefore = 0; + int haveEndpointAfter = 0; + int haveStreamAfter = 0; + + memset(&endpointBefore, 0, sizeof(endpointBefore)); + memset(&endpointAfter, 0, sizeof(endpointAfter)); + memset(&startInfo, 0, sizeof(startInfo)); + memset(&streamAfter, 0, sizeof(streamAfter)); + + if (!endpointId) { + DbgPrint("AUDIO final selftest: SKIP no default endpoint\n"); + return; + } + + KSTATUS status = KiAudioSelfTestQueryEndpoint(endpointId, &endpointBefore); + if (KFAILED(status) || !endpointBefore.Available || + endpointBefore.State != KeAudioEndpointReady) { + failurePhase = "endpoint-ready"; + failureStatus = KFAILED(status) ? status : NTLSTATUS_DEVICE_ERROR; + goto cleanup; + } + haveEndpointBefore = 1; + if (endpointBefore.HardwareFormat.SampleRate != KE_AUDIO_OUTPUT_SAMPLE_RATE || + endpointBefore.HardwareFormat.ChannelCount != KE_AUDIO_OUTPUT_CHANNELS || + endpointBefore.HardwareFormat.SampleFormat != IoAudioSampleS16) { + failurePhase = "endpoint-format"; + failureStatus = NTLSTATUS_NOT_SUPPORTED; + failureValue = endpointBefore.HardwareFormat.SampleFormat; + goto cleanup; + } + + DbgPrint("AUDIO final selftest: begin endpoint=%llu kind=%u format=48000/S16/stereo\n", + (unsigned long long)endpointId, + (unsigned)endpointBefore.Kind); + + IO_AUDIO_FORMAT format = { + .SampleRate = KE_AUDIO_OUTPUT_SAMPLE_RATE, + .ChannelCount = KE_AUDIO_OUTPUT_CHANNELS, + .SampleFormat = IoAudioSampleS16, + }; + status = KeAudioCreateStream(endpointId, &format, &stream); + if (KFAILED(status)) { + failurePhase = "stream-create"; + failureStatus = status; + goto cleanup; + } + + chunk = (int16_t*)MmAllocatePoolTag( + (size_t)chunkFrames * KE_AUDIO_OUTPUT_BYTES_PER_FRAME, + KE_AUDIO_SELFTEST_TAG); + if (!chunk) { + failurePhase = "tone-allocation"; + failureStatus = NTLSTATUS_NO_MEMORY; + goto cleanup; + } + + double sine = 0.0; + double cosine = 1.0; + uint64_t submitted = 0; + while (submitted < KE_AUDIO_STREAM_RING_FRAMES) { + uint32_t count = (uint32_t)(KE_AUDIO_STREAM_RING_FRAMES - submitted); + if (count > chunkFrames) + count = chunkFrames; + KiAudioSelfTestGenerateTone(chunk, count, submitted, &sine, &cosine); + uint32_t written = 0; + status = KeAudioWriteStream(stream, chunk, count, &written); + if (KFAILED(status) || written != count) { + failurePhase = "initial-prefill"; + failureStatus = KFAILED(status) ? status : NTLSTATUS_FAILURE; + failureValue = written; + goto cleanup; + } + submitted += written; + } + + status = KeAudioStartStream(stream); + if (KFAILED(status)) { + failurePhase = "stream-start"; + failureStatus = status; + goto cleanup; + } + + status = KeAudioQueryStream(stream, &startInfo); + if (KFAILED(status)) { + failurePhase = "start-query"; + failureStatus = status; + goto cleanup; + } + expectedDiscontinuity = startInfo.DiscontinuityGeneration; + + while (submitted < totalFrames) { + uint64_t remaining = totalFrames - submitted; + uint32_t count = remaining < chunkFrames ? (uint32_t)remaining : chunkFrames; + KiAudioSelfTestGenerateTone(chunk, count, submitted, &sine, &cosine); + status = KiAudioSelfTestWriteFrames(stream, + chunk, + count, + expectedDiscontinuity); + if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) { + failurePhase = "stream-write"; + failureStatus = status; + failureValue = submitted; + goto cleanup; + } + submitted += count; + } + + status = KeAudioDrainStream(stream); + if (KFAILED(status)) { + failurePhase = "stream-drain"; + failureStatus = status; + goto cleanup; + } + KE_AUDIO_STREAM_WAIT_CONDITION drainCondition = { + .Mask = KE_AUDIO_WAIT_DRAINED | KE_AUDIO_WAIT_FAILED | KE_AUDIO_WAIT_DISCONTINUITY, + .DrainedGeneration = startInfo.DrainedGeneration, + .DiscontinuityGeneration = expectedDiscontinuity, + }; + status = KeAudioArmStreamWait(stream, &drainCondition); + if (KFAILED(status)) { + failurePhase = "drain-arm"; + failureStatus = status; + goto cleanup; + } + status = KiAudioSelfTestWait(stream); + if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) { + failurePhase = "drain-wait"; + failureStatus = status; + goto cleanup; + } + + status = KeAudioQueryStream(stream, &streamAfter); + if (KFAILED(status)) { + failurePhase = "final-stream-query"; + failureStatus = status; + goto cleanup; + } + haveStreamAfter = 1; + if (streamAfter.State != KeAudioStreamStopped || + streamAfter.DrainedGeneration != startInfo.DrainedGeneration + 1u || + streamAfter.BufferedFrames != 0 || streamAfter.FramesSubmitted != totalFrames || + streamAfter.FramesConsumed != totalFrames || streamAfter.FramesPresented != totalFrames || + streamAfter.FramesDiscarded != 0 || streamAfter.Underruns != 0 || + streamAfter.DiscontinuityGeneration != expectedDiscontinuity) { + failurePhase = "final-stream-counters"; + failureStatus = NTLSTATUS_DEVICE_ERROR; + failureValue = streamAfter.FramesPresented; + goto cleanup; + } + + status = KiAudioSelfTestQueryEndpoint(endpointId, &endpointAfter); + if (KFAILED(status)) { + failurePhase = "final-endpoint-query"; + failureStatus = status; + goto cleanup; + } + haveEndpointAfter = 1; + uint64_t mixedPeriods = endpointAfter.PeriodsMixed - endpointBefore.PeriodsMixed; + uint64_t completedPeriods = endpointAfter.PeriodsCompleted - endpointBefore.PeriodsCompleted; + uint64_t interruptCount = endpointAfter.InterruptCount - endpointBefore.InterruptCount; + uint64_t endpointUnderruns = endpointAfter.Underruns - endpointBefore.Underruns; + uint64_t endpointDiscontinuities = endpointAfter.DiscontinuityGeneration - + endpointBefore.DiscontinuityGeneration; + uint64_t positionFrames = endpointAfter.HardwarePositionFrames - + endpointBefore.HardwarePositionFrames; + if (!mixedPeriods || !completedPeriods || !interruptCount || + endpointUnderruns || endpointDiscontinuities || + positionFrames + endpointAfter.FramesPerPeriod < totalFrames) { + failurePhase = "final-endpoint-counters"; + failureStatus = NTLSTATUS_DEVICE_ERROR; + failureValue = positionFrames; + goto cleanup; + } + +cleanup: + if (stream && !haveStreamAfter && KSUCCESS(KeAudioQueryStream(stream, &streamAfter))) + haveStreamAfter = 1; + if (haveEndpointBefore && !haveEndpointAfter && + KSUCCESS(KiAudioSelfTestQueryEndpoint(endpointId, &endpointAfter))) + haveEndpointAfter = 1; + if (haveEndpointBefore && haveEndpointAfter && haveStreamAfter) + KiAudioSelfTestPrintCounters(endpointId, + &endpointBefore, + &endpointAfter, + &streamAfter, + expectedDiscontinuity); + + if (stream) { + KE_AUDIO_STREAM_INFO cleanupInfo; + if (KSUCCESS(KeAudioQueryStream(stream, &cleanupInfo)) && + cleanupInfo.State != KeAudioStreamStopped && + cleanupInfo.State != KeAudioStreamFailed && + cleanupInfo.State != KeAudioStreamClosed) + (void)KeAudioStopStream(stream); + } + if (chunk) + MmFreePool(chunk); + if (stream) + ObDereferenceObject(&stream->Header); + + if (failurePhase) { + DbgPrint("AUDIO final selftest: FAIL phase=%s status=0x%x value=%llu\n", + failurePhase, + failureStatus, + (unsigned long long)failureValue); + return; + } + DbgPrint("AUDIO final selftest: PASS\n"); +} diff --git a/code/kernel/ke_cpu.c b/code/kernel/ke_cpu.c index 36e58e9..441d2b1 100644 --- a/code/kernel/ke_cpu.c +++ b/code/kernel/ke_cpu.c @@ -4,19 +4,148 @@ #include #include #include +#include +#include #define BOOTSTRAP_CPU_STACK_SIZE 0x4000u #define IA32_GS_BASE 0xC0000101u #define IA32_KERNEL_GS_BASE 0xC0000102u +#define IA32_MTRRCAP 0x000000FEu +#define IA32_MTRR_PHYSBASE0 0x00000200u +#define IA32_MTRR_PHYSMASK0 0x00000201u +#define IA32_PAT 0x00000277u +#define IA32_MTRR_DEF_TYPE 0x000002FFu +#define CPUID_FEATURE_MTRR (1u << 12) +#define CPUID_FEATURE_PAT (1u << 16) +#define MTRR_CAP_VARIABLE_COUNT 0x000000FFull +#define MTRR_MASK_VALID (1ull << 11) +#define MTRR_DEF_TYPE_MASK 0x000000FFull +#define MTRR_DEF_FIXED_ENABLE (1ull << 10) +#define MTRR_DEF_ENABLE (1ull << 11) +#define MEMORY_TYPE_UC 0x00ull +#define MEMORY_TYPE_WC 0x01ull +#define MEMORY_TYPE_WB 0x06ull +#define CR0_CACHE_DISABLE (1ull << 30) +#define CR0_NOT_WRITE_THROUGH (1ull << 29) +#define CR4_PAGE_GLOBAL_ENABLE (1ull << 7) +#define CR4_PCID_ENABLE (1ull << 17) +#define PAT_ENTRY(type, index) ((uint64_t)(type) << ((index) * 8u)) +#define CANONICAL_PAT (PAT_ENTRY(MEMORY_TYPE_WB, 0) | \ + PAT_ENTRY(MEMORY_TYPE_WC, 1) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 2) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 3) | \ + PAT_ENTRY(MEMORY_TYPE_WB, 4) | \ + PAT_ENTRY(MEMORY_TYPE_WC, 5) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 6) | \ + PAT_ENTRY(MEMORY_TYPE_UC, 7)) KCPU gCpus[KE_MAX_CPUS]; uint32_t gCpuCount = 0; static uint8_t gBootstrapCpuStack[BOOTSTRAP_CPU_STACK_SIZE]; static volatile long gKernelGsReady = 0; +static volatile long gWriteCombiningAvailable = 0; extern KCPU* KeReadCurrentCpuFromGs(void); +// Returns whether this CPU exposes the memory-type mechanisms required by the kernel policy. +static bool KiMemoryTypeFeaturesAvailable(void) { + int regs[4]; + __cpuid(regs, 1); + uint32_t features = (uint32_t)regs[3]; + return (features & (CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT)) == + (CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT); +} + +// Flushes all translations before memory-type state changes and leaves global pages disabled. +static uintptr_t KiDisableGlobalTranslations(void) { + uintptr_t cr4 = __readcr4(); + + // nt-lite does not enable PCID. Refuse inherited PCID state rather than + // pretending that a CR3 reload flushed every tagged translation. + if (cr4 & CR4_PCID_ENABLE) + HalStopProcessor(); + + if (cr4 & CR4_PAGE_GLOBAL_ENABLE) + __writecr4(cr4 & ~CR4_PAGE_GLOBAL_ENABLE); + else + __writecr3(__readcr3()); + + return cr4; +} + +// Returns whether this CPU accepted the complete canonical PAT and MTRR policy. +static bool KiMemoryTypePolicyInstalled(uint32_t variableCount) { + if (__readmsr(IA32_PAT) != CANONICAL_PAT) + return false; + + uint64_t expectedDefault = MTRR_DEF_ENABLE | MEMORY_TYPE_WB; + uint64_t defaultType = __readmsr(IA32_MTRR_DEF_TYPE); + uint64_t relevantDefaultBits = MTRR_DEF_TYPE_MASK | MTRR_DEF_FIXED_ENABLE | MTRR_DEF_ENABLE; + if ((defaultType & relevantDefaultBits) != expectedDefault) + return false; + + for (uint32_t i = 0; i < variableCount; i++) { + uint64_t base = __readmsr(IA32_MTRR_PHYSBASE0 + i * 2u); + uint64_t mask = __readmsr(IA32_MTRR_PHYSMASK0 + i * 2u); + if (base != 0 || (mask & MTRR_MASK_VALID) != 0) + return false; + } + + return true; +} + +// Programs the kernel-owned PAT policy over a uniform write-back MTRR baseline. +void KeInitializeMemoryTypes(void) { + if (!KiMemoryTypeFeaturesAvailable()) + HalStopProcessor(); + + uint32_t variableCount = (uint32_t)(__readmsr(IA32_MTRRCAP) & MTRR_CAP_VARIABLE_COUNT); + if (KiMemoryTypePolicyInstalled(variableCount)) { + _InterlockedExchange(&gWriteCombiningAvailable, 1); + return; + } + + uintptr_t interruptFlags = HalDisableInterrupts(); + uintptr_t cr0 = __readcr0(); + + // Every CPU runs this before entering normal kernel execution. The BSP + // installs the policy while it is alone, and each AP installs it before + // publishing its started state. + __writecr0((cr0 | CR0_CACHE_DISABLE) & ~CR0_NOT_WRITE_THROUGH); + __wbinvd(); + uintptr_t cr4 = KiDisableGlobalTranslations(); + + // MTRRs provide only a uniform WB baseline. PAT is the sole owner of the + // page-granular WB, WC, and UC policy selected by kernel mappings. + __writemsr(IA32_MTRR_DEF_TYPE, 0); + for (uint32_t i = 0; i < variableCount; i++) { + __writemsr(IA32_MTRR_PHYSBASE0 + i * 2u, 0); + __writemsr(IA32_MTRR_PHYSMASK0 + i * 2u, 0); + } + + __writemsr(IA32_PAT, CANONICAL_PAT); + __writemsr(IA32_MTRR_DEF_TYPE, MTRR_DEF_ENABLE | MEMORY_TYPE_WB); + + __wbinvd(); + __writecr3(__readcr3()); + __writecr0(cr0); + if (cr4 & CR4_PAGE_GLOBAL_ENABLE) + __writecr4(cr4); + + bool installed = KiMemoryTypePolicyInstalled(variableCount); + HalRestoreInterrupts(interruptFlags); + if (!installed) + HalStopProcessor(); + + _InterlockedExchange(&gWriteCombiningAvailable, 1); +} + +// Reports whether PAT write-combining is available on the current platform. +bool KeWriteCombiningAvailable(void) { + return _InterlockedCompareExchange(&gWriteCombiningAvailable, 0, 0) != 0; +} + // Loads kernel Gs Base. void KeLoadKernelGsBase(KCPU* cpu) { if (!cpu) @@ -69,6 +198,7 @@ void KeInitializeBootstrapCpu(void) { cpu->Tss.RSP0 = cpu->KernelStackTop; gCpuCount = 1; + KeInitializeMemoryTypes(); KeLoadKernelGsBase(cpu); KeSetUserGsBase(0); } diff --git a/code/kernel/ke_debug.c b/code/kernel/ke_debug.c index 239ebcc..8359cc7 100644 --- a/code/kernel/ke_debug.c +++ b/code/kernel/ke_debug.c @@ -220,7 +220,29 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) { line, length); } -// Flushes debug Log To File. +// Writes one debug-log segment and rejects successful zero-progress writes. +static KSTATUS KiDebugLogWriteAll(FILE_OBJECT* file, + const char* data, + size_t length, + size_t* totalWritten) { + if (!file || (!data && length) || !totalWritten) + return NTLSTATUS_INVALID_PARAMETER; + + while (length != 0) { + size_t written = 0; + KSTATUS st = IoWriteFile(file, data, length, &written); + if (!KSUCCESS(st)) + return st; + if (written == 0) + return NTLSTATUS_DEVICE_ERROR; + data += written; + length -= written; + *totalWritten += written; + } + return NTLSTATUS_SUCCESS; +} + +// Persists the in-memory debug ring and flushes it through the storage stack. __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0) return NTLSTATUS_INVALID_PARAMETER; @@ -237,11 +259,6 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0; KeReleaseSpinlock(&gDebugPrintLock, flags); - if (used == 0) { - _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0); - return NTLSTATUS_SUCCESS; - } - (void)IoDeleteFile(path); FILE_OBJECT* file = (FILE_OBJECT*)0; @@ -251,23 +268,32 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) { return st; } - size_t written = 0; - size_t totalWritten = 0; - size_t first = gDebugLogSize - start; - if (first > used) first = used; - st = IoWriteFile(file, gDebugLogBuffer + start, first, &written); - totalWritten += written; + // Shutdown logging must not depend on deferred file-cache writeback. FAT + // still updates the directory entry and cluster chain through its normal + // write path, while IoFlushFile pushes metadata and the device cache. + file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND; - if (KSUCCESS(st) && used > first) { - written = 0; - st = IoWriteFile(file, gDebugLogBuffer, used - first, &written); - totalWritten += written; + size_t totalWritten = 0; + if (used != 0) { + size_t first = gDebugLogSize - start; + if (first > used) + first = used; + + st = KiDebugLogWriteAll(file, gDebugLogBuffer + start, first, &totalWritten); + if (KSUCCESS(st) && used > first) { + st = KiDebugLogWriteAll(file, gDebugLogBuffer, + used - first, &totalWritten); + } } - IoCloseFile(file); + if (KSUCCESS(st) && totalWritten != used) + st = NTLSTATUS_DEVICE_ERROR; + if (KSUCCESS(st)) + st = IoFlushFile(file); + IoCloseFile(file); _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0); - return (KSUCCESS(st) && totalWritten == used) ? NTLSTATUS_SUCCESS : st; + return st; } // Attempts to persist the debug log during a panic when the filesystem is usable. @@ -863,6 +889,18 @@ __declspec(dllexport) KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) { if (!KeFaultFrameFromUser(frame)) KiPanicKernelFault(proc, thread, frame); + DbgPrint("USERFAULT: pid=%u tid=%u vector=%llu rip=0x%016llX rsp=0x%016llX cr2=0x%016llX error=0x%016llX rcx=0x%016llX rbx=0x%016llX rax=0x%016llX\n", + proc ? proc->ProcessId : 0, + thread ? thread->ThreadId : 0, + frame->Vector, + (uint64_t)frame->Rip, + (uint64_t)frame->Rsp, + (uint64_t)frame->Cr2, + (uint64_t)frame->ErrorCode, + (uint64_t)frame->Rcx, + (uint64_t)frame->Rbx, + (uint64_t)frame->Rax); + KSTATUS delivered = KeDeliverUserException(frame); if (KSUCCESS(delivered)) return NTLSTATUS_SUCCESS; diff --git a/code/kernel/ke_display.c b/code/kernel/ke_display.c index d8ddf0d..800b87e 100644 --- a/code/kernel/ke_display.c +++ b/code/kernel/ke_display.c @@ -19,6 +19,7 @@ #include #include #include +#include #define DISPLAY_DEFAULT_FG 0x00CCCCCCu #define DISPLAY_DEFAULT_BG 0x00012456u @@ -44,6 +45,15 @@ typedef struct DISPLAY_OUTPUT_GLYPH { uint32_t Background; } DISPLAY_OUTPUT_GLYPH; +typedef struct DISPLAY_OUTPUT_DRAW { + char Character; + uint32_t Foreground; + uint32_t Background; + int Column; + int Row; + size_t ScrollCount; +} DISPLAY_OUTPUT_DRAW; + typedef struct BOOTFB_CONTEXT { const HAL_API* Hal; uint32_t* Framebuffer; @@ -59,6 +69,7 @@ static DISPLAY_DEVICE* gDisplayList; static DISPLAY_DEVICE* gPrimaryDisplay; static SPINLOCK gDisplayLock = SPINLOCK_INIT; static MUTEX* gDisplayProviderMutex; +static MUTEX* gDisplayOutputMutex; static KWORK_ITEM gDisplayCursorWorkItem; static volatile long gDisplayCursorWorkInitialized; static bool gDisplayTransitionInProgress; @@ -104,8 +115,10 @@ static KWORK_ITEM gDisplayOutputWorkItem; static volatile long gDisplayOutputWorkInitialized; static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg); +static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count); static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode); -static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels); +static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); +static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color); static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h); static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride); @@ -119,6 +132,7 @@ static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) static const DISPLAY_DEVICE_OPS gBootFbOps = { .GetMode = BootFbGetMode, .Blit = BootFbBlit, + .PresentPitched = BootFbPresentPitched, .FillRect = BootFbFillRect, .CopyRect = BootFbCopyRect, .ReadRect = BootFbReadRect, @@ -192,10 +206,15 @@ static KSTATUS KiDisplayAcquireProviderMutex(bool* acquired) { if (KeGetDebugFlags() & KE_DEBUG_DISPLAY_ASSERTS) KeAssertIrql(PASSIVE_LEVEL); - KSTATUS s = KeAcquireMutex(gDisplayProviderMutex); - if (KSUCCESS(s) && acquired) + KSTATUS s = KeAcquireMutex(gDisplayProviderMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s)) + return s; + + if (acquired) *acquired = true; - return s; + + // Provider operations still own the mutex after an abandoned acquisition. + return NTLSTATUS_SUCCESS; } static void KiDisplayReleaseProviderMutex(bool acquired) { @@ -449,6 +468,34 @@ static bool KiDisplayConsoleGridFromMode(const DISPLAY_MODE* mode, int* outCols, return true; } +// Returns whether two dirty rectangles overlap or share an edge. +static bool KiDisplayDirtyRectsMergeable(const DISPLAY_RECT* a, const DISPLAY_RECT* b) { + if (!a || !b) + return false; + + int64_t aRight = (int64_t)a->X + a->W; + int64_t aBottom = (int64_t)a->Y + a->H; + int64_t bRight = (int64_t)b->X + b->W; + int64_t bBottom = (int64_t)b->Y + b->H; + return (int64_t)a->X <= bRight && (int64_t)b->X <= aRight && + (int64_t)a->Y <= bBottom && (int64_t)b->Y <= aBottom; +} + +// Unions two clipped dirty rectangles. +static DISPLAY_RECT KiDisplayUnionDirtyRects(const DISPLAY_RECT* a, const DISPLAY_RECT* b) { + int64_t left = a->X < b->X ? a->X : b->X; + int64_t top = a->Y < b->Y ? a->Y : b->Y; + int64_t aRight = (int64_t)a->X + a->W; + int64_t bRight = (int64_t)b->X + b->W; + int64_t aBottom = (int64_t)a->Y + a->H; + int64_t bBottom = (int64_t)b->Y + b->H; + int64_t right = aRight > bRight ? aRight : bRight; + int64_t bottom = aBottom > bBottom ? aBottom : bBottom; + DISPLAY_RECT result = { (int32_t)left, (int32_t)top, + (int32_t)(right - left), (int32_t)(bottom - top) }; + return result; +} + static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { if (!mode || gDirtyAll || w <= 0 || h <= 0) return; @@ -457,6 +504,19 @@ static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int if (!KiDisplayClipRect(mode, &rect)) return; + for (size_t i = 0; i < gDirtyRectCount;) { + if (!KiDisplayDirtyRectsMergeable(&rect, &gDirtyRects[i])) { + i++; + continue; + } + + rect = KiDisplayUnionDirtyRects(&rect, &gDirtyRects[i]); + gDirtyRectCount--; + if (i != gDirtyRectCount) + gDirtyRects[i] = gDirtyRects[gDirtyRectCount]; + i = 0; + } + if (rect.X == 0 && rect.Y == 0 && rect.W == (int32_t)mode->Width && rect.H == (int32_t)mode->Height) { KiDisplayDirtyAll(); @@ -478,6 +538,24 @@ static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, in KiDisplayAddDirtyForModeLocked(mode, x, y, w, h); } +static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { + if (!mode || w <= 0 || h <= 0) + return; + + DISPLAY_RECT rect = { x, y, w, h }; + if (!KiDisplayClipRect(mode, &rect)) + return; + + uint64_t bytes = (uint64_t)(uint32_t)rect.W * + (uint64_t)(uint32_t)rect.H * sizeof(uint32_t); + gDisplayStats.BlitCount++; + gDisplayStats.PresentCount++; + gDisplayStats.PresentRectCount++; + gDisplayStats.DirectPresentCount++; + gDisplayStats.ScanoutBytes += bytes; + gDisplayStats.BytesCopied += bytes; +} + static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) { gDisplayStats.FillRectCount++; gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t); @@ -660,7 +738,7 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device, uint32_t glyphPixels[FONT_W * FONT_H]; KiDisplayBuildGlyphPixels(c, fg, bg, glyphPixels); - KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels); + KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels, FONT_W * sizeof(uint32_t)); if (KSUCCESS(s)) { uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); if (device == gPrimaryDisplay) @@ -670,20 +748,26 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device, return s; } -static KSTATUS KiDisplayDrawBlitSnapshot(DISPLAY_DEVICE* device, - const DISPLAY_DEVICE_OPS* ops, - const DISPLAY_MODE* mode, - int x, - int y, - int w, - int h, - const uint32_t* pixels) { - if (!pixels || w <= 0 || h <= 0) +// Draws a pitched source rectangle and records it as one display operation. +static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device, + const DISPLAY_DEVICE_OPS* ops, + const DISPLAY_MODE* mode, + int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; - if (!device || !ops || !ops->Blit || !mode) + if (!device || !ops || !mode) return NTLSTATUS_NOT_SUPPORTED; - KSTATUS s = ops->Blit(device, x, y, w, h, pixels); + if (!ops->Blit) + return NTLSTATUS_NOT_SUPPORTED; + + KSTATUS s = ops->Blit(device, x, y, w, h, pixels, pitchBytes); + if (KSUCCESS(s)) { uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); if (device == gPrimaryDisplay) @@ -867,7 +951,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3 if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Blit) return NTLSTATUS_NOT_SUPPORTED; - KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels); + KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels, (uint32_t)w * sizeof(uint32_t)); if (KSUCCESS(s)) { gDisplayStats.BlitCount++; gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t); @@ -1043,11 +1127,18 @@ static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uin } // Drains buffered output through the active display provider. -static void KiDisplayOutputWorker(void* context) { - (void)context; +static KSTATUS KiDisplayDrainOutput(void) { DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT]; + bool mutexHeld = false; - for (;;) { + if (gDisplayOutputMutex) { + KSTATUS acquireStatus = KeAcquireMutex(gDisplayOutputMutex, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(acquireStatus)) + return acquireStatus; + mutexHeld = true; + } + + while (true) { size_t count = 0; uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock); while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) { @@ -1059,12 +1150,28 @@ static void KiDisplayOutputWorker(void* context) { if (count == 0) break; - for (size_t i = 0; i < count; i++) { - (void)KiDisplayPrintCharInternal(glyphs[i].Character, - glyphs[i].Foreground, - glyphs[i].Background); + KSTATUS drawStatus = KiDisplayPrintGlyphsInternal(glyphs, count); + if (!KSUCCESS(drawStatus)) { + if (mutexHeld) + KeReleaseMutex(gDisplayOutputMutex); + return drawStatus; } } + + if (mutexHeld) + KeReleaseMutex(gDisplayOutputMutex); + return NTLSTATUS_SUCCESS; +} + +// Drains buffered output from the passive work queue. +static void KiDisplayOutputWorker(void* context) { + (void)context; + (void)KiDisplayDrainOutput(); +} + +// Completes buffered console output before a caller overlays console pixels. +KSTATUS KeDisplayFlushConsoleOutput(void) { + return KiDisplayDrainOutput(); } // Queues the runtime display output worker when passive work is available. @@ -1157,6 +1264,8 @@ void KeDisplayInitialize(const HAL_API* hal) { _InterlockedExchange(&gDisplayClockInitialized, 1); if (!gDisplayProviderMutex) (void)KeMutexCreate(&gDisplayProviderMutex); + if (!gDisplayOutputMutex) + (void)KeMutexCreate(&gDisplayOutputMutex); KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0); _InterlockedExchange(&gDisplayCursorWorkInitialized, 1); KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0); @@ -1212,8 +1321,9 @@ void KeDisplayInitialize(const HAL_API* hal) { (void)KeDisplaySetPrimary(&gBootFbDevice); DISPLAY_MODE mode; if (KSUCCESS(KeDisplayGetMode(&mode))) { - DbgPrint("Display: bootfb primary %ux%u stride=%u\n", - mode.Width, mode.Height, mode.Stride); + DbgPrint("Display: bootfb primary %ux%u stride=%u cache=%s\n", + mode.Width, mode.Height, mode.Stride, + KeWriteCombiningAvailable() ? "WC" : "WT"); } // The registration messages above were routed through the HAL boot @@ -1277,7 +1387,7 @@ DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) { // Signals every display clock once for a provider-owned frame generation. static void KiDisplaySignalFrameGeneration(uint64_t generation) { - for (;;) { + while (true) { EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH]; uint32_t count = 0; @@ -1340,7 +1450,7 @@ DISPLAY_API void KeDisplayTick(void) { return; uint64_t now = KiDisplayClockReadSchedulerTick(); - for (;;) { + while (true) { EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH]; uint32_t count = 0; @@ -1424,7 +1534,8 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE s = oldDevice->Ops->ReadRect(oldDevice, 0, (int)y, (int)newMode.Width, 1, row, newMode.Width); if (!KSUCCESS(s)) break; - s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row); + s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row, + newMode.Width * sizeof(uint32_t)); if (!KSUCCESS(s)) break; } @@ -1610,6 +1721,8 @@ DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) { for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) count++; info.DeviceCount = count; + if (KeWriteCombiningAvailable()) + info.Flags |= DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE; info.ManagerStats = gDisplayStats; DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0; @@ -1660,7 +1773,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); if (KSUCCESS(s)) { (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); - s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y, w, h, pixels); + s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, + (uint32_t)w * sizeof(uint32_t)); s = KiDisplayFinishProviderDraw(device, ops, caps, s); } @@ -1668,8 +1782,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi return s; } -DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch) { - if (!source || !scratch || w <= 0 || h <= 0) +DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) { + if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; bool providerHeld = false; @@ -1684,14 +1798,7 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); if (KSUCCESS(s)) { (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); - for (int row = 0; row < h; ++row) { - s = source(context, row, scratch, (size_t)w); - if (!KSUCCESS(s)) - break; - s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y + row, w, 1, scratch); - if (!KSUCCESS(s)) - break; - } + s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, pitchBytes); s = KiDisplayFinishProviderDraw(device, ops, caps, s); } @@ -1699,6 +1806,47 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW return s; } +DISPLAY_API KSTATUS KeDisplayPresentPitched(int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t)) + return NTLSTATUS_INVALID_PARAMETER; + + bool providerHeld = false; + KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); + if (!KSUCCESS(s)) + return s; + + DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0; + const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0; + uint32_t caps = 0; + DISPLAY_MODE mode; + s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode); + if (KSUCCESS(s)) { + (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); + if (ops->PresentPitched) { + s = ops->PresentPitched(device, x, y, w, h, pixels, pitchBytes); + if (KSUCCESS(s)) { + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (device == gPrimaryDisplay) + KiDisplayRecordDirectPresentLocked(&mode, x, y, w, h); + KeReleaseSpinlock(&gDisplayLock, flags); + } + KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps); + } else { + s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, + x, y, w, h, pixels, pitchBytes); + s = KiDisplayFinishProviderDraw(device, ops, caps, s); + } + } + + KiDisplayReleaseProviderMutex(providerHeld); + return s; +} + DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) { bool providerHeld = false; KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); @@ -1835,18 +1983,28 @@ DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t return s; } -static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { +// Renders one or more console glyphs as one provider transaction and one present. +static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count) { + if (!glyphs && count != 0) + return NTLSTATUS_INVALID_PARAMETER; + if (count == 0) + return NTLSTATUS_SUCCESS; + if (count > DISPLAY_OUTPUT_DRAIN_COUNT) + return NTLSTATUS_INVALID_PARAMETER; + if (KiDisplayOutputSuspended()) { - if (!KeCanWait()) - return KiDisplayFallbackPrintChar(c, fg, bg); + if (!KeCanWait()) { + for (size_t i = 0; i < count; i++) + (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background); + } return NTLSTATUS_SUCCESS; } - // Once the scheduler/work-queue era begins, display provider callbacks are - // PASSIVE_LEVEL-only. The HAL output endpoint buffers normal text so this - // fallback is reserved for explicit display calls made at elevated IRQL. - if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) - return KiDisplayFallbackPrintChar(c, fg, bg); + if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) { + for (size_t i = 0; i < count; i++) + (void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background); + return NTLSTATUS_SUCCESS; + } bool providerHeld = false; KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld); @@ -1861,8 +2019,10 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { if (!KSUCCESS(s)) { KiDisplayReleaseProviderMutex(providerHeld); const HAL_API* hal = gDisplayHal; - if (hal && hal->Screen.PrintChar) - hal->Screen.PrintChar(c, fg, bg); + if (hal && hal->Screen.PrintChar) { + for (size_t i = 0; i < count; i++) + hal->Screen.PrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background); + } return s; } @@ -1875,71 +2035,94 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { (void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps); - bool drawChar = false; - bool needScroll = false; - int drawCol = 0; - int drawRow = 0; + DISPLAY_OUTPUT_DRAW draws[DISPLAY_OUTPUT_DRAIN_COUNT]; + size_t drawCount = 0; + size_t scrollCount = 0; - uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); - if (c == '\n') { - gConsoleX = 0; - gConsoleY++; - } else if (c == '\r') { - gConsoleX = 0; - } else if (c == '\b') { - if (gConsoleX > 0) { - gConsoleX--; - } else if (gConsoleY > 0) { - gConsoleY--; - gConsoleX = cols - 1; - } - } else { - drawChar = true; - drawCol = gConsoleX; - drawRow = gConsoleY; - if (++gConsoleX >= cols) { + for (size_t glyphIndex = 0; glyphIndex < count; glyphIndex++) { + char c = glyphs[glyphIndex].Character; + int drawCol = 0; + int drawRow = 0; + bool drawChar = false; + bool needScroll = false; + + uintptr_t flags = KeAcquireSpinlock(&gDisplayLock); + if (c == '\n') { gConsoleX = 0; gConsoleY++; + } else if (c == '\r') { + gConsoleX = 0; + } else if (c == '\b') { + if (gConsoleX > 0) { + gConsoleX--; + } else if (gConsoleY > 0) { + gConsoleY--; + gConsoleX = cols - 1; + } + } else { + drawChar = true; + drawCol = gConsoleX; + drawRow = gConsoleY; + if (++gConsoleX >= cols) { + gConsoleX = 0; + gConsoleY++; + } + + if (drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) { + size_t cell = KiDisplayConsoleIndex(drawCol, drawRow); + gConsoleCells[cell] = c; + gConsoleAttrs[cell] = (uint8_t)gConsoleAttribute; + } + } + + if (gConsoleY >= rows) { + KiDisplayConsoleScrollCellsLocked(cols, rows, 1); + gConsoleY = rows - 1; + if (gConsoleX < 0) gConsoleX = 0; + if (gConsoleX >= cols) gConsoleX = cols - 1; + needScroll = true; + } + KeReleaseSpinlock(&gDisplayLock, flags); + + if (drawChar) { + DISPLAY_OUTPUT_DRAW* draw = &draws[drawCount++]; + draw->Character = c; + draw->Foreground = glyphs[glyphIndex].Foreground; + draw->Background = glyphs[glyphIndex].Background; + draw->Column = drawCol; + draw->Row = drawRow; + draw->ScrollCount = scrollCount; + } + if (needScroll) + scrollCount++; + } + + if (scrollCount != 0) { + uint64_t scrollPixels64 = (uint64_t)scrollCount * FONT_H; + int scrollPixels = scrollPixels64 >= mode.Height ? (int)mode.Height : (int)scrollPixels64; + if ((uint32_t)scrollPixels >= mode.Height) { + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0, + (int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG); + } else { + s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, scrollPixels, + (int)mode.Width, (int)(mode.Height - (uint32_t)scrollPixels)); + if (KSUCCESS(s)) { + s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, + (int)(mode.Height - (uint32_t)scrollPixels), + (int)mode.Width, scrollPixels, DISPLAY_DEFAULT_BG); + } } } - if (gConsoleY >= rows) - needScroll = true; - KeReleaseSpinlock(&gDisplayLock, flags); - - if (drawChar) { - s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, drawCol * FONT_W, drawRow * FONT_H, fg, bg); - if (KSUCCESS(s)) { - flags = KeAcquireSpinlock(&gDisplayLock); - if (device == gPrimaryDisplay && drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) { - size_t i = KiDisplayConsoleIndex(drawCol, drawRow); - gConsoleCells[i] = c; - gConsoleAttrs[i] = (uint8_t)gConsoleAttribute; - } - KeReleaseSpinlock(&gDisplayLock, flags); - } - } else { - s = NTLSTATUS_SUCCESS; - } - - if (KSUCCESS(s) && needScroll) { - s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, FONT_H, - (int)mode.Width, (int)(mode.Height - FONT_H)); - if (KSUCCESS(s)) { - s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, - (int)(mode.Height - FONT_H), - (int)mode.Width, FONT_H, DISPLAY_DEFAULT_BG); - } - if (KSUCCESS(s)) { - flags = KeAcquireSpinlock(&gDisplayLock); - if (device == gPrimaryDisplay) { - KiDisplayConsoleScrollCellsLocked(cols, rows, 1); - gConsoleY = rows - 1; - if (gConsoleX < 0) gConsoleX = 0; - if (gConsoleX >= cols) gConsoleX = cols - 1; - } - KeReleaseSpinlock(&gDisplayLock, flags); - } + for (size_t i = 0; i < drawCount && KSUCCESS(s); i++) { + DISPLAY_OUTPUT_DRAW* draw = &draws[i]; + size_t laterScrolls = scrollCount - draw->ScrollCount; + int finalRow = draw->Row - (int)laterScrolls; + if (finalRow < 0 || finalRow >= rows) + continue; + s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, draw->Character, + draw->Column * FONT_W, finalRow * FONT_H, + draw->Foreground, draw->Background); } s = KiDisplayFinishProviderDraw(device, ops, caps, s); @@ -1947,6 +2130,12 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { return s; } +// Renders one console character through the shared batched console path. +static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) { + DISPLAY_OUTPUT_GLYPH glyph = { c, fg, bg }; + return KiDisplayPrintGlyphsInternal(&glyph, 1); +} + DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) { return KiDisplayPrintCharInternal(c, fg, bg); } @@ -2285,10 +2474,114 @@ static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) { return NTLSTATUS_SUCCESS; } -// Implements the boot Fb Blit operation. -static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) { +// Copies one pixel row into the write-combined scanout aperture. +static void BootFbCopyToScanout(uint32_t* destination, + const uint32_t* source, + size_t pixelCount) { + if (!destination || !source || pixelCount == 0) + return; + + if (!KeWriteCombiningAvailable()) { + memcpy(destination, source, pixelCount * sizeof(uint32_t)); + return; + } + + while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) { + *destination++ = *source++; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_stream_si128((__m128i*)(destination + 0), v0); + _mm_stream_si128((__m128i*)(destination + 4), v1); + _mm_stream_si128((__m128i*)(destination + 8), v2); + _mm_stream_si128((__m128i*)(destination + 12), v3); + source += 16; + destination += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_stream_si128((__m128i*)destination, value); + source += 4; + destination += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + *destination++ = *source++; + pixelCount--; + } +} + +// Copies one source row into the retained shadow and scanout in one read pass. +static void BootFbCopyToShadowAndScanout(uint32_t* shadow, + uint32_t* scanout, + const uint32_t* source, + size_t pixelCount) { + if (!shadow || !scanout || !source || pixelCount == 0) + return; + + if (!KeWriteCombiningAvailable()) { + memcpy(shadow, source, pixelCount * sizeof(uint32_t)); + memcpy(scanout, source, pixelCount * sizeof(uint32_t)); + return; + } + + while (pixelCount != 0 && ((uintptr_t)scanout & 15u) != 0) { + uint32_t value = *source++; + *shadow++ = value; + *scanout++ = value; + pixelCount--; + } + + while (pixelCount >= 16u) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12)); + _mm_storeu_si128((__m128i*)(shadow + 0), v0); + _mm_storeu_si128((__m128i*)(shadow + 4), v1); + _mm_storeu_si128((__m128i*)(shadow + 8), v2); + _mm_storeu_si128((__m128i*)(shadow + 12), v3); + _mm_stream_si128((__m128i*)(scanout + 0), v0); + _mm_stream_si128((__m128i*)(scanout + 4), v1); + _mm_stream_si128((__m128i*)(scanout + 8), v2); + _mm_stream_si128((__m128i*)(scanout + 12), v3); + source += 16; + shadow += 16; + scanout += 16; + pixelCount -= 16; + } + + while (pixelCount >= 4u) { + __m128i value = _mm_loadu_si128((const __m128i*)source); + _mm_storeu_si128((__m128i*)shadow, value); + _mm_stream_si128((__m128i*)scanout, value); + source += 4; + shadow += 4; + scanout += 4; + pixelCount -= 4; + } + + while (pixelCount != 0) { + uint32_t value = *source++; + *shadow++ = value; + *scanout++ = value; + pixelCount--; + } +} + +// Implements the boot framebuffer pitched blit operation. +static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) { BOOTFB_CONTEXT* fb = BootFbContext(device); - if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0) + if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) return NTLSTATUS_INVALID_PARAMETER; uint32_t* dstBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer; @@ -2309,12 +2602,61 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co for (int row = 0; row < endY - startY; row++) { uint32_t* dst = dstBase + ((size_t)(startY + row) * fb->Stride) + startX; - const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX; + const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX; memcpy(dst, src, bytesPerRow); } return NTLSTATUS_SUCCESS; } +// Presents one pitched source rectangle without a shadow-to-scanout second pass. +static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, + int x, + int y, + int w, + int h, + const uint32_t* pixels, + uint32_t pitchBytes) { + BOOTFB_CONTEXT* fb = BootFbContext(device); + if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 || + pitchBytes < (uint32_t)w * sizeof(uint32_t)) + return NTLSTATUS_INVALID_PARAMETER; + + int startX = x < 0 ? 0 : x; + int startY = y < 0 ? 0 : y; + int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w; + int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h; + if (startX >= endX || startY >= endY) + return NTLSTATUS_SUCCESS; + + int srcOffsetX = startX - x; + int srcOffsetY = startY - y; + size_t pixelCount = (size_t)(endX - startX); + uint64_t bytes = (uint64_t)pixelCount * (uint64_t)(endY - startY) * sizeof(uint32_t); + + fb->Stats.BlitCount++; + fb->Stats.PresentCount++; + fb->Stats.PresentRectCount++; + fb->Stats.DirectPresentCount++; + fb->Stats.ScanoutBytes += bytes; + fb->Stats.BytesCopied += fb->Backbuffer ? bytes * 2u : bytes; + + for (int row = 0; row < endY - startY; row++) { + const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes; + const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX; + uint32_t* scanout = fb->Framebuffer + ((size_t)(startY + row) * fb->Stride) + startX; + if (fb->Backbuffer) { + uint32_t* shadow = fb->Backbuffer + ((size_t)(startY + row) * fb->Stride) + startX; + BootFbCopyToShadowAndScanout(shadow, scanout, source, pixelCount); + } else { + BootFbCopyToScanout(scanout, source, pixelCount); + } + } + + _mm_sfence(); + return NTLSTATUS_SUCCESS; +} + // Implements the boot Fb Fill Rect operation. static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) { BOOTFB_CONTEXT* fb = BootFbContext(device); @@ -2425,14 +2767,18 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, DISPLAY_RECT r = rects[i]; if (!KiDisplayClipRect(&mode, &r)) continue; - fb->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t); - size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t); + uint64_t bytes = (uint64_t)(uint32_t)r.W * + (uint64_t)(uint32_t)r.H * sizeof(uint32_t); + fb->Stats.BytesCopied += bytes; + fb->Stats.ScanoutBytes += bytes; + size_t pixelCount = (size_t)r.W; for (int row = 0; row < r.H; row++) { uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X; uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X; - memcpy(dst, src, bytesPerRow); + BootFbCopyToScanout(dst, src, pixelCount); } } + _mm_sfence(); return NTLSTATUS_SUCCESS; } @@ -2540,6 +2886,7 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) { fb->Stats.CursorHideCount++; BootFbRestoreCursor(fb); } + _mm_sfence(); return NTLSTATUS_SUCCESS; } diff --git a/code/kernel/ke_dpc.c b/code/kernel/ke_dpc.c index d0b1a41..6e81e43 100644 --- a/code/kernel/ke_dpc.c +++ b/code/kernel/ke_dpc.c @@ -81,7 +81,7 @@ void KiDrainDpcQueue(void) { if (oldIrql < DISPATCH_LEVEL) KeRaiseIrql(DISPATCH_LEVEL); - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&gDpcLock); LIST_ENTRY* node = KeListPopHeadChecked(&gDpcQueue); if (!node) { diff --git a/code/kernel/ke_exception.c b/code/kernel/ke_exception.c index 9eb59f1..c8dafb6 100644 --- a/code/kernel/ke_exception.c +++ b/code/kernel/ke_exception.c @@ -60,7 +60,7 @@ static _Noreturn void KiTerminateCurrentExceptionProcess(KSTATUS exitStatus) { if (thread) KeExitThread(thread, exitCode); - for (;;) { + while (true) { KeYield(); } } diff --git a/code/kernel/ke_input.c b/code/kernel/ke_input.c index 5220ffc..ca54e56 100644 --- a/code/kernel/ke_input.c +++ b/code/kernel/ke_input.c @@ -390,7 +390,7 @@ static KSTATUS KiInputWaitKey(KE_INPUT_CONTEXT* context, uint8_t alertable, uint if (!context->WaitEvent) return NTLSTATUS_FAILURE; - for (;;) { + while (true) { EVENT* event = context->WaitEvent; uint32_t selected = 0; KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected); diff --git a/code/kernel/ke_irp.c b/code/kernel/ke_irp.c index 05ad272..d269f33 100644 --- a/code/kernel/ke_irp.c +++ b/code/kernel/ke_irp.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -38,12 +39,13 @@ __declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) { return CONTAINING_RECORD(node, IRP, Link); } -__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, - KSTATUS status, - IRP_QUEUE_CANCEL_CALLBACK callback, - void* context) { +// Cancels every queued IRP and invokes an optional ownership callback. +__declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q, + KSTATUS status, + IRP_QUEUE_CANCEL_CALLBACK callback, + void* context) { if (!q) return; - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&q->Lock); LIST_ENTRY* node = ListPopHead(&q->Head); KeReleaseSpinlock(&q->Lock, flags); @@ -60,7 +62,7 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, // Implements the irp Queue Cancel All operation. __declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) { - IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0); + IrpQueueCancelAllWithCallback(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0); } // Remove locks. @@ -284,6 +286,14 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t irp->Status = status; irp->Information = information; + // Completion callbacks may recycle an embedded IRP, while a synchronous + // waiter may release stack storage as soon as its event is signaled. + // Snapshot every completion target before either ownership handoff. + KAPC* completionApc = (KAPC*)irp->CompletionApc; + EVENT* completionEvent = (EVENT*)irp->CompletionEvent; + IRP_COMPLETION_FN completionFn = irp->CompletionFn; + void* completionCtx = irp->CompletionCtx; + IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp); DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0; if (dev) { @@ -292,12 +302,18 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t ios->DeviceObject = (DEVICE_OBJECT*)0; } - if (irp->CompletionApc) - KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status); - if (irp->CompletionEvent) - KeSignalEvent((EVENT*)irp->CompletionEvent); - if (irp->CompletionFn) - irp->CompletionFn(irp, irp->CompletionCtx); + if (completionApc) + KeInsertQueueApc(completionApc, irp, (void*)(uintptr_t)status); + + // The callback owns the final callback-based handoff. Do not dereference + // the IRP after it returns because embedded requests may be freed there. + if (completionFn) + completionFn(irp, completionCtx); + + // The event is the final synchronous ownership handoff. A waiter may + // invalidate a stack IRP immediately after this signal becomes visible. + if (completionEvent) + KeSignalEvent(completionEvent); return NTLSTATUS_SUCCESS; } @@ -314,6 +330,7 @@ __declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) { // Submits irp Sync. __declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) { if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER; + if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; EVENT* ev = (EVENT*)0; KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev); @@ -321,12 +338,22 @@ __declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) { irp->CompletionEvent = ev; + // Drivers may retain this IRP, its transfer buffer, and the caller's + // kernel stack until completion. Defer process termination across that + // ownership interval rather than letting teardown free live storage. + KeEnterTerminationDeferral(); st = IoSubmitIrp(device, irp); if (st == NTLSTATUS_PENDING) { - KeWaitForEvent(ev, 0); + do { + st = KeWaitForEvent(ev, 0); + } while (st != NTLSTATUS_SUCCESS); st = irp->Status; } + // Completion events are signaled only after IoCompleteRequest has finished + // every operation that can dereference a synchronous caller's stack IRP. + irp->CompletionEvent = (EVENT*)0; ObDereferenceObject(&ev->Header); + KeLeaveTerminationDeferral(); return st; } diff --git a/code/kernel/ke_irq.c b/code/kernel/ke_irq.c index e15526f..e5722ab 100644 --- a/code/kernel/ke_irq.c +++ b/code/kernel/ke_irq.c @@ -126,7 +126,9 @@ void KiExitInterrupt(void) { // high-IRQL region. if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) { THREAD* current = cpu->CurrentThread; - if (current && !current->IsIdleThread && current->Owner && KeIsProcessTerminating(current->Owner)) { + if (current && !current->IsIdleThread && current->Owner && + KeIsProcessTerminating(current->Owner) && + !KeThreadTerminationDeferred(current)) { KeExitThread(current, current->Owner->ExitCode); while (true) ; } diff --git a/code/kernel/ke_mutex.c b/code/kernel/ke_mutex.c index fee35fd..91fdb3a 100644 --- a/code/kernel/ke_mutex.c +++ b/code/kernel/ke_mutex.c @@ -1,4 +1,4 @@ -// Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups. +// Kernel mutex primitive. This file owns exclusive ownership, abandonment, wait queues, and priority inheritance. #include #include @@ -6,76 +6,285 @@ #include #include #include +#include #include #include -// Implements the mutex Destroy operation. -static void KiMutexDestroy(OBJECT* obj) { (void)obj; } +// Reports an invalid mutex ownership transition. +static void KiMutexBug(const char* where, MUTEX* mutex, THREAD* thread, const char* why) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "mutex invariant failed: %s: mutex=0x%016llX thread=0x%016llX tid=%u owner=0x%016llX refs=%ld waitEmpty=%u ownerLink=0x%016llX/0x%016llX threadWaiting=0x%016llX threadOwnedEmpty=%u: %s\n", + where, + (uint64_t)mutex, + (uint64_t)thread, + thread ? thread->ThreadId : 0, + mutex ? (uint64_t)mutex->Owner : 0, + mutex ? mutex->Header.RefCount : 0, + mutex ? (LIST_IS_EMPTY(&mutex->WaitList) ? 1u : 0u) : 1u, + mutex ? (uint64_t)mutex->OwnerLink.Fwd : 0, + mutex ? (uint64_t)mutex->OwnerLink.Bwd : 0, + thread ? (uint64_t)thread->WaitingMutex : 0, + thread ? (LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u) : 1u, + why ? why : "?"); +} -// Implements the mutex Create operation. -__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) { +// Stabilizes an owner pointer before it crosses the mutex lock boundary. +static void KiMutexReferenceOwnerLocked(MUTEX* mutex, THREAD* owner) { + if (!owner || !ObTryReferenceObject(&owner->Header)) + KiMutexBug("KiMutexReferenceOwnerLocked", mutex, owner, + "live mutex owner could not be referenced"); +} + +// Tests whether one scheduler link is currently attached to a list. +static int KiMutexWaiterIsLinked(const THREAD* thread) { + if (!thread) return 0; + return thread->SchedulerLink.Fwd != &thread->SchedulerLink || + thread->SchedulerLink.Bwd != &thread->SchedulerLink; +} + +// Propagates one effective-rank change without carrying raw owners across locks. +static void KiMutexPropagatePriority(THREAD* thread) { + int releaseThread = 0; + + for (uint32_t depth = 1; thread && depth <= KE_PRIORITY_INHERITANCE_MAX_DEPTH; depth++) { + MUTEX* waitingMutex = (MUTEX*)0; + int changed = KiSchedulerRefreshEffectivePriority(thread, &waitingMutex); + if (!changed) { + if (releaseThread) ObDereferenceObject(&thread->Header); + return; + } + + KiSchedulerNotePriorityPropagation(depth); + if (!waitingMutex) { + if (releaseThread) ObDereferenceObject(&thread->Header); + return; + } + + uintptr_t flags = KeAcquireSpinlock(&waitingMutex->Lock); + if (thread->WaitingMutex != waitingMutex || + thread->WaitObject != &waitingMutex->Header || + thread->WaitListHead != &waitingMutex->WaitList || + !KiMutexWaiterIsLinked(thread)) { + KeReleaseSpinlock(&waitingMutex->Lock, flags); + ObDereferenceObject(&waitingMutex->Header); + if (releaseThread) ObDereferenceObject(&thread->Header); + return; + } + + THREAD* owner = waitingMutex->Owner; + if (owner) KiMutexReferenceOwnerLocked(waitingMutex, owner); + int donationChanged = KiSchedulerRefreshMutexDonation(waitingMutex); + KeReleaseSpinlock(&waitingMutex->Lock, flags); + ObDereferenceObject(&waitingMutex->Header); + + if (releaseThread) ObDereferenceObject(&thread->Header); + if (!donationChanged || !owner) { + if (owner) ObDereferenceObject(&owner->Header); + return; + } + + thread = owner; + releaseThread = 1; + } + + if (thread) + KiMutexBug("KiMutexPropagatePriority", thread->WaitingMutex, thread, + "priority inheritance chain exceeded the supported depth"); +} + +// Initializes one mutex's ownership bookkeeping. +static void KiMutexInitialize(MUTEX* mutex) { + mutex->Lock = (SPINLOCK)SPINLOCK_INIT; + mutex->Owner = (THREAD*)0; + LIST_INIT(&mutex->WaitList); + LIST_INIT(&mutex->OwnerLink); + mutex->DonationClass = KeSchedulingNormal; + mutex->DonationPriority = PriorityLow; + mutex->HasDonation = 0; + mutex->Abandoned = 0; +} + +// Destroys one unowned mutex object. +static void KiMutexDestroy(OBJECT* object) { + MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header); + if (mutex->Owner || !LIST_IS_EMPTY(&mutex->WaitList) || + mutex->OwnerLink.Fwd != &mutex->OwnerLink || + mutex->OwnerLink.Bwd != &mutex->OwnerLink) { + KiMutexBug("KiMutexDestroy", mutex, mutex->Owner, + "owned or waited mutex reached object destruction"); + } +} + +// Assigns an unowned mutex to one thread and consumes abandonment state. +static KSTATUS KiMutexAcquireUnownedLocked(MUTEX* mutex, THREAD* thread) { + KSTATUS status = mutex->Abandoned ? NTLSTATUS_ABANDONED : NTLSTATUS_SUCCESS; + mutex->Abandoned = 0; + ObReferenceObject(&mutex->Header); + KiSchedulerAcquireMutexOwnership(thread, mutex); + return status; +} + +// Creates a mutex object. +__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out) { if (!out) return NTLSTATUS_INVALID_PARAMETER; - MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX)); - if (!m) return NTLSTATUS_NO_MEMORY; - m->Header.Destroy = KiMutexDestroy; - m->Lock = (SPINLOCK)SPINLOCK_INIT; - m->Owner = (struct THREAD *)0; - LIST_INIT(&m->WaitList); - *out = m; + MUTEX* mutex = (MUTEX*)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX)); + if (!mutex) return NTLSTATUS_NO_MEMORY; + mutex->Header.Destroy = KiMutexDestroy; + KiMutexInitialize(mutex); + *out = mutex; return NTLSTATUS_SUCCESS; } -// Implements the acquire Mutex operation. -__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) { +// Acquires a mutex within the requested timeout and donates priority while blocked. +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds) { if (!mutex) return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; + if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL; + THREAD* self = KeGetCurrentThread(); + if (!self) return NTLSTATUS_FAILURE; uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); if (!mutex->Owner) { - mutex->Owner = self; + KSTATUS status = KiMutexAcquireUnownedLocked(mutex, self); KeReleaseSpinlock(&mutex->Lock, flags); - return NTLSTATUS_SUCCESS; + return status; + } + + if (mutex->Owner == self) { + KeReleaseSpinlock(&mutex->Lock, flags); + return NTLSTATUS_DEADLOCK; + } + + if (timeoutMilliseconds == 0) { + KeReleaseSpinlock(&mutex->Lock, flags); + return NTLSTATUS_TIMEOUT; + } + + THREAD* owner = mutex->Owner; + KiMutexReferenceOwnerLocked(mutex, owner); + KSTATUS waitStatus = KiSchedulerPrepareMutexWait(self, mutex, owner); + if (!KSUCCESS(waitStatus)) { + KeReleaseSpinlock(&mutex->Lock, flags); + ObDereferenceObject(&owner->Header); + return waitStatus; } - KePrepareBlockThread(self, &mutex->Header, 0); KeListInsertTailChecked(&mutex->WaitList, &self->SchedulerLink); - KeSetThreadWaitList(self, &mutex->WaitList); + if (timeoutMilliseconds != KE_WAIT_INFINITE) { + KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT); + if (!KSUCCESS(timeoutStatus)) { + KeListRemoveChecked(&self->SchedulerLink); + KiSchedulerCancelMutexWait(self, mutex); + (void)KiSchedulerRefreshMutexDonation(mutex); + KeUnblockThreadWithStatus(self, timeoutStatus); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + ObDereferenceObject(&owner->Header); + return timeoutStatus; + } + } + (void)KiSchedulerRefreshMutexDonation(mutex); KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + ObDereferenceObject(&owner->Header); return KeCommitBlockThread(self, 0); } -// Implements the try Acquire Mutex operation. -__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) { - if (!mutex) return NTLSTATUS_INVALID_PARAMETER; - THREAD* self = KeGetCurrentThread(); - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - if (!mutex->Owner) { - mutex->Owner = self; - KeReleaseSpinlock(&mutex->Lock, flags); - return NTLSTATUS_SUCCESS; - } - KeReleaseSpinlock(&mutex->Lock, flags); - return NTLSTATUS_FAILURE; -} - -// Releases mutex. +// Releases a mutex and hands ownership to its highest-ranked waiter. __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) { if (!mutex) return; - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - if (!LIST_IS_EMPTY(&mutex->WaitList)) { - LIST_ENTRY* node = KeListPopHeadChecked(&mutex->WaitList); - THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); - KeSetThreadWaitList(waiter, (LIST_ENTRY*)0); - // Direct handoff: the selected waiter owns the mutex when its wait - // returns. Do not drop Owner to NULL, or another thread can steal it. - mutex->Owner = waiter; - KeReleaseSpinlock(&mutex->Lock, flags); + THREAD* self = KeGetCurrentThread(); + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (!self || mutex->Owner != self) + KiMutexBug("KeReleaseMutex", mutex, self, "release by non-owner"); + + THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex); + if (waiter) { + KeListRemoveChecked(&waiter->SchedulerLink); + KiSchedulerTransferMutexOwnership(self, waiter, mutex); + (void)KiSchedulerRefreshMutexDonation(mutex); + (void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0); KeUnblockThreadWithStatus(waiter, NTLSTATUS_SUCCESS); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(self); return; } - mutex->Owner = (struct THREAD *)0; + KiSchedulerReleaseMutexOwnership(self, mutex); + (void)KiSchedulerRefreshMutexDonation(mutex); KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(self); + ObDereferenceObject(&mutex->Header); +} + +// Removes one blocked mutex waiter during thread-wait rundown. +int KiMutexAbortWait(MUTEX* mutex, THREAD* thread) { + if (!mutex || !thread) return 0; + + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (thread->WaitingMutex != mutex || + thread->WaitObject != &mutex->Header || + thread->WaitListHead != &mutex->WaitList || + !KiMutexWaiterIsLinked(thread)) { + KeReleaseSpinlock(&mutex->Lock, flags); + return 0; + } + + KeListRemoveChecked(&thread->SchedulerLink); + KiSchedulerCancelMutexWait(thread, mutex); + THREAD* owner = mutex->Owner; + if (owner) KiMutexReferenceOwnerLocked(mutex, owner); + (void)KiSchedulerRefreshMutexDonation(mutex); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + if (owner) ObDereferenceObject(&owner->Header); + KiMutexPropagatePriority(thread); + return 1; +} + +// Abandons one mutex owned by a terminating thread. +static void KiMutexAbandonOwned(MUTEX* mutex, THREAD* owner) { + uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); + if (mutex->Owner != owner) + KiMutexBug("KiMutexAbandonOwned", mutex, owner, "ownership list mismatch"); + + THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex); + if (waiter) { + KeListRemoveChecked(&waiter->SchedulerLink); + KiSchedulerTransferMutexOwnership(owner, waiter, mutex); + mutex->Abandoned = 0; + (void)KiSchedulerRefreshMutexDonation(mutex); + (void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0); + KeUnblockThreadWithStatus(waiter, NTLSTATUS_ABANDONED); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + return; + } + + KiSchedulerReleaseMutexOwnership(owner, mutex); + mutex->Abandoned = 1; + (void)KiSchedulerRefreshMutexDonation(mutex); + KeReleaseSpinlock(&mutex->Lock, flags); + + KiMutexPropagatePriority(owner); + ObDereferenceObject(&mutex->Header); +} + +// Releases every mutex still owned by a terminating thread. +void KiMutexRundownThread(THREAD* thread) { + if (!thread) return; + + while (true) { + MUTEX* mutex = KiSchedulerFirstOwnedMutex(thread); + if (!mutex) + return; + KiMutexAbandonOwned(mutex, thread); + } } diff --git a/code/kernel/ke_process.c b/code/kernel/ke_process.c index 127c180..5670a48 100644 --- a/code/kernel/ke_process.c +++ b/code/kernel/ke_process.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include @@ -217,6 +219,7 @@ static uint32_t KiThreadStateToSys(THREAD_STATE state) { case ThreadRunning: return SYS_THREAD_STATE_RUNNING; case ThreadWaiting: return SYS_THREAD_STATE_WAITING; case ThreadDead: return SYS_THREAD_STATE_DEAD; + case ThreadSuspended: return SYS_THREAD_STATE_SUSPENDED; default: return SYS_THREAD_STATE_DEAD; } } @@ -265,25 +268,33 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s SYS_THREAD_INFO* info = &infoOut[count]; memset(info, 0, sizeof(*info)); info->Size = sizeof(*info); + KE_THREAD_SCHEDULER_SNAPSHOT snapshot; + KiSchedulerSnapshotThread(thread, &snapshot); + info->ProcessId = process->ProcessId; info->ThreadId = thread->ThreadId; - info->State = KiThreadStateToSys(thread->State); - info->Priority = (uint32_t)thread->Priority; - info->RunningCpu = thread->RunningCpu; - info->LastCpu = thread->LastCpu; - info->QuantumRemaining = thread->QuantumRemaining; - info->QuantumTicks = thread->QuantumTicks; - info->CpuBurstTicks = thread->CpuBurstTicks; - info->LastRunTicks = thread->LastRunTicks; - info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE; - info->WaitStatus = (uint32_t)thread->WaitStatus; + info->State = KiThreadStateToSys(snapshot.State); + info->BaseSchedulingClass = (uint32_t)snapshot.BaseSchedulingClass; + info->BasePriority = snapshot.BasePriority; + info->EffectiveSchedulingClass = (uint32_t)snapshot.EffectiveSchedulingClass; + info->EffectivePriority = snapshot.EffectivePriority; + info->RunningCpu = snapshot.RunningCpu; + info->LastCpu = snapshot.LastCpu; + info->QuantumRemaining = snapshot.QuantumRemaining; + info->QuantumTicks = snapshot.QuantumTicks; + info->CpuBurstTicks = snapshot.CpuBurstTicks; + info->LastRunTicks = snapshot.LastRunTicks; + info->WaitObjectType = snapshot.WaitObjectType != 0 + ? KiWaitObjectTypeToSys((OBJECT_TYPE)snapshot.WaitObjectType) + : SYS_THREAD_WAIT_NONE; + info->WaitStatus = (uint32_t)snapshot.WaitStatus; info->KernelApcDisable = thread->KernelApcDisable; if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT; if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE; if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL; - if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED; - if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT; + if (snapshot.Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED; + if (snapshot.AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT; if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING; if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING; } @@ -296,10 +307,26 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s return NTLSTATUS_SUCCESS; } -// Releases all kernel objects still reachable from a process handle table. -// This is part of process exit, not merely object destruction: a parent may keep -// a handle to an exited process for wait/query, but the dead process must not -// keep its own files, mappings, queues, sockets, or synchronization objects alive. +// Records type-specific ownership after a handle slot becomes visible. +static void KiProcessHandleOpened(OBJECT* object) { + if (!object) + return; + + if (object->Type == ObjectTypeAudioStream) + KeAudioStreamHandleOpened((KE_AUDIO_STREAM*)object); +} + +// Releases type-specific ownership before the handle-table reference drops. +static void KiProcessReleaseHandleObject(OBJECT* object) { + if (!object) + return; + + if (object->Type == ObjectTypeAudioStream) + KeAudioStreamHandleClosed((KE_AUDIO_STREAM*)object); + ObDereferenceObject(object); +} + +// Releases every handle-table reference owned by one process. static void KiProcessReleaseAllHandles(PROCESS* proc) { if (!proc) return; @@ -317,7 +344,7 @@ static void KiProcessReleaseAllHandles(PROCESS* proc) { if (obj) { KeNamedQueueUnpublishHandle(proc, (uint32_t)i, obj); - ObDereferenceObject(obj); + KiProcessReleaseHandleObject(obj); } } } @@ -671,14 +698,16 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) { if (t == self) continue; - // A terminating process must not be held open by worker threads that - // are blocked in non-alertable waits. User-mode DLLs such as Bink can - // leave IO/submit workers parked on semaphores; after the foreground - // thread crashes, those waiters would otherwise keep ThreadCount > 0 - // and prevent the process ExitEvent from waking the shell. - if (t->State == ThreadWaiting) + // Most user waits must be interrupted immediately. A thread can defer + // termination only while synchronous kernel work still owns its stack + // or process memory; that thread is allowed to finish the ownership + // interval before the normal termination gates reap it. + int deferred = KeThreadTerminationDeferred(t); + if (t->State == ThreadSuspended && !deferred) + (void)KeResumeThread(t, (uint32_t*)0); + else if (t->State == ThreadWaiting && !deferred) KeAbortThreadWait(t, NTLSTATUS_ALERTED); - else + else if (!deferred) (void)KeAlertThread(t); if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) { @@ -727,6 +756,7 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri process->Handles[i].Object = object; process->Handles[i].Rights = rights; process->Handles[i].Flags = 0; + KiProcessHandleOpened(object); *handleOut = (uint32_t)i; KeReleaseSpinlock(&process->HandleLock, flags); return NTLSTATUS_SUCCESS; @@ -756,7 +786,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) { if (obj->Type == ObjectTypeFile) (void)IoFlushFile((FILE_OBJECT*)obj); KeNamedQueueUnpublishHandle(process, handle, obj); - ObDereferenceObject(obj); + KiProcessReleaseHandleObject(obj); return; } KeReleaseSpinlock(&process->HandleLock, flags); diff --git a/code/kernel/ke_queue.c b/code/kernel/ke_queue.c index 17a8193..b11601d 100644 --- a/code/kernel/ke_queue.c +++ b/code/kernel/ke_queue.c @@ -133,7 +133,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMi if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE) deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds); - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&queue->Lock); if (queue->Count < QUEUE_DEPTH) { uint32_t previousCount = queue->Count; @@ -165,7 +165,7 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilli if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE) deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds); - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&queue->Lock); if (queue->Count > 0) { uint32_t previousCount = queue->Count; diff --git a/code/kernel/ke_sched.c b/code/kernel/ke_sched.c index 632cba6..67afa95 100644 --- a/code/kernel/ke_sched.c +++ b/code/kernel/ke_sched.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -15,17 +16,117 @@ #include #include #include +#include #include #include -static LIST_ENTRY gRunQueue[PriorityCount]; -static LIST_ENTRY gSleepList; +static LIST_ENTRY gNormalRunQueue[PriorityCount]; +static LIST_ENTRY gRealtimeRunQueue[KE_REALTIME_PRIORITY_COUNT]; +static LIST_ENTRY gTimeoutList; +static KWORK_ITEM gTimeoutWorkItem; static SPINLOCK gSchedLock; +static void KiTimeoutWorkRoutine(void* context); + static uint32_t gAgingCounter = 0; static KE_SCHED_STATS gSchedStats; static PROCESS* gIdleProcess; +// Tests whether one scheduling rank outranks another. +static int KiRankOutranks(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority, + KE_SCHEDULING_CLASS rightClass, uint32_t rightPriority) { + if (leftClass != rightClass) + return leftClass == KeSchedulingRealtime; + return leftPriority > rightPriority; +} + +// Tests whether two scheduling ranks are equal. +static int KiRanksEqual(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority, + KE_SCHEDULING_CLASS rightClass, uint32_t rightPriority) { + return leftClass == rightClass && leftPriority == rightPriority; +} + +// Compares two threads by effective scheduling rank. +static int KiSchedulerThreadOutranks(const THREAD* left, const THREAD* right) { + if (!left) return 0; + if (!right) return 1; + return KiRankOutranks(left->EffectiveSchedulingClass, left->EffectivePriority, + right->EffectiveSchedulingClass, right->EffectivePriority); +} + +// Copies scheduler-owned thread state into one coherent snapshot. +void KiSchedulerSnapshotThread(THREAD* thread, KE_THREAD_SCHEDULER_SNAPSHOT* snapshot) { + if (!thread || !snapshot) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + snapshot->State = thread->State; + snapshot->BaseSchedulingClass = thread->BaseSchedulingClass; + snapshot->BasePriority = thread->BasePriority; + snapshot->EffectiveSchedulingClass = thread->EffectiveSchedulingClass; + snapshot->EffectivePriority = thread->EffectivePriority; + snapshot->RunningCpu = thread->RunningCpu; + snapshot->LastCpu = thread->LastCpu; + snapshot->QuantumRemaining = thread->QuantumRemaining; + snapshot->QuantumTicks = thread->QuantumTicks; + snapshot->CpuBurstTicks = thread->CpuBurstTicks; + snapshot->LastRunTicks = thread->LastRunTicks; + snapshot->WaitObjectType = thread->WaitObject ? (uint32_t)thread->WaitObject->Type : 0; + snapshot->WaitStatus = thread->WaitStatus; + snapshot->WaitingMutex = thread->WaitingMutex; + snapshot->Queued = thread->Queued; + snapshot->AlertableWait = thread->AlertableWait; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Selects the highest-ranked waiter while the caller holds the mutex lock. +THREAD* KiSchedulerSelectMutexWaiter(MUTEX* mutex) { + if (!mutex) return (THREAD*)0; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + THREAD* best = (THREAD*)0; + LIST_ENTRY* head = &mutex->WaitList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!best || KiSchedulerThreadOutranks(waiter, best)) + best = waiter; + } + KeReleaseSpinlock(&gSchedLock, flags); + return best; +} + +// Refreshes one mutex donation while the caller holds the mutex lock. +int KiSchedulerRefreshMutexDonation(MUTEX* mutex) { + if (!mutex) return 0; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + THREAD* highest = (THREAD*)0; + LIST_ENTRY* head = &mutex->WaitList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!highest || KiSchedulerThreadOutranks(waiter, highest)) + highest = waiter; + } + + KE_SCHEDULING_CLASS oldClass = mutex->DonationClass; + uint32_t oldPriority = mutex->DonationPriority; + uint8_t oldHasDonation = mutex->HasDonation; + if (highest) { + mutex->DonationClass = highest->EffectiveSchedulingClass; + mutex->DonationPriority = highest->EffectivePriority; + mutex->HasDonation = 1; + } else { + mutex->DonationClass = KeSchedulingNormal; + mutex->DonationPriority = PriorityLow; + mutex->HasDonation = 0; + } + + int changed = oldHasDonation != mutex->HasDonation || + oldClass != mutex->DonationClass || + oldPriority != mutex->DonationPriority; + KeReleaseSpinlock(&gSchedLock, flags); + return changed; +} + // Implements the default Thread Affinity Mask operation. static uint64_t KiDefaultThreadAffinityMask(void) { // Normal kernel, user, and driver-created threads are BSP-affine by @@ -36,7 +137,9 @@ static uint64_t KiDefaultThreadAffinityMask(void) { // Implements the thread Is Killed Locked operation. static int KiThreadIsKilledLocked(const THREAD* thread) { - return thread && !thread->IsIdleThread && thread->Owner && KeIsProcessTerminating(thread->Owner); + return thread && !thread->IsIdleThread && thread->Owner && + KeIsProcessTerminating(thread->Owner) && + !KeThreadTerminationDeferred(thread); } // Implements the thread Can Run On Cpu Locked operation. @@ -124,8 +227,10 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) { gSchedStats.SmtPrimaryPokes++; KiRequestRescheduleCpuLocked(cpu); requestedIdle = 1; - } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) { + } else if (cur->State == ThreadRunning && KiSchedulerThreadOutranks(thread, cur)) { gSchedStats.PriorityPreemptions++; + if (thread->EffectiveSchedulingClass == KeSchedulingRealtime) + gSchedStats.RealtimePreemptions++; KiRequestRescheduleCpuLocked(cpu); } } @@ -150,8 +255,8 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) { } // Implements the base Quantum For Priority operation. -static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) { - switch (priority) { +static uint32_t KiBaseQuantumForPriority(uint32_t priority) { + switch ((THREAD_PRIORITY)priority) { case PriorityHigh: return SCHED_HIGH_QUANTUM_TICKS; case PriorityNormal: return SCHED_NORMAL_QUANTUM_TICKS; case PriorityLow: return SCHED_LOW_QUANTUM_TICKS; @@ -164,13 +269,16 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) { if (!thread || thread == KeGetCurrentCpu()->IdleThread) return 1; - uint32_t quantum = KiBaseQuantumForPriority(thread->Priority); + if (thread->EffectiveSchedulingClass == KeSchedulingRealtime) + return SCHED_REALTIME_QUANTUM_TICKS; + + uint32_t quantum = KiBaseQuantumForPriority(thread->EffectivePriority); // Low-priority CPU-bound threads get longer batches once they have proven // they mostly burn CPU. Higher-priority work still preempts them as soon // as it becomes ready, so this reduces switch overhead without hurting // interactive wake latency. - if (thread->Priority == PriorityLow && thread->CpuBurstTicks >= + if (thread->EffectivePriority == PriorityLow && thread->CpuBurstTicks >= (SCHED_DEMOTE_AFTER_BURST_TICKS * 2u)) { quantum += 4; } @@ -256,17 +364,330 @@ static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) { KiSchedulerBug(where, thread, "picked thread was not ready"); } -// Inserts ready Locked. +// Inserts one thread into the queue selected by its effective rank. static void KiInsertReadyLocked(THREAD* thread) { KiAssertReadyQueueCandidateLocked("KiInsertReadyLocked", thread); - int p = (int)thread->Priority; - if (p < 0) p = 0; - if (p >= PriorityCount) p = PriorityCount - 1; + LIST_ENTRY* queue; + if (thread->EffectiveSchedulingClass == KeSchedulingRealtime) { + uint32_t priority = thread->EffectivePriority; + if (priority > KE_REALTIME_PRIORITY_MAX) + priority = KE_REALTIME_PRIORITY_MAX; + queue = &gRealtimeRunQueue[priority]; + } else { + uint32_t priority = thread->EffectivePriority; + if (priority >= PriorityCount) + priority = PriorityCount - 1; + queue = &gNormalRunQueue[priority]; + } thread->State = ThreadReady; thread->Queued = 1; - KeListInsertTailChecked(&gRunQueue[p], &thread->SchedulerLink); + KeListInsertTailChecked(queue, &thread->SchedulerLink); +} + +// Tests whether any ready thread outranks one running thread on a CPU. +static int KiAnyRunnableOutrankingThreadForCpuLocked(KCPU* cpu, const THREAD* current) { + if (!cpu || !current) return 0; + + for (int p = (int)KE_REALTIME_PRIORITY_MAX; p >= 0; p--) { + LIST_ENTRY* head = &gRealtimeRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiSchedulerThreadOutranks(thread, current)) + return 1; + } + } + + for (int p = PriorityCount - 1; p >= 0; p--) { + LIST_ENTRY* head = &gNormalRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiSchedulerThreadOutranks(thread, current)) + return 1; + } + } + + return 0; +} + +// Tests whether a real-time peer at the current or higher priority is ready. +static int KiAnyRealtimePeerForCpuLocked(KCPU* cpu, uint32_t minimumPriority) { + if (!cpu) return 0; + if (minimumPriority > KE_REALTIME_PRIORITY_MAX) + minimumPriority = KE_REALTIME_PRIORITY_MAX; + + for (int p = (int)KE_REALTIME_PRIORITY_MAX; p >= (int)minimumPriority; p--) { + LIST_ENTRY* head = &gRealtimeRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (KiThreadCanRunOnCpuLocked(thread, cpu)) + return 1; + } + } + return 0; +} + +// Computes one thread's effective rank from its base rank and mutex donations. +static void KiComputeEffectiveRankLocked(THREAD* thread, + KE_SCHEDULING_CLASS* schedulingClassOut, + uint32_t* priorityOut) { + KE_SCHEDULING_CLASS schedulingClass = thread->BaseSchedulingClass; + uint32_t priority = thread->BasePriority; + + LIST_ENTRY* head = &thread->OwnedMutexList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + MUTEX* mutex = CONTAINING_RECORD(node, MUTEX, OwnerLink); + if (!mutex->HasDonation) + continue; + if (KiRankOutranks(mutex->DonationClass, mutex->DonationPriority, + schedulingClass, priority)) { + schedulingClass = mutex->DonationClass; + priority = mutex->DonationPriority; + } + } + + *schedulingClassOut = schedulingClass; + *priorityOut = priority; +} + +// Applies one effective-rank change and preserves ready-queue ordering. +static int KiApplyEffectiveRankLocked(THREAD* thread, + KE_SCHEDULING_CLASS schedulingClass, + uint32_t priority) { + KE_SCHEDULING_CLASS oldClass = thread->EffectiveSchedulingClass; + uint32_t oldPriority = thread->EffectivePriority; + if (KiRanksEqual(oldClass, oldPriority, schedulingClass, priority)) + return 0; + + int wasQueued = thread->Queued != 0; + if (wasQueued) { + KeListRemoveChecked(&thread->SchedulerLink); + thread->Queued = 0; + } + + thread->EffectiveSchedulingClass = schedulingClass; + thread->EffectivePriority = priority; + thread->QuantumRemaining = 0; + + if (wasQueued) { + KiInsertReadyLocked(thread); + KiPokeRunnableCpusLocked(thread); + } else if (thread->State == ThreadRunning && thread->RunningCpu < gCpuCount) { + KCPU* cpu = &gCpus[thread->RunningCpu]; + if (KiAnyRunnableOutrankingThreadForCpuLocked(cpu, thread)) + KiRequestRescheduleCpuLocked(cpu); + } + + return 1; +} + +// Recomputes effective rank from current base rank and owned mutexes. +int KiSchedulerRefreshEffectivePriority(THREAD* thread, MUTEX** waitingMutexOut) { + if (!thread) return 0; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + KE_SCHEDULING_CLASS oldClass = thread->EffectiveSchedulingClass; + uint32_t oldPriority = thread->EffectivePriority; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &priority); + int changed = KiApplyEffectiveRankLocked(thread, schedulingClass, priority); + if (changed) { + if (KiRankOutranks(schedulingClass, priority, oldClass, oldPriority)) + gSchedStats.PriorityInheritanceBoosts++; + else + gSchedStats.PriorityInheritanceDrops++; + } + if (waitingMutexOut) { + *waitingMutexOut = (MUTEX*)0; + if (changed && thread->WaitingMutex) { + if (!ObTryReferenceObject(&thread->WaitingMutex->Header)) + KiSchedulerBug("KiSchedulerRefreshEffectivePriority", thread, + "waiting mutex could not be referenced"); + *waitingMutexOut = thread->WaitingMutex; + } + } + KeReleaseSpinlock(&gSchedLock, flags); + return changed; +} + +// Publishes one wait edge atomically so concurrent acquisitions cannot create a cycle. +KSTATUS KiSchedulerPrepareMutexWait(THREAD* thread, MUTEX* mutex, THREAD* owner) { + if (!thread || !mutex || !owner) return NTLSTATUS_INVALID_PARAMETER; + + KCPU* cpu = KeGetCurrentCpu(); + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + + THREAD* current = owner; + uint32_t depth = 0; + while (current && depth < KE_PRIORITY_INHERITANCE_MAX_DEPTH) { + if (current == thread) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_DEADLOCK; + } + + MUTEX* waitingMutex = current->WaitingMutex; + current = waitingMutex ? waitingMutex->Owner : (THREAD*)0; + depth++; + } + + if (current) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_DEADLOCK; + } + + if (thread->IsIdleThread) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "idle thread cannot block"); + if (thread->Queued) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "queued thread cannot block"); + if (!cpu || thread != cpu->CurrentThread) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "only the current thread can prepare a mutex wait"); + if (thread->RunningCpu != cpu->CpuIndex) + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "thread running CPU does not match current CPU"); + if (thread->State != ThreadRunning || thread->WaitingMutex || thread->WaitObject || + thread->WaitListHead || thread->SchedulerLink.Fwd != &thread->SchedulerLink || + thread->SchedulerLink.Bwd != &thread->SchedulerLink || thread->TimeoutArmed || + thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KiSchedulerBug("KiSchedulerPrepareMutexWait", thread, "thread already has wait state"); + } + + thread->State = ThreadWaiting; + thread->WaitObject = &mutex->Header; + thread->WaitStatus = NTLSTATUS_PENDING; + thread->WaitListHead = &mutex->WaitList; + thread->AlertableWait = 0; + thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0; + thread->WaitRundownContext = (void*)0; + thread->WaitingMutex = mutex; + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_SUCCESS; +} + +// Removes one published wait edge after its wait-list node is detached. +void KiSchedulerCancelMutexWait(THREAD* thread, MUTEX* mutex) { + if (!thread || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->WaitingMutex != mutex || thread->WaitObject != &mutex->Header || + thread->WaitListHead != &mutex->WaitList || + thread->SchedulerLink.Fwd != &thread->SchedulerLink || + thread->SchedulerLink.Bwd != &thread->SchedulerLink) { + KiSchedulerBug("KiSchedulerCancelMutexWait", thread, "mutex wait state is inconsistent"); + } + + thread->WaitingMutex = (MUTEX*)0; + thread->WaitListHead = (LIST_ENTRY*)0; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Assigns one unowned mutex while the caller holds its lock. +void KiSchedulerAcquireMutexOwnership(THREAD* thread, MUTEX* mutex) { + if (!thread || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (mutex->Owner || + mutex->OwnerLink.Fwd != &mutex->OwnerLink || + mutex->OwnerLink.Bwd != &mutex->OwnerLink) { + KiSchedulerBug("KiSchedulerAcquireMutexOwnership", thread, + "mutex already belongs to an owner"); + } + + mutex->Owner = thread; + KeListInsertTailChecked(&thread->OwnedMutexList, &mutex->OwnerLink); + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Transfers one mutex directly to a blocked waiter. +void KiSchedulerTransferMutexOwnership(THREAD* oldOwner, THREAD* newOwner, + MUTEX* mutex) { + if (!oldOwner || !newOwner || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (mutex->Owner != oldOwner || + mutex->OwnerLink.Fwd == &mutex->OwnerLink || + mutex->OwnerLink.Bwd == &mutex->OwnerLink) { + KiSchedulerBug("KiSchedulerTransferMutexOwnership", oldOwner, + "mutex ownership graph is inconsistent"); + } + if (newOwner->WaitingMutex != mutex || + newOwner->State != ThreadWaiting || + newOwner->WaitObject != &mutex->Header || + newOwner->WaitListHead != &mutex->WaitList || + newOwner->SchedulerLink.Fwd != &newOwner->SchedulerLink || + newOwner->SchedulerLink.Bwd != &newOwner->SchedulerLink) { + KiSchedulerBug("KiSchedulerTransferMutexOwnership", newOwner, + "new owner was not in the mutex handoff state"); + } + + KeListRemoveChecked(&mutex->OwnerLink); + newOwner->WaitingMutex = (MUTEX*)0; + newOwner->WaitListHead = (LIST_ENTRY*)0; + mutex->Owner = newOwner; + KeListInsertTailChecked(&newOwner->OwnedMutexList, &mutex->OwnerLink); + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Removes one mutex from its current owner. +void KiSchedulerReleaseMutexOwnership(THREAD* thread, MUTEX* mutex) { + if (!thread || !mutex) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (mutex->Owner != thread || + mutex->OwnerLink.Fwd == &mutex->OwnerLink || + mutex->OwnerLink.Bwd == &mutex->OwnerLink) { + KiSchedulerBug("KiSchedulerReleaseMutexOwnership", thread, + "mutex was not linked to the stated owner"); + } + + KeListRemoveChecked(&mutex->OwnerLink); + mutex->Owner = (THREAD*)0; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Returns one owned mutex while the ownership list is stable. +MUTEX* KiSchedulerFirstOwnedMutex(THREAD* thread) { + if (!thread) return (MUTEX*)0; + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + MUTEX* mutex = LIST_IS_EMPTY(&thread->OwnedMutexList) + ? (MUTEX*)0 + : CONTAINING_RECORD(thread->OwnedMutexList.Fwd, MUTEX, OwnerLink); + KeReleaseSpinlock(&gSchedLock, flags); + return mutex; +} + +// Records one transitive priority-inheritance propagation step. +void KiSchedulerNotePriorityPropagation(uint32_t depth) { + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + gSchedStats.PriorityInheritancePropagations++; + if (depth > gSchedStats.PriorityInheritanceMaxDepth) + gSchedStats.PriorityInheritanceMaxDepth = depth; + KeReleaseSpinlock(&gSchedLock, flags); +} + +// Changes one normal thread's dynamic base priority. +KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority) { + if (!thread || priority >= PriorityCount) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->BaseSchedulingClass != KeSchedulingNormal) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_ACCESS_DENIED; + } + + thread->BasePriority = (uint32_t)priority; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t effectivePriority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &effectivePriority); + (void)KiApplyEffectiveRankLocked(thread, schedulingClass, effectivePriority); + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_SUCCESS; } static int KiAnyRunnableForCpuLocked(KCPU* cpu); @@ -282,7 +703,7 @@ static int KiAnyRunnableForCurrentCpu(void) { // Implements the idle Loop operation. static void KiIdleLoop(void) { - for (;;) { + while (true) { KCPU* cpu = KeGetCurrentCpu(); cpu->SchedulerTicks++; @@ -304,8 +725,10 @@ static void KiIdleLoop(void) { // Initializes scheduler Global. void KiInitializeSchedulerGlobal(void) { - for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]); - LIST_INIT(&gSleepList); + for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gNormalRunQueue[i]); + for (uint32_t i = 0; i < KE_REALTIME_PRIORITY_COUNT; i++) LIST_INIT(&gRealtimeRunQueue[i]); + LIST_INIT(&gTimeoutList); + KeInitializeWorkItem(&gTimeoutWorkItem, KiTimeoutWorkRoutine, (void*)0); gSchedLock = (SPINLOCK)SPINLOCK_INIT; KSTATUS s = KeCreateProcess("idle", &gIdleProcess); @@ -333,14 +756,20 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { idle->Header.Destroy = (void *)0; idle->ThreadId = cpu->CpuIndex ? (0x80000000u | cpu->CpuIndex) : 0; - idle->State = ThreadRunning; - idle->Priority = PriorityLow; - idle->QuantaConsumed = 0; + idle->State = ThreadRunning; + idle->BaseSchedulingClass = KeSchedulingNormal; + idle->BasePriority = PriorityLow; + idle->EffectiveSchedulingClass = KeSchedulingNormal; + idle->EffectivePriority = PriorityLow; + idle->QuantaConsumed = 0; idle->QuantumTicks = 1; idle->QuantumRemaining = 1; idle->CpuBurstTicks = 0; idle->LastRunTicks = 0; idle->RunningCpu = cpu->CpuIndex; + idle->SuspendCount = 0; + LIST_INIT(&idle->OwnedMutexList); + idle->WaitingMutex = (MUTEX*)0; idle->Queued = 0; idle->IsIdleThread = 1; idle->SchedulerFlags = 0; @@ -351,7 +780,9 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { idle->WaitStatus = NTLSTATUS_SUCCESS; idle->WaitListHead = (LIST_ENTRY*)0; idle->AlertableWait = 0; - idle->SleepUntilTick = 0; + LIST_INIT(&idle->TimeoutLink); + idle->TimeoutArmed = 0; + idle->TimeoutTick = 0; idle->TimeoutStatus = NTLSTATUS_SUCCESS; idle->UserStackBase = (void*)0; idle->Teb = (void*)0; @@ -425,14 +856,20 @@ void KeSchedulerPostSwitch(void) { THREAD* cur = cpu->CurrentThread; KeReleaseSpinlock(&gSchedLock, KiPostSwitchUnlockCookie(cur)); - KiDeliverKernelApcs(); - if (cpu->OutgoingThread && cpu->OutgoingThread->State == ThreadDead) { THREAD* dead = cpu->OutgoingThread; cpu->OutgoingThread = (THREAD*)0; dead->RunningCpu = KE_INVALID_CPU; + + // Forced process termination can bypass KeExitThread. Finalize thread + // ownership after the dead thread switches out, before releasing the + // scheduler lifetime reference. + KeFinalizeThreadExit(dead); + KiMutexRundownThread(dead); ObDereferenceObject(&dead->Header); } + + KiDeliverKernelApcs(); } // Updates thread Affinity Mask. @@ -465,6 +902,63 @@ void KeReadyThread(THREAD* thread) { KeReleaseSpinlock(&gSchedLock, flags); } +// Decrements one thread's suspend count and queues it when the count reaches zero. +__declspec(dllexport) KSTATUS KeResumeThread(THREAD* thread, + uint32_t* previousSuspendCountOut) { + if (!thread || thread->IsIdleThread) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->State == ThreadDead) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_OBJECT_DEAD; + } + + uint32_t previous = thread->SuspendCount; + if (previous != 0) { + if (thread->State != ThreadSuspended || thread->Queued || + thread->RunningCpu != KE_INVALID_CPU) { + KiSchedulerBug("KeResumeThread", thread, + "suspend count did not match scheduler state"); + } + + thread->SuspendCount = previous - 1; + if (thread->SuspendCount == 0) { + KiInsertReadyLocked(thread); + KiPokeRunnableCpusLocked(thread); + } + } + KeReleaseSpinlock(&gSchedLock, flags); + + if (previousSuspendCountOut) + *previousSuspendCountOut = previous; + return NTLSTATUS_SUCCESS; +} + +// Cancels a thread that has never left its initial suspended state. +KSTATUS KeCancelSuspendedThread(THREAD* thread) { + if (!thread || thread->IsIdleThread) + return NTLSTATUS_INVALID_PARAMETER; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + if (thread->State != ThreadSuspended || thread->SuspendCount == 0 || + thread->Queued || thread->RunningCpu != KE_INVALID_CPU) { + KeReleaseSpinlock(&gSchedLock, flags); + return NTLSTATUS_INVALID_PARAMETER; + } + + thread->SuspendCount = 0; + thread->State = ThreadDead; + KeReleaseSpinlock(&gSchedLock, flags); + + KeFinalizeThreadExit(thread); + + // Drop the scheduler-owned lifetime reference. The creator still owns the + // reference returned by KeCreateThread(). + ObDereferenceObject(&thread->Header); + return NTLSTATUS_SUCCESS; +} + // Implements the unready Thread operation. void KeUnreadyThread(THREAD* thread) { if (!thread) return; @@ -480,8 +974,20 @@ void KeUnreadyThread(THREAD* thread) { static int KiAnyRunnableForCpuLocked(KCPU* cpu) { if (!cpu) cpu = KeGetCurrentCpu(); + for (uint32_t p = 0; p < KE_REALTIME_PRIORITY_COUNT; p++) { + LIST_ENTRY* head = &gRealtimeRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) + continue; + return 1; + } + } + for (int p = 0; p < PriorityCount; p++) { - LIST_ENTRY* head = &gRunQueue[p]; + LIST_ENTRY* head = &gNormalRunQueue[p]; for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); if (!KiThreadCanRunOnCpuLocked(thread, cpu)) @@ -494,28 +1000,43 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) { return 0; } +// Removes the first eligible thread from one ready queue. +static THREAD* KiPickFromQueueLocked(KCPU* cpu, LIST_ENTRY* head) { + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) { + gSchedStats.SmtSiblingDeferrals++; + continue; + } + + KeListRemoveChecked(node); + thread->Queued = 0; + KiAssertPickedThreadLocked("KiPickFromQueueLocked", thread); + return thread; + } + return (THREAD*)0; +} + // Implements the pick Next For Cpu operation. static THREAD* KiPickNextForCpu(KCPU* cpu) { if (!cpu) cpu = KeGetCurrentCpu(); - for (int p = PriorityCount - 1; p >= 0; p--) { - LIST_ENTRY* head = &gRunQueue[p]; - for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { - THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); - if (!KiThreadCanRunOnCpuLocked(thread, cpu)) - continue; - if (KiShouldDeferToIdlePrimaryLocked(cpu, thread)) { - gSchedStats.SmtSiblingDeferrals++; - continue; - } - - KeListRemoveChecked(node); - thread->Queued = 0; - KiAssertPickedThreadLocked("KiPickNextForCpu", thread); + for (int p = (int)KE_REALTIME_PRIORITY_MAX; p >= 0; p--) { + THREAD* thread = KiPickFromQueueLocked(cpu, &gRealtimeRunQueue[p]); + if (thread) { + gSchedStats.RealtimeDispatches++; return thread; } } + for (int p = PriorityCount - 1; p >= 0; p--) { + THREAD* thread = KiPickFromQueueLocked(cpu, &gNormalRunQueue[p]); + if (thread) + return thread; + } + return cpu->IdleThread; } @@ -561,6 +1082,25 @@ static void KiSchedule(void) { } THREAD* next = KiPickNextForCpu(cpu); + + // A voluntary reschedule must not let lower-ranked work displace a running + // real-time thread. Equal-ranked peers may still rotate at the quantum + // boundary, and higher-ranked real-time work may preempt immediately. + if (cur != idle && cur->State == ThreadRunning && + cur->EffectiveSchedulingClass == KeSchedulingRealtime && + next != cur && + !KiSchedulerThreadOutranks(next, cur) && + !KiRanksEqual(next->EffectiveSchedulingClass, next->EffectivePriority, + cur->EffectiveSchedulingClass, cur->EffectivePriority)) { + if (next != idle) + KiInsertReadyLocked(next); + if (cur->QuantumRemaining == 0) + KiRefillQuantumLocked(cur); + gSchedStats.KeepCurrentDecisions++; + KeReleaseSpinlock(&gSchedLock, flags); + return; + } + if (next == idle) gSchedStats.IdleSelections++; if (next == cur) { if (cur != idle) @@ -620,11 +1160,36 @@ void KiSchedulerMarkThreadWaiting(THREAD* thread) { KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread running CPU does not match current CPU"); if (thread->State != ThreadRunning) KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread was not running before wait"); + if (thread->TimeoutArmed || + thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread already has a timeout"); + } thread->State = ThreadWaiting; KeReleaseSpinlock(&gSchedLock, flags); } +// Cancels one armed deadline while the scheduler lock is held. +static void KiCancelThreadTimeoutLocked(THREAD* thread) { + if (!thread) return; + + if (thread->TimeoutArmed) { + if (thread->TimeoutLink.Fwd == &thread->TimeoutLink || + thread->TimeoutLink.Bwd == &thread->TimeoutLink) { + KiSchedulerBug("KiCancelThreadTimeoutLocked", thread, "armed timeout was not linked"); + } + KeListRemoveChecked(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + } else if (thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KiSchedulerBug("KiCancelThreadTimeoutLocked", thread, "unarmed timeout remained linked"); + } + + thread->TimeoutTick = 0; + thread->TimeoutStatus = NTLSTATUS_SUCCESS; +} + // Implements the scheduler Note Voluntary Block operation. void KiSchedulerNoteVoluntaryBlock(THREAD* thread) { if (!thread) return; @@ -643,17 +1208,17 @@ static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) { // already on the object's wait list and has State=ThreadWaiting, but it is // still executing with RunningCpu set until it actually calls KeYield(). // That case must complete in-place, not enqueue the still-running thread. - if (thread->State != ThreadWaiting) { - thread->WaitStatus = status; + // Once another path has completed the wait, a stale abort or duplicate + // signal must not replace the result that made the thread runnable. + if (thread->State != ThreadWaiting) return; - } + + KiCancelThreadTimeoutLocked(thread); thread->WaitObject = (OBJECT*)0; thread->WaitStatus = status; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; thread->CpuBurstTicks = 0; thread->QuantaConsumed = 0; @@ -667,10 +1232,15 @@ static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) { return; } - if (thread->Priority < PriorityHigh) { - thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1); + if (thread->BaseSchedulingClass == KeSchedulingNormal && + thread->BasePriority < PriorityHigh) { + thread->BasePriority++; } + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &priority); + (void)KiApplyEffectiveRankLocked(thread, schedulingClass, priority); KiInsertReadyLocked(thread); KiPokeRunnableCpusLocked(thread); } @@ -682,8 +1252,6 @@ void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) { if (thread->State == ThreadWaiting && thread->WaitListHead) { KeListRemoveChecked(&thread->SchedulerLink); thread->WaitListHead = (LIST_ENTRY*)0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; } KiWakeThreadLocked(thread, status); KeReleaseSpinlock(&gSchedLock, flags); @@ -696,22 +1264,58 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { KeReleaseSpinlock(&gSchedLock, flags); } -// Wakes threads whose scheduler-tick sleep deadline has expired. -static void KiWakeExpiredSleepersLocked(uint64_t nowTick) { - LIST_ENTRY* node = gSleepList.Fwd; - while (node != &gSleepList) { - LIST_ENTRY* next = node->Fwd; - THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); +// Detaches one expired deadline while the scheduler lock is held. +static THREAD* KiTakeExpiredTimeoutLocked(uint64_t nowTick, KSTATUS* statusOut) { + LIST_ENTRY* head = &gTimeoutList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, TimeoutLink); + if (!thread->TimeoutArmed) + KiSchedulerBug("KiTakeExpiredTimeoutLocked", thread, "timeout list contains an unarmed thread"); + if (thread->State != ThreadWaiting) + KiSchedulerBug("KiTakeExpiredTimeoutLocked", thread, "timeout list contains a non-waiting thread"); + if (thread->TimeoutTick > nowTick) + continue; - if (thread->State == ThreadWaiting && - thread->WaitListHead == &gSleepList && - thread->SleepUntilTick <= nowTick) { - KeListRemoveChecked(node); - thread->WaitListHead = (LIST_ENTRY*)0; - KiWakeThreadLocked(thread, thread->TimeoutStatus); - } + KSTATUS timeoutStatus = thread->TimeoutStatus; + KeListRemoveChecked(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + thread->TimeoutTick = 0; + thread->TimeoutStatus = NTLSTATUS_SUCCESS; + ObReferenceObject(&thread->Header); + if (statusOut) *statusOut = timeoutStatus; + return thread; + } - node = next; + return (THREAD*)0; +} + +// Reports whether one deadline is ready while the scheduler lock is held. +static int KiHasExpiredTimeoutLocked(uint64_t nowTick) { + LIST_ENTRY* head = &gTimeoutList; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, TimeoutLink); + if (thread->TimeoutTick <= nowTick) + return 1; + } + return 0; +} + +// Resolves expired deadlines at passive level through each object's abort path. +static void KiTimeoutWorkRoutine(void* context) { + (void)context; + KeAssertIrql(PASSIVE_LEVEL); + + while (true) { + KSTATUS timeoutStatus = NTLSTATUS_TIMEOUT; + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + uint64_t nowTick = gSchedStats.Ticks; + THREAD* thread = KiTakeExpiredTimeoutLocked(nowTick, &timeoutStatus); + KeReleaseSpinlock(&gSchedLock, flags); + if (!thread) + return; + + (void)KeAbortThreadWait(thread, timeoutStatus); + ObDereferenceObject(&thread->Header); } } @@ -724,36 +1328,42 @@ static int KiCpuOwnsGlobalSchedulerTick(const KCPU* cpu) { return cpu && cpu->IsBsp; } -// Ages ready threads after enough global scheduler ticks have elapsed. +// Ages ready normal-class threads after enough global ticks have elapsed. static void KiAgeReadyThreadsLocked(void) { - // Periodic aging prevents indefinite starvation of PriorityLow and - // PriorityNormal threads. Ready threads promoted by aging get a fresh CPU - // burst history; they earned the boost by waiting, not by burning CPU. + // Real-time queues are static and never participate in aging. Normal + // threads promoted by aging get a fresh CPU-burst history because they + // earned the boost by waiting rather than by burning CPU. gAgingCounter++; if (gAgingCounter < SCHED_AGING_INTERVAL_TICKS) return; gAgingCounter = 0; for (int p = 0; p < PriorityHigh; p++) { - while (!LIST_IS_EMPTY(&gRunQueue[p])) { - LIST_ENTRY* node = KeListPopHeadChecked(&gRunQueue[p]); - THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink); - t->Priority = PriorityHigh; - t->CpuBurstTicks = 0; - t->QuantaConsumed = 0; - t->QuantumRemaining = 0; + while (!LIST_IS_EMPTY(&gNormalRunQueue[p])) { + LIST_ENTRY* node = KeListPopHeadChecked(&gNormalRunQueue[p]); + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + thread->Queued = 0; + thread->BasePriority = PriorityHigh; + thread->CpuBurstTicks = 0; + thread->QuantaConsumed = 0; + thread->QuantumRemaining = 0; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(thread, &schedulingClass, &priority); + thread->EffectiveSchedulingClass = schedulingClass; + thread->EffectivePriority = priority; gSchedStats.AgingPromotions++; - KeListInsertTailChecked(&gRunQueue[PriorityHigh], &t->SchedulerLink); + KiInsertReadyLocked(thread); } } } // Runs scheduler work that must be single-sourced for the whole system. -static void KiRunGlobalSchedulerTickLocked(void) { +static int KiRunGlobalSchedulerTickLocked(void) { gSchedStats.Ticks++; - KiWakeExpiredSleepersLocked(gSchedStats.Ticks); KeInputTimerTick(); KiAgeReadyThreadsLocked(); + return KiHasExpiredTimeoutLocked(gSchedStats.Ticks); } // Services the current CPU's local scheduler timer interrupt. @@ -766,16 +1376,22 @@ static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) { } else if (KiThreadIsKilledLocked(cur)) { requestReschedule = 1; } else { - cur->CpuBurstTicks++; - cur->QuantaConsumed = cur->CpuBurstTicks; + if (cur->EffectiveSchedulingClass == KeSchedulingNormal) { + cur->CpuBurstTicks++; + cur->QuantaConsumed = cur->CpuBurstTicks; - if (cur->CpuBurstTicks >= SCHED_DEMOTE_AFTER_BURST_TICKS) { - cur->CpuBurstTicks = 0; - cur->QuantaConsumed = 0; - if (cur->Priority > PriorityLow) { - cur->Priority = (THREAD_PRIORITY)(cur->Priority - 1); - gSchedStats.Demotions++; - gSchedStats.CpuBurstDemotions++; + if (cur->CpuBurstTicks >= SCHED_DEMOTE_AFTER_BURST_TICKS) { + cur->CpuBurstTicks = 0; + cur->QuantaConsumed = 0; + if (cur->BasePriority > PriorityLow) { + cur->BasePriority--; + KE_SCHEDULING_CLASS schedulingClass; + uint32_t priority; + KiComputeEffectiveRankLocked(cur, &schedulingClass, &priority); + (void)KiApplyEffectiveRankLocked(cur, schedulingClass, priority); + gSchedStats.Demotions++; + gSchedStats.CpuBurstDemotions++; + } } } @@ -786,13 +1402,15 @@ static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) { cur->QuantumRemaining--; if (cur->QuantumRemaining == 0) { - if (KiAnyRunnableForCpuLocked(cpu)) { + int havePeer = cur->EffectiveSchedulingClass == KeSchedulingRealtime + ? KiAnyRealtimePeerForCpuLocked(cpu, cur->EffectivePriority) + : KiAnyRunnableForCpuLocked(cpu); + if (havePeer) { gSchedStats.QuantumExpirations++; requestReschedule = 1; } else { - // No peer is ready, so keep the CPU hot and refill the budget. - // This avoids reintroducing the old every-other-tick idle dance - // as a pointless reschedule-pending chatter loop. + // No eligible peer is ready, so keep the CPU hot and refill the + // budget instead of turning a periodic tick into idle chatter. KiRefillQuantumLocked(cur); gSchedStats.KeepCurrentDecisions++; } @@ -810,18 +1428,21 @@ void KeSchedulerTick(void) { if (!cur) return; int ownsGlobalTick = KiCpuOwnsGlobalSchedulerTick(cpu); + int queueTimeoutWork = 0; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); KiArmPostSwitchUnlock(cur, flags); if (ownsGlobalTick) - KiRunGlobalSchedulerTickLocked(); + queueTimeoutWork = KiRunGlobalSchedulerTickLocked(); int requestReschedule = KiRunLocalSchedulerTickLocked(cpu, cur, idle); KeReleaseSpinlock(&gSchedLock, flags); if (ownsGlobalTick) { + if (queueTimeoutWork) + (void)KeQueueWorkItem(&gTimeoutWorkItem); KeTimerTick(); KeDisplayTick(); } @@ -834,7 +1455,7 @@ void KeSchedulerTick(void) { KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) { if (milliseconds == 0) return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) + if (KeGetCurrentIrql() > DISPATCH_LEVEL) return NTLSTATUS_INVALID_IRQL; THREAD* self = KeGetCurrentThread(); @@ -848,8 +1469,11 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) ticks = 1; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - if (self->State != ThreadWaiting || self->WaitListHead != (LIST_ENTRY*)0 || - self->RunningCpu != KeGetCurrentCpu()->CpuIndex) { + if (self->State != ThreadWaiting || + self->RunningCpu != KeGetCurrentCpu()->CpuIndex || + self->TimeoutArmed || + self->TimeoutLink.Fwd != &self->TimeoutLink || + self->TimeoutLink.Bwd != &self->TimeoutLink) { KeReleaseSpinlock(&gSchedLock, flags); return NTLSTATUS_INVALID_PARAMETER; } @@ -859,10 +1483,10 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) if (wakeTick < now) wakeTick = UINT64_MAX; - self->WaitListHead = &gSleepList; - self->SleepUntilTick = wakeTick; + self->TimeoutTick = wakeTick; self->TimeoutStatus = timeoutStatus; - KeListInsertTailChecked(&gSleepList, &self->SchedulerLink); + self->TimeoutArmed = 1; + KeListInsertTailChecked(&gTimeoutList, &self->TimeoutLink); KeReleaseSpinlock(&gSchedLock, flags); return NTLSTATUS_SUCCESS; } @@ -890,7 +1514,10 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) { uintptr_t flags = KeAcquireSpinlock(&gSchedLock); if (self->Queued || self->State != ThreadRunning || - self->RunningCpu != KeGetCurrentCpu()->CpuIndex) { + self->RunningCpu != KeGetCurrentCpu()->CpuIndex || + self->TimeoutArmed || + self->TimeoutLink.Fwd != &self->TimeoutLink || + self->TimeoutLink.Bwd != &self->TimeoutLink) { KeReleaseSpinlock(&gSchedLock, flags); return NTLSTATUS_INVALID_PARAMETER; } @@ -903,11 +1530,12 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) { self->State = ThreadWaiting; self->WaitObject = (OBJECT*)0; self->WaitStatus = NTLSTATUS_PENDING; - self->WaitListHead = &gSleepList; + self->WaitListHead = (LIST_ENTRY*)0; self->AlertableWait = 0; - self->SleepUntilTick = wakeTick; + self->TimeoutTick = wakeTick; self->TimeoutStatus = NTLSTATUS_SUCCESS; - KeListInsertTailChecked(&gSleepList, &self->SchedulerLink); + self->TimeoutArmed = 1; + KeListInsertTailChecked(&gTimeoutList, &self->TimeoutLink); KeReleaseSpinlock(&gSchedLock, flags); return KeCommitBlockThread(self, 0); diff --git a/code/kernel/ke_selftest.c b/code/kernel/ke_selftest.c index 7b3cab9..f745cdf 100644 --- a/code/kernel/ke_selftest.c +++ b/code/kernel/ke_selftest.c @@ -11,11 +11,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -240,9 +243,10 @@ static void KiRunKernelApcSelfTest(void) { s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: FinishedEvent create failed %d\n", s); + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker, (uintptr_t)&st, - PriorityNormal, + &workerParameters, &st.WorkerThread); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker create failed %d\n", s); @@ -310,9 +314,10 @@ static void KiRunKernelApcSelfTest(void) { s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert FinishedEvent create failed %d\n", s); + const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker, (uintptr_t)&st, - PriorityNormal, + &workerParameters, &st.WorkerThread); if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable worker create failed %d\n", s); @@ -584,6 +589,213 @@ static void KiRunPfnDatabaseSelfTest(void) { DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n"); } +// Verifies common-buffer, direct mapping, bounce mapping, and teardown accounting. +static void KiRunDmaSelfTest(void) { + const NTL_DDK_DMA* dma = IoGetDmaDdk(); + if (!dma) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: DDK table missing\n"); + + IO_DMA_STATS before; + IO_DMA_STATS after; + dma->QueryStats(&before); + + DRIVER_OBJECT driver; + memset(&driver, 0, sizeof(driver)); + IoInitializeDriverObject(&driver, SYS_SV_LITERAL("dma-selftest")); + + DEVICE_OBJECT* device = (DEVICE_OBJECT*)0; + KSTATUS status = IoCreateDevice(&driver, 0, &device); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: device creation failed status=%u\n", status); + + IO_DMA_REQUIREMENTS requirements; + memset(&requirements, 0, sizeof(requirements)); + requirements.MaximumDeviceAddress = UINT64_MAX; + requirements.Alignment = PAGE_SIZE; + requirements.MaximumSegmentLength = PAGE_SIZE; + requirements.MaximumSegments = 4; + + IO_DMA_ADAPTER* adapter = (IO_DMA_ADAPTER*)0; + IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0; + MM_PINNED_MEMORY* pinned = (MM_PINNED_MEMORY*)0; + IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)0; + + status = dma->GetAdapter(device, &requirements, &adapter); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: adapter creation failed status=%u\n", status); + + status = dma->AllocateCommonBuffer(adapter, PAGE_SIZE, PAGE_SIZE, &buffer); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: common buffer allocation failed status=%u\n", status); + + uint8_t* bytes = (uint8_t*)dma->GetCommonBufferAddress(buffer); + uint64_t deviceAddress = dma->GetCommonBufferDeviceAddress(buffer); + if (!bytes || !deviceAddress || (deviceAddress & (PAGE_SIZE - 1)) != 0 || + dma->GetCommonBufferSize(buffer) != PAGE_SIZE) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: common buffer metadata invalid address=0x%016llX\n", + (unsigned long long)deviceAddress); + + status = dma->PinKernelMemory(bytes, PAGE_SIZE, IoDmaBidirectional, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: kernel pin failed status=%u\n", status); + status = dma->MapPinnedMemory(adapter, + pinned, + 0, + PAGE_SIZE, + IoDmaBidirectional, + IO_DMA_MAP_NO_BOUNCE, + &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 || + dma->GetSegments(mapping)[0].DeviceAddress != deviceAddress) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: direct mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + + for (size_t i = 0; i < 128; i++) + bytes[i + 1] = (uint8_t)(i ^ 0x5Au); + status = dma->PinKernelMemory(bytes + 1, 128, IoDmaToDevice, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: unaligned kernel pin failed status=%u\n", status); + status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaToDevice, 0, &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 || + (dma->GetSegments(mapping)[0].DeviceAddress & (PAGE_SIZE - 1)) != 0 || + dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: bounce mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + status = dma->SynchronizeForDevice(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: bounce synchronization failed status=%u\n", status); + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + + memset(bytes + 1, 0, 128); + status = dma->PinKernelMemory(bytes + 1, 128, IoDmaFromDevice, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive pin failed status=%u\n", status); + status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaFromDevice, 0, &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 || + dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: receive bounce mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + status = dma->SynchronizeForDevice(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive handoff failed status=%u\n", status); + + uint64_t receiveDeviceAddress = dma->GetSegments(mapping)[0].DeviceAddress; + uint8_t* receiveDeviceAlias = (uint8_t*)MmPfnToVirtual(receiveDeviceAddress & PAGE_MASK); + if (!receiveDeviceAlias) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive device alias missing\n"); + receiveDeviceAlias += receiveDeviceAddress & (PAGE_SIZE - 1); + for (size_t i = 0; i < 128; i++) + receiveDeviceAlias[i] = (uint8_t)(i ^ 0xA5u); + + status = dma->SynchronizeForCpu(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive synchronization failed status=%u\n", status); + for (size_t i = 0; i < 128; i++) { + if (bytes[i + 1] != (uint8_t)(i ^ 0xA5u)) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: receive bounce copy mismatch offset=%llu\n", + (unsigned long long)i); + } + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + + PROCESS* process = KeGetCurrentProcess(); + if (!process) + process = KeGetSystemProcess(); + if (!process) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: no process for user pinning\n"); + + void* userBase = (void*)0; + status = MmVadCreateCommitted(process, + PAGE_SIZE, + PAGE_SIZE, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, + MmVadGeneric, + &userBase); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD creation failed status=%u\n", status); + status = MmVadEnsureResident(process, userBase, PAGE_SIZE, PAGE_WRITE | PAGE_USER); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD materialization failed status=%u\n", status); + ((uint8_t*)userBase)[0] = 0xD5u; + + status = dma->PinUserMemory(process, userBase, PAGE_SIZE, IoDmaToDevice, &pinned); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user pin failed status=%u\n", status); + status = MmVadDestroy(process, userBase); + if (status != NTLSTATUS_ACCESS_DENIED) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: pinned user VAD was destroyable status=%u\n", + status); + status = dma->MapPinnedMemory(adapter, + pinned, + 0, + PAGE_SIZE, + IoDmaToDevice, + IO_DMA_MAP_NO_BOUNCE, + &mapping); + if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1) + KePanic(KE_PANIC_CODE_SELFTEST, + "DMA selftest: user mapping failed status=%u segments=%u\n", + status, + dma->GetSegmentCount(mapping)); + status = dma->SynchronizeForDevice(mapping); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user synchronization failed status=%u\n", status); + dma->UnmapMemory(mapping); + mapping = (IO_DMA_MAPPING*)0; + dma->UnpinMemory(pinned); + pinned = (MM_PINNED_MEMORY*)0; + status = MmVadDestroy(process, userBase); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD destroy failed status=%u\n", status); + + dma->FreeCommonBuffer(buffer); + dma->PutAdapter(adapter); + IoDeleteDevice(device); + + dma->QueryStats(&after); + if (after.AdaptersCreated != before.AdaptersCreated + 1 || + after.AdaptersDestroyed != before.AdaptersDestroyed + 1 || + after.AdaptersInUse != before.AdaptersInUse || + after.CommonBuffersAllocated < before.CommonBuffersAllocated + 3 || + after.CommonBuffersFreed < before.CommonBuffersFreed + 3 || + after.CommonBuffersInUse != before.CommonBuffersInUse || + after.CommonBufferBytesInUse != before.CommonBufferBytesInUse || + after.PinnedRangesInUse != before.PinnedRangesInUse || + after.PinnedPagesInUse != before.PinnedPagesInUse || + after.PinnedKernelRanges != before.PinnedKernelRanges + 3 || + after.PinnedUserRanges != before.PinnedUserRanges + 1 || + after.PinnedRangesReleased != before.PinnedRangesReleased + 4 || + after.StreamingMappings != before.StreamingMappings + 4 || + after.StreamingMappingsUnmapped != before.StreamingMappingsUnmapped + 4 || + after.StreamingMappingsInUse != before.StreamingMappingsInUse || + after.DirectMappings != before.DirectMappings + 2 || + after.BounceMappings != before.BounceMappings + 2 || + after.BounceBytesCopiedToDevice < before.BounceBytesCopiedToDevice + 128 || + after.BounceBytesCopiedToCpu < before.BounceBytesCopiedToCpu + 128) + KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: accounting mismatch\n"); + + DbgPrint("DMA selftest: common/kernel-pin/user-pin/direct/send-bounce/receive-bounce/teardown PASS\n"); +} + // Verifies the page-table manager invariants that the higher VMM will depend on. static void KiRunMemoryMappingSelfTest(void) { PHYSICAL_ADDRESS phys = 0; @@ -1155,6 +1367,380 @@ static void KiRunAsyncFileIoSelfTest(void) { DbgPrint("FILE/ASYNC selftest: PASS\n"); } +typedef struct KI_SCHEDULER_SELFTEST_STATE { + MUTEX* GateMutex; + MUTEX* LinkMutex; + EVENT* ChainReadyEvent; + EVENT* RealtimeReadyEvent; + EVENT* ChainFinishedEvent; + EVENT* RealtimeFinishedEvent; + volatile KSTATUS ChainAcquireStatus; + volatile KSTATUS RealtimeAcquireStatus; +} KI_SCHEDULER_SELFTEST_STATE; + +// Waits until one worker has entered the expected mutex wait state. +static KE_THREAD_SCHEDULER_SNAPSHOT KiSchedulerAwaitMutexWait(THREAD* thread, MUTEX* mutex) { + KE_THREAD_SCHEDULER_SNAPSHOT snapshot; + memset(&snapshot, 0, sizeof(snapshot)); + + for (uint32_t attempt = 0; attempt < 100; attempt++) { + KiSchedulerSnapshotThread(thread, &snapshot); + if (snapshot.State == ThreadWaiting && snapshot.WaitingMutex == mutex) + return snapshot; + + KSTATUS status = KeDelayThread(SCHED_TICK_MS); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: state wait failed status=%u\n", status); + } + + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: thread %u did not block on mutex=0x%016llX state=%u waiting=0x%016llX\n", + thread ? thread->ThreadId : 0, + (uint64_t)mutex, + (unsigned)snapshot.State, + (uint64_t)snapshot.WaitingMutex); +} + +typedef struct KI_SYNC_TIMEOUT_SELFTEST_STATE { + MUTEX* Mutex; + EVENT* ReadyEvent; + EVENT* FinishedEvent; + volatile KSTATUS MutexStatus; +} KI_SYNC_TIMEOUT_SELFTEST_STATE; + +// Waits on the held mutex long enough for the timeout engine to remove donation state. +static void KiSynchronizationTimeoutWorker(void* argument) { + KI_SYNC_TIMEOUT_SELFTEST_STATE* state = (KI_SYNC_TIMEOUT_SELFTEST_STATE*)argument; + KeSignalEvent(state->ReadyEvent); + + state->MutexStatus = KeAcquireMutex(state->Mutex, SCHED_TICK_MS * 20ULL); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(state->MutexStatus)) + KeReleaseMutex(state->Mutex); + + KeSignalEvent(state->FinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +// Verifies finite mutex and semaphore waits consume no object state after timeout. +static void KiRunSynchronizationTimeoutSelfTest(void) { + DbgPrint("SYNC/TIMEOUT selftest: begin\n"); + + KI_SYNC_TIMEOUT_SELFTEST_STATE state; + memset(&state, 0, sizeof(state)); + state.MutexStatus = NTLSTATUS_PENDING; + + KSTATUS status = KeMutexCreate(&state.Mutex); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: mutex create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.ReadyEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: ready event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.FinishedEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finished event create failed status=%u\n", status); + + status = KeAcquireMutex(state.Mutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: owner acquire failed status=%u\n", status); + + THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_REALTIME(12); + status = KeCreateKernelThread((uintptr_t)KiSynchronizationTimeoutWorker, + (uintptr_t)&state, + ¶meters, + &worker); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: worker create failed status=%u\n", status); + + status = KeWaitForEvent(state.ReadyEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: ready wait failed status=%u\n", status); + + (void)KiSchedulerAwaitMutexWait(worker, state.Mutex); + + THREAD* self = KeGetCurrentThread(); + KE_THREAD_SCHEDULER_SNAPSHOT snapshot; + KiSchedulerSnapshotThread(self, &snapshot); + if (snapshot.EffectiveSchedulingClass != KeSchedulingRealtime || + snapshot.EffectivePriority != 12) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finite waiter donation missing class=%u priority=%u\n", + snapshot.EffectiveSchedulingClass, + snapshot.EffectivePriority); + } + + status = KeWaitForEvent(state.FinishedEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finished wait failed status=%u\n", status); + if (state.MutexStatus != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: mutex wait returned status=%u\n", + state.MutexStatus); + + KiSchedulerSnapshotThread(self, &snapshot); + if (snapshot.EffectiveSchedulingClass != snapshot.BaseSchedulingClass || + snapshot.EffectivePriority != snapshot.BasePriority) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: timed-out donation remained base=%u/%u effective=%u/%u\n", + snapshot.BaseSchedulingClass, + snapshot.BasePriority, + snapshot.EffectiveSchedulingClass, + snapshot.EffectivePriority); + } + + KeReleaseMutex(state.Mutex); + status = KeAcquireMutex(state.Mutex, 0); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: zero-timeout mutex acquire failed status=%u\n", status); + KeReleaseMutex(state.Mutex); + + SEMAPHORE* semaphore = (SEMAPHORE*)0; + status = KeSemaphoreCreate(0, 1, &semaphore); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: semaphore create failed status=%u\n", status); + + status = KeWaitSemaphore(semaphore, 0); + if (status != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: zero-timeout semaphore wait returned status=%u\n", status); + + status = KeWaitSemaphore(semaphore, SCHED_TICK_MS * 3ULL); + if (status != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: finite semaphore wait returned status=%u\n", status); + + KeSignalSemaphore(semaphore); + status = KeWaitSemaphore(semaphore, 0); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: signaled semaphore probe failed status=%u\n", status); + status = KeWaitSemaphore(semaphore, 0); + if (status != NTLSTATUS_TIMEOUT) + KePanic(KE_PANIC_CODE_SELFTEST, + "SYNC/TIMEOUT selftest: semaphore count was consumed twice status=%u\n", status); + + ObDereferenceObject(&worker->Header); + ObDereferenceObject(&semaphore->Header); + ObDereferenceObject(&state.Mutex->Header); + ObDereferenceObject(&state.ReadyEvent->Header); + ObDereferenceObject(&state.FinishedEvent->Header); + + DbgPrint("SYNC/TIMEOUT selftest: PASS\n"); +} + +// Owns the link mutex and then blocks behind the self-test thread. +static void KiSchedulerChainWorker(void* argument) { + KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument; + + KSTATUS status = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain link acquire failed status=%u\n", status); + + KeSignalEvent(state->ChainReadyEvent); + state->ChainAcquireStatus = KeAcquireMutex(state->GateMutex, KE_WAIT_INFINITE); + if (state->ChainAcquireStatus != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain gate acquire failed status=%u\n", + state->ChainAcquireStatus); + + KeReleaseMutex(state->GateMutex); + KeReleaseMutex(state->LinkMutex); + KeSignalEvent(state->ChainFinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +// Blocks a real-time thread behind the normal chain worker. +static void KiSchedulerRealtimeWorker(void* argument) { + KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument; + KeSignalEvent(state->RealtimeReadyEvent); + + state->RealtimeAcquireStatus = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE); + if (state->RealtimeAcquireStatus != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time acquire failed status=%u\n", + state->RealtimeAcquireStatus); + + KeReleaseMutex(state->LinkMutex); + KeSignalEvent(state->RealtimeFinishedEvent); + KeExitThread(KeGetCurrentThread(), 0); +} + +// Verifies real-time dispatch and transitive mutex priority inheritance. +static void KiRunRealtimeSchedulerSelfTest(void) { + DbgPrint("SCHED selftest: begin\n"); + + THREAD* self = KeGetCurrentThread(); + if (!self || self->BaseSchedulingClass != KeSchedulingNormal) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: caller is not a normal thread\n"); + + KE_SCHED_STATS before; + KE_SCHED_STATS after; + KeGetSchedulerStats(&before); + + KI_SCHEDULER_SELFTEST_STATE state; + memset(&state, 0, sizeof(state)); + state.ChainAcquireStatus = NTLSTATUS_PENDING; + state.RealtimeAcquireStatus = NTLSTATUS_PENDING; + + KSTATUS status = KeMutexCreate(&state.GateMutex); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: gate mutex create failed status=%u\n", status); + + status = KeMutexCreate(&state.LinkMutex); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: link mutex create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.ChainReadyEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain ready event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.RealtimeReadyEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time ready event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.ChainFinishedEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain finished event create failed status=%u\n", status); + + status = KeEventCreate(EventAutoReset, 0, &state.RealtimeFinishedEvent); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time finished event create failed status=%u\n", status); + + status = KeAcquireMutex(state.GateMutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: owner acquire failed status=%u\n", status); + + THREAD* chainWorker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS chainParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + status = KeCreateKernelThread((uintptr_t)KiSchedulerChainWorker, + (uintptr_t)&state, + &chainParameters, + &chainWorker); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain worker create failed status=%u\n", status); + + status = KeWaitForEvent(state.ChainReadyEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain ready wait failed status=%u\n", status); + + (void)KiSchedulerAwaitMutexWait(chainWorker, state.GateMutex); + + status = KeAcquireMutex(state.LinkMutex, KE_WAIT_INFINITE); + if (status != NTLSTATUS_DEADLOCK) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: mutex cycle was not rejected status=%u\n", status); + + THREAD* realtimeWorker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS realtimeParameters = KE_THREAD_CREATE_REALTIME(16); + status = KeCreateKernelThread((uintptr_t)KiSchedulerRealtimeWorker, + (uintptr_t)&state, + &realtimeParameters, + &realtimeWorker); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time worker create failed status=%u\n", status); + + status = KeWaitForEvent(state.RealtimeReadyEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time ready wait failed status=%u\n", status); + + KE_THREAD_SCHEDULER_SNAPSHOT realtimeSnapshot = + KiSchedulerAwaitMutexWait(realtimeWorker, state.LinkMutex); + (void)realtimeSnapshot; + + KE_THREAD_SCHEDULER_SNAPSHOT chainSnapshot; + KE_THREAD_SCHEDULER_SNAPSHOT selfSnapshot; + KiSchedulerSnapshotThread(chainWorker, &chainSnapshot); + KiSchedulerSnapshotThread(self, &selfSnapshot); + + if (chainSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime || + chainSnapshot.EffectivePriority != 16) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: intermediate donation missing class=%u priority=%u\n", + chainSnapshot.EffectiveSchedulingClass, + chainSnapshot.EffectivePriority); + + if (selfSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime || + selfSnapshot.EffectivePriority != 16) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: transitive donation missing class=%u priority=%u\n", + selfSnapshot.EffectiveSchedulingClass, + selfSnapshot.EffectivePriority); + + KeReleaseMutex(state.GateMutex); + + KiSchedulerSnapshotThread(self, &selfSnapshot); + if (selfSnapshot.EffectiveSchedulingClass != selfSnapshot.BaseSchedulingClass || + selfSnapshot.EffectivePriority != selfSnapshot.BasePriority) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: inherited priority was not restored base=%u/%u effective=%u/%u\n", + selfSnapshot.BaseSchedulingClass, + selfSnapshot.BasePriority, + selfSnapshot.EffectiveSchedulingClass, + selfSnapshot.EffectivePriority); + + status = KeWaitForEvent(state.ChainFinishedEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: chain finished wait failed status=%u\n", status); + + status = KeWaitForEvent(state.RealtimeFinishedEvent, 0); + if (!KSUCCESS(status)) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: real-time finished wait failed status=%u\n", status); + + if (state.ChainAcquireStatus != NTLSTATUS_SUCCESS || + state.RealtimeAcquireStatus != NTLSTATUS_SUCCESS) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: handoff statuses chain=%u realtime=%u\n", + state.ChainAcquireStatus, + state.RealtimeAcquireStatus); + + KeGetSchedulerStats(&after); + if (after.RealtimeDispatches <= before.RealtimeDispatches || + after.PriorityInheritanceBoosts < before.PriorityInheritanceBoosts + 2 || + after.PriorityInheritanceDrops < before.PriorityInheritanceDrops + 2 || + after.PriorityInheritancePropagations < before.PriorityInheritancePropagations + 2 || + after.PriorityInheritanceMaxDepth < 2) + KePanic(KE_PANIC_CODE_SELFTEST, + "SCHED selftest: scheduler counters did not prove transitive inheritance\n"); + + ObDereferenceObject(&chainWorker->Header); + ObDereferenceObject(&realtimeWorker->Header); + ObDereferenceObject(&state.GateMutex->Header); + ObDereferenceObject(&state.LinkMutex->Header); + ObDereferenceObject(&state.ChainReadyEvent->Header); + ObDereferenceObject(&state.RealtimeReadyEvent->Header); + ObDereferenceObject(&state.ChainFinishedEvent->Header); + ObDereferenceObject(&state.RealtimeFinishedEvent->Header); + + DbgPrint("SCHED selftest: PASS\n"); +} + // Runs trap Frame Self Test. static void KiRunTrapFrameSelfTest(void) { DbgPrint("TRAP selftest: begin\n"); @@ -1251,8 +1837,11 @@ void KiRunEarlyKernelSelfTests(void) { // Runs kernel Self Tests. void KiRunKernelSelfTests(const HAL_API* hal) { DbgPrint("Kernel selftests: begin\n"); + KiRunSynchronizationTimeoutSelfTest(); + KiRunRealtimeSchedulerSelfTest(); KiRunKernelApcSelfTest(); KiRunPfnDatabaseSelfTest(); + KiRunDmaSelfTest(); KiRunMemoryMappingSelfTest(); KiRunVadSparseSelfTest(); KiRunVadCopyOnWriteSelfTest(); diff --git a/code/kernel/ke_semaphore.c b/code/kernel/ke_semaphore.c index 7daa3fc..73ceb70 100644 --- a/code/kernel/ke_semaphore.c +++ b/code/kernel/ke_semaphore.c @@ -26,10 +26,10 @@ __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SE return NTLSTATUS_SUCCESS; } -// Waits for semaphore. -__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) { +// Waits for one semaphore count within the requested timeout. +__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem, uint64_t timeoutMilliseconds) { if (!sem) return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL; + if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL; THREAD* self = KeGetCurrentThread(); uintptr_t flags = KeAcquireSpinlock(&sem->Lock); @@ -39,24 +39,26 @@ __declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) { return NTLSTATUS_SUCCESS; } + if (timeoutMilliseconds == 0) { + KeReleaseSpinlock(&sem->Lock, flags); + return NTLSTATUS_TIMEOUT; + } + KePrepareBlockThread(self, &sem->Header, 0); KeListInsertTailChecked(&sem->WaitList, &self->SchedulerLink); KeSetThreadWaitList(self, &sem->WaitList); - KeReleaseSpinlock(&sem->Lock, flags); - return KeCommitBlockThread(self, 0); -} - -// Implements the try Semaphore operation. -__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) { - if (!sem) return NTLSTATUS_INVALID_PARAMETER; - uintptr_t flags = KeAcquireSpinlock(&sem->Lock); - if (sem->Count > 0) { - sem->Count--; - KeReleaseSpinlock(&sem->Lock, flags); - return NTLSTATUS_SUCCESS; + if (timeoutMilliseconds != KE_WAIT_INFINITE) { + KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT); + if (!KSUCCESS(timeoutStatus)) { + KeListRemoveChecked(&self->SchedulerLink); + KeSetThreadWaitList(self, (LIST_ENTRY*)0); + KeUnblockThreadWithStatus(self, timeoutStatus); + KeReleaseSpinlock(&sem->Lock, flags); + return timeoutStatus; + } } KeReleaseSpinlock(&sem->Lock, flags); - return NTLSTATUS_FAILURE; + return KeCommitBlockThread(self, 0); } // Signals semaphore. diff --git a/code/kernel/ke_smp.c b/code/kernel/ke_smp.c index 1607242..44f0321 100644 --- a/code/kernel/ke_smp.c +++ b/code/kernel/ke_smp.c @@ -289,6 +289,8 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) { if ((cr4 & AP_CR4_REQUIRED) != AP_CR4_REQUIRED) __writecr4(cr4 | AP_CR4_REQUIRED); + KeInitializeMemoryTypes(); + if (gApBootBlock) { gApBootBlock->Stage = AP_STAGE_ENTRY; gApBootBlock->State = AP_STATE_KERNEL_ENTRY; @@ -320,7 +322,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) { gApBootBlock->State = AP_STATE_KERNEL_PARKED; } - for (;;) { + while (true) { cpu->SchedulerTicks++; if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) { @@ -980,11 +982,12 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) { cpuIndex, st); } - st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker, - (uintptr_t)arg, - PriorityNormal, - (1ULL << cpuIndex), - (THREAD**)0); + KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + workerParameters.AffinityMask = 1ULL << cpuIndex; + st = KeCreateKernelThread((uintptr_t)KiSmpApControlledWorker, + (uintptr_t)arg, + &workerParameters, + (THREAD**)0); if (!KSUCCESS(st)) { KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: AP%u worker create failed status=0x%x\n", diff --git a/code/kernel/ke_switch.asm b/code/kernel/ke_switch.asm index f7b7874..09709fb 100644 --- a/code/kernel/ke_switch.asm +++ b/code/kernel/ke_switch.asm @@ -2,16 +2,16 @@ ; ; Struct offsets: ; sizeof(OBJECT) = 24 (type:4, refcount:4, state:4, flags:4, destroy:8) -; THREAD.SavedContext offset = 96 -; THREAD.TssRsp0 offset = 112 -; THREAD.Teb offset = 120 -; THREAD.FpuContext offset = 176 +; THREAD.SavedContext offset = 112 +; THREAD.TssRsp0 offset = 128 +; THREAD.Teb offset = 136 +; THREAD.FpuContext offset = 192 ; TSS64.RSP0 offset = 4 ; -THREAD_SavedContext EQU 96 -THREAD_TssRsp0 EQU 112 -THREAD_Teb EQU 120 -THREAD_FpuContext EQU 176 +THREAD_SavedContext EQU 112 +THREAD_TssRsp0 EQU 128 +THREAD_Teb EQU 136 +THREAD_FpuContext EQU 192 TSS_RSP0_OFFSET EQU 4 EXTERN KeSetCurrentTssRsp0 : PROC @@ -131,12 +131,14 @@ KeStartInitialThread ENDP ; We load RCX from the stack (first arg per MS x64), then IRETQ. ; ------------------------------------------------------------------------- EXTERN KeSchedulerPostSwitch : PROC +EXTERN KePrepareUserThreadStart : PROC EXTERN KeGetCurrentThread : PROC EXTERN KeExitThread : PROC KeThreadEntry PROC sub rsp, 40 ; Reserve shadow space and restore MS x64 call alignment. call KeSchedulerPostSwitch + call KePrepareUserThreadStart add rsp, 40 pop rcx ; load user arg swapgs ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU diff --git a/code/kernel/ke_syscall.c b/code/kernel/ke_syscall.c index ff87faa..969376f 100644 --- a/code/kernel/ke_syscall.c +++ b/code/kernel/ke_syscall.c @@ -71,7 +71,9 @@ #include #include #include +#include #include +#include #include // HAL pointer - set once at boot by KeSyscallSetHal(). @@ -182,7 +184,8 @@ typedef struct SURFACE_OBJECT { uint64_t PresentBytesCopied; PHYSICAL_ADDRESS* Frames; void* UserBase; - uint32_t* RowScratch; + MM_KERNEL_VIEW* KernelView; + void* KernelBase; MUTEX* Lock; } SURFACE_OBJECT; @@ -274,6 +277,12 @@ static void KiSurfaceDestroy(OBJECT* obj) { surface->UserBase = (void*)0; } + if (surface->KernelView) { + MmDestroyKernelView(surface->KernelView); + surface->KernelView = (MM_KERNEL_VIEW*)0; + surface->KernelBase = (void*)0; + } + if (surface->Lock) { ObDereferenceObject(&surface->Lock->Header); surface->Lock = (MUTEX*)0; @@ -288,52 +297,19 @@ static void KiSurfaceDestroy(OBJECT* obj) { surface->Frames = (PHYSICAL_ADDRESS*)0; } - if (surface->RowScratch) { - MmFreePool(surface->RowScratch); - surface->RowScratch = (uint32_t*)0; - } } -typedef struct SURFACE_ROW_CONTEXT { - SURFACE_OBJECT* Surface; - uint32_t SourceX; - uint32_t SourceY; -} SURFACE_ROW_CONTEXT; - -static KSTATUS KiSurfaceCopyRow(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount) { - SURFACE_ROW_CONTEXT* ctx = (SURFACE_ROW_CONTEXT*)context; - if (!ctx || !ctx->Surface || !scratch || rowIndex < 0) +// Creates the process mapping used for user drawing. +static KSTATUS KiSurfaceEnsureViewLocked(SURFACE_OBJECT* surface) { + if (!surface || !surface->Owner || !surface->Frames || !surface->PageCount) return NTLSTATUS_INVALID_PARAMETER; + if (surface->UserBase) + return NTLSTATUS_SUCCESS; - SURFACE_OBJECT* surface = ctx->Surface; - uint64_t row = (uint64_t)ctx->SourceY + (uint64_t)(uint32_t)rowIndex; - uint64_t offset = row * (uint64_t)surface->PitchBytes + (uint64_t)ctx->SourceX * sizeof(uint32_t); - uint64_t bytes = (uint64_t)pixelCount * sizeof(uint32_t); - if (offset > (uint64_t)surface->PayloadSize || bytes > (uint64_t)surface->PayloadSize - offset) - return NTLSTATUS_INVALID_PARAMETER; - - uint8_t* dst = (uint8_t*)scratch; - while (bytes) { - size_t frameIndex = (size_t)(offset / PAGE_SIZE); - size_t pageOffset = (size_t)(offset & (PAGE_SIZE - 1)); - if (frameIndex >= surface->PageCount || !surface->Frames[frameIndex]) - return NTLSTATUS_INVALID_PARAMETER; - - size_t chunk = PAGE_SIZE - pageOffset; - if (chunk > bytes) - chunk = (size_t)bytes; - - const uint8_t* src = (const uint8_t*)MmPfnToVirtual(surface->Frames[frameIndex]); - if (!src) - return NTLSTATUS_INVALID_PARAMETER; - memcpy(dst, src + pageOffset, chunk); - - dst += chunk; - offset += chunk; - bytes -= chunk; - } - - return NTLSTATUS_SUCCESS; + return MmVadCreateSharedViewFromFrames(surface->Owner, surface->Frames, surface->PageCount, + surface->PayloadSize, + PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, + MM_VAD_ZERO, MmVadShared, &surface->UserBase); } // Implements the syscall Init operation. @@ -513,8 +489,9 @@ static uint64_t SyscallExitThread(uint64_t a0, uint64_t a1, uint64_t a2, } // SYS_CREATE_THREAD a0=*SYS_CREATE_THREAD_REQUEST. -// Creates a runnable user thread and optionally returns both its waitable -// handle and diagnostic TID. The handle is the synchronization identity. +// Creates a user thread and optionally returns both its waitable handle and +// diagnostic TID. Suspended creation leaves the thread off every ready queue +// until SYS_RESUME_THREAD decrements its initial suspend count. static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; @@ -527,8 +504,11 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, if (!proc) return NTLSTATUS_FAILURE; if (!req.Entry) return NTLSTATUS_INVALID_PARAMETER; if (req.Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; - if (req.Flags) return NTLSTATUS_INVALID_PARAMETER; + if (req.Flags & ~SYS_CREATE_THREAD_FLAG_SUSPENDED) + return NTLSTATUS_INVALID_PARAMETER; if (!req.ThreadHandleOut && !req.ThreadIdOut) return NTLSTATUS_INVALID_PARAMETER; + if ((req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) && !req.ThreadHandleOut) + return NTLSTATUS_INVALID_PARAMETER; s = MmProbeUserRead((const void*)req.Entry, 1); if (!KSUCCESS(s)) return s; @@ -542,16 +522,22 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, } THREAD* newt = NULL; + KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(req.Priority); + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + threadParameters.Flags |= KE_THREAD_CREATE_FLAG_SUSPENDED; s = KeCreateThread(proc, req.Entry, req.Argument, - (THREAD_PRIORITY)req.Priority, &newt); + &threadParameters, &newt); if (!KSUCCESS(s)) return s; uint32_t threadHandle = 0; if (req.ThreadHandleOut) { s = KeAllocateHandleWithRights(proc, &newt->Header, - HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY, + HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY | + HANDLE_RIGHT_RESUME, &threadHandle); if (!KSUCCESS(s)) { + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + (void)KeCancelSuspendedThread(newt); ObDereferenceObject(&newt->Header); return s; } @@ -561,6 +547,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadHandleOut, threadHandle); if (!KSUCCESS(s)) { KeCloseHandle(proc, threadHandle); + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + (void)KeCancelSuspendedThread(newt); ObDereferenceObject(&newt->Header); return s; } @@ -570,6 +558,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadIdOut, newt->ThreadId); if (!KSUCCESS(s)) { if (threadHandle) KeCloseHandle(proc, threadHandle); + if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) + (void)KeCancelSuspendedThread(newt); ObDereferenceObject(&newt->Header); return s; } @@ -579,6 +569,35 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2, return NTLSTATUS_SUCCESS; } +// SYS_RESUME_THREAD a0=threadHandle, a1=previousSuspendCountOut. +static uint64_t SyscallResumeThread(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a2; (void)a3; (void)a4; (void)a5; + if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER; + + KSTATUS status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(uint32_t)); + if (!KSUCCESS(status)) return status; + + PROCESS* process = KeGetCurrentProcess(); + if (!process) return NTLSTATUS_FAILURE; + + OBJECT* object = KeLookupHandleWithRights(process, (uint32_t)a0, + HANDLE_RIGHT_RESUME); + if (!object) return NTLSTATUS_INVALID_HANDLE; + if (object->Type != ObjectTypeThread) { + ObDereferenceObject(object); + return NTLSTATUS_INVALID_HANDLE; + } + + THREAD* thread = CONTAINING_RECORD(object, THREAD, Header); + uint32_t previousSuspendCount = 0; + status = KeResumeThread(thread, &previousSuspendCount); + ObDereferenceObject(object); + if (!KSUCCESS(status)) return status; + + return KiCopyU32ToUser(a1, previousSuspendCount); +} + static void KiFinishAlertableUserWait(uint8_t alertable) { if (!alertable) return; @@ -619,6 +638,9 @@ static KSTATUS KiHandleToWaitEvent(PROCESS* proc, uint32_t handle, EVENT** event case ObjectTypeDisplayClock: event = KeDisplayClockWaitEvent((DISPLAY_CLOCK_OBJECT*)obj); break; + case ObjectTypeAudioStream: + event = KeAudioStreamWaitEvent((KE_AUDIO_STREAM*)obj); + break; default: ObDereferenceObject(obj); return NTLSTATUS_NOT_SUPPORTED; @@ -656,28 +678,14 @@ static uint64_t SyscallWaitSingle(uint64_t a0, uint64_t a1, uint64_t a2, if (obj->Type == ObjectTypeMutex) { if (alertable) s = NTLSTATUS_NOT_SUPPORTED; - else if (timeoutMs == 0) { - s = KeTryAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header)); - if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT; - } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) { - s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header)); - } else { - s = NTLSTATUS_NOT_SUPPORTED; - } + else s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header), timeoutMs); ObDereferenceObject(obj); return s; } if (obj->Type == ObjectTypeSemaphore) { if (alertable) s = NTLSTATUS_NOT_SUPPORTED; - else if (timeoutMs == 0) { - s = KeTrySemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header)); - if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT; - } else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) { - s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header)); - } else { - s = NTLSTATUS_NOT_SUPPORTED; - } + else s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header), timeoutMs); ObDereferenceObject(obj); return s; } @@ -992,8 +1000,7 @@ static uint64_t SyscallSetThreadPriority(uint64_t a0, uint64_t a1, uint64_t a2, if (a0 >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; THREAD* t = KeGetCurrentThread(); if (!t) return NTLSTATUS_FAILURE; - t->Priority = (THREAD_PRIORITY)a0; - return NTLSTATUS_SUCCESS; + return KiSchedulerSetNormalPriority(t, (THREAD_PRIORITY)a0); } // SYS_VM_* page-granular user virtual memory. @@ -1163,7 +1170,8 @@ static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex, HANDLE_RIGHT_WAIT); if (!mx) return NTLSTATUS_NOT_FOUND; - KSTATUS s = KeTryAcquireMutex(mx); + KSTATUS s = KeAcquireMutex(mx, 0); + if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE; ObDereferenceObject(&mx->Header); return s; } @@ -1204,7 +1212,8 @@ static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore, HANDLE_RIGHT_WAIT); if (!sem) return NTLSTATUS_NOT_FOUND; - KSTATUS s = KeTrySemaphore(sem); + KSTATUS s = KeWaitSemaphore(sem, 0); + if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE; ObDereferenceObject(&sem->Header); return s; } @@ -1672,7 +1681,7 @@ static void* KiAllocateSyscallIoBuffer(size_t requested, size_t* actualOut) { if (!requested) return NULL; size_t chunk = requested < SYSCALL_FILE_IO_CHUNK_BYTES ? requested : SYSCALL_FILE_IO_CHUNK_BYTES; - for (;;) { + while (true) { void* buf = MmAllocatePoolTag(chunk, MM_POOL_TAG('s', 'i', 'o', ' ')); if (buf) { if (actualOut) *actualOut = chunk; @@ -2275,9 +2284,8 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2, surface->PageCount = pageCount; surface->Frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS), MM_POOL_TAG('s', 'f', 'r', 'm')); - surface->RowScratch = (uint32_t*)MmAllocatePoolTag(rowBytes, MM_POOL_TAG('s', 'r', 'o', 'w')); s = KeMutexCreate(&surface->Lock); - if (!surface->Frames || !surface->RowScratch || !KSUCCESS(s)) { + if (!surface->Frames || !KSUCCESS(s)) { ObDereferenceObject(&surface->Header); return NTLSTATUS_NO_MEMORY; } @@ -2295,6 +2303,14 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2, memset(kva, 0, PAGE_SIZE); } + s = MmCreateKernelViewFromFrames(surface->Frames, surface->PageCount, + &surface->KernelView); + if (!KSUCCESS(s)) { + ObDereferenceObject(&surface->Header); + return s; + } + surface->KernelBase = MmKernelViewBase(surface->KernelView); + uint32_t handle = 0; s = KiAllocHandleForCurrentWithRights(&surface->Header, HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE | HANDLE_RIGHT_QUERY, &handle); if (!KSUCCESS(s)) @@ -2327,18 +2343,13 @@ static uint64_t SyscallMapSurface(uint64_t a0, uint64_t a1, uint64_t a2, return NTLSTATUS_ACCESS_DENIED; } - KSTATUS s = KeAcquireMutex(surface->Lock); - if (!KSUCCESS(s)) { + KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE); + if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s)) { ObDereferenceObject(&surface->Header); return s; } - if (!surface->UserBase) { - s = MmVadCreateSharedViewFromFrames(current, surface->Frames, surface->PageCount, - surface->PayloadSize, - PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, - MM_VAD_ZERO, MmVadShared, &surface->UserBase); - } + s = KiSurfaceEnsureViewLocked(surface); void* userBase = surface->UserBase; uint32_t pitchBytes = surface->PitchBytes; @@ -2374,8 +2385,8 @@ static uint64_t SyscallQuerySurface(uint64_t a0, uint64_t a1, uint64_t a2, SYS_SURFACE_DESC desc; memset(&desc, 0, sizeof(desc)); - KSTATUS s = KeAcquireMutex(surface->Lock); - if (KSUCCESS(s)) { + KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE); + if (KE_MUTEX_ACQUIRE_SUCCEEDED(s)) { desc.Width = surface->Width; desc.Height = surface->Height; desc.PitchBytes = surface->PitchBytes; @@ -2475,20 +2486,21 @@ static uint64_t SyscallPresentSurface(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t rowsCopied = 0; uint64_t bytesCopied = 0; - KSTATUS s = NTLSTATUS_SUCCESS; - for (uint32_t i = 0; i < rectCount; ++i) { + KSTATUS s = surface->KernelBase ? NTLSTATUS_SUCCESS : NTLSTATUS_DEVICE_ERROR; + const uint8_t* sourceBase = (const uint8_t*)surface->KernelBase; + + for (uint32_t i = 0; i < rectCount && KSUCCESS(s); ++i) { DISPLAY_RECT clipped; if (!KiClipSurfaceRect(surface, &rects[i], &clipped)) { emptyClips++; continue; } - SURFACE_ROW_CONTEXT ctx; - ctx.Surface = surface; - ctx.SourceX = (uint32_t)clipped.X; - ctx.SourceY = (uint32_t)clipped.Y; - s = KeDisplayBlitRows(clipped.X, clipped.Y, clipped.W, clipped.H, - KiSurfaceCopyRow, &ctx, surface->RowScratch); + const uint32_t* source = (const uint32_t*)(sourceBase + + (size_t)(uint32_t)clipped.Y * surface->PitchBytes + + (size_t)(uint32_t)clipped.X * sizeof(uint32_t)); + s = KeDisplayPresentPitched(clipped.X, clipped.Y, clipped.W, clipped.H, + source, surface->PitchBytes); if (!KSUCCESS(s)) break; @@ -2657,7 +2669,7 @@ static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; if (!a0) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { char ch = KeInputReadChar(); if (ch) return CopyToUser((void*)(uintptr_t)a0, &ch, sizeof(ch)); @@ -2674,7 +2686,7 @@ static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2, (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; if (!a0) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { INPUT_KEY key = KeInputReadKey(); if (key != INPUT_KEY_NONE) return CopyToUser((void*)(uintptr_t)a0, &key, sizeof(key)); @@ -2722,7 +2734,7 @@ static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2, if (!KSUCCESS(s)) return s; INPUT_KEY_RECORD record; - for (;;) { + while (true) { bool available = queue ? KeInputQueueReadEvent(queue, &record) : KeInputReadEvent(&record); if (available) { @@ -2759,7 +2771,7 @@ static uint64_t SyscallKbPeekInputEvent(uint64_t a0, uint64_t a1, uint64_t a2, if (!KSUCCESS(s)) return s; INPUT_KEY_RECORD record; - for (;;) { + while (true) { bool available = queue ? KeInputQueuePeekEvent(queue, &record) : KeInputPeekEvent(&record); if (available) { @@ -2995,7 +3007,8 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2, THREAD* thread; uintptr_t entryPoint = LdrGetEntryPoint(imageBase); - s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread); + const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + s = KeCreateThread(proc, entryPoint, 0, &threadParameters, &thread); if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint); MmVadDebugDumpProcess(proc); @@ -3070,7 +3083,7 @@ static uint64_t SyscallGetProcessExitCode(uint64_t a0, uint64_t a1, uint64_t a2, } // SYS_WRITE_CONSOLE (60) a0=*buf a1=len -// Writes user console text in white; distinct from green SYS_PRINT_DEBUG. +// Writes user console text and completes display delivery before returning. static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; @@ -3092,7 +3105,7 @@ static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2, KePrintConsole(chunk, todo); offset += todo; } - return NTLSTATUS_SUCCESS; + return KeDisplayFlushConsoleOutput(); } // SYS_OPEN_DIR (54) a0=*dirPath a1=*handleOut. @@ -3309,7 +3322,7 @@ void KeNamedQueueUnpublishHandle(PROCESS* process, uint32_t handle, OBJECT* obje if (!process || !object || object->Type != ObjectTypeQueue) return; - for (;;) { + while (true) { QUEUE* queue = (QUEUE*)0; uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock); for (int i = 0; i < NAMED_QUEUE_MAX; i++) { @@ -4473,7 +4486,7 @@ static uint64_t SyscallPoll(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t start = gHal ? gHal->Timer.QueryTicks() : 0; size_t ready = 0; - for (;;) { + while (true) { ready = 0; for (size_t i = 0; i < count; i++) { fds[i].ReturnedEvents = 0; @@ -4982,6 +4995,7 @@ static uint64_t SyscallQueryDisplayInfo(uint64_t a0, uint64_t a1, uint64_t a2, out.Mode = ki.Mode; out.Caps = ki.Caps; out.DeviceCount = ki.DeviceCount; + out.Flags = ki.Flags; out.ManagerStats = ki.ManagerStats; out.DeviceStats = ki.DeviceStats; @@ -5012,6 +5026,12 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, us.CpuBurstDemotions = ks.CpuBurstDemotions; us.SmtSiblingDeferrals = ks.SmtSiblingDeferrals; us.SmtPrimaryPokes = ks.SmtPrimaryPokes; + us.RealtimeDispatches = ks.RealtimeDispatches; + us.RealtimePreemptions = ks.RealtimePreemptions; + us.PriorityInheritanceBoosts = ks.PriorityInheritanceBoosts; + us.PriorityInheritanceDrops = ks.PriorityInheritanceDrops; + us.PriorityInheritancePropagations = ks.PriorityInheritancePropagations; + us.PriorityInheritanceMaxDepth = ks.PriorityInheritanceMaxDepth; return KiReturnKernelStats(a1, (size_t)a2, a3, &us, sizeof(us)); } @@ -5199,19 +5219,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, um.DropFailures = km.DropFailures; um.StandbyTransitions = km.StandbyTransitions; um.StandbyHits = km.StandbyHits; - um.ReadAheadRequests = km.ReadAheadRequests; - um.ReadAheadPagesRequested = km.ReadAheadPagesRequested; - um.ReadAheadPagesQueued = km.ReadAheadPagesQueued; - um.ReadAheadPagesLoaded = km.ReadAheadPagesLoaded; - um.ReadAheadPagesSkipped = km.ReadAheadPagesSkipped; - um.ReadAheadFailures = km.ReadAheadFailures; - um.ReadAheadPagesThrottled = km.ReadAheadPagesThrottled; - um.MaxReadAheadPagesPerRequest = km.MaxReadAheadPagesPerRequest; - um.BatchReads = km.BatchReads; - um.BatchPagesRequested = km.BatchPagesRequested; - um.BatchPagesLoaded = km.BatchPagesLoaded; - um.MaxBatchPagesRequested = km.MaxBatchPagesRequested; - um.MaxBatchPagesLoaded = km.MaxBatchPagesLoaded; um.ResidentPages = km.ResidentPages; um.ActivePages = km.ActivePages; um.StandbyPages = km.StandbyPages; @@ -5285,17 +5292,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, um.WriteSectors = km.WriteSectors; um.MaxReadSectors = km.MaxReadSectors; um.MaxWriteSectors = km.MaxWriteSectors; - um.ReadaheadHits = km.ReadaheadHits; - um.ReadaheadMisses = km.ReadaheadMisses; - um.ReadaheadStores = km.ReadaheadStores; - um.ReadaheadInvalidations = km.ReadaheadInvalidations; - um.ReadaheadBytesServed = km.ReadaheadBytesServed; - um.ReadaheadSectorsLoaded = km.ReadaheadSectorsLoaded; - um.ReadaheadSequentialDetections = km.ReadaheadSequentialDetections; - um.ReadaheadSuppressed = km.ReadaheadSuppressed; - um.ReadaheadAllocFailures = km.ReadaheadAllocFailures; - um.ReadaheadReadFailures = km.ReadaheadReadFailures; - um.MaxReadaheadSectors = km.MaxReadaheadSectors; return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um)); } @@ -5374,6 +5370,466 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2, } } +// Converts one public audio format into the kernel mixer format. +static KSTATUS KiImportAudioFormat(const SYS_AUDIO_FORMAT* source, IO_AUDIO_FORMAT* destination) { + if (!source || !destination) + return NTLSTATUS_INVALID_PARAMETER; + if (source->SampleFormat != SysAudioSampleS16 || + (source->ChannelCount != 1 && source->ChannelCount != 2) || + (source->SampleRate != 11025u && source->SampleRate != 22050u && + source->SampleRate != 44100u && source->SampleRate != 48000u)) + return NTLSTATUS_NOT_SUPPORTED; + + destination->SampleRate = source->SampleRate; + destination->ChannelCount = source->ChannelCount; + destination->SampleFormat = IoAudioSampleS16; + return NTLSTATUS_SUCCESS; +} + +// Converts one kernel audio format into the public syscall representation. +static SYS_AUDIO_FORMAT KiExportAudioFormat(const IO_AUDIO_FORMAT* source) { + SYS_AUDIO_FORMAT destination; + destination.SampleRate = source->SampleRate; + destination.ChannelCount = source->ChannelCount; + destination.SampleFormat = source->SampleFormat == IoAudioSampleS16 + ? SysAudioSampleS16 + : 0; + return destination; +} + +// Converts one normalized kernel gain into unsigned Q16. +static uint32_t KiExportAudioGainQ16(float gain) { + if (gain <= 0.0f) return 0; + if (gain >= 1.0f) return SYS_AUDIO_GAIN_Q16_UNITY; + return (uint32_t)(gain * (float)SYS_AUDIO_GAIN_Q16_UNITY + 0.5f); +} + +// Converts one normalized kernel pan into signed Q16. +static int32_t KiExportAudioPanQ16(float pan) { + if (pan <= -1.0f) return SYS_AUDIO_PAN_Q16_LEFT; + if (pan >= 1.0f) return SYS_AUDIO_PAN_Q16_RIGHT; + float scaled = pan * (float)SYS_AUDIO_GAIN_Q16_UNITY; + return (int32_t)(scaled + (scaled >= 0.0f ? 0.5f : -0.5f)); +} + +// Converts one public unsigned Q16 gain into the kernel representation. +static float KiImportAudioGainQ16(uint32_t gainQ16) { + return (float)gainQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY; +} + +// Converts one public signed Q16 pan into the kernel representation. +static float KiImportAudioPanQ16(int32_t panQ16) { + return (float)panQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY; +} + +// Converts one kernel endpoint kind into its public ABI value. +static uint32_t KiExportAudioEndpointKind(IO_AUDIO_ENDPOINT_KIND kind) { + switch (kind) { + case IoAudioEndpointLineOut: return SysAudioEndpointLineOut; + case IoAudioEndpointSpeakerHeadphone: return SysAudioEndpointSpeakerHeadphone; + case IoAudioEndpointOtherAnalog: return SysAudioEndpointOtherAnalog; + default: return 0; + } +} + +// Converts provider endpoint capabilities into public capability bits. +static uint32_t KiExportAudioEndpointCapabilities(uint32_t capabilities) { + uint32_t result = 0; + if (capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE) + result |= SYS_AUDIO_ENDPOINT_CAP_JACK_SENSE; + if (capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) + result |= SYS_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN; + return result; +} + +// Converts one endpoint lifecycle value into its public ABI value. +static uint32_t KiExportAudioEndpointState(KE_AUDIO_ENDPOINT_STATE state) { + switch (state) { + case KeAudioEndpointStarting: return SysAudioEndpointStarting; + case KeAudioEndpointReady: return SysAudioEndpointReady; + case KeAudioEndpointStopping: return SysAudioEndpointStopping; + case KeAudioEndpointFailed: return SysAudioEndpointFailed; + case KeAudioEndpointRemoved: return SysAudioEndpointRemoved; + default: return 0; + } +} + +// Converts one stream lifecycle value into its public ABI value. +static uint32_t KiExportAudioStreamState(KE_AUDIO_STREAM_STATE state) { + switch (state) { + case KeAudioStreamCreated: return SysAudioStreamCreated; + case KeAudioStreamRunning: return SysAudioStreamRunning; + case KeAudioStreamPaused: return SysAudioStreamPaused; + case KeAudioStreamDraining: return SysAudioStreamDraining; + case KeAudioStreamStopped: return SysAudioStreamStopped; + case KeAudioStreamFailed: return SysAudioStreamFailed; + case KeAudioStreamClosed: + default: + return 0; + } +} + +// Converts public wait flags into the kernel stream condition mask. +static uint32_t KiImportAudioWaitMask(uint32_t mask) { + uint32_t result = 0; + if (mask & SYS_AUDIO_WAIT_WRITABLE) result |= KE_AUDIO_WAIT_WRITABLE; + if (mask & SYS_AUDIO_WAIT_CONSUMED) result |= KE_AUDIO_WAIT_CONSUMED; + if (mask & SYS_AUDIO_WAIT_PRESENTED) result |= KE_AUDIO_WAIT_PRESENTED; + if (mask & SYS_AUDIO_WAIT_STOPPED) result |= KE_AUDIO_WAIT_STOPPED; + if (mask & SYS_AUDIO_WAIT_DRAINED) result |= KE_AUDIO_WAIT_DRAINED; + if (mask & SYS_AUDIO_WAIT_FAILED) result |= KE_AUDIO_WAIT_FAILED; + if (mask & SYS_AUDIO_WAIT_DISCONTINUITY) result |= KE_AUDIO_WAIT_DISCONTINUITY; + return result; +} + +// Looks up one audio stream handle with the requested operation rights. +static KE_AUDIO_STREAM* KiLookupAudioStream(uint32_t handle, uint32_t rights) { + OBJECT* object = KiHandleToObject(handle, ObjectTypeAudioStream, rights); + return object ? (KE_AUDIO_STREAM*)object : (KE_AUDIO_STREAM*)0; +} + +// SYS_AUDIO_ENUMERATE_ENDPOINTS a0=*SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST. +static uint64_t SyscallAudioEnumerateEndpoints(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 || + !request.CountOut || (request.Capacity && !request.Endpoints)) + return NTLSTATUS_INVALID_PARAMETER; + + status = MmProbeUserWrite(request.CountOut, sizeof(*request.CountOut)); + if (!KSUCCESS(status)) return status; + uint32_t copyCapacity = request.Capacity < SYS_AUDIO_MAX_ENDPOINTS + ? request.Capacity + : SYS_AUDIO_MAX_ENDPOINTS; + if (copyCapacity) { + status = MmProbeUserWrite(request.Endpoints, + (size_t)copyCapacity * sizeof(*request.Endpoints)); + if (!KSUCCESS(status)) return status; + } + + KE_AUDIO_ENDPOINT_INFO kernelInfo[KE_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + status = KeAudioQueryEndpoints(kernelInfo, KE_AUDIO_MAX_ENDPOINTS, &count); + if (KFAILED(status)) return status; + + SYS_AUDIO_ENDPOINT_INFO publicInfo[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t copyCount = copyCapacity < count ? copyCapacity : count; + for (uint32_t index = 0; index < copyCount; index++) { + const KE_AUDIO_ENDPOINT_INFO* source = &kernelInfo[index]; + SYS_AUDIO_ENDPOINT_INFO* destination = &publicInfo[index]; + memset(destination, 0, sizeof(*destination)); + destination->Size = sizeof(*destination); + destination->Version = SYS_AUDIO_ABI_VERSION; + destination->EndpointId = source->EndpointId; + destination->Kind = KiExportAudioEndpointKind(source->Kind); + if (source->Available) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE; + if (source->IsDefault) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_DEFAULT; + destination->Capabilities = KiExportAudioEndpointCapabilities(source->Capabilities); + destination->State = KiExportAudioEndpointState(source->State); + destination->HardwareFormat = KiExportAudioFormat(&source->HardwareFormat); + destination->PeriodCount = source->PeriodCount; + destination->FramesPerPeriod = source->FramesPerPeriod; + destination->MasterVolumeQ16 = KiExportAudioGainQ16(source->MasterRequestedVolume); + destination->MasterMuted = source->MasterMuted ? 1u : 0u; + destination->FailureStatus = source->FailureStatus; + } + + if (copyCount) { + status = CopyToUser(request.Endpoints, + publicInfo, + (size_t)copyCount * sizeof(publicInfo[0])); + if (!KSUCCESS(status)) return status; + } + status = CopyToUser(request.CountOut, &count, sizeof(count)); + if (!KSUCCESS(status)) return status; + return request.Capacity < count ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS; +} + +// SYS_AUDIO_OPEN_STREAM a0=*SYS_AUDIO_OPEN_STREAM_REQUEST. +static uint64_t SyscallAudioOpenStream(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_OPEN_STREAM_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 || + !request.EndpointId || !request.StreamHandleOut) + return NTLSTATUS_INVALID_PARAMETER; + + status = MmProbeUserWrite(request.StreamHandleOut, sizeof(*request.StreamHandleOut)); + if (!KSUCCESS(status)) return status; + + IO_AUDIO_FORMAT format; + status = KiImportAudioFormat(&request.Format, &format); + if (!KSUCCESS(status)) return status; + + KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)0; + status = KeAudioCreateStream(request.EndpointId, &format, &stream); + if (!KSUCCESS(status)) return status; + + uint32_t handle = 0; + status = KiAllocHandleForCurrentWithRights((OBJECT*)stream, + HANDLE_RIGHT_WRITE | + HANDLE_RIGHT_WAIT | + HANDLE_RIGHT_QUERY, + &handle); + if (!KSUCCESS(status)) return status; + + status = CopyToUser(request.StreamHandleOut, &handle, sizeof(handle)); + if (!KSUCCESS(status)) { + PROCESS* process = KeGetCurrentProcess(); + if (process) KeCloseHandle(process, handle); + } + return status; +} + +// SYS_AUDIO_WRITE a0=*SYS_AUDIO_WRITE_REQUEST. +static uint64_t SyscallAudioWrite(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_WRITE_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle || + !request.WrittenFramesOut || (request.FrameCount && !request.Frames)) + return NTLSTATUS_INVALID_PARAMETER; + + status = MmProbeUserWrite(request.WrittenFramesOut, sizeof(*request.WrittenFramesOut)); + if (!KSUCCESS(status)) return status; + uint32_t written = 0; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + if (!request.FrameCount) { + ObDereferenceObject((OBJECT*)stream); + return CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + } + + KE_AUDIO_STREAM_INFO info; + status = KeAudioQueryStream(stream, &info); + if (!KSUCCESS(status)) { + ObDereferenceObject((OBJECT*)stream); + return status; + } + if (info.State == KeAudioStreamFailed) { + status = KFAILED(info.FailureStatus) ? info.FailureStatus : NTLSTATUS_DEVICE_ERROR; + ObDereferenceObject((OBJECT*)stream); + (void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + return status; + } + if (info.State == KeAudioStreamDraining) { + ObDereferenceObject((OBJECT*)stream); + (void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + return NTLSTATUS_INVALID_PARAMETER; + } + if (!info.WritableFrames) { + ObDereferenceObject((OBJECT*)stream); + status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + return KSUCCESS(status) ? NTLSTATUS_WOULD_BLOCK : status; + } + + uint32_t frameCount = request.FrameCount < info.WritableFrames + ? request.FrameCount + : info.WritableFrames; + uint32_t frameBytes = (uint32_t)info.Format.ChannelCount * sizeof(int16_t); + size_t byteCount = (size_t)frameCount * frameBytes; + status = MmProbeUserRead(request.Frames, byteCount); + if (!KSUCCESS(status)) { + ObDereferenceObject((OBJECT*)stream); + return status; + } + + void* buffer = MmAllocatePoolTag(byteCount, MM_POOL_TAG('a', 'u', 'w', 'r')); + if (!buffer) { + ObDereferenceObject((OBJECT*)stream); + return NTLSTATUS_NO_MEMORY; + } + status = CopyFromUser(buffer, request.Frames, byteCount); + if (KSUCCESS(status)) + status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); + if (KSUCCESS(status)) + status = KeAudioWriteStream(stream, buffer, frameCount, &written); + MmFreePool(buffer); + ObDereferenceObject((OBJECT*)stream); + if (!KSUCCESS(status)) return status; + return CopyToUser(request.WrittenFramesOut, &written, sizeof(written)); +} + +// SYS_AUDIO_ARM_WAIT a0=*SYS_AUDIO_ARM_WAIT_REQUEST. +static uint64_t SyscallAudioArmWait(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_ARM_WAIT_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 || + !request.StreamHandle || !request.Mask || + (request.Mask & ~SYS_AUDIO_WAIT_VALID_MASK) != 0) + return NTLSTATUS_INVALID_PARAMETER; + + KE_AUDIO_STREAM_WAIT_CONDITION condition; + condition.Mask = KiImportAudioWaitMask(request.Mask); + condition.WritableFrames = request.WritableFrames; + condition.ConsumedFrames = request.ConsumedFrames; + condition.PresentedFrames = request.PresentedFrames; + condition.DrainedGeneration = request.DrainedGeneration; + condition.DiscontinuityGeneration = request.DiscontinuityGeneration; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WAIT); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + status = KeAudioArmStreamWait(stream, &condition); + ObDereferenceObject((OBJECT*)stream); + return status; +} + +// SYS_AUDIO_QUERY_STREAM a0=streamHandle a1=*SYS_AUDIO_STREAM_INFO. +static uint64_t SyscallAudioQueryStream(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a2; (void)a3; (void)a4; (void)a5; + if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER; + + uint32_t userSize = 0; + KSTATUS status = CopyFromUser(&userSize, (const void*)(uintptr_t)a1, sizeof(userSize)); + if (!KSUCCESS(status)) return status; + if (userSize != sizeof(SYS_AUDIO_STREAM_INFO)) + return NTLSTATUS_INVALID_PARAMETER; + status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(SYS_AUDIO_STREAM_INFO)); + if (!KSUCCESS(status)) return status; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream((uint32_t)a0, HANDLE_RIGHT_QUERY); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + KE_AUDIO_STREAM_INFO kernelInfo; + status = KeAudioQueryStream(stream, &kernelInfo); + ObDereferenceObject((OBJECT*)stream); + if (!KSUCCESS(status)) return status; + + SYS_AUDIO_STREAM_INFO publicInfo; + memset(&publicInfo, 0, sizeof(publicInfo)); + publicInfo.Size = sizeof(publicInfo); + publicInfo.Version = SYS_AUDIO_ABI_VERSION; + publicInfo.State = KiExportAudioStreamState(kernelInfo.State); + publicInfo.Format = KiExportAudioFormat(&kernelInfo.Format); + publicInfo.CapacityFrames = kernelInfo.CapacityFrames; + publicInfo.BufferedFrames = kernelInfo.BufferedFrames; + publicInfo.WritableFrames = kernelInfo.WritableFrames; + publicInfo.FailureStatus = kernelInfo.FailureStatus; + publicInfo.FramesSubmitted = kernelInfo.FramesSubmitted; + publicInfo.FramesConsumed = kernelInfo.FramesConsumed; + publicInfo.FramesPresented = kernelInfo.FramesPresented; + publicInfo.FramesDiscarded = kernelInfo.FramesDiscarded; + publicInfo.OutputFramesMixed = kernelInfo.OutputFramesMixed; + publicInfo.SilenceFramesInserted = kernelInfo.SilenceFramesInserted; + publicInfo.Underruns = kernelInfo.Underruns; + publicInfo.DrainedGeneration = kernelInfo.DrainedGeneration; + publicInfo.DiscontinuityGeneration = kernelInfo.DiscontinuityGeneration; + publicInfo.VolumeQ16 = KiExportAudioGainQ16(kernelInfo.Volume); + publicInfo.PanQ16 = KiExportAudioPanQ16(kernelInfo.Pan); + publicInfo.Muted = kernelInfo.Muted; + return CopyToUser((void*)(uintptr_t)a1, &publicInfo, sizeof(publicInfo)); +} + +typedef KSTATUS (*KI_AUDIO_STREAM_OPERATION)(KE_AUDIO_STREAM* stream); + +// Invokes one existing kernel stream operation through a write-capable handle. +static KSTATUS KiCallAudioStreamOperation(uint32_t handle, KI_AUDIO_STREAM_OPERATION operation) { + if (!handle || !operation) return NTLSTATUS_INVALID_PARAMETER; + KE_AUDIO_STREAM* stream = KiLookupAudioStream(handle, HANDLE_RIGHT_WRITE); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + KSTATUS status = operation(stream); + ObDereferenceObject((OBJECT*)stream); + return status; +} + +// SYS_AUDIO_START a0=streamHandle. +static uint64_t SyscallAudioStart(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStartStream); +} + +// SYS_AUDIO_PAUSE a0=streamHandle. +static uint64_t SyscallAudioPause(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioPauseStream); +} + +// SYS_AUDIO_RESUME a0=streamHandle. +static uint64_t SyscallAudioResume(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResumeStream); +} + +// SYS_AUDIO_DRAIN a0=streamHandle. +static uint64_t SyscallAudioDrain(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioDrainStream); +} + +// SYS_AUDIO_STOP a0=streamHandle. +static uint64_t SyscallAudioStop(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStopStream); +} + +// SYS_AUDIO_RESET a0=streamHandle. +static uint64_t SyscallAudioReset(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResetStream); +} + +// SYS_AUDIO_SET_VOLUME a0=*SYS_AUDIO_SET_VOLUME_REQUEST. +static uint64_t SyscallAudioSetVolume(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_SET_VOLUME_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle || + request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY || + request.PanQ16 < SYS_AUDIO_PAN_Q16_LEFT || + request.PanQ16 > SYS_AUDIO_PAN_Q16_RIGHT) + return NTLSTATUS_INVALID_PARAMETER; + + KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE); + if (!stream) return NTLSTATUS_INVALID_HANDLE; + status = KeAudioSetStreamVolume(stream, + KiImportAudioGainQ16(request.VolumeQ16), + KiImportAudioPanQ16(request.PanQ16), + request.Muted != 0); + ObDereferenceObject((OBJECT*)stream); + return status; +} + +// SYS_AUDIO_SET_MASTER_VOLUME a0=*SYS_AUDIO_SET_MASTER_VOLUME_REQUEST. +static uint64_t SyscallAudioSetMasterVolume(uint64_t a0, uint64_t a1, uint64_t a2, + uint64_t a3, uint64_t a4, uint64_t a5) { + (void)a1; (void)a2; (void)a3; (void)a4; (void)a5; + + SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request; + KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request)); + if (!KSUCCESS(status)) return status; + if (request.Size != sizeof(request) || request.Flags != 0 || !request.EndpointId || + request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY) + return NTLSTATUS_INVALID_PARAMETER; + + return KeAudioSetMasterVolume(request.EndpointId, + KiImportAudioGainQ16(request.VolumeQ16), + request.Muted != 0); +} + // Dispatch table keyed by SYS_* constants. typedef uint64_t (*SYSCALL_HANDLER)(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); @@ -5389,6 +5845,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { [SYS_GET_PROCESS_LIST] = SyscallGetProcessList, [SYS_GET_PROCESS_PID] = SyscallGetProcessPid, [SYS_CREATE_THREAD] = SyscallCreateThread, + [SYS_RESUME_THREAD] = SyscallResumeThread, [SYS_GET_THREAD_LIST] = SyscallGetThreadList, [SYS_LOAD_MODULE] = SyscallLoadModule, [SYS_UNLOAD_MODULE] = SyscallUnloadModule, @@ -5480,6 +5937,21 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = { [SYS_QUERY_SURFACE] = SyscallQuerySurface, [SYS_PRESENT_SURFACE] = SyscallPresentSurface, + // Native render audio. + [SYS_AUDIO_ENUMERATE_ENDPOINTS] = SyscallAudioEnumerateEndpoints, + [SYS_AUDIO_OPEN_STREAM] = SyscallAudioOpenStream, + [SYS_AUDIO_WRITE] = SyscallAudioWrite, + [SYS_AUDIO_ARM_WAIT] = SyscallAudioArmWait, + [SYS_AUDIO_QUERY_STREAM] = SyscallAudioQueryStream, + [SYS_AUDIO_START] = SyscallAudioStart, + [SYS_AUDIO_PAUSE] = SyscallAudioPause, + [SYS_AUDIO_RESUME] = SyscallAudioResume, + [SYS_AUDIO_DRAIN] = SyscallAudioDrain, + [SYS_AUDIO_STOP] = SyscallAudioStop, + [SYS_AUDIO_RESET] = SyscallAudioReset, + [SYS_AUDIO_SET_VOLUME] = SyscallAudioSetVolume, + [SYS_AUDIO_SET_MASTER_VOLUME] = SyscallAudioSetMasterVolume, + // Input. [SYS_POLL_KEYBOARD] = SyscallKbPoll, [SYS_IS_KEY_DOWN] = SyscallKbIsKeyDown, @@ -5532,6 +6004,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1, THREAD* self = KeGetCurrentThread(); PROCESS* owner = self ? self->Owner : (PROCESS*)0; if (owner && KeIsProcessTerminating(owner) && + !KeThreadTerminationDeferred(self) && nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) { int32_t exitCode = owner->ExitCode; KeExitThread(self, exitCode); @@ -5547,6 +6020,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1, // has been marked terminating. owner = self ? self->Owner : (PROCESS*)0; if (owner && KeIsProcessTerminating(owner) && + !KeThreadTerminationDeferred(self) && nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) { int32_t exitCode = owner->ExitCode; KeExitThread(self, exitCode); diff --git a/code/kernel/ke_system_root.c b/code/kernel/ke_system_root.c index eb7bd14..ab9c2ad 100644 --- a/code/kernel/ke_system_root.c +++ b/code/kernel/ke_system_root.c @@ -7,10 +7,15 @@ #include #include #include +#include static uint8_t gSystemRootDrive; static bool gSystemRootReady; +#define KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS 30000ull +#define KE_SYSTEM_ROOT_WAIT_SLICE_MS 1000ull +#define KE_SYSTEM_ROOT_LOG_INTERVAL_MS 5000ull + // Builds a drive-qualified path without depending on any process CWD. static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) { if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES) @@ -50,23 +55,76 @@ static bool KiDriveLooksLikeSystemRoot(uint8_t drive) { return true; } -// Selects the mounted volume that contains the kernel/system tree. -KSTATUS KeInitializeSystemRootVolume(void) { +// Returns the current scheduler time in milliseconds. +static uint64_t KiSystemRootNowMilliseconds(void) { + KE_SCHED_STATS stats; + KeGetSchedulerStats(&stats); + return stats.Ticks * SCHED_TICK_MS; +} + +// Scans the currently mounted volumes for the system tree. +static KSTATUS KiTrySelectSystemRootVolume(uint32_t* mountedOut) { + uint32_t mounted = 0; for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) { if (!IoFilesystemPresent(drive)) continue; + mounted++; if (!KiDriveLooksLikeSystemRoot(drive)) continue; gSystemRootDrive = drive; gSystemRootReady = true; + if (mountedOut) + *mountedOut = mounted; DbgPrint("System root: drive=%u\n", drive); return NTLSTATUS_SUCCESS; } + if (mountedOut) + *mountedOut = mounted; return NTLSTATUS_NOT_FOUND; } +// Waits for the mounted boot volume instead of racing asynchronous bus discovery. +KSTATUS KeInitializeSystemRootVolume(void) { + uint64_t start = KiSystemRootNowMilliseconds(); + uint64_t deadline = start + KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS; + uint64_t nextLog = start; + + for (;;) { + uint64_t generation = IoGetVolumeNamespaceGeneration(); + uint32_t mounted = 0; + KSTATUS status = KiTrySelectSystemRootVolume(&mounted); + if (KSUCCESS(status)) + return status; + + uint64_t now = KiSystemRootNowMilliseconds(); + if (now >= deadline) { + DbgPrint("System root: discovery timed out after %llu ms; mounted=%u generation=%llu\n", + (unsigned long long)KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS, + mounted, + (unsigned long long)generation); + return NTLSTATUS_TIMEOUT; + } + + if (now >= nextLog) { + DbgPrint("System root: waiting for boot volume; mounted=%u generation=%llu remaining=%llu ms\n", + mounted, + (unsigned long long)generation, + (unsigned long long)(deadline - now)); + nextLog = now + KE_SYSTEM_ROOT_LOG_INTERVAL_MS; + } + + uint64_t remaining = deadline - now; + uint64_t wait = remaining < KE_SYSTEM_ROOT_WAIT_SLICE_MS + ? remaining + : KE_SYSTEM_ROOT_WAIT_SLICE_MS; + KSTATUS waitStatus = IoWaitForVolumeNamespaceChange(generation, wait); + if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT) + return waitStatus; + } +} + // Returns whether the system root volume has been selected. bool KeSystemRootReady(void) { return gSystemRootReady; diff --git a/code/kernel/ke_thread.c b/code/kernel/ke_thread.c index 47e5d39..a5f7a2a 100644 --- a/code/kernel/ke_thread.c +++ b/code/kernel/ke_thread.c @@ -27,14 +27,105 @@ _Static_assert(offsetof(type, field) == (expected), \ #type "." #field " offset must match assembly") -STATIC_ASSERT_OFFSET(THREAD, SavedContext, 96); -STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 112); -STATIC_ASSERT_OFFSET(THREAD, Teb, 120); -STATIC_ASSERT_OFFSET(THREAD, FpuContext, 176); +STATIC_ASSERT_OFFSET(THREAD, SavedContext, 112); +STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 128); +STATIC_ASSERT_OFFSET(THREAD, Teb, 136); +STATIC_ASSERT_OFFSET(THREAD, FpuContext, 192); static uint32_t gNextThreadId = 1; extern void KeThreadEntry(void); +// Prevents a newly scheduled user thread from entering a terminating process. +void KePrepareUserThreadStart(void) { + THREAD* thread = KeGetCurrentThread(); + PROCESS* owner = thread ? thread->Owner : (PROCESS*)0; + if (!thread || !owner || !KeIsProcessTerminating(owner)) + return; + + KeExitThread(thread, owner->ExitCode); + while (true) { + KeYield(); + } +} + +// Finalizes one dead thread's completion, user memory, and process membership. +void KeFinalizeThreadExit(THREAD* thread) { + if (!thread) + return; + if (thread->State != ThreadDead) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread finalized before death: thread=0x%016llX tid=%u state=%u\n", + (uint64_t)thread, + thread->ThreadId, + (unsigned)thread->State); + } + + // Thread completion is manual-reset and idempotent. Publish it before + // process-owned handles can be released by the final live-thread exit. + if (thread->ExitEvent) + KeSignalEvent(thread->ExitEvent); + + PROCESS* owner = thread->Owner; + if (!owner) + return; + + // Every exiting thread releases its own user allocations before leaving + // the live-thread set. Therefore the final detach can destroy the remaining + // process VADs without racing a later THREAD destructor. + if (!owner->ResourcesReleased) { + if (thread->Teb && thread->TebSize) + MmFreeUserMemoryForProcess(owner, thread->Teb); + if (thread->UserStackBase) + MmFreeUserMemoryForProcess(owner, thread->UserStackBase); + } + thread->Teb = (void*)0; + thread->TebSize = 0; + thread->UserStackBase = (void*)0; + + uint32_t remaining = 0; + int detached = 0; + + uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock); + int fwdDetached = thread->ProcessLink.Fwd == &thread->ProcessLink; + int bwdDetached = thread->ProcessLink.Bwd == &thread->ProcessLink; + if (fwdDetached != bwdDetached) { + KeReleaseSpinlock(&owner->HandleLock, flags); + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread process linkage corrupt: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->ProcessLink.Fwd, + (uint64_t)thread->ProcessLink.Bwd); + } + + if (!fwdDetached) { + if (owner->ThreadCount == 0) { + KeReleaseSpinlock(&owner->HandleLock, flags); + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread process count underflow: thread=0x%016llX tid=%u pid=%u\n", + (uint64_t)thread, + thread->ThreadId, + owner->ProcessId); + } + + KeListRemoveChecked(&thread->ProcessLink); + owner->ThreadCount--; + detached = 1; + } + remaining = owner->ThreadCount; + KeReleaseSpinlock(&owner->HandleLock, flags); + + if (!detached) + return; + + if (remaining == 0) { + // ThreadCount represents live process membership, not retained thread + // handles. Dead THREAD objects may remain waitable after process exit. + _InterlockedExchange(&owner->Terminating, 1); + KeCompleteProcessExitIfLastThread(owner); + } +} + // Implements the thread Destroy operation. static void KiThreadDestroy(OBJECT* obj) { THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header); @@ -44,6 +135,26 @@ static void KiThreadDestroy(OBJECT* obj) { // wait syscall to run normal cleanup. KeRundownThreadWait(thread, NTLSTATUS_ALERTED); + if (thread->WaitingMutex || !LIST_IS_EMPTY(&thread->OwnedMutexList)) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed with mutex state: thread=0x%016llX tid=%u waiting=0x%016llX ownedEmpty=%u\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->WaitingMutex, + LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u); + } + if (thread->TimeoutArmed || + thread->TimeoutLink.Fwd != &thread->TimeoutLink || + thread->TimeoutLink.Bwd != &thread->TimeoutLink) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed with timeout state: thread=0x%016llX tid=%u armed=%u link=0x%016llX/0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (unsigned)thread->TimeoutArmed, + (uint64_t)thread->TimeoutLink.Fwd, + (uint64_t)thread->TimeoutLink.Bwd); + } + KiRundownApcState(thread); if (thread->ExitEvent) { @@ -51,39 +162,28 @@ static void KiThreadDestroy(OBJECT* obj) { thread->ExitEvent = (EVENT*)0; } - PROCESS* ownerForVadList = thread->Owner; - if (ownerForVadList) { - if (thread->Teb && thread->TebSize) { - MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb); - thread->Teb = (void*)0; - thread->TebSize = 0; - } + if (thread->Teb || thread->TebSize || thread->UserStackBase) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed with user memory: thread=0x%016llX tid=%u teb=0x%016llX tebSize=%llu stack=0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->Teb, + (uint64_t)thread->TebSize, + (uint64_t)thread->UserStackBase); + } - if (thread->UserStackBase) { - MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase); - thread->UserStackBase = (void *)0; - } + if (thread->ProcessLink.Fwd != &thread->ProcessLink || + thread->ProcessLink.Bwd != &thread->ProcessLink) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread destroyed while still process-linked: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n", + (uint64_t)thread, + thread->ThreadId, + (uint64_t)thread->ProcessLink.Fwd, + (uint64_t)thread->ProcessLink.Bwd); } if (thread->Owner) { - PROCESS* owner = thread->Owner; - uint32_t remaining = 0; - uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock); - KeListRemoveChecked(&thread->ProcessLink); - if (owner->ThreadCount) owner->ThreadCount--; - remaining = owner->ThreadCount; - KeReleaseSpinlock(&owner->HandleLock, flags); - - if (remaining == 0) { - // The last thread leaving a process is process exit even if user - // code called ExitThread instead of ExitProcess. KeExitProcess / - // KeTerminateProcess set Terminating earlier; this covers the last - // thread voluntarily exiting without an explicit process-exit call. - _InterlockedExchange(&owner->Terminating, 1); - KeCompleteProcessExitIfLastThread(owner); - } - - ObDereferenceObject(&owner->Header); + ObDereferenceObject(&thread->Owner->Header); thread->Owner = (PROCESS *)0; } @@ -94,35 +194,51 @@ static void KiThreadDestroy(OBJECT* obj) { } __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out) { - if (!owner || !out) return NTLSTATUS_INVALID_PARAMETER; - if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) { + if (!owner || !parameters || !out) return NTLSTATUS_INVALID_PARAMETER; + if (parameters->SchedulingClass != KeSchedulingNormal || + parameters->Priority >= PriorityCount || + (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED)) + return NTLSTATUS_INVALID_PARAMETER; + + uint8_t initiallySuspended = + (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u; THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); if (!thread) return NTLSTATUS_NO_MEMORY; thread->Header.Destroy = KiThreadDestroy; thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1); - thread->State = ThreadReady; - thread->Priority = priority; - thread->QuantaConsumed = 0; + thread->State = initiallySuspended ? ThreadSuspended : ThreadReady; + thread->BaseSchedulingClass = parameters->SchedulingClass; + thread->BasePriority = parameters->Priority; + thread->EffectiveSchedulingClass = parameters->SchedulingClass; + thread->EffectivePriority = parameters->Priority; + thread->QuantaConsumed = 0; thread->QuantumTicks = 0; thread->QuantumRemaining = 0; thread->CpuBurstTicks = 0; thread->LastRunTicks = 0; thread->RunningCpu = KE_INVALID_CPU; + thread->SuspendCount = initiallySuspended ? 1u : 0u; + LIST_INIT(&thread->OwnedMutexList); + thread->WaitingMutex = (MUTEX*)0; thread->Queued = 0; thread->IsIdleThread = 0; thread->SchedulerFlags = 0; thread->LastCpu = KE_INVALID_CPU; - thread->AffinityMask = KeGetDefaultThreadAffinityMask(); + thread->AffinityMask = parameters->AffinityMask + ? parameters->AffinityMask + : KeGetDefaultThreadAffinityMask(); LIST_INIT(&thread->ProcessLink); LIST_INIT(&thread->SchedulerLink); thread->WaitObject = (OBJECT *)0; thread->WaitStatus = NTLSTATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; + LIST_INIT(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + thread->TimeoutTick = 0; thread->TimeoutStatus = NTLSTATUS_SUCCESS; thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); @@ -131,6 +247,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui thread->UserExceptionNoncontinuableMask = 0; thread->UserExceptionFrameUserAddress = 0; memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus)); + thread->TerminationDeferralCount = 0; void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k')); if (!kstack) { @@ -245,24 +362,58 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui ObReferenceObject(&thread->Header); *out = thread; - KeReadyThread(thread); + if (!initiallySuspended) + KeReadyThread(thread); return NTLSTATUS_SUCCESS; } + +// Enters a region that must retain the current thread during process termination. +void KeEnterTerminationDeferral(void) { + KeAssertIrql(PASSIVE_LEVEL); + THREAD* thread = KeGetCurrentThread(); + if (!thread || thread->IsIdleThread) + return; + _InterlockedIncrement(&thread->TerminationDeferralCount); +} + +// Leaves a process-termination deferral region. +void KeLeaveTerminationDeferral(void) { + KeAssertIrql(PASSIVE_LEVEL); + THREAD* thread = KeGetCurrentThread(); + if (!thread || thread->IsIdleThread) + return; + + long remaining = _InterlockedDecrement(&thread->TerminationDeferralCount); + if (remaining < 0) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "thread termination deferral underflow: thread=0x%016llX tid=%u\n", + (uint64_t)thread, + thread->ThreadId); + } + + if (remaining == 0 && thread->Owner && KeIsProcessTerminating(thread->Owner)) + KiRequestReschedule(); +} + +// Returns whether one thread still owns termination-sensitive kernel work. +int KeThreadTerminationDeferred(const THREAD* thread) { + return thread && _InterlockedCompareExchange((volatile long*)&thread->TerminationDeferralCount, + 0, + 0) > 0; +} + // Implements the exit Thread operation. __declspec(dllexport) void KeExitThread(THREAD* thread, int code) { (void)code; if (!thread) return; KiRundownApcState(thread); - - // Wake any pthread_join waiters before the thread is marked dead and - // potentially freed. Manual-reset means all concurrent waiters unblock. - if (thread->ExitEvent) - KeSignalEvent(thread->ExitEvent); + KiMutexRundownThread(thread); thread->State = ThreadDead; + KeFinalizeThreadExit(thread); KeYield(); } @@ -277,10 +428,9 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) { // [ksp - 16 - sizeof(KCONTEXT)] KCONTEXT { .Rip = KeKernelThreadEntry }. extern void KeKernelThreadEntry(void); -static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, - uint64_t affinityMask, - THREAD** out) { +static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg, + const KE_THREAD_CREATE_PARAMETERS* parameters, + THREAD** out) { // Kernel worker threads are owned by the system process. The scheduler's // hand-built CPU idle thread stays attached to PID 0 (idle), while driver, // async I/O, and init worker threads are grouped under PID 1 (system). @@ -289,34 +439,57 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, THREAD* currentThread = KeGetCurrentThread(); owner = currentThread ? currentThread->Owner : (PROCESS*)0; } - if (!owner) return NTLSTATUS_INVALID_PARAMETER; - if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; + if (!owner || !parameters) return NTLSTATUS_INVALID_PARAMETER; + if (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED) + return NTLSTATUS_INVALID_PARAMETER; + if ((parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) && !out) + return NTLSTATUS_INVALID_PARAMETER; + if (parameters->SchedulingClass == KeSchedulingNormal) { + if (parameters->Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER; + } else if (parameters->SchedulingClass == KeSchedulingRealtime) { + if (parameters->Priority > KE_REALTIME_PRIORITY_MAX) return NTLSTATUS_INVALID_PARAMETER; + } else { + return NTLSTATUS_INVALID_PARAMETER; + } + + uint8_t initiallySuspended = + (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u; THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); if (!thread) return NTLSTATUS_NO_MEMORY; thread->Header.Destroy = KiThreadDestroy; thread->ThreadId = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1); - thread->State = ThreadReady; - thread->Priority = priority; - thread->QuantaConsumed = 0; + thread->State = initiallySuspended ? ThreadSuspended : ThreadReady; + thread->BaseSchedulingClass = parameters->SchedulingClass; + thread->BasePriority = parameters->Priority; + thread->EffectiveSchedulingClass = parameters->SchedulingClass; + thread->EffectivePriority = parameters->Priority; + thread->QuantaConsumed = 0; thread->QuantumTicks = 0; thread->QuantumRemaining = 0; thread->CpuBurstTicks = 0; thread->LastRunTicks = 0; thread->RunningCpu = KE_INVALID_CPU; + thread->SuspendCount = initiallySuspended ? 1u : 0u; + LIST_INIT(&thread->OwnedMutexList); + thread->WaitingMutex = (MUTEX*)0; thread->Queued = 0; thread->IsIdleThread = 0; thread->SchedulerFlags = 0; thread->LastCpu = KE_INVALID_CPU; - thread->AffinityMask = affinityMask ? affinityMask : KeGetDefaultThreadAffinityMask(); + thread->AffinityMask = parameters->AffinityMask + ? parameters->AffinityMask + : KeGetDefaultThreadAffinityMask(); LIST_INIT(&thread->ProcessLink); LIST_INIT(&thread->SchedulerLink); thread->WaitObject = (OBJECT*)0; thread->WaitStatus = NTLSTATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; + LIST_INIT(&thread->TimeoutLink); + thread->TimeoutArmed = 0; + thread->TimeoutTick = 0; thread->TimeoutStatus = NTLSTATUS_SUCCESS; thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); @@ -325,6 +498,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, thread->UserExceptionNoncontinuableMask = 0; thread->UserExceptionFrameUserAddress = 0; memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus)); + thread->TerminationDeferralCount = 0; thread->UserStackBase = (void*)0; // no user stack thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx thread->TebSize = 0; @@ -371,20 +545,15 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, } KeInitThreadFpuContext(thread); - KeReadyThread(thread); + if (!initiallySuspended) + KeReadyThread(thread); return NTLSTATUS_SUCCESS; } __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out) { - return KiCreateKernelThreadWithAffinity(fn, arg, priority, - KeGetDefaultThreadAffinityMask(), out); -} - -__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, uint64_t affinityMask, + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) { - return KiCreateKernelThreadWithAffinity(fn, arg, priority, affinityMask, out); + return KiCreateKernelThread(fn, arg, parameters, out); } // Prepares block Thread. @@ -438,16 +607,6 @@ void KeRundownThreadWait(THREAD* thread, KSTATUS status) { routine(thread, context, status); } -// Implements the finish Wait operation. -static void KiFinishWait(THREAD* thread, KSTATUS status) { - thread->WaitObject = (OBJECT*)0; - thread->WaitStatus = status; - thread->WaitListHead = (LIST_ENTRY*)0; - thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; -} - // Implements the commit Block Thread operation. KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { KeAssertIrql(PASSIVE_LEVEL); @@ -463,19 +622,14 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { if (thread->State != ThreadWaiting) { KSTATUS status = thread->WaitStatus; thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; return status; } // If a user APC was already pending before the thread fully blocked, // consume the alertable wait immediately instead of going to sleep. if (alertable && thread->UserApcPending && thread->UserApcDispatcher) { - if (thread->WaitListHead) - KeListRemoveChecked(&thread->SchedulerLink); - KiFinishWait(thread, NTLSTATUS_USER_APC); - thread->State = ThreadRunning; - return NTLSTATUS_USER_APC; + if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC)) + return NTLSTATUS_USER_APC; } KiSchedulerNoteVoluntaryBlock(thread); @@ -486,8 +640,6 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) { KiDeliverKernelApcs(); thread->AlertableWait = 0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; return thread->WaitStatus; } @@ -525,8 +677,6 @@ static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* wa KeListRemoveChecked(&thread->SchedulerLink); thread->WaitListHead = (LIST_ENTRY*)0; - thread->SleepUntilTick = 0; - thread->TimeoutStatus = NTLSTATUS_SUCCESS; return 1; } @@ -546,10 +696,7 @@ static int KiAbortObjectWait(THREAD* thread) { } case ObjectTypeMutex: { MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header); - uintptr_t flags = KeAcquireSpinlock(&mutex->Lock); - int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList); - KeReleaseSpinlock(&mutex->Lock, flags); - return removed; + return KiMutexAbortWait(mutex, thread); } case ObjectTypeSemaphore: { SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header); @@ -569,24 +716,29 @@ static int KiAbortObjectWait(THREAD* thread) { // unsignaled after the foreground thread has faulted. Object wait-list removal // must use the object's own lock; otherwise signal/abort races can leave stale // scheduler links in an event, mutex, or semaphore wait queue. -void KeAbortThreadWait(THREAD* thread, KSTATUS status) { - if (!thread) return; - if (thread->State != ThreadWaiting) return; +int KeAbortThreadWait(THREAD* thread, KSTATUS status) { + if (!thread) return 0; + if (thread->State != ThreadWaiting) return 0; if (thread->WaitRundownRoutine) { KeRundownThreadWait(thread, status); - return; + return thread->State != ThreadWaiting; } if (KiAbortObjectWait(thread)) { KeUnblockThreadWithStatus(thread, status); - return; + return 1; } - // Sleep/timeouts and already-dequeued handoffs are owned by the scheduler. - // This path removes from the scheduler sleep list when present and otherwise - // completes the wait in place. + // A signaler that already dequeued an object waiter owns completion. Let + // that handoff finish rather than returning timeout after ownership or a + // semaphore count has already transferred to this thread. + if (thread->WaitObject) + return 0; + + // Delays and auxiliary scheduler waits have no object-owned queue. KeUnblockThreadFromWaitListWithStatus(thread, status); + return 1; } // Implements the unblock Thread operation. @@ -601,11 +753,9 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) { if (thread->State != ThreadWaiting || !thread->AlertableWait) return NTLSTATUS_PENDING; - if (thread->WaitListHead) - KeListRemoveChecked(&thread->SchedulerLink); - - KeUnblockThreadWithStatus(thread, status); - return NTLSTATUS_SUCCESS; + return KeAbortThreadWait(thread, status) + ? NTLSTATUS_SUCCESS + : NTLSTATUS_PENDING; } // Implements the alert Thread operation. diff --git a/code/kernel/ke_time.c b/code/kernel/ke_time.c index 86a6a90..fe069ed 100644 --- a/code/kernel/ke_time.c +++ b/code/kernel/ke_time.c @@ -99,7 +99,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out uint32_t sod = (uint32_t)(epochSeconds % 86400u); uint32_t year = 1970u; - for (;;) { + while (true) { uint32_t diy = KiIsLeapYear(year) ? 366u : 365u; if (days < diy) break; days -= diy; @@ -107,7 +107,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out } uint32_t month = 1u; - for (;;) { + while (true) { uint32_t dim = KiDaysInMonth(year, month); if (days < dim) break; days -= dim; @@ -519,6 +519,13 @@ __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime) { return NTLSTATUS_SUCCESS; } +// Returns the monotonic platform counter in microseconds. +uint64_t KeQueryMonotonicTicks(void) { + return (gTimeHal && gTimeHal->Timer.QueryTicks) + ? gTimeHal->Timer.QueryTicks() + : 0; +} + // Returns unix Time Seconds. __declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void) { SYS_SYSTEM_TIME t; diff --git a/code/kernel/ke_work.c b/code/kernel/ke_work.c index 609877c..78122ec 100644 --- a/code/kernel/ke_work.c +++ b/code/kernel/ke_work.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -17,17 +18,31 @@ static SPINLOCK gWorkQueueLock = SPINLOCK_INIT; static LIST_ENTRY gWorkQueueList; static EVENT* gWorkQueueEvent; +static KDPC gWorkQueueWakeDpc; static volatile long gWorkQueueListReady; static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started. +// Signals the passive worker after an enqueue from hardware interrupt context. +static void KiWorkQueueWakeDpc(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { + (void)dpc; + (void)context; + (void)systemArgument1; + (void)systemArgument2; + KeAssertIrql(DISPATCH_LEVEL); + + EVENT* event = gWorkQueueEvent; + if (event) + KeSignalEvent(event); +} + static void KiWorkQueueWorker(void* arg) { (void)arg; KeAssertIrql(PASSIVE_LEVEL); - for (;;) { + while (true) { (void)KeWaitForEvent(gWorkQueueEvent, 0); - for (;;) { + while (true) { KWORK_ITEM* item = (KWORK_ITEM*)0; uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock); if (!LIST_IS_EMPTY(&gWorkQueueList)) { @@ -79,8 +94,14 @@ bool KeQueueWorkItem(KWORK_ITEM* item) { EVENT* event = gWorkQueueEvent; KeReleaseSpinlock(&gWorkQueueLock, flags); - if (event) - KeSignalEvent(event); + if (event) { + // Event signaling can detach waiters, so callers above passive level + // defer the wake until no caller-owned spinlock remains held. + if (KeGetCurrentIrql() == PASSIVE_LEVEL) + KeSignalEvent(event); + else + (void)KeInsertQueueDpc(&gWorkQueueWakeDpc, (void*)0, (void*)0); + } return true; } @@ -95,14 +116,16 @@ KSTATUS KeStartWorkQueue(void) { } LIST_INIT(&gWorkQueueList); + KeInitializeDpc(&gWorkQueueWakeDpc, KiWorkQueueWakeDpc, (void*)0); _InterlockedExchange(&gWorkQueueListReady, 1); KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker, 0, - PriorityNormal, + ¶meters, &worker); // Persistent system worker; keep the creator reference for kernel life. (void)worker; diff --git a/code/kernel/kmain.c b/code/kernel/kmain.c index ed11472..13dbb1d 100644 --- a/code/kernel/kmain.c +++ b/code/kernel/kmain.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -422,7 +423,8 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) { uintptr_t entryPoint = LdrGetEntryPoint(imageBase); // Create the main thread last. KeCreateThread queues it before returning. - s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread); + const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); + s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, &threadParameters, &thread); if (!KSUCCESS(s)) { DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s); proc->ImageBase = 0; @@ -447,6 +449,9 @@ void SystemInitThread(void* arg) { KiRunEarlyKernelSelfTests(); KiInitializeDrivers(args->bootInfo, args->hal); + + if (args->bootInfo->Debug & KE_DEBUG_AUDIO_SELFTEST) + KiRunAudioFinalPathSelfTest(); KeTimeReloadSettings(); KiInitializeSwapFromSettings(); @@ -532,6 +537,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) { gKmainHal = hal; KeInitDebug(hal, bootInfo->Debug); + DbgPrint("CPU: PAT owns WB/WC/UC policy; MTRR default=WB fixed=off variable=off.\n"); KeTimeInitialize(bootInfo, hal); KeDisplayInitialize(hal); @@ -566,6 +572,15 @@ extern void KernelMain(BOOT_INFO* bootInfo) { KeInitializeScheduler(); + KSTATUS volumeNamespaceStatus = IoInitializeVolumeNamespace(); + if (!KSUCCESS(volumeNamespaceStatus)) { + KePanic(KE_PANIC_CODE_INIT, + "I/O: failed to initialize volume namespace notifications, status=0x%x\n", + volumeNamespaceStatus); + } + + KeAudioInitialize(); + KSTATUS workStatus = KeStartWorkQueue(); if (!KSUCCESS(workStatus)) { KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus); @@ -585,9 +600,10 @@ extern void KernelMain(BOOT_INFO* bootInfo) { // Spawn the Initialization Thread. No creator reference is needed; the // scheduler owns the live thread lifetime. + const KE_THREAD_CREATE_PARAMETERS initThreadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread, (uintptr_t)initArgs, - PriorityNormal, (THREAD**)0); + &initThreadParameters, (THREAD**)0); if (!KSUCCESS(st)) { KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st); diff --git a/code/kernel/ldr_section.c b/code/kernel/ldr_section.c index a23d770..b105a68 100644 --- a/code/kernel/ldr_section.c +++ b/code/kernel/ldr_section.c @@ -327,8 +327,18 @@ static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section, if (!size) return NTLSTATUS_SUCCESS; + // Image prototype pages are the canonical cache for executable image bytes. + // Read the backing filesystem directly so the fault path has one cache owner. + if (!section->BackingFile->Fs || !section->BackingFile->Fs->Read) + return NTLSTATUS_NOT_SUPPORTED; + size_t got = 0; - KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got); + KSTATUS s = section->BackingFile->Fs->Read(section->BackingFile->Fs, + section->BackingFile->FileCtx, + fileOffset, + dst, + size, + &got); if (!KSUCCESS(s)) return s; if (got != size) diff --git a/code/kernel/mm_file_cache.c b/code/kernel/mm_file_cache.c index 37b6b82..b22e50d 100644 --- a/code/kernel/mm_file_cache.c +++ b/code/kernel/mm_file_cache.c @@ -16,19 +16,12 @@ #include #include #include -#include #include #include #include #define MM_FILE_CACHE_BUCKETS 64u #define MM_FILE_CACHE_SOFT_MAX_PAGES 512u -#define MM_FILE_CACHE_READAHEAD_MAX_REQUESTS 64u -#define MM_FILE_CACHE_READAHEAD_MAX_PAGES 32u -#define MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET 128u -#define MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES 16u -#define MM_FILE_CACHE_BATCH_MAX_PAGES 16u -#define MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES 64u #define MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES 128u #define MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES 192u #define MM_FILE_CACHE_WRITEBACK_BATCH_PAGES 16u @@ -48,13 +41,6 @@ typedef struct MM_FILE_CACHE_ENTRY { uint64_t DirtySequence; } MM_FILE_CACHE_ENTRY; -typedef struct MM_FILE_CACHE_READAHEAD_REQUEST { - LIST_ENTRY Link; - FILE_OBJECT* File; - uint64_t Offset; - uint32_t Pages; -} MM_FILE_CACHE_READAHEAD_REQUEST; - static SPINLOCK gFileCacheLock = SPINLOCK_INIT; static MM_FILE_CACHE_ENTRY* gFileCacheBuckets[MM_FILE_CACHE_BUCKETS]; static uint64_t gFileCacheResidentPages; @@ -65,13 +51,6 @@ static uint64_t gFileCacheMaxDirtyPages; static uint64_t gFileCacheMaxResidentPages; static MM_FILE_CACHE_STATS gFileCacheStats; -static SPINLOCK gReadAheadLock = SPINLOCK_INIT; -static LIST_ENTRY gReadAheadQueue; -static SEMAPHORE* gReadAheadSemaphore; -static uint32_t gReadAheadQueuedRequests; -static uint32_t gReadAheadQueuedPages; -static int gReadAheadQueueReady; -static int gReadAheadStartState; static SPINLOCK gWritebackLock = SPINLOCK_INIT; static SEMAPHORE* gWritebackSemaphore; @@ -253,7 +232,8 @@ static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOff return NTLSTATUS_NOT_FOUND; } -static void KiFileCacheAddReadAheadStat(uint64_t* field, uint64_t value) { +// Adds to one file-cache statistic under the cache lock. +static void KiFileCacheAddStat(uint64_t* field, uint64_t value) { uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); *field += value; KeReleaseSpinlock(&gFileCacheLock, flags); @@ -265,17 +245,6 @@ static void KiFileCacheSignalWriteback(void) { KeSignalSemaphore(gWritebackSemaphore); } -// Updates a maximum counter while the file-cache stats lock is held. -static void KiFileCacheMaxStatLocked(uint64_t* field, uint64_t value) { - if (*field < value) - *field = value; -} - -static int KiFileCachePageExistsLocked(FILE_OBJECT* file, uint64_t offset) { - uint32_t bucket = KiFileCacheHash(file, offset); - return KiFileCacheFindLocked(file, offset, bucket) != (MM_FILE_CACHE_ENTRY*)0; -} - // Finds the lowest dirty page for a file, or globally when file is NULL. static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) { MM_FILE_CACHE_ENTRY* best = (MM_FILE_CACHE_ENTRY*)0; @@ -292,115 +261,6 @@ static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) return best; } -static MM_FILE_CACHE_READAHEAD_REQUEST* KiFileCacheReadAheadDequeue(void) { - uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); - LIST_ENTRY* node = ListPopHead(&gReadAheadQueue); - if (node) { - MM_FILE_CACHE_READAHEAD_REQUEST* req = CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link); - if (gReadAheadQueuedRequests) - gReadAheadQueuedRequests--; - if (gReadAheadQueuedPages >= req->Pages) - gReadAheadQueuedPages -= req->Pages; - else - gReadAheadQueuedPages = 0; - } - KeReleaseSpinlock(&gReadAheadLock, flags); - - if (!node) - return (MM_FILE_CACHE_READAHEAD_REQUEST*)0; - return CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link); -} - -static void KiFileCacheReadAheadWorker(void* arg) { - (void)arg; - KeAssertIrql(PASSIVE_LEVEL); - - for (;;) { - KSTATUS ws = KeWaitSemaphore(gReadAheadSemaphore); - if (!KSUCCESS(ws)) { - KeYield(); - continue; - } - - MM_FILE_CACHE_READAHEAD_REQUEST* req = KiFileCacheReadAheadDequeue(); - if (!req) - continue; - - for (uint32_t i = 0; i < req->Pages; ++i) { - uint64_t offset = req->Offset + ((uint64_t)i * PAGE_SIZE); - - uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); - if (gFileCacheResidentPages + MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES >= - MM_FILE_CACHE_SOFT_MAX_PAGES) { - gFileCacheStats.ReadAheadPagesThrottled += (uint64_t)(req->Pages - i); - KeReleaseSpinlock(&gFileCacheLock, flags); - break; - } - int exists = KiFileCachePageExistsLocked(req->File, offset); - if (exists) - gFileCacheStats.ReadAheadPagesSkipped++; - KeReleaseSpinlock(&gFileCacheLock, flags); - if (exists) - continue; - - MM_FILE_CACHE_PAGE page; - KSTATUS s = MmFileCacheGetPage(req->File, offset, &page); - flags = KeAcquireSpinlock(&gFileCacheLock); - if (KSUCCESS(s)) - gFileCacheStats.ReadAheadPagesLoaded++; - else - gFileCacheStats.ReadAheadFailures++; - KeReleaseSpinlock(&gFileCacheLock, flags); - - if (!KSUCCESS(s)) - break; - MmFileCacheReleasePage(&page); - } - - ObDereferenceObject(&req->File->Header); - MmFreePool(req); - } -} - -static KSTATUS KiFileCacheEnsureReadAheadStarted(void) { - uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); - if (gReadAheadStartState == 2) { - KeReleaseSpinlock(&gReadAheadLock, flags); - return NTLSTATUS_SUCCESS; - } - if (gReadAheadStartState == 1) { - KeReleaseSpinlock(&gReadAheadLock, flags); - while (gReadAheadStartState == 1) - KeYield(); - return gReadAheadStartState == 2 ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE; - } - - gReadAheadStartState = 1; - KeReleaseSpinlock(&gReadAheadLock, flags); - - LIST_INIT(&gReadAheadQueue); - gReadAheadQueuedRequests = 0; - gReadAheadQueuedPages = 0; - gReadAheadQueueReady = 1; - - KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gReadAheadSemaphore); - if (KSUCCESS(s)) { - THREAD* worker = (THREAD*)0; - s = KeCreateKernelThread((uintptr_t)KiFileCacheReadAheadWorker, - 0, - PriorityNormal, - &worker); - (void)worker; - } - - flags = KeAcquireSpinlock(&gReadAheadLock); - gReadAheadStartState = KSUCCESS(s) ? 2 : 0; - if (!KSUCCESS(s)) - gReadAheadQueueReady = 0; - KeReleaseSpinlock(&gReadAheadLock, flags); - return s; -} - // Flushes the next dirty cache page for file, or globally when file is NULL. static KSTATUS KiFileCacheFlushOneDirtyPage(FILE_OBJECT* file, int* didWork) { if (didWork) *didWork = 0; @@ -462,8 +322,8 @@ static void KiFileCacheWritebackWorker(void* arg) { (void)arg; KeAssertIrql(PASSIVE_LEVEL); - for (;;) { - KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore); + while (true) { + KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(ws)) { KeYield(); continue; @@ -504,9 +364,10 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) { KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gWritebackSemaphore); if (KSUCCESS(s)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); s = KeCreateKernelThread((uintptr_t)KiFileCacheWritebackWorker, 0, - PriorityNormal, + ¶meters, &worker); (void)worker; } @@ -517,15 +378,13 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) { return s; } -static size_t KiMinSize(size_t a, size_t b) { return a < b ? a : b; } - static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_PAGE* outPage, int waitForLoading) { if (outPage) memset(outPage, 0, sizeof(*outPage)); uint32_t bucket = KiFileCacheHash(file, aligned); int counted = 0; - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); if (!counted) { gFileCacheStats.Lookups++; @@ -589,7 +448,7 @@ static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CA } static void KiFileCacheKeepBounded(FILE_OBJECT* skipFile, uint64_t skipOffset) { - for (;;) { + while (true) { uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); int over = gFileCacheResidentPages > MM_FILE_CACHE_SOFT_MAX_PAGES; KeReleaseSpinlock(&gFileCacheLock, flags); @@ -718,42 +577,12 @@ static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry, return NTLSTATUS_SUCCESS; } -static uint32_t KiFileCacheChooseForegroundBatchPages(void) { - uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); - uint64_t resident = gFileCacheResidentPages; - KeReleaseSpinlock(&gFileCacheLock, flags); - - if (resident >= MM_FILE_CACHE_SOFT_MAX_PAGES) - return 1; - - uint64_t headroom = MM_FILE_CACHE_SOFT_MAX_PAGES - resident; - if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) - return 1; - if (headroom <= MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES) - return 2; - if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 2u)) - return 4; - if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 4u)) - return 8; - return MM_FILE_CACHE_BATCH_MAX_PAGES; -} - -static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file, +// Loads exactly one demanded file page into the coherent cache. +static KSTATUS KiFileCacheLoadPage(FILE_OBJECT* file, uint64_t aligned, - uint32_t maxPages, - MM_FILE_CACHE_PAGE* requestedPage, - uint32_t* loadedPagesOut) { - if (loadedPagesOut) - *loadedPagesOut = 0; - if (!file || maxPages == 0) + MM_FILE_CACHE_PAGE* outPage) { + if (!file || !outPage) return NTLSTATUS_INVALID_PARAMETER; - if (maxPages > MM_FILE_CACHE_BATCH_MAX_PAGES) - maxPages = MM_FILE_CACHE_BATCH_MAX_PAGES; - - uintptr_t statFlags = KeAcquireSpinlock(&gFileCacheLock); - gFileCacheStats.BatchPagesRequested += maxPages; - KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesRequested, maxPages); - KeReleaseSpinlock(&gFileCacheLock, statFlags); uintptr_t fileFlags = KeAcquireSpinlock(&file->Lock); uint64_t fileSize = file->Size; @@ -761,134 +590,55 @@ static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file, if (aligned >= fileSize) return NTLSTATUS_SUCCESS; - MM_FILE_CACHE_ENTRY* entries[MM_FILE_CACHE_BATCH_MAX_PAGES]; - PHYSICAL_ADDRESS frames[MM_FILE_CACHE_BATCH_MAX_PAGES]; - memset(entries, 0, sizeof(entries)); - memset(frames, 0, sizeof(frames)); + MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)0; + KSTATUS s = KiFileCacheReserveLoadingPage(file, aligned, &entry); + if (s == NTLSTATUS_ALREADY_EXISTS) + return KiFileCacheTryHit(file, aligned, outPage, 1); + if (!KSUCCESS(s)) + return s; - uint32_t pages = 0; - KSTATUS reserveStatus = NTLSTATUS_SUCCESS; - while (pages < maxPages) { - uint64_t pageOffset = aligned + ((uint64_t)pages * PAGE_SIZE); - if (pageOffset >= fileSize) - break; - - reserveStatus = KiFileCacheReserveLoadingPage(file, pageOffset, &entries[pages]); - if (!KSUCCESS(reserveStatus)) - break; - pages++; - } - - if (pages == 0) { - if (requestedPage) - return KiFileCacheTryHit(file, aligned, requestedPage, 1); - return reserveStatus == NTLSTATUS_ALREADY_EXISTS ? NTLSTATUS_ALREADY_EXISTS : reserveStatus; - } - - uint8_t* readBuffer = (uint8_t*)MmAllocatePoolTag((size_t)pages * PAGE_SIZE, MM_POOL_TAG('f', 'c', 'b', 'l')); - if (!readBuffer) { - for (uint32_t i = 0; i < pages; ++i) - KiFileCacheCancelLoadingPage(entries[i]); - KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); + PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseCleanFileCache); + if (!frame) { + KiFileCacheCancelLoadingPage(entry); + KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1); return NTLSTATUS_NO_MEMORY; } - memset(readBuffer, 0, (size_t)pages * PAGE_SIZE); - for (uint32_t i = 0; i < pages; ++i) { - frames[i] = MmPfnAllocate(MmPfnUseCleanFileCache); - if (!frames[i]) { - for (uint32_t j = 0; j < i; ++j) - (void)MmPfnRelease(frames[j]); - for (uint32_t j = 0; j < pages; ++j) - KiFileCacheCancelLoadingPage(entries[j]); - MmFreePool(readBuffer); - KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); - return NTLSTATUS_NO_MEMORY; - } + void* page = MmPfnToVirtual(frame); + if (!page) { + (void)MmPfnRelease(frame); + KiFileCacheCancelLoadingPage(entry); + KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1); + return NTLSTATUS_FAILURE; } + memset(page, 0, PAGE_SIZE); + + size_t toRead = PAGE_SIZE; + uint64_t remaining = fileSize - aligned; + if (remaining < PAGE_SIZE) + toRead = (size_t)remaining; - size_t toRead = KiMinSize((size_t)pages * PAGE_SIZE, (size_t)(fileSize - aligned)); size_t got = 0; - KSTATUS s = NTLSTATUS_NOT_SUPPORTED; + s = NTLSTATUS_NOT_SUPPORTED; if (file->Fs && file->Fs->Read) - s = file->Fs->Read(file->Fs, file->FileCtx, aligned, readBuffer, toRead, &got); + s = file->Fs->Read(file->Fs, file->FileCtx, aligned, page, toRead, &got); if (!KSUCCESS(s)) { - for (uint32_t i = 0; i < pages; ++i) { - if (frames[i]) - (void)MmPfnRelease(frames[i]); - KiFileCacheCancelLoadingPage(entries[i]); - } - MmFreePool(readBuffer); - KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1); + (void)MmPfnRelease(frame); + KiFileCacheCancelLoadingPage(entry); + KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1); return s; } if (got > toRead) got = toRead; + if (got < PAGE_SIZE) + memset((uint8_t*)page + got, 0, PAGE_SIZE - got); - uint64_t published = 0; - KSTATUS requestedStatus = NTLSTATUS_NOT_FOUND; - for (uint32_t i = 0; i < pages; ++i) { - uint64_t pageOffset = aligned + ((uint64_t)i * PAGE_SIZE); - size_t pageBase = (size_t)i * PAGE_SIZE; - if (pageBase >= got && pageOffset >= fileSize) { - (void)MmPfnRelease(frames[i]); - frames[i] = 0; - KiFileCacheCancelLoadingPage(entries[i]); - continue; - } - - uint32_t validBytes = 0; - if (pageBase < got) { - size_t avail = got - pageBase; - if (avail > PAGE_SIZE) avail = PAGE_SIZE; - validBytes = (uint32_t)avail; - } - - void* page = MmPfnToVirtual(frames[i]); - if (!page) { - (void)MmPfnRelease(frames[i]); - frames[i] = 0; - KiFileCacheCancelLoadingPage(entries[i]); - continue; - } - memcpy(page, readBuffer + pageBase, validBytes); - if (validBytes < PAGE_SIZE) - memset((uint8_t*)page + validBytes, 0, PAGE_SIZE - validBytes); - - MM_FILE_CACHE_PAGE* out = (pageOffset == aligned) ? requestedPage : (MM_FILE_CACHE_PAGE*)0; - KSTATUS ps = KiFileCacheCompleteLoadingPage(entries[i], frames[i], validBytes, out, &published); - if (KSUCCESS(ps)) { - if (pageOffset == aligned) - requestedStatus = NTLSTATUS_SUCCESS; - frames[i] = 0; - } else { - if (pageOffset == aligned) - requestedStatus = ps; - frames[i] = 0; - } - } - - MmFreePool(readBuffer); - - uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); - if (published) { - gFileCacheStats.BatchReads++; - gFileCacheStats.BatchPagesLoaded += published; - KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesLoaded, published); - } - KeReleaseSpinlock(&gFileCacheLock, flags); - - if (loadedPagesOut) - *loadedPagesOut = (uint32_t)published; + s = KiFileCacheCompleteLoadingPage(entry, frame, (uint32_t)got, outPage, (uint64_t*)0); + if (!KSUCCESS(s)) + return s; KiFileCacheKeepBounded(file, aligned); - - if (requestedPage) { - if (KSUCCESS(requestedStatus)) - return NTLSTATUS_SUCCESS; - return KiFileCacheTryHit(file, aligned, requestedPage, 1); - } - return published ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND; + return NTLSTATUS_SUCCESS; } // Returns a retained cache page suitable for in-place buffered writes. @@ -906,7 +656,7 @@ static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file, return NTLSTATUS_SUCCESS; if (needExistingBytes && aligned < oldFileSize) { - s = KiFileCacheLoadSpan(file, aligned, 1, outPage, (uint32_t*)0); + s = KiFileCacheLoadPage(file, aligned, outPage); if (KSUCCESS(s)) return NTLSTATUS_SUCCESS; if (s != NTLSTATUS_NOT_FOUND) @@ -1063,7 +813,7 @@ KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file) { KeReleaseSpinlock(&gFileCacheLock, flags); KSTATUS firstError = NTLSTATUS_SUCCESS; - for (;;) { + while (true) { int didWork = 0; KSTATUS s = KiFileCacheFlushOneDirtyPage(file, &didWork); if (!didWork) @@ -1090,7 +840,7 @@ KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAG KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1); if (KSUCCESS(s)) return NTLSTATUS_SUCCESS; - return KiFileCacheLoadSpan(file, aligned, KiFileCacheChooseForegroundBatchPages(), outPage, (uint32_t*)0); + return KiFileCacheLoadPage(file, aligned, outPage); } void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page) { @@ -1111,7 +861,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) { } for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { - for (;;) { + while (true) { MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0; uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; @@ -1145,7 +895,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) { void MmFileCacheInvalidateAll(FILE_OBJECT* file) { if (!file) return; for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) { - for (;;) { + while (true) { MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0; uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0; @@ -1179,83 +929,6 @@ KSTATUS MmFileCacheDropOnePageForPressure(void) { return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0); } -KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages) { - if (!file || pages == 0) - return NTLSTATUS_INVALID_PARAMETER; - if (!KeCanWait()) - return NTLSTATUS_INVALID_IRQL; - - uint32_t requestedPages = pages; - if (pages > MM_FILE_CACHE_READAHEAD_MAX_PAGES) - pages = MM_FILE_CACHE_READAHEAD_MAX_PAGES; - - uintptr_t cacheFlags = KeAcquireSpinlock(&gFileCacheLock); - gFileCacheStats.ReadAheadPagesRequested += requestedPages; - KiFileCacheMaxStatLocked(&gFileCacheStats.MaxReadAheadPagesPerRequest, requestedPages); - uint64_t resident = gFileCacheResidentPages; - uint64_t headroom = resident < MM_FILE_CACHE_SOFT_MAX_PAGES - ? MM_FILE_CACHE_SOFT_MAX_PAGES - resident - : 0; - if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) { - gFileCacheStats.ReadAheadPagesThrottled += pages; - KeReleaseSpinlock(&gFileCacheLock, cacheFlags); - return NTLSTATUS_QUEUE_FULL; - } - headroom -= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES; - if ((uint64_t)pages > headroom) - pages = (uint32_t)headroom; - KeReleaseSpinlock(&gFileCacheLock, cacheFlags); - if (pages == 0) - return NTLSTATUS_QUEUE_FULL; - - KSTATUS s = KiFileCacheEnsureReadAheadStarted(); - if (!KSUCCESS(s)) { - KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1); - return s; - } - - MM_FILE_CACHE_READAHEAD_REQUEST* req = - (MM_FILE_CACHE_READAHEAD_REQUEST*)MmAllocatePoolTag(sizeof(*req), MM_POOL_TAG('r', 'd', 'a', 'h')); - if (!req) { - KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1); - return NTLSTATUS_NO_MEMORY; - } - - LIST_INIT(&req->Link); - req->File = file; - req->Offset = offset & ~(uint64_t)(PAGE_SIZE - 1); - req->Pages = pages; - ObReferenceObject(&file->Header); - - uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock); - if (!gReadAheadQueueReady || - gReadAheadQueuedRequests >= MM_FILE_CACHE_READAHEAD_MAX_REQUESTS || - gReadAheadQueuedPages >= MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET) { - KeReleaseSpinlock(&gReadAheadLock, flags); - ObDereferenceObject(&file->Header); - MmFreePool(req); - KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadPagesThrottled, pages); - return NTLSTATUS_QUEUE_FULL; - } - uint32_t queueHeadroom = MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET - gReadAheadQueuedPages; - if (pages > queueHeadroom) - pages = queueHeadroom; - req->Pages = pages; - - ListInsertTail(&gReadAheadQueue, &req->Link); - gReadAheadQueuedRequests++; - gReadAheadQueuedPages += pages; - KeReleaseSpinlock(&gReadAheadLock, flags); - - flags = KeAcquireSpinlock(&gFileCacheLock); - gFileCacheStats.ReadAheadRequests++; - gFileCacheStats.ReadAheadPagesQueued += pages; - KeReleaseSpinlock(&gFileCacheLock, flags); - - KeSignalSemaphore(gReadAheadSemaphore); - return NTLSTATUS_SUCCESS; -} - void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out) { if (!out) return; uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock); diff --git a/code/kernel/mm_phys.c b/code/kernel/mm_phys.c index 1680ff7..0a636fc 100644 --- a/code/kernel/mm_phys.c +++ b/code/kernel/mm_phys.c @@ -472,8 +472,13 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min return (PHYSICAL_ADDRESS)0; } -// Allocates a physically contiguous run for DMA or hardware structures. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) { +// Allocates a physically contiguous run inside an address and alignment window. +PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count, + PHYSICAL_ADDRESS minInclusive, + PHYSICAL_ADDRESS maxExclusive, + size_t alignment, + size_t boundary, + MM_PFN_USAGE usage) { if (!count || count > gMaxPfn) { uintptr_t flags = KeAcquireSpinlock(&gPfnLock); gPfnFailedContiguousAllocations++; @@ -482,6 +487,50 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_ return (PHYSICAL_ADDRESS)0; } + if (!alignment) + alignment = PAGE_SIZE; + if (alignment < PAGE_SIZE) + alignment = PAGE_SIZE; + if ((alignment & (alignment - 1)) != 0 || + (boundary && (boundary & (boundary - 1)) != 0)) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + + if (count > ((size_t)-1) / PAGE_SIZE) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + + size_t byteCount = count * PAGE_SIZE; + if (boundary && byteCount > boundary) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + + minInclusive = KiAlignUp(minInclusive); + maxExclusive = KiAlignDown(maxExclusive); + PHYSICAL_ADDRESS managedLimit = (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE; + if (maxExclusive > managedLimit) + maxExclusive = managedLimit; + if (maxExclusive <= minInclusive || + (PHYSICAL_ADDRESS)byteCount > maxExclusive - minInclusive) { + uintptr_t flags = KeAcquireSpinlock(&gPfnLock); + gPfnFailedContiguousAllocations++; + gPfnFailedContiguousInvalidRequest++; + KeReleaseSpinlock(&gPfnLock, flags); + return (PHYSICAL_ADDRESS)0; + } + if (!KiPfnUsageMayUseReserve(usage)) KiPfnReclaimForPressure(); @@ -493,23 +542,31 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_ return (PHYSICAL_ADDRESS)0; } - size_t runStart = 0; - size_t runLength = 0; - - for (size_t pfn = 1; pfn < gMaxPfn; pfn++) { - if (KiPfnAvailableLocked(pfn)) { - if (!runLength) - runStart = pfn; - runLength++; - if (runLength == count) { - for (size_t i = 0; i < count; i++) - KiClaimPfnLocked(runStart + i, usage); - KeReleaseSpinlock(&gPfnLock, flags); - return MmPfnAddress(runStart); - } - } else { - runLength = 0; + size_t firstPfn = (size_t)(minInclusive / PAGE_SIZE); + size_t limitPfn = (size_t)(maxExclusive / PAGE_SIZE); + for (size_t runStart = firstPfn; runStart + count <= limitPfn; ++runStart) { + PHYSICAL_ADDRESS runAddress = MmPfnAddress(runStart); + if ((runAddress & (alignment - 1)) != 0) + continue; + if (boundary) { + PHYSICAL_ADDRESS runEnd = runAddress + (PHYSICAL_ADDRESS)byteCount - 1; + if ((runAddress & ~((PHYSICAL_ADDRESS)boundary - 1)) != + (runEnd & ~((PHYSICAL_ADDRESS)boundary - 1))) + continue; } + + size_t available = 0; + while (available < count && KiPfnAvailableLocked(runStart + available)) + available++; + if (available != count) { + runStart += available; + continue; + } + + for (size_t i = 0; i < count; ++i) + KiClaimPfnLocked(runStart + i, usage); + KeReleaseSpinlock(&gPfnLock, flags); + return runAddress; } gPfnFailedContiguousAllocations++; @@ -518,6 +575,16 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_ return (PHYSICAL_ADDRESS)0; } +// Allocates a physically contiguous run for DMA or hardware structures. +__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) { + return MmPfnAllocateContiguousInRange(count, + PAGE_SIZE, + (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE, + PAGE_SIZE, + 0, + usage); +} + // Increments the reference count on an allocated frame. __declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame) { size_t pfn = MmPfnIndex(frame); diff --git a/code/kernel/mm_pte.c b/code/kernel/mm_pte.c index 0383584..2a71465 100644 --- a/code/kernel/mm_pte.c +++ b/code/kernel/mm_pte.c @@ -60,6 +60,14 @@ static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose return purpose; } +// Returns whether flags select one canonical PAT cache policy. +static bool KiPteCacheFlagsValid(uint64_t flags) { + uint64_t cacheFlags = flags & PAGE_CACHE_MASK; + return cacheFlags == PAGE_CACHE_WB || + cacheFlags == PAGE_CACHE_WC || + cacheFlags == PAGE_CACHE_UC; +} + // Ensures that one parent entry points to a child page table. // New child tables are zeroed before publication so no stale mappings become visible. static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys, @@ -200,6 +208,10 @@ int MmPteHandleShootdownNmi(void) { // Maps or replaces one 4 KiB leaf PTE in the shared page tables. __declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) { KiPteStatInc(&gPteStats.MapCalls); + if (!KiPteCacheFlagsValid(flags)) { + KiPteStatInc(&gPteStats.MapFailures); + return NTLSTATUS_INVALID_PARAMETER; + } unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0; uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); @@ -286,6 +298,10 @@ __declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) { // Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them. __declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) { KiPteStatInc(&gPteStats.ProtectCalls); + if (!KiPteCacheFlagsValid(flags)) { + KiPteStatInc(&gPteStats.ProtectFailures); + return NTLSTATUS_INVALID_PARAMETER; + } if (!size) return NTLSTATUS_SUCCESS; // The protection operation is page-granular even when callers pass an interior address. diff --git a/code/kernel/mm_virt.c b/code/kernel/mm_virt.c index 422a3ff..218cced 100644 --- a/code/kernel/mm_virt.c +++ b/code/kernel/mm_virt.c @@ -7,6 +7,7 @@ // FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings. // FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map. // FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys. +// FFFFA00000000000-FFFFAFFFFFFFFFFF is the kernel shared-view arena. // FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool. // // The current VM model uses one shared PML4, so user pointers are globally @@ -112,6 +113,12 @@ typedef struct MM_VAD_PAGE { uint32_t SwapSlot; } MM_VAD_PAGE; +struct MM_KERNEL_VIEW { + LIST_ENTRY Link; + uintptr_t Base; + size_t PageCount; +}; + typedef struct MM_VAD { LIST_ENTRY ProcessLink; LIST_ENTRY GlobalLink; @@ -143,13 +150,19 @@ static LIST_ENTRY gVadList; static SPINLOCK gVadLock = SPINLOCK_INIT; static int gVadListReady = 0; static MM_VAD_STATS gVadStats; +static LIST_ENTRY gKernelViewList; +static SPINLOCK gKernelViewLock = SPINLOCK_INIT; -#define MM_VAD_ARENA_BASE 0x0000100000000000ULL +#define MM_VAD_ARENA_BASE 0x0000100000000000ULL +#define MM_KERNEL_VIEW_BASE 0xFFFFA00000000000ULL +#define MM_KERNEL_VIEW_LIMIT 0xFFFFB00000000000ULL // Initializes the VAD list after the shared page table and kernel pool are ready. static void KiVadInitialize(void) { LIST_INIT(&gVadList); + LIST_INIT(&gKernelViewList); gVadLock = (SPINLOCK)SPINLOCK_INIT; + gKernelViewLock = (SPINLOCK)SPINLOCK_INIT; memset(&gVadStats, 0, sizeof(gVadStats)); gVadListReady = 1; } @@ -170,6 +183,108 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) { (void)MmPfnPin(gVadZeroPageFrame); } +// Inserts a kernel view before one list node while preserving address order. +static void KiKernelViewInsertBefore(LIST_ENTRY* before, MM_KERNEL_VIEW* view) { + view->Link.Fwd = before; + view->Link.Bwd = before->Bwd; + before->Bwd->Fwd = &view->Link; + before->Bwd = &view->Link; +} + +// Maps existing frames into the kernel shared-view arena. +KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames, + size_t pageCount, + MM_KERNEL_VIEW** outView) { + if (!outView) + return NTLSTATUS_INVALID_PARAMETER; + *outView = (MM_KERNEL_VIEW*)0; + if (!frames || pageCount == 0 || pageCount > (SIZE_MAX / PAGE_SIZE)) + return NTLSTATUS_INVALID_PARAMETER; + + size_t viewBytes = pageCount * PAGE_SIZE; + MM_KERNEL_VIEW* view = (MM_KERNEL_VIEW*)MmAllocatePoolTag(sizeof(MM_KERNEL_VIEW), + MM_POOL_TAG('k', 'v', 'i', 'w')); + if (!view) + return NTLSTATUS_NO_MEMORY; + memset(view, 0, sizeof(*view)); + LIST_INIT(&view->Link); + + uintptr_t candidate = MM_KERNEL_VIEW_BASE; + LIST_ENTRY* before = &gKernelViewList; + uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock); + for (LIST_ENTRY* it = gKernelViewList.Fwd; it != &gKernelViewList; it = it->Fwd) { + MM_KERNEL_VIEW* existing = CONTAINING_RECORD(it, MM_KERNEL_VIEW, Link); + if (candidate <= existing->Base && + viewBytes <= (size_t)(existing->Base - candidate)) { + before = it; + break; + } + + uintptr_t existingEnd = existing->Base + existing->PageCount * PAGE_SIZE; + if (existingEnd > candidate) + candidate = existingEnd; + } + + if (candidate > MM_KERNEL_VIEW_LIMIT || viewBytes > MM_KERNEL_VIEW_LIMIT - candidate) { + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + MmFreePool(view); + return NTLSTATUS_NO_MEMORY; + } + + view->Base = candidate; + view->PageCount = pageCount; + KiKernelViewInsertBefore(before, view); + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + + const uint64_t hardwareFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX; + size_t mappedPages = 0; + KSTATUS s = NTLSTATUS_SUCCESS; + for (; mappedPages < pageCount; mappedPages++) { + if (!frames[mappedPages]) { + s = NTLSTATUS_INVALID_PARAMETER; + break; + } + s = MmPteMapPage(candidate + mappedPages * PAGE_SIZE, frames[mappedPages], + hardwareFlags, MmPteMapFailIfPresent); + if (!KSUCCESS(s)) + break; + } + + if (!KSUCCESS(s)) { + while (mappedPages != 0) { + mappedPages--; + (void)MmPteUnmapPage(candidate + mappedPages * PAGE_SIZE); + } + lockFlags = KeAcquireSpinlock(&gKernelViewLock); + ListRemove(&view->Link); + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + MmFreePool(view); + return s; + } + + *outView = view; + return NTLSTATUS_SUCCESS; +} + +// Returns the base address of one kernel shared view. +void* MmKernelViewBase(const MM_KERNEL_VIEW* view) { + return view ? (void*)view->Base : (void*)0; +} + +// Unmaps and releases one kernel shared view without releasing its frames. +void MmDestroyKernelView(MM_KERNEL_VIEW* view) { + if (!view) + return; + + for (size_t i = 0; i < view->PageCount; i++) + (void)MmPteUnmapPage(view->Base + i * PAGE_SIZE); + + uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock); + ListRemove(&view->Link); + KeReleaseSpinlock(&gKernelViewLock, lockFlags); + MmFreePool(view); +} + // VAD utility helpers. static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) { if (align < PAGE_SIZE) align = PAGE_SIZE; @@ -199,7 +314,7 @@ static KSTATUS KiFindVadRange(size_t reservationSize, size_t alignment, uintptr_ uintptr_t candidate = KiAlignUpPtr(MM_VAD_ARENA_BASE, alignment); if (!candidate) return NTLSTATUS_INVALID_PARAMETER; - for (;;) { + while (true) { if (candidate < MM_VAD_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate)) return NTLSTATUS_NO_MEMORY; @@ -438,7 +553,7 @@ static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) { static int KiVadTryBeginPaging(MM_VAD_PAGE* page) { if (!page) return 0; - for (;;) { + while (true) { uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0); if (oldFlags & MM_VAD_PAGE_PAGING) return 0; @@ -454,7 +569,7 @@ static int KiVadTryBeginPaging(MM_VAD_PAGE* page) { static void KiVadEndPaging(MM_VAD_PAGE* page) { if (!page) return; - for (;;) { + while (true) { uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0); uint32_t newFlags = oldFlags & ~MM_VAD_PAGE_PAGING; if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, @@ -1157,6 +1272,18 @@ __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, return NTLSTATUS_SUCCESS; } +// Returns the page count held by a locked user buffer. +size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer) { + return buffer ? buffer->PageCount : 0; +} + +// Returns one pinned frame held by a locked user buffer. +PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index) { + if (!buffer || index >= buffer->PageCount) + return (PHYSICAL_ADDRESS)0; + return buffer->Pages[index].Frame; +} + // Unlocks a user buffer locked for asynchronous kernel access. __declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer) { if (!buffer) @@ -2256,7 +2383,7 @@ void MmVadDestroyAllForProcess(PROCESS* process) { KeAssertIrql(PASSIVE_LEVEL); if (!process) return; - for (;;) { + while (true) { uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock); LIST_ENTRY* node = (LIST_ENTRY*)0; int sawLocked = 0; diff --git a/code/kernel/net_socket.c b/code/kernel/net_socket.c index 307a23e..2638d8d 100644 --- a/code/kernel/net_socket.c +++ b/code/kernel/net_socket.c @@ -379,8 +379,8 @@ static void NetAsyncSocketExecute(NET_ASYNC_SOCKET_REQUEST_OBJECT* req, static void NetAsyncSocketWorker(void* arg) { (void)arg; - for (;;) { - KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore); + while (true) { + KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(waitStatus)) { KeYield(); continue; @@ -426,9 +426,10 @@ static KSTATUS NetEnsureAsyncSocketWorkerStarted(void) { KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncSocketSemaphore); if (KSUCCESS(status)) { THREAD* worker = (THREAD*)0; + const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal); status = KeCreateKernelThread((uintptr_t)NetAsyncSocketWorker, 0, - PriorityNormal, + ¶meters, &worker); (void)worker; } diff --git a/code/kernel/ob_object.c b/code/kernel/ob_object.c index bde8914..3e59ab7 100644 --- a/code/kernel/ob_object.c +++ b/code/kernel/ob_object.c @@ -31,6 +31,8 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) { case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' '); case ObjectTypeModule: return MM_POOL_TAG('m', 'o', 'd', 'u'); case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k'); + case ObjectTypeAudioEndpoint: return MM_POOL_TAG('a', 'e', 'p', 't'); + case ObjectTypeAudioStream: return MM_POOL_TAG('a', 's', 't', 'r'); default: return MM_POOL_TAG('o', 'b', 'j', ' '); } } @@ -64,7 +66,7 @@ __declspec(dllexport) bool ObTryReferenceObject(OBJECT* object) { if (!object) return false; - for (;;) { + while (true) { if (_InterlockedCompareExchange(&object->LifetimeState, 0, 0) != ObjectAlive) return false; if ((_InterlockedCompareExchange(&object->Flags, 0, 0) & OBJECT_FLAG_INSERTED) == 0) diff --git a/code/public/ntlite/audio.h b/code/public/ntlite/audio.h new file mode 100644 index 0000000..5c93c48 --- /dev/null +++ b/code/public/ntlite/audio.h @@ -0,0 +1,220 @@ +// Public native render-stream syscall ABI declarations. + +#pragma once + +#include +#include + +#define SYS_AUDIO_ABI_VERSION 3u +#define SYS_AUDIO_MAX_ENDPOINTS 32u +#define SYS_AUDIO_STREAM_CAPACITY_FRAMES 4096u + +#define SYS_AUDIO_GAIN_Q16_UNITY 65536u +#define SYS_AUDIO_PAN_Q16_LEFT (-65536) +#define SYS_AUDIO_PAN_Q16_CENTER 0 +#define SYS_AUDIO_PAN_Q16_RIGHT 65536 + +// Identifies one logical render destination. +typedef enum SYS_AUDIO_ENDPOINT_KIND { + SysAudioEndpointLineOut = 1, + SysAudioEndpointSpeakerHeadphone, + SysAudioEndpointOtherAnalog +} SYS_AUDIO_ENDPOINT_KIND; + +// Identifies the interleaved PCM sample representation accepted by native streams. +typedef enum SYS_AUDIO_SAMPLE_FORMAT { + SysAudioSampleS16 = 1 +} SYS_AUDIO_SAMPLE_FORMAT; + +// Describes one interleaved PCM stream format. +typedef struct SYS_AUDIO_FORMAT { + uint32_t SampleRate; + uint16_t ChannelCount; + uint16_t SampleFormat; +} SYS_AUDIO_FORMAT; + +#define SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE 0x00000001u +#define SYS_AUDIO_ENDPOINT_FLAG_DEFAULT 0x00000002u +#define SYS_AUDIO_ENDPOINT_FLAG_VALID_MASK 0x00000003u + +#define SYS_AUDIO_ENDPOINT_CAP_JACK_SENSE 0x00000001u +#define SYS_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN 0x00000002u +#define SYS_AUDIO_ENDPOINT_CAP_VALID_MASK 0x00000003u + +// Identifies the lifecycle state reported for one render endpoint. +typedef enum SYS_AUDIO_ENDPOINT_STATE { + SysAudioEndpointStarting = 1, + SysAudioEndpointReady, + SysAudioEndpointStopping, + SysAudioEndpointFailed, + SysAudioEndpointRemoved +} SYS_AUDIO_ENDPOINT_STATE; + +// Reports one endpoint snapshot keyed by its stable kernel identifier. +typedef struct SYS_AUDIO_ENDPOINT_INFO { + uint32_t Size; + uint32_t Version; + uint64_t EndpointId; + uint32_t Kind; + uint32_t Flags; + uint32_t Capabilities; + uint32_t State; + SYS_AUDIO_FORMAT HardwareFormat; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + uint32_t MasterVolumeQ16; + uint32_t MasterMuted; + KSTATUS FailureStatus; + uint32_t Reserved; +} SYS_AUDIO_ENDPOINT_INFO; + +// Selects the endpoint array used by one enumeration call. +typedef struct SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST { + uint32_t Size; + uint32_t Flags; + SYS_AUDIO_ENDPOINT_INFO* Endpoints; + uint32_t Capacity; + uint32_t Reserved; + uint32_t* CountOut; +} SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST; + +// Selects one endpoint and source format for a new render stream handle. +typedef struct SYS_AUDIO_OPEN_STREAM_REQUEST { + uint32_t Size; + uint32_t Flags; + uint64_t EndpointId; + SYS_AUDIO_FORMAT Format; + uint32_t Reserved; + uint32_t* StreamHandleOut; +} SYS_AUDIO_OPEN_STREAM_REQUEST; + +// Describes one bounded nonblocking stream write. +typedef struct SYS_AUDIO_WRITE_REQUEST { + uint32_t Size; + uint32_t Flags; + uint32_t StreamHandle; + uint32_t FrameCount; + const void* Frames; + uint32_t* WrittenFramesOut; +} SYS_AUDIO_WRITE_REQUEST; + +#define SYS_AUDIO_WAIT_WRITABLE 0x00000001u +#define SYS_AUDIO_WAIT_CONSUMED 0x00000002u +#define SYS_AUDIO_WAIT_PRESENTED 0x00000004u +#define SYS_AUDIO_WAIT_STOPPED 0x00000008u +#define SYS_AUDIO_WAIT_DRAINED 0x00000010u +#define SYS_AUDIO_WAIT_FAILED 0x00000020u +#define SYS_AUDIO_WAIT_DISCONTINUITY 0x00000040u +#define SYS_AUDIO_WAIT_VALID_MASK 0x0000007Fu + +// Arms the stream handle event for one or more progress conditions. +typedef struct SYS_AUDIO_ARM_WAIT_REQUEST { + uint32_t Size; + uint32_t Flags; + uint32_t StreamHandle; + uint32_t Mask; + // Frame counters are inclusive absolute targets and generations are change baselines. + uint32_t WritableFrames; + uint32_t Reserved; + uint64_t ConsumedFrames; + uint64_t PresentedFrames; + uint64_t DrainedGeneration; + uint64_t DiscontinuityGeneration; +} SYS_AUDIO_ARM_WAIT_REQUEST; + +// Identifies the public lifecycle state of one native render stream. +typedef enum SYS_AUDIO_STREAM_STATE { + SysAudioStreamCreated = 1, + SysAudioStreamRunning, + SysAudioStreamPaused, + SysAudioStreamDraining, + SysAudioStreamStopped, + SysAudioStreamFailed +} SYS_AUDIO_STREAM_STATE; + +// Reports one coherent stream snapshot. +typedef struct SYS_AUDIO_STREAM_INFO { + uint32_t Size; + uint32_t Version; + uint32_t State; + uint32_t Reserved; + SYS_AUDIO_FORMAT Format; + uint32_t CapacityFrames; + uint32_t BufferedFrames; + uint32_t WritableFrames; + KSTATUS FailureStatus; + uint64_t FramesSubmitted; + uint64_t FramesConsumed; + uint64_t FramesPresented; + uint64_t FramesDiscarded; + uint64_t OutputFramesMixed; + uint64_t SilenceFramesInserted; + // Underruns count recoverable source starvation periods. + uint64_t Underruns; + uint64_t DrainedGeneration; + // Discontinuities report explicit control or hardware timeline changes. + uint64_t DiscontinuityGeneration; + uint32_t VolumeQ16; + int32_t PanQ16; + uint8_t Muted; + uint8_t ReservedBytes[7]; +} SYS_AUDIO_STREAM_INFO; + +// Updates one stream's software gain, balance, and mute targets. +// Volume uses unsigned Q16 in [0, 1] and pan uses signed Q16 in [-1, 1]. +typedef struct SYS_AUDIO_SET_VOLUME_REQUEST { + uint32_t Size; + uint32_t Flags; + uint32_t StreamHandle; + uint32_t Muted; + uint32_t VolumeQ16; + int32_t PanQ16; +} SYS_AUDIO_SET_VOLUME_REQUEST; + +// Updates one endpoint's master software gain and mute state. +typedef struct SYS_AUDIO_SET_MASTER_VOLUME_REQUEST { + uint32_t Size; + uint32_t Flags; + uint64_t EndpointId; + uint32_t Muted; + uint32_t VolumeQ16; +} SYS_AUDIO_SET_MASTER_VOLUME_REQUEST; + +// Enumerates render endpoints and returns the complete count even when capacity is insufficient. +KSTATUS SysAudioEnumerateEndpoints(const SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST* request); + +// Opens one waitable, process-owned native render stream handle. +KSTATUS SysAudioOpenStream(const SYS_AUDIO_OPEN_STREAM_REQUEST* request); + +// Copies as many complete frames as currently fit and returns WOULD_BLOCK when none fit. +KSTATUS SysAudioWrite(const SYS_AUDIO_WRITE_REQUEST* request); + +// Replaces the any-condition progress targets that drive one stream handle's wait state. +KSTATUS SysAudioArmWait(const SYS_AUDIO_ARM_WAIT_REQUEST* request); + +// Copies one coherent stream snapshot after the caller initializes infoOut->Size. +KSTATUS SysAudioQueryStream(uint32_t streamHandle, SYS_AUDIO_STREAM_INFO* infoOut); + +// Starts one created or stopped stream. +KSTATUS SysAudioStart(uint32_t streamHandle); + +// Pauses future mixing while preserving unconsumed source frames. +KSTATUS SysAudioPause(uint32_t streamHandle); + +// Resumes one paused stream. +KSTATUS SysAudioResume(uint32_t streamHandle); + +// Rejects new writes and stops after all submitted frames reach a terminal disposition. +KSTATUS SysAudioDrain(uint32_t streamHandle); + +// Stops future mixing and discards source frames not yet committed to hardware periods. +KSTATUS SysAudioStop(uint32_t streamHandle); + +// Returns one stopped stream to an empty position-zero state. +KSTATUS SysAudioReset(uint32_t streamHandle); + +// Updates one stream's software volume, pan, and mute state. +KSTATUS SysAudioSetVolume(const SYS_AUDIO_SET_VOLUME_REQUEST* request); + +// Updates one endpoint's master software volume and mute state. +KSTATUS SysAudioSetMasterVolume(const SYS_AUDIO_SET_MASTER_VOLUME_REQUEST* request); diff --git a/code/public/ntlite/ddi/display.h b/code/public/ntlite/ddi/display.h index bdb1392..4eb5508 100644 --- a/code/public/ntlite/ddi/display.h +++ b/code/public/ntlite/ddi/display.h @@ -25,7 +25,13 @@ typedef struct DISPLAY_DEVICE_OPS { KSTATUS (*Blit)(DISPLAY_DEVICE* device, int x, int y, int w, int h, - const uint32_t* pixels); + const uint32_t* pixels, + uint32_t pitchBytes); + + KSTATUS (*PresentPitched)(DISPLAY_DEVICE* device, + int x, int y, int w, int h, + const uint32_t* pixels, + uint32_t pitchBytes); KSTATUS (*FillRect)(DISPLAY_DEVICE* device, int x, int y, int w, int h, diff --git a/code/public/ntlite/ddi/io_driver.h b/code/public/ntlite/ddi/io_driver.h index ec5a7b5..6d7ccf4 100644 --- a/code/public/ntlite/ddi/io_driver.h +++ b/code/public/ntlite/ddi/io_driver.h @@ -20,6 +20,7 @@ typedef void (*DRIVER_UNLOAD_FN)(void); typedef struct DRIVER_OBJECT DRIVER_OBJECT; typedef struct DEVICE_OBJECT DEVICE_OBJECT; typedef struct IRP IRP; +struct IO_DMA_ADAPTER; typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, IRP* irp); typedef KSTATUS (*PDRIVER_ADD_DEVICE)(DRIVER_OBJECT* driver, @@ -89,6 +90,7 @@ struct DEVICE_OBJECT { DEVICE_OBJECT* LowerDevice; // device directly below this FDO/filter void* DeviceExtension; + struct IO_DMA_ADAPTER* DmaAdapter; uint32_t Flags; uint32_t Characteristics; @@ -183,17 +185,6 @@ typedef struct IO_DISK_STATS { uint64_t WriteSectors; uint64_t MaxReadSectors; uint64_t MaxWriteSectors; - uint64_t ReadaheadHits; - uint64_t ReadaheadMisses; - uint64_t ReadaheadStores; - uint64_t ReadaheadInvalidations; - uint64_t ReadaheadBytesServed; - uint64_t ReadaheadSectorsLoaded; - uint64_t ReadaheadSequentialDetections; - uint64_t ReadaheadSuppressed; - uint64_t ReadaheadAllocFailures; - uint64_t ReadaheadReadFailures; - uint64_t MaxReadaheadSectors; } IO_DISK_STATS; __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex, @@ -220,4 +211,5 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex, uint64_t lba, uint32_t count, const void* buffer); +__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex); __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out); diff --git a/code/public/ntlite/ddk/audio.h b/code/public/ntlite/ddk/audio.h new file mode 100644 index 0000000..ecabab2 --- /dev/null +++ b/code/public/ntlite/ddk/audio.h @@ -0,0 +1,105 @@ +// Public nt-lite kernel-audio provider declarations. + +#pragma once + +#include +#include +#include + +struct DEVICE_OBJECT; +typedef struct IO_AUDIO_ENDPOINT IO_AUDIO_ENDPOINT; + +// Identifies one logical render destination rather than one codec pin. +typedef enum IO_AUDIO_ENDPOINT_KIND { + IoAudioEndpointLineOut = 1, + IoAudioEndpointSpeakerHeadphone, + IoAudioEndpointOtherAnalog +} IO_AUDIO_ENDPOINT_KIND; + +// Identifies the canonical interleaved PCM sample representation. +typedef enum IO_AUDIO_SAMPLE_FORMAT { + IoAudioSampleS16 = 1 +} IO_AUDIO_SAMPLE_FORMAT; + +// Describes interleaved PCM frames. +typedef struct IO_AUDIO_FORMAT { + uint32_t SampleRate; + uint16_t ChannelCount; + uint16_t SampleFormat; +} IO_AUDIO_FORMAT; + +#define IO_AUDIO_ENDPOINT_CAP_JACK_SENSE 0x00000001u +#define IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN 0x00000002u +#define IO_AUDIO_ENDPOINT_CAP_VALID_MASK 0x00000003u + +// Bounds provider-advertised hardware gain ranges in millidecibels. +#define IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB 200000 + +// Distinguishes orderly teardown from surprise removal where registers may no longer be accessible. +typedef enum IO_AUDIO_REMOVAL_MODE { + IoAudioOrderlyRemoval = 1, + IoAudioSurpriseRemoval +} IO_AUDIO_REMOVAL_MODE; + +#define IO_AUDIO_FAILURE_HARDWARE_INACCESSIBLE 0x00000001u +#define IO_AUDIO_FAILURE_VALID_MASK 0x00000001u + +// Provider operations are invoked only at PASSIVE_LEVEL by the endpoint mixer thread. +typedef struct IO_AUDIO_PROVIDER_OPERATIONS { + // Starts DMA only after the ring contains at least two prepared periods. + // Failure must leave the provider stopped; Stop may still be called for partial-start cleanup. + KSTATUS (*Start)(void* providerContext); + // Stops DMA before unregister permits provider storage to be released. + void (*Stop)(void* providerContext); + // Applies endpoint gain and mute only with IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN. + KSTATUS (*SetGain)(void* providerContext, int32_t gainMilliDb, uint8_t muted); +} IO_AUDIO_PROVIDER_OPERATIONS; + +// Registers one fixed-format render ring owned by a hardware provider. +// The provider must keep DMA stopped and the physical output muted until Start is called. +typedef struct IO_AUDIO_ENDPOINT_REGISTRATION { + struct DEVICE_OBJECT* Device; + IO_AUDIO_ENDPOINT_KIND Kind; + uint32_t Capabilities; + uint8_t InitiallyAvailable; + void* ProviderContext; + IO_AUDIO_PROVIDER_OPERATIONS Operations; + void* PeriodBuffer; + size_t PeriodBufferBytes; + uint32_t PeriodCount; + uint32_t FramesPerPeriod; + size_t PeriodStrideBytes; + IO_AUDIO_FORMAT HardwareFormat; + // Gain fields must be zero unless IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN is present. + int32_t HardwareGainMinimumMilliDb; + int32_t HardwareGainMaximumMilliDb; + uint32_t HardwareGainStepMilliDb; +} IO_AUDIO_ENDPOINT_REGISTRATION; + +// Driver-visible services for registering and advancing kernel audio endpoints. +typedef struct NTL_DDK_AUDIO { + // Registers one endpoint and returns a provider-owned reference valid until unregister returns. + KSTATUS (*RegisterEndpoint)(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, + IO_AUDIO_ENDPOINT** outEndpoint); + + // Stops the endpoint, removes it from enumeration, and consumes the provider-owned reference. + void (*UnregisterEndpoint)(IO_AUDIO_ENDPOINT* endpoint, IO_AUDIO_REMOVAL_MODE mode); + + // Reports one completed hardware period from ISR/DPC completion processing. + // presentedFrames, hardwarePositionFrames, and interruptCount must be monotonic for the + // endpoint lifetime. timestampTicks uses the DDK monotonic microsecond source. + void (*NotifyPeriod)(IO_AUDIO_ENDPOINT* endpoint, + uint32_t completedPeriodIndex, + uint64_t presentedFrames, + uint64_t hardwarePositionFrames, + uint64_t interruptCount, + uint64_t timestampTicks); + + // Updates jack-derived endpoint availability without changing endpoint identity. + void (*SetEndpointAvailable)(IO_AUDIO_ENDPOINT* endpoint, uint8_t available); + + // Fails an endpoint and wakes every stream owner. + void (*ReportEndpointFailure)(IO_AUDIO_ENDPOINT* endpoint, + KSTATUS status, + uint32_t flags); +} NTL_DDK_AUDIO; diff --git a/code/public/ntlite/ddk/ddk.h b/code/public/ntlite/ddk/ddk.h index 9ad13de..b24ee96 100644 --- a/code/public/ntlite/ddk/ddk.h +++ b/code/public/ntlite/ddk/ddk.h @@ -4,7 +4,9 @@ #include #include +#include #include +#include #include #include #include @@ -25,12 +27,14 @@ #include #include -#define NTL_DDK_VERSION 2u +#define NTL_DDK_VERSION 1u // Root DDK table passed to kernel drivers by the driver loader. typedef struct NTL_DDK { uint32_t Version; const NTL_DDK_DEBUG* Debug; + const NTL_DDK_AUDIO* Audio; + const NTL_DDK_DMA* Dma; const NTL_DDK_INTERRUPT* Interrupt; const NTL_DDK_MEMORY* Memory; const NTL_DDK_PLATFORM* Platform; diff --git a/code/public/ntlite/ddk/debug.h b/code/public/ntlite/ddk/debug.h index 18ccd3b..b6863ee 100644 --- a/code/public/ntlite/ddk/debug.h +++ b/code/public/ntlite/ddk/debug.h @@ -2,11 +2,29 @@ #pragma once +#include #include +#define KE_DEBUG_CONSOLE 0x00000001u +#define KE_DEBUG_SELFTEST 0x00000002u +#define KE_DEBUG_FILE_LOG 0x00000004u +#define KE_DEBUG_POOL_CANARY 0x00000008u +#define KE_DEBUG_POOL_POISON 0x00000010u +#define KE_DEBUG_POOL_QUARANTINE 0x00000020u +#define KE_DEBUG_POOL_CALLERS 0x00000040u +#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u +#define KE_DEBUG_DISPLAY_TRACE 0x00000100u +#define KE_DEBUG_INPUT_TRACE 0x00000200u +#define KE_DEBUG_VM_ASSERTS 0x00000400u +#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u +#define KE_DEBUG_LIST_CHECKS 0x00001000u +#define KE_DEBUG_AUDIO 0x00002000u +#define KE_DEBUG_AUDIO_SELFTEST 0x00004000u + // Driver-visible debug output service table passed through NTL_DDK. typedef struct NTL_DDK_DEBUG { void (*Print)(const char* fmt, ...); + uint32_t (*GetFlags)(void); } NTL_DDK_DEBUG; // Current fatal categories used by kernel-mode driver diagnostics. diff --git a/code/public/ntlite/ddk/display.h b/code/public/ntlite/ddk/display.h index 0f571b6..0c7c96a 100644 --- a/code/public/ntlite/ddk/display.h +++ b/code/public/ntlite/ddk/display.h @@ -21,6 +21,8 @@ typedef struct KE_DISPLAY_INFO { DISPLAY_MODE Mode; uint32_t Caps; uint32_t DeviceCount; + uint32_t Flags; + uint32_t Reserved; DISPLAY_STATS ManagerStats; DISPLAY_STATS DeviceStats; } KE_DISPLAY_INFO; @@ -40,10 +42,9 @@ DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode); // Queries the current display size. DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight); -typedef KSTATUS (*KE_DISPLAY_ROW_SOURCE)(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount); - DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels); -DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch); +DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); +DISPLAY_API KSTATUS KeDisplayPresentPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes); DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color); DISPLAY_API KSTATUS KeDisplayClear(uint32_t color); DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h); diff --git a/code/public/ntlite/ddk/dma.h b/code/public/ntlite/ddk/dma.h new file mode 100644 index 0000000..eb651ec --- /dev/null +++ b/code/public/ntlite/ddk/dma.h @@ -0,0 +1,146 @@ +// Public nt-lite DMA declarations. + +#pragma once + +#include +#include +#include + +struct DEVICE_OBJECT; +struct PROCESS; +typedef struct IO_DMA_ADAPTER IO_DMA_ADAPTER; +typedef struct IO_DMA_BUFFER IO_DMA_BUFFER; +typedef struct IO_DMA_MAPPING IO_DMA_MAPPING; +typedef struct MM_PINNED_MEMORY MM_PINNED_MEMORY; + +typedef enum IO_DMA_DIRECTION { + IoDmaToDevice = 1, + IoDmaFromDevice, + IoDmaBidirectional +} IO_DMA_DIRECTION; + +// Fails a streaming mapping instead of allocating a bounce buffer. +#define IO_DMA_MAP_NO_BOUNCE 0x00000001u + +// Describes immutable constraints shared by one device adapter. Address, +// alignment, and boundary limits apply to common buffers. Segment limits apply +// to streaming mappings. Zero selects the unconstrained default for each field. +typedef struct IO_DMA_REQUIREMENTS { + uint64_t MaximumDeviceAddress; + size_t Alignment; + size_t Boundary; + size_t MaximumSegmentLength; + uint32_t MaximumSegments; +} IO_DMA_REQUIREMENTS; + +// Describes one device-visible span in a streaming mapping. +typedef struct IO_DMA_SEGMENT { + uint64_t DeviceAddress; + size_t Length; +} IO_DMA_SEGMENT; + +// Reports cumulative activity and live ownership for validation. +typedef struct IO_DMA_STATS { + uint64_t AdaptersCreated; + uint64_t AdaptersDestroyed; + uint64_t AdaptersInUse; + uint64_t CommonBuffersAllocated; + uint64_t CommonBuffersFreed; + uint64_t CommonBuffersInUse; + uint64_t CommonBufferBytesInUse; + uint64_t PinnedKernelRanges; + uint64_t PinnedUserRanges; + uint64_t PinnedRangesReleased; + uint64_t PinnedRangesInUse; + uint64_t PinnedPagesInUse; + uint64_t StreamingMappings; + uint64_t StreamingMappingsUnmapped; + uint64_t StreamingMappingsInUse; + uint64_t DirectMappings; + uint64_t BounceMappings; + uint64_t BounceBytesCopiedToDevice; + uint64_t BounceBytesCopiedToCpu; + uint64_t FailedAllocations; + uint64_t FailedMappings; +} IO_DMA_STATS; + +// Driver-visible DMA services. Device addresses are intentionally distinct +// from physical addresses even while the first implementation maps them one to +// one on x86-64. +typedef struct NTL_DDK_DMA { + // Returns the one adapter and requirement set owned by a device object. + KSTATUS (*GetAdapter)(struct DEVICE_OBJECT* device, + const IO_DMA_REQUIREMENTS* requirements, + IO_DMA_ADAPTER** outAdapter); + + // Releases one caller-owned adapter reference. + void (*PutAdapter)(IO_DMA_ADAPTER* adapter); + + // Allocates coherent memory with stable CPU and base device addresses. The + // caller must quiesce every device reference before releasing the buffer. + KSTATUS (*AllocateCommonBuffer)(IO_DMA_ADAPTER* adapter, + size_t size, + size_t alignment, + IO_DMA_BUFFER** outBuffer); + + // Releases a coherent common buffer. + void (*FreeCommonBuffer)(IO_DMA_BUFFER* buffer); + + // Returns the stable CPU address of a common buffer. + void* (*GetCommonBufferAddress)(const IO_DMA_BUFFER* buffer); + + // Returns the stable device address of a common buffer. + uint64_t (*GetCommonBufferDeviceAddress)(const IO_DMA_BUFFER* buffer); + + // Returns the caller-requested size of a common buffer. + size_t (*GetCommonBufferSize)(const IO_DMA_BUFFER* buffer); + + // Stabilizes a kernel range for one or more streaming mappings. Pinning + // preserves the pages, but does not transfer ownership to a device. + KSTATUS (*PinKernelMemory)(void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory); + + // Stabilizes a process-owned user range for one or more mappings. The + // descriptor retains the process and range until the final unpin. + KSTATUS (*PinUserMemory)(struct PROCESS* process, + void* address, + size_t size, + IO_DMA_DIRECTION direction, + MM_PINNED_MEMORY** outMemory); + + // Releases one pinned-memory reference after all mappings leave. + void (*UnpinMemory)(MM_PINNED_MEMORY* memory); + + // Maps a pinned subrange and creates a constrained segment list. A new + // mapping begins CPU-owned and requires external operation serialization. + KSTATUS (*MapPinnedMemory)(IO_DMA_ADAPTER* adapter, + MM_PINNED_MEMORY* memory, + size_t offset, + size_t length, + IO_DMA_DIRECTION direction, + uint32_t flags, + IO_DMA_MAPPING** outMapping); + + // Releases a mapping and performs a final receive-side bounce copy only + // when ownership was transferred to the device and not returned to the CPU. + void (*UnmapMemory)(IO_DMA_MAPPING* mapping); + + // Transfers mapping ownership from the CPU to the device. Streaming DMA + // must not begin before this operation succeeds. + KSTATUS (*SynchronizeForDevice)(IO_DMA_MAPPING* mapping); + + // Transfers mapping ownership from the device to the CPU. CPU readers must + // not consume receive data before this operation succeeds. + KSTATUS (*SynchronizeForCpu)(IO_DMA_MAPPING* mapping); + + // Returns the immutable segment count owned by a mapping. + uint32_t (*GetSegmentCount)(const IO_DMA_MAPPING* mapping); + + // Returns the immutable segment array owned by a mapping. + const IO_DMA_SEGMENT* (*GetSegments)(const IO_DMA_MAPPING* mapping); + + // Copies current DMA counters into caller-owned storage. + void (*QueryStats)(IO_DMA_STATS* outStats); +} NTL_DDK_DMA; diff --git a/code/public/ntlite/ddk/file.h b/code/public/ntlite/ddk/file.h index ec0b760..ad27a7c 100644 --- a/code/public/ntlite/ddk/file.h +++ b/code/public/ntlite/ddk/file.h @@ -59,6 +59,7 @@ typedef struct IO_FILESYSTEM { KSTATUS (*QueryInfo)(struct IO_FILESYSTEM* fs, void* fileCtx, SYS_FILE_STAT* out); KSTATUS (*Stat)(struct IO_FILESYSTEM* fs, SYS_STRING_VIEW path, SYS_FILE_STAT* out); KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx, IO_FILE_IDENTITY* out); + KSTATUS (*Flush)(struct IO_FILESYSTEM* fs, void* fileCtx); void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx); void (*QueryStats)(struct IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out); void* DriverCtx; @@ -84,10 +85,6 @@ typedef struct FILE_OBJECT { uint64_t Size; uint64_t Offset; uint32_t Flags; - uint32_t ReadaheadScore; - uint64_t ReadaheadNextOffset; - uint64_t ReadaheadIssuedUntil; - uint64_t ReadaheadSuppressedUntil; } FILE_OBJECT; typedef struct DIR_OBJECT { diff --git a/code/public/ntlite/ddk/irp.h b/code/public/ntlite/ddk/irp.h index c6a5bbe..ece9216 100644 --- a/code/public/ntlite/ddk/irp.h +++ b/code/public/ntlite/ddk/irp.h @@ -23,11 +23,12 @@ typedef enum IO_MAJOR_FUNCTION { IRP_MJ_CLOSE = 1, IRP_MJ_READ = 2, IRP_MJ_WRITE = 3, - IRP_MJ_DEVICE_CONTROL = 4, - IRP_MJ_INTERNAL_DEVICE_CONTROL = 5, - IRP_MJ_PNP = 6, - IRP_MJ_POWER = 7, - IRP_MJ_MAXIMUM_FUNCTION = 8, + IRP_MJ_FLUSH_BUFFERS = 4, + IRP_MJ_DEVICE_CONTROL = 5, + IRP_MJ_INTERNAL_DEVICE_CONTROL = 6, + IRP_MJ_PNP = 7, + IRP_MJ_POWER = 8, + IRP_MJ_MAXIMUM_FUNCTION = 9, } IO_MAJOR_FUNCTION; // PnP minor functions. @@ -159,10 +160,10 @@ __declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp); __declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q); // Cancels every IRP in a queue and invokes a callback for each one. -__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q, - KSTATUS status, - IRP_QUEUE_CANCEL_CALLBACK callback, - void* context); +__declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q, + KSTATUS status, + IRP_QUEUE_CANCEL_CALLBACK callback, + void* context); // Cancels every IRP in a queue. __declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status); diff --git a/code/public/ntlite/ddk/memory.h b/code/public/ntlite/ddk/memory.h index c9450ec..f1fe3d2 100644 --- a/code/public/ntlite/ddk/memory.h +++ b/code/public/ntlite/ddk/memory.h @@ -14,12 +14,14 @@ #define PAGE_MASK (~(PAGE_SIZE - 1ULL)) #endif -#define NTL_DDK_PAGE_SIZE PAGE_SIZE -#define NTL_DDK_PAGE_PRESENT (1ULL << 0) -#define NTL_DDK_PAGE_WRITE (1ULL << 1) -#define NTL_DDK_PAGE_WRITETHROUGH (1ULL << 3) -#define NTL_DDK_PAGE_NOCACHE (1ULL << 4) -#define NTL_DDK_PAGE_NX (1ULL << 63) +#define NTL_DDK_PAGE_SIZE PAGE_SIZE +#define NTL_DDK_PAGE_PRESENT (1ULL << 0) +#define NTL_DDK_PAGE_WRITE (1ULL << 1) +#define NTL_DDK_PAGE_CACHE_WB 0ULL +#define NTL_DDK_PAGE_CACHE_WC (1ULL << 3) +#define NTL_DDK_PAGE_CACHE_UC (1ULL << 4) +#define NTL_DDK_PAGE_CACHE_MASK (NTL_DDK_PAGE_CACHE_WC | NTL_DDK_PAGE_CACHE_UC) +#define NTL_DDK_PAGE_NX (1ULL << 63) #ifndef PAGE_PRESENT #define PAGE_PRESENT (1ULL << 0) @@ -30,11 +32,17 @@ #ifndef PAGE_USER #define PAGE_USER (1ULL << 2) #endif -#ifndef PAGE_WRITETHROUGH -#define PAGE_WRITETHROUGH (1ULL << 3) +#ifndef PAGE_CACHE_WB +#define PAGE_CACHE_WB 0ULL #endif -#ifndef PAGE_NOCACHE -#define PAGE_NOCACHE (1ULL << 4) +#ifndef PAGE_CACHE_WC +#define PAGE_CACHE_WC (1ULL << 3) +#endif +#ifndef PAGE_CACHE_UC +#define PAGE_CACHE_UC (1ULL << 4) +#endif +#ifndef PAGE_CACHE_MASK +#define PAGE_CACHE_MASK (PAGE_CACHE_WC | PAGE_CACHE_UC) #endif #ifndef PAGE_ACCESSED #define PAGE_ACCESSED (1ULL << 5) @@ -56,55 +64,6 @@ typedef enum NTL_DDK_MAP_MODE { NtlDdkMapReplaceExisting = 1, } NTL_DDK_MAP_MODE; -#ifndef NTL_MM_PTE_MAP_MODE_DEFINED -typedef enum MM_PTE_MAP_MODE { - MmPteMapFailIfPresent = 0, - MmPteMapReplaceExisting = 1, -} MM_PTE_MAP_MODE; -#define NTL_MM_PTE_MAP_MODE_DEFINED 1 -#endif - -#ifndef NTL_MM_PFN_TYPES_DEFINED -typedef enum MM_PFN_STATE { - MmPfnStateInvalid = 0, - MmPfnStateReserved, - MmPfnStateFree, - MmPfnStateZeroed, - MmPfnStateActive, - MmPfnStateStandby, - MmPfnStateModified, -} MM_PFN_STATE; - -typedef enum MM_PFN_USAGE { - MmPfnUseNone = 0, - MmPfnUseKernel, - MmPfnUsePageTable, - MmPfnUseUser, - MmPfnUseCleanFileCache, - MmPfnUseImage, - MmPfnUseMappedFile, - MmPfnUseAnonymous, - MmPfnUseCow, - MmPfnUseSurface, - MmPfnUseDriver, - MmPfnUseDma, - MmPfnUseBootloader, - MmPfnUsePfnDatabase, - MmPfnUseMmio, -} MM_PFN_USAGE; - -typedef struct MM_PFN { - uint16_t State; - uint16_t Usage; - uint16_t RefCount; - uint16_t PinCount; - uint32_t Flags; - uint32_t MappedCount; - void* Owner; - uint64_t BackingPage; -} MM_PFN; -#define NTL_MM_PFN_TYPES_DEFINED 1 -#endif #define MM_POOL_TAG(a, b, c, d) \ ((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \ @@ -131,56 +90,3 @@ __declspec(dllexport) void MmFreePool(void* ptr); // Reallocates kernel pool memory. __declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize); - -// Allocates one physical frame for a specific driver-visible usage. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage); - -// Allocates one physical frame below maxExclusive. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage); - -// Allocates one physical frame in a range. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive, - PHYSICAL_ADDRESS maxExclusive, - MM_PFN_USAGE usage); - -// Allocates contiguous physical frames. -__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage); - -// Releases a frame reference. -__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame); - -// Retains a physical frame. -__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame); - -// Releases a retained physical frame. -__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame); - -// Pins a physical frame. -__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame); - -// Unpins a physical frame. -__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame); - -// Returns the PFN database entry for a frame. -__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame); - -// Returns the direct-map alias for a physical frame. -__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame); - -// Maps one page into the shared kernel address space. -__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, - PHYSICAL_ADDRESS phys, - uint64_t flags, - MM_PTE_MAP_MODE mode); - -// Removes one page mapping from the shared kernel address space. -__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt); - -// Replaces flags on a mapped range. -__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags); - -// Translates a virtual address through the shared kernel page tables. -__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt); - -// Returns the physical address and flags for a mapped page. -__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut); diff --git a/code/public/ntlite/ddk/mutex.h b/code/public/ntlite/ddk/mutex.h index 1dc405b..b00628f 100644 --- a/code/public/ntlite/ddk/mutex.h +++ b/code/public/ntlite/ddk/mutex.h @@ -13,16 +13,23 @@ typedef struct MUTEX { SPINLOCK Lock; THREAD* Owner; LIST_ENTRY WaitList; + LIST_ENTRY OwnerLink; + KE_SCHEDULING_CLASS DonationClass; + uint32_t DonationPriority; + uint8_t HasDonation; + uint8_t Abandoned; } MUTEX; +// Tests whether a mutex acquisition transferred ownership to the caller. +#define KE_MUTEX_ACQUIRE_SUCCEEDED(status) \ + ((status) == NTLSTATUS_SUCCESS || (status) == NTLSTATUS_ABANDONED) + // Creates a mutex object. __declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out); -// Acquires a mutex, blocking when necessary. -__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex); - -// Attempts to acquire a mutex without blocking. -__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex); +// Acquires a mutex within the requested timeout and reports abandonment on transfer. +// Zero performs a nonblocking probe; KE_WAIT_INFINITE waits without a timeout. +__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds); // Releases a mutex. __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex); diff --git a/code/public/ntlite/ddk/object.h b/code/public/ntlite/ddk/object.h index 98bf575..17a61b9 100644 --- a/code/public/ntlite/ddk/object.h +++ b/code/public/ntlite/ddk/object.h @@ -29,6 +29,8 @@ typedef enum OBJECT_TYPE { ObjectTypeTimer = 19, ObjectTypeModule = 20, ObjectTypeDisplayClock = 21, + ObjectTypeAudioEndpoint = 22, + ObjectTypeAudioStream = 23, } OBJECT_TYPE; typedef enum OBJECT_LIFETIME_STATE { diff --git a/code/public/ntlite/ddk/semaphore.h b/code/public/ntlite/ddk/semaphore.h index 74db887..35eac0b 100644 --- a/code/public/ntlite/ddk/semaphore.h +++ b/code/public/ntlite/ddk/semaphore.h @@ -19,11 +19,9 @@ typedef struct SEMAPHORE { // Creates a counting semaphore. __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE** out); -// Waits for a semaphore count to become available. -__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore); - -// Attempts to take a semaphore without blocking. -__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* semaphore); +// Waits for a semaphore count within the requested timeout. +// Zero performs a nonblocking probe; KE_WAIT_INFINITE waits without a timeout. +__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore, uint64_t timeoutMilliseconds); // Releases one semaphore count. __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* semaphore); diff --git a/code/public/ntlite/ddk/sync.h b/code/public/ntlite/ddk/sync.h index d4cb9f0..56c548d 100644 --- a/code/public/ntlite/ddk/sync.h +++ b/code/public/ntlite/ddk/sync.h @@ -14,6 +14,9 @@ typedef struct SPINLOCK { #define SPINLOCK_INIT { 0 } +// Requests a synchronization wait without a timeout. +#define KE_WAIT_INFINITE UINT64_MAX + #define KI_SPINLOCK_IRQL_SHIFT ((unsigned)(sizeof(uintptr_t) * 8u - 8u)) #define KI_SPINLOCK_FLAGS_MASK (~(((uintptr_t)0xFFu) << KI_SPINLOCK_IRQL_SHIFT)) diff --git a/code/public/ntlite/ddk/thread.h b/code/public/ntlite/ddk/thread.h index 28187e3..d0c0228 100644 --- a/code/public/ntlite/ddk/thread.h +++ b/code/public/ntlite/ddk/thread.h @@ -17,18 +17,49 @@ typedef enum THREAD_PRIORITY { #define NTL_THREAD_PRIORITY_DEFINED 1 #endif -// Creates a system-process kernel thread for a kernel-mode driver. +// Selects dynamic normal scheduling or static real-time scheduling. +typedef enum KE_SCHEDULING_CLASS { + KeSchedulingNormal = 0, + KeSchedulingRealtime = 1, + KeSchedulingClassCount = 2, +} KE_SCHEDULING_CLASS; + +// Higher real-time values outrank lower values. Real-time threads always +// outrank normal threads and round-robin with equal-priority peers. +#define KE_REALTIME_PRIORITY_MIN 0u +#define KE_REALTIME_PRIORITY_MAX 31u +#define KE_REALTIME_PRIORITY_COUNT 32u + +// Creates a thread without placing it on a scheduler ready queue. The caller +// must retain the returned THREAD reference and call KeResumeThread(). +#define KE_THREAD_CREATE_FLAG_SUSPENDED 0x00000001u + +// Describes initial scheduling, execution state, and affinity for one new +// thread. Zero affinity selects the kernel default rather than an empty CPU +// set. +typedef struct KE_THREAD_CREATE_PARAMETERS { + KE_SCHEDULING_CLASS SchedulingClass; + uint32_t Priority; + uint32_t Flags; + uint64_t AffinityMask; +} KE_THREAD_CREATE_PARAMETERS; + +#define KE_THREAD_CREATE_NORMAL(priorityValue) \ + { KeSchedulingNormal, (uint32_t)(priorityValue), 0u, 0ULL } + +#define KE_THREAD_CREATE_REALTIME(priorityValue) \ + { KeSchedulingRealtime, (uint32_t)(priorityValue), 0u, 0ULL } + +// Creates a system-process kernel thread with explicit scheduling parameters. __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, + const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out); -// Creates a system-process kernel thread with an explicit affinity mask. -__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, - uintptr_t arg, - THREAD_PRIORITY priority, - uint64_t affinityMask, - THREAD** out); +// Decrements one thread's suspend count and queues it when the count reaches +// zero. previousSuspendCountOut may be NULL. +__declspec(dllexport) KSTATUS KeResumeThread(THREAD* thread, + uint32_t* previousSuspendCountOut); // Yields the current processor to another runnable thread. __declspec(dllexport) void KeYield(void); diff --git a/code/public/ntlite/display.h b/code/public/ntlite/display.h index c978269..9bc9f66 100644 --- a/code/public/ntlite/display.h +++ b/code/public/ntlite/display.h @@ -77,6 +77,8 @@ typedef struct DISPLAY_STATS { uint64_t ReadRectCount; uint64_t PresentCount; uint64_t PresentRectCount; + uint64_t DirectPresentCount; + uint64_t ScanoutBytes; uint64_t DirtyRectCount; uint64_t DirtyAllCount; uint64_t BytesCopied; @@ -98,11 +100,15 @@ typedef struct DISPLAY_STATS { #define SYS_DISPLAY_NAME_MAX 32u +#define DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE 0x00000001u + typedef struct SYS_DISPLAY_INFO { char PrimaryName[SYS_DISPLAY_NAME_MAX]; DISPLAY_MODE Mode; uint32_t Caps; uint32_t DeviceCount; + uint32_t Flags; + uint32_t Reserved; DISPLAY_STATS ManagerStats; DISPLAY_STATS DeviceStats; } SYS_DISPLAY_INFO; diff --git a/code/public/ntlite/status.h b/code/public/ntlite/status.h index fba063f..5362e37 100644 --- a/code/public/ntlite/status.h +++ b/code/public/ntlite/status.h @@ -36,6 +36,7 @@ typedef uint32_t KSTATUS; #define NTLSTATUS_ALERTED KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000003u) #define NTLSTATUS_KERNEL_APC KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000004u) #define NTLSTATUS_USER_APC KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000005u) +#define NTLSTATUS_ABANDONED KSTATUS_MAKE_SUCCESS(NTLSTATUS_FACILITY_WAIT, 0x000006u) // System-facility failures. // NTLSTATUS_WOULD_BLOCK is the canonical result for immediate nonblocking operations that cannot make progress. @@ -59,6 +60,8 @@ typedef uint32_t KSTATUS; #define NTLSTATUS_NO_MORE_FILES KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000011u) #define NTLSTATUS_INVALID_IRQL KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000012u) #define NTLSTATUS_WOULD_BLOCK KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000013u) +#define NTLSTATUS_DEADLOCK KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000014u) +#define NTLSTATUS_DEVICE_REMOVED KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_SYSTEM, 0x000015u) // Process termination and fault statuses use the process facility. #define NTLSTATUS_PROCESS_BASE KSTATUS_MAKE_ERROR(NTLSTATUS_FACILITY_PROCESS, 0x000000u) diff --git a/code/public/ntlite/syscall.h b/code/public/ntlite/syscall.h index 2b8961c..f181621 100644 --- a/code/public/ntlite/syscall.h +++ b/code/public/ntlite/syscall.h @@ -12,6 +12,7 @@ #include #include #include +#include // Syscall ABI convention: status-returning calls use KSTATUS. New or revised // operations use a request struct when they need five or more logical arguments @@ -494,6 +495,9 @@ typedef struct SYS_CREATE_THREAD_REQUEST { uint32_t* ThreadIdOut; } SYS_CREATE_THREAD_REQUEST; +#define SYS_CREATE_THREAD_FLAG_NONE 0x00000000u +#define SYS_CREATE_THREAD_FLAG_SUSPENDED 0x00000001u + void SysExitProcess(int32_t exitCode); void SysExitThread(int32_t exitCode); KSTATUS SysCreateProcess(const SYS_CREATE_PROCESS_REQUEST* request); @@ -533,6 +537,10 @@ KSTATUS SysKillProcess(uint32_t processHandle, int32_t exitCode); #define SYS_THREAD_STATE_RUNNING 1u #define SYS_THREAD_STATE_WAITING 2u #define SYS_THREAD_STATE_DEAD 3u +#define SYS_THREAD_STATE_SUSPENDED 4u + +#define SYS_SCHEDULING_NORMAL 0u +#define SYS_SCHEDULING_REALTIME 1u #define SYS_THREAD_FLAG_CURRENT 0x00000001u #define SYS_THREAD_FLAG_IDLE 0x00000002u @@ -569,7 +577,11 @@ typedef struct SYS_THREAD_INFO { uint32_t ThreadId; uint32_t State; - uint32_t Priority; + uint32_t BaseSchedulingClass; + uint32_t BasePriority; + uint32_t EffectiveSchedulingClass; + uint32_t EffectivePriority; + uint32_t Flags; uint32_t RunningCpu; uint32_t LastCpu; @@ -674,6 +686,7 @@ KSTATUS SysGetModuleExport(const SYS_GET_MODULE_EXPORT_REQUEST* request); KSTATUS SysGetModuleList(SYS_MODULE_INFO* modulesOut, size_t maxCount, size_t* countOut); KSTATUS SysCreateThread(const SYS_CREATE_THREAD_REQUEST* request); +KSTATUS SysResumeThread(uint32_t threadHandle, uint32_t* previousSuspendCountOut); uint32_t SysGetCurrentTid(void); uint32_t SysGetCurrentPid(void); KSTATUS SysSetThreadPriority(uint32_t priority); @@ -771,7 +784,8 @@ KSTATUS SysCloseHandle(uint32_t handle); #define SYS_HANDLE_RIGHT_QUERY 0x00000020u #define SYS_HANDLE_RIGHT_TERMINATE 0x00000040u #define SYS_HANDLE_RIGHT_LOCK 0x00000080u -#define SYS_HANDLE_RIGHT_ALL 0x000000FFu +#define SYS_HANDLE_RIGHT_RESUME 0x00000100u +#define SYS_HANDLE_RIGHT_ALL 0x000001FFu #define SYS_DUPLICATE_HANDLE_FLAG_NONE 0x00000000u #define SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS 0x00000001u @@ -1080,7 +1094,7 @@ KSTATUS SysRecvSocket(uint32_t socket, void* buffer, size_t length, uint32_t fla KSTATUS SysShutdownSocket(uint32_t socket, uint32_t how); KSTATUS SysSendToSocket(uint32_t socket, const SYS_SOCKET_SEND_TO_REQUEST* request); KSTATUS SysRecvFromSocket(uint32_t socket, const SYS_SOCKET_RECV_FROM_REQUEST* request); -KSTATUS SysResolveHost(const SYS_RESOLVE_HOST_REQUEST* request); // Cry some more, microkernel purists! +KSTATUS SysResolveHost(const SYS_RESOLVE_HOST_REQUEST* request); // Cry about it. KSTATUS SysSetSocketOption(uint32_t socket, const SYS_SOCKET_OPTION_REQUEST* request); KSTATUS SysGetSocketOption(uint32_t socket, const SYS_SOCKET_OPTION_REQUEST* request); KSTATUS SysPoll(const SYS_POLL_REQUEST* request); @@ -1147,6 +1161,12 @@ typedef struct SYS_SCHEDULER_STATS { uint64_t CpuBurstDemotions; uint64_t SmtSiblingDeferrals; uint64_t SmtPrimaryPokes; + uint64_t RealtimeDispatches; + uint64_t RealtimePreemptions; + uint64_t PriorityInheritanceBoosts; + uint64_t PriorityInheritanceDrops; + uint64_t PriorityInheritancePropagations; + uint64_t PriorityInheritanceMaxDepth; } SYS_SCHEDULER_STATS; typedef struct SYS_PMM_STATS { @@ -1310,19 +1330,6 @@ typedef struct SYS_FILE_CACHE_STATS { uint64_t DropFailures; uint64_t StandbyTransitions; uint64_t StandbyHits; - uint64_t ReadAheadRequests; - uint64_t ReadAheadPagesRequested; - uint64_t ReadAheadPagesQueued; - uint64_t ReadAheadPagesLoaded; - uint64_t ReadAheadPagesSkipped; - uint64_t ReadAheadFailures; - uint64_t ReadAheadPagesThrottled; - uint64_t MaxReadAheadPagesPerRequest; - uint64_t BatchReads; - uint64_t BatchPagesRequested; - uint64_t BatchPagesLoaded; - uint64_t MaxBatchPagesRequested; - uint64_t MaxBatchPagesLoaded; uint64_t ResidentPages; uint64_t ActivePages; uint64_t StandbyPages; @@ -1360,17 +1367,6 @@ typedef struct SYS_DISK_STATS { uint64_t WriteSectors; uint64_t MaxReadSectors; uint64_t MaxWriteSectors; - uint64_t ReadaheadHits; - uint64_t ReadaheadMisses; - uint64_t ReadaheadStores; - uint64_t ReadaheadInvalidations; - uint64_t ReadaheadBytesServed; - uint64_t ReadaheadSectorsLoaded; - uint64_t ReadaheadSequentialDetections; - uint64_t ReadaheadSuppressed; - uint64_t ReadaheadAllocFailures; - uint64_t ReadaheadReadFailures; - uint64_t MaxReadaheadSectors; } SYS_DISK_STATS; #define SYS_NET_IRP_OPERATION_COUNT 16u diff --git a/code/user/apps/doom/PureDOOM.h b/code/user/apps/doom/PureDOOM.h index 7275eb5..262750d 100644 --- a/code/user/apps/doom/PureDOOM.h +++ b/code/user/apps/doom/PureDOOM.h @@ -7549,15 +7549,25 @@ void doom_init(int argc, const char** argv, int flags) if (!doom_exit) doom_exit = doom_exit_impl; if (!doom_getenv) doom_getenv = doom_getenv_impl; + doom_print("doom_init: allocating screen buffer\n"); screen_buffer = (unsigned char*)doom_malloc(SCREENWIDTH * SCREENHEIGHT); + doom_print("doom_init: screen buffer ready\n"); + + doom_print("doom_init: allocating final screen buffer\n"); final_screen_buffer = (unsigned char*)doom_malloc(SCREENWIDTH * SCREENHEIGHT * 4); + doom_print("doom_init: final screen buffer ready\n"); + + doom_print("doom_init: reading initial time\n"); last_update_time = I_GetTime(); + doom_print("doom_init: initial time ready\n"); myargc = argc; myargv = (char**)argv; // unsafe my ass doom_flags = flags; + doom_print("doom_init: entering D_DoomMain\n"); D_DoomMain(); + doom_print("doom_init: D_DoomMain returned\n"); } @@ -9790,7 +9800,27 @@ void FindResponseFile(void) { int i; + doom_print("FindResponseFile: entered\n"); + if (myargc <= 0) + { + doom_print("FindResponseFile: argc <= 0\n"); + return; + } + if (myargc == 1) + { + doom_print("FindResponseFile: argc == 1, returning\n"); + return; + } + doom_print("FindResponseFile: argc > 1\n"); + for (i = 1; i < myargc; i++) + { + doom_print("FindResponseFile: checking argument\n"); + if (!myargv || !myargv[i]) + { + doom_print("FindResponseFile: invalid argv entry\n"); + return; + } if (myargv[i][0] == '@') { void* handle = 0; @@ -9860,6 +9890,9 @@ void FindResponseFile(void) break; } + } + + doom_print("FindResponseFile: scan complete\n"); } @@ -9871,9 +9904,13 @@ void D_DoomMain(void) int p; char file[256]; + doom_print("D_DoomMain: FindResponseFile\n"); FindResponseFile(); + doom_print("D_DoomMain: FindResponseFile done\n"); + doom_print("D_DoomMain: IdentifyVersion\n"); IdentifyVersion(); + doom_print("D_DoomMain: IdentifyVersion done\n"); modifiedgame = doom_false; @@ -23741,19 +23778,67 @@ void M_SaveDefaults(void) // // M_LoadDefaults // +// Reads one complete defaults line without allowing malformed files to escape the caller's buffer. +static int M_ReadDefaultsLine(void* handle, char* line, int capacity) +{ + int length = 0; + doom_boolean overflow = doom_false; + + if (!handle || !line || capacity < 2) + return 0; + + for (;;) + { + char c = 0; + int read = doom_read(handle, &c, 1); + if (read != 1) + { + if (!length && !overflow) + return 0; + break; + } + + if (c == '\n') + break; + if (c == '\r') + continue; + + if (length + 1 < capacity) + line[length++] = c; + else + overflow = doom_true; + } + + line[length] = 0; + return overflow ? -1 : 1; +} + + +// Returns the first non-whitespace byte in a mutable defaults token. +static char* M_SkipDefaultWhitespace(char* text) +{ + while (*text == ' ' || *text == '\t') + text++; + return text; +} + + +// Removes horizontal whitespace from the end of a mutable defaults token. +static void M_TrimDefaultWhitespace(char* text) +{ + int length = (int)doom_strlen(text); + while (length > 0 && (text[length - 1] == ' ' || text[length - 1] == '\t')) + text[--length] = 0; +} + + +// Loads bounded name/value records from the optional defaults file. void M_LoadDefaults(void) { + char line[384]; int i; - int len; - void* f; - char def[80]; - char strparm[100]; - char* newstring; - int parm; - doom_boolean isstring; - // set everything to base values - // numdefaults = sizeof(defaults)/sizeof(defaults[0]); + // Set everything to base values. for (i = 0; i < numdefaults; i++) { if (defaults[i].defaultvalue == 0xFFFF) @@ -23762,104 +23847,80 @@ void M_LoadDefaults(void) *defaults[i].location = (int)defaults[i].defaultvalue; } - // check for a custom default file + // Check for a custom default file. i = M_CheckParm((char*)"-config"); if (i && i < myargc - 1) { defaultfile = myargv[i + 1]; - //doom_print(" default file: %s\n", defaultfile); doom_print(" default file: "); doom_print(defaultfile); doom_print("\n"); } else - defaultfile = basedefault; - - // read the file in, overriding any set defaults - f = doom_open(defaultfile, "r"); - if (f) { - while (!doom_eof(f)) + defaultfile = basedefault; + } + + void* f = doom_open(defaultfile, "r"); + if (!f) + return; + + for (;;) + { + int lineStatus = M_ReadDefaultsLine(f, line, (int)sizeof(line)); + if (!lineStatus) + break; + if (lineStatus < 0) { - // def - int arg_read = 0; - char c; - for (i = 0; i < 79; ++i) - { - doom_read(f, &c, 1); - if (c == ' ' || c == '\n' || c == '\t') - { - if (i > 0) arg_read++; - break; - } - def[i] = c; - } - def[i] = '\0'; - - // Ignore spaces - if (c != '\n') - { - while (1) - { - doom_read(f, &c, 1); - if (c != ' ' && c != '\t') break; - } - - // strparam - i = 0; - if (c != '\n') - { - for (; i < 260;) - { - strparm[i++] = c; - doom_read(f, &c, 1); - if (c == '\n') - { - if (i > 0) arg_read++; - break; - } - } - } - strparm[i] = '\0'; - } - - isstring = doom_false; - //if (fscanf(f, "%79s %[^\n]\n", def, strparm) == 2) - if (arg_read == 2) - { - if (strparm[0] == '"') - { - // get a string default - isstring = doom_true; - len = (int)doom_strlen(strparm); - newstring = (char*)doom_malloc(len); - strparm[len - 1] = 0; - doom_strcpy(newstring, strparm + 1); - } - else if (strparm[0] == '0' && strparm[1] == 'x') - { - //sscanf(strparm + 2, "%x", &parm); - parm = doom_atox(strparm + 2); - } - else - { - //sscanf(strparm, "%i", &parm); - parm = doom_atoi(strparm); - } - for (i = 0; i < numdefaults; i++) - if (!doom_strcmp(def, defaults[i].name)) - { - if (!isstring) - *defaults[i].location = parm; - else - *defaults[i].text_location = newstring; - break; - } - } + doom_print("M_LoadDefaults: ignored an overlong defaults line.\n"); + continue; } - doom_close(f); + char* name = M_SkipDefaultWhitespace(line); + if (!*name || *name == '#') + continue; + + char* value = name; + while (*value && *value != ' ' && *value != '\t') + value++; + if (!*value) + continue; + *value++ = 0; + value = M_SkipDefaultWhitespace(value); + M_TrimDefaultWhitespace(value); + if (!*value) + continue; + + default_t* setting = get_default(name); + if (!setting) + continue; + + if (value[0] == '"') + { + int length = (int)doom_strlen(value); + if (length < 2 || value[length - 1] != '"' || !setting->text_location) + continue; + + int textLength = length - 2; + char* newstring = (char*)doom_malloc(textLength + 1); + if (!newstring) + continue; + doom_memcpy(newstring, value + 1, textLength); + newstring[textLength] = 0; + *setting->text_location = newstring; + } + else if (setting->location) + { + int parsed; + if (value[0] == '0' && value[1] == 'x') + parsed = doom_atox(value + 2); + else + parsed = doom_atoi(value); + *setting->location = parsed; + } } + + doom_close(f); } @@ -37068,10 +37129,6 @@ void R_InitTextures(void) int* directory; - int temp1; - int temp2; - int temp3; - // Load the patch names from pnames.lmp. name[8] = 0; names = (char*)W_CacheLumpName((char*)"PNAMES", PU_STATIC); @@ -37118,23 +37175,10 @@ void R_InitTextures(void) totalwidth = 0; - // Really complex printing shit... - temp1 = W_GetNumForName((char*)"S_START"); // P_??????? - temp2 = W_GetNumForName((char*)"S_END") - 1; - temp3 = ((temp2 - temp1 + 63) / 64) + ((numtextures + 63) / 64); - doom_print("["); - for (i = 0; i < temp3; i++) - doom_print(" "); - doom_print(" ]"); - for (i = 0; i < temp3; i++) - doom_print("\x8"); - doom_print("\x8\x8\x8\x8\x8\x8\x8\x8\x8\x8"); + // Cursor-control progress output can block simple platform consoles. for (i = 0; i < numtextures; i++, directory++) { - if (!(i & 63)) - doom_print("."); - if (i == numtextures1) { // Start looking in second texture file. @@ -37285,14 +37329,14 @@ void R_InitColormaps(void) // void R_InitData(void) { - R_InitTextures(); doom_print("\nInitTextures"); - R_InitFlats(); + R_InitTextures(); doom_print("\nInitFlats"); - R_InitSpriteLumps(); + R_InitFlats(); doom_print("\nInitSprites"); - R_InitColormaps(); + R_InitSpriteLumps(); doom_print("\nInitColormaps"); + R_InitColormaps(); } diff --git a/code/user/apps/doom/doom.c b/code/user/apps/doom/doom.c index c71bc39..07924ea 100644 --- a/code/user/apps/doom/doom.c +++ b/code/user/apps/doom/doom.c @@ -1,6 +1,6 @@ // doom.c — PureDOOM port // Native personality carrying all the weight. -// Drop doom1.wad (or doom.wad) next to this binary on your FAT32 volume. +// Place the Doom WAD and optional doom.sf2 in the process working directory. #include #include @@ -10,6 +10,8 @@ #include #include +#include "doom_audio.h" + #define DOOM_IMPLEMENTATION #include "PureDOOM.h" @@ -189,6 +191,7 @@ static void doom_gettime_wrapper(int* sec, int* usec) // --------------------------------------------------------------------------- static void doom_exit_wrapper(int code) { + DoomAudioAbort(); SysExitProcess(code); } @@ -445,7 +448,7 @@ static int doom_now_tic(void) // Waits until Doom has a new simulation tic to render. static int wait_for_next_doom_tic(int lastTic) { - for (;;) { + while (true) { int now = doom_now_tic(); if (now != lastTic) return now; @@ -495,9 +498,18 @@ int main(void) char argvStorage[512]; const char* argv[32]; int argc = build_argv_from_command_line(cmdline, argvStorage, sizeof(argvStorage), argv, 32); + if (argc == 1) + doom_print_wrapper("doom: parsed argc == 1\n"); + else if (argc > 1) + doom_print_wrapper("doom: parsed argc > 1\n"); + else + doom_print_wrapper("doom: parsed argc <= 0\n"); init_surface(); init_display_clock(); + do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: starting doom\n") }; SysWriteConsole(&req); } while (0); doom_init(argc, argv, 0); + do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: starting audio\n") }; SysWriteConsole(&req); } while (0); + DoomAudioStart(); do { SYS_WRITE_CONSOLE_REQUEST req = { SYS_SV_LITERAL("doom: entering game loop\n") }; SysWriteConsole(&req); } while (0); @@ -508,9 +520,12 @@ int main(void) int frameTic = wait_for_next_doom_tic(lastFrameTic); lastFrameTic = frameTic; + DoomAudioLockCore(); poll_input(); doom_update(); - render_frame((const uint8_t*)doom_get_framebuffer(4 /* RGBA */)); + const uint8_t* framebuffer = (const uint8_t*)doom_get_framebuffer(4 /* RGBA */); + DoomAudioUnlockCore(); + render_frame(framebuffer); } SysExitProcess(0); diff --git a/code/user/apps/doom/doom_audio.c b/code/user/apps/doom/doom_audio.c new file mode 100644 index 0000000..b9e8932 --- /dev/null +++ b/code/user/apps/doom/doom_audio.c @@ -0,0 +1,802 @@ +// Doom audio mixes PureDOOM SFX with optional SoundFont MIDI into one native render stream. + +#include +#include +#include +#include +#include +#include + +#include "doom_audio.h" +#include "PureDOOM.h" + +#define DOOM_AUDIO_SAMPLE_RATE DOOM_SAMPLERATE +#define DOOM_AUDIO_CHANNELS 2u +#define DOOM_AUDIO_BLOCK_FRAMES 512u +#define DOOM_AUDIO_WRITE_QUANTUM_FRAMES 256u +#define DOOM_AUDIO_PREFILL_FRAMES DOOM_AUDIO_BLOCK_FRAMES +#define DOOM_AUDIO_REFILL_WATERMARK_FRAMES DOOM_AUDIO_WRITE_QUANTUM_FRAMES +#define DOOM_AUDIO_MIDI_BATCH_MESSAGES 256u +#define DOOM_AUDIO_SOUNDFONT_PATH "doom.sf2" +#define DOOM_AUDIO_MAX_SOUNDFONT_BYTES (64u * 1024u * 1024u) +#define DOOM_AUDIO_WAIT_MILLISECONDS 100u +#define DOOM_AUDIO_WORKER_PRIORITY 2u +#define DOOM_AUDIO_SYNTH_LOADER_PRIORITY 0u +#define DOOM_AUDIO_MAX_VOICES 128 +#define DOOM_AUDIO_MUSIC_GAIN 1.0f + +// Computes a natural logarithm for the positive finite values used by TinySoundFont. +static double DoomAudioLog(double value) { + union { + double Value; + uint64_t Bits; + } bits; + const double ln2 = 0.69314718055994530942; + int exponent; + double mantissa; + double ratio; + double ratio2; + double term; + double sum; + + if (value <= 0.0) + return -1.0e300; + + bits.Value = value; + exponent = (int)((bits.Bits >> 52) & 0x7FFu) - 1023; + bits.Bits = (bits.Bits & UINT64_C(0x000FFFFFFFFFFFFF)) | + UINT64_C(0x3FF0000000000000); + mantissa = bits.Value; + ratio = (mantissa - 1.0) / (mantissa + 1.0); + ratio2 = ratio * ratio; + term = ratio; + sum = term; + term *= ratio2; + sum += term / 3.0; + term *= ratio2; + sum += term / 5.0; + term *= ratio2; + sum += term / 7.0; + term *= ratio2; + sum += term / 9.0; + term *= ratio2; + sum += term / 11.0; + term *= ratio2; + sum += term / 13.0; + return (double)exponent * ln2 + 2.0 * sum; +} + +// Scales a normal double by an integral power of two without a runtime math library. +static double DoomAudioScalePowerOfTwo(double value, int exponent) { + union { + double Value; + uint64_t Bits; + } scale; + + if (exponent > 1023) + return 1.0e300; + if (exponent < -1022) + return 0.0; + + scale.Bits = (uint64_t)(exponent + 1023) << 52; + return value * scale.Value; +} + +// Computes an exponential over the bounded ranges used by TinySoundFont envelopes. +static double DoomAudioExp(double value) { + const double ln2 = 0.69314718055994530942; + const double inverseLn2 = 1.4426950408889634074; + int exponent; + double reduced; + double result; + + if (value >= 700.0) + return 1.0e300; + if (value <= -700.0) + return 0.0; + + exponent = (int)(value * inverseLn2 + (value >= 0.0 ? 0.5 : -0.5)); + reduced = value - (double)exponent * ln2; + result = 1.0 + reduced * + (1.0 + reduced * + (0.5 + reduced * + (0.16666666666666666667 + reduced * + (0.04166666666666666667 + reduced * + (0.00833333333333333333 + reduced * + (0.00138888888888888889 + reduced * + 0.00019841269841269841)))))); + return DoomAudioScalePowerOfTwo(result, exponent); +} + +// Computes a positive real power for TinySoundFont frequency and gain conversion. +static double DoomAudioPow(double base, double exponent) { + if (base <= 0.0) + return 0.0; + return DoomAudioExp(exponent * DoomAudioLog(base)); +} + +// Computes the float exponential required by TinySoundFont envelopes. +static float DoomAudioExpf(float value) { + return (float)DoomAudioExp((double)value); +} + +// Computes the float power required by TinySoundFont frequency and gain conversion. +static float DoomAudioPowf(float base, float exponent) { + return (float)DoomAudioPow((double)base, (double)exponent); +} + +// Computes base-ten logarithms from the shared natural logarithm approximation. +static double DoomAudioLog10(double value) { + return DoomAudioLog(value) * 0.43429448190325182765; +} + +// Computes square roots with a bit-derived seed and stable Newton refinement. +static double DoomAudioSqrt(double value) { + union { + double Value; + uint64_t Bits; + } estimate; + + if (value <= 0.0) + return 0.0; + + estimate.Value = value; + estimate.Bits = (estimate.Bits >> 1) + UINT64_C(0x1FF8000000000000); + for (int iteration = 0; iteration < 6; iteration++) + estimate.Value = 0.5 * (estimate.Value + value / estimate.Value); + return estimate.Value; +} + +// Computes the float square root required by TinySoundFont panning. +static float DoomAudioSqrtf(float value) { + return (float)DoomAudioSqrt((double)value); +} + +// Computes tangent accurately over TinySoundFont's normalized low-pass range. +static double DoomAudioTan(double value) { + double squared = value * value; + double sine = value * + (1.0 + squared * + (-0.16666666666666666667 + squared * + (0.00833333333333333333 + squared * + (-0.00019841269841269841 + squared * + (0.00000275573192239859 + squared * + (-0.00000002505210838544 + squared * + 0.00000000016059043837)))))); + double cosine = 1.0 + squared * + (-0.5 + squared * + (0.04166666666666666667 + squared * + (-0.00138888888888888889 + squared * + (0.00002480158730158730 + squared * + (-0.00000027557319223986 + squared * + (0.00000000208767569879 + squared * + -0.00000000001147074560)))))); + + if (cosine > -1.0e-12 && cosine < 1.0e-12) + cosine = cosine < 0.0 ? -1.0e-12 : 1.0e-12; + return sine / cosine; +} + +#pragma warning(push) +#pragma warning(disable: 4244 4267 4706) +#define TSF_NO_STDIO +#define TSF_MALLOC malloc +#define TSF_REALLOC realloc +#define TSF_FREE free +#define TSF_MEMCPY memcpy +#define TSF_MEMSET memset +#define TSF_POW DoomAudioPow +#define TSF_POWF DoomAudioPowf +#define TSF_EXPF DoomAudioExpf +#define TSF_LOG DoomAudioLog +#define TSF_TAN DoomAudioTan +#define TSF_LOG10 DoomAudioLog10 +#define TSF_SQRT DoomAudioSqrt +#define TSF_SQRTF DoomAudioSqrtf +#define TSF_IMPLEMENTATION +#include "third_party/tsf.h" +#pragma warning(pop) + +typedef struct DOOM_AUDIO_STATE { + uint32_t CoreMutex; + uint32_t StreamHandle; + uint32_t WorkerHandle; + uint32_t WorkerId; + uint32_t SynthLoaderHandle; + uint32_t SynthLoaderId; + volatile long StopRequested; + tsf* volatile Synth; +} DOOM_AUDIO_STATE; + +static DOOM_AUDIO_STATE g_doomAudio; + +// Writes one complete diagnostic line to the process console. +static void DoomAudioPrint(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Opens the default available native render endpoint. +static KSTATUS DoomAudioFindEndpoint(uint64_t* endpointIdOut) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + uint32_t count = 0; + const SYS_AUDIO_ENDPOINT_INFO* fallback = NULL; + KSTATUS status; + + if (!endpointIdOut) + return NTLSTATUS_INVALID_PARAMETER; + *endpointIdOut = 0; + + memset(endpoints, 0, sizeof(endpoints)); + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + request.CountOut = &count; + status = SysAudioEnumerateEndpoints(&request); + if (status != NTLSTATUS_SUCCESS) + return status; + if (count > SYS_AUDIO_MAX_ENDPOINTS) + count = SYS_AUDIO_MAX_ENDPOINTS; + + for (uint32_t index = 0; index < count; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) + fallback = endpoint; + if (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT) { + *endpointIdOut = endpoint->EndpointId; + return NTLSTATUS_SUCCESS; + } + } + + if (!fallback) + return NTLSTATUS_NOT_FOUND; + *endpointIdOut = fallback->EndpointId; + return NTLSTATUS_SUCCESS; +} + +// Returns the currently published synthesizer without taking the PureDOOM lock. +static tsf* DoomAudioGetSynth(void) { + return (tsf*)_InterlockedCompareExchangePointer( + (void* volatile*)&g_doomAudio.Synth, NULL, NULL); +} + +// Maps the optional SoundFont so TinySoundFont can parse it without a separate input copy. +static KSTATUS DoomAudioMapSoundFont(void** dataOut, uint32_t* sizeOut) { + uint32_t fileHandle = 0; + uint32_t mappingHandle = 0; + uint64_t fileSize = 0; + void* base = NULL; + KSTATUS status; + + if (!dataOut || !sizeOut) + return NTLSTATUS_INVALID_PARAMETER; + *dataOut = NULL; + *sizeOut = 0; + + SYS_OPEN_FILE_REQUEST open = SysMakeOpenFileRequest( + SysSvFromCstr(DOOM_AUDIO_SOUNDFONT_PATH), + SYS_FILE_ACCESS_QUERY | SYS_FILE_ACCESS_READ, + SYS_FILE_OPEN_EXISTING, + SYS_FILE_FLAG_NONE, + &fileHandle); + status = SysOpenFile(&open); + if (status != NTLSTATUS_SUCCESS) + return status; + + status = SysGetFileSize(fileHandle, &fileSize); + if (status != NTLSTATUS_SUCCESS || fileSize == 0 || + fileSize > DOOM_AUDIO_MAX_SOUNDFONT_BYTES || + fileSize > INT32_MAX) { + (void)SysCloseHandle(fileHandle); + return status == NTLSTATUS_SUCCESS ? NTLSTATUS_INVALID_PARAMETER : status; + } + + status = SysCreateFileMapping(fileHandle, SYS_VM_READ, SYS_MAP_FLAG_NONE, + fileSize, &mappingHandle); + if (status != NTLSTATUS_SUCCESS || !mappingHandle) { + (void)SysCloseHandle(fileHandle); + return status == NTLSTATUS_SUCCESS ? NTLSTATUS_FAILURE : status; + } + + status = SysMapFile(mappingHandle, 0, (size_t)fileSize, + SYS_MAP_ACCESS_READ, SYS_MAP_FLAG_NONE, &base); + (void)SysCloseHandle(mappingHandle); + (void)SysCloseHandle(fileHandle); + if (status != NTLSTATUS_SUCCESS || !base) + return status == NTLSTATUS_SUCCESS ? NTLSTATUS_FAILURE : status; + + *dataOut = base; + *sizeOut = (uint32_t)fileSize; + return NTLSTATUS_SUCCESS; +} + +// Configures one parsed General MIDI synthesizer for PureDOOM's native sample rate. +static KSTATUS DoomAudioConfigureSynth(tsf* synth) { + if (!synth) + return NTLSTATUS_INVALID_PARAMETER; + + tsf_set_output(synth, TSF_STEREO_INTERLEAVED, + (int)DOOM_AUDIO_SAMPLE_RATE, 0.0f); + if (!tsf_set_max_voices(synth, DOOM_AUDIO_MAX_VOICES)) + return NTLSTATUS_NO_MEMORY; + + for (int channel = 0; channel < 16; channel++) { + if (!tsf_channel_set_presetnumber(synth, channel, 0, channel == 9)) + return NTLSTATUS_FAILURE; + } + return NTLSTATUS_SUCCESS; +} + +// Loads optional MIDI state at low priority without blocking Doom or SFX startup. +static void DoomAudioSynthLoader(uintptr_t argument) { + uint32_t soundFontSize = 0; + void* soundFontBase = NULL; + tsf* synth = NULL; + KSTATUS status; + (void)argument; + + status = DoomAudioMapSoundFont(&soundFontBase, &soundFontSize); + if (status == NTLSTATUS_NOT_FOUND) { + DoomAudioPrint("doom: doom.sf2 unavailable; continuing with sound effects only\n"); + SysExitThread(0); + } + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: doom.sf2 could not be mapped; continuing with sound effects only\n"); + SysExitThread(0); + } + + if (_InterlockedCompareExchange(&g_doomAudio.StopRequested, 0, 0)) { + (void)SysFreeVm(soundFontBase); + SysExitThread(0); + } + + synth = tsf_load_memory(soundFontBase, (int)soundFontSize); + (void)SysFreeVm(soundFontBase); + if (!synth) { + DoomAudioPrint("doom: doom.sf2 is invalid or synth allocation failed; continuing with sound effects only\n"); + SysExitThread(0); + } + + status = DoomAudioConfigureSynth(synth); + if (status != NTLSTATUS_SUCCESS || + _InterlockedCompareExchange(&g_doomAudio.StopRequested, 0, 0)) { + if (status != NTLSTATUS_SUCCESS) + DoomAudioPrint("doom: MIDI synthesizer initialization failed; continuing with sound effects only\n"); + tsf_close(synth); + SysExitThread(0); + } + + (void)_InterlockedExchangePointer( + (void* volatile*)&g_doomAudio.Synth, synth); + DoomAudioPrint("doom: MIDI synthesizer ready\n"); + SysExitThread(0); +} + +// Applies one packed WinMM-style short MIDI message to TinySoundFont. +static void DoomAudioDispatchMidi(tsf* synth, uint32_t message) { + uint8_t status = (uint8_t)(message & 0xFFu); + uint8_t data1 = (uint8_t)((message >> 8) & 0x7Fu); + uint8_t data2 = (uint8_t)((message >> 16) & 0x7Fu); + int channel = status & 0x0F; + + if (!synth) + return; + + switch (status & 0xF0) { + case 0x80: + tsf_channel_note_off(synth, channel, data1); + break; + case 0x90: + if (data2) + (void)tsf_channel_note_on(synth, channel, data1, + (float)data2 / 127.0f); + else + tsf_channel_note_off(synth, channel, data1); + break; + case 0xB0: + (void)tsf_channel_midi_control(synth, channel, data1, data2); + break; + case 0xC0: + (void)tsf_channel_set_presetnumber(synth, channel, data1, + channel == 9); + break; + case 0xE0: + (void)tsf_channel_set_pitchwheel(synth, channel, + (int)data1 | ((int)data2 << 7)); + break; + default: + break; + } +} + +// Advances one PureDOOM MIDI tick and dispatches every event outside the core lock. +static void DoomAudioTickMidi(tsf* synth) { + uint32_t messages[DOOM_AUDIO_MIDI_BATCH_MESSAGES]; + int tickComplete = 0; + + if (!synth) + return; + + while (!tickComplete) { + uint32_t count = 0; + + DoomAudioLockCore(); + while (count < DOOM_AUDIO_MIDI_BATCH_MESSAGES) { + uint32_t message = (uint32_t)doom_tick_midi(); + if (!message) { + tickComplete = 1; + break; + } + messages[count++] = message; + } + DoomAudioUnlockCore(); + + for (uint32_t index = 0; index < count; index++) + DoomAudioDispatchMidi(synth, messages[index]); + } +} + +// Copies one complete PureDOOM SFX block while holding the shared core lock. +static void DoomAudioReadSfxBlock(int16_t* frames) { + short* source; + + DoomAudioLockCore(); + source = doom_get_sound_buffer(); + memcpy(frames, source, + DOOM_AUDIO_BLOCK_FRAMES * DOOM_AUDIO_CHANNELS * sizeof(int16_t)); + DoomAudioUnlockCore(); +} + +// Mixes rendered floating-point music into signed 16-bit PureDOOM SFX. +static void DoomAudioMixMusic(int16_t* destination, + const float* music, + uint32_t frameOffset, + uint32_t frameCount) { + uint32_t firstSample = frameOffset * DOOM_AUDIO_CHANNELS; + uint32_t sampleCount = frameCount * DOOM_AUDIO_CHANNELS; + + for (uint32_t index = 0; index < sampleCount; index++) { + float scaled = music[index] * (32767.0f * DOOM_AUDIO_MUSIC_GAIN); + int32_t synthSample; + int32_t mixed; + + if (scaled > 32767.0f) + synthSample = 32767; + else if (scaled < -32768.0f) + synthSample = -32768; + else + synthSample = (int32_t)scaled; + + mixed = (int32_t)destination[firstSample + index] + synthSample; + if (mixed > 32767) + mixed = 32767; + else if (mixed < -32768) + mixed = -32768; + destination[firstSample + index] = (int16_t)mixed; + } +} + +// Renders MIDI at exactly 140 ticks per second into one native PureDOOM block. +static void DoomAudioRenderMusic(tsf* synth, + int16_t* frames, + uint32_t* phaseInOut) { + float music[DOOM_AUDIO_BLOCK_FRAMES * DOOM_AUDIO_CHANNELS]; + uint32_t frameOffset = 0; + + memset(music, 0, sizeof(music)); + while (frameOffset < DOOM_AUDIO_BLOCK_FRAMES) { + uint32_t remaining; + uint32_t framesUntilTick; + + while (*phaseInOut >= DOOM_AUDIO_SAMPLE_RATE) { + *phaseInOut -= DOOM_AUDIO_SAMPLE_RATE; + DoomAudioTickMidi(synth); + } + + remaining = DOOM_AUDIO_BLOCK_FRAMES - frameOffset; + framesUntilTick = + (DOOM_AUDIO_SAMPLE_RATE - *phaseInOut + DOOM_MIDI_RATE - 1u) / + DOOM_MIDI_RATE; + if (framesUntilTick > remaining) + framesUntilTick = remaining; + + tsf_render_float(synth, + music + frameOffset * DOOM_AUDIO_CHANNELS, + (int)framesUntilTick, + 0); + DoomAudioMixMusic(frames, music + frameOffset * DOOM_AUDIO_CHANNELS, + frameOffset, framesUntilTick); + frameOffset += framesUntilTick; + *phaseInOut += framesUntilTick * DOOM_MIDI_RATE; + } +} + +// Renders one complete native-rate block containing synchronized music and SFX. +static void DoomAudioRenderBlock(int16_t* frames, + tsf** activeSynthInOut, + uint32_t* midiPhaseInOut) { + tsf* synth; + + DoomAudioReadSfxBlock(frames); + synth = DoomAudioGetSynth(); + if (synth != *activeSynthInOut) { + *activeSynthInOut = synth; + *midiPhaseInOut = DOOM_AUDIO_SAMPLE_RATE; + } + if (synth) + DoomAudioRenderMusic(synth, frames, midiPhaseInOut); +} + +// Queries one coherent native stream snapshot. +static KSTATUS DoomAudioQueryStream(SYS_AUDIO_STREAM_INFO* infoOut) { + if (!infoOut) + return NTLSTATUS_INVALID_PARAMETER; + memset(infoOut, 0, sizeof(*infoOut)); + infoOut->Size = sizeof(*infoOut); + return SysAudioQueryStream(g_doomAudio.StreamHandle, infoOut); +} + +// Arms a wait for writable space or a terminal stream condition. +static KSTATUS DoomAudioWaitWritable(const SYS_AUDIO_STREAM_INFO* info, + uint32_t writableFrames) { + SYS_AUDIO_ARM_WAIT_REQUEST request; + KSTATUS status; + + if (!info || !writableFrames || writableFrames > info->CapacityFrames) + return NTLSTATUS_INVALID_PARAMETER; + + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = g_doomAudio.StreamHandle; + request.Mask = SYS_AUDIO_WAIT_WRITABLE | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY; + request.WritableFrames = writableFrames; + request.DiscontinuityGeneration = info->DiscontinuityGeneration; + status = SysAudioArmWait(&request); + if (status != NTLSTATUS_SUCCESS) + return status; + return SysWaitSingle(g_doomAudio.StreamHandle, + SYS_WAIT_FLAG_NONE, + DOOM_AUDIO_WAIT_MILLISECONDS); +} + +// Waits until the buffered source has fallen to the low-latency refill watermark. +static KSTATUS DoomAudioWaitForRefill(const SYS_AUDIO_STREAM_INFO* info) { + if (!info || info->CapacityFrames <= DOOM_AUDIO_REFILL_WATERMARK_FRAMES) + return NTLSTATUS_INVALID_PARAMETER; + return DoomAudioWaitWritable( + info, info->CapacityFrames - DOOM_AUDIO_REFILL_WATERMARK_FRAMES); +} + +// Writes as much of one pending block as the nonblocking stream accepts. +static KSTATUS DoomAudioWritePending(const int16_t* frames, + uint32_t* frameOffsetInOut) { + SYS_AUDIO_WRITE_REQUEST request; + uint32_t written = 0; + KSTATUS status; + + if (!frames || !frameOffsetInOut || + *frameOffsetInOut >= DOOM_AUDIO_BLOCK_FRAMES) + return NTLSTATUS_INVALID_PARAMETER; + + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = g_doomAudio.StreamHandle; + request.FrameCount = DOOM_AUDIO_BLOCK_FRAMES - *frameOffsetInOut; + if (request.FrameCount > DOOM_AUDIO_WRITE_QUANTUM_FRAMES) + request.FrameCount = DOOM_AUDIO_WRITE_QUANTUM_FRAMES; + request.Frames = frames + *frameOffsetInOut * DOOM_AUDIO_CHANNELS; + request.WrittenFramesOut = &written; + status = SysAudioWrite(&request); + if (status != NTLSTATUS_SUCCESS) + return status; + if (!written) + return NTLSTATUS_WOULD_BLOCK; + + *frameOffsetInOut += written; + return NTLSTATUS_SUCCESS; +} + +// Keeps a bounded low-latency stream queue at PureDOOM's native cadence. +static void DoomAudioWorker(uintptr_t argument) { + int16_t block[DOOM_AUDIO_BLOCK_FRAMES * DOOM_AUDIO_CHANNELS]; + uint32_t pendingOffset = DOOM_AUDIO_BLOCK_FRAMES; + uint32_t midiPhase = 0; + tsf* activeSynth = NULL; + uint8_t started = 0; + (void)argument; + + while (true) { + SYS_AUDIO_STREAM_INFO info; + uint32_t requiredFrames; + KSTATUS status; + + if (_InterlockedCompareExchange(&g_doomAudio.StopRequested, 0, 0)) + break; + + status = DoomAudioQueryStream(&info); + if (status != NTLSTATUS_SUCCESS || info.State == SysAudioStreamFailed) { + DoomAudioPrint("doom: native audio stream failed\n"); + break; + } + + if (!started && info.BufferedFrames >= DOOM_AUDIO_PREFILL_FRAMES) { + status = SysAudioStart(g_doomAudio.StreamHandle); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: native audio start failed\n"); + break; + } + started = 1; + continue; + } + + if (pendingOffset == DOOM_AUDIO_BLOCK_FRAMES) { + if (started && + info.BufferedFrames > DOOM_AUDIO_REFILL_WATERMARK_FRAMES) { + status = DoomAudioWaitForRefill(&info); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_TIMEOUT) { + DoomAudioPrint("doom: native audio wait failed\n"); + break; + } + continue; + } + + DoomAudioRenderBlock(block, &activeSynth, &midiPhase); + pendingOffset = 0; + } + + requiredFrames = DOOM_AUDIO_BLOCK_FRAMES - pendingOffset; + if (requiredFrames > DOOM_AUDIO_WRITE_QUANTUM_FRAMES) + requiredFrames = DOOM_AUDIO_WRITE_QUANTUM_FRAMES; + if (info.WritableFrames < requiredFrames) { + status = DoomAudioWaitWritable(&info, requiredFrames); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_TIMEOUT) { + DoomAudioPrint("doom: native audio wait failed\n"); + break; + } + continue; + } + + status = DoomAudioWritePending(block, &pendingOffset); + if (status == NTLSTATUS_WOULD_BLOCK) + continue; + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: native audio write failed\n"); + break; + } + } + + if (started) + (void)SysAudioStop(g_doomAudio.StreamHandle); + SysExitThread(0); +} + +// Releases resources owned before or after the producer thread lifetime. +static void DoomAudioReleaseResources(void) { + if (g_doomAudio.StreamHandle) { + (void)SysCloseHandle(g_doomAudio.StreamHandle); + g_doomAudio.StreamHandle = 0; + } + tsf* synth = (tsf*)_InterlockedExchangePointer( + (void* volatile*)&g_doomAudio.Synth, NULL); + if (synth) + tsf_close(synth); + if (g_doomAudio.CoreMutex) { + (void)SysCloseHandle(g_doomAudio.CoreMutex); + g_doomAudio.CoreMutex = 0; + } +} + +// Starts one process-owned audio stream and its exclusive producer thread. +void DoomAudioStart(void) { + uint64_t endpointId = 0; + uint32_t streamHandle = 0; + KSTATUS status; + + if (DOOM_AUDIO_PREFILL_FRAMES >= SYS_AUDIO_STREAM_CAPACITY_FRAMES || + DOOM_AUDIO_REFILL_WATERMARK_FRAMES >= DOOM_AUDIO_PREFILL_FRAMES || + DOOM_AUDIO_WRITE_QUANTUM_FRAMES > DOOM_AUDIO_BLOCK_FRAMES) { + DoomAudioPrint("doom: invalid native audio buffering configuration\n"); + return; + } + + memset(&g_doomAudio, 0, sizeof(g_doomAudio)); + status = SysCreateMutex(&g_doomAudio.CoreMutex); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: audio synchronization unavailable; continuing without sound\n"); + return; + } + + status = DoomAudioFindEndpoint(&endpointId); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: no audio endpoint; continuing without sound\n"); + DoomAudioReleaseResources(); + return; + } + + SYS_AUDIO_OPEN_STREAM_REQUEST open; + memset(&open, 0, sizeof(open)); + open.Size = sizeof(open); + open.EndpointId = endpointId; + open.Format.SampleRate = DOOM_AUDIO_SAMPLE_RATE; + open.Format.ChannelCount = DOOM_AUDIO_CHANNELS; + open.Format.SampleFormat = SysAudioSampleS16; + open.StreamHandleOut = &streamHandle; + status = SysAudioOpenStream(&open); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: native audio open failed; continuing without sound\n"); + DoomAudioReleaseResources(); + return; + } + + g_doomAudio.StreamHandle = streamHandle; + + SYS_CREATE_THREAD_REQUEST thread; + memset(&thread, 0, sizeof(thread)); + thread.Entry = (uintptr_t)DoomAudioWorker; + thread.Priority = DOOM_AUDIO_WORKER_PRIORITY; + thread.ThreadHandleOut = &g_doomAudio.WorkerHandle; + thread.ThreadIdOut = &g_doomAudio.WorkerId; + status = SysCreateThread(&thread); + if (status != NTLSTATUS_SUCCESS) { + DoomAudioPrint("doom: audio worker creation failed; continuing without sound\n"); + DoomAudioReleaseResources(); + return; + } + + memset(&thread, 0, sizeof(thread)); + thread.Entry = (uintptr_t)DoomAudioSynthLoader; + thread.Priority = DOOM_AUDIO_SYNTH_LOADER_PRIORITY; + thread.ThreadHandleOut = &g_doomAudio.SynthLoaderHandle; + thread.ThreadIdOut = &g_doomAudio.SynthLoaderId; + status = SysCreateThread(&thread); + if (status != NTLSTATUS_SUCCESS) + DoomAudioPrint("doom: MIDI loader creation failed; continuing with sound effects only\n"); +} + +// Stops playback immediately without waiting for the producer thread. +void DoomAudioAbort(void) { + _InterlockedExchange(&g_doomAudio.StopRequested, 1); + if (g_doomAudio.StreamHandle) + (void)SysAudioStop(g_doomAudio.StreamHandle); +} + +// Stops the producer before releasing its stream, synthesizer, and core lock. +void DoomAudioShutdown(void) { + DoomAudioAbort(); + if (g_doomAudio.WorkerHandle) { + (void)SysWaitSingle(g_doomAudio.WorkerHandle, + SYS_WAIT_FLAG_NONE, + SYS_WAIT_TIMEOUT_INFINITE); + (void)SysCloseHandle(g_doomAudio.WorkerHandle); + g_doomAudio.WorkerHandle = 0; + } + if (g_doomAudio.SynthLoaderHandle) { + (void)SysWaitSingle(g_doomAudio.SynthLoaderHandle, + SYS_WAIT_FLAG_NONE, + SYS_WAIT_TIMEOUT_INFINITE); + (void)SysCloseHandle(g_doomAudio.SynthLoaderHandle); + g_doomAudio.SynthLoaderHandle = 0; + } + DoomAudioReleaseResources(); +} + +// Enters the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioLockCore(void) { + if (g_doomAudio.CoreMutex) + (void)SysWaitSingle(g_doomAudio.CoreMutex, + SYS_WAIT_FLAG_NONE, + SYS_WAIT_TIMEOUT_INFINITE); +} + +// Leaves the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioUnlockCore(void) { + if (g_doomAudio.CoreMutex) + (void)SysReleaseMutex(g_doomAudio.CoreMutex); +} diff --git a/code/user/apps/doom/doom_audio.h b/code/user/apps/doom/doom_audio.h new file mode 100644 index 0000000..9762512 --- /dev/null +++ b/code/user/apps/doom/doom_audio.h @@ -0,0 +1,18 @@ +// Doom audio owns PureDOOM sound extraction, MIDI synthesis, and native stream submission. + +#pragma once + +// Starts the optional audio worker after PureDOOM has initialized its sound state. +void DoomAudioStart(void); + +// Stops playback immediately without waiting for the worker. +void DoomAudioAbort(void); + +// Stops the audio worker and releases its process-owned resources. +void DoomAudioShutdown(void); + +// Enters the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioLockCore(void); + +// Leaves the lock shared by the game loop and PureDOOM audio callbacks. +void DoomAudioUnlockCore(void); diff --git a/code/user/apps/doom/third_party/LICENSE.tsf b/code/user/apps/doom/third_party/LICENSE.tsf new file mode 100644 index 0000000..533da73 --- /dev/null +++ b/code/user/apps/doom/third_party/LICENSE.tsf @@ -0,0 +1,19 @@ +Copyright (C) 2017-2023 Bernhard Schelling (Based on SFZero, Copyright (C) 2012 Steve Folta, https://github.com/stevefolta/SFZero) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/code/user/apps/doom/third_party/README.tsf b/code/user/apps/doom/third_party/README.tsf new file mode 100644 index 0000000..f741372 --- /dev/null +++ b/code/user/apps/doom/third_party/README.tsf @@ -0,0 +1,11 @@ +TinySoundFont v0.9 + +Upstream: https://github.com/schellingb/TinySoundFont +File: tsf.h +License: MIT; see LICENSE.tsf. + +The vendored header is unchanged apart from normalized line endings. nt-lite +supplies allocator, memory, and math macros from doom_audio.c so the synthesizer +stays within the native user runtime. Doom loads the optional small `doom.sf2` +asset asynchronously, then releases the input mapping after TinySoundFont has +copied the sample bank into its owned representation. diff --git a/code/user/apps/doom/third_party/tsf.h b/code/user/apps/doom/third_party/tsf.h new file mode 100644 index 0000000..22a2f2a --- /dev/null +++ b/code/user/apps/doom/third_party/tsf.h @@ -0,0 +1,2045 @@ +/* TinySoundFont - v0.9 - SoundFont2 synthesizer - https://github.com/schellingb/TinySoundFont + no warranty implied; use at your own risk + Do this: + #define TSF_IMPLEMENTATION + before you include this file in *one* C or C++ file to create the implementation. + // i.e. it should look like this: + #include ... + #include ... + #define TSF_IMPLEMENTATION + #include "tsf.h" + + [OPTIONAL] #define TSF_NO_STDIO to remove stdio dependency + [OPTIONAL] #define TSF_MALLOC, TSF_REALLOC, and TSF_FREE to avoid stdlib.h + [OPTIONAL] #define TSF_MEMCPY, TSF_MEMSET to avoid string.h + [OPTIONAL] #define TSF_POW, TSF_POWF, TSF_EXPF, TSF_LOG, TSF_TAN, TSF_LOG10, TSF_SQRT to avoid math.h + + NOT YET IMPLEMENTED + - Support for ChorusEffectsSend and ReverbEffectsSend generators + - Better low-pass filter without lowering performance too much + - Support for modulators + + LICENSE (MIT) + + Copyright (C) 2017-2023 Bernhard Schelling + Based on SFZero, Copyright (C) 2012 Steve Folta (https://github.com/stevefolta/SFZero) + + Permission is hereby granted, free of charge, to any person obtaining a copy of this + software and associated documentation files (the "Software"), to deal in the Software + without restriction, including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + +#ifndef TSF_INCLUDE_TSF_INL +#define TSF_INCLUDE_TSF_INL + +#ifdef __cplusplus +extern "C" { +# define CPP_DEFAULT0 = 0 +#else +# define CPP_DEFAULT0 +#endif + +//define this if you want the API functions to be static +#ifdef TSF_STATIC +#define TSFDEF static +#else +#define TSFDEF extern +#endif + +// The load functions will return a pointer to a struct tsf which all functions +// thereafter take as the first parameter. +// On error the tsf_load* functions will return NULL most likely due to invalid +// data (or if the file did not exist in tsf_load_filename). +typedef struct tsf tsf; + +#ifndef TSF_NO_STDIO +// Directly load a SoundFont from a .sf2 file path +TSFDEF tsf* tsf_load_filename(const char* filename); +#endif + +// Load a SoundFont from a block of memory +TSFDEF tsf* tsf_load_memory(const void* buffer, int size); + +// Stream structure for the generic loading +struct tsf_stream +{ + // Custom data given to the functions as the first parameter + void* data; + + // Function pointer will be called to read 'size' bytes into ptr (returns number of read bytes) + int (*read)(void* data, void* ptr, unsigned int size); + + // Function pointer will be called to skip ahead over 'count' bytes (returns 1 on success, 0 on error) + int (*skip)(void* data, unsigned int count); +}; + +// Generic SoundFont loading method using the stream structure above +TSFDEF tsf* tsf_load(struct tsf_stream* stream); + +// Copy a tsf instance from an existing one, use tsf_close to close it as well. +// All copied tsf instances and their original instance are linked, and share the underlying soundfont. +// This allows loading a soundfont only once, but using it for multiple independent playbacks. +// (This function isn't thread-safe without locking.) +TSFDEF tsf* tsf_copy(tsf* f); + +// Free the memory related to this tsf instance +TSFDEF void tsf_close(tsf* f); + +// Stop all playing notes immediately and reset all channel parameters +TSFDEF void tsf_reset(tsf* f); + +// Returns the preset index from a bank and preset number, or -1 if it does not exist in the loaded SoundFont +TSFDEF int tsf_get_presetindex(const tsf* f, int bank, int preset_number); + +// Returns the number of presets in the loaded SoundFont +TSFDEF int tsf_get_presetcount(const tsf* f); + +// Returns the name of a preset index >= 0 and < tsf_get_presetcount() +TSFDEF const char* tsf_get_presetname(const tsf* f, int preset_index); + +// Returns the name of a preset by bank and preset number +TSFDEF const char* tsf_bank_get_presetname(const tsf* f, int bank, int preset_number); + +// Supported output modes by the render methods +enum TSFOutputMode +{ + // Two channels with single left/right samples one after another + TSF_STEREO_INTERLEAVED, + // Two channels with all samples for the left channel first then right + TSF_STEREO_UNWEAVED, + // A single channel (stereo instruments are mixed into center) + TSF_MONO +}; + +// Thread safety: +// +// 1. Rendering / voices: +// +// Your audio output which calls the tsf_render* functions will most likely +// run on a different thread than where the playback tsf_note* functions +// are called. In which case some sort of concurrency control like a +// mutex needs to be used so they are not called at the same time. +// Alternatively, you can pre-allocate a maximum number of voices that can +// play simultaneously by calling tsf_set_max_voices after loading. +// That way memory re-allocation will not happen during tsf_note_on and +// TSF should become mostly thread safe. +// There is a theoretical chance that ending notes would negatively influence +// a voice that is rendering at the time but it is hard to say. +// Also be aware, this has not been tested much. +// +// 2. Channels: +// +// Calls to tsf_channel_set_... functions may allocate new channels +// if no channel with that number was previously used. Make sure to +// create all channels at the beginning as required if you call tsf_render* +// from a different thread. + +// Setup the parameters for the voice render methods +// outputmode: if mono or stereo and how stereo channel data is ordered +// samplerate: the number of samples per second (output frequency) +// global_gain_db: volume gain in decibels (>0 means higher, <0 means lower) +TSFDEF void tsf_set_output(tsf* f, enum TSFOutputMode outputmode, int samplerate, float global_gain_db CPP_DEFAULT0); + +// Set the global gain as a volume factor +// global_gain: the desired volume where 1.0 is 100% +TSFDEF void tsf_set_volume(tsf* f, float global_gain); + +// Set the maximum number of voices to play simultaneously +// Depending on the soundfond, one note can cause many new voices to be started, +// so don't keep this number too low or otherwise sounds may not play. +// max_voices: maximum number to pre-allocate and set the limit to +// (tsf_set_max_voices returns 0 if allocation failed, otherwise 1) +TSFDEF int tsf_set_max_voices(tsf* f, int max_voices); + +// Start playing a note +// preset_index: preset index >= 0 and < tsf_get_presetcount() +// key: note value between 0 and 127 (60 being middle C) +// vel: velocity as a float between 0.0 (equal to note off) and 1.0 (full) +// bank: instrument bank number (alternative to preset_index) +// preset_number: preset number (alternative to preset_index) +// (tsf_note_on returns 0 if the allocation of a new voice failed, otherwise 1) +// (tsf_bank_note_on returns 0 if preset does not exist or allocation failed, otherwise 1) +TSFDEF int tsf_note_on(tsf* f, int preset_index, int key, float vel); +TSFDEF int tsf_bank_note_on(tsf* f, int bank, int preset_number, int key, float vel); + +// Stop playing a note +// (bank_note_off returns 0 if preset does not exist, otherwise 1) +TSFDEF void tsf_note_off(tsf* f, int preset_index, int key); +TSFDEF int tsf_bank_note_off(tsf* f, int bank, int preset_number, int key); + +// Stop playing all notes (end with sustain and release) +TSFDEF void tsf_note_off_all(tsf* f); + +// Returns the number of active voices +TSFDEF int tsf_active_voice_count(tsf* f); + +// Render output samples into a buffer +// You can either render as signed 16-bit values (tsf_render_short) or +// as 32-bit float values (tsf_render_float) +// buffer: target buffer of size samples * output_channels * sizeof(type) +// samples: number of samples to render +// flag_mixing: if 0 clear the buffer first, otherwise mix into existing data +TSFDEF void tsf_render_short(tsf* f, short* buffer, int samples, int flag_mixing CPP_DEFAULT0); +TSFDEF void tsf_render_float(tsf* f, float* buffer, int samples, int flag_mixing CPP_DEFAULT0); + +// Higher level channel based functions, set up channel parameters +// channel: channel number +// preset_index: preset index >= 0 and < tsf_get_presetcount() +// preset_number: preset number (alternative to preset_index) +// flag_mididrums: 0 for normal channels, otherwise apply MIDI drum channel rules +// bank: instrument bank number (alternative to preset_index) +// pan: stereo panning value from 0.0 (left) to 1.0 (right) (default 0.5 center) +// volume: linear volume scale factor (default 1.0 full) +// pitch_wheel: pitch wheel position 0 to 16383 (default 8192 unpitched) +// pitch_range: range of the pitch wheel in semitones (default 2.0, total +/- 2 semitones) +// tuning: tuning of all playing voices in semitones (default 0.0, standard (A440) tuning) +// (tsf_set_preset_number and set_bank_preset return 0 if preset does not exist, otherwise 1) +// (tsf_channel_set_... return 0 if a new channel needed allocation and that failed, otherwise 1) +TSFDEF int tsf_channel_set_presetindex(tsf* f, int channel, int preset_index); +TSFDEF int tsf_channel_set_presetnumber(tsf* f, int channel, int preset_number, int flag_mididrums CPP_DEFAULT0); +TSFDEF int tsf_channel_set_bank(tsf* f, int channel, int bank); +TSFDEF int tsf_channel_set_bank_preset(tsf* f, int channel, int bank, int preset_number); +TSFDEF int tsf_channel_set_pan(tsf* f, int channel, float pan); +TSFDEF int tsf_channel_set_volume(tsf* f, int channel, float volume); +TSFDEF int tsf_channel_set_pitchwheel(tsf* f, int channel, int pitch_wheel); +TSFDEF int tsf_channel_set_pitchrange(tsf* f, int channel, float pitch_range); +TSFDEF int tsf_channel_set_tuning(tsf* f, int channel, float tuning); + +// Start or stop playing notes on a channel (needs channel preset to be set) +// channel: channel number +// key: note value between 0 and 127 (60 being middle C) +// vel: velocity as a float between 0.0 (equal to note off) and 1.0 (full) +// (tsf_channel_note_on returns 0 on allocation failure of new voice, otherwise 1) +TSFDEF int tsf_channel_note_on(tsf* f, int channel, int key, float vel); +TSFDEF void tsf_channel_note_off(tsf* f, int channel, int key); +TSFDEF void tsf_channel_note_off_all(tsf* f, int channel); //end with sustain and release +TSFDEF void tsf_channel_sounds_off_all(tsf* f, int channel); //end immediately + +// Apply a MIDI control change to the channel (not all controllers are supported!) +// (tsf_channel_midi_control returns 0 on allocation failure of new channel, otherwise 1) +TSFDEF int tsf_channel_midi_control(tsf* f, int channel, int controller, int control_value); + +// Get current values set on the channels +TSFDEF int tsf_channel_get_preset_index(tsf* f, int channel); +TSFDEF int tsf_channel_get_preset_bank(tsf* f, int channel); +TSFDEF int tsf_channel_get_preset_number(tsf* f, int channel); +TSFDEF float tsf_channel_get_pan(tsf* f, int channel); +TSFDEF float tsf_channel_get_volume(tsf* f, int channel); +TSFDEF int tsf_channel_get_pitchwheel(tsf* f, int channel); +TSFDEF float tsf_channel_get_pitchrange(tsf* f, int channel); +TSFDEF float tsf_channel_get_tuning(tsf* f, int channel); + +#ifdef __cplusplus +# undef CPP_DEFAULT0 +} +#endif + +// end header +// --------------------------------------------------------------------------------------------------------- +#endif //TSF_INCLUDE_TSF_INL + +#ifdef TSF_IMPLEMENTATION +#undef TSF_IMPLEMENTATION + +// The lower this block size is the more accurate the effects are. +// Increasing the value significantly lowers the CPU usage of the voice rendering. +// If LFO affects the low-pass filter it can be hearable even as low as 8. +#ifndef TSF_RENDER_EFFECTSAMPLEBLOCK +#define TSF_RENDER_EFFECTSAMPLEBLOCK 64 +#endif + +// When using tsf_render_short, to do the conversion a buffer of a fixed size is +// allocated on the stack. On low memory platforms this could be made smaller. +// Increasing this above 512 should not have a significant impact on performance. +// The value should be a multiple of TSF_RENDER_EFFECTSAMPLEBLOCK. +#ifndef TSF_RENDER_SHORTBUFFERBLOCK +#define TSF_RENDER_SHORTBUFFERBLOCK 512 +#endif + +// Grace release time for quick voice off (avoid clicking noise) +#define TSF_FASTRELEASETIME 0.01f + +#if !defined(TSF_MALLOC) || !defined(TSF_FREE) || !defined(TSF_REALLOC) +# include +# define TSF_MALLOC malloc +# define TSF_FREE free +# define TSF_REALLOC realloc +#endif + +#if !defined(TSF_MEMCPY) || !defined(TSF_MEMSET) +# include +# define TSF_MEMCPY memcpy +# define TSF_MEMSET memset +#endif + +#if !defined(TSF_POW) || !defined(TSF_POWF) || !defined(TSF_EXPF) || !defined(TSF_LOG) || !defined(TSF_TAN) || !defined(TSF_LOG10) || !defined(TSF_SQRT) +# include +# if !defined(__cplusplus) && !defined(NAN) && !defined(powf) && !defined(expf) && !defined(sqrtf) +# define powf (float)pow // deal with old math.h +# define expf (float)exp // files that come without +# define sqrtf (float)sqrt // powf, expf and sqrtf +# endif +# define TSF_POW pow +# define TSF_POWF powf +# define TSF_EXPF expf +# define TSF_LOG log +# define TSF_TAN tan +# define TSF_LOG10 log10 +# define TSF_SQRTF sqrtf +#endif + +#ifndef TSF_NO_STDIO +# include +#endif + +#define TSF_TRUE 1 +#define TSF_FALSE 0 +#define TSF_BOOL char +#define TSF_PI 3.14159265358979323846264338327950288 +#define TSF_NULL 0 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef char tsf_fourcc[4]; +typedef signed char tsf_s8; +typedef unsigned char tsf_u8; +typedef unsigned short tsf_u16; +typedef signed short tsf_s16; +typedef unsigned int tsf_u32; +typedef char tsf_char20[20]; + +#define TSF_FourCCEquals(value1, value2) (value1[0] == value2[0] && value1[1] == value2[1] && value1[2] == value2[2] && value1[3] == value2[3]) + +struct tsf +{ + struct tsf_preset* presets; + float* fontSamples; + struct tsf_voice* voices; + struct tsf_channels* channels; + + int presetNum; + int voiceNum; + int maxVoiceNum; + unsigned int voicePlayIndex; + + enum TSFOutputMode outputmode; + float outSampleRate; + float globalGainDB; + int* refCount; +}; + +#ifndef TSF_NO_STDIO +static int tsf_stream_stdio_read(FILE* f, void* ptr, unsigned int size) { return (int)fread(ptr, 1, size, f); } +static int tsf_stream_stdio_skip(FILE* f, unsigned int count) { return !fseek(f, count, SEEK_CUR); } +TSFDEF tsf* tsf_load_filename(const char* filename) +{ + tsf* res; + struct tsf_stream stream = { TSF_NULL, (int(*)(void*,void*,unsigned int))&tsf_stream_stdio_read, (int(*)(void*,unsigned int))&tsf_stream_stdio_skip }; + #if __STDC_WANT_SECURE_LIB__ + FILE* f = TSF_NULL; fopen_s(&f, filename, "rb"); + #else + FILE* f = fopen(filename, "rb"); + #endif + if (!f) + { + //if (e) *e = TSF_FILENOTFOUND; + return TSF_NULL; + } + stream.data = f; + res = tsf_load(&stream); + fclose(f); + return res; +} +#endif + +struct tsf_stream_memory { const char* buffer; unsigned int total, pos; }; +static int tsf_stream_memory_read(struct tsf_stream_memory* m, void* ptr, unsigned int size) { if (size > m->total - m->pos) size = m->total - m->pos; TSF_MEMCPY(ptr, m->buffer+m->pos, size); m->pos += size; return size; } +static int tsf_stream_memory_skip(struct tsf_stream_memory* m, unsigned int count) { if (m->pos + count > m->total) return 0; m->pos += count; return 1; } +TSFDEF tsf* tsf_load_memory(const void* buffer, int size) +{ + struct tsf_stream stream = { TSF_NULL, (int(*)(void*,void*,unsigned int))&tsf_stream_memory_read, (int(*)(void*,unsigned int))&tsf_stream_memory_skip }; + struct tsf_stream_memory f = { 0, 0, 0 }; + f.buffer = (const char*)buffer; + f.total = size; + stream.data = &f; + return tsf_load(&stream); +} + +enum { TSF_LOOPMODE_NONE, TSF_LOOPMODE_CONTINUOUS, TSF_LOOPMODE_SUSTAIN }; + +enum { TSF_SEGMENT_NONE, TSF_SEGMENT_DELAY, TSF_SEGMENT_ATTACK, TSF_SEGMENT_HOLD, TSF_SEGMENT_DECAY, TSF_SEGMENT_SUSTAIN, TSF_SEGMENT_RELEASE, TSF_SEGMENT_DONE }; + +struct tsf_hydra +{ + struct tsf_hydra_phdr *phdrs; struct tsf_hydra_pbag *pbags; struct tsf_hydra_pmod *pmods; + struct tsf_hydra_pgen *pgens; struct tsf_hydra_inst *insts; struct tsf_hydra_ibag *ibags; + struct tsf_hydra_imod *imods; struct tsf_hydra_igen *igens; struct tsf_hydra_shdr *shdrs; + int phdrNum, pbagNum, pmodNum, pgenNum, instNum, ibagNum, imodNum, igenNum, shdrNum; +}; + +union tsf_hydra_genamount { struct { tsf_u8 lo, hi; } range; tsf_s16 shortAmount; tsf_u16 wordAmount; }; +struct tsf_hydra_phdr { tsf_char20 presetName; tsf_u16 preset, bank, presetBagNdx; tsf_u32 library, genre, morphology; }; +struct tsf_hydra_pbag { tsf_u16 genNdx, modNdx; }; +struct tsf_hydra_pmod { tsf_u16 modSrcOper, modDestOper; tsf_s16 modAmount; tsf_u16 modAmtSrcOper, modTransOper; }; +struct tsf_hydra_pgen { tsf_u16 genOper; union tsf_hydra_genamount genAmount; }; +struct tsf_hydra_inst { tsf_char20 instName; tsf_u16 instBagNdx; }; +struct tsf_hydra_ibag { tsf_u16 instGenNdx, instModNdx; }; +struct tsf_hydra_imod { tsf_u16 modSrcOper, modDestOper; tsf_s16 modAmount; tsf_u16 modAmtSrcOper, modTransOper; }; +struct tsf_hydra_igen { tsf_u16 genOper; union tsf_hydra_genamount genAmount; }; +struct tsf_hydra_shdr { tsf_char20 sampleName; tsf_u32 start, end, startLoop, endLoop, sampleRate; tsf_u8 originalPitch; tsf_s8 pitchCorrection; tsf_u16 sampleLink, sampleType; }; + +#define TSFR(FIELD) stream->read(stream->data, &i->FIELD, sizeof(i->FIELD)); +static void tsf_hydra_read_phdr(struct tsf_hydra_phdr* i, struct tsf_stream* stream) { TSFR(presetName) TSFR(preset) TSFR(bank) TSFR(presetBagNdx) TSFR(library) TSFR(genre) TSFR(morphology) } +static void tsf_hydra_read_pbag(struct tsf_hydra_pbag* i, struct tsf_stream* stream) { TSFR(genNdx) TSFR(modNdx) } +static void tsf_hydra_read_pmod(struct tsf_hydra_pmod* i, struct tsf_stream* stream) { TSFR(modSrcOper) TSFR(modDestOper) TSFR(modAmount) TSFR(modAmtSrcOper) TSFR(modTransOper) } +static void tsf_hydra_read_pgen(struct tsf_hydra_pgen* i, struct tsf_stream* stream) { TSFR(genOper) TSFR(genAmount) } +static void tsf_hydra_read_inst(struct tsf_hydra_inst* i, struct tsf_stream* stream) { TSFR(instName) TSFR(instBagNdx) } +static void tsf_hydra_read_ibag(struct tsf_hydra_ibag* i, struct tsf_stream* stream) { TSFR(instGenNdx) TSFR(instModNdx) } +static void tsf_hydra_read_imod(struct tsf_hydra_imod* i, struct tsf_stream* stream) { TSFR(modSrcOper) TSFR(modDestOper) TSFR(modAmount) TSFR(modAmtSrcOper) TSFR(modTransOper) } +static void tsf_hydra_read_igen(struct tsf_hydra_igen* i, struct tsf_stream* stream) { TSFR(genOper) TSFR(genAmount) } +static void tsf_hydra_read_shdr(struct tsf_hydra_shdr* i, struct tsf_stream* stream) { TSFR(sampleName) TSFR(start) TSFR(end) TSFR(startLoop) TSFR(endLoop) TSFR(sampleRate) TSFR(originalPitch) TSFR(pitchCorrection) TSFR(sampleLink) TSFR(sampleType) } +#undef TSFR + +struct tsf_riffchunk { tsf_fourcc id; tsf_u32 size; }; +struct tsf_envelope { float delay, attack, hold, decay, sustain, release, keynumToHold, keynumToDecay; }; +struct tsf_voice_envelope { float level, slope; int samplesUntilNextSegment; short segment, midiVelocity; struct tsf_envelope parameters; TSF_BOOL segmentIsExponential, isAmpEnv; }; +struct tsf_voice_lowpass { double QInv, a0, a1, b1, b2, z1, z2; TSF_BOOL active; }; +struct tsf_voice_lfo { int samplesUntil; float level, delta; }; + +struct tsf_region +{ + int loop_mode; + unsigned int sample_rate; + unsigned char lokey, hikey, lovel, hivel; + unsigned int group, offset, end, loop_start, loop_end; + int transpose, tune, pitch_keycenter, pitch_keytrack; + float attenuation, pan; + struct tsf_envelope ampenv, modenv; + int initialFilterQ, initialFilterFc; + int modEnvToPitch, modEnvToFilterFc, modLfoToFilterFc, modLfoToVolume; + float delayModLFO; + int freqModLFO, modLfoToPitch; + float delayVibLFO; + int freqVibLFO, vibLfoToPitch; +}; + +struct tsf_preset +{ + tsf_char20 presetName; + tsf_u16 preset, bank; + struct tsf_region* regions; + int regionNum; +}; + +struct tsf_voice +{ + int playingPreset, playingKey, playingChannel; + struct tsf_region* region; + double pitchInputTimecents, pitchOutputFactor; + double sourceSamplePosition; + float noteGainDB, panFactorLeft, panFactorRight; + unsigned int playIndex, loopStart, loopEnd; + struct tsf_voice_envelope ampenv, modenv; + struct tsf_voice_lowpass lowpass; + struct tsf_voice_lfo modlfo, viblfo; +}; + +struct tsf_channel +{ + unsigned short presetIndex, bank, pitchWheel, midiPan, midiVolume, midiExpression, midiRPN, midiData; + float panOffset, gainDB, pitchRange, tuning; +}; + +struct tsf_channels +{ + void (*setupVoice)(tsf* f, struct tsf_voice* voice); + int channelNum, activeChannel; + struct tsf_channel channels[1]; +}; + +static double tsf_timecents2Secsd(double timecents) { return TSF_POW(2.0, timecents / 1200.0); } +static float tsf_timecents2Secsf(float timecents) { return TSF_POWF(2.0f, timecents / 1200.0f); } +static float tsf_cents2Hertz(float cents) { return 8.176f * TSF_POWF(2.0f, cents / 1200.0f); } +static float tsf_decibelsToGain(float db) { return (db > -100.f ? TSF_POWF(10.0f, db * 0.05f) : 0); } +static float tsf_gainToDecibels(float gain) { return (gain <= .00001f ? -100.f : (float)(20.0 * TSF_LOG10(gain))); } + +static TSF_BOOL tsf_riffchunk_read(struct tsf_riffchunk* parent, struct tsf_riffchunk* chunk, struct tsf_stream* stream) +{ + TSF_BOOL IsRiff, IsList; + if (parent && sizeof(tsf_fourcc) + sizeof(tsf_u32) > parent->size) return TSF_FALSE; + if (!stream->read(stream->data, &chunk->id, sizeof(tsf_fourcc)) || *chunk->id <= ' ' || *chunk->id >= 'z') return TSF_FALSE; + if (!stream->read(stream->data, &chunk->size, sizeof(tsf_u32))) return TSF_FALSE; + if (parent && sizeof(tsf_fourcc) + sizeof(tsf_u32) + chunk->size > parent->size) return TSF_FALSE; + if (parent) parent->size -= sizeof(tsf_fourcc) + sizeof(tsf_u32) + chunk->size; + IsRiff = TSF_FourCCEquals(chunk->id, "RIFF"), IsList = TSF_FourCCEquals(chunk->id, "LIST"); + if (IsRiff && parent) return TSF_FALSE; //not allowed + if (!IsRiff && !IsList) return TSF_TRUE; //custom type without sub type + if (!stream->read(stream->data, &chunk->id, sizeof(tsf_fourcc)) || *chunk->id <= ' ' || *chunk->id >= 'z') return TSF_FALSE; + chunk->size -= sizeof(tsf_fourcc); + return TSF_TRUE; +} + +static void tsf_region_clear(struct tsf_region* i, TSF_BOOL for_relative) +{ + TSF_MEMSET(i, 0, sizeof(struct tsf_region)); + i->hikey = i->hivel = 127; + i->pitch_keycenter = 60; // C4 + if (for_relative) return; + + i->pitch_keytrack = 100; + + i->pitch_keycenter = -1; + + // SF2 defaults in timecents. + i->ampenv.delay = i->ampenv.attack = i->ampenv.hold = i->ampenv.decay = i->ampenv.release = -12000.0f; + i->modenv.delay = i->modenv.attack = i->modenv.hold = i->modenv.decay = i->modenv.release = -12000.0f; + + i->initialFilterFc = 13500; + + i->delayModLFO = -12000.0f; + i->delayVibLFO = -12000.0f; +} + +static void tsf_region_operator(struct tsf_region* region, tsf_u16 genOper, union tsf_hydra_genamount* amount, struct tsf_region* merge_region) +{ + enum + { + _GEN_TYPE_MASK = 0x0F, + GEN_FLOAT = 0x01, + GEN_INT = 0x02, + GEN_UINT_ADD = 0x03, + GEN_UINT_ADD15 = 0x04, + GEN_KEYRANGE = 0x05, + GEN_VELRANGE = 0x06, + GEN_LOOPMODE = 0x07, + GEN_GROUP = 0x08, + GEN_KEYCENTER = 0x09, + + _GEN_LIMIT_MASK = 0xF0, + GEN_INT_LIMIT12K = 0x10, //min -12000, max 12000 + GEN_INT_LIMITFC = 0x20, //min 1500, max 13500 + GEN_INT_LIMITQ = 0x30, //min 0, max 960 + GEN_INT_LIMIT960 = 0x40, //min -960, max 960 + GEN_INT_LIMIT16K4500 = 0x50, //min -16000, max 4500 + GEN_FLOAT_LIMIT12K5K = 0x60, //min -12000, max 5000 + GEN_FLOAT_LIMIT12K8K = 0x70, //min -12000, max 8000 + GEN_FLOAT_LIMIT1200 = 0x80, //min -1200, max 1200 + GEN_FLOAT_LIMITPAN = 0x90, //* .001f, min -.5f, max .5f, + GEN_FLOAT_LIMITATTN = 0xA0, //* .1f, min 0, max 144.0 + GEN_FLOAT_MAX1000 = 0xB0, //min 0, max 1000 + GEN_FLOAT_MAX1440 = 0xC0, //min 0, max 1440 + + _GEN_MAX = 59 + }; + #define _TSFREGIONOFFSET(TYPE, FIELD) (unsigned char)(((TYPE*)&((struct tsf_region*)0)->FIELD) - (TYPE*)0) + #define _TSFREGIONENVOFFSET(TYPE, ENV, FIELD) (unsigned char)(((TYPE*)&((&(((struct tsf_region*)0)->ENV))->FIELD)) - (TYPE*)0) + static const struct { unsigned char mode, offset; } genMetas[_GEN_MAX] = + { + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, offset ) }, // 0 StartAddrsOffset + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, end ) }, // 1 EndAddrsOffset + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, loop_start ) }, // 2 StartloopAddrsOffset + { GEN_UINT_ADD , _TSFREGIONOFFSET(unsigned int, loop_end ) }, // 3 EndloopAddrsOffset + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, offset ) }, // 4 StartAddrsCoarseOffset + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modLfoToPitch ) }, // 5 ModLfoToPitch + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, vibLfoToPitch ) }, // 6 VibLfoToPitch + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modEnvToPitch ) }, // 7 ModEnvToPitch + { GEN_INT | GEN_INT_LIMITFC , _TSFREGIONOFFSET( int, initialFilterFc ) }, // 8 InitialFilterFc + { GEN_INT | GEN_INT_LIMITQ , _TSFREGIONOFFSET( int, initialFilterQ ) }, // 9 InitialFilterQ + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modLfoToFilterFc ) }, //10 ModLfoToFilterFc + { GEN_INT | GEN_INT_LIMIT12K , _TSFREGIONOFFSET( int, modEnvToFilterFc ) }, //11 ModEnvToFilterFc + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, end ) }, //12 EndAddrsCoarseOffset + { GEN_INT | GEN_INT_LIMIT960 , _TSFREGIONOFFSET( int, modLfoToVolume ) }, //13 ModLfoToVolume + { 0 , (0 ) }, // Unused + { 0 , (0 ) }, //15 ChorusEffectsSend (unsupported) + { 0 , (0 ) }, //16 ReverbEffectsSend (unsupported) + { GEN_FLOAT | GEN_FLOAT_LIMITPAN , _TSFREGIONOFFSET( float, pan ) }, //17 Pan + { 0 , (0 ) }, // Unused + { 0 , (0 ) }, // Unused + { 0 , (0 ) }, // Unused + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONOFFSET( float, delayModLFO ) }, //21 DelayModLFO + { GEN_INT | GEN_INT_LIMIT16K4500 , _TSFREGIONOFFSET( int, freqModLFO ) }, //22 FreqModLFO + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONOFFSET( float, delayVibLFO ) }, //23 DelayVibLFO + { GEN_INT | GEN_INT_LIMIT16K4500 , _TSFREGIONOFFSET( int, freqVibLFO ) }, //24 FreqVibLFO + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, modenv, delay ) }, //25 DelayModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, modenv, attack ) }, //26 AttackModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, modenv, hold ) }, //27 HoldModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, modenv, decay ) }, //28 DecayModEnv + { GEN_FLOAT | GEN_FLOAT_MAX1000 , _TSFREGIONENVOFFSET( float, modenv, sustain ) }, //29 SustainModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, modenv, release ) }, //30 ReleaseModEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, modenv, keynumToHold ) }, //31 KeynumToModEnvHold + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, modenv, keynumToDecay) }, //32 KeynumToModEnvDecay + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, ampenv, delay ) }, //33 DelayVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, ampenv, attack ) }, //34 AttackVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K5K , _TSFREGIONENVOFFSET( float, ampenv, hold ) }, //35 HoldVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, ampenv, decay ) }, //36 DecayVolEnv + { GEN_FLOAT | GEN_FLOAT_MAX1440 , _TSFREGIONENVOFFSET( float, ampenv, sustain ) }, //37 SustainVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT12K8K , _TSFREGIONENVOFFSET( float, ampenv, release ) }, //38 ReleaseVolEnv + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, ampenv, keynumToHold ) }, //39 KeynumToVolEnvHold + { GEN_FLOAT | GEN_FLOAT_LIMIT1200 , _TSFREGIONENVOFFSET( float, ampenv, keynumToDecay) }, //40 KeynumToVolEnvDecay + { 0 , (0 ) }, // Instrument (special) + { 0 , (0 ) }, // Reserved + { GEN_KEYRANGE , (0 ) }, //43 KeyRange + { GEN_VELRANGE , (0 ) }, //44 VelRange + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, loop_start ) }, //45 StartloopAddrsCoarseOffset + { 0 , (0 ) }, //46 Keynum (special) + { 0 , (0 ) }, //47 Velocity (special) + { GEN_FLOAT | GEN_FLOAT_LIMITATTN , _TSFREGIONOFFSET( float, attenuation ) }, //48 InitialAttenuation + { 0 , (0 ) }, // Reserved + { GEN_UINT_ADD15 , _TSFREGIONOFFSET(unsigned int, loop_end ) }, //50 EndloopAddrsCoarseOffset + { GEN_INT , _TSFREGIONOFFSET( int, transpose ) }, //51 CoarseTune + { GEN_INT , _TSFREGIONOFFSET( int, tune ) }, //52 FineTune + { 0 , (0 ) }, // SampleID (special) + { GEN_LOOPMODE , _TSFREGIONOFFSET( int, loop_mode ) }, //54 SampleModes + { 0 , (0 ) }, // Reserved + { GEN_INT , _TSFREGIONOFFSET( int, pitch_keytrack ) }, //56 ScaleTuning + { GEN_GROUP , _TSFREGIONOFFSET(unsigned int, group ) }, //57 ExclusiveClass + { GEN_KEYCENTER , _TSFREGIONOFFSET( int, pitch_keycenter ) }, //58 OverridingRootKey + }; + #undef _TSFREGIONOFFSET + #undef _TSFREGIONENVOFFSET + if (amount) + { + int offset; + if (genOper >= _GEN_MAX) return; + offset = genMetas[genOper].offset; + switch (genMetas[genOper].mode & _GEN_TYPE_MASK) + { + case GEN_FLOAT: (( float*)region)[offset] = amount->shortAmount; return; + case GEN_INT: (( int*)region)[offset] = amount->shortAmount; return; + case GEN_UINT_ADD: ((unsigned int*)region)[offset] += amount->shortAmount; return; + case GEN_UINT_ADD15: ((unsigned int*)region)[offset] += amount->shortAmount<<15; return; + case GEN_KEYRANGE: region->lokey = amount->range.lo; region->hikey = amount->range.hi; return; + case GEN_VELRANGE: region->lovel = amount->range.lo; region->hivel = amount->range.hi; return; + case GEN_LOOPMODE: region->loop_mode = ((amount->wordAmount&3) == 3 ? TSF_LOOPMODE_SUSTAIN : ((amount->wordAmount&3) == 1 ? TSF_LOOPMODE_CONTINUOUS : TSF_LOOPMODE_NONE)); return; + case GEN_GROUP: region->group = amount->wordAmount; return; + case GEN_KEYCENTER: region->pitch_keycenter = amount->shortAmount; return; + } + } + else //merge regions and clamp values + { + for (genOper = 0; genOper != _GEN_MAX; genOper++) + { + int offset = genMetas[genOper].offset; + switch (genMetas[genOper].mode & _GEN_TYPE_MASK) + { + case GEN_FLOAT: + { + float *val = &((float*)region)[offset], vfactor, vmin, vmax; + *val += ((float*)merge_region)[offset]; + switch (genMetas[genOper].mode & _GEN_LIMIT_MASK) + { + case GEN_FLOAT_LIMIT12K5K: vfactor = 1.0f; vmin = -12000.0f; vmax = 5000.0f; break; + case GEN_FLOAT_LIMIT12K8K: vfactor = 1.0f; vmin = -12000.0f; vmax = 8000.0f; break; + case GEN_FLOAT_LIMIT1200: vfactor = 1.0f; vmin = -1200.0f; vmax = 1200.0f; break; + case GEN_FLOAT_LIMITPAN: vfactor = 0.001f; vmin = -0.5f; vmax = 0.5f; break; + case GEN_FLOAT_LIMITATTN: vfactor = 0.1f; vmin = 0.0f; vmax = 144.0f; break; + case GEN_FLOAT_MAX1000: vfactor = 1.0f; vmin = 0.0f; vmax = 1000.0f; break; + case GEN_FLOAT_MAX1440: vfactor = 1.0f; vmin = 0.0f; vmax = 1440.0f; break; + default: continue; + } + *val *= vfactor; + if (*val < vmin) *val = vmin; + else if (*val > vmax) *val = vmax; + continue; + } + case GEN_INT: + { + int *val = &((int*)region)[offset], vmin, vmax; + *val += ((int*)merge_region)[offset]; + switch (genMetas[genOper].mode & _GEN_LIMIT_MASK) + { + case GEN_INT_LIMIT12K: vmin = -12000; vmax = 12000; break; + case GEN_INT_LIMITFC: vmin = 1500; vmax = 13500; break; + case GEN_INT_LIMITQ: vmin = 0; vmax = 960; break; + case GEN_INT_LIMIT960: vmin = -960; vmax = 960; break; + case GEN_INT_LIMIT16K4500: vmin = -16000; vmax = 4500; break; + default: continue; + } + if (*val < vmin) *val = vmin; + else if (*val > vmax) *val = vmax; + continue; + } + case GEN_UINT_ADD: + { + ((unsigned int*)region)[offset] += ((unsigned int*)merge_region)[offset]; + continue; + } + } + } + } +} + +static void tsf_region_envtosecs(struct tsf_envelope* p, TSF_BOOL sustainIsGain) +{ + // EG times need to be converted from timecents to seconds. + // Pin very short EG segments. Timecents don't get to zero, and our EG is + // happier with zero values. + p->delay = (p->delay < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->delay)); + p->attack = (p->attack < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->attack)); + p->release = (p->release < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->release)); + + // If we have dynamic hold or decay times depending on key number we need + // to keep the values in timecents so we can calculate it during startNote + if (!p->keynumToHold) p->hold = (p->hold < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->hold)); + if (!p->keynumToDecay) p->decay = (p->decay < -11950.0f ? 0.0f : tsf_timecents2Secsf(p->decay)); + + if (p->sustain < 0.0f) p->sustain = 0.0f; + else if (sustainIsGain) p->sustain = tsf_decibelsToGain(-p->sustain / 10.0f); + else p->sustain = 1.0f - (p->sustain / 1000.0f); +} + +static int tsf_load_presets(tsf* res, struct tsf_hydra *hydra, unsigned int fontSampleCount) +{ + enum { GenInstrument = 41, GenKeyRange = 43, GenVelRange = 44, GenSampleID = 53 }; + // Read each preset. + struct tsf_hydra_phdr *pphdr, *pphdrMax; + res->presetNum = hydra->phdrNum - 1; + res->presets = (struct tsf_preset*)TSF_MALLOC(res->presetNum * sizeof(struct tsf_preset)); + if (!res->presets) return 0; + else { int i; for (i = 0; i != res->presetNum; i++) res->presets[i].regions = TSF_NULL; } + for (pphdr = hydra->phdrs, pphdrMax = pphdr + hydra->phdrNum - 1; pphdr != pphdrMax; pphdr++) + { + int sortedIndex = 0, region_index = 0; + struct tsf_hydra_phdr *otherphdr; + struct tsf_preset* preset; + struct tsf_hydra_pbag *ppbag, *ppbagEnd; + struct tsf_region globalRegion; + for (otherphdr = hydra->phdrs; otherphdr != pphdrMax; otherphdr++) + { + if (otherphdr == pphdr || otherphdr->bank > pphdr->bank) continue; + else if (otherphdr->bank < pphdr->bank) sortedIndex++; + else if (otherphdr->preset > pphdr->preset) continue; + else if (otherphdr->preset < pphdr->preset) sortedIndex++; + else if (otherphdr < pphdr) sortedIndex++; + } + + preset = &res->presets[sortedIndex]; + TSF_MEMCPY(preset->presetName, pphdr->presetName, sizeof(preset->presetName)); + preset->presetName[sizeof(preset->presetName)-1] = '\0'; //should be zero terminated in source file but make sure + preset->bank = pphdr->bank; + preset->preset = pphdr->preset; + preset->regionNum = 0; + + //count regions covered by this preset + for (ppbag = hydra->pbags + pphdr->presetBagNdx, ppbagEnd = hydra->pbags + pphdr[1].presetBagNdx; ppbag != ppbagEnd; ppbag++) + { + unsigned char plokey = 0, phikey = 127, plovel = 0, phivel = 127; + struct tsf_hydra_pgen *ppgen, *ppgenEnd; struct tsf_hydra_inst *pinst; struct tsf_hydra_ibag *pibag, *pibagEnd; struct tsf_hydra_igen *pigen, *pigenEnd; + for (ppgen = hydra->pgens + ppbag->genNdx, ppgenEnd = hydra->pgens + ppbag[1].genNdx; ppgen != ppgenEnd; ppgen++) + { + if (ppgen->genOper == GenKeyRange) { plokey = ppgen->genAmount.range.lo; phikey = ppgen->genAmount.range.hi; continue; } + if (ppgen->genOper == GenVelRange) { plovel = ppgen->genAmount.range.lo; phivel = ppgen->genAmount.range.hi; continue; } + if (ppgen->genOper != GenInstrument) continue; + if (ppgen->genAmount.wordAmount >= hydra->instNum) continue; + pinst = hydra->insts + ppgen->genAmount.wordAmount; + for (pibag = hydra->ibags + pinst->instBagNdx, pibagEnd = hydra->ibags + pinst[1].instBagNdx; pibag != pibagEnd; pibag++) + { + unsigned char ilokey = 0, ihikey = 127, ilovel = 0, ihivel = 127; + for (pigen = hydra->igens + pibag->instGenNdx, pigenEnd = hydra->igens + pibag[1].instGenNdx; pigen != pigenEnd; pigen++) + { + if (pigen->genOper == GenKeyRange) { ilokey = pigen->genAmount.range.lo; ihikey = pigen->genAmount.range.hi; continue; } + if (pigen->genOper == GenVelRange) { ilovel = pigen->genAmount.range.lo; ihivel = pigen->genAmount.range.hi; continue; } + if (pigen->genOper == GenSampleID && ihikey >= plokey && ilokey <= phikey && ihivel >= plovel && ilovel <= phivel) preset->regionNum++; + } + } + } + } + + preset->regions = (struct tsf_region*)TSF_MALLOC(preset->regionNum * sizeof(struct tsf_region)); + if (!preset->regions) + { + int i; for (i = 0; i != res->presetNum; i++) TSF_FREE(res->presets[i].regions); + TSF_FREE(res->presets); + return 0; + } + tsf_region_clear(&globalRegion, TSF_TRUE); + + // Zones. + for (ppbag = hydra->pbags + pphdr->presetBagNdx, ppbagEnd = hydra->pbags + pphdr[1].presetBagNdx; ppbag != ppbagEnd; ppbag++) + { + struct tsf_hydra_pgen *ppgen, *ppgenEnd; struct tsf_hydra_inst *pinst; struct tsf_hydra_ibag *pibag, *pibagEnd; struct tsf_hydra_igen *pigen, *pigenEnd; + struct tsf_region presetRegion = globalRegion; + int hadGenInstrument = 0; + + // Generators. + for (ppgen = hydra->pgens + ppbag->genNdx, ppgenEnd = hydra->pgens + ppbag[1].genNdx; ppgen != ppgenEnd; ppgen++) + { + // Instrument. + if (ppgen->genOper == GenInstrument) + { + struct tsf_region instRegion; + tsf_u16 whichInst = ppgen->genAmount.wordAmount; + if (whichInst >= hydra->instNum) continue; + + tsf_region_clear(&instRegion, TSF_FALSE); + pinst = &hydra->insts[whichInst]; + for (pibag = hydra->ibags + pinst->instBagNdx, pibagEnd = hydra->ibags + pinst[1].instBagNdx; pibag != pibagEnd; pibag++) + { + // Generators. + struct tsf_region zoneRegion = instRegion; + int hadSampleID = 0; + for (pigen = hydra->igens + pibag->instGenNdx, pigenEnd = hydra->igens + pibag[1].instGenNdx; pigen != pigenEnd; pigen++) + { + if (pigen->genOper == GenSampleID) + { + struct tsf_hydra_shdr* pshdr; + + //preset region key and vel ranges are a filter for the zone regions + if (zoneRegion.hikey < presetRegion.lokey || zoneRegion.lokey > presetRegion.hikey) continue; + if (zoneRegion.hivel < presetRegion.lovel || zoneRegion.lovel > presetRegion.hivel) continue; + if (presetRegion.lokey > zoneRegion.lokey) zoneRegion.lokey = presetRegion.lokey; + if (presetRegion.hikey < zoneRegion.hikey) zoneRegion.hikey = presetRegion.hikey; + if (presetRegion.lovel > zoneRegion.lovel) zoneRegion.lovel = presetRegion.lovel; + if (presetRegion.hivel < zoneRegion.hivel) zoneRegion.hivel = presetRegion.hivel; + + //sum regions + tsf_region_operator(&zoneRegion, 0, TSF_NULL, &presetRegion); + + // EG times need to be converted from timecents to seconds. + tsf_region_envtosecs(&zoneRegion.ampenv, TSF_TRUE); + tsf_region_envtosecs(&zoneRegion.modenv, TSF_FALSE); + + // LFO times need to be converted from timecents to seconds. + zoneRegion.delayModLFO = (zoneRegion.delayModLFO < -11950.0f ? 0.0f : tsf_timecents2Secsf(zoneRegion.delayModLFO)); + zoneRegion.delayVibLFO = (zoneRegion.delayVibLFO < -11950.0f ? 0.0f : tsf_timecents2Secsf(zoneRegion.delayVibLFO)); + + // Fixup sample positions + pshdr = &hydra->shdrs[pigen->genAmount.wordAmount]; + zoneRegion.offset += pshdr->start; + zoneRegion.end += pshdr->end; + zoneRegion.loop_start += pshdr->startLoop; + zoneRegion.loop_end += pshdr->endLoop; + if (pshdr->endLoop > 0) zoneRegion.loop_end -= 1; + if (zoneRegion.loop_end > fontSampleCount) zoneRegion.loop_end = fontSampleCount; + if (zoneRegion.pitch_keycenter == -1) zoneRegion.pitch_keycenter = pshdr->originalPitch; + zoneRegion.tune += pshdr->pitchCorrection; + zoneRegion.sample_rate = pshdr->sampleRate; + if (zoneRegion.end && zoneRegion.end < fontSampleCount) zoneRegion.end++; + else zoneRegion.end = fontSampleCount; + + preset->regions[region_index] = zoneRegion; + region_index++; + hadSampleID = 1; + } + else tsf_region_operator(&zoneRegion, pigen->genOper, &pigen->genAmount, TSF_NULL); + } + + // Handle instrument's global zone. + if (pibag == hydra->ibags + pinst->instBagNdx && !hadSampleID) + instRegion = zoneRegion; + + // Modulators (TODO) + //if (ibag->instModNdx < ibag[1].instModNdx) addUnsupportedOpcode("any modulator"); + } + hadGenInstrument = 1; + } + else tsf_region_operator(&presetRegion, ppgen->genOper, &ppgen->genAmount, TSF_NULL); + } + + // Modulators (TODO) + //if (pbag->modNdx < pbag[1].modNdx) addUnsupportedOpcode("any modulator"); + + // Handle preset's global zone. + if (ppbag == hydra->pbags + pphdr->presetBagNdx && !hadGenInstrument) + globalRegion = presetRegion; + } + } + return 1; +} + +#ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H +static int tsf_decode_ogg(const tsf_u8 *pSmpl, const tsf_u8 *pSmplEnd, float** pRes, tsf_u32* pResNum, tsf_u32* pResMax, tsf_u32 resInitial) +{ + float *res = *pRes, *oldres; tsf_u32 resNum = *pResNum; tsf_u32 resMax = *pResMax; stb_vorbis *v; + + // Use whatever stb_vorbis API that is available (either pull or push) + #if !defined(STB_VORBIS_NO_PULLDATA_API) && !defined(STB_VORBIS_NO_FROMMEMORY) + v = stb_vorbis_open_memory(pSmpl, (int)(pSmplEnd - pSmpl), TSF_NULL, TSF_NULL); + #else + { int use, err; v = stb_vorbis_open_pushdata(pSmpl, (int)(pSmplEnd - pSmpl), &use, &err, TSF_NULL); pSmpl += use; } + #endif + if (v == TSF_NULL) return 0; + + while (true) + { + float** outputs; int n_samples; + + // Decode one frame of vorbis samples with whatever stb_vorbis API that is available + #if !defined(STB_VORBIS_NO_PULLDATA_API) && !defined(STB_VORBIS_NO_FROMMEMORY) + n_samples = stb_vorbis_get_frame_float(v, TSF_NULL, &outputs); + if (!n_samples) break; + #else + if (pSmpl >= pSmplEnd) break; + { int use = stb_vorbis_decode_frame_pushdata(v, pSmpl, (int)(pSmplEnd - pSmpl), TSF_NULL, &outputs, &n_samples); pSmpl += use; } + if (!n_samples) continue; + #endif + + // Expand our output buffer if necessary then copy over the decoded frame samples + resNum += n_samples; + if (resNum > resMax) + { + do { resMax += (resMax ? (resMax < 1048576 ? resMax : 1048576) : resInitial); } while (resNum > resMax); + res = (float*)TSF_REALLOC((oldres = res), resMax * sizeof(float)); + if (!res) { TSF_FREE(oldres); stb_vorbis_close(v); return 0; } + } + TSF_MEMCPY(res + resNum - n_samples, outputs[0], n_samples * sizeof(float)); + } + stb_vorbis_close(v); + *pRes = res; *pResNum = resNum; *pResMax = resMax; + return 1; +} + +static int tsf_decode_sf3_samples(const void* rawBuffer, float** pFloatBuffer, unsigned int* pSmplCount, struct tsf_hydra *hydra) +{ + const tsf_u8* smplBuffer = (const tsf_u8*)rawBuffer; + tsf_u32 smplLength = *pSmplCount, resNum = 0, resMax = 0, resInitial = (smplLength > 0x100000 ? (smplLength & ~0xFFFFF) : 65536); + float *res = TSF_NULL, *oldres; + int i, shdrLast = hydra->shdrNum - 1, is_sf3 = 0; + for (i = 0; i <= shdrLast; i++) + { + struct tsf_hydra_shdr *shdr = &hydra->shdrs[i]; + if (shdr->sampleType & 0x30) // compression flags (sometimes Vorbis flag) + { + const tsf_u8 *pSmpl = smplBuffer + shdr->start, *pSmplEnd = smplBuffer + shdr->end; + if (pSmpl + 4 > pSmplEnd || !TSF_FourCCEquals(pSmpl, "OggS")) + { + shdr->start = shdr->end = shdr->startLoop = shdr->endLoop = 0; + continue; + } + + // Fix up sample indices in shdr (end index is set after decoding) + shdr->start = resNum; + shdr->startLoop += resNum; + shdr->endLoop += resNum; + if (!tsf_decode_ogg(pSmpl, pSmplEnd, &res, &resNum, &resMax, resInitial)) { TSF_FREE(res); return 0; } + shdr->end = resNum; + is_sf3 = 1; + } + else // raw PCM sample + { + float *out; short *in = (short*)smplBuffer + resNum, *inEnd; tsf_u32 oldResNum = resNum; + if (is_sf3) // Fix up sample indices in shdr + { + tsf_u32 fix_offset = resNum - shdr->start; + in -= fix_offset; + shdr->start = resNum; + shdr->end += fix_offset; + shdr->startLoop += fix_offset; + shdr->endLoop += fix_offset; + } + inEnd = in + ((shdr->end >= shdr->endLoop ? shdr->end : shdr->endLoop) - resNum); + if (i == shdrLast || (tsf_u8*)inEnd > (smplBuffer + smplLength)) inEnd = (short*)(smplBuffer + smplLength); + if (inEnd <= in) continue; + + // expand our output buffer if necessary then convert the PCM data from short to float + resNum += (tsf_u32)(inEnd - in); + if (resNum > resMax) + { + do { resMax += (resMax ? (resMax < 1048576 ? resMax : 1048576) : resInitial); } while (resNum > resMax); + res = (float*)TSF_REALLOC((oldres = res), resMax * sizeof(float)); + if (!res) { TSF_FREE(oldres); return 0; } + } + + // Convert the samples from short to float + for (out = res + oldResNum; in < inEnd;) + *(out++) = (float)(*(in++) / 32767.0); + } + } + + // Trim the sample buffer down then return success (unless out of memory) + if (!(*pFloatBuffer = (float*)TSF_REALLOC(res, resNum * sizeof(float)))) *pFloatBuffer = res; + *pFloatBuffer = res; + *pSmplCount = resNum; + return (res ? 1 : 0); +} +#endif + +static int tsf_load_samples(void** pRawBuffer, float** pFloatBuffer, unsigned int* pSmplCount, struct tsf_riffchunk *chunkSmpl, struct tsf_stream* stream) +{ + #ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H + // With OGG Vorbis support we cannot pre-allocate the memory for tsf_decode_sf3_samples + tsf_u32 resNum, resMax; float* oldres; + *pSmplCount = chunkSmpl->size; + *pRawBuffer = (void*)TSF_MALLOC(*pSmplCount); + if (!*pRawBuffer || !stream->read(stream->data, *pRawBuffer, chunkSmpl->size)) return 0; + if (chunkSmpl->id[3] != 'o') return 1; + + // Decode custom .sfo 'smpo' format where all samples are in a single ogg stream + resNum = resMax = 0; + if (!tsf_decode_ogg((tsf_u8*)*pRawBuffer, (tsf_u8*)*pRawBuffer + chunkSmpl->size, pFloatBuffer, &resNum, &resMax, 65536)) return 0; + if (!(*pFloatBuffer = (float*)TSF_REALLOC((oldres = *pFloatBuffer), resNum * sizeof(float)))) *pFloatBuffer = oldres; + *pSmplCount = resNum; + return (*pFloatBuffer ? 1 : 0); + #else + // Inline convert the samples from short to float + float *res, *out; const short *in; + *pSmplCount = chunkSmpl->size / (unsigned int)sizeof(short); + *pFloatBuffer = (float*)TSF_MALLOC(*pSmplCount * sizeof(float)); + if (!*pFloatBuffer || !stream->read(stream->data, *pFloatBuffer, chunkSmpl->size)) return 0; + for (res = *pFloatBuffer, out = res + *pSmplCount, in = (short*)res + *pSmplCount; out != res;) + *(--out) = (float)(*(--in) / 32767.0); + return 1; + #endif +} + +static int tsf_voice_envelope_release_samples(struct tsf_voice_envelope* e, float outSampleRate) +{ + return (int)((e->parameters.release <= 0 ? TSF_FASTRELEASETIME : e->parameters.release) * outSampleRate); +} + +static void tsf_voice_envelope_nextsegment(struct tsf_voice_envelope* e, short active_segment, float outSampleRate) +{ + switch (active_segment) + { + case TSF_SEGMENT_NONE: + e->samplesUntilNextSegment = (int)(e->parameters.delay * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + e->segment = TSF_SEGMENT_DELAY; + e->segmentIsExponential = TSF_FALSE; + e->level = 0.0; + e->slope = 0.0; + return; + } + /* fall through */ + case TSF_SEGMENT_DELAY: + e->samplesUntilNextSegment = (int)(e->parameters.attack * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + if (!e->isAmpEnv) + { + //mod env attack duration scales with velocity (velocity of 1 is full duration, max velocity is 0.125 times duration) + e->samplesUntilNextSegment = (int)(e->parameters.attack * ((145 - e->midiVelocity) / 144.0f) * outSampleRate); + } + e->segment = TSF_SEGMENT_ATTACK; + e->segmentIsExponential = TSF_FALSE; + e->level = 0.0f; + e->slope = 1.0f / e->samplesUntilNextSegment; + return; + } + /* fall through */ + case TSF_SEGMENT_ATTACK: + e->samplesUntilNextSegment = (int)(e->parameters.hold * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + e->segment = TSF_SEGMENT_HOLD; + e->segmentIsExponential = TSF_FALSE; + e->level = 1.0f; + e->slope = 0.0f; + return; + } + /* fall through */ + case TSF_SEGMENT_HOLD: + e->samplesUntilNextSegment = (int)(e->parameters.decay * outSampleRate); + if (e->samplesUntilNextSegment > 0) + { + e->segment = TSF_SEGMENT_DECAY; + e->level = 1.0f; + if (e->isAmpEnv) + { + // I don't truly understand this; just following what LinuxSampler does. + float mysterySlope = -9.226f / e->samplesUntilNextSegment; + e->slope = TSF_EXPF(mysterySlope); + e->segmentIsExponential = TSF_TRUE; + if (e->parameters.sustain > 0.0f) + { + // Again, this is following LinuxSampler's example, which is similar to + // SF2-style decay, where "decay" specifies the time it would take to + // get to zero, not to the sustain level. The SFZ spec is not that + // specific about what "decay" means, so perhaps it's really supposed + // to specify the time to reach the sustain level. + e->samplesUntilNextSegment = (int)(TSF_LOG(e->parameters.sustain) / mysterySlope); + } + } + else + { + e->slope = -1.0f / e->samplesUntilNextSegment; + e->samplesUntilNextSegment = (int)(e->parameters.decay * (1.0f - e->parameters.sustain) * outSampleRate); + e->segmentIsExponential = TSF_FALSE; + } + return; + } + /* fall through */ + case TSF_SEGMENT_DECAY: + e->segment = TSF_SEGMENT_SUSTAIN; + e->level = e->parameters.sustain; + e->slope = 0.0f; + e->samplesUntilNextSegment = 0x7FFFFFFF; + e->segmentIsExponential = TSF_FALSE; + return; + case TSF_SEGMENT_SUSTAIN: + e->segment = TSF_SEGMENT_RELEASE; + e->samplesUntilNextSegment = tsf_voice_envelope_release_samples(e, outSampleRate); + if (e->isAmpEnv) + { + // I don't truly understand this; just following what LinuxSampler does. + float mysterySlope = -9.226f / e->samplesUntilNextSegment; + e->slope = TSF_EXPF(mysterySlope); + e->segmentIsExponential = TSF_TRUE; + } + else + { + e->slope = -e->level / e->samplesUntilNextSegment; + e->segmentIsExponential = TSF_FALSE; + } + return; + case TSF_SEGMENT_RELEASE: + default: + e->segment = TSF_SEGMENT_DONE; + e->segmentIsExponential = TSF_FALSE; + e->level = e->slope = 0.0f; + e->samplesUntilNextSegment = 0x7FFFFFF; + } +} + +static void tsf_voice_envelope_setup(struct tsf_voice_envelope* e, struct tsf_envelope* new_parameters, int midiNoteNumber, short midiVelocity, TSF_BOOL isAmpEnv, float outSampleRate) +{ + e->parameters = *new_parameters; + if (e->parameters.keynumToHold) + { + e->parameters.hold += e->parameters.keynumToHold * (60.0f - midiNoteNumber); + e->parameters.hold = (e->parameters.hold < -10000.0f ? 0.0f : tsf_timecents2Secsf(e->parameters.hold)); + } + if (e->parameters.keynumToDecay) + { + e->parameters.decay += e->parameters.keynumToDecay * (60.0f - midiNoteNumber); + e->parameters.decay = (e->parameters.decay < -10000.0f ? 0.0f : tsf_timecents2Secsf(e->parameters.decay)); + } + e->midiVelocity = midiVelocity; + e->isAmpEnv = isAmpEnv; + tsf_voice_envelope_nextsegment(e, TSF_SEGMENT_NONE, outSampleRate); +} + +static void tsf_voice_envelope_process(struct tsf_voice_envelope* e, int numSamples, float outSampleRate) +{ + if (e->slope) + { + if (e->segmentIsExponential) e->level *= TSF_POWF(e->slope, (float)numSamples); + else e->level += (e->slope * numSamples); + } + if ((e->samplesUntilNextSegment -= numSamples) <= 0) + tsf_voice_envelope_nextsegment(e, e->segment, outSampleRate); +} + +static void tsf_voice_lowpass_setup(struct tsf_voice_lowpass* e, float Fc) +{ + // Lowpass filter from http://www.earlevel.com/main/2012/11/26/biquad-c-source-code/ + double K = TSF_TAN(TSF_PI * Fc), KK = K * K; + double norm = 1 / (1 + K * e->QInv + KK); + e->a0 = KK * norm; + e->a1 = 2 * e->a0; + e->b1 = 2 * (KK - 1) * norm; + e->b2 = (1 - K * e->QInv + KK) * norm; +} + +static float tsf_voice_lowpass_process(struct tsf_voice_lowpass* e, double In) +{ + double Out = In * e->a0 + e->z1; e->z1 = In * e->a1 + e->z2 - e->b1 * Out; e->z2 = In * e->a0 - e->b2 * Out; return (float)Out; +} + +static void tsf_voice_lfo_setup(struct tsf_voice_lfo* e, float delay, int freqCents, float outSampleRate) +{ + e->samplesUntil = (int)(delay * outSampleRate); + e->delta = (4.0f * tsf_cents2Hertz((float)freqCents) / outSampleRate); + e->level = 0; +} + +static void tsf_voice_lfo_process(struct tsf_voice_lfo* e, int blockSamples) +{ + if (e->samplesUntil > blockSamples) { e->samplesUntil -= blockSamples; return; } + e->level += e->delta * blockSamples; + if (e->level > 1.0f) { e->delta = -e->delta; e->level = 2.0f - e->level; } + else if (e->level < -1.0f) { e->delta = -e->delta; e->level = -2.0f - e->level; } +} + +static void tsf_voice_kill(struct tsf_voice* v) +{ + v->playingPreset = -1; +} + +static void tsf_voice_end(tsf* f, struct tsf_voice* v) +{ + // if maxVoiceNum is set, assume that voice rendering and note queuing are on separate threads + // so to minimize the chance that voice rendering would advance the segment at the same time + // we just do it twice here and hope that it sticks + int repeats = (f->maxVoiceNum ? 2 : 1); + while (repeats--) + { + tsf_voice_envelope_nextsegment(&v->ampenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + tsf_voice_envelope_nextsegment(&v->modenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + if (v->region->loop_mode == TSF_LOOPMODE_SUSTAIN) + { + // Continue playing, but stop looping. + v->loopEnd = v->loopStart; + } + } +} + +static void tsf_voice_endquick(tsf* f, struct tsf_voice* v) +{ + // if maxVoiceNum is set, assume that voice rendering and note queuing are on separate threads + // so to minimize the chance that voice rendering would advance the segment at the same time + // we just do it twice here and hope that it sticks + int repeats = (f->maxVoiceNum ? 2 : 1); + while (repeats--) + { + v->ampenv.parameters.release = 0.0f; tsf_voice_envelope_nextsegment(&v->ampenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + v->modenv.parameters.release = 0.0f; tsf_voice_envelope_nextsegment(&v->modenv, TSF_SEGMENT_SUSTAIN, f->outSampleRate); + } +} + +static void tsf_voice_calcpitchratio(struct tsf_voice* v, float pitchShift, float outSampleRate) +{ + double note = v->playingKey + v->region->transpose + v->region->tune / 100.0; + double adjustedPitch = v->region->pitch_keycenter + (note - v->region->pitch_keycenter) * (v->region->pitch_keytrack / 100.0); + if (pitchShift) adjustedPitch += pitchShift; + v->pitchInputTimecents = adjustedPitch * 100.0; + v->pitchOutputFactor = v->region->sample_rate / (tsf_timecents2Secsd(v->region->pitch_keycenter * 100.0) * outSampleRate); +} + +static void tsf_voice_render(tsf* f, struct tsf_voice* v, float* outputBuffer, int numSamples) +{ + struct tsf_region* region = v->region; + float* input = f->fontSamples; + float* outL = outputBuffer; + float* outR = (f->outputmode == TSF_STEREO_UNWEAVED ? outL + numSamples : TSF_NULL); + + // Cache some values, to give them at least some chance of ending up in registers. + TSF_BOOL updateModEnv = (region->modEnvToPitch || region->modEnvToFilterFc); + TSF_BOOL updateModLFO = (v->modlfo.delta && (region->modLfoToPitch || region->modLfoToFilterFc || region->modLfoToVolume)); + TSF_BOOL updateVibLFO = (v->viblfo.delta && (region->vibLfoToPitch)); + TSF_BOOL isLooping = (v->loopStart < v->loopEnd); + unsigned int tmpLoopStart = v->loopStart, tmpLoopEnd = v->loopEnd; + double tmpSampleEndDbl = (double)region->end, tmpLoopEndDbl = (double)tmpLoopEnd + 1.0; + double tmpSourceSamplePosition = v->sourceSamplePosition; + struct tsf_voice_lowpass tmpLowpass = v->lowpass; + + TSF_BOOL dynamicLowpass = (region->modLfoToFilterFc || region->modEnvToFilterFc); + float tmpSampleRate = f->outSampleRate, tmpInitialFilterFc, tmpModLfoToFilterFc, tmpModEnvToFilterFc; + + TSF_BOOL dynamicPitchRatio = (region->modLfoToPitch || region->modEnvToPitch || region->vibLfoToPitch); + double pitchRatio; + float tmpModLfoToPitch, tmpVibLfoToPitch, tmpModEnvToPitch; + + TSF_BOOL dynamicGain = (region->modLfoToVolume != 0); + float noteGain = 0, tmpModLfoToVolume; + + if (dynamicLowpass) tmpInitialFilterFc = (float)region->initialFilterFc, tmpModLfoToFilterFc = (float)region->modLfoToFilterFc, tmpModEnvToFilterFc = (float)region->modEnvToFilterFc; + else tmpInitialFilterFc = 0, tmpModLfoToFilterFc = 0, tmpModEnvToFilterFc = 0; + + if (dynamicPitchRatio) pitchRatio = 0, tmpModLfoToPitch = (float)region->modLfoToPitch, tmpVibLfoToPitch = (float)region->vibLfoToPitch, tmpModEnvToPitch = (float)region->modEnvToPitch; + else pitchRatio = tsf_timecents2Secsd(v->pitchInputTimecents) * v->pitchOutputFactor, tmpModLfoToPitch = 0, tmpVibLfoToPitch = 0, tmpModEnvToPitch = 0; + + if (dynamicGain) tmpModLfoToVolume = (float)region->modLfoToVolume * 0.1f; + else noteGain = tsf_decibelsToGain(v->noteGainDB), tmpModLfoToVolume = 0; + + while (numSamples) + { + float gainMono, gainLeft, gainRight; + int blockSamples = (numSamples > TSF_RENDER_EFFECTSAMPLEBLOCK ? TSF_RENDER_EFFECTSAMPLEBLOCK : numSamples); + numSamples -= blockSamples; + + if (dynamicLowpass) + { + float fres = tmpInitialFilterFc + v->modlfo.level * tmpModLfoToFilterFc + v->modenv.level * tmpModEnvToFilterFc; + float lowpassFc = (fres <= 13500 ? tsf_cents2Hertz(fres) / tmpSampleRate : 1.0f); + tmpLowpass.active = (lowpassFc < 0.499f); + if (tmpLowpass.active) tsf_voice_lowpass_setup(&tmpLowpass, lowpassFc); + } + + if (dynamicPitchRatio) + pitchRatio = tsf_timecents2Secsd(v->pitchInputTimecents + (v->modlfo.level * tmpModLfoToPitch + v->viblfo.level * tmpVibLfoToPitch + v->modenv.level * tmpModEnvToPitch)) * v->pitchOutputFactor; + + if (dynamicGain) + noteGain = tsf_decibelsToGain(v->noteGainDB + (v->modlfo.level * tmpModLfoToVolume)); + + gainMono = noteGain * v->ampenv.level; + + // Update EG. + tsf_voice_envelope_process(&v->ampenv, blockSamples, tmpSampleRate); + if (updateModEnv) tsf_voice_envelope_process(&v->modenv, blockSamples, tmpSampleRate); + + // Update LFOs. + if (updateModLFO) tsf_voice_lfo_process(&v->modlfo, blockSamples); + if (updateVibLFO) tsf_voice_lfo_process(&v->viblfo, blockSamples); + + switch (f->outputmode) + { + case TSF_STEREO_INTERLEAVED: + gainLeft = gainMono * v->panFactorLeft, gainRight = gainMono * v->panFactorRight; + while (blockSamples-- && tmpSourceSamplePosition < tmpSampleEndDbl) + { + unsigned int pos = (unsigned int)tmpSourceSamplePosition, nextPos = (pos >= tmpLoopEnd && isLooping ? tmpLoopStart : pos + 1); + + // Simple linear interpolation. + float alpha = (float)(tmpSourceSamplePosition - pos), val = (input[pos] * (1.0f - alpha) + input[nextPos] * alpha); + + // Low-pass filter. + if (tmpLowpass.active) val = tsf_voice_lowpass_process(&tmpLowpass, val); + + *outL++ += val * gainLeft; + *outL++ += val * gainRight; + + // Next sample. + tmpSourceSamplePosition += pitchRatio; + if (tmpSourceSamplePosition >= tmpLoopEndDbl && isLooping) tmpSourceSamplePosition -= (tmpLoopEnd - tmpLoopStart + 1.0); + } + break; + + case TSF_STEREO_UNWEAVED: + gainLeft = gainMono * v->panFactorLeft, gainRight = gainMono * v->panFactorRight; + while (blockSamples-- && tmpSourceSamplePosition < tmpSampleEndDbl) + { + unsigned int pos = (unsigned int)tmpSourceSamplePosition, nextPos = (pos >= tmpLoopEnd && isLooping ? tmpLoopStart : pos + 1); + + // Simple linear interpolation. + float alpha = (float)(tmpSourceSamplePosition - pos), val = (input[pos] * (1.0f - alpha) + input[nextPos] * alpha); + + // Low-pass filter. + if (tmpLowpass.active) val = tsf_voice_lowpass_process(&tmpLowpass, val); + + *outL++ += val * gainLeft; + *outR++ += val * gainRight; + + // Next sample. + tmpSourceSamplePosition += pitchRatio; + if (tmpSourceSamplePosition >= tmpLoopEndDbl && isLooping) tmpSourceSamplePosition -= (tmpLoopEnd - tmpLoopStart + 1.0); + } + break; + + case TSF_MONO: + while (blockSamples-- && tmpSourceSamplePosition < tmpSampleEndDbl) + { + unsigned int pos = (unsigned int)tmpSourceSamplePosition, nextPos = (pos >= tmpLoopEnd && isLooping ? tmpLoopStart : pos + 1); + + // Simple linear interpolation. + float alpha = (float)(tmpSourceSamplePosition - pos), val = (input[pos] * (1.0f - alpha) + input[nextPos] * alpha); + + // Low-pass filter. + if (tmpLowpass.active) val = tsf_voice_lowpass_process(&tmpLowpass, val); + + *outL++ += val * gainMono; + + // Next sample. + tmpSourceSamplePosition += pitchRatio; + if (tmpSourceSamplePosition >= tmpLoopEndDbl && isLooping) tmpSourceSamplePosition -= (tmpLoopEnd - tmpLoopStart + 1.0); + } + break; + } + + if (tmpSourceSamplePosition >= tmpSampleEndDbl || v->ampenv.segment == TSF_SEGMENT_DONE) + { + tsf_voice_kill(v); + return; + } + } + + v->sourceSamplePosition = tmpSourceSamplePosition; + if (tmpLowpass.active || dynamicLowpass) v->lowpass = tmpLowpass; +} + +TSFDEF tsf* tsf_load(struct tsf_stream* stream) +{ + tsf* res = TSF_NULL; + struct tsf_riffchunk chunkHead; + struct tsf_riffchunk chunkList; + struct tsf_hydra hydra; + void* rawBuffer = TSF_NULL; + float* floatBuffer = TSF_NULL; + tsf_u32 smplCount = 0; + + if (!tsf_riffchunk_read(TSF_NULL, &chunkHead, stream) || !TSF_FourCCEquals(chunkHead.id, "sfbk")) + { + //if (e) *e = TSF_INVALID_NOSF2HEADER; + return res; + } + + // Read hydra and locate sample data. + TSF_MEMSET(&hydra, 0, sizeof(hydra)); + while (tsf_riffchunk_read(&chunkHead, &chunkList, stream)) + { + struct tsf_riffchunk chunk; + if (TSF_FourCCEquals(chunkList.id, "pdta")) + { + while (tsf_riffchunk_read(&chunkList, &chunk, stream)) + { + #define HandleChunk(chunkName) (TSF_FourCCEquals(chunk.id, #chunkName) && !(chunk.size % chunkName##SizeInFile)) \ + { \ + int num = chunk.size / chunkName##SizeInFile, i; \ + hydra.chunkName##Num = num; \ + hydra.chunkName##s = (struct tsf_hydra_##chunkName*)TSF_MALLOC(num * sizeof(struct tsf_hydra_##chunkName)); \ + if (!hydra.chunkName##s) goto out_of_memory; \ + for (i = 0; i < num; ++i) tsf_hydra_read_##chunkName(&hydra.chunkName##s[i], stream); \ + } + enum + { + phdrSizeInFile = 38, pbagSizeInFile = 4, pmodSizeInFile = 10, + pgenSizeInFile = 4, instSizeInFile = 22, ibagSizeInFile = 4, + imodSizeInFile = 10, igenSizeInFile = 4, shdrSizeInFile = 46 + }; + if HandleChunk(phdr) else if HandleChunk(pbag) else if HandleChunk(pmod) + else if HandleChunk(pgen) else if HandleChunk(inst) else if HandleChunk(ibag) + else if HandleChunk(imod) else if HandleChunk(igen) else if HandleChunk(shdr) + else stream->skip(stream->data, chunk.size); + #undef HandleChunk + } + } + else if (TSF_FourCCEquals(chunkList.id, "sdta")) + { + while (tsf_riffchunk_read(&chunkList, &chunk, stream)) + { + if ((TSF_FourCCEquals(chunk.id, "smpl") + #ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H + || TSF_FourCCEquals(chunk.id, "smpo") + #endif + ) && !rawBuffer && !floatBuffer && chunk.size >= sizeof(short)) + { + if (!tsf_load_samples(&rawBuffer, &floatBuffer, &smplCount, &chunk, stream)) goto out_of_memory; + } + else stream->skip(stream->data, chunk.size); + } + } + else stream->skip(stream->data, chunkList.size); + } + if (!hydra.phdrs || !hydra.pbags || !hydra.pmods || !hydra.pgens || !hydra.insts || !hydra.ibags || !hydra.imods || !hydra.igens || !hydra.shdrs) + { + //if (e) *e = TSF_INVALID_INCOMPLETE; + } + else if (!rawBuffer && !floatBuffer) + { + //if (e) *e = TSF_INVALID_NOSAMPLEDATA; + } + else + { + #ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H + if (!floatBuffer && !tsf_decode_sf3_samples(rawBuffer, &floatBuffer, &smplCount, &hydra)) goto out_of_memory; + #endif + res = (tsf*)TSF_MALLOC(sizeof(tsf)); + if (res) TSF_MEMSET(res, 0, sizeof(tsf)); + if (!res || !tsf_load_presets(res, &hydra, smplCount)) goto out_of_memory; + res->outSampleRate = 44100.0f; + res->fontSamples = floatBuffer; + floatBuffer = TSF_NULL; // don't free below + } + if (0) + { + out_of_memory: + TSF_FREE(res); + res = TSF_NULL; + //if (e) *e = TSF_OUT_OF_MEMORY; + } + TSF_FREE(hydra.phdrs); TSF_FREE(hydra.pbags); TSF_FREE(hydra.pmods); + TSF_FREE(hydra.pgens); TSF_FREE(hydra.insts); TSF_FREE(hydra.ibags); + TSF_FREE(hydra.imods); TSF_FREE(hydra.igens); TSF_FREE(hydra.shdrs); + TSF_FREE(rawBuffer); TSF_FREE(floatBuffer); + return res; +} + +TSFDEF tsf* tsf_copy(tsf* f) +{ + tsf* res; + if (!f) return TSF_NULL; + if (!f->refCount) + { + f->refCount = (int*)TSF_MALLOC(sizeof(int)); + if (!f->refCount) return TSF_NULL; + *f->refCount = 1; + } + res = (tsf*)TSF_MALLOC(sizeof(tsf)); + if (!res) return TSF_NULL; + TSF_MEMCPY(res, f, sizeof(tsf)); + res->voices = TSF_NULL; + res->voiceNum = 0; + res->channels = TSF_NULL; + (*res->refCount)++; + return res; +} + +TSFDEF void tsf_close(tsf* f) +{ + if (!f) return; + if (!f->refCount || !--(*f->refCount)) + { + struct tsf_preset *preset = f->presets, *presetEnd = preset + f->presetNum; + for (; preset != presetEnd; preset++) TSF_FREE(preset->regions); + TSF_FREE(f->presets); + TSF_FREE(f->fontSamples); + TSF_FREE(f->refCount); + } + TSF_FREE(f->channels); + TSF_FREE(f->voices); + TSF_FREE(f); +} + +TSFDEF void tsf_reset(tsf* f) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) + if (v->playingPreset != -1 && (v->ampenv.segment < TSF_SEGMENT_RELEASE || v->ampenv.parameters.release)) + tsf_voice_endquick(f, v); + if (f->channels) { TSF_FREE(f->channels); f->channels = TSF_NULL; } +} + +TSFDEF int tsf_get_presetindex(const tsf* f, int bank, int preset_number) +{ + const struct tsf_preset *presets; + int i, iMax; + for (presets = f->presets, i = 0, iMax = f->presetNum; i < iMax; i++) + if (presets[i].preset == preset_number && presets[i].bank == bank) + return i; + return -1; +} + +TSFDEF int tsf_get_presetcount(const tsf* f) +{ + return f->presetNum; +} + +TSFDEF const char* tsf_get_presetname(const tsf* f, int preset) +{ + return (preset < 0 || preset >= f->presetNum ? TSF_NULL : f->presets[preset].presetName); +} + +TSFDEF const char* tsf_bank_get_presetname(const tsf* f, int bank, int preset_number) +{ + return tsf_get_presetname(f, tsf_get_presetindex(f, bank, preset_number)); +} + +TSFDEF void tsf_set_output(tsf* f, enum TSFOutputMode outputmode, int samplerate, float global_gain_db) +{ + f->outputmode = outputmode; + f->outSampleRate = (float)(samplerate >= 1 ? samplerate : 44100.0f); + f->globalGainDB = global_gain_db; +} + +TSFDEF void tsf_set_volume(tsf* f, float global_volume) +{ + f->globalGainDB = (global_volume == 1.0f ? 0 : -tsf_gainToDecibels(1.0f / global_volume)); +} + +TSFDEF int tsf_set_max_voices(tsf* f, int max_voices) +{ + int i = f->voiceNum; + int newVoiceNum = (f->voiceNum > max_voices ? f->voiceNum : max_voices); + struct tsf_voice *newVoices = (struct tsf_voice*)TSF_REALLOC(f->voices, newVoiceNum * sizeof(struct tsf_voice)); + if (!newVoices) return 0; + f->voices = newVoices; + f->voiceNum = f->maxVoiceNum = newVoiceNum; + for (; i < max_voices; i++) + f->voices[i].playingPreset = -1; + return 1; +} + +TSFDEF int tsf_note_on(tsf* f, int preset_index, int key, float vel) +{ + short midiVelocity = (short)(vel * 127); + int voicePlayIndex; + struct tsf_region *region, *regionEnd; + + if (preset_index < 0 || preset_index >= f->presetNum) return 1; + if (vel <= 0.0f) { tsf_note_off(f, preset_index, key); return 1; } + + // Play all matching regions. + voicePlayIndex = f->voicePlayIndex++; + for (region = f->presets[preset_index].regions, regionEnd = region + f->presets[preset_index].regionNum; region != regionEnd; region++) + { + struct tsf_voice *voice, *v, *vEnd; TSF_BOOL doLoop; float lowpassFilterQDB, lowpassFc; + if (key < region->lokey || key > region->hikey || midiVelocity < region->lovel || midiVelocity > region->hivel) continue; + + voice = TSF_NULL, v = f->voices, vEnd = v + f->voiceNum; + if (region->group) + { + for (; v != vEnd; v++) + if (v->playingPreset == preset_index && v->region->group == region->group) tsf_voice_endquick(f, v); + else if (v->playingPreset == -1 && !voice) voice = v; + } + else for (; v != vEnd; v++) if (v->playingPreset == -1) { voice = v; break; } + + if (!voice) + { + if (f->maxVoiceNum) + { + // Voices have been pre-allocated and limited to a maximum, try to kill a voice off in its release envelope + int bestKillReleaseSamplePos = -999999999; + for (v = f->voices; v != vEnd; v++) + { + if (v->ampenv.segment == TSF_SEGMENT_RELEASE) + { + // We're looking for the voice furthest into its release + int releaseSamplesDone = tsf_voice_envelope_release_samples(&v->ampenv, f->outSampleRate) - v->ampenv.samplesUntilNextSegment; + if (releaseSamplesDone > bestKillReleaseSamplePos) + { + bestKillReleaseSamplePos = releaseSamplesDone; + voice = v; + } + } + } + if (!voice) + continue; + tsf_voice_kill(voice); + } + else + { + // Allocate more voices so we don't need to kill one off. + struct tsf_voice* newVoices; + f->voiceNum += 4; + newVoices = (struct tsf_voice*)TSF_REALLOC(f->voices, f->voiceNum * sizeof(struct tsf_voice)); + if (!newVoices) return 0; + f->voices = newVoices; + voice = &f->voices[f->voiceNum - 4]; + voice[1].playingPreset = voice[2].playingPreset = voice[3].playingPreset = -1; + } + } + + voice->region = region; + voice->playingPreset = preset_index; + voice->playingKey = key; + voice->playIndex = voicePlayIndex; + voice->noteGainDB = f->globalGainDB - region->attenuation - tsf_gainToDecibels(1.0f / vel); + + if (f->channels) + { + f->channels->setupVoice(f, voice); + } + else + { + tsf_voice_calcpitchratio(voice, 0, f->outSampleRate); + // The SFZ spec is silent about the pan curve, but a 3dB pan law seems common. This sqrt() curve matches what Dimension LE does; Alchemy Free seems closer to sin(adjustedPan * pi/2). + voice->panFactorLeft = TSF_SQRTF(0.5f - region->pan); + voice->panFactorRight = TSF_SQRTF(0.5f + region->pan); + } + + // Offset/end. + voice->sourceSamplePosition = region->offset; + + // Loop. + doLoop = (region->loop_mode != TSF_LOOPMODE_NONE && region->loop_start < region->loop_end); + voice->loopStart = (doLoop ? region->loop_start : 0); + voice->loopEnd = (doLoop ? region->loop_end : 0); + + // Setup envelopes. + tsf_voice_envelope_setup(&voice->ampenv, ®ion->ampenv, key, midiVelocity, TSF_TRUE, f->outSampleRate); + tsf_voice_envelope_setup(&voice->modenv, ®ion->modenv, key, midiVelocity, TSF_FALSE, f->outSampleRate); + + // Setup lowpass filter. + lowpassFc = (region->initialFilterFc <= 13500 ? tsf_cents2Hertz((float)region->initialFilterFc) / f->outSampleRate : 1.0f); + lowpassFilterQDB = region->initialFilterQ / 10.0f; + voice->lowpass.QInv = 1.0 / TSF_POW(10.0, (lowpassFilterQDB / 20.0)); + voice->lowpass.z1 = voice->lowpass.z2 = 0; + voice->lowpass.active = (lowpassFc < 0.499f); + if (voice->lowpass.active) tsf_voice_lowpass_setup(&voice->lowpass, lowpassFc); + + // Setup LFO filters. + tsf_voice_lfo_setup(&voice->modlfo, region->delayModLFO, region->freqModLFO, f->outSampleRate); + tsf_voice_lfo_setup(&voice->viblfo, region->delayVibLFO, region->freqVibLFO, f->outSampleRate); + } + return 1; +} + +TSFDEF int tsf_bank_note_on(tsf* f, int bank, int preset_number, int key, float vel) +{ + int preset_index = tsf_get_presetindex(f, bank, preset_number); + if (preset_index == -1) return 0; + return tsf_note_on(f, preset_index, key, vel); +} + +TSFDEF void tsf_note_off(tsf* f, int preset_index, int key) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum, *vMatchFirst = TSF_NULL, *vMatchLast = TSF_NULL; + for (; v != vEnd; v++) + { + //Find the first and last entry in the voices list with matching preset, key and look up the smallest play index + if (v->playingPreset != preset_index || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE) continue; + else if (!vMatchFirst || v->playIndex < vMatchFirst->playIndex) vMatchFirst = vMatchLast = v; + else if (v->playIndex == vMatchFirst->playIndex) vMatchLast = v; + } + if (!vMatchFirst) return; + for (v = vMatchFirst; v <= vMatchLast; v++) + { + //Stop all voices with matching preset, key and the smallest play index which was enumerated above + if (v != vMatchFirst && v != vMatchLast && + (v->playIndex != vMatchFirst->playIndex || v->playingPreset != preset_index || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE)) continue; + tsf_voice_end(f, v); + } +} + +TSFDEF int tsf_bank_note_off(tsf* f, int bank, int preset_number, int key) +{ + int preset_index = tsf_get_presetindex(f, bank, preset_number); + if (preset_index == -1) return 0; + tsf_note_off(f, preset_index, key); + return 1; +} + +TSFDEF void tsf_note_off_all(tsf* f) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) if (v->playingPreset != -1 && v->ampenv.segment < TSF_SEGMENT_RELEASE) + tsf_voice_end(f, v); +} + +TSFDEF int tsf_active_voice_count(tsf* f) +{ + int count = 0; + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) if (v->playingPreset != -1) count++; + return count; +} + +TSFDEF void tsf_render_short(tsf* f, short* buffer, int samples, int flag_mixing) +{ + float outputSamples[TSF_RENDER_SHORTBUFFERBLOCK]; + int channels = (f->outputmode == TSF_MONO ? 1 : 2), maxChannelSamples = TSF_RENDER_SHORTBUFFERBLOCK / channels; + while (samples > 0) + { + int channelSamples = (samples > maxChannelSamples ? maxChannelSamples : samples); + short* bufferEnd = buffer + channelSamples * channels; + float *floatSamples = outputSamples; + tsf_render_float(f, floatSamples, channelSamples, TSF_FALSE); + samples -= channelSamples; + + if (flag_mixing) + while (buffer != bufferEnd) + { + float v = *floatSamples++; + int vi = *buffer + (v < -1.00004566f ? (int)-32768 : (v > 1.00001514f ? (int)32767 : (int)(v * 32767.5f))); + *buffer++ = (vi < -32768 ? (short)-32768 : (vi > 32767 ? (short)32767 : (short)vi)); + } + else + while (buffer != bufferEnd) + { + float v = *floatSamples++; + *buffer++ = (v < -1.00004566f ? (short)-32768 : (v > 1.00001514f ? (short)32767 : (short)(v * 32767.5f))); + } + } +} + +TSFDEF void tsf_render_float(tsf* f, float* buffer, int samples, int flag_mixing) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + if (!flag_mixing) TSF_MEMSET(buffer, 0, (f->outputmode == TSF_MONO ? 1 : 2) * sizeof(float) * samples); + for (; v != vEnd; v++) + if (v->playingPreset != -1) + tsf_voice_render(f, v, buffer, samples); +} + +static void tsf_channel_setup_voice(tsf* f, struct tsf_voice* v) +{ + struct tsf_channel* c = &f->channels->channels[f->channels->activeChannel]; + float newpan = v->region->pan + c->panOffset; + v->playingChannel = f->channels->activeChannel; + v->noteGainDB += c->gainDB; + tsf_voice_calcpitchratio(v, (c->pitchWheel == 8192 ? c->tuning : ((c->pitchWheel / 16383.0f * c->pitchRange * 2.0f) - c->pitchRange + c->tuning)), f->outSampleRate); + if (newpan <= -0.5f) { v->panFactorLeft = 1.0f; v->panFactorRight = 0.0f; } + else if (newpan >= 0.5f) { v->panFactorLeft = 0.0f; v->panFactorRight = 1.0f; } + else { v->panFactorLeft = TSF_SQRTF(0.5f - newpan); v->panFactorRight = TSF_SQRTF(0.5f + newpan); } +} + +static struct tsf_channel* tsf_channel_init(tsf* f, int channel) +{ + int i; + if (f->channels && channel < f->channels->channelNum) return &f->channels->channels[channel]; + if (!f->channels) + { + f->channels = (struct tsf_channels*)TSF_MALLOC(sizeof(struct tsf_channels) + sizeof(struct tsf_channel) * channel); + if (!f->channels) return TSF_NULL; + f->channels->setupVoice = &tsf_channel_setup_voice; + f->channels->channelNum = 0; + f->channels->activeChannel = 0; + } + else + { + struct tsf_channels *newChannels = (struct tsf_channels*)TSF_REALLOC(f->channels, sizeof(struct tsf_channels) + sizeof(struct tsf_channel) * channel); + if (!newChannels) return TSF_NULL; + f->channels = newChannels; + } + i = f->channels->channelNum; + f->channels->channelNum = channel + 1; + for (; i <= channel; i++) + { + struct tsf_channel* c = &f->channels->channels[i]; + c->presetIndex = c->bank = 0; + c->pitchWheel = c->midiPan = 8192; + c->midiVolume = c->midiExpression = 16383; + c->midiRPN = 0xFFFF; + c->midiData = 0; + c->panOffset = 0.0f; + c->gainDB = 0.0f; + c->pitchRange = 2.0f; + c->tuning = 0.0f; + } + return &f->channels->channels[channel]; +} + +static void tsf_channel_applypitch(tsf* f, int channel, struct tsf_channel* c) +{ + struct tsf_voice *v, *vEnd; + float pitchShift = (c->pitchWheel == 8192 ? c->tuning : ((c->pitchWheel / 16383.0f * c->pitchRange * 2.0f) - c->pitchRange + c->tuning)); + for (v = f->voices, vEnd = v + f->voiceNum; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel) + tsf_voice_calcpitchratio(v, pitchShift, f->outSampleRate); +} + +TSFDEF int tsf_channel_set_presetindex(tsf* f, int channel, int preset_index) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + c->presetIndex = (unsigned short)preset_index; + return 1; +} + +TSFDEF int tsf_channel_set_presetnumber(tsf* f, int channel, int preset_number, int flag_mididrums) +{ + int preset_index; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (flag_mididrums) + { + preset_index = tsf_get_presetindex(f, 128 | (c->bank & 0x7FFF), preset_number); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, 128, preset_number); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, 128, 0); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, (c->bank & 0x7FFF), preset_number); + } + else preset_index = tsf_get_presetindex(f, (c->bank & 0x7FFF), preset_number); + if (preset_index == -1) preset_index = tsf_get_presetindex(f, 0, preset_number); + if (preset_index != -1) + { + c->presetIndex = (unsigned short)preset_index; + return 1; + } + return 0; +} + +TSFDEF int tsf_channel_set_bank(tsf* f, int channel, int bank) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + c->bank = (unsigned short)bank; + return 1; +} + +TSFDEF int tsf_channel_set_bank_preset(tsf* f, int channel, int bank, int preset_number) +{ + int preset_index; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + preset_index = tsf_get_presetindex(f, bank, preset_number); + if (preset_index == -1) return 0; + c->presetIndex = (unsigned short)preset_index; + c->bank = (unsigned short)bank; + return 1; +} + +TSFDEF int tsf_channel_set_pan(tsf* f, int channel, float pan) +{ + struct tsf_voice *v, *vEnd; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + for (v = f->voices, vEnd = v + f->voiceNum; v != vEnd; v++) + if (v->playingChannel == channel && v->playingPreset != -1) + { + float newpan = v->region->pan + pan - 0.5f; + if (newpan <= -0.5f) { v->panFactorLeft = 1.0f; v->panFactorRight = 0.0f; } + else if (newpan >= 0.5f) { v->panFactorLeft = 0.0f; v->panFactorRight = 1.0f; } + else { v->panFactorLeft = TSF_SQRTF(0.5f - newpan); v->panFactorRight = TSF_SQRTF(0.5f + newpan); } + } + c->panOffset = pan - 0.5f; + return 1; +} + +TSFDEF int tsf_channel_set_volume(tsf* f, int channel, float volume) +{ + float gainDB = tsf_gainToDecibels(volume), gainDBChange; + struct tsf_voice *v, *vEnd; + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (gainDB == c->gainDB) return 1; + for (v = f->voices, vEnd = v + f->voiceNum, gainDBChange = gainDB - c->gainDB; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel) + v->noteGainDB += gainDBChange; + c->gainDB = gainDB; + return 1; +} + +TSFDEF int tsf_channel_set_pitchwheel(tsf* f, int channel, int pitch_wheel) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (c->pitchWheel == pitch_wheel) return 1; + c->pitchWheel = (unsigned short)pitch_wheel; + tsf_channel_applypitch(f, channel, c); + return 1; +} + +TSFDEF int tsf_channel_set_pitchrange(tsf* f, int channel, float pitch_range) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (c->pitchRange == pitch_range) return 1; + c->pitchRange = pitch_range; + if (c->pitchWheel != 8192) tsf_channel_applypitch(f, channel, c); + return 1; +} + +TSFDEF int tsf_channel_set_tuning(tsf* f, int channel, float tuning) +{ + struct tsf_channel *c = tsf_channel_init(f, channel); + if (!c) return 0; + if (c->tuning == tuning) return 1; + c->tuning = tuning; + tsf_channel_applypitch(f, channel, c); + return 1; +} + +TSFDEF int tsf_channel_note_on(tsf* f, int channel, int key, float vel) +{ + if (!f->channels || channel >= f->channels->channelNum) return 1; + f->channels->activeChannel = channel; + return tsf_note_on(f, f->channels->channels[channel].presetIndex, key, vel); +} + +TSFDEF void tsf_channel_note_off(tsf* f, int channel, int key) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum, *vMatchFirst = TSF_NULL, *vMatchLast = TSF_NULL; + for (; v != vEnd; v++) + { + //Find the first and last entry in the voices list with matching channel, key and look up the smallest play index + if (v->playingPreset == -1 || v->playingChannel != channel || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE) continue; + else if (!vMatchFirst || v->playIndex < vMatchFirst->playIndex) vMatchFirst = vMatchLast = v; + else if (v->playIndex == vMatchFirst->playIndex) vMatchLast = v; + } + if (!vMatchFirst) return; + for (v = vMatchFirst; v <= vMatchLast; v++) + { + //Stop all voices with matching channel, key and the smallest play index which was enumerated above + if (v != vMatchFirst && v != vMatchLast && + (v->playIndex != vMatchFirst->playIndex || v->playingPreset == -1 || v->playingChannel != channel || v->playingKey != key || v->ampenv.segment >= TSF_SEGMENT_RELEASE)) continue; + tsf_voice_end(f, v); + } +} + +TSFDEF void tsf_channel_note_off_all(tsf* f, int channel) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel && v->ampenv.segment < TSF_SEGMENT_RELEASE) + tsf_voice_end(f, v); +} + +TSFDEF void tsf_channel_sounds_off_all(tsf* f, int channel) +{ + struct tsf_voice *v = f->voices, *vEnd = v + f->voiceNum; + for (; v != vEnd; v++) + if (v->playingPreset != -1 && v->playingChannel == channel && (v->ampenv.segment < TSF_SEGMENT_RELEASE || v->ampenv.parameters.release)) + tsf_voice_endquick(f, v); +} + +TSFDEF int tsf_channel_midi_control(tsf* f, int channel, int controller, int control_value) +{ + struct tsf_channel* c = tsf_channel_init(f, channel); + if (!c) return 0; + switch (controller) + { + case 7 /*VOLUME_MSB*/ : c->midiVolume = (unsigned short)((c->midiVolume & 0x7F ) | (control_value << 7)); goto TCMC_SET_VOLUME; + case 39 /*VOLUME_LSB*/ : c->midiVolume = (unsigned short)((c->midiVolume & 0x3F80) | control_value); goto TCMC_SET_VOLUME; + case 11 /*EXPRESSION_MSB*/ : c->midiExpression = (unsigned short)((c->midiExpression & 0x7F ) | (control_value << 7)); goto TCMC_SET_VOLUME; + case 43 /*EXPRESSION_LSB*/ : c->midiExpression = (unsigned short)((c->midiExpression & 0x3F80) | control_value); goto TCMC_SET_VOLUME; + case 10 /*PAN_MSB*/ : c->midiPan = (unsigned short)((c->midiPan & 0x7F ) | (control_value << 7)); goto TCMC_SET_PAN; + case 42 /*PAN_LSB*/ : c->midiPan = (unsigned short)((c->midiPan & 0x3F80) | control_value); goto TCMC_SET_PAN; + case 6 /*DATA_ENTRY_MSB*/ : c->midiData = (unsigned short)((c->midiData & 0x7F) | (control_value << 7)); goto TCMC_SET_DATA; + case 38 /*DATA_ENTRY_LSB*/ : c->midiData = (unsigned short)((c->midiData & 0x3F80) | control_value); goto TCMC_SET_DATA; + case 0 /*BANK_SELECT_MSB*/ : c->bank = (unsigned short)(0x8000 | control_value); return 1; //bank select MSB alone acts like LSB + case 32 /*BANK_SELECT_LSB*/ : c->bank = (unsigned short)((c->bank & 0x8000 ? ((c->bank & 0x7F) << 7) : 0) | control_value); return 1; + case 101 /*RPN_MSB*/ : c->midiRPN = (unsigned short)(((c->midiRPN == 0xFFFF ? 0 : c->midiRPN) & 0x7F ) | (control_value << 7)); return 1; + case 100 /*RPN_LSB*/ : c->midiRPN = (unsigned short)(((c->midiRPN == 0xFFFF ? 0 : c->midiRPN) & 0x3F80) | control_value); return 1; + case 98 /*NRPN_LSB*/ : c->midiRPN = 0xFFFF; return 1; + case 99 /*NRPN_MSB*/ : c->midiRPN = 0xFFFF; return 1; + case 120 /*ALL_SOUND_OFF*/ : tsf_channel_sounds_off_all(f, channel); return 1; + case 123 /*ALL_NOTES_OFF*/ : tsf_channel_note_off_all(f, channel); return 1; + case 121 /*ALL_CTRL_OFF*/ : + c->midiVolume = c->midiExpression = 16383; + c->midiPan = 8192; + c->bank = 0; + c->midiRPN = 0xFFFF; + c->midiData = 0; + tsf_channel_set_volume(f, channel, 1.0f); + tsf_channel_set_pan(f, channel, 0.5f); + tsf_channel_set_pitchrange(f, channel, 2.0f); + tsf_channel_set_tuning(f, channel, 0); + return 1; + } + return 1; +TCMC_SET_VOLUME: + //Raising to the power of 3 seems to result in a decent sounding volume curve for MIDI + tsf_channel_set_volume(f, channel, TSF_POWF((c->midiVolume / 16383.0f) * (c->midiExpression / 16383.0f), 3.0f)); + return 1; +TCMC_SET_PAN: + tsf_channel_set_pan(f, channel, c->midiPan / 16383.0f); + return 1; +TCMC_SET_DATA: + if (c->midiRPN == 0) tsf_channel_set_pitchrange(f, channel, (c->midiData >> 7) + 0.01f * (c->midiData & 0x7F)); + else if (c->midiRPN == 1) tsf_channel_set_tuning(f, channel, (int)c->tuning + ((float)c->midiData - 8192.0f) / 8192.0f); //fine tune + else if (c->midiRPN == 2 && controller == 6) tsf_channel_set_tuning(f, channel, ((float)control_value - 64.0f) + (c->tuning - (int)c->tuning)); //coarse tune + return 1; +} + +TSFDEF int tsf_channel_get_preset_index(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].presetIndex : 0); +} + +TSFDEF int tsf_channel_get_preset_bank(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? (f->channels->channels[channel].bank & 0x7FFF) : 0); +} + +TSFDEF int tsf_channel_get_preset_number(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->presets[f->channels->channels[channel].presetIndex].preset : 0); +} + +TSFDEF float tsf_channel_get_pan(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].panOffset - 0.5f : 0.5f); +} + +TSFDEF float tsf_channel_get_volume(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? tsf_decibelsToGain(f->channels->channels[channel].gainDB) : 1.0f); +} + +TSFDEF int tsf_channel_get_pitchwheel(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].pitchWheel : 8192); +} + +TSFDEF float tsf_channel_get_pitchrange(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].pitchRange : 2.0f); +} + +TSFDEF float tsf_channel_get_tuning(tsf* f, int channel) +{ + return (f->channels && channel < f->channels->channelNum ? f->channels->channels[channel].tuning : 0.0f); +} + +#ifdef __cplusplus +} +#endif + +#endif //TSF_IMPLEMENTATION diff --git a/code/user/apps/edit/edit.c b/code/user/apps/edit/edit.c index 9a4a204..7ba0191 100644 --- a/code/user/apps/edit/edit.c +++ b/code/user/apps/edit/edit.c @@ -426,7 +426,7 @@ static void load_file(void) { char buf[512]; size_t got = 0; bool truncated = false; - for (;;) { + while (true) { SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(h, buf, sizeof(buf), &got); if (SysReadFile(&readReq) != 0 || got == 0) break; for (size_t i = 0; i < got; i++) { diff --git a/code/user/apps/httpget/httpget.c b/code/user/apps/httpget/httpget.c index 663ca52..3eeed2f 100644 --- a/code/user/apps/httpget/httpget.c +++ b/code/user/apps/httpget/httpget.c @@ -279,7 +279,7 @@ int main(void) { char buf[513]; size_t total = 0; - for (;;) { + while (true) { size_t got = 0; st = SysRecvSocket(sock, buf, sizeof(buf) - 1, 0, &got); if (st != NTLSTATUS_SUCCESS) { diff --git a/code/user/apps/ping/ping.c b/code/user/apps/ping/ping.c index 5ac7f6b..9c71a46 100644 --- a/code/user/apps/ping/ping.c +++ b/code/user/apps/ping/ping.c @@ -332,7 +332,7 @@ int main(void) { bool matched = false; uint8_t lastType = 0, lastCode = 0; - for (;;) { + while (true) { uint8_t recvBuf[512]; SYS_SOCKET_ADDRESS from; size_t fromSize = sizeof(from); diff --git a/code/user/apps/shell_native/shell_native.c b/code/user/apps/shell_native/shell_native.c index 67450f0..caefe96 100644 --- a/code/user/apps/shell_native/shell_native.c +++ b/code/user/apps/shell_native/shell_native.c @@ -12,6 +12,12 @@ #define MAX_PROCS 64 // Maximum number of processes we will fetch at once #define MAX_THREADS 128 #define SHELL_PROMPT "> " +#define SHELL_GLYPH_WIDTH 8 +#define SHELL_GLYPH_HEIGHT 16 +#define SHELL_TEXT_FOREGROUND 0x00CCCCCCu +#define SHELL_TEXT_BACKGROUND 0x00012456u +#define SHELL_CURSOR_FOREGROUND 0x00000000u +#define SHELL_CURSOR_BACKGROUND 0x00E0E0E0u // Output helpers. static void print(const char* s) { SYS_WRITE_CONSOLE_REQUEST req = { SysSvFromCstr(s) }; SysWriteConsole(&req); } @@ -192,6 +198,20 @@ static void printStringWidth(const char* s, size_t width) { static char gHistory[SHELL_HISTORY_DEPTH][MAX_CMD]; static size_t gHistoryCount; +typedef struct SHELL_EDITOR { + char* Buffer; + int Capacity; + int Length; + int Cursor; +} SHELL_EDITOR; + +typedef struct SHELL_VISIBLE_CURSOR { + int Column; + int Row; + char Character; + bool Visible; +} SHELL_VISIBLE_CURSOR; + static const char* const gShellBuiltinCommands[] = { "cd", "clear", @@ -201,6 +221,7 @@ static const char* const gShellBuiltinCommands[] = { "dir", "display", "exit", + "fbbench", "gpu", "help", "kill", @@ -211,6 +232,7 @@ static const char* const gShellBuiltinCommands[] = { "threads", "time", "type", + "volume", "write", }; @@ -280,23 +302,112 @@ static void shellHistoryAdd(const char* cmd) { gHistoryCount++; } -// Replaces the visible line while keeping the cursor at the end. -static void shellEditorReplaceLine(char* buf, int len, int* pos, const char* text) { - int oldPos = *pos; - for (int i = 0; i < oldPos; i++) print("\b"); - for (int i = 0; i < oldPos; i++) print(" "); - for (int i = 0; i < oldPos; i++) print("\b"); +// Restores the character underneath the visible edit cursor. +static void shellEditorHideVisibleCursor(SHELL_VISIBLE_CURSOR* cursor) { + if (!cursor || !cursor->Visible) return; - int newPos = 0; - if (text) { - while (text[newPos] && newPos < len - 1) { - buf[newPos] = text[newPos]; - newPos++; + (void)SysPutScreenChar(cursor->Character, + cursor->Column * SHELL_GLYPH_WIDTH, + cursor->Row * SHELL_GLYPH_HEIGHT, + SHELL_TEXT_FOREGROUND, + SHELL_TEXT_BACKGROUND); + cursor->Visible = false; +} + +// Draws the same steady block cursor used by edit without changing console state. +static void shellEditorShowVisibleCursor(SHELL_VISIBLE_CURSOR* cursor, const SHELL_EDITOR* editor) { + if (!cursor || !editor) return; + shellEditorHideVisibleCursor(cursor); + + SYS_CONSOLE_INFO info; + if (SysGetConsoleInfo(&info) != NTLSTATUS_SUCCESS) return; + + char character = editor->Cursor < editor->Length + ? editor->Buffer[editor->Cursor] + : ' '; + if (SysPutScreenChar(character, + (int)info.CursorX * SHELL_GLYPH_WIDTH, + (int)info.CursorY * SHELL_GLYPH_HEIGHT, + SHELL_CURSOR_FOREGROUND, + SHELL_CURSOR_BACKGROUND) != NTLSTATUS_SUCCESS) + return; + + cursor->Column = (int)info.CursorX; + cursor->Row = (int)info.CursorY; + cursor->Character = character; + cursor->Visible = true; +} + +// Moves the visible cursor left by the requested number of cells. +static void shellEditorMoveLeftCells(int count) { + while (count-- > 0) print("\b"); +} + +// Moves the editor cursor without changing the buffer. +static void shellEditorMoveCursor(SHELL_EDITOR* editor, int target) { + if (!editor) return; + if (target < 0) target = 0; + if (target > editor->Length) target = editor->Length; + + if (target < editor->Cursor) { + shellEditorMoveLeftCells(editor->Cursor - target); + } else { + while (editor->Cursor < target) { + char echo[2] = { editor->Buffer[editor->Cursor], '\0' }; + print(echo); + editor->Cursor++; } } - buf[newPos] = '\0'; - *pos = newPos; - print(buf); + editor->Cursor = target; +} + +// Redraws the changed tail and restores the requested cursor position. +static void shellEditorRedrawTail(SHELL_EDITOR* editor, int start, int oldLength, int targetCursor) { + if (!editor) return; + + print(editor->Buffer + start); + if (oldLength > editor->Length) printSpaces((size_t)(oldLength - editor->Length)); + + int displayedEnd = oldLength > editor->Length ? oldLength : editor->Length; + shellEditorMoveLeftCells(displayedEnd - targetCursor); + editor->Cursor = targetCursor; +} + +// Replaces one buffer range and keeps the visible line synchronized. +static bool shellEditorReplaceRange(SHELL_EDITOR* editor, int start, int end, const char* text) { + if (!editor || !editor->Buffer || editor->Capacity <= 0 || !text) return false; + if (start < 0 || end < start || end > editor->Length) return false; + + shellEditorMoveCursor(editor, start); + + int oldLength = editor->Length; + int removed = end - start; + int available = editor->Capacity - 1 - (oldLength - removed); + int replacementLength = 0; + while (text[replacementLength] && replacementLength < available) replacementLength++; + + memmove(editor->Buffer + start + replacementLength, + editor->Buffer + end, + (size_t)(oldLength - end + 1)); + if (replacementLength > 0) + memcpy(editor->Buffer + start, text, (size_t)replacementLength); + + editor->Length = oldLength - removed + replacementLength; + int targetCursor = start + replacementLength; + shellEditorRedrawTail(editor, start, oldLength, targetCursor); + return removed != replacementLength || replacementLength != 0; +} + +// Inserts text at the current cursor position. +static bool shellEditorInsert(SHELL_EDITOR* editor, const char* text) { + if (!editor) return false; + return shellEditorReplaceRange(editor, editor->Cursor, editor->Cursor, text); +} + +// Replaces the complete visible line and leaves the cursor at its end. +static void shellEditorReplaceLine(SHELL_EDITOR* editor, const char* text) { + if (!editor) return; + (void)shellEditorReplaceRange(editor, 0, editor->Length, text ? text : ""); } // Returns true when a character separates shell tokens. @@ -340,23 +451,6 @@ static void shellUpdateCommonPrefix(char* common, size_t commonSize, const char* common[i] = '\0'; } -// Appends text to the edited line and echoes it. -static bool shellEditorAppend(char* buf, int len, int* pos, const char* text) { - if (!buf || len <= 0 || !pos || !text) return false; - - bool changed = false; - while (*text && *pos < len - 1) { - buf[*pos] = *text; - (*pos)++; - buf[*pos] = '\0'; - char echo[2] = { *text, '\0' }; - print(echo); - text++; - changed = true; - } - return changed; -} - // Prints one completion candidate. static void shellPrintCompletionCandidate(const char* name, bool directory) { print(" "); @@ -454,18 +548,23 @@ static void shellCollectPathCompletions(const char* token, size_t tokenLength, } // Redraws the prompt and current edit buffer after a completion list. -static void shellRedrawPromptAndLine(const char* buf) { +static void shellRedrawPromptAndLine(SHELL_EDITOR* editor) { print(SHELL_PROMPT); - print(buf ? buf : ""); + if (!editor) return; + print(editor->Buffer); + shellEditorMoveLeftCells(editor->Length - editor->Cursor); } -// Completes the token at the end of the current line. -static void shellCompleteLine(char* buf, int len, int* pos) { - if (!buf || len <= 0 || !pos || *pos < 0) return; +// Completes the token around the current cursor position. +static void shellCompleteLine(SHELL_EDITOR* editor) { + if (!editor || !editor->Buffer || editor->Capacity <= 0 || editor->Cursor < 0) return; - size_t cursor = (size_t)*pos; + char* buf = editor->Buffer; + size_t cursor = (size_t)editor->Cursor; size_t tokenStart = cursor; while (tokenStart > 0 && !shellIsTokenSeparator(buf[tokenStart - 1])) tokenStart--; + size_t tokenEnd = cursor; + while (tokenEnd < (size_t)editor->Length && !shellIsTokenSeparator(buf[tokenEnd])) tokenEnd++; const char* token = buf + tokenStart; size_t tokenLength = cursor - tokenStart; @@ -490,14 +589,20 @@ static void shellCompleteLine(char* buf, int len, int* pos) { size_t commonLength = strlen(state.Common); bool expanded = commonLength > namePrefixLength; if (expanded) { - shellEditorAppend(buf, len, pos, state.Common + namePrefixLength); + (void)shellEditorReplaceRange(editor, editor->Cursor, (int)tokenEnd, + state.Common + namePrefixLength); } if (state.MatchCount == 1) { if (state.LastIsDirectory) { - if (*pos < len - 1 && *pos > 0 && !shellIsPathSeparator(buf[*pos - 1])) shellEditorAppend(buf, len, pos, "/"); + if (editor->Cursor < editor->Capacity - 1 && editor->Cursor > 0 && + !shellIsPathSeparator(buf[editor->Cursor - 1])) + (void)shellEditorInsert(editor, "/"); } else if (state.LastAddsSpace) { - if (*pos < len - 1 && (*pos == 0 || buf[*pos - 1] != ' ')) shellEditorAppend(buf, len, pos, " "); + if (editor->Cursor < editor->Capacity - 1 && + (editor->Cursor == editor->Length || + !shellIsTokenSeparator(buf[editor->Cursor]))) + (void)shellEditorInsert(editor, " "); } return; } @@ -509,17 +614,27 @@ static void shellCompleteLine(char* buf, int len, int* pos) { shellCollectBuiltinCompletions(token, tokenLength, NULL, true); } shellCollectPathCompletions(token, tokenLength, NULL, true); - shellRedrawPromptAndLine(buf); + shellRedrawPromptAndLine(editor); } // Reads one editable command line from the keyboard. static bool readLine(char* buf, int len) { - int pos = 0; + SHELL_EDITOR editor; + SHELL_VISIBLE_CURSOR visibleCursor; + char historyDraft[MAX_CMD]; size_t historyCursor = gHistoryCount; if (len <= 0) return false; + + editor.Buffer = buf; + editor.Capacity = len; + editor.Length = 0; + editor.Cursor = 0; + memset(&visibleCursor, 0, sizeof(visibleCursor)); buf[0] = '\0'; + historyDraft[0] = '\0'; while (true) { + shellEditorShowVisibleCursor(&visibleCursor, &editor); INPUT_KEY_RECORD input; SYS_INPUT_EVENT_REQUEST inputReq; inputReq.Size = sizeof(inputReq); @@ -527,7 +642,9 @@ static bool readLine(char* buf, int len) { inputReq.Flags = SYS_INPUT_EVENT_FLAG_NONE; inputReq.TimeoutMilliseconds = SYS_WAIT_TIMEOUT_INFINITE; inputReq.RecordOut = &input; - if (SysReadInputEvent(&inputReq) != NTLSTATUS_SUCCESS) { SysYield(); continue; } + KSTATUS inputStatus = SysReadInputEvent(&inputReq); + shellEditorHideVisibleCursor(&visibleCursor); + if (inputStatus != NTLSTATUS_SUCCESS) { SysYield(); continue; } INPUT_KEY key = input.Key; if (key == INPUT_KEY_NONE) { SysYield(); continue; } @@ -545,18 +662,45 @@ static bool readLine(char* buf, int len) { } if (key == INPUT_KEY_BACKSPACE) { - if (pos > 0) { - pos--; - buf[pos] = '\0'; - print("\b \b"); - } + if (editor.Cursor > 0) + (void)shellEditorReplaceRange(&editor, editor.Cursor - 1, editor.Cursor, ""); + continue; + } + + if (key == INPUT_KEY_DELETE) { + if (editor.Cursor < editor.Length) + (void)shellEditorReplaceRange(&editor, editor.Cursor, editor.Cursor + 1, ""); + continue; + } + + if (key == INPUT_KEY_LEFT) { + shellEditorMoveCursor(&editor, editor.Cursor - 1); + continue; + } + + if (key == INPUT_KEY_RIGHT) { + shellEditorMoveCursor(&editor, editor.Cursor + 1); + continue; + } + + if (key == INPUT_KEY_HOME) { + shellEditorMoveCursor(&editor, 0); + continue; + } + + if (key == INPUT_KEY_END) { + shellEditorMoveCursor(&editor, editor.Length); continue; } if (key == INPUT_KEY_UP) { if (historyCursor > 0) { + if (historyCursor == gHistoryCount) { + strncpy(historyDraft, editor.Buffer, sizeof(historyDraft) - 1); + historyDraft[sizeof(historyDraft) - 1] = '\0'; + } historyCursor--; - shellEditorReplaceLine(buf, len, &pos, gHistory[historyCursor]); + shellEditorReplaceLine(&editor, gHistory[historyCursor]); } continue; } @@ -564,25 +708,21 @@ static bool readLine(char* buf, int len) { if (key == INPUT_KEY_DOWN) { if (historyCursor < gHistoryCount) { historyCursor++; - shellEditorReplaceLine(buf, len, &pos, - historyCursor < gHistoryCount ? gHistory[historyCursor] : ""); + shellEditorReplaceLine(&editor, + historyCursor < gHistoryCount ? gHistory[historyCursor] : historyDraft); } continue; } if (key == INPUT_KEY_TAB) { - shellCompleteLine(buf, len, &pos); + shellCompleteLine(&editor); continue; } char c = input.Char ? input.Char : shellKeyToChar(key, modifiers); if (c < 0x20 || c == 127) continue; - if (pos < len - 1) { - buf[pos++] = c; - buf[pos] = '\0'; - char echo[2] = { c, '\0' }; - print(echo); - } + char text[2] = { c, '\0' }; + (void)shellEditorInsert(&editor, text); } return false; @@ -604,6 +744,7 @@ static void cmd_help() { " mem - Show memory usage\n" " time - Show system wall-clock time\n" " display - Show display provider and counters\n" + " fbbench - Benchmark Doom-size and full-screen presents\n" " stats - Show scheduler/CPU/MM telemetry\n" " ps - List processes\n" " threads - Show one process's thread statuses\n" @@ -611,6 +752,9 @@ static void cmd_help() { " dir [path] - List directory contents\n" " cd - Change directory\n" " type - Display file contents\n" + " volume [0-100] - Query or set master audio volume\n" + " volume mute - Mute the default audio endpoint\n" + " volume unmute - Unmute the default audio endpoint\n" " del - Delete a file\n" " write - Create a file with text\n" " clear - Clear the screen\n" @@ -639,6 +783,117 @@ static void printCpuFlags(uint32_t flags) { if (flags & SYS_CPU_FLAG_PRESENT) print("present "); } +// Selects the available default audio endpoint, or the first available endpoint. +static bool shellQueryDefaultAudioEndpoint(SYS_AUDIO_ENDPOINT_INFO* endpointOut) { + if (!endpointOut) return false; + + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + request.CountOut = &count; + + KSTATUS status = SysAudioEnumerateEndpoints(&request); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_BUFFER_TOO_SMALL) { + print("volume: endpoint query failed status="); + printUInt(status); + print("\n"); + return false; + } + + const SYS_AUDIO_ENDPOINT_INFO* fallback = NULL; + uint32_t availableCount = count < SYS_AUDIO_MAX_ENDPOINTS + ? count + : SYS_AUDIO_MAX_ENDPOINTS; + for (uint32_t index = 0; index < availableCount; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->Size != sizeof(*endpoint) || + endpoint->Version != SYS_AUDIO_ABI_VERSION || + endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) fallback = endpoint; + if (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT) { + *endpointOut = *endpoint; + return true; + } + } + + if (fallback) { + *endpointOut = *fallback; + return true; + } + + print("volume: no available audio endpoint\n"); + return false; +} + +// Converts one public Q16 gain into a rounded percentage. +static uint32_t shellAudioVolumePercent(uint32_t volumeQ16) { + uint64_t scaled = (uint64_t)volumeQ16 * 100u + SYS_AUDIO_GAIN_Q16_UNITY / 2u; + return (uint32_t)(scaled / SYS_AUDIO_GAIN_Q16_UNITY); +} + +// Converts one percentage into public Q16 gain. +static uint32_t shellAudioVolumeQ16(uint32_t percent) { + return (uint32_t)(((uint64_t)percent * SYS_AUDIO_GAIN_Q16_UNITY + 50u) / 100u); +} + +// Queries or updates the default endpoint master volume. +static void cmd_volume(const char* args) { + SYS_AUDIO_ENDPOINT_INFO endpoint; + if (!shellQueryDefaultAudioEndpoint(&endpoint)) return; + + while (args && (*args == ' ' || *args == '\t')) args++; + if (!args || *args == '\0') { + print("Volume: "); + printUInt(shellAudioVolumePercent(endpoint.MasterVolumeQ16)); + print("%"); + if (endpoint.MasterMuted) print(" (muted)"); + print("\n"); + return; + } + + uint32_t volumeQ16 = endpoint.MasterVolumeQ16; + uint32_t muted = endpoint.MasterMuted ? 1u : 0u; + if (strcmp(args, "mute") == 0) { + muted = 1u; + } else if (strcmp(args, "unmute") == 0) { + muted = 0u; + } else { + uint32_t percent = 0; + if (!parseUInt(args, &percent) || percent > 100u) { + print("Usage: volume [0-100|mute|unmute]\n"); + return; + } + volumeQ16 = shellAudioVolumeQ16(percent); + muted = 0u; + } + + SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.EndpointId = endpoint.EndpointId; + request.Muted = muted; + request.VolumeQ16 = volumeQ16; + KSTATUS status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_SUCCESS) { + print("volume: update failed status="); + printUInt(status); + print("\n"); + return; + } + + print("Volume: "); + printUInt(shellAudioVolumePercent(volumeQ16)); + print("%"); + if (muted) print(" (muted)"); + print("\n"); +} + // Implements the cmd stats operation. static void cmd_stats(void) { SYS_SCHEDULER_STATS sched; @@ -801,6 +1056,12 @@ static void cmd_stats(void) { print(" voluntary blocks: "); printUInt64(sched.VoluntaryBlocks); print("\n"); print(" SMT deferrals: "); printUInt64(sched.SmtSiblingDeferrals); print("\n"); print(" SMT primary pokes: "); printUInt64(sched.SmtPrimaryPokes); print("\n"); + print(" RT dispatches: "); printUInt64(sched.RealtimeDispatches); print("\n"); + print(" RT preemptions: "); printUInt64(sched.RealtimePreemptions); print("\n"); + print(" PI boosts: "); printUInt64(sched.PriorityInheritanceBoosts); print("\n"); + print(" PI drops: "); printUInt64(sched.PriorityInheritanceDrops); print("\n"); + print(" PI propagations: "); printUInt64(sched.PriorityInheritancePropagations); print("\n"); + print(" PI max depth: "); printUInt64(sched.PriorityInheritanceMaxDepth); print("\n"); print("Exceptions:\n"); print(" delivered: "); printUInt64(exceptionStats.Delivered); print("\n"); @@ -975,20 +1236,6 @@ static void cmd_stats(void) { print(" standby="); printUInt64(fileCache.StandbyTransitions); print(" standby-hits="); printUInt64(fileCache.StandbyHits); print(" drop-fail="); printUInt64(fileCache.DropFailures); print("\n"); - print(" read-ahead: requests="); printUInt64(fileCache.ReadAheadRequests); - print(" requested="); printUInt64(fileCache.ReadAheadPagesRequested); - print(" queued="); printUInt64(fileCache.ReadAheadPagesQueued); - print(" loaded="); printUInt64(fileCache.ReadAheadPagesLoaded); - print(" skipped="); printUInt64(fileCache.ReadAheadPagesSkipped); - print(" fail="); printUInt64(fileCache.ReadAheadFailures); - print(" throttled="); printUInt64(fileCache.ReadAheadPagesThrottled); print("\n"); - print(" read-ahead max: pages/request="); printUInt64(fileCache.MaxReadAheadPagesPerRequest); print("\n"); - print(" batch fills: reads="); printUInt64(fileCache.BatchReads); - print(" requested="); printUInt64(fileCache.BatchPagesRequested); - print(" loaded="); printUInt64(fileCache.BatchPagesLoaded); - print(" max-request="); printUInt64(fileCache.MaxBatchPagesRequested); - print(" max-loaded="); printUInt64(fileCache.MaxBatchPagesLoaded); print("\n"); - print("Filesystem:\n"); print(" path cache: lookups="); printUInt64(filesystem.PathLookups); print(" hits="); printUInt64(filesystem.PathCacheHits); @@ -1012,18 +1259,6 @@ static void cmd_stats(void) { print(" writes: requests="); printUInt64(disk.WriteRequests); print(" sectors="); printUInt64(disk.WriteSectors); print(" max-sectors="); printUInt64(disk.MaxWriteSectors); print("\n"); - print(" read-ahead: hits="); printUInt64(disk.ReadaheadHits); - print(" misses="); printUInt64(disk.ReadaheadMisses); - print(" stores="); printUInt64(disk.ReadaheadStores); - print(" sectors="); printUInt64(disk.ReadaheadSectorsLoaded); - print(" max-sectors="); printUInt64(disk.MaxReadaheadSectors); print("\n"); - print(" read-ahead misc: served-bytes="); printUInt64(disk.ReadaheadBytesServed); - print(" seq="); printUInt64(disk.ReadaheadSequentialDetections); - print(" invalidations="); printUInt64(disk.ReadaheadInvalidations); - print(" suppressed="); printUInt64(disk.ReadaheadSuppressed); - print(" alloc-fail="); printUInt64(disk.ReadaheadAllocFailures); - print(" read-fail="); printUInt64(disk.ReadaheadReadFailures); print("\n"); - print("Surfaces:\n"); print(" lifetime: created="); printUInt64(surface.CreatedSurfaces); print(" destroyed="); printUInt64(surface.DestroyedSurfaces); @@ -1123,9 +1358,11 @@ static void printDisplayStats(const char* title, const DISPLAY_STATS* st) { print(" fill: "); printUInt64(st->FillRectCount); print("\n"); print(" copy: "); printUInt64(st->CopyRectCount); print("\n"); print(" read: "); printUInt64(st->ReadRectCount); print("\n"); - print(" present: "); printUInt64(st->PresentCount); print(" rects="); printUInt64(st->PresentRectCount); print("\n"); + print(" present: "); printUInt64(st->PresentCount); print(" rects="); printUInt64(st->PresentRectCount); + print(" direct="); printUInt64(st->DirectPresentCount); print("\n"); print(" dirty: "); printUInt64(st->DirtyRectCount); print(" all="); printUInt64(st->DirtyAllCount); print("\n"); - print(" bytes: "); printUInt64(st->BytesCopied); print("\n"); + print(" bytes: logical="); printUInt64(st->BytesCopied); + print(" scanout="); printUInt64(st->ScanoutBytes); print("\n"); print(" sync: "); printUInt64(st->SyncCount); print("\n"); print(" cursor: move="); printUInt64(st->CursorMoveCount); print(" show="); printUInt64(st->CursorShowCount); @@ -1149,10 +1386,205 @@ static void cmd_display(void) { print(" mode: "); printUInt(info.Mode.Width); print("x"); printUInt(info.Mode.Height); print(" stride="); printUInt(info.Mode.Stride); print(" format="); printUInt((uint32_t)info.Mode.Format); print("\n"); print(" caps: 0x"); printHex32(info.Caps); print(" ["); printDisplayCaps(info.Caps); print("]\n"); + print(" scanout: "); + print((info.Flags & DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE) ? "PAT WC available" : "PAT WC unavailable"); + print("\n"); printDisplayStats("Manager counters", &info.ManagerStats); printDisplayStats("Provider counters", &info.DeviceStats); } +typedef struct SHELL_FB_BENCH_RESULT { + uint32_t Width; + uint32_t Height; + uint32_t Frames; + uint32_t Status; + uint64_t ElapsedTicks; + uint64_t Frequency; + uint64_t ExpectedBytes; + uint64_t ScanoutBytes; + uint64_t DirectPresents; +} SHELL_FB_BENCH_RESULT; + +// Computes a decimal fixed-point ratio with one digit after the decimal point. +static uint64_t shellRatioTenths(uint64_t numerator, uint64_t denominator) { + if (!denominator) + return 0; + if (numerator > UINT64_MAX / 10u) + return (numerator / denominator) * 10u; + return (numerator * 10u) / denominator; +} + +// Fills a benchmark surface with a deterministic high-contrast pattern. +static void shellFillBenchmarkSurface(uint32_t* pixels, + uint32_t pitchPixels, + uint32_t width, + uint32_t height) { + for (uint32_t y = 0; y < height; y++) { + uint32_t* row = pixels + (size_t)y * pitchPixels; + for (uint32_t x = 0; x < width; x++) { + uint32_t red = (x * 255u) / (width ? width : 1u); + uint32_t green = (y * 255u) / (height ? height : 1u); + uint32_t blue = ((x ^ y) & 0xFFu); + row[x] = (red << 16) | (green << 8) | blue; + } + } +} + +// Runs one synchronous surface-present benchmark case. +static void shellRunFramebufferBenchmark(uint32_t width, + uint32_t height, + uint32_t frames, + uint64_t frequency, + SHELL_FB_BENCH_RESULT* result) { + memset(result, 0, sizeof(*result)); + result->Width = width; + result->Height = height; + result->Frames = frames; + result->Frequency = frequency; + + uint32_t surface = 0; + SYS_CREATE_SURFACE_REQUEST create; + memset(&create, 0, sizeof(create)); + create.Size = sizeof(create); + create.Width = width; + create.Height = height; + create.Format = SYS_SURFACE_FORMAT_XRGB8888; + create.Flags = SYS_SURFACE_FLAG_NONE; + create.HandleOut = &surface; + + uint32_t status = SysCreateSurface(&create); + if (status != NTLSTATUS_SUCCESS || !surface) { + result->Status = status ? status : NTLSTATUS_FAILURE; + return; + } + + void* base = 0; + uint32_t pitchBytes = 0; + status = SysMapSurface(surface, &base, &pitchBytes); + if (status != NTLSTATUS_SUCCESS || !base || pitchBytes < width * sizeof(uint32_t)) { + result->Status = status ? status : NTLSTATUS_FAILURE; + SysCloseHandle(surface); + return; + } + + shellFillBenchmarkSurface((uint32_t*)base, pitchBytes / sizeof(uint32_t), width, height); + + DISPLAY_RECT rect = { 0, 0, (int32_t)width, (int32_t)height }; + for (uint32_t i = 0; i < 2; i++) { + status = SysPresentSurface(surface, &rect, 1); + if (status != NTLSTATUS_SUCCESS) + break; + } + + SYS_DISPLAY_INFO before; + SYS_DISPLAY_INFO after; + memset(&before, 0, sizeof(before)); + memset(&after, 0, sizeof(after)); + if (status == NTLSTATUS_SUCCESS) + status = SysQueryDisplayInfo(&before); + + uint64_t begin = 0; + uint64_t end = 0; + uint32_t completed = 0; + if (status == NTLSTATUS_SUCCESS) { + begin = SysGetTick(); + for (; completed < frames; completed++) { + status = SysPresentSurface(surface, &rect, 1); + if (status != NTLSTATUS_SUCCESS) + break; + } + end = SysGetTick(); + uint32_t queryStatus = SysQueryDisplayInfo(&after); + if (status == NTLSTATUS_SUCCESS && queryStatus != NTLSTATUS_SUCCESS) + status = queryStatus; + } + + result->Frames = completed; + result->Status = status; + result->ElapsedTicks = end - begin; + result->ExpectedBytes = (uint64_t)width * (uint64_t)height * sizeof(uint32_t) * completed; + if (after.DeviceStats.ScanoutBytes >= before.DeviceStats.ScanoutBytes) + result->ScanoutBytes = after.DeviceStats.ScanoutBytes - before.DeviceStats.ScanoutBytes; + if (after.DeviceStats.DirectPresentCount >= before.DeviceStats.DirectPresentCount) + result->DirectPresents = after.DeviceStats.DirectPresentCount - before.DeviceStats.DirectPresentCount; + + SysCloseHandle(surface); +} + +// Prints one framebuffer benchmark result. +static void shellPrintFramebufferBenchmark(const char* name, + const SHELL_FB_BENCH_RESULT* result) { + print(" "); print(name); print(" "); + printUInt(result->Width); print("x"); printUInt(result->Height); print(": "); + if (result->Status != NTLSTATUS_SUCCESS || !result->ElapsedTicks || !result->Frequency) { + print("failed status=0x"); printHex32(result->Status); print("\n"); + return; + } + + uint64_t elapsedUs = SysTicksToMicroseconds(result->ElapsedTicks, result->Frequency); + uint64_t fpsTenths = shellRatioTenths((uint64_t)result->Frames * result->Frequency, + result->ElapsedTicks); + uint64_t bytes = result->ScanoutBytes ? result->ScanoutBytes : result->ExpectedBytes; + uint64_t mibTenths = shellRatioTenths(bytes * result->Frequency, + result->ElapsedTicks * 1024u * 1024u); + + print("frames="); printUInt(result->Frames); + print(" time="); printUInt64(elapsedUs / 1000u); print(" ms"); + print(" fps="); printUInt64(fpsTenths / 10u); print("."); printUInt64(fpsTenths % 10u); + print(" scanout="); printUInt64(mibTenths / 10u); print("."); printUInt64(mibTenths % 10u); print(" MiB/s"); + print(" direct="); printUInt64(result->DirectPresents); + print(" bytes="); printUInt64(bytes); + print("\n"); +} + +// Benchmarks the synchronous surface-to-scanout path without console traffic. +static void cmd_fbbench(void) { + SYS_DISPLAY_INFO info; + uint32_t status = SysQueryDisplayInfo(&info); + if (status != NTLSTATUS_SUCCESS) { + print("fbbench: display query failed status=0x"); printHex32(status); print("\n"); + return; + } + + uint64_t frequency = 0; + status = SysGetTickFrequency(&frequency); + if (status != NTLSTATUS_SUCCESS || !frequency) { + print("fbbench: timer query failed status=0x"); printHex32(status); print("\n"); + return; + } + + uint32_t doomWidth = info.Mode.Width < 960u ? info.Mode.Width : 960u; + uint32_t doomHeight = info.Mode.Height < 600u ? info.Mode.Height : 600u; + uint32_t fullWidth = info.Mode.Width; + uint32_t fullHeight = info.Mode.Height; + + uint64_t targetBytes = 128ull * 1024ull * 1024ull; + uint64_t doomFrameBytes = (uint64_t)doomWidth * doomHeight * sizeof(uint32_t); + uint64_t fullFrameBytes = (uint64_t)fullWidth * fullHeight * sizeof(uint32_t); + uint32_t doomFrames = doomFrameBytes ? (uint32_t)((targetBytes + doomFrameBytes - 1u) / doomFrameBytes) : 1u; + uint32_t fullFrames = fullFrameBytes ? (uint32_t)((targetBytes + fullFrameBytes - 1u) / fullFrameBytes) : 1u; + if (doomFrames < 8u) doomFrames = 8u; + if (doomFrames > 128u) doomFrames = 128u; + if (fullFrames < 8u) fullFrames = 8u; + if (fullFrames > 64u) fullFrames = 64u; + + print("fbbench: running; the framebuffer will be overwritten temporarily\n"); + SHELL_FB_BENCH_RESULT doom; + SHELL_FB_BENCH_RESULT full; + shellRunFramebufferBenchmark(doomWidth, doomHeight, doomFrames, frequency, &doom); + shellRunFramebufferBenchmark(fullWidth, fullHeight, fullFrames, frequency, &full); + + SysClearScreen(0x00012456u); + print("Framebuffer benchmark:\n"); + print(" provider: "); print(info.PrimaryName[0] ? info.PrimaryName : "?"); + print(" mode="); printUInt(info.Mode.Width); print("x"); printUInt(info.Mode.Height); + print(" stride="); printUInt(info.Mode.Stride); + print(" cache="); print((info.Flags & DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE) ? "PAT-WC" : "fallback"); + print("\n"); + shellPrintFramebufferBenchmark("doom", &doom); + shellPrintFramebufferBenchmark("full", &full); +} + // Implements the cmd mem operation. static void cmd_mem(void) { SYS_MEMORY_STATUS st; @@ -1200,6 +1632,7 @@ static const char* threadStateName(uint32_t state) { case SYS_THREAD_STATE_RUNNING: return "run"; case SYS_THREAD_STATE_WAITING: return "wait"; case SYS_THREAD_STATE_DEAD: return "dead"; + case SYS_THREAD_STATE_SUSPENDED: return "suspend"; default: return "?"; } } @@ -1331,7 +1764,7 @@ static void cmd_threads(const char* args) { print(" (Count: "); printUInt((uint32_t)count); print("):\n"); - print(" TID State Pri CPU LastCPU QRem QTick Burst Wait WStat Flags\n"); + print(" TID State Base Eff CPU LastCPU QRem QTick Burst Wait WStat Flags\n"); print(" Flags: *=current I=idle K=kernel Q=queued A=alertable k=kernel-apc u=user-apc.\n"); size_t displayCount = count < MAX_THREADS ? count : MAX_THREADS; @@ -1342,7 +1775,11 @@ static void cmd_threads(const char* args) { print(" "); printStringWidth(threadStateName(t->State), 5); print(" "); - printUIntWidth(t->Priority, 3); + if (t->BaseSchedulingClass == SYS_SCHEDULING_REALTIME) print("R"); else print("N"); + printUIntWidth(t->BasePriority, 2); + print(" "); + if (t->EffectiveSchedulingClass == SYS_SCHEDULING_REALTIME) print("R"); else print("N"); + printUIntWidth(t->EffectivePriority, 2); print(" "); if (t->RunningCpu == 0xffffffffu) printStringWidth("-", 3); else printUIntWidth(t->RunningCpu, 3); @@ -1431,7 +1868,7 @@ static int shellCompareNames(const char* a, const char* b) { if (!a) a = ""; if (!b) b = ""; - for (;;) { + while (true) { char ca = shellAsciiLower(a[i]); char cb = shellAsciiLower(b[i]); if (ca != cb) @@ -1540,7 +1977,7 @@ static void cmd_dir(const char* path) { uint32_t count = 0; uint32_t capacity = 0; - for (;;) { + while (true) { SYS_DIR_ENTRY ke; st = SysReadDir(hDir, &ke); if (st != 0) @@ -1575,7 +2012,7 @@ static void cmd_type(const char* filename) { uint32_t hFile; if (open_file(filename, &hFile) != 0) { print("type: cannot open file\n"); return; } char buffer[512]; size_t bytesRead; - for (;;) { + while (true) { SYS_FILE_IO_REQUEST readReq = SysMakeFileIoRequest(hFile, buffer, sizeof(buffer) - 1, &bytesRead); if (SysReadFile(&readReq) != 0 || bytesRead == 0) break; buffer[bytesRead] = '\0'; print(buffer); @@ -1604,78 +2041,39 @@ static bool extractProgramPath(const char* cmdline, char* out, size_t outSize) { return pos != 0; } -// Consumes queued Ctrl-C input without taking ordinary foreground stdin. -static bool shellConsumeQueuedCtrlC(void) { - for (int i = 0; i < 4; i++) { - INPUT_KEY_RECORD input; - SYS_INPUT_EVENT_REQUEST inputReq; - inputReq.Size = sizeof(inputReq); - inputReq.QueueHandle = 0; - inputReq.Flags = SYS_INPUT_EVENT_FLAG_NONE; - inputReq.TimeoutMilliseconds = 0; - inputReq.RecordOut = &input; - if (SysPeekInputEvent(&inputReq) != NTLSTATUS_SUCCESS) return false; - - if ((input.Modifiers & INPUT_MOD_CTRL) && input.Key == INPUT_KEY_C) { - (void)SysReadInputEvent(&inputReq); - return true; - } - - if (input.Key == INPUT_KEY_LEFT_CTRL || input.Key == INPUT_KEY_RIGHT_CTRL) { - (void)SysReadInputEvent(&inputReq); - continue; - } - - return false; - } - - return false; -} - // Returns whether one input key is currently held. static bool shellIsKeyDown(INPUT_KEY key) { bool down = false; return SysIsKeyDown(key, &down) == NTLSTATUS_SUCCESS && down; } -// Waits for a foreground process while watching Ctrl-C without stealing ordinary stdin. +// Waits for a foreground process while polling Ctrl-C without consuming child input. static uint32_t waitForegroundProcess(uint32_t hProc) { bool sentBreak = false; - uint32_t inputQueue = 0; - if (SysOpenInputQueue(0, &inputQueue) != 0) - return SysWait(hProc, SYS_WAIT_FLAG_NONE); - - uint32_t handles[2] = { hProc, inputQueue }; - uint32_t index = 0; - SYS_WAIT_MANY_REQUEST waitReq; - waitReq.Size = sizeof(waitReq); - waitReq.Flags = SYS_WAIT_FLAG_NONE; - waitReq.Handles = handles; - waitReq.Count = 2; - waitReq.SignaledIndexOut = &index; - waitReq.TimeoutMilliseconds = SYS_WAIT_TIMEOUT_INFINITE; - - for (;;) { - index = 0; - uint32_t status = SysWaitMany(&waitReq); - if (status != NTLSTATUS_SUCCESS) { - SysCloseHandle(inputQueue); + while (true) { + uint32_t status = SysWaitSingle(hProc, + SYS_WAIT_FLAG_NONE, + SHELL_FOREGROUND_POLL_MS); + if (status == NTLSTATUS_SUCCESS) return status; - } - - if (index == 0) { - SysCloseHandle(inputQueue); + if (status != NTLSTATUS_TIMEOUT) return status; - } uint32_t modifiers = SysGetKeyModifiers(); - if (((modifiers & INPUT_MOD_CTRL) && shellIsKeyDown(INPUT_KEY_C)) || shellConsumeQueuedCtrlC()) { - if (!sentBreak) { - print("^C\n"); - (void)SysKillProcess(hProc, (int32_t)NTLSTATUS_PROCESS_KILLED); - sentBreak = true; + if (!sentBreak && (modifiers & INPUT_MOD_CTRL) && + shellIsKeyDown(INPUT_KEY_C)) { + print("^C\n"); + uint32_t killStatus = SysKillProcess( + hProc, + (int32_t)NTLSTATUS_PROCESS_KILLED); + if (killStatus != NTLSTATUS_SUCCESS) { + print("shell: foreground termination failed status="); + printUInt(killStatus); + print("\n"); + return killStatus; } + sentBreak = true; } } } @@ -1727,6 +2125,9 @@ static int runCommand(const char* cmd) { if (strcmp(cmd, "mem") == 0) { cmd_mem(); return 0; } if (strcmp(cmd, "time") == 0 || strcmp(cmd, "date") == 0) { cmd_time(); return 0; } if (strcmp(cmd, "display") == 0 || strcmp(cmd, "gpu") == 0) { cmd_display(); return 0; } + if (strcmp(cmd, "fbbench") == 0) { cmd_fbbench(); return 0; } + if (strcmp(cmd, "volume") == 0) { cmd_volume(""); return 0; } + if (strncmp(cmd, "volume ", 7) == 0) { cmd_volume(cmd + 7); return 0; } if (strcmp(cmd, "stats") == 0 || strcmp(cmd, "telemetry") == 0) { cmd_stats(); return 0; } if (strcmp(cmd, "ps") == 0) { cmd_ps(); return 0; } if (strncmp(cmd, "threads ", 8) == 0) { cmd_threads(cmd + 8); return 0; } diff --git a/code/user/apps/shell_posix/shell_posix.c b/code/user/apps/shell_posix/shell_posix.c index 81190af..49f8b9b 100644 --- a/code/user/apps/shell_posix/shell_posix.c +++ b/code/user/apps/shell_posix/shell_posix.c @@ -142,7 +142,7 @@ static int shellCompareNames(const char* a, const char* b) { if (!a) a = ""; if (!b) b = ""; - for (;;) { + while (true) { char ca = shellAsciiLower(a[i]); char cb = shellAsciiLower(b[i]); if (ca != cb) diff --git a/code/user/apps/shell_win32/shell_win32.c b/code/user/apps/shell_win32/shell_win32.c index 3f44767..12dee7e 100644 --- a/code/user/apps/shell_win32/shell_win32.c +++ b/code/user/apps/shell_win32/shell_win32.c @@ -112,7 +112,7 @@ static int shellCompareNames(const char* a, const char* b) { if (!a) a = ""; if (!b) b = ""; - for (;;) { + while (true) { char ca = shellAsciiLower(a[i]); char cb = shellAsciiLower(b[i]); if (ca != cb) diff --git a/code/user/apps/video/video.c b/code/user/apps/video/video.c index e40589c..70b57ad 100644 --- a/code/user/apps/video/video.c +++ b/code/user/apps/video/video.c @@ -50,11 +50,11 @@ static void fail_status(const char* what, uint32_t status) { print(" failed status="); print_u64(status); print("\n"); - for (;;) SysExitProcess(1); + while (true) SysExitProcess(1); } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static SYS_STRING_VIEW command_line_view(void) { @@ -227,6 +227,16 @@ int main(void) { print("video: input file mapped\n"); + if (!BinkSoundUseWaveOut()) { + print("video: BinkSoundUseWaveOut failed: "); + const char* err = BinkGetError(); + print(err ? err : "unknown"); + print("\n"); + SysFreeVm(buf); + exit_now(1); + } + print("video: WaveOut sound system ready\n"); + HBINK b = BinkOpen((const char*)buf, BINKFROMMEMORY); if (!b) { print("video: BinkOpen failed: "); @@ -237,6 +247,25 @@ int main(void) { exit_now(1); } + print("video: Bink open tracks="); + print_u64((uint64_t)(b->NumTracks < 0 ? 0 : b->NumTracks)); + print(" playing="); + print_u64(b->playingtracks); + print(" soundon="); + print_u64(b->soundon); + print("\n"); + for (int32_t track = 0; track < b->NumTracks; track++) { + print("video: track "); + print_u64((uint64_t)track); + print(" id="); + print_u64(BinkGetTrackID(b, (uint32_t)track)); + print(" type="); + print_u64(BinkGetTrackType(b, (uint32_t)track)); + print(" maxBytes="); + print_u64(BinkGetTrackMaxSize(b, (uint32_t)track)); + print("\n"); + } + uint32_t width = b->Width; uint32_t height = b->Height; if (!width || !height) { @@ -273,12 +302,16 @@ int main(void) { } print("video: native surface ready\n"); + print("video: decode loop entered\n"); DISPLAY_RECT fullFrame = { 0, 0, (int32_t)width, (int32_t)height }; + int firstFramePresented = 0; while (1) { if (BinkWait(b)) { - _mm_pause(); + // Bink can be waiting on its decode, I/O, or WaveOut worker. Yield + // a millisecond instead of monopolizing the CPU with a hot poll. + SysSleep(1); continue; } @@ -299,6 +332,16 @@ int main(void) { SysFreeVm(buf); fail_status("SysPresentSurface", status); } + if (!firstFramePresented) { + firstFramePresented = 1; + print("video: first frame presented sndBytes="); + print_u64(b->SndSize); + print(" soundon="); + print_u64(b->soundon); + print(" playing="); + print_u64(b->playingtracks); + print("\n"); + } if (b->FrameNum == b->Frames) break; @@ -306,8 +349,17 @@ int main(void) { BinkNextFrame(b); } + print("video: playback complete frame="); + print_u64(b->FrameNum); + print(" sndBytes="); + print_u64(b->SndSize); + print("\n"); + print("video: closing surface\n"); SysCloseHandle(surface); + print("video: closing Bink\n"); BinkClose(b); + print("video: Bink closed\n"); + print("video: freeing input mapping\n"); SysFreeVm(buf); print("video: goodbye\n"); diff --git a/code/user/personalities/win32/kernel32.c b/code/user/personalities/win32/kernel32.c index 0f2211a..c87d8d4 100644 --- a/code/user/personalities/win32/kernel32.c +++ b/code/user/personalities/win32/kernel32.c @@ -39,6 +39,54 @@ typedef long LONG; typedef struct { LONG x; LONG y; } POINT; typedef struct { SHORT X; SHORT Y; } COORD; typedef struct { SHORT Left; SHORT Top; SHORT Right; SHORT Bottom; } SMALL_RECT; +typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION; +typedef struct { void* _ptr; } CONDITION_VARIABLE; + +// Forward declarations used by the process-local GetProcAddress surface. +__declspec(dllexport) HANDLE CreateEventA(void*, BOOL, BOOL, const char*); +__declspec(dllexport) HANDLE CreateEventW(void*, BOOL, BOOL, const unsigned short*); +__declspec(dllexport) BOOL SetEvent(HANDLE); +__declspec(dllexport) BOOL ResetEvent(HANDLE); +__declspec(dllexport) _Noreturn void ExitThread(DWORD); +__declspec(dllexport) HANDLE CreateThread(void*, size_t, void*, void*, DWORD, DWORD*); +__declspec(dllexport) DWORD WaitForSingleObject(HANDLE, DWORD); +__declspec(dllexport) HANDLE CreateMutexA(void*, BOOL, const char*); +__declspec(dllexport) HANDLE CreateMutexW(void*, BOOL, const unsigned short*); +__declspec(dllexport) BOOL ReleaseMutex(HANDLE); +__declspec(dllexport) BOOL CloseHandle(HANDLE); +__declspec(dllexport) void InitializeCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) BOOL InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION*, DWORD); +__declspec(dllexport) void DeleteCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) void LeaveCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) BOOL TryEnterCriticalSection(CRITICAL_SECTION*); +__declspec(dllexport) void InitializeConditionVariable(CONDITION_VARIABLE*); +__declspec(dllexport) BOOL SleepConditionVariableCS(CONDITION_VARIABLE*, CRITICAL_SECTION*, DWORD); +__declspec(dllexport) void WakeConditionVariable(CONDITION_VARIABLE*); +__declspec(dllexport) void WakeAllConditionVariable(CONDITION_VARIABLE*); +__declspec(dllexport) BOOL IsDebuggerPresent(void); +extern void waveOutGetNumDevs(void); +extern void waveOutOpen(void); +extern void waveOutClose(void); +extern void waveOutPrepareHeader(void); +extern void waveOutUnprepareHeader(void); +extern void waveOutWrite(void); +extern void waveOutPause(void); +extern void waveOutRestart(void); +extern void waveOutReset(void); +extern void waveOutSetVolume(void); +extern void waveOutGetVolume(void); +extern void waveOutGetPosition(void); +extern void waveOutGetID(void); +extern void waveOutGetDevCapsA(void); +extern void waveOutGetDevCapsW(void); +extern void waveOutGetErrorTextA(void); +extern void waveOutGetErrorTextW(void); +extern void waveOutBreakLoop(void); +extern void waveOutMessage(void); +extern void timeGetTime(void); +extern void timeBeginPeriod(void); +extern void timeEndPeriod(void); typedef struct _CONSOLE_SCREEN_BUFFER_INFO { COORD dwSize; @@ -318,6 +366,9 @@ static DWORD* K32LastErrorSlot(void) { #define gLastError (*K32LastErrorSlot()) +// Returns a stable nonzero identifier for the calling thread. +static DWORD K32CurrentTidOrOne(void); + // Implements the file Exists A operation. static BOOL KiFileExistsA(const char* path) { if (!path || !*path) return FALSE; @@ -976,11 +1027,189 @@ __declspec(dllexport) BOOL DeleteFileA(const char* lpFileName) { return TRUE; } -// CloseHandle - unified close for files, threads, mutexes, semaphores, events, queues, and processes. -__declspec(dllexport) BOOL CloseHandle(HANDLE hObject) { - if (!hObject || hObject == INVALID_HANDLE_VALUE) { gLastError = 6; return FALSE; } +#define K32_MUTEX_MAGIC 0x58544D4Bu - SysCloseHandle(HANDLE_TO_KH(hObject)); +typedef struct K32_MUTEX_HANDLE { + struct K32_MUTEX_HANDLE* Next; + uint32_t Magic; + volatile long StateLock; + volatile long ApiUsers; + uint32_t NativeHandle; + DWORD OwnerTid; + uint32_t Recursion; + uint32_t Waiters; + volatile long Closing; +} K32_MUTEX_HANDLE; + +static K32_MUTEX_HANDLE* gK32MutexRegistry; +static volatile long gK32MutexRegistryLock; + +// Acquires one short process-local mutex registry lock. +static void K32MutexRegistryLock(void) { + while (_InterlockedCompareExchange(&gK32MutexRegistryLock, 1, 0) != 0) + (void)SysYield(); +} + +// Releases the process-local mutex registry lock. +static void K32MutexRegistryUnlock(void) { + _InterlockedExchange(&gK32MutexRegistryLock, 0); +} + +// Acquires one short mutex-wrapper state lock. +static void K32MutexStateLock(K32_MUTEX_HANDLE* mutex) { + while (_InterlockedCompareExchange(&mutex->StateLock, 1, 0) != 0) + (void)SysYield(); +} + +// Releases one mutex-wrapper state lock. +static void K32MutexStateUnlock(K32_MUTEX_HANDLE* mutex) { + _InterlockedExchange(&mutex->StateLock, 0); +} + +// Registers one process-local Win32 mutex wrapper. +static void K32RegisterMutex(K32_MUTEX_HANDLE* mutex) { + K32MutexRegistryLock(); + mutex->Next = gK32MutexRegistry; + gK32MutexRegistry = mutex; + K32MutexRegistryUnlock(); +} + +// References one registered Win32 mutex without dereferencing unknown handles. +static K32_MUTEX_HANDLE* K32ReferenceMutex(HANDLE handle) { + K32_MUTEX_HANDLE* found = NULL; + K32MutexRegistryLock(); + for (K32_MUTEX_HANDLE* current = gK32MutexRegistry; + current; + current = current->Next) { + if ((HANDLE)current == handle && + current->Magic == K32_MUTEX_MAGIC && + _InterlockedCompareExchange(¤t->Closing, 0, 0) == 0) { + _InterlockedIncrement(¤t->ApiUsers); + found = current; + break; + } + } + K32MutexRegistryUnlock(); + return found; +} + +// Releases one mutex-wrapper API reference. +static void K32DereferenceMutex(K32_MUTEX_HANDLE* mutex) { + if (mutex) + _InterlockedDecrement(&mutex->ApiUsers); +} + +// Removes one mutex wrapper so no new API references can be acquired. +static BOOL K32RemoveMutex(K32_MUTEX_HANDLE* mutex) { + BOOL removed = FALSE; + K32MutexRegistryLock(); + K32_MUTEX_HANDLE** link = &gK32MutexRegistry; + while (*link) { + if (*link == mutex) { + *link = mutex->Next; + mutex->Next = NULL; + removed = TRUE; + break; + } + link = &(*link)->Next; + } + K32MutexRegistryUnlock(); + return removed; +} + +// Waits on one recursive Win32 mutex wrapper. +static DWORD K32WaitMutex(K32_MUTEX_HANDLE* mutex, DWORD milliseconds) { + DWORD tid = K32CurrentTidOrOne(); + K32MutexStateLock(mutex); + if (_InterlockedCompareExchange(&mutex->Closing, 0, 0) != 0) { + K32MutexStateUnlock(mutex); + gLastError = 6; + return 0xFFFFFFFFUL; + } + if (mutex->OwnerTid == tid) { + if (mutex->Recursion == UINT32_MAX) { + K32MutexStateUnlock(mutex); + gLastError = 8; + return 0xFFFFFFFFUL; + } + mutex->Recursion++; + K32MutexStateUnlock(mutex); + return 0; + } + mutex->Waiters++; + K32MutexStateUnlock(mutex); + + uint64_t timeout = milliseconds == 0xFFFFFFFFUL + ? SYS_WAIT_TIMEOUT_INFINITE + : (uint64_t)milliseconds; + uint32_t status = SysWaitSingle(mutex->NativeHandle, SYS_WAIT_FLAG_NONE, timeout); + + K32MutexStateLock(mutex); + if (mutex->Waiters) + mutex->Waiters--; + if (status == NTLSTATUS_SUCCESS || status == NTLSTATUS_ABANDONED) { + mutex->OwnerTid = tid; + mutex->Recursion = 1; + } + K32MutexStateUnlock(mutex); + + if (status == NTLSTATUS_SUCCESS) + return 0; + if (status == NTLSTATUS_ABANDONED) + return 0x00000080UL; + if (status == NTLSTATUS_TIMEOUT) + return 0x00000102UL; + gLastError = 6; + return 0xFFFFFFFFUL; +} + +// Closes one idle Win32 mutex wrapper. +static BOOL K32CloseMutex(K32_MUTEX_HANDLE* mutex) { + K32MutexStateLock(mutex); + BOOL busy = mutex->OwnerTid != 0 || mutex->Waiters != 0; + if (!busy && _InterlockedCompareExchange(&mutex->Closing, 1, 0) != 0) + busy = TRUE; + K32MutexStateUnlock(mutex); + if (busy) { + gLastError = 170; + K32DereferenceMutex(mutex); + return FALSE; + } + if (!K32RemoveMutex(mutex)) { + gLastError = 6; + K32DereferenceMutex(mutex); + return FALSE; + } + + while (mutex->ApiUsers > 1) + (void)SysYield(); + uint32_t status = SysCloseHandle(mutex->NativeHandle); + mutex->Magic = 0; + K32DereferenceMutex(mutex); + free(mutex); + if (status != NTLSTATUS_SUCCESS) { + gLastError = 6; + return FALSE; + } + return TRUE; +} + +// Closes one Win32 or native kernel handle. +__declspec(dllexport) BOOL CloseHandle(HANDLE hObject) { + if (!hObject || hObject == INVALID_HANDLE_VALUE) { + gLastError = 6; + return FALSE; + } + + K32_MUTEX_HANDLE* mutex = K32ReferenceMutex(hObject); + if (mutex) + return K32CloseMutex(mutex); + + uint32_t status = SysCloseHandle(HANDLE_TO_KH(hObject)); + if (status != NTLSTATUS_SUCCESS) { + gLastError = 6; + return FALSE; + } return TRUE; } @@ -1087,12 +1316,32 @@ static int KiStricmpA(const char* a, const char* b) { return ((unsigned char)*a | 0x20) - ((unsigned char)*b | 0x20); } -// Checks whether win32 Dll Name A. +// Matches one Win32 module basename with its optional DLL suffix. +static BOOL KiMatchesWin32DllNameA(const char* name, const char* baseName) { + if (!name || !baseName) + return FALSE; + + while (*name && *baseName && + (((unsigned char)*name | 0x20u) == ((unsigned char)*baseName | 0x20u))) { + name++; + baseName++; + } + + if (*baseName) + return FALSE; + if (!*name) + return TRUE; + return KiStricmpA(name, ".dll") == 0; +} + +// Checks whether one name identifies a process-local Win32 personality module. static BOOL KiIsWin32DllNameA(const char* name) { - return name && ( - !KiStricmpA(name, "kernel32.dll") || !KiStricmpA(name, "user32.dll") || - !KiStricmpA(name, "gdi32.dll") || !KiStricmpA(name, "ole32.dll") || - !KiStricmpA(name, "winmm.dll") || !KiStricmpA(name, "win32.dll")); + return KiMatchesWin32DllNameA(name, "kernel32") || + KiMatchesWin32DllNameA(name, "user32") || + KiMatchesWin32DllNameA(name, "gdi32") || + KiMatchesWin32DllNameA(name, "ole32") || + KiMatchesWin32DllNameA(name, "winmm") || + KiMatchesWin32DllNameA(name, "win32"); } // Implements the dbg Write A operation. @@ -1100,11 +1349,13 @@ static void KiDbgWriteA(const char* s) { if (s) K32WriteConsoleSpan(s, strlen(s)); } -// Returns module Handle A. +// Returns the process-local module handle for one ANSI module name. __declspec(dllexport) HANDLE GetModuleHandleA(const char* lpModuleName) { - if (!lpModuleName) return WIN32_HMODULE_EXE; - if (KiIsWin32DllNameA(lpModuleName)) return WIN32_HMODULE_WIN32; - gLastError = 126; // ERROR_MOD_NOT_FOUND + if (!lpModuleName) + return WIN32_HMODULE_EXE; + if (KiIsWin32DllNameA(lpModuleName)) + return WIN32_HMODULE_WIN32; + gLastError = 126; // ERROR_MOD_NOT_FOUND. return NULL; } @@ -1175,13 +1426,31 @@ __declspec(dllexport) void* GetProcAddress(HANDLE hModule, const char* lpProcNam return (void*)address; } -#define GP(name) if (strcmp(lpProcName, #name) == 0) return (void*)&name +#define GP(name) do { \ + if (strcmp(lpProcName, #name) == 0) return (void*)&name; \ +} while (0) GP(GetLastError); GP(SetLastError); GP(LoadLibraryA); GP(GetProcAddress); + GP(IsDebuggerPresent); GP(FreeLibrary); GP(VirtualAlloc); GP(VirtualFree); GP(VirtualProtect); GP(VirtualQuery); GP(GetTickCount); GP(GetTickCount64); GP(Sleep); + GP(CreateEventA); GP(CreateEventW); GP(SetEvent); GP(ResetEvent); + GP(CreateThread); GP(ExitThread); GP(WaitForSingleObject); + GP(CreateMutexA); GP(CreateMutexW); GP(ReleaseMutex); GP(CloseHandle); + GP(InitializeCriticalSection); GP(InitializeCriticalSectionAndSpinCount); + GP(DeleteCriticalSection); GP(EnterCriticalSection); GP(LeaveCriticalSection); + GP(TryEnterCriticalSection); GP(InitializeConditionVariable); + GP(SleepConditionVariableCS); GP(WakeConditionVariable); GP(WakeAllConditionVariable); GP(TlsAlloc); GP(TlsFree); GP(TlsGetValue); GP(TlsSetValue); GP(GetConsoleScreenBufferInfo); GP(FillConsoleOutputCharacterA); GP(FillConsoleOutputAttribute); GP(SetConsoleCursorPosition); + GP(waveOutGetNumDevs); GP(waveOutOpen); GP(waveOutClose); + GP(waveOutPrepareHeader); GP(waveOutUnprepareHeader); GP(waveOutWrite); + GP(waveOutPause); GP(waveOutRestart); GP(waveOutReset); + GP(waveOutSetVolume); GP(waveOutGetVolume); GP(waveOutGetPosition); GP(waveOutGetID); + GP(waveOutGetDevCapsA); GP(waveOutGetDevCapsW); + GP(waveOutGetErrorTextA); GP(waveOutGetErrorTextW); + GP(waveOutBreakLoop); GP(waveOutMessage); + GP(timeGetTime); GP(timeBeginPeriod); GP(timeEndPeriod); #undef GP KiDbgWriteA("GetProcAddress miss: "); KiDbgWriteA(lpProcName); @@ -1275,53 +1544,163 @@ __declspec(dllexport) char* GetCommandLineA(void) { #define WAIT_OBJECT_0 0x00000000UL #define WAIT_TIMEOUT 0x00000102UL #define INFINITE 0xFFFFFFFFUL +#define CREATE_SUSPENDED 0x00000004UL +typedef DWORD (__stdcall *K32_THREAD_START_ROUTINE)(void* parameter); + +typedef struct K32_THREAD_START_CONTEXT { + K32_THREAD_START_ROUTINE StartRoutine; + void* Parameter; +} K32_THREAD_START_CONTEXT; + +// Invokes one Win32 thread procedure and terminates the native thread on return. +static void K32ThreadStart(uintptr_t argument) { + K32_THREAD_START_CONTEXT* context = (K32_THREAD_START_CONTEXT*)argument; + K32_THREAD_START_ROUTINE startRoutine = context->StartRoutine; + void* parameter = context->Parameter; + free(context); + + DWORD exitCode = startRoutine(parameter); + SysExitThread((int32_t)exitCode); + while (true) { + (void)SysYield(); + } +} + +// Creates one Win32 thread with return-to-ExitThread semantics. __declspec(dllexport) HANDLE CreateThread(void* lpThreadAttributes, size_t dwStackSize, void* lpStartAddress, void* lpParameter, DWORD dwCreationFlags, DWORD* lpThreadId) { - (void)lpThreadAttributes; (void)dwStackSize; (void)dwCreationFlags; + (void)lpThreadAttributes; + (void)dwStackSize; + if (!lpStartAddress || (dwCreationFlags & ~CREATE_SUSPENDED)) { + gLastError = 87; + return NULL; + } + + K32_THREAD_START_CONTEXT* context = + (K32_THREAD_START_CONTEXT*)malloc(sizeof(*context)); + if (!context) { + gLastError = 8; + return NULL; + } + context->StartRoutine = (K32_THREAD_START_ROUTINE)lpStartAddress; + context->Parameter = lpParameter; + uint32_t tid = 0; uint32_t kh = 0; SYS_CREATE_THREAD_REQUEST req = { - (uintptr_t)lpStartAddress, - (uintptr_t)lpParameter, - 0, + (uintptr_t)K32ThreadStart, + (uintptr_t)context, 0, + (dwCreationFlags & CREATE_SUSPENDED) + ? SYS_CREATE_THREAD_FLAG_SUSPENDED + : SYS_CREATE_THREAD_FLAG_NONE, &kh, &tid }; - uint32_t s = SysCreateThread(&req); - if (s != 0) { gLastError = 11; return INVALID_HANDLE_VALUE; } - if (lpThreadId) *lpThreadId = (DWORD)tid; + uint32_t status = SysCreateThread(&req); + if (status != NTLSTATUS_SUCCESS) { + free(context); + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 11; + return NULL; + } + if (lpThreadId) + *lpThreadId = (DWORD)tid; return KH_TO_HANDLE(kh); } -// WaitForSingleObject - blocks on native waitable handles. A zero timeout is -// only a probe for mutexes/semaphores until native timeout waits exist. +// Terminates the calling Win32 thread with the supplied exit code. +__declspec(dllexport) _Noreturn void ExitThread(DWORD exitCode) { + SysExitThread((int32_t)exitCode); + while (true) { + (void)SysYield(); + } +} + +// Waits for one Win32 mutex wrapper or native waitable handle. __declspec(dllexport) DWORD WaitForSingleObject(HANDLE hHandle, DWORD dwMilliseconds) { - if (!IS_VALID_KERNEL_HANDLE(hHandle)) { gLastError = 6; return 0xFFFFFFFFUL; } - - uint32_t kh = HANDLE_TO_KH(hHandle); - if (dwMilliseconds == 0) { - uint32_t s = SysTryAcquireMutex(kh); - if (s == 0) return WAIT_OBJECT_0; - - s = SysTryWaitSem(kh); - if (s == 0) return WAIT_OBJECT_0; - - return WAIT_TIMEOUT; + if (!hHandle || hHandle == INVALID_HANDLE_VALUE) { + gLastError = 6; + return 0xFFFFFFFFUL; } - uint32_t s = SysWait(kh, SYS_WAIT_FLAG_NONE); - if (s == 0) return WAIT_OBJECT_0; + K32_MUTEX_HANDLE* mutex = K32ReferenceMutex(hHandle); + if (mutex) { + DWORD result = K32WaitMutex(mutex, dwMilliseconds); + K32DereferenceMutex(mutex); + return result; + } + if (!IS_VALID_KERNEL_HANDLE(hHandle)) { + gLastError = 6; + return 0xFFFFFFFFUL; + } + + uint64_t timeout = dwMilliseconds == INFINITE + ? SYS_WAIT_TIMEOUT_INFINITE + : (uint64_t)dwMilliseconds; + uint32_t s = SysWaitSingle(HANDLE_TO_KH(hHandle), SYS_WAIT_FLAG_NONE, timeout); + if (s == NTLSTATUS_SUCCESS) return WAIT_OBJECT_0; + if (s == NTLSTATUS_ABANDONED) return 0x00000080UL; + if (s == NTLSTATUS_TIMEOUT) return WAIT_TIMEOUT; gLastError = 6; return 0xFFFFFFFFUL; } +// Creates one unnamed native event with Win32 reset semantics. +__declspec(dllexport) HANDLE CreateEventA(void* lpEventAttributes, + BOOL bManualReset, + BOOL bInitialState, + const char* lpName) { + (void)lpEventAttributes; + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + + uint32_t handle = 0; + uint32_t status = SysCreateEvent(bManualReset ? 1u : 0u, + bInitialState ? 1u : 0u, + &handle); + if (status != NTLSTATUS_SUCCESS) { + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 87; + return NULL; + } + return KH_TO_HANDLE(handle); +} + +// Creates one unnamed native event from the wide Win32 entry point. +__declspec(dllexport) HANDLE CreateEventW(void* lpEventAttributes, + BOOL bManualReset, + BOOL bInitialState, + const unsigned short* lpName) { + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + return CreateEventA(lpEventAttributes, bManualReset, bInitialState, NULL); +} + +// Signals one native event. +__declspec(dllexport) BOOL SetEvent(HANDLE hEvent) { + if (!IS_VALID_KERNEL_HANDLE(hEvent)) { gLastError = 6; return FALSE; } + uint32_t status = SysSignalEvent(HANDLE_TO_KH(hEvent)); + if (status != NTLSTATUS_SUCCESS) { gLastError = 6; return FALSE; } + return TRUE; +} + +// Clears one native event. +__declspec(dllexport) BOOL ResetEvent(HANDLE hEvent) { + if (!IS_VALID_KERNEL_HANDLE(hEvent)) { gLastError = 6; return FALSE; } + uint32_t status = SysClearEvent(HANDLE_TO_KH(hEvent)); + if (status != NTLSTATUS_SUCCESS) { gLastError = 6; return FALSE; } + return TRUE; +} + // Implements the extract Application Path A operation. static BOOL ExtractApplicationPathA(const char* commandLine, char* out, size_t outSize) { if (!commandLine || !out || outSize == 0) return FALSE; @@ -1449,10 +1828,22 @@ __declspec(dllexport) BOOL GetExitCodeProcess(HANDLE hProcess, DWORD* lpExitCode return TRUE; } -// Implements the resume Thread operation. +// Decrements a thread's suspend count and makes it runnable at zero. __declspec(dllexport) DWORD ResumeThread(HANDLE hThread) { - (void)hThread; - return 0; // previous suspend count + if (!hThread || hThread == INVALID_HANDLE_VALUE) { + gLastError = 6; + return 0xFFFFFFFFu; + } + + uint32_t previousSuspendCount = 0; + uint32_t status = SysResumeThread(HANDLE_TO_KH(hThread), + &previousSuspendCount); + if (status != NTLSTATUS_SUCCESS) { + gLastError = status == NTLSTATUS_ACCESS_DENIED ? 5 : 6; + return 0xFFFFFFFFu; + } + + return previousSuspendCount; } #define THREAD_PRIORITY_NORMAL 0 @@ -1876,7 +2267,7 @@ __declspec(dllexport) HANDLE FindFirstFileA(const char* lpFileName, // Read the first entry that matches the pattern. KDIR_ENTRY ke; - for (;;) { + while (true) { uint32_t rs = SysReadDir(kh, &ke); if (rs != 0) { // Directory exhausted or error. @@ -1902,7 +2293,7 @@ __declspec(dllexport) BOOL FindNextFileA(HANDLE hFindFile, FIND_HANDLE* fh = (FIND_HANDLE*)hFindFile; KDIR_ENTRY ke; - for (;;) { + while (true) { uint32_t rs = SysReadDir(fh->kh, &ke); if (rs != 0) { gLastError = 18; // ERROR_NO_MORE_FILES @@ -1979,28 +2370,88 @@ __declspec(dllexport) DWORD GetCurrentDirectoryA(DWORD nBufferLength, char* lpBu return (DWORD)(needed - 1); // length without NUL, matching Win32 } +// Creates one unnamed recursive Win32 mutex over a native kernel mutex. __declspec(dllexport) HANDLE CreateMutexA(void* lpMutexAttributes, BOOL bInitialOwner, const char* lpName) { - (void)lpMutexAttributes; (void)lpName; - uint32_t kh = 0; - uint32_t s = SysCreateMutex(&kh); - if (s != 0 || !kh) { gLastError = 8; return NULL; } + (void)lpMutexAttributes; + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + + K32_MUTEX_HANDLE* mutex = (K32_MUTEX_HANDLE*)calloc(1, sizeof(*mutex)); + if (!mutex) { + gLastError = 8; + return NULL; + } + uint32_t status = SysCreateMutex(&mutex->NativeHandle); + if (status != NTLSTATUS_SUCCESS || !mutex->NativeHandle) { + free(mutex); + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 1; + return NULL; + } + mutex->Magic = K32_MUTEX_MAGIC; if (bInitialOwner) { - s = SysWait(kh, SYS_WAIT_FLAG_NONE); - if (s != 0) { - SysCloseHandle(kh); + status = SysWait(mutex->NativeHandle, SYS_WAIT_FLAG_NONE); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_ABANDONED) { + (void)SysCloseHandle(mutex->NativeHandle); + free(mutex); gLastError = 1; return NULL; } + mutex->OwnerTid = K32CurrentTidOrOne(); + mutex->Recursion = 1; } - return KH_TO_HANDLE(kh); + K32RegisterMutex(mutex); + return (HANDLE)mutex; } -// Releases mutex. +// Creates one unnamed recursive Win32 mutex from a wide-name request. +__declspec(dllexport) HANDLE CreateMutexW(void* lpMutexAttributes, BOOL bInitialOwner, + const unsigned short* lpName) { + if (lpName && lpName[0]) { + gLastError = 120; + return NULL; + } + return CreateMutexA(lpMutexAttributes, bInitialOwner, NULL); +} + +// Releases one recursion level from a Win32 mutex wrapper. __declspec(dllexport) BOOL ReleaseMutex(HANDLE hMutex) { - if (!IS_VALID_KERNEL_HANDLE(hMutex)) { gLastError = 6; return FALSE; } - uint32_t s = SysReleaseMutex(HANDLE_TO_KH(hMutex)); - if (s != 0) { gLastError = 6; return FALSE; } + K32_MUTEX_HANDLE* mutex = K32ReferenceMutex(hMutex); + if (!mutex) { + gLastError = 6; + return FALSE; + } + + DWORD tid = K32CurrentTidOrOne(); + K32MutexStateLock(mutex); + if (mutex->OwnerTid != tid || mutex->Recursion == 0) { + K32MutexStateUnlock(mutex); + K32DereferenceMutex(mutex); + gLastError = 288; + return FALSE; + } + mutex->Recursion--; + if (mutex->Recursion != 0) { + K32MutexStateUnlock(mutex); + K32DereferenceMutex(mutex); + return TRUE; + } + mutex->OwnerTid = 0; + K32MutexStateUnlock(mutex); + + uint32_t status = SysReleaseMutex(mutex->NativeHandle); + if (status != NTLSTATUS_SUCCESS) { + K32MutexStateLock(mutex); + mutex->OwnerTid = tid; + mutex->Recursion = 1; + K32MutexStateUnlock(mutex); + K32DereferenceMutex(mutex); + gLastError = 288; + return FALSE; + } + K32DereferenceMutex(mutex); return TRUE; } @@ -2030,8 +2481,19 @@ __declspec(dllexport) BOOL ReleaseSemaphore(HANDLE hSemaphore, long lReleaseCoun // using the storage as a small recursive spin lock. Bink uses these from its // worker/copy paths, so the old no-op implementation allowed internal state // races as soon as CreateThread started working. -typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION; -typedef struct { void* _ptr; } CONDITION_VARIABLE; +typedef struct K32_CONDITION_WAITER K32_CONDITION_WAITER; +typedef struct K32_CONDITION_STATE { + volatile long Lock; + K32_CONDITION_WAITER* Head; + K32_CONDITION_WAITER* Tail; +} K32_CONDITION_STATE; + +struct K32_CONDITION_WAITER { + K32_CONDITION_WAITER* Previous; + K32_CONDITION_WAITER* Next; + uint32_t Event; + BOOL Queued; +}; #define K32_CS_LOCK 0 #define K32_CS_OWNER 1 @@ -2067,8 +2529,12 @@ __declspec(dllexport) void EnterCriticalSection(CRITICAL_SECTION* lpCS) { lpCS->_opaque[K32_CS_RECURSION]++; return; } - while (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0) + uint32_t spins = 0; + while (_InterlockedCompareExchange(K32CsLockWord(lpCS), 1, 0) != 0) { _mm_pause(); + if ((++spins & 0xFFFu) == 0) + (void)SysYield(); + } lpCS->_opaque[K32_CS_OWNER] = tid; lpCS->_opaque[K32_CS_RECURSION] = 1; } @@ -2100,19 +2566,183 @@ __declspec(dllexport) BOOL TryEnterCriticalSection(CRITICAL_SECTION* lpCS) { return TRUE; } -// Initializes condition Variable. -__declspec(dllexport) void InitializeConditionVariable(CONDITION_VARIABLE* cv) { (void)cv; } +// Acquires one condition-variable queue lock. +static void K32ConditionLock(K32_CONDITION_STATE* state) { + uint32_t spins = 0; + while (_InterlockedCompareExchange(&state->Lock, 1, 0) != 0) { + _mm_pause(); + if ((++spins & 0xFFFu) == 0) + (void)SysYield(); + } +} +// Releases one condition-variable queue lock. +static void K32ConditionUnlock(K32_CONDITION_STATE* state) { + _ReadWriteBarrier(); + _InterlockedExchange(&state->Lock, 0); +} + +// Returns the lazily allocated process-local condition-variable state. +static K32_CONDITION_STATE* K32ConditionState(CONDITION_VARIABLE* cv, BOOL create) { + if (!cv) + return NULL; + + K32_CONDITION_STATE* state = (K32_CONDITION_STATE*)cv->_ptr; + if (state || !create) + return state; + + K32_CONDITION_STATE* allocated = (K32_CONDITION_STATE*)calloc(1, sizeof(*allocated)); + if (!allocated) + return NULL; + + void* previous = _InterlockedCompareExchangePointer((void* volatile*)&cv->_ptr, + allocated, + NULL); + if (previous) { + free(allocated); + state = (K32_CONDITION_STATE*)previous; + } else { + state = allocated; + } + return state; +} + +// Removes one waiter while the condition-variable queue lock is held. +static void K32ConditionRemoveWaiter(K32_CONDITION_STATE* state, + K32_CONDITION_WAITER* waiter) { + if (!waiter->Queued) + return; + + if (waiter->Previous) + waiter->Previous->Next = waiter->Next; + else + state->Head = waiter->Next; + + if (waiter->Next) + waiter->Next->Previous = waiter->Previous; + else + state->Tail = waiter->Previous; + + waiter->Previous = NULL; + waiter->Next = NULL; + waiter->Queued = FALSE; +} + +// Initializes one process-local condition variable. +__declspec(dllexport) void InitializeConditionVariable(CONDITION_VARIABLE* cv) { + if (cv) + cv->_ptr = NULL; +} + +// Atomically releases a critical section and waits for one condition wake. __declspec(dllexport) BOOL SleepConditionVariableCS(CONDITION_VARIABLE* cv, CRITICAL_SECTION* cs, DWORD dwMilliseconds) { - (void)cv; (void)cs; (void)dwMilliseconds; - return TRUE; + if (!cv || !cs) { + gLastError = 87; + return FALSE; + } + + DWORD tid = K32CurrentTidOrOne(); + uintptr_t recursion = cs->_opaque[K32_CS_RECURSION]; + if ((DWORD)cs->_opaque[K32_CS_OWNER] != tid || recursion == 0) { + gLastError = 87; + return FALSE; + } + + K32_CONDITION_STATE* state = K32ConditionState(cv, TRUE); + if (!state) { + gLastError = 8; + return FALSE; + } + + uint32_t event = 0; + KSTATUS status = SysCreateEvent(0, 0, &event); + if (status != NTLSTATUS_SUCCESS) { + gLastError = status == NTLSTATUS_NO_MEMORY ? 8 : 87; + return FALSE; + } + + // The waiter is heap-backed because process termination can stop a user + // thread at syscall return before it can remove a stack-backed queue node. + K32_CONDITION_WAITER* waiter = (K32_CONDITION_WAITER*)calloc(1, sizeof(*waiter)); + if (!waiter) { + (void)SysCloseHandle(event); + gLastError = 8; + return FALSE; + } + waiter->Event = event; + waiter->Queued = TRUE; + + K32ConditionLock(state); + waiter->Previous = state->Tail; + if (state->Tail) + state->Tail->Next = waiter; + else + state->Head = waiter; + state->Tail = waiter; + K32ConditionUnlock(state); + + // The waiter is visible before the lock is released, so a wake that races + // this transition is retained by the private auto-reset event. + cs->_opaque[K32_CS_RECURSION] = 0; + cs->_opaque[K32_CS_OWNER] = 0; + _ReadWriteBarrier(); + _InterlockedExchange(K32CsLockWord(cs), 0); + + uint64_t timeout = dwMilliseconds == INFINITE + ? SYS_WAIT_TIMEOUT_INFINITE + : (uint64_t)dwMilliseconds; + status = SysWaitSingle(event, SYS_WAIT_FLAG_NONE, timeout); + + K32ConditionLock(state); + BOOL wakeWon = waiter->Queued ? FALSE : TRUE; + K32ConditionRemoveWaiter(state, waiter); + K32ConditionUnlock(state); + + EnterCriticalSection(cs); + cs->_opaque[K32_CS_RECURSION] = recursion; + (void)SysCloseHandle(event); + free(waiter); + + if (status == NTLSTATUS_SUCCESS || wakeWon) + return TRUE; + if (status == NTLSTATUS_TIMEOUT) + gLastError = 1460; + else + gLastError = 6; + return FALSE; +} + +// Wakes the oldest waiter on one condition variable. +__declspec(dllexport) void WakeConditionVariable(CONDITION_VARIABLE* cv) { + K32_CONDITION_STATE* state = K32ConditionState(cv, FALSE); + if (!state) + return; + + K32ConditionLock(state); + K32_CONDITION_WAITER* waiter = state->Head; + if (waiter) { + K32ConditionRemoveWaiter(state, waiter); + (void)SysSignalEvent(waiter->Event); + } + K32ConditionUnlock(state); +} + +// Wakes every waiter currently queued on one condition variable. +__declspec(dllexport) void WakeAllConditionVariable(CONDITION_VARIABLE* cv) { + K32_CONDITION_STATE* state = K32ConditionState(cv, FALSE); + if (!state) + return; + + K32ConditionLock(state); + while (state->Head) { + K32_CONDITION_WAITER* waiter = state->Head; + K32ConditionRemoveWaiter(state, waiter); + (void)SysSignalEvent(waiter->Event); + } + K32ConditionUnlock(state); } -// Implements the wake Condition Variable operation. -__declspec(dllexport) void WakeConditionVariable(CONDITION_VARIABLE* cv) { (void)cv; } -// Implements the wake All Condition Variable operation. -__declspec(dllexport) void WakeAllConditionVariable(CONDITION_VARIABLE* cv) { (void)cv; } // Timing & Performance Counters. @@ -2961,7 +3591,7 @@ SetUnhandledExceptionFilter(LPTOP_LEVEL_EXCEPTION_FILTER lpFilter) { return NULL; // previous filter: none } -// Checks whether debugger Present. +// Reports that no user-mode debugger is attached. __declspec(dllexport) BOOL IsDebuggerPresent(void) { return FALSE; } diff --git a/code/user/personalities/win32/winmm.c b/code/user/personalities/win32/winmm.c index f3e4eaf..eceea58 100644 --- a/code/user/personalities/win32/winmm.c +++ b/code/user/personalities/win32/winmm.c @@ -1,126 +1,28 @@ -// Win32 multimedia compatibility stubs. This file provides minimal timing and sound API entry points. +// Win32 multimedia timing entry points. #include -#include #include -// Types. -typedef unsigned int MMRESULT; -typedef void* HWAVEOUT; +typedef unsigned int MMRESULT; -#define MMSYSERR_NOERROR 0 -#define MMSYSERR_NOTSUPPORTED 8 -#define WAVERR_STILLPLAYING 33 - -// WAVEFORMATEX - standard PCM audio format descriptor. -typedef struct { - unsigned short wFormatTag; - unsigned short nChannels; - unsigned int nSamplesPerSec; - unsigned int nAvgBytesPerSec; - unsigned short nBlockAlign; - unsigned short wBitsPerSample; - unsigned short cbSize; -} WAVEFORMATEX; - -// WAVEHDR - buffer header submitted to waveOutWrite. -typedef struct _WAVEHDR { - char* lpData; - unsigned int dwBufferLength; - unsigned int dwBytesRecorded; - uintptr_t dwUser; - unsigned int dwFlags; - unsigned int dwLoops; - struct _WAVEHDR* lpNext; - uintptr_t reserved; -} WAVEHDR; - -// MMTIME - used by waveOutGetPosition. -typedef struct { - unsigned int wType; - union { - unsigned int ms; - unsigned int sample; - unsigned int cb; - unsigned int ticks; - } u; -} MMTIME; - -// Timer. +#define MMSYSERR_NOERROR 0u +// Returns monotonic milliseconds since boot. __declspec(dllexport) unsigned long timeGetTime(void) { uint64_t frequency = 0; - if (SysGetTickFrequency(&frequency) != NTLSTATUS_SUCCESS || !frequency) return 0; + if (SysGetTickFrequency(&frequency) != NTLSTATUS_SUCCESS || !frequency) + return 0; return (unsigned long)SysTicksToMilliseconds(SysGetTick(), frequency); } -// Waveform audio (stubs - no audio hardware at this stage). - -__declspec(dllexport) MMRESULT waveOutOpen(HWAVEOUT* phwo, unsigned int deviceId, - const WAVEFORMATEX* fmt, uintptr_t callback, - uintptr_t callbackInstance, unsigned long flags) { - (void)deviceId; (void)fmt; (void)callback; (void)callbackInstance; (void)flags; - if (phwo) *phwo = (void*)0; - return MMSYSERR_NOTSUPPORTED; -} - -// Implements the wave Out Close operation. -__declspec(dllexport) MMRESULT waveOutClose(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -__declspec(dllexport) MMRESULT waveOutPrepareHeader(HWAVEOUT hwo, WAVEHDR* hdr, - unsigned int size) { - (void)hwo; (void)hdr; (void)size; return MMSYSERR_NOERROR; -} - -__declspec(dllexport) MMRESULT waveOutUnprepareHeader(HWAVEOUT hwo, WAVEHDR* hdr, - unsigned int size) { - (void)hwo; (void)hdr; (void)size; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Write operation. -__declspec(dllexport) MMRESULT waveOutWrite(HWAVEOUT hwo, WAVEHDR* hdr, unsigned int size) { - (void)hwo; (void)hdr; (void)size; return MMSYSERR_NOTSUPPORTED; -} - -// Implements the wave Out Pause operation. -__declspec(dllexport) MMRESULT waveOutPause(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Restart operation. -__declspec(dllexport) MMRESULT waveOutRestart(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Reset operation. -__declspec(dllexport) MMRESULT waveOutReset(HWAVEOUT hwo) { - (void)hwo; return MMSYSERR_NOERROR; -} - -// Implements the wave Out Set Volume operation. -__declspec(dllexport) MMRESULT waveOutSetVolume(HWAVEOUT hwo, unsigned long volume) { - (void)hwo; (void)volume; return MMSYSERR_NOTSUPPORTED; -} - -__declspec(dllexport) MMRESULT waveOutGetPosition(HWAVEOUT hwo, MMTIME* mmt, - unsigned int size) { - (void)hwo; (void)size; - if (mmt) { mmt->wType = 0; mmt->u.ms = 0; } +// Accepts timer-resolution requests without reprogramming the native timer. +__declspec(dllexport) MMRESULT timeBeginPeriod(unsigned int periodMilliseconds) { + (void)periodMilliseconds; return MMSYSERR_NOERROR; } -// Timer resolution control -// No hardware timer reprogramming at this stage; pretend success so callers -// that wrap their audio/game loop in timeBeginPeriod(1) don't bail out. -__declspec(dllexport) MMRESULT timeBeginPeriod(unsigned int uPeriod) { - (void)uPeriod; - return MMSYSERR_NOERROR; -} - -// Implements the time End Period operation. -__declspec(dllexport) MMRESULT timeEndPeriod(unsigned int uPeriod) { - (void)uPeriod; +// Releases one timer-resolution request without changing native timer policy. +__declspec(dllexport) MMRESULT timeEndPeriod(unsigned int periodMilliseconds) { + (void)periodMilliseconds; return MMSYSERR_NOERROR; } diff --git a/code/user/personalities/win32/winmm_waveout.c b/code/user/personalities/win32/winmm_waveout.c new file mode 100644 index 0000000..a709585 --- /dev/null +++ b/code/user/personalities/win32/winmm_waveout.c @@ -0,0 +1,1510 @@ +// WinMM waveform output over the native render-stream syscall ABI. + +#include +#include +#include +#include +#include +#include + +typedef unsigned int MMRESULT; +typedef void* HWAVEOUT; +typedef unsigned int UINT; +typedef uint32_t DWORD; +typedef uintptr_t DWORD_PTR; +typedef unsigned short WORD; +typedef int BOOL; + +#define FALSE 0 +#define TRUE 1 + +#define MMSYSERR_NOERROR 0u +#define MMSYSERR_ERROR 1u +#define MMSYSERR_BADDEVICEID 2u +#define MMSYSERR_ALLOCATED 4u +#define MMSYSERR_INVALHANDLE 5u +#define MMSYSERR_NOMEM 7u +#define MMSYSERR_NOTSUPPORTED 8u +#define MMSYSERR_INVALFLAG 10u +#define MMSYSERR_INVALPARAM 11u +#define MMSYSERR_HANDLEBUSY 12u + +#define WAVERR_BADFORMAT 32u +#define WAVERR_STILLPLAYING 33u +#define WAVERR_UNPREPARED 34u + +#define WAVE_FORMAT_PCM 1u +#define WAVE_MAPPER 0xFFFFFFFFu + +#define WAVE_FORMAT_QUERY 0x00000001u +#define WAVE_ALLOWSYNC 0x00000002u +#define WAVE_FORMAT_DIRECT 0x00000008u +#define CALLBACK_TYPEMASK 0x00070000u +#define CALLBACK_NULL 0x00000000u +#define CALLBACK_WINDOW 0x00010000u +#define CALLBACK_THREAD 0x00020000u +#define CALLBACK_FUNCTION 0x00030000u +#define CALLBACK_EVENT 0x00050000u +#define WINMM_WAVE_OPEN_SUPPORTED_FLAGS \ + (WAVE_FORMAT_QUERY | WAVE_ALLOWSYNC | WAVE_FORMAT_DIRECT | CALLBACK_TYPEMASK) + +#define WHDR_DONE 0x00000001u +#define WHDR_PREPARED 0x00000002u +#define WHDR_BEGINLOOP 0x00000004u +#define WHDR_ENDLOOP 0x00000008u +#define WHDR_INQUEUE 0x00000010u + +#define WOM_OPEN 0x03BBu +#define WOM_CLOSE 0x03BCu +#define WOM_DONE 0x03BDu + +#define TIME_MS 0x0001u +#define TIME_SAMPLES 0x0002u +#define TIME_BYTES 0x0004u + +#define WAVECAPS_VOLUME 0x0001u +#define WAVECAPS_LRVOLUME 0x0002u + +#define WAVE_FORMAT_1M16 0x00000004u +#define WAVE_FORMAT_1S16 0x00000008u +#define WAVE_FORMAT_2M16 0x00000040u +#define WAVE_FORMAT_2S16 0x00000080u +#define WAVE_FORMAT_4M16 0x00000400u +#define WAVE_FORMAT_4S16 0x00000800u +#define WAVE_FORMAT_48M16 0x00004000u +#define WAVE_FORMAT_48S16 0x00008000u + +#define WINMM_WAVEOUT_MAGIC 0x4F564157u +#define WINMM_WAIT_MILLISECONDS 5000ull +#define WINMM_WORKER_PRIORITY 1u +#define WINMM_WRITE_CHUNK_FRAMES 1024u + +typedef struct WAVEFORMATEX { + WORD wFormatTag; + WORD nChannels; + DWORD nSamplesPerSec; + DWORD nAvgBytesPerSec; + WORD nBlockAlign; + WORD wBitsPerSample; + WORD cbSize; +} WAVEFORMATEX; + +typedef struct WAVEHDR { + char* lpData; + DWORD dwBufferLength; + DWORD dwBytesRecorded; + DWORD_PTR dwUser; + DWORD dwFlags; + DWORD dwLoops; + struct WAVEHDR* lpNext; + DWORD_PTR reserved; +} WAVEHDR; + +typedef struct MMTIME { + UINT wType; + union { + DWORD ms; + DWORD sample; + DWORD cb; + DWORD ticks; + } u; +} MMTIME; + +typedef struct WAVEOUTCAPSA { + WORD wMid; + WORD wPid; + DWORD vDriverVersion; + char szPname[32]; + DWORD dwFormats; + WORD wChannels; + WORD wReserved1; + DWORD dwSupport; +} WAVEOUTCAPSA; + +typedef struct WAVEOUTCAPSW { + WORD wMid; + WORD wPid; + DWORD vDriverVersion; + unsigned short szPname[32]; + DWORD dwFormats; + WORD wChannels; + WORD wReserved1; + DWORD dwSupport; +} WAVEOUTCAPSW; + +typedef char WINMM_WAVEFORMATEX_SIZE[(sizeof(WAVEFORMATEX) == 20u) ? 1 : -1]; +typedef char WINMM_WAVEHDR_SIZE[(sizeof(WAVEHDR) == 48u) ? 1 : -1]; +typedef char WINMM_MMTIME_SIZE[(sizeof(MMTIME) == 8u) ? 1 : -1]; +typedef char WINMM_WAVEOUTCAPSA_SIZE[(sizeof(WAVEOUTCAPSA) == 52u) ? 1 : -1]; +typedef char WINMM_WAVEOUTCAPSW_SIZE[(sizeof(WAVEOUTCAPSW) == 84u) ? 1 : -1]; + +typedef void (__stdcall *WAVEOUTPROC)(HWAVEOUT waveOut, + UINT message, + DWORD_PTR instance, + DWORD_PTR parameter1, + DWORD_PTR parameter2); + +typedef enum WINMM_WAVE_COMMAND { + WinmmWaveCommandNone = 0, + WinmmWaveCommandPause, + WinmmWaveCommandRestart, + WinmmWaveCommandReset +} WINMM_WAVE_COMMAND; + +typedef struct WINMM_WAVE_BUFFER { + struct WINMM_WAVE_BUFFER* Next; + WAVEHDR* Header; + int16_t* Frames; + uint32_t FrameCount; + uint32_t SubmittedFrames; + uint64_t PresentedTarget; + uint8_t FullySubmitted; + uint64_t Sequence; +} WINMM_WAVE_BUFFER; + +typedef struct WINMM_WAVEOUT { + struct WINMM_WAVEOUT* RegistryNext; + uint32_t Magic; + volatile long Lock; + volatile long ApiUsers; + uint32_t NativeStream; + uint32_t WakeEvent; + uint32_t CommandEvent; + uint32_t ReadyEvent; + uint32_t WorkerHandle; + uint32_t WorkerTid; + uint64_t EndpointId; + WAVEFORMATEX SourceFormat; + uintptr_t Callback; + uintptr_t CallbackInstance; + uint32_t CallbackMode; + uint32_t DeviceId; + uint32_t Volume; + uint32_t Closing; + uint32_t UserPaused; + uint32_t AutoPaused; + uint32_t Started; + uint32_t TerminalError; + WINMM_WAVE_COMMAND Command; + MMRESULT CommandResult; + WINMM_WAVE_BUFFER* QueueHead; + WINMM_WAVE_BUFFER* QueueTail; + WINMM_WAVE_BUFFER* SubmitNode; + uint64_t NativeSubmitted; + uint64_t KnownDiscontinuity; + uint64_t HeadersQueued; + uint64_t HeadersCompleted; + uint64_t FramesQueued; +} WINMM_WAVEOUT; + +static WINMM_WAVEOUT* gWaveOutRegistry; +static volatile long gWaveOutRegistryLock; + + +typedef struct WINMM_DEBUG_LINE { + char Buffer[192]; + size_t Length; +} WINMM_DEBUG_LINE; + +// Appends text to one bounded WinMM diagnostic line. +static void WinmmDebugText(WINMM_DEBUG_LINE* line, const char* text) { + if (!line || !text) + return; + while (*text && line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = *text++; +} + +// Appends one unsigned decimal value to a WinMM diagnostic line. +static void WinmmDebugU64(WINMM_DEBUG_LINE* line, uint64_t value) { + char digits[24]; + size_t count = 0; + do { + digits[count++] = (char)('0' + value % 10u); + value /= 10u; + } while (value && count < sizeof(digits)); + while (count && line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = digits[--count]; +} + +// Appends one fixed-width hexadecimal value to a WinMM diagnostic line. +static void WinmmDebugHex(WINMM_DEBUG_LINE* line, uint64_t value) { + static const char digits[] = "0123456789ABCDEF"; + WinmmDebugText(line, "0x"); + int started = 0; + for (int shift = 60; shift >= 0; shift -= 4) { + uint8_t digit = (uint8_t)((value >> shift) & 0xFu); + if (!started && digit == 0 && shift) + continue; + started = 1; + if (line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = digits[digit]; + } +} + +// Emits one complete WinMM diagnostic line through the native debug console. +static void WinmmDebugCommit(WINMM_DEBUG_LINE* line) { + if (!line) + return; + if (line->Length + 1 < sizeof(line->Buffer)) + line->Buffer[line->Length++] = '\n'; + line->Buffer[line->Length] = '\0'; + SYS_PRINT_DEBUG_REQUEST request = { SysSvFromParts(line->Buffer, line->Length) }; + (void)SysPrintDebug(&request); +} + +// Logs one waveform-open request before validation changes its control flow. +static void WinmmDebugOpenRequest(UINT deviceId, + const WAVEFORMATEX* format, + DWORD flags) { + WINMM_DEBUG_LINE line = {0}; + WinmmDebugText(&line, "WINMM: open request device="); + WinmmDebugU64(&line, deviceId); + WinmmDebugText(&line, " tag="); + WinmmDebugU64(&line, format ? format->wFormatTag : 0); + WinmmDebugText(&line, " rate="); + WinmmDebugU64(&line, format ? format->nSamplesPerSec : 0); + WinmmDebugText(&line, " channels="); + WinmmDebugU64(&line, format ? format->nChannels : 0); + WinmmDebugText(&line, " bits="); + WinmmDebugU64(&line, format ? format->wBitsPerSample : 0); + WinmmDebugText(&line, " align="); + WinmmDebugU64(&line, format ? format->nBlockAlign : 0); + WinmmDebugText(&line, " avg="); + WinmmDebugU64(&line, format ? format->nAvgBytesPerSec : 0); + WinmmDebugText(&line, " flags="); + WinmmDebugHex(&line, flags); + WinmmDebugCommit(&line); +} + +// Logs one WinMM result with its handle and native stream identity. +static void WinmmDebugResult(const char* operation, + const WINMM_WAVEOUT* waveOut, + uint64_t value) { + WINMM_DEBUG_LINE line = {0}; + WinmmDebugText(&line, "WINMM: "); + WinmmDebugText(&line, operation); + if (waveOut) { + WinmmDebugText(&line, " handle="); + WinmmDebugHex(&line, (uintptr_t)waveOut); + WinmmDebugText(&line, " stream="); + WinmmDebugU64(&line, waveOut->NativeStream); + } + WinmmDebugText(&line, " value="); + WinmmDebugU64(&line, value); + WinmmDebugCommit(&line); +} + +// Acquires one short user-mode spin lock without entering the kernel. +static void WinmmLock(volatile long* lock) { + while (_InterlockedCompareExchange(lock, 1, 0) != 0) + (void)SysYield(); +} + +// Releases one short user-mode spin lock. +static void WinmmUnlock(volatile long* lock) { + _InterlockedExchange(lock, 0); +} + +// Maps one native status to the closest waveform-open result. +static MMRESULT WinmmMapNativeStatus(KSTATUS status) { + if (status == NTLSTATUS_SUCCESS) + return MMSYSERR_NOERROR; + if (status == NTLSTATUS_NO_MEMORY) + return MMSYSERR_NOMEM; + if (status == NTLSTATUS_NOT_FOUND || status == NTLSTATUS_DEVICE_REMOVED) + return MMSYSERR_BADDEVICEID; + if (status == NTLSTATUS_NOT_SUPPORTED) + return WAVERR_BADFORMAT; + if (status == NTLSTATUS_INVALID_PARAMETER) + return MMSYSERR_INVALPARAM; + return MMSYSERR_ERROR; +} + +// Returns whether one source rate is accepted by the native stream ABI. +static int WinmmRateIsSupported(uint32_t sampleRate) { + return sampleRate == 11025u || sampleRate == 22050u || + sampleRate == 44100u || sampleRate == 48000u; +} + +// Validates the narrow PCM surface implemented by this personality. +static MMRESULT WinmmValidateFormat(const WAVEFORMATEX* format) { + if (!format) + return MMSYSERR_INVALPARAM; + if (format->wFormatTag != WAVE_FORMAT_PCM || + (format->nChannels != 1u && format->nChannels != 2u) || + !WinmmRateIsSupported(format->nSamplesPerSec) || + (format->wBitsPerSample != 16u && format->wBitsPerSample != 24u)) + return WAVERR_BADFORMAT; + + uint32_t bytesPerSample = format->wBitsPerSample / 8u; + uint32_t blockAlign = (uint32_t)format->nChannels * bytesPerSample; + uint64_t averageBytes = (uint64_t)format->nSamplesPerSec * blockAlign; + if (format->nBlockAlign != blockAlign || + averageBytes > UINT32_MAX || + format->nAvgBytesPerSec != (uint32_t)averageBytes) + return WAVERR_BADFORMAT; + return MMSYSERR_NOERROR; +} + +// Enumerates available native endpoints into the caller's fixed snapshot. +static MMRESULT WinmmEnumerateEndpoints(SYS_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t capacity, + uint32_t* countOut) { + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = capacity; + request.CountOut = countOut; + return WinmmMapNativeStatus(SysAudioEnumerateEndpoints(&request)); +} + +// Counts endpoints that are currently ready and available. +static uint32_t WinmmAvailableEndpointCount(void) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + if (WinmmEnumerateEndpoints(endpoints, SYS_AUDIO_MAX_ENDPOINTS, &count) != MMSYSERR_NOERROR) + return 0; + if (count > SYS_AUDIO_MAX_ENDPOINTS) + count = SYS_AUDIO_MAX_ENDPOINTS; + + uint32_t available = 0; + for (uint32_t index = 0; index < count; index++) { + if (endpoints[index].State == SysAudioEndpointReady && + (endpoints[index].Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE)) + available++; + } + return available; +} + +// Resolves WAVE_MAPPER or one available-device index to a stable endpoint ID. +static MMRESULT WinmmResolveEndpoint(uint32_t deviceId, + uint64_t* endpointIdOut, + uint32_t* resolvedDeviceIdOut) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + MMRESULT result = WinmmEnumerateEndpoints(endpoints, SYS_AUDIO_MAX_ENDPOINTS, &count); + if (result != MMSYSERR_NOERROR) + return result; + if (count > SYS_AUDIO_MAX_ENDPOINTS) + count = SYS_AUDIO_MAX_ENDPOINTS; + + const SYS_AUDIO_ENDPOINT_INFO* fallback = NULL; + uint32_t fallbackIndex = 0; + uint32_t availableIndex = 0; + for (uint32_t index = 0; index < count; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) { + fallback = endpoint; + fallbackIndex = availableIndex; + } + if (deviceId == WAVE_MAPPER && + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT)) { + *endpointIdOut = endpoint->EndpointId; + *resolvedDeviceIdOut = availableIndex; + return MMSYSERR_NOERROR; + } + if (deviceId != WAVE_MAPPER && availableIndex == deviceId) { + *endpointIdOut = endpoint->EndpointId; + *resolvedDeviceIdOut = availableIndex; + return MMSYSERR_NOERROR; + } + availableIndex++; + } + + if (deviceId == WAVE_MAPPER && fallback) { + *endpointIdOut = fallback->EndpointId; + *resolvedDeviceIdOut = fallbackIndex; + return MMSYSERR_NOERROR; + } + return MMSYSERR_BADDEVICEID; +} + +// Adds one live waveform handle to the process-local registry. +static void WinmmRegisterWaveOut(WINMM_WAVEOUT* waveOut) { + WinmmLock(&gWaveOutRegistryLock); + waveOut->RegistryNext = gWaveOutRegistry; + gWaveOutRegistry = waveOut; + WinmmUnlock(&gWaveOutRegistryLock); +} + +// References one registered waveform handle without dereferencing unknown pointers. +static WINMM_WAVEOUT* WinmmReferenceWaveOut(HWAVEOUT handle) { + WINMM_WAVEOUT* found = NULL; + WinmmLock(&gWaveOutRegistryLock); + for (WINMM_WAVEOUT* current = gWaveOutRegistry; + current; + current = current->RegistryNext) { + if ((HWAVEOUT)current == handle && current->Magic == WINMM_WAVEOUT_MAGIC) { + _InterlockedIncrement(¤t->ApiUsers); + found = current; + break; + } + } + WinmmUnlock(&gWaveOutRegistryLock); + return found; +} + +// Releases one API-use reference acquired through the registry. +static void WinmmReleaseWaveOut(WINMM_WAVEOUT* waveOut) { + if (waveOut) + _InterlockedDecrement(&waveOut->ApiUsers); +} + +// Removes one waveform handle so no new API references can be acquired. +static int WinmmRemoveWaveOut(WINMM_WAVEOUT* waveOut) { + int removed = 0; + WinmmLock(&gWaveOutRegistryLock); + WINMM_WAVEOUT** link = &gWaveOutRegistry; + while (*link) { + if (*link == waveOut) { + *link = waveOut->RegistryNext; + waveOut->RegistryNext = NULL; + removed = 1; + break; + } + link = &(*link)->RegistryNext; + } + WinmmUnlock(&gWaveOutRegistryLock); + return removed; +} + +// Delivers one WinMM callback through a supported user-mode mechanism. +static void WinmmNotify(WINMM_WAVEOUT* waveOut, + uint32_t message, + uintptr_t parameter1, + uintptr_t parameter2) { + if (waveOut->CallbackMode == CALLBACK_FUNCTION) { + WAVEOUTPROC callback = (WAVEOUTPROC)waveOut->Callback; + callback((HWAVEOUT)waveOut, + message, + waveOut->CallbackInstance, + parameter1, + parameter2); + } else if (waveOut->CallbackMode == CALLBACK_EVENT) { + (void)SysSignalEvent((uint32_t)waveOut->Callback); + } +} + +// Converts one packed PCM buffer into personality-owned native S16 frames. +static MMRESULT WinmmCopyFrames(const WINMM_WAVEOUT* waveOut, + const WAVEHDR* header, + int16_t** framesOut, + uint32_t* frameCountOut) { + if (!header || !framesOut || !frameCountOut) + return MMSYSERR_INVALPARAM; + if (header->dwBufferLength && !header->lpData) + return MMSYSERR_INVALPARAM; + if (header->dwBufferLength % waveOut->SourceFormat.nBlockAlign) + return WAVERR_BADFORMAT; + + uint32_t frameCount = header->dwBufferLength / waveOut->SourceFormat.nBlockAlign; + size_t sampleCount = (size_t)frameCount * waveOut->SourceFormat.nChannels; + if (frameCount && sampleCount > SIZE_MAX / sizeof(int16_t)) + return MMSYSERR_NOMEM; + int16_t* output = sampleCount ? (int16_t*)malloc(sampleCount * sizeof(int16_t)) : NULL; + if (sampleCount && !output) + return MMSYSERR_NOMEM; + + if (waveOut->SourceFormat.wBitsPerSample == 16u) { + if (sampleCount) + memcpy(output, header->lpData, sampleCount * sizeof(int16_t)); + } else { + const uint8_t* input = (const uint8_t*)header->lpData; + for (size_t sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) { + uint32_t packed = (uint32_t)input[sampleIndex * 3u] | + ((uint32_t)input[sampleIndex * 3u + 1u] << 8u) | + ((uint32_t)input[sampleIndex * 3u + 2u] << 16u); + int32_t signedSample = (packed & 0x00800000u) + ? (int32_t)(packed | 0xFF000000u) + : (int32_t)packed; + output[sampleIndex] = (int16_t)(signedSample / 256); + } + } + + *framesOut = output; + *frameCountOut = frameCount; + return MMSYSERR_NOERROR; +} + +// Pops and completes every header whose source frames reached hardware presentation. +static void WinmmCompletePresentedBuffers(WINMM_WAVEOUT* waveOut, + uint64_t framesPresented) { + while (true) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* buffer = waveOut->QueueHead; + if (!buffer || !buffer->FullySubmitted || + buffer->PresentedTarget > framesPresented) { + WinmmUnlock(&waveOut->Lock); + return; + } + waveOut->QueueHead = buffer->Next; + if (!waveOut->QueueHead) + waveOut->QueueTail = NULL; + if (waveOut->SubmitNode == buffer) + waveOut->SubmitNode = buffer->Next; + buffer->Header->dwFlags &= ~WHDR_INQUEUE; + buffer->Header->dwFlags |= WHDR_DONE; + waveOut->HeadersCompleted++; + WinmmUnlock(&waveOut->Lock); + WinmmNotify(waveOut, WOM_DONE, (uintptr_t)buffer->Header, 0); + free(buffer->Frames); + free(buffer); + } +} + +// Detaches every queued buffer for reset or terminal failure completion. +static WINMM_WAVE_BUFFER* WinmmDetachAllBuffers(WINMM_WAVEOUT* waveOut) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* buffers = waveOut->QueueHead; + waveOut->QueueHead = NULL; + waveOut->QueueTail = NULL; + waveOut->SubmitNode = NULL; + WinmmUnlock(&waveOut->Lock); + return buffers; +} + +// Completes and releases one detached buffer list in queue order. +static void WinmmCompleteDetachedBuffers(WINMM_WAVEOUT* waveOut, + WINMM_WAVE_BUFFER* buffers) { + while (buffers) { + WINMM_WAVE_BUFFER* next = buffers->Next; + buffers->Header->dwFlags &= ~WHDR_INQUEUE; + buffers->Header->dwFlags |= WHDR_DONE; + waveOut->HeadersCompleted++; + WinmmNotify(waveOut, WOM_DONE, (uintptr_t)buffers->Header, 0); + free(buffers->Frames); + free(buffers); + buffers = next; + } +} + +// Queries one coherent native stream snapshot. +static KSTATUS WinmmQueryNativeStream(WINMM_WAVEOUT* waveOut, + SYS_AUDIO_STREAM_INFO* info) { + memset(info, 0, sizeof(*info)); + info->Size = sizeof(*info); + return SysAudioQueryStream(waveOut->NativeStream, info); +} + +// Updates the controlled discontinuity baseline after stop or reset. +static void WinmmRefreshDiscontinuity(WINMM_WAVEOUT* waveOut) { + SYS_AUDIO_STREAM_INFO info; + if (WinmmQueryNativeStream(waveOut, &info) == NTLSTATUS_SUCCESS) + waveOut->KnownDiscontinuity = info.DiscontinuityGeneration; +} + +// Processes one synchronous control command on the worker thread. +static void WinmmProcessCommand(WINMM_WAVEOUT* waveOut) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_COMMAND command = waveOut->Command; + WinmmUnlock(&waveOut->Lock); + if (command == WinmmWaveCommandNone) + return; + + MMRESULT result = MMSYSERR_NOERROR; + WINMM_WAVE_BUFFER* resetBuffers = NULL; + if (command == WinmmWaveCommandPause) { + if (!waveOut->UserPaused) { + waveOut->UserPaused = 1; + if (waveOut->Started && !waveOut->AutoPaused) { + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status == NTLSTATUS_SUCCESS && info.State == SysAudioStreamRunning) + status = SysAudioPause(waveOut->NativeStream); + if (status != NTLSTATUS_SUCCESS) + result = WinmmMapNativeStatus(status); + } + } + } else if (command == WinmmWaveCommandRestart) { + if (waveOut->UserPaused) { + waveOut->UserPaused = 0; + if (waveOut->Started && !waveOut->AutoPaused) { + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status == NTLSTATUS_SUCCESS && info.State == SysAudioStreamPaused) + status = SysAudioResume(waveOut->NativeStream); + if (status != NTLSTATUS_SUCCESS) + result = WinmmMapNativeStatus(status); + } + } + } else if (command == WinmmWaveCommandReset) { + (void)SysAudioStop(waveOut->NativeStream); + KSTATUS status = SysAudioReset(waveOut->NativeStream); + if (status != NTLSTATUS_SUCCESS) + result = WinmmMapNativeStatus(status); + else + waveOut->TerminalError = 0; + resetBuffers = WinmmDetachAllBuffers(waveOut); + waveOut->NativeSubmitted = 0; + waveOut->Started = 0; + waveOut->UserPaused = 0; + waveOut->AutoPaused = 0; + WinmmRefreshDiscontinuity(waveOut); + } + + if (resetBuffers) + WinmmCompleteDetachedBuffers(waveOut, resetBuffers); + + WinmmLock(&waveOut->Lock); + waveOut->CommandResult = result; + waveOut->Command = WinmmWaveCommandNone; + WinmmUnlock(&waveOut->Lock); + (void)SysSignalEvent(waveOut->CommandEvent); +} + +// Returns the next queued node that still has source frames to submit. +static WINMM_WAVE_BUFFER* WinmmFindSubmitNode(WINMM_WAVEOUT* waveOut) { + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* buffer = waveOut->SubmitNode; + while (buffer && buffer->FullySubmitted) + buffer = buffer->Next; + waveOut->SubmitNode = buffer; + WinmmUnlock(&waveOut->Lock); + return buffer; +} + +// Feeds queued buffers into the native ring until it becomes full or empty. +static KSTATUS WinmmFeedNativeStream(WINMM_WAVEOUT* waveOut, + uint32_t* madeProgressOut) { + *madeProgressOut = 0; + while (true) { + WINMM_WAVE_BUFFER* buffer = WinmmFindSubmitNode(waveOut); + if (!buffer) + return NTLSTATUS_SUCCESS; + + uint32_t remaining = buffer->FrameCount - buffer->SubmittedFrames; + if (!remaining) { + buffer->FullySubmitted = 1; + buffer->PresentedTarget = waveOut->NativeSubmitted; + continue; + } + uint32_t requestFrames = remaining < WINMM_WRITE_CHUNK_FRAMES + ? remaining + : WINMM_WRITE_CHUNK_FRAMES; + uint32_t writtenFrames = 0; + SYS_AUDIO_WRITE_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = waveOut->NativeStream; + request.FrameCount = requestFrames; + request.Frames = buffer->Frames + + (size_t)buffer->SubmittedFrames * waveOut->SourceFormat.nChannels; + request.WrittenFramesOut = &writtenFrames; + KSTATUS status = SysAudioWrite(&request); + if (status == NTLSTATUS_WOULD_BLOCK) + return status; + if (status != NTLSTATUS_SUCCESS) + return status; + if (!writtenFrames) + return NTLSTATUS_WOULD_BLOCK; + + buffer->SubmittedFrames += writtenFrames; + waveOut->NativeSubmitted += writtenFrames; + *madeProgressOut = 1; + if (buffer->SubmittedFrames == buffer->FrameCount) { + buffer->FullySubmitted = 1; + buffer->PresentedTarget = waveOut->NativeSubmitted; + } + } +} + +// Determines whether any queued buffer still needs native-ring submission. +static int WinmmHasUnsubmittedBuffer(WINMM_WAVEOUT* waveOut) { + return WinmmFindSubmitNode(waveOut) != NULL; +} + +// Starts or resumes playback after a useful prefill is available. +static KSTATUS WinmmEnsureStarted(WINMM_WAVEOUT* waveOut) { + if (waveOut->UserPaused) + return NTLSTATUS_SUCCESS; + if (!waveOut->Started) { + KSTATUS status = SysAudioStart(waveOut->NativeStream); + if (status == NTLSTATUS_SUCCESS) { + waveOut->Started = 1; + waveOut->AutoPaused = 0; + } + return status; + } + if (waveOut->AutoPaused) { + KSTATUS status = SysAudioResume(waveOut->NativeStream); + if (status == NTLSTATUS_SUCCESS) + waveOut->AutoPaused = 0; + return status; + } + return NTLSTATUS_SUCCESS; +} + +// Stops a completed empty stream without resetting its public position counters. +static void WinmmStopEmptyStream(WINMM_WAVEOUT* waveOut) { + if (!waveOut->Started) + return; + (void)SysAudioStop(waveOut->NativeStream); + waveOut->Started = 0; + waveOut->AutoPaused = 0; + WinmmRefreshDiscontinuity(waveOut); +} + +// Arms native progress conditions and waits for either stream progress or new work. +static KSTATUS WinmmWaitForWork(WINMM_WAVEOUT* waveOut, + const SYS_AUDIO_STREAM_INFO* info, + int hasUnsubmitted) { + SYS_AUDIO_ARM_WAIT_REQUEST arm; + memset(&arm, 0, sizeof(arm)); + arm.Size = sizeof(arm); + arm.StreamHandle = waveOut->NativeStream; + arm.Mask = SYS_AUDIO_WAIT_FAILED | SYS_AUDIO_WAIT_DISCONTINUITY; + arm.DiscontinuityGeneration = waveOut->KnownDiscontinuity; + + if (hasUnsubmitted && !waveOut->UserPaused) { + arm.Mask |= SYS_AUDIO_WAIT_WRITABLE; + arm.WritableFrames = 1; + } + WinmmLock(&waveOut->Lock); + WINMM_WAVE_BUFFER* head = waveOut->QueueHead; + if (head && head->FullySubmitted) { + arm.Mask |= SYS_AUDIO_WAIT_PRESENTED; + arm.PresentedFrames = head->PresentedTarget; + } + WinmmUnlock(&waveOut->Lock); + if (!hasUnsubmitted && waveOut->Started && + info->FramesConsumed < waveOut->NativeSubmitted) { + arm.Mask |= SYS_AUDIO_WAIT_CONSUMED; + arm.ConsumedFrames = waveOut->NativeSubmitted; + } + + KSTATUS status = SysAudioArmWait(&arm); + if (status != NTLSTATUS_SUCCESS) + return status; + + uint32_t handles[2] = { waveOut->WakeEvent, waveOut->NativeStream }; + uint32_t selected = 0; + SYS_WAIT_MANY_REQUEST wait; + memset(&wait, 0, sizeof(wait)); + wait.Size = sizeof(wait); + wait.Handles = handles; + wait.Count = 2; + wait.SignaledIndexOut = &selected; + wait.TimeoutMilliseconds = SYS_WAIT_TIMEOUT_INFINITE; + return SysWaitMany(&wait); +} + +// Retires a failed stream and returns all queued headers to the application. +static void WinmmFailStream(WINMM_WAVEOUT* waveOut, MMRESULT error) { + waveOut->TerminalError = error ? error : MMSYSERR_ERROR; + WinmmDebugResult("stream failure", waveOut, waveOut->TerminalError); + (void)SysAudioStop(waveOut->NativeStream); + waveOut->Started = 0; + waveOut->AutoPaused = 0; + WINMM_WAVE_BUFFER* buffers = WinmmDetachAllBuffers(waveOut); + WinmmCompleteDetachedBuffers(waveOut, buffers); +} + +// Runs one waveform-output worker until close or process teardown. +static void WinmmWaveWorker(uintptr_t argument) { + WINMM_WAVEOUT* waveOut = (WINMM_WAVEOUT*)argument; + waveOut->WorkerTid = SysGetCurrentTid(); + WinmmNotify(waveOut, WOM_OPEN, 0, 0); + (void)SysSignalEvent(waveOut->ReadyEvent); + + while (true) { + WinmmProcessCommand(waveOut); + if (waveOut->Closing) { + WinmmStopEmptyStream(waveOut); + WinmmNotify(waveOut, WOM_CLOSE, 0, 0); + SysExitThread(0); + } + if (waveOut->TerminalError) { + uint32_t handles[1] = { waveOut->WakeEvent }; + uint32_t selected = 0; + SYS_WAIT_MANY_REQUEST wait = { + sizeof(wait), 0, handles, 1, &selected, SYS_WAIT_TIMEOUT_INFINITE + }; + (void)SysWaitMany(&wait); + continue; + } + + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS || info.State == SysAudioStreamFailed) { + WinmmFailStream(waveOut, WinmmMapNativeStatus( + status != NTLSTATUS_SUCCESS ? status : info.FailureStatus)); + continue; + } + if (info.DiscontinuityGeneration != waveOut->KnownDiscontinuity) { + // A native discontinuity is a timeline notification, not a stream + // failure. The failed state remains the terminal error authority. + waveOut->KnownDiscontinuity = info.DiscontinuityGeneration; + WinmmDebugResult("stream discontinuity", waveOut, + waveOut->KnownDiscontinuity); + } + + WinmmCompletePresentedBuffers(waveOut, info.FramesPresented); + WinmmLock(&waveOut->Lock); + int queueEmpty = waveOut->QueueHead == NULL; + WinmmUnlock(&waveOut->Lock); + if (queueEmpty) { + WinmmStopEmptyStream(waveOut); + status = WinmmWaitForWork(waveOut, &info, 0); + if (status != NTLSTATUS_SUCCESS) + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + + if (!waveOut->UserPaused) { + uint32_t madeProgress = 0; + status = WinmmFeedNativeStream(waveOut, &madeProgress); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_WOULD_BLOCK) { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + if (waveOut->NativeSubmitted && (!waveOut->Started || waveOut->AutoPaused)) { + status = WinmmEnsureStarted(waveOut); + if (status != NTLSTATUS_SUCCESS) { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + } + if (madeProgress) + continue; + } + + status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS) { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + int hasUnsubmitted = WinmmHasUnsubmittedBuffer(waveOut); + if (!hasUnsubmitted && waveOut->Started && !waveOut->UserPaused && + !waveOut->AutoPaused && info.FramesConsumed >= waveOut->NativeSubmitted) { + status = SysAudioPause(waveOut->NativeStream); + if (status == NTLSTATUS_SUCCESS) + waveOut->AutoPaused = 1; + else { + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + continue; + } + } + + status = WinmmWaitForWork(waveOut, &info, hasUnsubmitted); + if (status != NTLSTATUS_SUCCESS) + WinmmFailStream(waveOut, WinmmMapNativeStatus(status)); + } +} + +// Submits one synchronous worker command and waits for its completion. +static MMRESULT WinmmSubmitCommand(WINMM_WAVEOUT* waveOut, + WINMM_WAVE_COMMAND command) { + if (waveOut->WorkerTid == SysGetCurrentTid()) + return MMSYSERR_HANDLEBUSY; + + WinmmLock(&waveOut->Lock); + if (waveOut->Command != WinmmWaveCommandNone) { + WinmmUnlock(&waveOut->Lock); + return MMSYSERR_ALLOCATED; + } + waveOut->Command = command; + waveOut->CommandResult = MMSYSERR_ERROR; + WinmmUnlock(&waveOut->Lock); + (void)SysSignalEvent(waveOut->WakeEvent); + + KSTATUS status = SysWait(waveOut->CommandEvent, SYS_WAIT_FLAG_NONE); + if (status != NTLSTATUS_SUCCESS) + return WinmmMapNativeStatus(status); + return waveOut->CommandResult; +} + +// Returns the number of available logical waveform output devices. +__declspec(dllexport) UINT waveOutGetNumDevs(void) { + return WinmmAvailableEndpointCount(); +} + +// Opens one waveform output handle or validates one format query. +__declspec(dllexport) MMRESULT waveOutOpen(HWAVEOUT* handleOut, + UINT deviceId, + const WAVEFORMATEX* format, + DWORD_PTR callback, + DWORD_PTR callbackInstance, + DWORD flags) { + WinmmDebugOpenRequest(deviceId, format, flags); + MMRESULT result = WinmmValidateFormat(format); + if (result != MMSYSERR_NOERROR) { + WinmmDebugResult("open rejected format", NULL, result); + return result; + } + if (flags & ~WINMM_WAVE_OPEN_SUPPORTED_FLAGS) { + WinmmDebugResult("open rejected flags", NULL, flags & ~WINMM_WAVE_OPEN_SUPPORTED_FLAGS); + return MMSYSERR_INVALFLAG; + } + + uint32_t callbackMode = flags & CALLBACK_TYPEMASK; + if (callbackMode == CALLBACK_WINDOW || callbackMode == CALLBACK_THREAD) { + WinmmDebugResult("open rejected callback", NULL, callbackMode); + return MMSYSERR_NOTSUPPORTED; + } + if (callbackMode != CALLBACK_NULL && + callbackMode != CALLBACK_FUNCTION && + callbackMode != CALLBACK_EVENT) + return MMSYSERR_INVALFLAG; + if ((callbackMode == CALLBACK_FUNCTION || callbackMode == CALLBACK_EVENT) && !callback) + return MMSYSERR_INVALPARAM; + + uint64_t endpointId = 0; + uint32_t resolvedDeviceId = 0; + result = WinmmResolveEndpoint(deviceId, &endpointId, &resolvedDeviceId); + if (result != MMSYSERR_NOERROR) { + WinmmDebugResult("open rejected endpoint", NULL, result); + return result; + } + if (flags & WAVE_FORMAT_QUERY) { + return MMSYSERR_NOERROR; + } + if (!handleOut) + return MMSYSERR_INVALPARAM; + *handleOut = NULL; + + WINMM_WAVEOUT* waveOut = (WINMM_WAVEOUT*)calloc(1, sizeof(*waveOut)); + if (!waveOut) + return MMSYSERR_NOMEM; + waveOut->Magic = WINMM_WAVEOUT_MAGIC; + waveOut->EndpointId = endpointId; + waveOut->SourceFormat = *format; + waveOut->Callback = callback; + waveOut->CallbackInstance = callbackInstance; + waveOut->CallbackMode = callbackMode; + waveOut->DeviceId = resolvedDeviceId; + waveOut->Volume = 0xFFFFFFFFu; + + SYS_AUDIO_OPEN_STREAM_REQUEST open; + memset(&open, 0, sizeof(open)); + open.Size = sizeof(open); + open.EndpointId = endpointId; + open.Format.SampleRate = format->nSamplesPerSec; + open.Format.ChannelCount = format->nChannels; + open.Format.SampleFormat = SysAudioSampleS16; + open.StreamHandleOut = &waveOut->NativeStream; + KSTATUS status = SysAudioOpenStream(&open); + if (status != NTLSTATUS_SUCCESS) { + result = WinmmMapNativeStatus(status); + WinmmDebugResult("native stream open failed", waveOut, status); + free(waveOut); + return result; + } + + status = SysCreateEvent(0, 0, &waveOut->WakeEvent); + if (status == NTLSTATUS_SUCCESS) + status = SysCreateEvent(0, 0, &waveOut->CommandEvent); + if (status == NTLSTATUS_SUCCESS) + status = SysCreateEvent(0, 0, &waveOut->ReadyEvent); + if (status != NTLSTATUS_SUCCESS) { + result = WinmmMapNativeStatus(status); + WinmmDebugResult("event creation failed", waveOut, status); + if (waveOut->ReadyEvent) (void)SysCloseHandle(waveOut->ReadyEvent); + if (waveOut->CommandEvent) (void)SysCloseHandle(waveOut->CommandEvent); + if (waveOut->WakeEvent) (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return result; + } + + SYS_AUDIO_STREAM_INFO info; + status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS) { + result = WinmmMapNativeStatus(status); + (void)SysCloseHandle(waveOut->ReadyEvent); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return result; + } + waveOut->KnownDiscontinuity = info.DiscontinuityGeneration; + + WinmmRegisterWaveOut(waveOut); + SYS_CREATE_THREAD_REQUEST thread; + memset(&thread, 0, sizeof(thread)); + thread.Entry = (uintptr_t)WinmmWaveWorker; + thread.Argument = (uintptr_t)waveOut; + thread.Priority = WINMM_WORKER_PRIORITY; + thread.ThreadHandleOut = &waveOut->WorkerHandle; + thread.ThreadIdOut = &waveOut->WorkerTid; + status = SysCreateThread(&thread); + if (status != NTLSTATUS_SUCCESS) { + (void)WinmmRemoveWaveOut(waveOut); + result = WinmmMapNativeStatus(status); + WinmmDebugResult("worker creation failed", waveOut, status); + (void)SysCloseHandle(waveOut->ReadyEvent); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return result; + } + + status = SysWaitSingle(waveOut->ReadyEvent, + SYS_WAIT_FLAG_NONE, + WINMM_WAIT_MILLISECONDS); + (void)SysCloseHandle(waveOut->ReadyEvent); + waveOut->ReadyEvent = 0; + if (status != NTLSTATUS_SUCCESS) { + waveOut->Closing = 1; + (void)SysSignalEvent(waveOut->WakeEvent); + (void)SysWait(waveOut->WorkerHandle, SYS_WAIT_FLAG_NONE); + (void)WinmmRemoveWaveOut(waveOut); + (void)SysCloseHandle(waveOut->WorkerHandle); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + free(waveOut); + return WinmmMapNativeStatus(status); + } + + *handleOut = (HWAVEOUT)waveOut; + WINMM_DEBUG_LINE line = {0}; + WinmmDebugText(&line, "WINMM: open success handle="); + WinmmDebugHex(&line, (uintptr_t)waveOut); + WinmmDebugText(&line, " endpoint="); + WinmmDebugU64(&line, endpointId); + WinmmDebugText(&line, " stream="); + WinmmDebugU64(&line, waveOut->NativeStream); + WinmmDebugText(&line, " worker="); + WinmmDebugU64(&line, waveOut->WorkerTid); + WinmmDebugCommit(&line); + return MMSYSERR_NOERROR; +} + +// Closes one idle waveform handle after its worker sends WOM_CLOSE. +__declspec(dllexport) MMRESULT waveOutClose(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (waveOut->WorkerTid == SysGetCurrentTid()) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_HANDLEBUSY; + } + + WinmmLock(&waveOut->Lock); + int busy = waveOut->QueueHead != NULL || waveOut->Command != WinmmWaveCommandNone; + WinmmUnlock(&waveOut->Lock); + if (busy) { + WinmmDebugResult("close still playing", waveOut, waveOut->HeadersQueued - waveOut->HeadersCompleted); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if (!WinmmRemoveWaveOut(waveOut)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALHANDLE; + } + + waveOut->Closing = 1; + (void)SysSignalEvent(waveOut->WakeEvent); + KSTATUS status = SysWait(waveOut->WorkerHandle, SYS_WAIT_FLAG_NONE); + while (waveOut->ApiUsers > 1) + (void)SysYield(); + + waveOut->Magic = 0; + (void)SysCloseHandle(waveOut->WorkerHandle); + (void)SysCloseHandle(waveOut->CommandEvent); + (void)SysCloseHandle(waveOut->WakeEvent); + (void)SysCloseHandle(waveOut->NativeStream); + WinmmReleaseWaveOut(waveOut); + MMRESULT result = status == NTLSTATUS_SUCCESS ? MMSYSERR_NOERROR : WinmmMapNativeStatus(status); + WinmmDebugResult("close complete", waveOut, result); + free(waveOut); + return result; +} + +// Marks one caller-owned header ready for waveform submission. +__declspec(dllexport) MMRESULT waveOutPrepareHeader(HWAVEOUT handle, + WAVEHDR* header, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!header || size < sizeof(*header)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + WinmmLock(&waveOut->Lock); + if (header->dwFlags & WHDR_INQUEUE) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if ((header->dwFlags & WHDR_PREPARED) && + header->reserved != (DWORD_PTR)waveOut) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_ALLOCATED; + } + header->dwFlags &= ~WHDR_DONE; + header->dwFlags |= WHDR_PREPARED; + header->reserved = (DWORD_PTR)waveOut; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Releases preparation state after one header is no longer queued. +__declspec(dllexport) MMRESULT waveOutUnprepareHeader(HWAVEOUT handle, + WAVEHDR* header, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!header || size < sizeof(*header)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + WinmmLock(&waveOut->Lock); + if (header->dwFlags & WHDR_INQUEUE) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if ((header->dwFlags & WHDR_PREPARED) == 0 || + header->reserved != (DWORD_PTR)waveOut) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + header->dwFlags &= ~WHDR_PREPARED; + header->reserved = 0; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Copies one prepared buffer immediately and appends it to the user-mode queue. +__declspec(dllexport) MMRESULT waveOutWrite(HWAVEOUT handle, + WAVEHDR* header, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!header || size < sizeof(*header)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + WinmmLock(&waveOut->Lock); + if ((header->dwFlags & WHDR_PREPARED) == 0 || + header->reserved != (DWORD_PTR)waveOut) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_UNPREPARED; + } + if (header->dwFlags & WHDR_INQUEUE) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return WAVERR_STILLPLAYING; + } + if ((header->dwFlags & (WHDR_BEGINLOOP | WHDR_ENDLOOP)) || header->dwLoops) { + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOTSUPPORTED; + } + if (waveOut->TerminalError) { + MMRESULT error = waveOut->TerminalError; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return error; + } + header->dwFlags &= ~WHDR_DONE; + header->dwFlags |= WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + + WINMM_WAVE_BUFFER* buffer = (WINMM_WAVE_BUFFER*)calloc(1, sizeof(*buffer)); + if (!buffer) { + WinmmLock(&waveOut->Lock); + header->dwFlags &= ~WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOMEM; + } + MMRESULT result = WinmmCopyFrames(waveOut, + header, + &buffer->Frames, + &buffer->FrameCount); + if (result != MMSYSERR_NOERROR) { + WinmmLock(&waveOut->Lock); + header->dwFlags &= ~WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + free(buffer); + WinmmReleaseWaveOut(waveOut); + return result; + } + buffer->Header = header; + if (!buffer->FrameCount) { + buffer->FullySubmitted = 1; + buffer->PresentedTarget = waveOut->NativeSubmitted; + } + + WinmmLock(&waveOut->Lock); + if (waveOut->Closing) { + header->dwFlags &= ~WHDR_INQUEUE; + WinmmUnlock(&waveOut->Lock); + free(buffer->Frames); + free(buffer); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALHANDLE; + } + waveOut->HeadersQueued++; + waveOut->FramesQueued += buffer->FrameCount; + buffer->Sequence = waveOut->HeadersQueued; + if (waveOut->QueueTail) + waveOut->QueueTail->Next = buffer; + else + waveOut->QueueHead = buffer; + waveOut->QueueTail = buffer; + if (!waveOut->SubmitNode) + waveOut->SubmitNode = buffer; + WinmmUnlock(&waveOut->Lock); + (void)SysSignalEvent(waveOut->WakeEvent); + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Pauses future source consumption while preserving queued headers. +__declspec(dllexport) MMRESULT waveOutPause(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + MMRESULT result = WinmmSubmitCommand(waveOut, WinmmWaveCommandPause); + WinmmReleaseWaveOut(waveOut); + return result; +} + +// Resumes a user-paused waveform queue. +__declspec(dllexport) MMRESULT waveOutRestart(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + MMRESULT result = WinmmSubmitCommand(waveOut, WinmmWaveCommandRestart); + WinmmReleaseWaveOut(waveOut); + return result; +} + +// Stops playback and returns every queued header as completed. +__declspec(dllexport) MMRESULT waveOutReset(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + MMRESULT result = WinmmSubmitCommand(waveOut, WinmmWaveCommandReset); + WinmmReleaseWaveOut(waveOut); + return result; +} + +// Applies independent WinMM left and right volume values through native gain and pan. +__declspec(dllexport) MMRESULT waveOutSetVolume(HWAVEOUT handle, DWORD volume) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + + uint32_t left = volume & 0xFFFFu; + uint32_t right = (volume >> 16u) & 0xFFFFu; + uint32_t maximum = left > right ? left : right; + uint32_t volumeQ16 = maximum ? (uint32_t)(((uint64_t)maximum * 65536u) / 65535u) : 0u; + int32_t panQ16 = 0; + if (left > right && left) + panQ16 = -(int32_t)(((uint64_t)(left - right) * 65536u) / left); + else if (right > left && right) + panQ16 = (int32_t)(((uint64_t)(right - left) * 65536u) / right); + + SYS_AUDIO_SET_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = waveOut->NativeStream; + request.VolumeQ16 = volumeQ16; + request.PanQ16 = panQ16; + KSTATUS status = SysAudioSetVolume(&request); + if (status == NTLSTATUS_SUCCESS) + waveOut->Volume = volume; + WinmmReleaseWaveOut(waveOut); + return WinmmMapNativeStatus(status); +} + +// Returns the last WinMM left and right volume value accepted for this handle. +__declspec(dllexport) MMRESULT waveOutGetVolume(HWAVEOUT handle, DWORD* volumeOut) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!volumeOut) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + *volumeOut = waveOut->Volume; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Reports source-frame progress using the requested MMTIME unit. +__declspec(dllexport) MMRESULT waveOutGetPosition(HWAVEOUT handle, + MMTIME* timeOut, + UINT size) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!timeOut || size < sizeof(*timeOut)) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + + SYS_AUDIO_STREAM_INFO info; + KSTATUS status = WinmmQueryNativeStream(waveOut, &info); + if (status != NTLSTATUS_SUCCESS) { + WinmmReleaseWaveOut(waveOut); + return WinmmMapNativeStatus(status); + } + uint64_t frames = info.FramesPresented; + if (timeOut->wType == TIME_BYTES) { + uint64_t bytes = frames * waveOut->SourceFormat.nBlockAlign; + timeOut->u.cb = bytes > UINT32_MAX ? UINT32_MAX : (uint32_t)bytes; + } else if (timeOut->wType == TIME_SAMPLES) { + timeOut->u.sample = frames > UINT32_MAX ? UINT32_MAX : (uint32_t)frames; + } else { + uint64_t milliseconds = frames * 1000u / waveOut->SourceFormat.nSamplesPerSec; + timeOut->wType = TIME_MS; + timeOut->u.ms = milliseconds > UINT32_MAX ? UINT32_MAX : (uint32_t)milliseconds; + } + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Returns the device index selected when the waveform handle was opened. +__declspec(dllexport) MMRESULT waveOutGetID(HWAVEOUT handle, UINT* deviceIdOut) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + if (!deviceIdOut) { + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_INVALPARAM; + } + *deviceIdOut = waveOut->DeviceId; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOERROR; +} + +// Fills the minimal ANSI device capabilities supported by this personality. +__declspec(dllexport) MMRESULT waveOutGetDevCapsA(UINT deviceId, + WAVEOUTCAPSA* caps, + UINT size) { + uint64_t endpointId = 0; + uint32_t resolvedDeviceId = 0; + MMRESULT result = WinmmResolveEndpoint(deviceId, &endpointId, &resolvedDeviceId); + if (result != MMSYSERR_NOERROR) { + WinmmDebugResult("device caps rejected", NULL, result); + return result; + } + (void)endpointId; + (void)resolvedDeviceId; + if (!caps || size < sizeof(*caps)) + return MMSYSERR_INVALPARAM; + + memset(caps, 0, sizeof(*caps)); + caps->vDriverVersion = 0x00010000u; + memcpy(caps->szPname, "nt-lite analog output", sizeof("nt-lite analog output")); + caps->dwFormats = WAVE_FORMAT_1M16 | WAVE_FORMAT_1S16 | + WAVE_FORMAT_2M16 | WAVE_FORMAT_2S16 | + WAVE_FORMAT_4M16 | WAVE_FORMAT_4S16 | + WAVE_FORMAT_48M16 | WAVE_FORMAT_48S16; + caps->wChannels = 2; + caps->dwSupport = WAVECAPS_VOLUME | WAVECAPS_LRVOLUME; + return MMSYSERR_NOERROR; +} + +// Fills the minimal wide device capabilities supported by this personality. +__declspec(dllexport) MMRESULT waveOutGetDevCapsW(UINT deviceId, + WAVEOUTCAPSW* caps, + UINT size) { + if (!caps || size < sizeof(*caps)) + return MMSYSERR_INVALPARAM; + WAVEOUTCAPSA ansi; + MMRESULT result = waveOutGetDevCapsA(deviceId, &ansi, sizeof(ansi)); + if (result != MMSYSERR_NOERROR) + return result; + memset(caps, 0, sizeof(*caps)); + caps->wMid = ansi.wMid; + caps->wPid = ansi.wPid; + caps->vDriverVersion = ansi.vDriverVersion; + for (uint32_t index = 0; index < 32u && ansi.szPname[index]; index++) + caps->szPname[index] = (unsigned short)(uint8_t)ansi.szPname[index]; + caps->dwFormats = ansi.dwFormats; + caps->wChannels = ansi.wChannels; + caps->dwSupport = ansi.dwSupport; + return MMSYSERR_NOERROR; +} + +// Copies one fixed WinMM error description to an ANSI buffer. +__declspec(dllexport) MMRESULT waveOutGetErrorTextA(MMRESULT error, + char* text, + UINT characters) { + if (!text || !characters) + return MMSYSERR_INVALPARAM; + const char* message = "Unknown waveform output error"; + switch (error) { + case MMSYSERR_NOERROR: message = "No error"; break; + case MMSYSERR_BADDEVICEID: message = "Bad waveform output device ID"; break; + case MMSYSERR_INVALHANDLE: message = "Invalid waveform output handle"; break; + case MMSYSERR_NOMEM: message = "Not enough memory"; break; + case MMSYSERR_NOTSUPPORTED: message = "Operation not supported"; break; + case MMSYSERR_INVALFLAG: message = "Invalid waveform output flags"; break; + case MMSYSERR_INVALPARAM: message = "Invalid waveform output parameter"; break; + case WAVERR_BADFORMAT: message = "Unsupported waveform format"; break; + case WAVERR_STILLPLAYING: message = "Waveform buffer is still playing"; break; + case WAVERR_UNPREPARED: message = "Waveform header is not prepared"; break; + default: break; + } + uint32_t index = 0; + while (index + 1u < characters && message[index]) { + text[index] = message[index]; + index++; + } + text[index] = '\0'; + return MMSYSERR_NOERROR; +} + +// Copies one fixed WinMM error description to a wide buffer. +__declspec(dllexport) MMRESULT waveOutGetErrorTextW(MMRESULT error, + unsigned short* text, + UINT characters) { + if (!text || !characters) + return MMSYSERR_INVALPARAM; + char ansi[96]; + MMRESULT result = waveOutGetErrorTextA(error, ansi, sizeof(ansi)); + if (result != MMSYSERR_NOERROR) + return result; + uint32_t index = 0; + while (index + 1u < characters && ansi[index]) { + text[index] = (unsigned short)(uint8_t)ansi[index]; + index++; + } + text[index] = 0; + return MMSYSERR_NOERROR; +} + +// Rejects loop-breaking because looped headers are not part of this implementation. +__declspec(dllexport) MMRESULT waveOutBreakLoop(HWAVEOUT handle) { + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOTSUPPORTED; +} + +// Rejects driver-specific waveform messages instead of pretending they succeeded. +__declspec(dllexport) MMRESULT waveOutMessage(HWAVEOUT handle, + UINT message, + DWORD_PTR parameter1, + DWORD_PTR parameter2) { + (void)message; + (void)parameter1; + (void)parameter2; + WINMM_WAVEOUT* waveOut = WinmmReferenceWaveOut(handle); + if (!waveOut) + return MMSYSERR_INVALHANDLE; + WinmmReleaseWaveOut(waveOut); + return MMSYSERR_NOTSUPPORTED; +} diff --git a/code/user/runtime/heap.c b/code/user/runtime/heap.c index 39d5cc8..7af9dd0 100644 --- a/code/user/runtime/heap.c +++ b/code/user/runtime/heap.c @@ -164,7 +164,7 @@ void* _malloc(size_t size) { size = AlignSize(size); HeapLock(); - for (;;) { + while (true) { for (HEAP_SEGMENT* seg = gSegments; seg; seg = seg->Next) { for (HEAP_BLOCK* block = seg->First; block; block = block->Next) { if (block->Magic == USER_HEAP_MAGIC && block->Free && block->Size >= size) { diff --git a/code/user/runtime/native_exception.c b/code/user/runtime/native_exception.c index e3f9c5b..b9b3d0f 100644 --- a/code/user/runtime/native_exception.c +++ b/code/user/runtime/native_exception.c @@ -53,7 +53,7 @@ static void NtlpTerminateExceptionDispatch(void) { request.Disposition = SYS_EXCEPTION_DISPOSITION_TERMINATE_PROCESS; uint32_t status = SysContinueException(&request); - for (;;) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); + while (true) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); } // Asks the kernel to continue searching from the top-level native dispatcher. @@ -64,7 +64,7 @@ static void NtlpContinueExceptionSearch(void) { request.Disposition = SYS_EXCEPTION_DISPOSITION_CONTINUE_SEARCH; uint32_t status = SysContinueException(&request); - for (;;) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); + while (true) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); } // Asks the kernel to resume execution with a handler-edited user context. @@ -76,7 +76,7 @@ static void NtlpContinueExecution(SYS_USER_CONTEXT* context) { request.Context = context; uint32_t status = SysContinueException(&request); - for (;;) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); + while (true) SysExitProcess((int32_t)NtlpStatusOrFailure(status)); } // Returns whether one kernel-provided exception frame has the expected ABI shape. diff --git a/code/user/runtime/native_start.c b/code/user/runtime/native_start.c index fdcdf35..765862f 100644 --- a/code/user/runtime/native_start.c +++ b/code/user/runtime/native_start.c @@ -10,9 +10,9 @@ extern int main(void); void NtlNativeStart(void) { uint32_t status = NtlInitializeExceptionRuntime(); if (status != NTLSTATUS_SUCCESS) { - for (;;) SysExitProcess((int32_t)status); + while (true) SysExitProcess((int32_t)status); } int exitCode = main(); - for (;;) SysExitProcess(exitCode); + while (true) SysExitProcess(exitCode); } diff --git a/code/user/runtime/posix_start.c b/code/user/runtime/posix_start.c index 4615d22..5f3a196 100644 --- a/code/user/runtime/posix_start.c +++ b/code/user/runtime/posix_start.c @@ -12,11 +12,11 @@ extern void PosixFdInit(void); void PosixStart(void) { uint32_t status = NtlInitializeExceptionRuntime(); if (status != NTLSTATUS_SUCCESS) { - for (;;) SysExitProcess((int32_t)status); + while (true) SysExitProcess((int32_t)status); } PosixFdInit(); int exitCode = main(); exit(exitCode); - for (;;) {} + while (true) {} } diff --git a/code/user/syscall/syscall_nr.inc b/code/user/syscall/syscall_nr.inc index 7fa798e..4d7599a 100644 --- a/code/user/syscall/syscall_nr.inc +++ b/code/user/syscall/syscall_nr.inc @@ -10,138 +10,154 @@ SYS_KILL_PROCESS EQU 5 SYS_GET_PROCESS_LIST EQU 6 SYS_GET_PROCESS_PID EQU 7 SYS_CREATE_THREAD EQU 8 -SYS_GET_THREAD_LIST EQU 9 -SYS_LOAD_MODULE EQU 10 -SYS_UNLOAD_MODULE EQU 11 -SYS_GET_MODULE_INFO EQU 12 -SYS_GET_MODULE_EXPORT EQU 13 -SYS_GET_MODULE_LIST EQU 14 -SYS_GET_CURRENT_TID EQU 15 -SYS_GET_CURRENT_PID EQU 16 -SYS_SET_THREAD_PRIORITY EQU 17 -SYS_YIELD EQU 18 -SYS_SLEEP EQU 19 -SYS_WAIT_SINGLE EQU 20 -SYS_WAIT_MANY EQU 21 -SYS_SET_USER_APC_DISPATCHER EQU 22 -SYS_APC_RETURN EQU 23 -SYS_SET_EXCEPTION_DISPATCHER EQU 24 -SYS_GET_EXCEPTION_DISPATCHER EQU 25 -SYS_CONTINUE_EXCEPTION EQU 26 -SYS_RAISE_EXCEPTION EQU 27 +SYS_RESUME_THREAD EQU 9 +SYS_GET_THREAD_LIST EQU 10 +SYS_LOAD_MODULE EQU 11 +SYS_UNLOAD_MODULE EQU 12 +SYS_GET_MODULE_INFO EQU 13 +SYS_GET_MODULE_EXPORT EQU 14 +SYS_GET_MODULE_LIST EQU 15 +SYS_GET_CURRENT_TID EQU 16 +SYS_GET_CURRENT_PID EQU 17 +SYS_SET_THREAD_PRIORITY EQU 18 +SYS_YIELD EQU 19 +SYS_SLEEP EQU 20 +SYS_WAIT_SINGLE EQU 21 +SYS_WAIT_MANY EQU 22 +SYS_SET_USER_APC_DISPATCHER EQU 23 +SYS_APC_RETURN EQU 24 +SYS_SET_EXCEPTION_DISPATCHER EQU 25 +SYS_GET_EXCEPTION_DISPATCHER EQU 26 +SYS_CONTINUE_EXCEPTION EQU 27 +SYS_RAISE_EXCEPTION EQU 28 ; Virtual memory, mappings, and handles. -SYS_ALLOC_VM EQU 28 -SYS_FREE_VM EQU 29 -SYS_PROTECT_VM EQU 30 -SYS_QUERY_VM EQU 31 -SYS_CLOSE_HANDLE EQU 32 -SYS_DUPLICATE_HANDLE EQU 33 -SYS_CREATE_FILE_MAPPING EQU 34 -SYS_MAP_FILE EQU 35 +SYS_ALLOC_VM EQU 29 +SYS_FREE_VM EQU 30 +SYS_PROTECT_VM EQU 31 +SYS_QUERY_VM EQU 32 +SYS_CLOSE_HANDLE EQU 33 +SYS_DUPLICATE_HANDLE EQU 34 +SYS_CREATE_FILE_MAPPING EQU 35 +SYS_MAP_FILE EQU 36 ; Synchronization objects. -SYS_CREATE_MUTEX EQU 36 -SYS_TRY_ACQUIRE_MUTEX EQU 37 -SYS_RELEASE_MUTEX EQU 38 -SYS_CREATE_SEM EQU 39 -SYS_TRY_WAIT_SEM EQU 40 -SYS_SIGNAL_SEM EQU 41 -SYS_CREATE_EVENT EQU 42 -SYS_SIGNAL_EVENT EQU 43 -SYS_CLEAR_EVENT EQU 44 +SYS_CREATE_MUTEX EQU 37 +SYS_TRY_ACQUIRE_MUTEX EQU 38 +SYS_RELEASE_MUTEX EQU 39 +SYS_CREATE_SEM EQU 40 +SYS_TRY_WAIT_SEM EQU 41 +SYS_SIGNAL_SEM EQU 42 +SYS_CREATE_EVENT EQU 43 +SYS_SIGNAL_EVENT EQU 44 +SYS_CLEAR_EVENT EQU 45 ; Queues and named queues. -SYS_CREATE_QUEUE EQU 45 -SYS_SEND_QUEUE EQU 46 -SYS_RECV_QUEUE EQU 47 -SYS_PUBLISH_QUEUE EQU 48 -SYS_OPEN_NAMED_QUEUE EQU 49 -SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS EQU 50 +SYS_CREATE_QUEUE EQU 46 +SYS_SEND_QUEUE EQU 47 +SYS_RECV_QUEUE EQU 48 +SYS_PUBLISH_QUEUE EQU 49 +SYS_OPEN_NAMED_QUEUE EQU 50 +SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS EQU 51 ; Files, directories, and current directory. -SYS_OPEN_FILE EQU 51 -SYS_READ_FILE EQU 52 -SYS_WRITE_FILE EQU 53 -SYS_READ_FILE_ASYNC EQU 54 -SYS_WRITE_FILE_ASYNC EQU 55 -SYS_CANCEL_IO EQU 56 -SYS_SEEK_FILE EQU 57 -SYS_GET_FILE_POS EQU 58 -SYS_GET_FILE_SIZE EQU 59 -SYS_GET_FILE_STAT EQU 60 -SYS_STAT_FILE EQU 61 -SYS_TRUNCATE_FILE EQU 62 -SYS_FLUSH_FILE EQU 63 -SYS_DELETE_FILE EQU 64 -SYS_RENAME_FILE EQU 65 -SYS_LOCK_FILE EQU 66 -SYS_UNLOCK_FILE EQU 67 -SYS_OPEN_DIR EQU 68 -SYS_READ_DIR EQU 69 -SYS_CREATE_DIR EQU 70 -SYS_REMOVE_DIR EQU 71 -SYS_SET_CWD EQU 72 -SYS_REAL_PATH EQU 73 +SYS_OPEN_FILE EQU 52 +SYS_READ_FILE EQU 53 +SYS_WRITE_FILE EQU 54 +SYS_READ_FILE_ASYNC EQU 55 +SYS_WRITE_FILE_ASYNC EQU 56 +SYS_CANCEL_IO EQU 57 +SYS_SEEK_FILE EQU 58 +SYS_GET_FILE_POS EQU 59 +SYS_GET_FILE_SIZE EQU 60 +SYS_GET_FILE_STAT EQU 61 +SYS_STAT_FILE EQU 62 +SYS_TRUNCATE_FILE EQU 63 +SYS_FLUSH_FILE EQU 64 +SYS_DELETE_FILE EQU 65 +SYS_RENAME_FILE EQU 66 +SYS_LOCK_FILE EQU 67 +SYS_UNLOCK_FILE EQU 68 +SYS_OPEN_DIR EQU 69 +SYS_READ_DIR EQU 70 +SYS_CREATE_DIR EQU 71 +SYS_REMOVE_DIR EQU 72 +SYS_SET_CWD EQU 73 +SYS_REAL_PATH EQU 74 ; Console, display, and surfaces. -SYS_WRITE_CONSOLE EQU 74 -SYS_GET_CONSOLE_INFO EQU 75 -SYS_SET_CONSOLE_CURSOR EQU 76 -SYS_FILL_CONSOLE_CHARACTER EQU 77 -SYS_FILL_CONSOLE_ATTRIBUTE EQU 78 -SYS_PUT_SCREEN_CHAR EQU 79 -SYS_CLEAR_SCREEN EQU 80 -SYS_GET_SCREEN_SIZE EQU 81 -SYS_QUERY_DISPLAY_INFO EQU 82 -SYS_OPEN_DISPLAY_CLOCK EQU 83 -SYS_CREATE_SURFACE EQU 84 -SYS_MAP_SURFACE EQU 85 -SYS_QUERY_SURFACE EQU 86 -SYS_PRESENT_SURFACE EQU 87 +SYS_WRITE_CONSOLE EQU 75 +SYS_GET_CONSOLE_INFO EQU 76 +SYS_SET_CONSOLE_CURSOR EQU 77 +SYS_FILL_CONSOLE_CHARACTER EQU 78 +SYS_FILL_CONSOLE_ATTRIBUTE EQU 79 +SYS_PUT_SCREEN_CHAR EQU 80 +SYS_CLEAR_SCREEN EQU 81 +SYS_GET_SCREEN_SIZE EQU 82 +SYS_QUERY_DISPLAY_INFO EQU 83 +SYS_OPEN_DISPLAY_CLOCK EQU 84 +SYS_CREATE_SURFACE EQU 85 +SYS_MAP_SURFACE EQU 86 +SYS_QUERY_SURFACE EQU 87 +SYS_PRESENT_SURFACE EQU 88 + +; Native render audio. +SYS_AUDIO_ENUMERATE_ENDPOINTS EQU 89 +SYS_AUDIO_OPEN_STREAM EQU 90 +SYS_AUDIO_WRITE EQU 91 +SYS_AUDIO_ARM_WAIT EQU 92 +SYS_AUDIO_QUERY_STREAM EQU 93 +SYS_AUDIO_START EQU 94 +SYS_AUDIO_PAUSE EQU 95 +SYS_AUDIO_RESUME EQU 96 +SYS_AUDIO_DRAIN EQU 97 +SYS_AUDIO_STOP EQU 98 +SYS_AUDIO_RESET EQU 99 +SYS_AUDIO_SET_VOLUME EQU 100 +SYS_AUDIO_SET_MASTER_VOLUME EQU 101 ; Input. -SYS_POLL_KEYBOARD EQU 88 -SYS_IS_KEY_DOWN EQU 89 -SYS_READ_KEYBOARD_CHAR EQU 90 -SYS_READ_KEY EQU 91 -SYS_GET_KEY_MODIFIERS EQU 92 -SYS_QUERY_GLOBAL_INPUT_STATE EQU 93 -SYS_OPEN_INPUT_QUEUE EQU 94 -SYS_READ_INPUT_EVENT EQU 95 -SYS_PEEK_INPUT_EVENT EQU 96 -SYS_PEEK_MOUSE EQU 97 -SYS_READ_MOUSE EQU 98 -SYS_GET_MOUSE_POS EQU 99 +SYS_POLL_KEYBOARD EQU 102 +SYS_IS_KEY_DOWN EQU 103 +SYS_READ_KEYBOARD_CHAR EQU 104 +SYS_READ_KEY EQU 105 +SYS_GET_KEY_MODIFIERS EQU 106 +SYS_QUERY_GLOBAL_INPUT_STATE EQU 107 +SYS_OPEN_INPUT_QUEUE EQU 108 +SYS_READ_INPUT_EVENT EQU 109 +SYS_PEEK_INPUT_EVENT EQU 110 +SYS_PEEK_MOUSE EQU 111 +SYS_READ_MOUSE EQU 112 +SYS_GET_MOUSE_POS EQU 113 ; Time, debug, and system information. -SYS_CREATE_TIMER EQU 100 -SYS_SET_TIMER EQU 101 -SYS_CANCEL_TIMER EQU 102 -SYS_GET_TICK EQU 103 -SYS_GET_TICK_FREQUENCY EQU 104 -SYS_GET_SYSTEM_TIME EQU 105 -SYS_PRINT_DEBUG EQU 106 -SYS_QUERY_TRAP_FRAME EQU 107 -SYS_GET_MEMORY_STATUS EQU 108 -SYS_QUERY_KERNEL_STATS EQU 109 +SYS_CREATE_TIMER EQU 114 +SYS_SET_TIMER EQU 115 +SYS_CANCEL_TIMER EQU 116 +SYS_GET_TICK EQU 117 +SYS_GET_TICK_FREQUENCY EQU 118 +SYS_GET_SYSTEM_TIME EQU 119 +SYS_PRINT_DEBUG EQU 120 +SYS_QUERY_TRAP_FRAME EQU 121 +SYS_GET_MEMORY_STATUS EQU 122 +SYS_QUERY_KERNEL_STATS EQU 123 ; Native sockets and polling. -SYS_CREATE_SOCKET EQU 110 -SYS_SOCKET_IO_ASYNC EQU 111 -SYS_BIND_SOCKET EQU 112 -SYS_LISTEN_SOCKET EQU 113 -SYS_ACCEPT_SOCKET EQU 114 -SYS_CONNECT_SOCKET EQU 115 -SYS_SEND_SOCKET EQU 116 -SYS_RECV_SOCKET EQU 117 -SYS_SHUTDOWN_SOCKET EQU 118 -SYS_SEND_TO_SOCKET EQU 119 -SYS_RECV_FROM_SOCKET EQU 120 -SYS_RESOLVE_HOST EQU 121 -SYS_SET_SOCKET_OPTION EQU 122 -SYS_GET_SOCKET_OPTION EQU 123 -SYS_POLL EQU 124 -SYS_UNPUBLISH_QUEUE EQU 125 +SYS_CREATE_SOCKET EQU 124 +SYS_SOCKET_IO_ASYNC EQU 125 +SYS_BIND_SOCKET EQU 126 +SYS_LISTEN_SOCKET EQU 127 +SYS_ACCEPT_SOCKET EQU 128 +SYS_CONNECT_SOCKET EQU 129 +SYS_SEND_SOCKET EQU 130 +SYS_RECV_SOCKET EQU 131 +SYS_SHUTDOWN_SOCKET EQU 132 +SYS_SEND_TO_SOCKET EQU 133 +SYS_RECV_FROM_SOCKET EQU 134 +SYS_RESOLVE_HOST EQU 135 +SYS_SET_SOCKET_OPTION EQU 136 +SYS_GET_SOCKET_OPTION EQU 137 +SYS_POLL EQU 138 +SYS_UNPUBLISH_QUEUE EQU 139 -SYS_MAX EQU 126 +SYS_MAX EQU 140 diff --git a/code/user/syscall/syscall_stubs.asm b/code/user/syscall/syscall_stubs.asm index 87dc71c..b0983ca 100644 --- a/code/user/syscall/syscall_stubs.asm +++ b/code/user/syscall/syscall_stubs.asm @@ -15,11 +15,6 @@ ; SYSCALL_STUB2/3 - 4 or fewer args: pure register passing, no stack read. ; SYSCALL_STUB5 - exactly 5 args: loads [rsp+28h] -> R10 only. ; SYSCALL_STUB6 - 5 or 6 args: loads [rsp+28h] -> R10 and [rsp+30h] -> R11. -; -; Handle lifecycle: -; SysCloseHandle is the single close stub for every object type. -; There is no SysCloseFile - files are closed through the unified path, -; exactly like mutexes, semaphores, events, and queues. INCLUDE syscall_nr.inc @@ -97,6 +92,7 @@ SYSCALL_STUB2 GetModuleExport, SYS_GET_MODULE_EXPORT SYSCALL_STUB2 GetModuleList, SYS_GET_MODULE_LIST SYSCALL_STUB2 CreateThread, SYS_CREATE_THREAD +SYSCALL_STUB2 ResumeThread, SYS_RESUME_THREAD SYSCALL_STUB2 GetCurrentTid, SYS_GET_CURRENT_TID SYSCALL_STUB2 GetCurrentPid, SYS_GET_CURRENT_PID SYSCALL_STUB2 SetThreadPriority, SYS_SET_THREAD_PRIORITY @@ -218,7 +214,24 @@ SYSCALL_STUB2 PresentSurface, SYS_PRESENT_SURFACE SYSCALL_STUB2 ClearScreen, SYS_CLEAR_SCREEN SYSCALL_STUB2 GetScreenSize, SYS_GET_SCREEN_SIZE SYSCALL_STUB2 QueryDisplayInfo, SYS_QUERY_DISPLAY_INFO -SYSCALL_STUB2 OpenDisplayClock, SYS_OPEN_DISPLAY_CLOCK +SYSCALL_STUB2 OpenDisplayClock, SYS_OPEN_DISPLAY_CLOCK + +; --------------------------------------------------------------------------- +; Native render audio +; --------------------------------------------------------------------------- +SYSCALL_STUB2 AudioEnumerateEndpoints, SYS_AUDIO_ENUMERATE_ENDPOINTS +SYSCALL_STUB2 AudioOpenStream, SYS_AUDIO_OPEN_STREAM +SYSCALL_STUB2 AudioWrite, SYS_AUDIO_WRITE +SYSCALL_STUB2 AudioArmWait, SYS_AUDIO_ARM_WAIT +SYSCALL_STUB2 AudioQueryStream, SYS_AUDIO_QUERY_STREAM +SYSCALL_STUB2 AudioStart, SYS_AUDIO_START +SYSCALL_STUB2 AudioPause, SYS_AUDIO_PAUSE +SYSCALL_STUB2 AudioResume, SYS_AUDIO_RESUME +SYSCALL_STUB2 AudioDrain, SYS_AUDIO_DRAIN +SYSCALL_STUB2 AudioStop, SYS_AUDIO_STOP +SYSCALL_STUB2 AudioReset, SYS_AUDIO_RESET +SYSCALL_STUB2 AudioSetVolume, SYS_AUDIO_SET_VOLUME +SYSCALL_STUB2 AudioSetMasterVolume, SYS_AUDIO_SET_MASTER_VOLUME ; --------------------------------------------------------------------------- ; Input diff --git a/code/user/tests/all_tests/all_tests.c b/code/user/tests/all_tests/all_tests.c index d53c95e..4c69636 100644 --- a/code/user/tests/all_tests/all_tests.c +++ b/code/user/tests/all_tests/all_tests.c @@ -57,7 +57,7 @@ static void print_hex32(uint32_t v) { // Exits the process without returning to the runtime. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Converts ASCII to lowercase for stable case-insensitive comparisons. @@ -69,7 +69,7 @@ static char ascii_lower(char c) { // Compares two names case-insensitively, with a case-sensitive tie-breaker. static int compare_names(const char* a, const char* b) { size_t i = 0; - for (;;) { + while (true) { char ca = ascii_lower(a[i]); char cb = ascii_lower(b[i]); if (ca != cb) return (unsigned char)ca < (unsigned char)cb ? -1 : 1; @@ -219,7 +219,7 @@ static uint32_t collect_tests(const char* dir, const char* selfName, TEST_ENTRY* uint32_t count = 0; uint32_t capacity = 0; - for (;;) { + while (true) { SYS_DIR_ENTRY entry; status = SysReadDir(hDir, &entry); if (status != NTLSTATUS_SUCCESS) break; diff --git a/code/user/tests/audio_stream_smoke/audio_stream_smoke.c b/code/user/tests/audio_stream_smoke/audio_stream_smoke.c new file mode 100644 index 0000000..7c77a15 --- /dev/null +++ b/code/user/tests/audio_stream_smoke/audio_stream_smoke.c @@ -0,0 +1,729 @@ +// Native render-stream syscall smoke test. + +#include +#include +#include + +#define TEST_WAIT_MILLISECONDS 3000ull +#define TEST_EXTRA_FRAMES 512u +#define TEST_REFILL_FRAMES 1024u +#define TEST_TONE_FRAMES 48000u +#define TEST_AUDIBLE_FRAMES SYS_AUDIO_STREAM_CAPACITY_FRAMES +#define TEST_AUDIBLE_RAMP_FRAMES 240u +#define TEST_CLOSE_RUNDOWN_ITERATIONS 72u +#define TEST_CLOSE_RUNDOWN_SLEEP_MILLISECONDS 5ull +#define TEST_IDLE_QUIESCE_MILLISECONDS 300ull +#define TEST_TONE_PHASE_MODULUS 1200u +#define TEST_TONE_PHASE_STEP 11u +#define TEST_TONE_QUARTER_PHASE 300u +#define TEST_TONE_PEAK 8192 + +static int16_t gFrames[TEST_TONE_FRAMES * 2u]; +static int16_t gAudibleFrames[TEST_AUDIBLE_FRAMES * 2u]; +static uint32_t gOpenStream; + +typedef char TEST_AUDIO_FORMAT_SIZE[(sizeof(SYS_AUDIO_FORMAT) == 8) ? 1 : -1]; +typedef char TEST_AUDIO_ENDPOINT_INFO_SIZE[(sizeof(SYS_AUDIO_ENDPOINT_INFO) == 64) ? 1 : -1]; +typedef char TEST_AUDIO_ENUMERATE_REQUEST_SIZE[(sizeof(SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST) == 32) ? 1 : -1]; +typedef char TEST_AUDIO_OPEN_REQUEST_SIZE[(sizeof(SYS_AUDIO_OPEN_STREAM_REQUEST) == 40) ? 1 : -1]; +typedef char TEST_AUDIO_WRITE_REQUEST_SIZE[(sizeof(SYS_AUDIO_WRITE_REQUEST) == 32) ? 1 : -1]; +typedef char TEST_AUDIO_ARM_WAIT_REQUEST_SIZE[(sizeof(SYS_AUDIO_ARM_WAIT_REQUEST) == 56) ? 1 : -1]; +typedef char TEST_AUDIO_STREAM_INFO_SIZE[(sizeof(SYS_AUDIO_STREAM_INFO) == 128) ? 1 : -1]; +typedef char TEST_AUDIO_VOLUME_REQUEST_SIZE[(sizeof(SYS_AUDIO_SET_VOLUME_REQUEST) == 24) ? 1 : -1]; +typedef char TEST_AUDIO_MASTER_VOLUME_REQUEST_SIZE[(sizeof(SYS_AUDIO_SET_MASTER_VOLUME_REQUEST) == 24) ? 1 : -1]; + +// Writes one NUL-terminated string to the console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Writes one unsigned integer to the console. +static void print_u64(uint64_t value) { + char digits[32]; + uint32_t count = 0; + if (!value) { + print("0"); + return; + } + while (value && count < sizeof(digits)) { + digits[count++] = (char)('0' + value % 10u); + value /= 10u; + } + while (count) { + char character = digits[--count]; + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromParts(&character, 1) }; + (void)SysWriteConsole(&request); + } +} + +// Stops and closes the canonical stream after success, failure, or process interruption. +static void cleanup_stream(void) { + if (!gOpenStream) + return; + (void)SysAudioStop(gOpenStream); + (void)SysCloseHandle(gOpenStream); + gOpenStream = 0; +} + +// Terminates the test process. +static void exit_now(int32_t code) { + cleanup_stream(); + while (true) (void)SysExitProcess(code); +} + +// Fails the test with one status value. +static void fail_status(const char* operation, KSTATUS status) { + print("audio_stream_smoke: "); + print(operation); + print(" status="); + print_u64(status); + print("\n"); + exit_now(1); +} + +// Fails the test with one plain diagnostic. +static void fail(const char* message) { + print("audio_stream_smoke: "); + print(message); + print("\n"); + exit_now(1); +} + +// Returns one exact 440 Hz integer triangle-wave sample for the supplied phase. +static int16_t tone_sample(uint32_t phase) { + int32_t sample; + if (phase < TEST_TONE_QUARTER_PHASE) { + sample = (int32_t)(phase * TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } else if (phase < TEST_TONE_QUARTER_PHASE * 2u) { + sample = (int32_t)((TEST_TONE_QUARTER_PHASE * 2u - phase) * + TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } else if (phase < TEST_TONE_QUARTER_PHASE * 3u) { + sample = -(int32_t)((phase - TEST_TONE_QUARTER_PHASE * 2u) * + TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } else { + sample = -(int32_t)((TEST_TONE_PHASE_MODULUS - phase) * + TEST_TONE_PEAK / TEST_TONE_QUARTER_PHASE); + } + return (int16_t)sample; +} + +// Fills contract and audible buffers without requiring floating-point support. +static void fill_test_frames(void) { + uint32_t phase = 0; + for (uint32_t frame = 0; frame < TEST_TONE_FRAMES; frame++) { + int16_t sample = tone_sample(phase); + gFrames[frame * 2u] = sample; + gFrames[frame * 2u + 1u] = sample; + phase += TEST_TONE_PHASE_STEP; + if (phase >= TEST_TONE_PHASE_MODULUS) + phase -= TEST_TONE_PHASE_MODULUS; + } + + phase = 0; + for (uint32_t frame = 0; frame < TEST_AUDIBLE_FRAMES; frame++) { + int32_t sample = tone_sample(phase); + uint32_t gain = TEST_AUDIBLE_RAMP_FRAMES; + if (frame < gain) + gain = frame; + uint32_t remaining = TEST_AUDIBLE_FRAMES - 1u - frame; + if (remaining < gain) + gain = remaining; + sample = sample * (int32_t)gain / (int32_t)TEST_AUDIBLE_RAMP_FRAMES; + gAudibleFrames[frame * 2u] = (int16_t)sample; + gAudibleFrames[frame * 2u + 1u] = (int16_t)sample; + phase += TEST_TONE_PHASE_STEP; + if (phase >= TEST_TONE_PHASE_MODULUS) + phase -= TEST_TONE_PHASE_MODULUS; + } +} + +// Queries one stream with the required public structure size. +static SYS_AUDIO_STREAM_INFO query_stream(uint32_t stream) { + SYS_AUDIO_STREAM_INFO info; + memset(&info, 0, sizeof(info)); + info.Size = sizeof(info); + KSTATUS status = SysAudioQueryStream(stream, &info); + if (status != NTLSTATUS_SUCCESS) + fail_status("query stream", status); + if (info.Version != SYS_AUDIO_ABI_VERSION) + fail("unexpected audio ABI version"); + return info; +} + +// Applies one valid stream volume, pan, and mute setting. +static void set_stream_mix(uint32_t stream, + uint32_t volumeQ16, + int32_t panQ16, + uint32_t muted) { + SYS_AUDIO_SET_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = stream; + request.Muted = muted; + request.VolumeQ16 = volumeQ16; + request.PanQ16 = panQ16; + KSTATUS status = SysAudioSetVolume(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("set stream mix", status); +} + +// Selects the available default endpoint, or the first available endpoint. +static const SYS_AUDIO_ENDPOINT_INFO* select_endpoint(const SYS_AUDIO_ENDPOINT_INFO* endpoints, + uint32_t count) { + const SYS_AUDIO_ENDPOINT_INFO* fallback = 0; + for (uint32_t index = 0; index < count; index++) { + const SYS_AUDIO_ENDPOINT_INFO* endpoint = &endpoints[index]; + if (endpoint->Size != sizeof(*endpoint) || + endpoint->Version != SYS_AUDIO_ABI_VERSION || + endpoint->State != SysAudioEndpointReady || + (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE) == 0) + continue; + if (!fallback) fallback = endpoint; + if (endpoint->Flags & SYS_AUDIO_ENDPOINT_FLAG_DEFAULT) + return endpoint; + } + return fallback; +} + +// Queries one endpoint by its stable identifier. +static SYS_AUDIO_ENDPOINT_INFO query_endpoint(uint64_t endpointId) { + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + uint32_t count = 0; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.Endpoints = endpoints; + request.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + request.CountOut = &count; + KSTATUS status = SysAudioEnumerateEndpoints(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("query endpoint", status); + + for (uint32_t index = 0; index < count; index++) { + if (endpoints[index].EndpointId == endpointId) + return endpoints[index]; + } + fail("endpoint disappeared during control test"); + return endpoints[0]; +} + +// Verifies endpoint master volume, mute, validation, and restoration. +static void test_master_volume(const SYS_AUDIO_ENDPOINT_INFO* endpoint) { + SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.EndpointId = endpoint->EndpointId; + request.Muted = 1u; + request.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY / 2u; + + KSTATUS status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("set master volume", status); + SYS_AUDIO_ENDPOINT_INFO updated = query_endpoint(endpoint->EndpointId); + if (updated.MasterVolumeQ16 != SYS_AUDIO_GAIN_Q16_UNITY / 2u || + updated.MasterMuted != 1u) + fail("master volume snapshot mismatch"); + + request.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY + 1u; + status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_INVALID_PARAMETER) + fail_status("reject master volume above unity", status); + + request.Muted = endpoint->MasterMuted; + request.VolumeQ16 = endpoint->MasterVolumeQ16; + status = SysAudioSetMasterVolume(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("restore master volume", status); +} + +// Opens one stream format and checks the expected status. +static uint32_t open_stream_format(uint64_t endpointId, + uint32_t sampleRate, + uint16_t channelCount, + uint16_t sampleFormat, + KSTATUS expectedStatus) { + uint32_t stream = 0; + SYS_AUDIO_OPEN_STREAM_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.EndpointId = endpointId; + request.Format.SampleRate = sampleRate; + request.Format.ChannelCount = channelCount; + request.Format.SampleFormat = sampleFormat; + request.StreamHandleOut = &stream; + KSTATUS status = SysAudioOpenStream(&request); + if (status != expectedStatus) + fail_status("open stream format", status); + if (status == NTLSTATUS_SUCCESS && !stream) + fail("open returned a null handle"); + if (status != NTLSTATUS_SUCCESS && stream) + fail("rejected format returned a handle"); + return stream; +} + +// Proves the complete accepted and rejected source-format surface. +static void test_format_contract(uint64_t endpointId) { + static const uint32_t rates[] = { 11025u, 22050u, 44100u, 48000u }; + for (uint32_t rateIndex = 0; rateIndex < sizeof(rates) / sizeof(rates[0]); rateIndex++) { + for (uint16_t channels = 1; channels <= 2; channels++) { + uint32_t stream = open_stream_format(endpointId, + rates[rateIndex], + channels, + SysAudioSampleS16, + NTLSTATUS_SUCCESS); + KSTATUS status = SysCloseHandle(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("close format stream", status); + } + } + + (void)open_stream_format(endpointId, 48000u, 2, 2, NTLSTATUS_NOT_SUPPORTED); + (void)open_stream_format(endpointId, 32000u, 2, SysAudioSampleS16, NTLSTATUS_NOT_SUPPORTED); + (void)open_stream_format(endpointId, 48000u, 3, SysAudioSampleS16, NTLSTATUS_NOT_SUPPORTED); +} + +// Opens the canonical test stream. +static uint32_t open_stream(uint64_t endpointId) { + return open_stream_format(endpointId, + 48000u, + 2, + SysAudioSampleS16, + NTLSTATUS_SUCCESS); +} + +// Writes one sample run and returns the accepted frame count. +static uint32_t write_samples(uint32_t stream, + const int16_t* frames, + uint32_t frameCount, + KSTATUS expectedStatus) { + uint32_t written = 0xFFFFFFFFu; + SYS_AUDIO_WRITE_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = stream; + request.FrameCount = frameCount; + request.Frames = frameCount ? frames : 0; + request.WrittenFramesOut = &written; + KSTATUS status = SysAudioWrite(&request); + if (status != expectedStatus) fail_status("write stream", status); + if (status != NTLSTATUS_SUCCESS && written != 0) + fail("failed write reported accepted frames"); + return written; +} + +// Writes one contract-tone range and returns the accepted frame count. +static uint32_t write_frames(uint32_t stream, + uint32_t frameOffset, + uint32_t frameCount, + KSTATUS expectedStatus) { + if (frameOffset > TEST_TONE_FRAMES || + frameCount > TEST_TONE_FRAMES - frameOffset) + fail("test write range exceeded the tone buffer"); + return write_samples(stream, + frameCount ? gFrames + (size_t)frameOffset * 2u : 0, + frameCount, + expectedStatus); +} + +// Arms one exact set of stream progress targets. +static void arm_wait(uint32_t stream, + uint32_t mask, + uint32_t writableFrames, + uint64_t consumedFrames, + uint64_t presentedFrames, + uint64_t drainedGeneration, + uint64_t discontinuityGeneration) { + SYS_AUDIO_ARM_WAIT_REQUEST request; + memset(&request, 0, sizeof(request)); + request.Size = sizeof(request); + request.StreamHandle = stream; + request.Mask = mask; + request.WritableFrames = writableFrames; + request.ConsumedFrames = consumedFrames; + request.PresentedFrames = presentedFrames; + request.DrainedGeneration = drainedGeneration; + request.DiscontinuityGeneration = discontinuityGeneration; + KSTATUS status = SysAudioArmWait(&request); + if (status != NTLSTATUS_SUCCESS) + fail_status("arm wait", status); +} + +// Waits once and converts timeout or alert completion into a focused failure. +static void wait_stream(uint32_t stream, const char* operation) { + KSTATUS status = SysWaitSingle(stream, SYS_WAIT_FLAG_NONE, TEST_WAIT_MILLISECONDS); + if (status != NTLSTATUS_SUCCESS) + fail_status(operation, status); +} + +// Proves that closing the final handle retires an active stream with queued periods. +static void test_active_close_rundown(uint64_t endpointId) { + for (uint32_t iteration = 0; + iteration < TEST_CLOSE_RUNDOWN_ITERATIONS; + iteration++) { + uint32_t stream = open_stream(endpointId); + gOpenStream = stream; + set_stream_mix(stream, + SYS_AUDIO_GAIN_Q16_UNITY, + SYS_AUDIO_PAN_Q16_CENTER, + 1u); + uint32_t written = write_frames(stream, + 0, + SYS_AUDIO_STREAM_CAPACITY_FRAMES, + NTLSTATUS_SUCCESS); + if (written != SYS_AUDIO_STREAM_CAPACITY_FRAMES) + fail("active-close prefill was partial"); + + SYS_AUDIO_STREAM_INFO before = query_stream(stream); + arm_wait(stream, + SYS_AUDIO_WAIT_CONSUMED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + before.FramesConsumed + TEST_EXTRA_FRAMES, + before.FramesPresented, + before.DrainedGeneration, + before.DiscontinuityGeneration); + KSTATUS status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("active-close start", status); + wait_stream(stream, "active-close consumed wait"); + + SYS_AUDIO_STREAM_INFO active = query_stream(stream); + if (active.State == SysAudioStreamFailed) + fail_status("active-close stream failure", active.FailureStatus); + if (active.DiscontinuityGeneration != before.DiscontinuityGeneration || + active.FramesConsumed < before.FramesConsumed + TEST_EXTRA_FRAMES) + fail("active-close progress mismatch"); + + status = SysCloseHandle(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("active-close final handle", status); + gOpenStream = 0; + status = SysSleep(TEST_CLOSE_RUNDOWN_SLEEP_MILLISECONDS); + if (status != NTLSTATUS_SUCCESS) + fail_status("active-close rundown sleep", status); + } +} + +// Plays one bounded audible tone without depending on real-time user refills. +static void play_continuous_tone(uint32_t stream) { + KSTATUS status = SysAudioReset(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("audible reset", status); + set_stream_mix(stream, + SYS_AUDIO_GAIN_Q16_UNITY, + SYS_AUDIO_PAN_Q16_CENTER, + 0u); + + SYS_AUDIO_STREAM_INFO baseline = query_stream(stream); + uint32_t submitted = write_samples(stream, + gAudibleFrames, + TEST_AUDIBLE_FRAMES, + NTLSTATUS_SUCCESS); + if (submitted != TEST_AUDIBLE_FRAMES) + fail("audible prefill was partial"); + + status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("audible start", status); + status = SysAudioDrain(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("audible drain", status); + print("audio_stream_smoke: audible 440 Hz tone begin\n"); + + arm_wait(stream, + SYS_AUDIO_WAIT_DRAINED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + baseline.FramesConsumed, + baseline.FramesPresented, + baseline.DrainedGeneration, + baseline.DiscontinuityGeneration); + wait_stream(stream, "audible drained wait"); + + SYS_AUDIO_STREAM_INFO drained = query_stream(stream); + if (drained.State == SysAudioStreamFailed) + fail_status("audible stream failure", drained.FailureStatus); + if (drained.State != SysAudioStreamStopped || + drained.DrainedGeneration != baseline.DrainedGeneration + 1u || + drained.DiscontinuityGeneration != baseline.DiscontinuityGeneration || + drained.FramesSubmitted != TEST_AUDIBLE_FRAMES || + drained.FramesConsumed != TEST_AUDIBLE_FRAMES || + drained.FramesPresented != TEST_AUDIBLE_FRAMES || + drained.BufferedFrames != 0 || + drained.Underruns != 0 || + drained.SilenceFramesInserted != 0) + fail("audible tone completion mismatch"); + print("audio_stream_smoke: audible 440 Hz tone complete\n"); +} + +// Runs the complete native stream contract against one endpoint. +static void test_stream(const SYS_AUDIO_ENDPOINT_INFO* endpoint) { + test_format_contract(endpoint->EndpointId); + test_active_close_rundown(endpoint->EndpointId); + + uint32_t stream = open_stream(endpoint->EndpointId); + gOpenStream = stream; + KSTATUS status; + SYS_AUDIO_STREAM_INFO info = query_stream(stream); + if (info.State != SysAudioStreamCreated || + info.CapacityFrames != SYS_AUDIO_STREAM_CAPACITY_FRAMES || + info.Format.SampleFormat != SysAudioSampleS16) + fail("initial stream snapshot mismatch"); + + SYS_AUDIO_SET_VOLUME_REQUEST volume; + memset(&volume, 0, sizeof(volume)); + volume.Size = sizeof(volume); + volume.StreamHandle = stream; + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY / 2u; + volume.PanQ16 = SYS_AUDIO_PAN_Q16_CENTER; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_SUCCESS) + fail_status("set volume", status); + info = query_stream(stream); + if (info.VolumeQ16 != SYS_AUDIO_GAIN_Q16_UNITY / 2u || + info.PanQ16 != SYS_AUDIO_PAN_Q16_CENTER || info.Muted) + fail("volume snapshot mismatch"); + + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY + 1u; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_INVALID_PARAMETER) + fail_status("reject volume above unity", status); + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY / 2u; + volume.PanQ16 = SYS_AUDIO_PAN_Q16_RIGHT + 1; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_INVALID_PARAMETER) + fail_status("reject pan outside range", status); + + volume.VolumeQ16 = SYS_AUDIO_GAIN_Q16_UNITY; + volume.PanQ16 = SYS_AUDIO_PAN_Q16_CENTER; + volume.Muted = 1u; + status = SysAudioSetVolume(&volume); + if (status != NTLSTATUS_SUCCESS) + fail_status("mute control stream", status); + + uint32_t written = write_frames(stream, 0, 0, NTLSTATUS_SUCCESS); + if (written != 0) fail("zero-length write accepted frames"); + written = write_frames(stream, + 0, + SYS_AUDIO_STREAM_CAPACITY_FRAMES + TEST_EXTRA_FRAMES, + NTLSTATUS_SUCCESS); + if (written != SYS_AUDIO_STREAM_CAPACITY_FRAMES) + fail("partial write did not fill exactly one stream ring"); + written = write_frames(stream, 0, 1, NTLSTATUS_WOULD_BLOCK); + if (written != 0) fail("full-ring write accepted a frame"); + + info = query_stream(stream); + arm_wait(stream, + SYS_AUDIO_WAIT_WRITABLE | + SYS_AUDIO_WAIT_CONSUMED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + TEST_EXTRA_FRAMES, + info.FramesConsumed + TEST_EXTRA_FRAMES, + 0, + info.DrainedGeneration, + info.DiscontinuityGeneration); + status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("start", status); + wait_stream(stream, "writable wait"); + + SYS_AUDIO_STREAM_INFO progressed = query_stream(stream); + if (progressed.State == SysAudioStreamFailed) + fail_status("stream failure", progressed.FailureStatus); + if (progressed.DiscontinuityGeneration != info.DiscontinuityGeneration) + fail("unexpected discontinuity before writable target"); + if (progressed.WritableFrames < TEST_EXTRA_FRAMES || + progressed.FramesConsumed < info.FramesConsumed + TEST_EXTRA_FRAMES) + fail("writable or consumed wait did not observe its target"); + + status = SysAudioPause(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("pause", status); + if (query_stream(stream).State != SysAudioStreamPaused) + fail("pause state mismatch"); + status = SysAudioResume(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("resume", status); + + uint32_t submitted = SYS_AUDIO_STREAM_CAPACITY_FRAMES; + written = write_frames(stream, submitted, TEST_EXTRA_FRAMES, NTLSTATUS_SUCCESS); + if (written != TEST_EXTRA_FRAMES) + fail("resume refill was partial"); + submitted += written; + + while (submitted < TEST_TONE_FRAMES) { + SYS_AUDIO_STREAM_INFO refill = query_stream(stream); + if (refill.State == SysAudioStreamFailed) + fail_status("stream failure", refill.FailureStatus); + if (refill.DiscontinuityGeneration != info.DiscontinuityGeneration) + fail("unexpected discontinuity during muted control refill"); + + uint32_t remaining = TEST_TONE_FRAMES - submitted; + uint32_t refillFrames = remaining < TEST_REFILL_FRAMES + ? remaining + : TEST_REFILL_FRAMES; + if (refill.WritableFrames < refillFrames) { + arm_wait(stream, + SYS_AUDIO_WAIT_WRITABLE | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + refillFrames, + refill.FramesConsumed, + refill.FramesPresented, + refill.DrainedGeneration, + info.DiscontinuityGeneration); + wait_stream(stream, "muted control refill wait"); + continue; + } + + written = write_frames(stream, submitted, refillFrames, NTLSTATUS_SUCCESS); + if (written != refillFrames) + fail("muted control refill was partial despite reserved capacity"); + submitted += written; + } + + SYS_AUDIO_STREAM_INFO beforeDrain = query_stream(stream); + if (beforeDrain.FramesSubmitted != TEST_TONE_FRAMES) + fail("muted control frame submission mismatch"); + uint64_t presentedTarget = beforeDrain.FramesSubmitted; + status = SysAudioDrain(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("drain", status); + arm_wait(stream, + SYS_AUDIO_WAIT_PRESENTED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + beforeDrain.FramesConsumed, + presentedTarget, + beforeDrain.DrainedGeneration, + beforeDrain.DiscontinuityGeneration); + wait_stream(stream, "presentation wait"); + + SYS_AUDIO_STREAM_INFO presented = query_stream(stream); + if (presented.State == SysAudioStreamFailed) + fail_status("stream failure", presented.FailureStatus); + if (presented.DiscontinuityGeneration != beforeDrain.DiscontinuityGeneration) + fail("unexpected discontinuity before presentation target"); + if (presented.FramesPresented < presentedTarget) + fail("presented wait did not observe its target"); + + if (presented.State != SysAudioStreamStopped) { + arm_wait(stream, + SYS_AUDIO_WAIT_DRAINED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + presented.FramesConsumed, + presented.FramesPresented, + beforeDrain.DrainedGeneration, + beforeDrain.DiscontinuityGeneration); + wait_stream(stream, "drained wait"); + } + + SYS_AUDIO_STREAM_INFO drained = query_stream(stream); + if (drained.State == SysAudioStreamFailed) + fail_status("stream failure", drained.FailureStatus); + if (drained.DiscontinuityGeneration != beforeDrain.DiscontinuityGeneration) + fail("unexpected discontinuity during drain"); + if (drained.State != SysAudioStreamStopped || + drained.DrainedGeneration != beforeDrain.DrainedGeneration + 1u || + drained.BufferedFrames != 0 || + drained.FramesPresented < presentedTarget || + drained.FramesPresented + drained.FramesDiscarded < drained.FramesSubmitted) + fail("drained stream snapshot mismatch"); + + status = SysAudioReset(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("reset", status); + SYS_AUDIO_STREAM_INFO reset = query_stream(stream); + if (reset.State != SysAudioStreamStopped || reset.FramesSubmitted != 0 || + reset.FramesConsumed != 0 || reset.FramesPresented != 0 || + reset.DrainedGeneration != drained.DrainedGeneration) + fail("reset stream snapshot mismatch"); + + written = write_frames(stream, 0, 128, NTLSTATUS_SUCCESS); + if (written != 128) fail("stop test write was partial"); + status = SysAudioStart(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("restart", status); + SYS_AUDIO_STREAM_INFO beforeStop = query_stream(stream); + arm_wait(stream, + SYS_AUDIO_WAIT_STOPPED | + SYS_AUDIO_WAIT_FAILED | + SYS_AUDIO_WAIT_DISCONTINUITY, + 0, + beforeStop.FramesConsumed, + beforeStop.FramesPresented, + beforeStop.DrainedGeneration, + beforeStop.DiscontinuityGeneration); + status = SysAudioStop(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("stop", status); + wait_stream(stream, "stopped wait"); + if (query_stream(stream).State != SysAudioStreamStopped) + fail("stop state mismatch"); + + play_continuous_tone(stream); + + status = SysCloseHandle(stream); + if (status != NTLSTATUS_SUCCESS) + fail_status("close stream", status); + gOpenStream = 0; + status = SysSleep(TEST_IDLE_QUIESCE_MILLISECONDS); + if (status != NTLSTATUS_SUCCESS) + fail_status("endpoint quiesce sleep", status); +} + +// Executes the native audio syscall smoke test. +int main(void) { + fill_test_frames(); + print("audio_stream_smoke: begin native render contract test\n"); + + uint32_t count = 0; + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST countRequest; + memset(&countRequest, 0, sizeof(countRequest)); + countRequest.Size = sizeof(countRequest); + countRequest.CountOut = &count; + KSTATUS status = SysAudioEnumerateEndpoints(&countRequest); + if (status != NTLSTATUS_SUCCESS && status != NTLSTATUS_BUFFER_TOO_SMALL) + fail_status("count endpoints", status); + if (!count) { + print("audio_stream_smoke: SKIP no render endpoint\n"); + exit_now(0); + } + if (count > SYS_AUDIO_MAX_ENDPOINTS) + fail("endpoint count exceeded ABI maximum"); + + SYS_AUDIO_ENDPOINT_INFO endpoints[SYS_AUDIO_MAX_ENDPOINTS]; + memset(endpoints, 0, sizeof(endpoints)); + SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST enumerate; + memset(&enumerate, 0, sizeof(enumerate)); + enumerate.Size = sizeof(enumerate); + enumerate.Endpoints = endpoints; + enumerate.Capacity = SYS_AUDIO_MAX_ENDPOINTS; + enumerate.CountOut = &count; + status = SysAudioEnumerateEndpoints(&enumerate); + if (status != NTLSTATUS_SUCCESS) + fail_status("enumerate endpoints", status); + + const SYS_AUDIO_ENDPOINT_INFO* endpoint = select_endpoint(endpoints, count); + if (!endpoint) { + print("audio_stream_smoke: SKIP no available render endpoint\n"); + exit_now(0); + } + + print("audio_stream_smoke: endpoint="); + print_u64(endpoint->EndpointId); + print("\n"); + test_master_volume(endpoint); + test_stream(endpoint); + print("audio_stream_smoke: PASS endpoint="); + print_u64(endpoint->EndpointId); + print("\n"); + exit_now(0); +} diff --git a/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c b/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c index 787ea18..6fccb5f 100644 --- a/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c +++ b/code/user/tests/cwd_peb_smoke/cwd_peb_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/dirty_writeback/dirty_writeback.c b/code/user/tests/dirty_writeback/dirty_writeback.c index 40b8d0d..bab15a1 100644 --- a/code/user/tests/dirty_writeback/dirty_writeback.c +++ b/code/user/tests/dirty_writeback/dirty_writeback.c @@ -38,7 +38,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { @@ -62,9 +62,10 @@ static uint32_t query_cache(SYS_FILE_CACHE_STATS* out) { return SysQueryKernelStats(SYS_KERNEL_STATS_FILE_CACHE, out, sizeof(*out), 0); } -static void delete_path(const char* path) { +// Deletes one test path. +static uint32_t delete_path(const char* path) { SYS_PATH_REQUEST req = { SysSvFromCstr(path) }; - (void)SysDeleteFile(&req); + return SysDeleteFile(&req); } static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { @@ -80,7 +81,7 @@ static uint32_t create_temp(uint32_t* fileOut, const char** pathOut) { for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); ++i) { uint32_t h = 0; - delete_path(paths[i]); + (void)delete_path(paths[i]); SYS_OPEN_FILE_REQUEST req = SysMakeOpenFileRequest(SysSvFromCstr(paths[i]), SYS_FILE_ACCESS_READ | SYS_FILE_ACCESS_WRITE | SYS_FILE_ACCESS_QUERY, SYS_FILE_CREATE_NEW, @@ -222,7 +223,16 @@ int main(void) { s = SysCloseHandle(file); file = 0; if (s != NTLSTATUS_SUCCESS) fail_status("close reopened", s); - delete_path(path); + s = delete_path(path); + if (s != NTLSTATUS_SUCCESS) fail_status("delete temp", s); + + uint32_t deletedHandle = 0; + s = open_path(path, &deletedHandle); + if (s == NTLSTATUS_SUCCESS) { + (void)SysCloseHandle(deletedHandle); + fail("deleted file reopened"); + } + if (s != NTLSTATUS_NOT_FOUND) fail_status("verify delete", s); print("dirty_writeback: dirty-marks+="); print_u64(afterClose.DirtyMarks - before.DirtyMarks); diff --git a/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c b/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c index 1e7f2d0..f640acd 100644 --- a/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c +++ b/code/user/tests/exception_dispatcher_smoke/exception_dispatcher_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/exception_smoke/exception_smoke.c b/code/user/tests/exception_smoke/exception_smoke.c index bdf6853..93987b3 100644 --- a/code/user/tests/exception_smoke/exception_smoke.c +++ b/code/user/tests/exception_smoke/exception_smoke.c @@ -40,7 +40,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/file_lock_smoke/file_lock_smoke.c b/code/user/tests/file_lock_smoke/file_lock_smoke.c index b9bbd73..54d5110 100644 --- a/code/user/tests/file_lock_smoke/file_lock_smoke.c +++ b/code/user/tests/file_lock_smoke/file_lock_smoke.c @@ -39,7 +39,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/file_stat_smoke/file_stat_smoke.c b/code/user/tests/file_stat_smoke/file_stat_smoke.c index cdd5085..32a6eee 100644 --- a/code/user/tests/file_stat_smoke/file_stat_smoke.c +++ b/code/user/tests/file_stat_smoke/file_stat_smoke.c @@ -36,7 +36,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/handle_dup_smoke/handle_dup_smoke.c b/code/user/tests/handle_dup_smoke/handle_dup_smoke.c index 64f4b63..1d95351 100644 --- a/code/user/tests/handle_dup_smoke/handle_dup_smoke.c +++ b/code/user/tests/handle_dup_smoke/handle_dup_smoke.c @@ -42,7 +42,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/input_api_smoke/input_api_smoke.c b/code/user/tests/input_api_smoke/input_api_smoke.c index e370778..9b9fc2e 100644 --- a/code/user/tests/input_api_smoke/input_api_smoke.c +++ b/code/user/tests/input_api_smoke/input_api_smoke.c @@ -32,7 +32,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/input_model_smoke/input_model_smoke.c b/code/user/tests/input_model_smoke/input_model_smoke.c index 98ea49d..8f19d80 100644 --- a/code/user/tests/input_model_smoke/input_model_smoke.c +++ b/code/user/tests/input_model_smoke/input_model_smoke.c @@ -31,7 +31,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/mmap_smoke/mmap_smoke.c b/code/user/tests/mmap_smoke/mmap_smoke.c index 438e407..8830bbb 100644 --- a/code/user/tests/mmap_smoke/mmap_smoke.c +++ b/code/user/tests/mmap_smoke/mmap_smoke.c @@ -35,7 +35,7 @@ static void print_u64(uint64_t v) { static void exit_now(int code) { // SysExitProcess should not return, but keep tests from falling through to // the synthetic entry frame if a broken kernel build returns anyway. - for (;;) + while (true) SysExitProcess(code); } diff --git a/code/user/tests/module_load_smoke/module_load_smoke.c b/code/user/tests/module_load_smoke/module_load_smoke.c index 4120dc7..f26d51d 100644 --- a/code/user/tests/module_load_smoke/module_load_smoke.c +++ b/code/user/tests/module_load_smoke/module_load_smoke.c @@ -32,7 +32,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c b/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c index 5b0d180..8bc79d6 100644 --- a/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c +++ b/code/user/tests/named_queue_lifetime_smoke/named_queue_lifetime_smoke.c @@ -45,7 +45,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c b/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c index 3dae79f..9fcda32 100644 --- a/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c +++ b/code/user/tests/native_exception_runtime_smoke/native_exception_runtime_smoke.c @@ -35,7 +35,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c b/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c index 72a0124..bbfac44 100644 --- a/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c +++ b/code/user/tests/native_raise_exception_smoke/native_raise_exception_smoke.c @@ -36,7 +36,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/net_irp_smoke/net_irp_smoke.c b/code/user/tests/net_irp_smoke/net_irp_smoke.c index f9b6cf7..88fb0c3 100644 --- a/code/user/tests/net_irp_smoke/net_irp_smoke.c +++ b/code/user/tests/net_irp_smoke/net_irp_smoke.c @@ -36,7 +36,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/pcache_invalidate/pcache_invalidate.c b/code/user/tests/pcache_invalidate/pcache_invalidate.c index 1e369b2..4bf2918 100644 --- a/code/user/tests/pcache_invalidate/pcache_invalidate.c +++ b/code/user/tests/pcache_invalidate/pcache_invalidate.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c b/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c index 21cdac9..d15d711 100644 --- a/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c +++ b/code/user/tests/pcache_mmap_read_reuse/pcache_mmap_read_reuse.c @@ -30,7 +30,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/pcache_multithread/pcache_multithread.c b/code/user/tests/pcache_multithread/pcache_multithread.c index 6ae8442..77c6ce8 100644 --- a/code/user/tests/pcache_multithread/pcache_multithread.c +++ b/code/user/tests/pcache_multithread/pcache_multithread.c @@ -41,7 +41,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/pcache_pressure/pcache_pressure.c b/code/user/tests/pcache_pressure/pcache_pressure.c index d33e45f..915aeb3 100644 --- a/code/user/tests/pcache_pressure/pcache_pressure.c +++ b/code/user/tests/pcache_pressure/pcache_pressure.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { @@ -147,11 +147,6 @@ int main(void) { uint64_t loads = after.Loads - before.Loads; uint64_t hits = after.Hits - before.Hits; uint64_t drops = after.Drops - before.Drops; - uint64_t raReq = after.ReadAheadRequests - before.ReadAheadRequests; - uint64_t raLoad = after.ReadAheadPagesLoaded - before.ReadAheadPagesLoaded; - uint64_t raThrottle = after.ReadAheadPagesThrottled - before.ReadAheadPagesThrottled; - uint64_t batchReads = after.BatchReads - before.BatchReads; - uint64_t batchPages = after.BatchPagesLoaded - before.BatchPagesLoaded; print("pcache_pressure: read bytes="); print_u64(total); print(" loads+="); print_u64(loads); @@ -161,11 +156,6 @@ int main(void) { print(" active="); print_u64(after.ActivePages); print(" standby="); print_u64(after.StandbyPages); print(" max="); print_u64(after.MaxResidentPages); - print(" readahead-req+="); print_u64(raReq); - print(" readahead-load+="); print_u64(raLoad); - print(" readahead-throttle+="); print_u64(raThrottle); - print(" batch-reads+="); print_u64(batchReads); - print(" batch-pages+="); print_u64(batchPages); print("\n"); if (size >= MIN_PRESSURE_BYTES) { diff --git a/code/user/tests/pcache_smoke/pcache_smoke.c b/code/user/tests/pcache_smoke/pcache_smoke.c index 3fcc798..c88d35d 100644 --- a/code/user/tests/pcache_smoke/pcache_smoke.c +++ b/code/user/tests/pcache_smoke/pcache_smoke.c @@ -32,7 +32,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/posix_module_smoke/posix_module_smoke.c b/code/user/tests/posix_module_smoke/posix_module_smoke.c index 67ee83d..00904bc 100644 --- a/code/user/tests/posix_module_smoke/posix_module_smoke.c +++ b/code/user/tests/posix_module_smoke/posix_module_smoke.c @@ -3,6 +3,7 @@ #include #include #include +#include #define RTLD_NOW 0x0002 #define RTLD_LOCAL 0x0004 @@ -23,7 +24,7 @@ static void print(const char* s) { // Terminates the current process. static void exit_now(int code) { - for (;;) exit(code); + while (true) exit(code); } // Fails the smoke test with one message. diff --git a/code/user/tests/process_protection_smoke/process_protection_smoke.c b/code/user/tests/process_protection_smoke/process_protection_smoke.c index a9c8234..aed3ceb 100644 --- a/code/user/tests/process_protection_smoke/process_protection_smoke.c +++ b/code/user/tests/process_protection_smoke/process_protection_smoke.c @@ -37,7 +37,7 @@ static void print_u32(uint32_t value) { // Terminates the current test process. static void exit_now(int32_t code) { - for (;;) (void)SysExitProcess(code); + while (true) (void)SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/realpath_smoke/realpath_smoke.c b/code/user/tests/realpath_smoke/realpath_smoke.c index e4c19c6..01973d4 100644 --- a/code/user/tests/realpath_smoke/realpath_smoke.c +++ b/code/user/tests/realpath_smoke/realpath_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/shared_mmap/shared_mmap.c b/code/user/tests/shared_mmap/shared_mmap.c index 650371d..eb610d1 100644 --- a/code/user/tests/shared_mmap/shared_mmap.c +++ b/code/user/tests/shared_mmap/shared_mmap.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/socket_addr_smoke/socket_addr_smoke.c b/code/user/tests/socket_addr_smoke/socket_addr_smoke.c index dfe2efc..79f136c 100644 --- a/code/user/tests/socket_addr_smoke/socket_addr_smoke.c +++ b/code/user/tests/socket_addr_smoke/socket_addr_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/stats_abi_smoke/stats_abi_smoke.c b/code/user/tests/stats_abi_smoke/stats_abi_smoke.c index 28ecb31..f0e2353 100644 --- a/code/user/tests/stats_abi_smoke/stats_abi_smoke.c +++ b/code/user/tests/stats_abi_smoke/stats_abi_smoke.c @@ -33,7 +33,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/code/user/tests/surface_invalid_rect/surface_invalid_rect.c b/code/user/tests/surface_invalid_rect/surface_invalid_rect.c index 2173215..61f074a 100644 --- a/code/user/tests/surface_invalid_rect/surface_invalid_rect.c +++ b/code/user/tests/surface_invalid_rect/surface_invalid_rect.c @@ -30,7 +30,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/surface_multithread_present/surface_multithread_present.c b/code/user/tests/surface_multithread_present/surface_multithread_present.c index d242ccc..91f6725 100644 --- a/code/user/tests/surface_multithread_present/surface_multithread_present.c +++ b/code/user/tests/surface_multithread_present/surface_multithread_present.c @@ -43,7 +43,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/surface_smoke/surface_smoke.c b/code/user/tests/surface_smoke/surface_smoke.c index 0c9b9dd..14b257a 100644 --- a/code/user/tests/surface_smoke/surface_smoke.c +++ b/code/user/tests/surface_smoke/surface_smoke.c @@ -30,7 +30,7 @@ static void print_u64(uint64_t v) { } static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } static void fail(const char* msg) { diff --git a/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c b/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c index 70be498..c7e77c5 100644 --- a/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c +++ b/code/user/tests/syscall_contract_smoke/syscall_contract_smoke.c @@ -34,7 +34,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Checks whether a string view equals a NUL-terminated string. diff --git a/code/user/tests/timer_smoke/timer_smoke.c b/code/user/tests/timer_smoke/timer_smoke.c index d124a66..51dda9c 100644 --- a/code/user/tests/timer_smoke/timer_smoke.c +++ b/code/user/tests/timer_smoke/timer_smoke.c @@ -31,7 +31,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one status value. diff --git a/code/user/tests/win32_condition_variable_smoke/win32_condition_variable_smoke.c b/code/user/tests/win32_condition_variable_smoke/win32_condition_variable_smoke.c new file mode 100644 index 0000000..20aa869 --- /dev/null +++ b/code/user/tests/win32_condition_variable_smoke/win32_condition_variable_smoke.c @@ -0,0 +1,137 @@ +// Win32 condition-variable smoke test for Bink worker synchronization. + +#include +#include +#include + +typedef void* HANDLE; +typedef unsigned long DWORD; +typedef int BOOL; +typedef struct { uintptr_t _opaque[5]; } CRITICAL_SECTION; +typedef struct { void* _ptr; } CONDITION_VARIABLE; +typedef DWORD (__stdcall *THREAD_START_ROUTINE)(void* parameter); + +#define FALSE 0 +#define TRUE 1 +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u +#define INVALID_HANDLE_VALUE ((HANDLE)(intptr_t)-1) +#define ERROR_TIMEOUT 1460u + +__declspec(dllimport) void InitializeCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void DeleteCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void EnterCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void LeaveCriticalSection(CRITICAL_SECTION* criticalSection); +__declspec(dllimport) void InitializeConditionVariable(CONDITION_VARIABLE* conditionVariable); +__declspec(dllimport) BOOL SleepConditionVariableCS(CONDITION_VARIABLE* conditionVariable, + CRITICAL_SECTION* criticalSection, + DWORD milliseconds); +__declspec(dllimport) void WakeConditionVariable(CONDITION_VARIABLE* conditionVariable); +__declspec(dllimport) void WakeAllConditionVariable(CONDITION_VARIABLE* conditionVariable); +__declspec(dllimport) HANDLE CreateThread(void* attributes, + size_t stackSize, + THREAD_START_ROUTINE startRoutine, + void* parameter, + DWORD creationFlags, + DWORD* threadId); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE handle, DWORD milliseconds); +__declspec(dllimport) BOOL CloseHandle(HANDLE handle); +__declspec(dllimport) DWORD GetLastError(void); + +static CRITICAL_SECTION gLock; +static CONDITION_VARIABLE gCondition; +static uint32_t gReady; +static uint32_t gRelease; +static uint32_t gWoke; +static uint32_t gWorkerFailure; + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the smoke test after one failure. +static void fail(const char* message) { + print("win32_condition_variable_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Waits until the main thread releases all condition-variable workers. +static DWORD __stdcall worker_main(void* parameter) { + (void)parameter; + EnterCriticalSection(&gLock); + gReady++; + while (!gRelease) { + if (!SleepConditionVariableCS(&gCondition, &gLock, INFINITE)) { + gWorkerFailure = 1; + LeaveCriticalSection(&gLock); + return 1; + } + } + gWoke++; + LeaveCriticalSection(&gLock); + return 0; +} + +// Waits for both workers to enter their condition wait. +static void wait_for_workers(void) { + for (uint32_t attempt = 0; attempt < 5000u; attempt++) { + EnterCriticalSection(&gLock); + uint32_t ready = gReady; + LeaveCriticalSection(&gLock); + if (ready == 2u) + return; + (void)SysSleep(1); + } + fail("workers did not reach condition wait"); +} + +// Runs wake-all and finite-timeout condition-variable checks. +int main(void) { + print("win32_condition_variable_smoke: begin\n"); + InitializeCriticalSection(&gLock); + InitializeConditionVariable(&gCondition); + + HANDLE workers[2]; + workers[0] = CreateThread(NULL, 0, worker_main, NULL, 0, NULL); + workers[1] = CreateThread(NULL, 0, worker_main, NULL, 0, NULL); + if (!workers[0] || workers[0] == INVALID_HANDLE_VALUE || + !workers[1] || workers[1] == INVALID_HANDLE_VALUE) + fail("CreateThread failed"); + + wait_for_workers(); + print("win32_condition_variable_smoke: workers waiting\n"); + EnterCriticalSection(&gLock); + gRelease = 1; + WakeAllConditionVariable(&gCondition); + LeaveCriticalSection(&gLock); + print("win32_condition_variable_smoke: wake-all sent\n"); + + for (uint32_t index = 0; index < 2u; index++) { + if (WaitForSingleObject(workers[index], 5000u) != WAIT_OBJECT_0) + fail("worker did not exit after WakeAllConditionVariable"); + if (!CloseHandle(workers[index])) + fail("CloseHandle failed"); + } + if (gWorkerFailure || gWoke != 2u) + fail("wake-all result mismatch"); + print("win32_condition_variable_smoke: workers joined\n"); + + EnterCriticalSection(&gLock); + BOOL signaled = SleepConditionVariableCS(&gCondition, &gLock, 25u); + DWORD error = GetLastError(); + gRelease = 2; + LeaveCriticalSection(&gLock); + if (signaled || error != ERROR_TIMEOUT || gRelease != 2u) + fail("finite timeout or critical-section reacquire failed"); + print("win32_condition_variable_smoke: timeout complete\n"); + + // A wake with no waiter is intentionally not retained. + WakeConditionVariable(&gCondition); + DeleteCriticalSection(&gLock); + print("win32_condition_variable_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); +} diff --git a/code/user/tests/win32_exception_smoke/win32_exception_smoke.c b/code/user/tests/win32_exception_smoke/win32_exception_smoke.c index 461e3e2..c2033f4 100644 --- a/code/user/tests/win32_exception_smoke/win32_exception_smoke.c +++ b/code/user/tests/win32_exception_smoke/win32_exception_smoke.c @@ -41,7 +41,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/win32_module_lookup_smoke/win32_module_lookup_smoke.c b/code/user/tests/win32_module_lookup_smoke/win32_module_lookup_smoke.c new file mode 100644 index 0000000..f2dfb6b --- /dev/null +++ b/code/user/tests/win32_module_lookup_smoke/win32_module_lookup_smoke.c @@ -0,0 +1,51 @@ +// Win32 process-local module lookup smoke test. + +#include +#include + +typedef void* HANDLE; +typedef int BOOL; + +typedef BOOL (__stdcall *IS_DEBUGGER_PRESENT_ROUTINE)(void); + +__declspec(dllimport) HANDLE GetModuleHandleA(const char* moduleName); +__declspec(dllimport) void* GetProcAddress(HANDLE module, const char* procedureName); + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test after one contract failure. +static void fail(const char* message) { + print("win32_module_lookup_smoke: "); + print(message); + print("\n"); + while (true) + (void)SysExitProcess(1); +} + +// Verifies extensionless system-module lookup and dynamic debugger probing. +int main(void) { + print("win32_module_lookup_smoke: begin\n"); + + HANDLE fullName = GetModuleHandleA("kernel32.dll"); + HANDLE baseName = GetModuleHandleA("KERNEL32"); + if (!fullName || !baseName || fullName != baseName) + fail("extensionless KERNEL32 lookup failed"); + + if (GetModuleHandleA("kernel32.invalid")) + fail("invalid module suffix was accepted"); + + IS_DEBUGGER_PRESENT_ROUTINE isDebuggerPresent = + (IS_DEBUGGER_PRESENT_ROUTINE)GetProcAddress(baseName, "IsDebuggerPresent"); + if (!isDebuggerPresent) + fail("IsDebuggerPresent was absent from GetProcAddress"); + if (isDebuggerPresent()) + fail("IsDebuggerPresent reported a debugger"); + + print("win32_module_lookup_smoke: PASS\n"); + while (true) + (void)SysExitProcess(0); +} diff --git a/code/user/tests/win32_mutex_smoke/win32_mutex_smoke.c b/code/user/tests/win32_mutex_smoke/win32_mutex_smoke.c new file mode 100644 index 0000000..eb0f313 --- /dev/null +++ b/code/user/tests/win32_mutex_smoke/win32_mutex_smoke.c @@ -0,0 +1,147 @@ +// Win32 recursive mutex smoke test for Bink worker teardown. + +#include +#include +#include + +typedef void* HANDLE; +typedef unsigned long DWORD; +typedef int BOOL; +typedef DWORD (__stdcall *THREAD_START_ROUTINE)(void* parameter); + +#define FALSE 0 +#define TRUE 1 +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u +#define WAIT_TIMEOUT 0x00000102u +#define WAIT_FAILED 0xFFFFFFFFu +#define INVALID_HANDLE_VALUE ((HANDLE)(intptr_t)-1) + +__declspec(dllimport) HANDLE CreateMutexA(void* attributes, BOOL initialOwner, const char* name); +__declspec(dllimport) BOOL ReleaseMutex(HANDLE mutex); +__declspec(dllimport) HANDLE CreateThread(void* attributes, + size_t stackSize, + THREAD_START_ROUTINE startRoutine, + void* parameter, + DWORD creationFlags, + DWORD* threadId); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE handle, DWORD milliseconds); +__declspec(dllimport) BOOL CloseHandle(HANDLE handle); +__declspec(dllimport) DWORD GetLastError(void); + +static HANDLE gMutex; +static DWORD gWorkerWait; +static DWORD gWorkerError; +static BOOL gWorkerRelease; + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test after one contract failure. +static void fail(const char* message) { + print("win32_mutex_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Writes one fixed-width hexadecimal value to the native console. +static void print_hex32(DWORD value) { + static const char digits[] = "0123456789ABCDEF"; + char text[] = "0x00000000"; + for (unsigned i = 0; i < 8; i++) { + text[9 - i] = digits[value & 0xFu]; + value >>= 4; + } + print(text); +} + +// Reports a failed worker wait with both Win32 result channels. +static void fail_worker_wait(const char* message) { + print("win32_mutex_smoke: "); + print(message); + print(" result="); + print_hex32(gWorkerWait); + print(" error="); + print_hex32(gWorkerError); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Attempts one wait and releases the mutex only after successful acquisition. +static DWORD __stdcall mutex_worker(void* parameter) { + DWORD timeout = (DWORD)(uintptr_t)parameter; + gWorkerWait = WaitForSingleObject(gMutex, timeout); + gWorkerError = gWorkerWait == WAIT_FAILED ? GetLastError() : 0; + if (gWorkerWait == WAIT_OBJECT_0) + gWorkerRelease = ReleaseMutex(gMutex); + return 0; +} + +// Runs one worker and waits for its normal return. +static void run_worker(DWORD timeout) { + gWorkerWait = 0xFFFFFFFFu; + gWorkerError = 0; + gWorkerRelease = FALSE; + HANDLE worker = CreateThread(NULL, 0, mutex_worker, + (void*)(uintptr_t)timeout, 0, NULL); + if (!worker || worker == INVALID_HANDLE_VALUE) + fail("CreateThread failed"); + if (WaitForSingleObject(worker, 5000u) != WAIT_OBJECT_0) + fail("worker did not return"); + if (!CloseHandle(worker)) + fail("worker CloseHandle failed"); +} + +// Verifies recursive ownership, non-owner safety, and final release transfer. +int main(void) { + print("win32_mutex_smoke: begin\n"); + gMutex = CreateMutexA(NULL, FALSE, NULL); + if (!gMutex || gMutex == INVALID_HANDLE_VALUE) + fail("CreateMutexA failed"); + + if (WaitForSingleObject(gMutex, INFINITE) != WAIT_OBJECT_0 || + WaitForSingleObject(gMutex, INFINITE) != WAIT_OBJECT_0) + fail("recursive acquisition failed"); + + run_worker(25u); + if (gWorkerWait == WAIT_FAILED) + fail_worker_wait("doubly-owned finite wait failed"); + if (gWorkerWait != WAIT_TIMEOUT || gWorkerRelease) + fail("worker acquired doubly-owned mutex"); + + if (!ReleaseMutex(gMutex)) + fail("first recursive release failed"); + run_worker(25u); + if (gWorkerWait == WAIT_FAILED) + fail_worker_wait("singly-owned finite wait failed"); + if (gWorkerWait != WAIT_TIMEOUT || gWorkerRelease) + fail("worker acquired singly-owned mutex"); + + if (!ReleaseMutex(gMutex)) + fail("final recursive release failed"); + run_worker(1000u); + if (gWorkerWait == WAIT_FAILED) + fail_worker_wait("released finite wait failed"); + if (gWorkerWait != WAIT_OBJECT_0 || !gWorkerRelease) + fail("worker did not acquire released mutex"); + + if (!CloseHandle(gMutex)) + fail("mutex CloseHandle failed"); + + gMutex = CreateMutexA(NULL, TRUE, NULL); + if (!gMutex || gMutex == INVALID_HANDLE_VALUE) + fail("initial-owner mutex failed"); + if (WaitForSingleObject(gMutex, INFINITE) != WAIT_OBJECT_0) + fail("initial-owner recursion failed"); + if (!ReleaseMutex(gMutex) || !ReleaseMutex(gMutex)) + fail("initial-owner release failed"); + if (!CloseHandle(gMutex)) + fail("initial-owner close failed"); + + print("win32_mutex_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); +} diff --git a/code/user/tests/win32_seh_smoke/win32_seh_smoke.c b/code/user/tests/win32_seh_smoke/win32_seh_smoke.c index 84d59a9..11c3f34 100644 --- a/code/user/tests/win32_seh_smoke/win32_seh_smoke.c +++ b/code/user/tests/win32_seh_smoke/win32_seh_smoke.c @@ -54,7 +54,7 @@ static void print_u64(uint64_t v) { // Terminates the current process. static void exit_now(int32_t code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one message. diff --git a/code/user/tests/win32_thread_suspend_smoke/win32_thread_suspend_smoke.c b/code/user/tests/win32_thread_suspend_smoke/win32_thread_suspend_smoke.c new file mode 100644 index 0000000..31d99db --- /dev/null +++ b/code/user/tests/win32_thread_suspend_smoke/win32_thread_suspend_smoke.c @@ -0,0 +1,97 @@ +// Win32 suspended-thread creation and resume smoke test. + +#include +#include +#include + +typedef void* HANDLE; +typedef unsigned long DWORD; +typedef int BOOL; +typedef DWORD (__stdcall *THREAD_START_ROUTINE)(void* parameter); + +#define CREATE_SUSPENDED 0x00000004u +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u +#define INVALID_HANDLE_VALUE ((HANDLE)(intptr_t)-1) + +__declspec(dllimport) HANDLE CreateThread(void* attributes, + size_t stackSize, + THREAD_START_ROUTINE startRoutine, + void* parameter, + DWORD creationFlags, + DWORD* threadId); +__declspec(dllimport) DWORD ResumeThread(HANDLE thread); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE handle, DWORD milliseconds); +__declspec(dllimport) void Sleep(DWORD milliseconds); +__declspec(dllimport) BOOL CloseHandle(HANDLE handle); + +static volatile long gStarted; +static volatile long gRelease; + +// Writes one test message to the native console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test after one contract failure. +static void fail(const char* message) { + print("win32_thread_suspend_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Records that the suspended worker reached its user entry point. +static DWORD __stdcall suspended_worker(void* parameter) { + (void)parameter; + gStarted = 1; + while (!gRelease) + Sleep(1u); + return 0; +} + +// Exits immediately so process teardown can retain its thread handle. +static DWORD __stdcall exited_worker(void* parameter) { + (void)parameter; + return 0; +} + +// Verifies that CREATE_SUSPENDED prevents execution until ResumeThread. +int main(void) { + print("win32_thread_suspend_smoke: begin\n"); + + DWORD threadId = 0; + HANDLE thread = CreateThread(NULL, 0, suspended_worker, NULL, + CREATE_SUSPENDED, &threadId); + if (!thread || thread == INVALID_HANDLE_VALUE || threadId == 0) + fail("CreateThread(CREATE_SUSPENDED) failed"); + + Sleep(25u); + if (gStarted) + fail("worker ran before ResumeThread"); + + if (ResumeThread(thread) != 1u) + fail("first ResumeThread did not return one"); + if (ResumeThread(thread) != 0u) + fail("second ResumeThread did not return zero"); + + gRelease = 1; + if (WaitForSingleObject(thread, INFINITE) != WAIT_OBJECT_0) + fail("worker wait failed"); + if (!gStarted) + fail("worker did not run after ResumeThread"); + if (!CloseHandle(thread)) + fail("CloseHandle failed"); + + // Keep one completed thread handle open until process exit. Dead THREAD + // objects may remain waitable, but they must not keep the process alive. + HANDLE retained = CreateThread(NULL, 0, exited_worker, NULL, 0, NULL); + if (!retained || retained == INVALID_HANDLE_VALUE) + fail("retained CreateThread failed"); + if (WaitForSingleObject(retained, INFINITE) != WAIT_OBJECT_0) + fail("retained worker wait failed"); + + print("win32_thread_suspend_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); +} diff --git a/code/user/tests/winmm_waveout_smoke/winmm_waveout_smoke.c b/code/user/tests/winmm_waveout_smoke/winmm_waveout_smoke.c new file mode 100644 index 0000000..77f8a26 --- /dev/null +++ b/code/user/tests/winmm_waveout_smoke/winmm_waveout_smoke.c @@ -0,0 +1,455 @@ +// WinMM waveform-output smoke test over the native render-stream personality path. + +#include +#include +#include +#include +#include + +typedef void* HANDLE; +typedef void* HWAVEOUT; +typedef unsigned int MMRESULT; +typedef unsigned int UINT; +typedef uint32_t DWORD; +typedef uintptr_t DWORD_PTR; +typedef unsigned short WORD; +typedef int BOOL; + +#define FALSE 0 +#define TRUE 1 +#define INFINITE 0xFFFFFFFFu +#define WAIT_OBJECT_0 0u + +#define MMSYSERR_NOERROR 0u +#define MMSYSERR_NOTSUPPORTED 8u +#define WAVERR_BADFORMAT 32u +#define WAVERR_STILLPLAYING 33u + +#define WAVE_FORMAT_PCM 1u +#define WAVE_FORMAT_QUERY 0x00000001u +#define WAVE_ALLOWSYNC 0x00000002u +#define WAVE_FORMAT_DIRECT 0x00000008u +#define CALLBACK_WINDOW 0x00010000u +#define CALLBACK_THREAD 0x00020000u +#define CALLBACK_FUNCTION 0x00030000u +#define CALLBACK_EVENT 0x00050000u +#define WAVE_MAPPER 0xFFFFFFFFu + +#define WHDR_DONE 0x00000001u +#define WHDR_PREPARED 0x00000002u +#define WHDR_BEGINLOOP 0x00000004u +#define WHDR_ENDLOOP 0x00000008u +#define WHDR_INQUEUE 0x00000010u + +#define WOM_OPEN 0x03BBu +#define WOM_CLOSE 0x03BCu +#define WOM_DONE 0x03BDu + +#define TIME_SAMPLES 0x0002u + +#define TEST_FRAMES 4096u +#define TEST_CONTROL_FRAMES 16384u +#define TEST_PHASE_MODULUS 1200u +#define TEST_PHASE_STEP 11u +#define TEST_QUARTER_PHASE 300u +#define TEST_PEAK 8192 + +typedef struct WAVEFORMATEX { + WORD wFormatTag; + WORD nChannels; + DWORD nSamplesPerSec; + DWORD nAvgBytesPerSec; + WORD nBlockAlign; + WORD wBitsPerSample; + WORD cbSize; +} WAVEFORMATEX; + +typedef struct WAVEHDR { + char* lpData; + DWORD dwBufferLength; + DWORD dwBytesRecorded; + DWORD_PTR dwUser; + DWORD dwFlags; + DWORD dwLoops; + struct WAVEHDR* lpNext; + DWORD_PTR reserved; +} WAVEHDR; + +typedef struct MMTIME { + UINT wType; + union { + DWORD ms; + DWORD sample; + DWORD cb; + DWORD ticks; + } u; +} MMTIME; + +typedef struct WAVEOUTCAPSA { + WORD wMid; + WORD wPid; + DWORD vDriverVersion; + char szPname[32]; + DWORD dwFormats; + WORD wChannels; + WORD wReserved1; + DWORD dwSupport; +} WAVEOUTCAPSA; + +typedef char TEST_WAVEFORMATEX_SIZE[(sizeof(WAVEFORMATEX) == 20u) ? 1 : -1]; +typedef char TEST_WAVEHDR_SIZE[(sizeof(WAVEHDR) == 48u) ? 1 : -1]; +typedef char TEST_MMTIME_SIZE[(sizeof(MMTIME) == 8u) ? 1 : -1]; +typedef char TEST_WAVEOUTCAPSA_SIZE[(sizeof(WAVEOUTCAPSA) == 52u) ? 1 : -1]; + +typedef void (__stdcall *WAVEOUTPROC)(HWAVEOUT, UINT, DWORD_PTR, DWORD_PTR, DWORD_PTR); + +__declspec(dllimport) UINT waveOutGetNumDevs(void); +__declspec(dllimport) MMRESULT waveOutOpen(HWAVEOUT*, UINT, const WAVEFORMATEX*, DWORD_PTR, DWORD_PTR, DWORD); +__declspec(dllimport) MMRESULT waveOutClose(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutPrepareHeader(HWAVEOUT, WAVEHDR*, UINT); +__declspec(dllimport) MMRESULT waveOutUnprepareHeader(HWAVEOUT, WAVEHDR*, UINT); +__declspec(dllimport) MMRESULT waveOutWrite(HWAVEOUT, WAVEHDR*, UINT); +__declspec(dllimport) MMRESULT waveOutPause(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutRestart(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutReset(HWAVEOUT); +__declspec(dllimport) MMRESULT waveOutSetVolume(HWAVEOUT, DWORD); +__declspec(dllimport) MMRESULT waveOutGetVolume(HWAVEOUT, DWORD*); +__declspec(dllimport) MMRESULT waveOutGetPosition(HWAVEOUT, MMTIME*, UINT); +__declspec(dllimport) MMRESULT waveOutGetID(HWAVEOUT, UINT*); +__declspec(dllimport) MMRESULT waveOutGetDevCapsA(UINT, WAVEOUTCAPSA*, UINT); +__declspec(dllimport) MMRESULT waveOutGetErrorTextA(MMRESULT, char*, UINT); +__declspec(dllimport) HANDLE CreateEventA(void*, BOOL, BOOL, const char*); +__declspec(dllimport) BOOL SetEvent(HANDLE); +__declspec(dllimport) DWORD WaitForSingleObject(HANDLE, DWORD); +__declspec(dllimport) BOOL CloseHandle(HANDLE); + +static int16_t gS16Frames[TEST_FRAMES * 2u]; +static uint8_t gS24Frames[TEST_FRAMES * 2u * 3u]; +static int16_t gControlFrames[TEST_CONTROL_FRAMES * 2u]; +static HANDLE gFunctionEvent; +static volatile long gOpenCallbacks; +static volatile long gDoneCallbacks; +static volatile long gCloseCallbacks; + +// Writes one NUL-terminated string to the kernel console. +static void print(const char* text) { + SYS_WRITE_CONSOLE_REQUEST request = { SysSvFromCstr(text) }; + (void)SysWriteConsole(&request); +} + +// Terminates the test process after one failure. +static void fail(const char* message) { + print("winmm_waveout_smoke: "); + print(message); + print("\n"); + while (true) (void)SysExitProcess(1); +} + +// Verifies one WinMM result exactly. +static void expect_result(const char* operation, MMRESULT actual, MMRESULT expected) { + if (actual != expected) { + print("winmm_waveout_smoke: unexpected result from "); + print(operation); + print("\n"); + while (true) (void)SysExitProcess(1); + } +} + +// Returns one exact integer 440 Hz triangle-wave sample at 48 kHz. +static int16_t tone_sample(uint32_t phase) { + int32_t sample; + if (phase < TEST_QUARTER_PHASE) { + sample = (int32_t)(phase * TEST_PEAK / TEST_QUARTER_PHASE); + } else if (phase < TEST_QUARTER_PHASE * 2u) { + sample = (int32_t)((TEST_QUARTER_PHASE * 2u - phase) * + TEST_PEAK / TEST_QUARTER_PHASE); + } else if (phase < TEST_QUARTER_PHASE * 3u) { + sample = -(int32_t)((phase - TEST_QUARTER_PHASE * 2u) * + TEST_PEAK / TEST_QUARTER_PHASE); + } else { + sample = -(int32_t)((TEST_PHASE_MODULUS - phase) * + TEST_PEAK / TEST_QUARTER_PHASE); + } + return (int16_t)sample; +} + +// Fills S16, packed-S24, and muted control buffers without floating point. +static void fill_buffers(void) { + uint32_t phase = 0; + for (uint32_t frame = 0; frame < TEST_FRAMES; frame++) { + int16_t sample = tone_sample(phase); + gS16Frames[frame * 2u] = sample; + gS16Frames[frame * 2u + 1u] = sample; + int32_t sample24 = (int32_t)sample * 256; + for (uint32_t channel = 0; channel < 2u; channel++) { + size_t offset = ((size_t)frame * 2u + channel) * 3u; + gS24Frames[offset] = (uint8_t)sample24; + gS24Frames[offset + 1u] = (uint8_t)(sample24 >> 8u); + gS24Frames[offset + 2u] = (uint8_t)(sample24 >> 16u); + } + phase += TEST_PHASE_STEP; + if (phase >= TEST_PHASE_MODULUS) + phase -= TEST_PHASE_MODULUS; + } + memset(gControlFrames, 0, sizeof(gControlFrames)); +} + +// Receives function callbacks on the per-HWAVEOUT worker thread. +static void __stdcall wave_callback(HWAVEOUT waveOut, + UINT message, + DWORD_PTR instance, + DWORD_PTR parameter1, + DWORD_PTR parameter2) { + (void)waveOut; + (void)instance; + (void)parameter1; + (void)parameter2; + if (message == WOM_OPEN) + _InterlockedIncrement(&gOpenCallbacks); + else if (message == WOM_DONE) { + _InterlockedIncrement(&gDoneCallbacks); + (void)SetEvent(gFunctionEvent); + } else if (message == WOM_CLOSE) + _InterlockedIncrement(&gCloseCallbacks); +} + +// Waits for one callback event with a finite deadline. +static void wait_event(HANDLE event, const char* operation) { + if (WaitForSingleObject(event, 5000u) != WAIT_OBJECT_0) + fail(operation); +} + +// Returns one canonical PCM format descriptor. +static WAVEFORMATEX make_format_at(uint32_t sampleRate, + uint16_t channels, + uint16_t bitsPerSample) { + WAVEFORMATEX format; + memset(&format, 0, sizeof(format)); + format.wFormatTag = WAVE_FORMAT_PCM; + format.nChannels = channels; + format.nSamplesPerSec = sampleRate; + format.wBitsPerSample = bitsPerSample; + format.nBlockAlign = (uint16_t)(channels * (bitsPerSample / 8u)); + format.nAvgBytesPerSec = format.nSamplesPerSec * format.nBlockAlign; + return format; +} + +// Returns the default stereo 48 kHz PCM descriptor used for playback tests. +static WAVEFORMATEX make_format(uint16_t bitsPerSample) { + return make_format_at(48000u, 2u, bitsPerSample); +} + + +// Verifies device enumeration, capabilities, and fixed error descriptions. +static void test_device_surface(void) { + WAVEOUTCAPSA caps; + memset(&caps, 0, sizeof(caps)); + expect_result("device capabilities", + waveOutGetDevCapsA(WAVE_MAPPER, &caps, sizeof(caps)), + MMSYSERR_NOERROR); + if (caps.wChannels != 2u || !caps.szPname[0]) + fail("device capabilities mismatch"); + + char errorText[64]; + memset(errorText, 0, sizeof(errorText)); + expect_result("error text", + waveOutGetErrorTextA(WAVERR_BADFORMAT, errorText, sizeof(errorText)), + MMSYSERR_NOERROR); + if (!errorText[0]) + fail("error text was empty"); +} + +// Verifies function callbacks, queued completion, immediate copy, and position. +static void test_function_callback(void) { + WAVEFORMATEX format = make_format(16u); + gFunctionEvent = CreateEventA(NULL, FALSE, FALSE, NULL); + if (!gFunctionEvent) + fail("CreateEventA for function callback failed"); + + HWAVEOUT waveOut = NULL; + expect_result("function waveOutOpen", + waveOutOpen(&waveOut, + WAVE_MAPPER, + &format, + (DWORD_PTR)(WAVEOUTPROC)wave_callback, + 0, + CALLBACK_FUNCTION | WAVE_ALLOWSYNC | WAVE_FORMAT_DIRECT), + MMSYSERR_NOERROR); + if (!waveOut || gOpenCallbacks != 1) + fail("function WOM_OPEN callback missing"); + UINT deviceId = UINT32_MAX; + expect_result("get device ID", waveOutGetID(waveOut, &deviceId), MMSYSERR_NOERROR); + if (deviceId >= waveOutGetNumDevs()) + fail("resolved device ID is out of range"); + + WAVEHDR first; + WAVEHDR second; + memset(&first, 0, sizeof(first)); + memset(&second, 0, sizeof(second)); + first.lpData = (char*)gS16Frames; + first.dwBufferLength = sizeof(gS16Frames) / 2u; + second.lpData = (char*)gS16Frames + first.dwBufferLength; + second.dwBufferLength = sizeof(gS16Frames) - first.dwBufferLength; + expect_result("prepare first", waveOutPrepareHeader(waveOut, &first, sizeof(first)), MMSYSERR_NOERROR); + expect_result("prepare second", waveOutPrepareHeader(waveOut, &second, sizeof(second)), MMSYSERR_NOERROR); + expect_result("write first", waveOutWrite(waveOut, &first, sizeof(first)), MMSYSERR_NOERROR); + expect_result("write second", waveOutWrite(waveOut, &second, sizeof(second)), MMSYSERR_NOERROR); + + memset(gS16Frames, 0, sizeof(gS16Frames)); + while (gDoneCallbacks < 2) + wait_event(gFunctionEvent, "function WOM_DONE timed out"); + if (gDoneCallbacks != 2 || + (first.dwFlags & (WHDR_DONE | WHDR_PREPARED)) != (WHDR_DONE | WHDR_PREPARED) || + (second.dwFlags & (WHDR_DONE | WHDR_PREPARED)) != (WHDR_DONE | WHDR_PREPARED) || + (first.dwFlags & WHDR_INQUEUE) || (second.dwFlags & WHDR_INQUEUE)) + fail("function header completion mismatch"); + + MMTIME position; + memset(&position, 0, sizeof(position)); + position.wType = TIME_SAMPLES; + expect_result("get position", waveOutGetPosition(waveOut, &position, sizeof(position)), MMSYSERR_NOERROR); + if (position.u.sample < TEST_FRAMES) + fail("position did not reach submitted frames"); + + expect_result("set volume", waveOutSetVolume(waveOut, 0xC0008000u), MMSYSERR_NOERROR); + DWORD volume = 0; + expect_result("get volume", waveOutGetVolume(waveOut, &volume), MMSYSERR_NOERROR); + if (volume != 0xC0008000u) + fail("volume did not round trip"); + + expect_result("unprepare first", waveOutUnprepareHeader(waveOut, &first, sizeof(first)), MMSYSERR_NOERROR); + expect_result("unprepare second", waveOutUnprepareHeader(waveOut, &second, sizeof(second)), MMSYSERR_NOERROR); + expect_result("function close", waveOutClose(waveOut), MMSYSERR_NOERROR); + if (gCloseCallbacks != 1) + fail("function WOM_CLOSE callback missing"); + if (!CloseHandle(gFunctionEvent)) + fail("function callback event close failed"); + gFunctionEvent = NULL; +} + +// Verifies CALLBACK_EVENT and packed-S24 conversion through the native S16 stream. +static void test_event_s24(void) { + WAVEFORMATEX format = make_format(24u); + HANDLE event = CreateEventA(NULL, FALSE, FALSE, NULL); + if (!event) + fail("CreateEventA for event callback failed"); + + HWAVEOUT waveOut = NULL; + expect_result("event waveOutOpen", + waveOutOpen(&waveOut, + WAVE_MAPPER, + &format, + (DWORD_PTR)event, + 0, + CALLBACK_EVENT), + MMSYSERR_NOERROR); + wait_event(event, "event WOM_OPEN timed out"); + + WAVEHDR header; + memset(&header, 0, sizeof(header)); + header.lpData = (char*)gS24Frames; + header.dwBufferLength = sizeof(gS24Frames); + expect_result("event prepare", waveOutPrepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("event write", waveOutWrite(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + memset(gS24Frames, 0, sizeof(gS24Frames)); + wait_event(event, "event WOM_DONE timed out"); + if ((header.dwFlags & WHDR_DONE) == 0 || (header.dwFlags & WHDR_INQUEUE)) + fail("event header completion mismatch"); + expect_result("event unprepare", waveOutUnprepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("event close", waveOutClose(waveOut), MMSYSERR_NOERROR); + wait_event(event, "event WOM_CLOSE timed out"); + if (!CloseHandle(event)) + fail("event callback handle close failed"); +} + +// Verifies queued-close refusal and synchronous pause, restart, and reset. +static void test_control_operations(void) { + WAVEFORMATEX format = make_format(16u); + HWAVEOUT waveOut = NULL; + expect_result("control waveOutOpen", + waveOutOpen(&waveOut, WAVE_MAPPER, &format, 0, 0, 0), + MMSYSERR_NOERROR); + expect_result("control mute", waveOutSetVolume(waveOut, 0), MMSYSERR_NOERROR); + + WAVEHDR header; + memset(&header, 0, sizeof(header)); + header.lpData = (char*)gControlFrames; + header.dwBufferLength = sizeof(gControlFrames); + expect_result("control prepare", waveOutPrepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + header.dwFlags |= WHDR_BEGINLOOP | WHDR_ENDLOOP; + header.dwLoops = 2u; + expect_result("loop rejection", waveOutWrite(waveOut, &header, sizeof(header)), MMSYSERR_NOTSUPPORTED); + header.dwFlags &= ~(WHDR_BEGINLOOP | WHDR_ENDLOOP); + header.dwLoops = 0; + expect_result("control write", waveOutWrite(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("close while queued", waveOutClose(waveOut), WAVERR_STILLPLAYING); + expect_result("unprepare while queued", + waveOutUnprepareHeader(waveOut, &header, sizeof(header)), + WAVERR_STILLPLAYING); + expect_result("pause", waveOutPause(waveOut), MMSYSERR_NOERROR); + expect_result("second pause", waveOutPause(waveOut), MMSYSERR_NOERROR); + expect_result("restart", waveOutRestart(waveOut), MMSYSERR_NOERROR); + expect_result("second restart", waveOutRestart(waveOut), MMSYSERR_NOERROR); + expect_result("reset", waveOutReset(waveOut), MMSYSERR_NOERROR); + if ((header.dwFlags & WHDR_DONE) == 0 || (header.dwFlags & WHDR_INQUEUE)) + fail("reset did not complete queued header"); + expect_result("control unprepare", waveOutUnprepareHeader(waveOut, &header, sizeof(header)), MMSYSERR_NOERROR); + expect_result("control close", waveOutClose(waveOut), MMSYSERR_NOERROR); +} + +// Verifies the complete accepted PCM matrix and explicit rejection paths. +static void test_formats_and_rejections(void) { + static const uint32_t rates[] = { 11025u, 22050u, 44100u, 48000u }; + static const uint16_t bits[] = { 16u, 24u }; + for (uint32_t rateIndex = 0; rateIndex < sizeof(rates) / sizeof(rates[0]); rateIndex++) { + for (uint16_t channels = 1u; channels <= 2u; channels++) { + for (uint32_t bitsIndex = 0; bitsIndex < sizeof(bits) / sizeof(bits[0]); bitsIndex++) { + WAVEFORMATEX format = make_format_at(rates[rateIndex], channels, bits[bitsIndex]); + expect_result("accepted PCM query", + waveOutOpen(NULL, WAVE_MAPPER, &format, 0, 0, WAVE_FORMAT_QUERY), + MMSYSERR_NOERROR); + } + } + } + + WAVEFORMATEX s16 = make_format(16u); + HWAVEOUT waveOut = NULL; + expect_result("window callback rejection", + waveOutOpen(&waveOut, WAVE_MAPPER, &s16, 1, 0, CALLBACK_WINDOW), + MMSYSERR_NOTSUPPORTED); + expect_result("thread callback rejection", + waveOutOpen(&waveOut, WAVE_MAPPER, &s16, 1, 0, CALLBACK_THREAD), + MMSYSERR_NOTSUPPORTED); + + WAVEFORMATEX floating = make_format_at(48000u, 2u, 32u); + floating.wFormatTag = 3u; + expect_result("Float32 rejection", + waveOutOpen(NULL, WAVE_MAPPER, &floating, 0, 0, WAVE_FORMAT_QUERY), + WAVERR_BADFORMAT); + + WAVEFORMATEX badRate = make_format_at(32000u, 2u, 16u); + expect_result("32 kHz rejection", + waveOutOpen(NULL, WAVE_MAPPER, &badRate, 0, 0, WAVE_FORMAT_QUERY), + WAVERR_BADFORMAT); + + WAVEFORMATEX badChannels = make_format_at(48000u, 3u, 16u); + expect_result("three-channel rejection", + waveOutOpen(NULL, WAVE_MAPPER, &badChannels, 0, 0, WAVE_FORMAT_QUERY), + WAVERR_BADFORMAT); +} + +// Runs the focused WinMM waveform-output contract test. +int main(void) { + print("winmm_waveout_smoke: begin\n"); + if (!waveOutGetNumDevs()) + fail("no waveform output devices"); + fill_buffers(); + test_device_surface(); + test_formats_and_rejections(); + test_control_operations(); + test_function_callback(); + test_event_s24(); + print("winmm_waveout_smoke: PASS\n"); + while (true) (void)SysExitProcess(0); + return 0; +} diff --git a/code/user/tests/would_block_smoke/would_block_smoke.c b/code/user/tests/would_block_smoke/would_block_smoke.c index 097fc44..89b3c56 100644 --- a/code/user/tests/would_block_smoke/would_block_smoke.c +++ b/code/user/tests/would_block_smoke/would_block_smoke.c @@ -74,7 +74,7 @@ static void print_u64(uint64_t v) { // Terminates the current test process. static void exit_now(int code) { - for (;;) SysExitProcess(code); + while (true) SysExitProcess(code); } // Fails the test with one plain message. diff --git a/docs.md b/docs.md index b7d440d..6cca3fe 100644 --- a/docs.md +++ b/docs.md @@ -95,7 +95,9 @@ Breaking changes are allowed. When making one: ## comments from developer -- 0x%016llX over %p -- never add Ex, it makes me mad, never +- 0x%016llX over %p, it looks prettier and is standard. +- never add Ex, it makes me mad, never, rearchitect instead - i do put a focus on overengineering if it means it will be cleaner and futureproof -- please do ask questions, preferably about a hundred per feature +- ASK QUESTIONS! never assume you know the right answer, if you do, you are probably wrong +- shift the syscall number table every time you add a syscall, do not append to the end +- do not worry if a patch breaks something, it's better to break, document and fix rather than work around it. diff --git a/readme.md b/readme.md index be176ce..9bf660e 100644 --- a/readme.md +++ b/readme.md @@ -79,7 +79,8 @@ Async is first-class here too, same machinery as the filesystem. ## how to build -- open the VS x64 build tools as administrator +- get mtools +- open the VS x64 build tools - change into this directory - run `build.bat` - use one of: @@ -144,6 +145,9 @@ A: Whatever I fixate on next. Q: Does it run Doom? A: `doom.exe` +Q: Can it play Crysis? +A: Into the bucket list. + Q: Does USB work? A: It did for everything I threw at it. diff --git a/run.bat b/run.bat index eb82548..a347a30 100644 --- a/run.bat +++ b/run.bat @@ -1 +1 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=raw,file=fat:rw:fatroot -serial stdio -netdev user,id=net0 -device rtl8139,netdev=net0 +@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=vpc,file=disk.vhd -serial stdio -netdev user,id=net0 -device rtl8139,netdev=net0 -audiodev dsound,id=snd0 -device intel-hda -device hda-output,audiodev=snd0 diff --git a/runa.bat b/runa.bat index 13e3da0..bda7825 100644 --- a/runa.bat +++ b/runa.bat @@ -1,20 +1,23 @@ -@start qemu-system-x86_64 ^ --vga vmware ^ --drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd ^ --drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd ^ --device ahci,id=ahci ^ --drive format=raw,file=fat:rw:fatroot,if=none,id=disk0 ^ --device ide-hd,bus=ahci.0,drive=disk0 ^ --serial stdio ^ --M q35 ^ --m 128M ^ --smp cpus=4,cores=4,threads=1,sockets=1 ^ --netdev user,id=net0,hostfwd=tcp::8080-:8080 ^ --device rtl8139,netdev=net0 ^ --display gtk,show-cursor=on ^ --device qemu-xhci,id=usb ^ --device usb-kbd,bus=usb.0 ^ --device usb-tablet,bus=usb.0 ^ --device usb-hub,id=myhub,bus=usb.0,port=3 ^ --drive if=none,id=usbflash,format=raw,file=fat:rw:usbfatroot ^ --device usb-storage,drive=usbflash,bus=usb.0,port=3.1,id=myusbflash +@start qemu-system-x86_64 ^ +-vga vmware ^ +-drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd ^ +-drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd ^ +-device ahci,id=ahci ^ +-drive format=vpc,file=disk.vhd,if=none,id=disk0 ^ +-device ide-hd,bus=ahci.0,drive=disk0 ^ +-serial stdio ^ +-M q35 ^ +-m 256M ^ +-smp cpus=4,cores=4,threads=1,sockets=1 ^ +-netdev user,id=net0,hostfwd=tcp::8080-:8080 ^ +-device rtl8139,netdev=net0 ^ +-display gtk,show-cursor=on ^ +-device qemu-xhci,id=usb ^ +-device usb-kbd,bus=usb.0 ^ +-device usb-tablet,bus=usb.0 ^ +-device usb-hub,id=myhub,bus=usb.0,port=3 ^ +-drive if=none,id=usbflash,format=raw,file=fat:rw:usbfatroot ^ +-device usb-storage,drive=usbflash,bus=usb.0,port=3.1,id=myusbflash ^ +-audiodev dsound,id=snd0 ^ +-device intel-hda ^ +-device hda-output,audiodev=snd0 diff --git a/runad.bat b/runad.bat index bcaecc6..f3bf91c 100644 --- a/runad.bat +++ b/runad.bat @@ -1 +1 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=raw,file=fat:rw:fatroot,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0 +@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=vpc,file=disk.vhd,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0 diff --git a/rund.bat b/rund.bat index 27e9dc5..f8d797c 100644 --- a/rund.bat +++ b/rund.bat @@ -1 +1 @@ -@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=raw,file=fat:rw:fatroot -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0 +@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -drive format=vpc,file=disk.vhd -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown -netdev user,id=net0 -device rtl8139,netdev=net0